hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fixed hidden sys tenant pin log disk in each server.

Browse Source
This commit is contained in:
HaHaJeff
2023-02-07 14:20:40 +08:00
committed by ob-robot
parent fbc56d967c
commit cde81bf4b1
7 changed files with 97 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

93
src/observer/omt/ob_multi_tenant.cpp
View File

@ -523,6 +523,8 @@ int ObMultiTenant::create_hidden_sys_tenant()
LOG_ERROR("fail to construct meta", K(ret));
} else if (OB_FAIL(create_tenant(meta, true/* write_slog*/))) {
LOG_ERROR("create hidden sys tenant failed", K(ret));
} else {
LOG_INFO("create hidden sys tenant success", K(ret), K(meta));
}
return ret;
@ -658,10 +660,10 @@ int ObMultiTenant::convert_hidden_to_real_sys_tenant(const ObUnitInfoGetter::ObT
LOG_WARN("must be hidden sys tenant", K(ret));
} else if (FALSE_IT(new_super_block = tenant->get_super_block())) {
} else if (FALSE_IT(new_super_block.is_hidden_ = false)) {
} else if (OB_FAIL(update_tenant_unit_no_lock(unit))) {
} else if (OB_FAIL(update_tenant_unit_no_lock(unit, UpdateTenantConfigOpt::CONVERT_HIDDEN_TO_REAL_SYS_TENANT))) {
LOG_WARN("fail to update_tenant_unit_no_lock", K(ret), K(unit));
} else if (ObServerCheckpointSlogHandler::get_instance()
.write_tenant_super_block_slog(new_super_block)) {
} else if (OB_FAIL(ObServerCheckpointSlogHandler::get_instance()
.write_tenant_super_block_slog(new_super_block))) {
LOG_WARN("fail to write_tenant_super_block_slog", K(ret), K(new_super_block));
} else {
tenant->set_tenant_super_block(new_super_block);
@ -692,6 +694,7 @@ int ObMultiTenant::create_tenant(const ObTenantMeta &meta, bool write_slog, cons
int64_t bucket_lock_idx = -1;
const int64_t log_disk_size = meta.unit_.config_.log_disk_size();
int64_t lock_timeout_ts = abs_timeout_us - 5000000; // reserve 5s for creating tenant
bool is_hidden_sys = meta.is_hidden();
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
@ -739,6 +742,20 @@ int ObMultiTenant::create_tenant(const ObTenantMeta &meta, bool write_slog, cons
create_step = ObTenantCreateStep::STEP_CTX_MEM_CONFIG_SETTED; // step1
}
}
if (OB_SUCC(ret)) {
if (!is_virtual_tenant_id(tenant_id)
&& !is_hidden_sys
&& OB_FAIL(GCTX.log_block_mgr_->create_tenant(log_disk_size))) {
LOG_ERROR("create_tenatn in ObServerLogBlockMgr failed", KR(ret));
}
// if create_tenant in ObServerLogBlockMGR success, the log disk size need by this tenant has been pinned,
// otherwise, the assigned log disk size of ObServerLogBlockMGR is origin.
if (OB_SUCC(ret)) {
create_step = ObTenantCreateStep::STEP_LOG_DISK_SIZE_PINNED;
}
}
if (OB_FAIL(ret)) {
// do nothing
} else if (OB_ISNULL(GCTX.cgroup_ctrl_)) {
@ -748,15 +765,14 @@ int ObMultiTenant::create_tenant(const ObTenantMeta &meta, bool write_slog, cons
ObTenant, ObModIds::OMT, tenant_id, GCONF.workers_per_cpu_quota.get_value(), *GCTX.cgroup_ctrl_))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("new tenant fail", K(ret));
} else if (FALSE_IT(create_step = ObTenantCreateStep::STEP_TENANT_NEWED)) { //step2
} else if (FALSE_IT(create_step = ObTenantCreateStep::STEP_TENANT_NEWED)) { //step3
} else if (OB_FAIL(tenant->init_ctx())) {
LOG_WARN("init ctx fail", K(tenant_id), K(ret));
} else if (write_slog) {
if (OB_FAIL(write_create_tenant_prepare_slog(meta))) {
LOG_ERROR("fail to write create tenant prepare slog", K(ret));
} else {
create_step = ObTenantCreateStep::STEP_WRITE_PREPARE_SLOG; // step3
create_step = ObTenantCreateStep::STEP_WRITE_PREPARE_SLOG; // step4
}
}
@ -791,7 +807,7 @@ int ObMultiTenant::create_tenant(const ObTenantMeta &meta, bool write_slog, cons
LOG_ERROR("fail to write create tenant commit slog", K(ret), K(tenant_id));
} else {
tenant->set_create_status(ObTenantCreateStatus::CREATE_COMMIT);
create_step = ObTenantCreateStep::STEP_FINISH; // step4
create_step = ObTenantCreateStep::STEP_FINISH; // step5
}
}
@ -802,19 +818,9 @@ int ObMultiTenant::create_tenant(const ObTenantMeta &meta, bool write_slog, cons
if (OB_SUCC(get_tenant_unsafe(tenant_id, tmp_tenant))) {
ret = OB_TENANT_EXIST;
LOG_ERROR("tenant exist", K(ret), K(tenant_id));
// NB: hold log disk firstly, guarded by lock_ to avoid create tenant concurrently.
} else if (false == is_virtual_tenant_id(tenant_id) && OB_FAIL(GCTX.log_block_mgr_->create_tenant(log_disk_size))) {
LOG_WARN("create_tenant in ObServerLogBlockMgr failed", K(ret), K(meta));
} else if (OB_FAIL(tenants_.insert(tenant, iter, compare_tenant))) {
LOG_ERROR("fail to insert tenant", K(ret), K(tenant_id));
}
// Need ensure that after insert tenant into tenants_ success, means create tenant success.
// NB:
// 1. need call ObServerLogBlockMgr::abort_create_tenant after create tenant failed.
// 2. need guarded by lock, avoid concurrent with create_tenant or del_tenant
if (false == is_valid_tenant_id(tenant_id) && OB_FAIL(ret)) {
GCTX.log_block_mgr_->abort_create_tenant(log_disk_size);
}
}
// TODO: @lingyang 预期不能失败
if (!is_virtual_tenant_id(tenant_id) && OB_TMP_FAIL(update_tenant_config(tenant_id))) {
@ -854,6 +860,13 @@ int ObMultiTenant::create_tenant(const ObTenantMeta &meta, bool write_slog, cons
}
} while (OB_SUCCESS != tmp_ret);
do {
tmp_ret = OB_SUCCESS;
if (create_step >= ObTenantCreateStep::STEP_LOG_DISK_SIZE_PINNED) {
GCTX.log_block_mgr_->abort_create_tenant(log_disk_size);
}
} while (OB_SUCCESS != tmp_ret);
if (create_step >= ObTenantCreateStep::STEP_WRITE_PREPARE_SLOG) {
if (OB_SUCCESS != (tmp_ret = write_create_tenant_abort_slog(tenant_id))) {
LOG_ERROR("fail to write create tenant abort slog", K(tmp_ret));
@ -869,7 +882,8 @@ int ObMultiTenant::create_tenant(const ObTenantMeta &meta, bool write_slog, cons
return ret;
}
int ObMultiTenant::update_tenant_unit_no_lock(const ObUnitInfoGetter::ObTenantConfig &unit)
int ObMultiTenant::update_tenant_unit_no_lock(const ObUnitInfoGetter::ObTenantConfig &unit,
const UpdateTenantConfigOpt &opt)
{
int ret = OB_SUCCESS;
@ -878,6 +892,9 @@ int ObMultiTenant::update_tenant_unit_no_lock(const ObUnitInfoGetter::ObTenantCo
const double max_cpu = static_cast<double>(unit.config_.max_cpu());
const uint64_t tenant_id = unit.tenant_id_;
int64_t allowed_mem_limit = 0;
// when opt is UpdateTenantConfigOpt::CONVERT_REAL_TO_HIDDEN_SYS_TENANT, the new_log_disk_size should set to zero.
const int64_t new_log_disk_size =
opt == UpdateTenantConfigOpt::CONVERT_REAL_TO_HIDDEN_SYS_TENANT ? 0 : unit.config_.log_disk_size();
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
@ -895,7 +912,7 @@ int ObMultiTenant::update_tenant_unit_no_lock(const ObUnitInfoGetter::ObTenantCo
LOG_WARN("fail to update_tenant_freezer_mem_limit", K(ret), K(tenant_id));
} else if (OB_FAIL(tenant->update_thread_cnt(max_cpu))) {
LOG_WARN("fail to update mtl module thread_cnt", K(ret), K(tenant_id));
} else if (OB_FAIL(update_tenant_log_disk_size(tenant_id, unit.config_.log_disk_size()))) {
} else if (OB_FAIL(update_tenant_log_disk_size(tenant_id, new_log_disk_size, opt))) {
LOG_WARN("fail to update tenant log disk size", K(ret), K(tenant_id));
} else if (FALSE_IT(tenant->update_memory_size(unit.config_.memory_size()))) {
// unreachable
@ -926,7 +943,7 @@ int ObMultiTenant::update_tenant_unit(const ObUnitInfoGetter::ObTenantConfig &un
} else if (OB_FAIL(bucket_lock_.wrlock(bucket_lock_idx = get_tenant_lock_bucket_idx(tenant_id)))) {
LOG_WARN("fail to try_wrlock for update tenant unit", K(ret), K(tenant_id), K(bucket_lock_idx));
} else if (FALSE_IT(lock_succ = true)) {
} else if (OB_FAIL(update_tenant_unit_no_lock(unit))) {
} else if (OB_FAIL(update_tenant_unit_no_lock(unit, UpdateTenantConfigOpt::NORMAL_UPDATE_TENANT_CONFIG_OPT))) {
LOG_WARN("fail to update_tenant_unit_no_lock", K(ret), K(unit));
}
@ -976,7 +993,8 @@ int ObMultiTenant::update_tenant_memory(const uint64_t tenant_id, const int64_t
}
int ObMultiTenant::update_tenant_log_disk_size(const uint64_t tenant_id,
const int64_t expected_log_disk_size)
const int64_t expected_log_disk_size,
const UpdateTenantConfigOpt &opt)
{
int ret = OB_SUCCESS;
MAKE_TENANT_SWITCH_SCOPE_GUARD(guard);
@ -986,23 +1004,26 @@ int ObMultiTenant::update_tenant_log_disk_size(const uint64_t tenant_id,
ObLogService *log_service = MTL(ObLogService *);
if (OB_ISNULL(log_service)) {
ret = OB_ERR_UNEXPECTED;
} else if (OB_FAIL(log_service->get_palf_disk_usage(unused_log_disk_size, old_log_disk_size))) {
// when opt is UpdateTenantConfigOpt::CONVERT_HIDDEN_TO_REAL_SYS_TENANT, the old_log_disk_size is zero.
} else if (opt != UpdateTenantConfigOpt::CONVERT_HIDDEN_TO_REAL_SYS_TENANT
&& OB_FAIL(log_service->get_palf_disk_usage(unused_log_disk_size, old_log_disk_size))) {
LOG_WARN("failed to get_palf_disk_usage", K(ret), K(unused_log_disk_size), K(old_log_disk_size),
K(expected_log_disk_size));
} else if (old_log_disk_size == expected_log_disk_size) {
LOG_INFO("no need to update tenant log disk size", K(tenant_id), K(old_log_disk_size), K(expected_log_disk_size));
} else if(false == is_valid_tenant_id(tenant_id)
&& OB_FAIL(GCTX.log_block_mgr_->update_tenant(old_log_disk_size, expected_log_disk_size))) {
} else if(OB_FAIL(GCTX.log_block_mgr_->update_tenant(old_log_disk_size, expected_log_disk_size))) {
LOG_WARN("failed to update_tenant in ObServerLogBlockMgr", K(ret), K(old_log_disk_size),
K(expected_log_disk_size));
} else {
if (OB_FAIL(log_service->update_log_disk_usage_limit_size(expected_log_disk_size))) {
LOG_WARN("failed to update_log_disk_usage_limit_size", K(ret), K(tenant_id),
K(expected_log_disk_size));
} else {
LOG_INFO("update_tenant_log_disk_size success", K(ret), K(tenant_id),
K(expected_log_disk_size));
// Only when opt is UpdateTenantConfigOpt::NORMAL_UPDATE_TENANT_CONFIG_OPT update palf.
if (opt == UpdateTenantConfigOpt::NORMAL_UPDATE_TENANT_CONFIG_OPT) {
if (OB_FAIL(log_service->update_log_disk_usage_limit_size(expected_log_disk_size))) {
LOG_WARN("failed to update_log_disk_usage_limit_size", K(ret), K(tenant_id),
K(expected_log_disk_size));
} else {
LOG_INFO("update_tenant_log_disk_size success", K(ret), K(tenant_id), K(opt),
K(expected_log_disk_size));
}
}
if (false == is_virtual_tenant_id(tenant_id) && OB_FAIL(ret)) {
GCTX.log_block_mgr_->abort_update_tenant(expected_log_disk_size, old_log_disk_size);
}
@ -1382,6 +1403,11 @@ int ObMultiTenant::remove_tenant(const uint64_t tenant_id, bool &try_clock_succ)
KP(removed_tenant), K(removed_tenant->lock_));
removed_tenant->lock_.ld_.print();
} else {
const share::ObUnitInfoGetter::ObTenantConfig &config = removed_tenant->get_unit();
const int64_t log_disk_size = config.config_.log_disk_size();
if (!removed_tenant->is_hidden() && !is_virtual_tenant_id(tenant_id)) {
GCTX.log_block_mgr_->remove_tenant(log_disk_size);
}
ObTenant *removed_tenant_tmp = nullptr;
SpinWLockGuard guard(lock_);
@ -1395,7 +1421,6 @@ int ObMultiTenant::remove_tenant(const uint64_t tenant_id, bool &try_clock_succ)
}
if (OB_SUCC(ret)) {
GCTX.log_block_mgr_->remove_tenant();
removed_tenant->destroy();
ob_delete(removed_tenant);
LOG_INFO("remove tenant success", K(tenant_id));
@ -1594,7 +1619,7 @@ int ObMultiTenant::convert_real_to_hidden_sys_tenant()
} else if (OB_FAIL(bucket_lock_.try_wrlock(bucket_lock_idx = get_tenant_lock_bucket_idx(tenant_id)))) {
LOG_WARN("fail to try_wrlock for delete tenant", K(ret), K(tenant_id), K(bucket_lock_idx));
} else if (FALSE_IT(lock_succ = true)) {
} else if (OB_FAIL(update_tenant_unit_no_lock(tenant_meta.unit_))) {
} else if (OB_FAIL(update_tenant_unit_no_lock(tenant_meta.unit_, UpdateTenantConfigOpt::CONVERT_REAL_TO_HIDDEN_SYS_TENANT))) {
LOG_WARN("fail to update_tenant_unit_no_lock", K(ret), K(tenant_meta));
} else {
ObTenantSwitchGuard guard(tenant);