hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fixed hidden sys tenant pin log disk in each server.

Browse Source
This commit is contained in:
HaHaJeff
2023-02-07 14:20:40 +08:00
committed by ob-robot
parent fbc56d967c
commit cde81bf4b1
7 changed files with 97 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
src/logservice/ob_server_log_block_mgr.cpp
View File

@ -301,11 +301,13 @@ int ObServerLogBlockMgr::create_tenant(const int64_t log_disk_size)
{
int ret = OB_SUCCESS;
ObSpinLockGuard guard(resize_lock_);
if ((min_log_disk_size_for_all_tenants_ += log_disk_size) > get_total_size_guarded_by_lock_()) {
int64_t tmp_log_disk_size = min_log_disk_size_for_all_tenants_;
if ((tmp_log_disk_size += log_disk_size) > get_total_size_guarded_by_lock_()) {
ret = OB_MACHINE_RESOURCE_NOT_ENOUGH;
CLOG_LOG(ERROR, "ObServerLogBlockMGR can not hold any new tenants",
K(ret), KPC(this), K(log_disk_size));
} else {
min_log_disk_size_for_all_tenants_ = tmp_log_disk_size;
CLOG_LOG(INFO, "ObServerLogBlockMGR create_tenant success", KPC(this), K(log_disk_size));
}
return ret;
@ -324,14 +326,15 @@ int ObServerLogBlockMgr::update_tenant(const int64_t old_log_disk_size, const in
{
int ret = OB_SUCCESS;
ObSpinLockGuard guard(resize_lock_);
min_log_disk_size_for_all_tenants_ -= old_log_disk_size;
if (old_log_disk_size == new_log_disk_size) {
} else if ((min_log_disk_size_for_all_tenants_+=new_log_disk_size) > get_total_size_guarded_by_lock_()) {
int64_t tmp_log_disk_size = min_log_disk_size_for_all_tenants_;
tmp_log_disk_size -= old_log_disk_size;
if ((tmp_log_disk_size +=new_log_disk_size) > get_total_size_guarded_by_lock_()) {
ret = OB_MACHINE_RESOURCE_NOT_ENOUGH;
CLOG_LOG(ERROR, "ObServerLogBlockMGR can not hold any new tenants",
K(ret), KPC(this), K(old_log_disk_size), K(new_log_disk_size));
} else {
CLOG_LOG(INFO, "ObServerLogBlockMGR update_tenant success", K(ret), KPC(this), K(old_log_disk_size), K(new_log_disk_size));
min_log_disk_size_for_all_tenants_ = tmp_log_disk_size;
CLOG_LOG(INFO, "ObServerLogBlockMGR update_tenant success", KPC(this), K(old_log_disk_size), K(new_log_disk_size));
}
return ret;
}
@ -346,15 +349,17 @@ void ObServerLogBlockMgr::abort_update_tenant(const int64_t old_log_disk_size, c
CLOG_LOG(INFO, "ObServerLogBlockMGR abort_update_tenant success", KPC(this), K(old_log_disk_size), K(new_log_disk_size));
}
int ObServerLogBlockMgr::remove_tenant()
int ObServerLogBlockMgr::remove_tenant(const int64_t log_disk_size)
{
int ret = OB_SUCCESS;
ObSpinLockGuard guard(resize_lock_);
min_log_disk_size_for_all_tenants_ = 0;
if (OB_FAIL(get_tenants_log_disk_size_func_(min_log_disk_size_for_all_tenants_))) {
CLOG_LOG(ERROR, "get_tenants_log_disk_size_func_ failed", K(ret), KPC(this));
if (min_log_disk_size_for_all_tenants_ - log_disk_size < 0) {
ret = OB_ERR_UNEXPECTED;
CLOG_LOG(ERROR, "unexpected error, min_log_disk_size_for_all_tenants_ is small than zero",
K(ret), KPC(this), K(log_disk_size));
} else {
CLOG_LOG(INFO, "remove_tenant success", K(ret), KPC(this));
min_log_disk_size_for_all_tenants_ -= log_disk_size;
CLOG_LOG(INFO, "remove tenant from ObServerLogBlockMGR success", KPC(this), K(log_disk_size));
}
return ret;
}

2
src/logservice/ob_server_log_block_mgr.h
View File

@ -198,7 +198,7 @@ public:
// @brief after 'del_tenant' in ObMultiTenant success, need remove it from ObServerLogBlockMgr
// NB: accurately, when tenant not exist in 'omt_', we can remove it from ObServerLogBlockMgr
int remove_tenant();
int remove_tenant(const int64_t log_disk_size);
TO_STRING_KV("dir:",
log_pool_path_, K_(dir_fd), K_(meta_fd), K_(log_pool_meta),

93
src/observer/omt/ob_multi_tenant.cpp
View File

@ -523,6 +523,8 @@ int ObMultiTenant::create_hidden_sys_tenant()
LOG_ERROR("fail to construct meta", K(ret));
} else if (OB_FAIL(create_tenant(meta, true/* write_slog*/))) {
LOG_ERROR("create hidden sys tenant failed", K(ret));
} else {
LOG_INFO("create hidden sys tenant success", K(ret), K(meta));
}
return ret;
@ -658,10 +660,10 @@ int ObMultiTenant::convert_hidden_to_real_sys_tenant(const ObUnitInfoGetter::ObT
LOG_WARN("must be hidden sys tenant", K(ret));
} else if (FALSE_IT(new_super_block = tenant->get_super_block())) {
} else if (FALSE_IT(new_super_block.is_hidden_ = false)) {
} else if (OB_FAIL(update_tenant_unit_no_lock(unit))) {
} else if (OB_FAIL(update_tenant_unit_no_lock(unit, UpdateTenantConfigOpt::CONVERT_HIDDEN_TO_REAL_SYS_TENANT))) {
LOG_WARN("fail to update_tenant_unit_no_lock", K(ret), K(unit));
} else if (ObServerCheckpointSlogHandler::get_instance()
.write_tenant_super_block_slog(new_super_block)) {
} else if (OB_FAIL(ObServerCheckpointSlogHandler::get_instance()
.write_tenant_super_block_slog(new_super_block))) {
LOG_WARN("fail to write_tenant_super_block_slog", K(ret), K(new_super_block));
} else {
tenant->set_tenant_super_block(new_super_block);
@ -692,6 +694,7 @@ int ObMultiTenant::create_tenant(const ObTenantMeta &meta, bool write_slog, cons
int64_t bucket_lock_idx = -1;
const int64_t log_disk_size = meta.unit_.config_.log_disk_size();
int64_t lock_timeout_ts = abs_timeout_us - 5000000; // reserve 5s for creating tenant
bool is_hidden_sys = meta.is_hidden();
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
@ -739,6 +742,20 @@ int ObMultiTenant::create_tenant(const ObTenantMeta &meta, bool write_slog, cons
create_step = ObTenantCreateStep::STEP_CTX_MEM_CONFIG_SETTED; // step1
}
}
if (OB_SUCC(ret)) {
if (!is_virtual_tenant_id(tenant_id)
&& !is_hidden_sys
&& OB_FAIL(GCTX.log_block_mgr_->create_tenant(log_disk_size))) {
LOG_ERROR("create_tenatn in ObServerLogBlockMgr failed", KR(ret));
}
// if create_tenant in ObServerLogBlockMGR success, the log disk size need by this tenant has been pinned,
// otherwise, the assigned log disk size of ObServerLogBlockMGR is origin.
if (OB_SUCC(ret)) {
create_step = ObTenantCreateStep::STEP_LOG_DISK_SIZE_PINNED;
}
}
if (OB_FAIL(ret)) {
// do nothing
} else if (OB_ISNULL(GCTX.cgroup_ctrl_)) {
@ -748,15 +765,14 @@ int ObMultiTenant::create_tenant(const ObTenantMeta &meta, bool write_slog, cons
ObTenant, ObModIds::OMT, tenant_id, GCONF.workers_per_cpu_quota.get_value(), *GCTX.cgroup_ctrl_))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("new tenant fail", K(ret));
} else if (FALSE_IT(create_step = ObTenantCreateStep::STEP_TENANT_NEWED)) { //step2
} else if (FALSE_IT(create_step = ObTenantCreateStep::STEP_TENANT_NEWED)) { //step3
} else if (OB_FAIL(tenant->init_ctx())) {
LOG_WARN("init ctx fail", K(tenant_id), K(ret));
} else if (write_slog) {
if (OB_FAIL(write_create_tenant_prepare_slog(meta))) {
LOG_ERROR("fail to write create tenant prepare slog", K(ret));
} else {
create_step = ObTenantCreateStep::STEP_WRITE_PREPARE_SLOG; // step3
create_step = ObTenantCreateStep::STEP_WRITE_PREPARE_SLOG; // step4
}
}
@ -791,7 +807,7 @@ int ObMultiTenant::create_tenant(const ObTenantMeta &meta, bool write_slog, cons
LOG_ERROR("fail to write create tenant commit slog", K(ret), K(tenant_id));
} else {
tenant->set_create_status(ObTenantCreateStatus::CREATE_COMMIT);
create_step = ObTenantCreateStep::STEP_FINISH; // step4
create_step = ObTenantCreateStep::STEP_FINISH; // step5
}
}
@ -802,19 +818,9 @@ int ObMultiTenant::create_tenant(const ObTenantMeta &meta, bool write_slog, cons
if (OB_SUCC(get_tenant_unsafe(tenant_id, tmp_tenant))) {
ret = OB_TENANT_EXIST;
LOG_ERROR("tenant exist", K(ret), K(tenant_id));
// NB: hold log disk firstly, guarded by lock_ to avoid create tenant concurrently.
} else if (false == is_virtual_tenant_id(tenant_id) && OB_FAIL(GCTX.log_block_mgr_->create_tenant(log_disk_size))) {
LOG_WARN("create_tenant in ObServerLogBlockMgr failed", K(ret), K(meta));
} else if (OB_FAIL(tenants_.insert(tenant, iter, compare_tenant))) {
LOG_ERROR("fail to insert tenant", K(ret), K(tenant_id));
}
// Need ensure that after insert tenant into tenants_ success, means create tenant success.
// NB:
// 1. need call ObServerLogBlockMgr::abort_create_tenant after create tenant failed.
// 2. need guarded by lock, avoid concurrent with create_tenant or del_tenant
if (false == is_valid_tenant_id(tenant_id) && OB_FAIL(ret)) {
GCTX.log_block_mgr_->abort_create_tenant(log_disk_size);
}
}
// TODO: @lingyang 预期不能失败
if (!is_virtual_tenant_id(tenant_id) && OB_TMP_FAIL(update_tenant_config(tenant_id))) {
@ -854,6 +860,13 @@ int ObMultiTenant::create_tenant(const ObTenantMeta &meta, bool write_slog, cons
}
} while (OB_SUCCESS != tmp_ret);
do {
tmp_ret = OB_SUCCESS;
if (create_step >= ObTenantCreateStep::STEP_LOG_DISK_SIZE_PINNED) {
GCTX.log_block_mgr_->abort_create_tenant(log_disk_size);
}
} while (OB_SUCCESS != tmp_ret);
if (create_step >= ObTenantCreateStep::STEP_WRITE_PREPARE_SLOG) {
if (OB_SUCCESS != (tmp_ret = write_create_tenant_abort_slog(tenant_id))) {
LOG_ERROR("fail to write create tenant abort slog", K(tmp_ret));
@ -869,7 +882,8 @@ int ObMultiTenant::create_tenant(const ObTenantMeta &meta, bool write_slog, cons
return ret;
}
int ObMultiTenant::update_tenant_unit_no_lock(const ObUnitInfoGetter::ObTenantConfig &unit)
int ObMultiTenant::update_tenant_unit_no_lock(const ObUnitInfoGetter::ObTenantConfig &unit,
const UpdateTenantConfigOpt &opt)
{
int ret = OB_SUCCESS;
@ -878,6 +892,9 @@ int ObMultiTenant::update_tenant_unit_no_lock(const ObUnitInfoGetter::ObTenantCo
const double max_cpu = static_cast<double>(unit.config_.max_cpu());
const uint64_t tenant_id = unit.tenant_id_;
int64_t allowed_mem_limit = 0;
// when opt is UpdateTenantConfigOpt::CONVERT_REAL_TO_HIDDEN_SYS_TENANT, the new_log_disk_size should set to zero.
const int64_t new_log_disk_size =
opt == UpdateTenantConfigOpt::CONVERT_REAL_TO_HIDDEN_SYS_TENANT ? 0 : unit.config_.log_disk_size();
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
@ -895,7 +912,7 @@ int ObMultiTenant::update_tenant_unit_no_lock(const ObUnitInfoGetter::ObTenantCo
LOG_WARN("fail to update_tenant_freezer_mem_limit", K(ret), K(tenant_id));
} else if (OB_FAIL(tenant->update_thread_cnt(max_cpu))) {
LOG_WARN("fail to update mtl module thread_cnt", K(ret), K(tenant_id));
} else if (OB_FAIL(update_tenant_log_disk_size(tenant_id, unit.config_.log_disk_size()))) {
} else if (OB_FAIL(update_tenant_log_disk_size(tenant_id, new_log_disk_size, opt))) {
LOG_WARN("fail to update tenant log disk size", K(ret), K(tenant_id));
} else if (FALSE_IT(tenant->update_memory_size(unit.config_.memory_size()))) {
// unreachable
@ -926,7 +943,7 @@ int ObMultiTenant::update_tenant_unit(const ObUnitInfoGetter::ObTenantConfig &un
} else if (OB_FAIL(bucket_lock_.wrlock(bucket_lock_idx = get_tenant_lock_bucket_idx(tenant_id)))) {
LOG_WARN("fail to try_wrlock for update tenant unit", K(ret), K(tenant_id), K(bucket_lock_idx));
} else if (FALSE_IT(lock_succ = true)) {
} else if (OB_FAIL(update_tenant_unit_no_lock(unit))) {
} else if (OB_FAIL(update_tenant_unit_no_lock(unit, UpdateTenantConfigOpt::NORMAL_UPDATE_TENANT_CONFIG_OPT))) {
LOG_WARN("fail to update_tenant_unit_no_lock", K(ret), K(unit));
}
@ -976,7 +993,8 @@ int ObMultiTenant::update_tenant_memory(const uint64_t tenant_id, const int64_t
}
int ObMultiTenant::update_tenant_log_disk_size(const uint64_t tenant_id,
const int64_t expected_log_disk_size)
const int64_t expected_log_disk_size,
const UpdateTenantConfigOpt &opt)
{
int ret = OB_SUCCESS;
MAKE_TENANT_SWITCH_SCOPE_GUARD(guard);
@ -986,23 +1004,26 @@ int ObMultiTenant::update_tenant_log_disk_size(const uint64_t tenant_id,
ObLogService *log_service = MTL(ObLogService *);
if (OB_ISNULL(log_service)) {
ret = OB_ERR_UNEXPECTED;
} else if (OB_FAIL(log_service->get_palf_disk_usage(unused_log_disk_size, old_log_disk_size))) {
// when opt is UpdateTenantConfigOpt::CONVERT_HIDDEN_TO_REAL_SYS_TENANT, the old_log_disk_size is zero.
} else if (opt != UpdateTenantConfigOpt::CONVERT_HIDDEN_TO_REAL_SYS_TENANT
&& OB_FAIL(log_service->get_palf_disk_usage(unused_log_disk_size, old_log_disk_size))) {
LOG_WARN("failed to get_palf_disk_usage", K(ret), K(unused_log_disk_size), K(old_log_disk_size),
K(expected_log_disk_size));
} else if (old_log_disk_size == expected_log_disk_size) {
LOG_INFO("no need to update tenant log disk size", K(tenant_id), K(old_log_disk_size), K(expected_log_disk_size));
} else if(false == is_valid_tenant_id(tenant_id)
&& OB_FAIL(GCTX.log_block_mgr_->update_tenant(old_log_disk_size, expected_log_disk_size))) {
} else if(OB_FAIL(GCTX.log_block_mgr_->update_tenant(old_log_disk_size, expected_log_disk_size))) {
LOG_WARN("failed to update_tenant in ObServerLogBlockMgr", K(ret), K(old_log_disk_size),
K(expected_log_disk_size));
} else {
if (OB_FAIL(log_service->update_log_disk_usage_limit_size(expected_log_disk_size))) {
LOG_WARN("failed to update_log_disk_usage_limit_size", K(ret), K(tenant_id),
K(expected_log_disk_size));
} else {
LOG_INFO("update_tenant_log_disk_size success", K(ret), K(tenant_id),
K(expected_log_disk_size));
// Only when opt is UpdateTenantConfigOpt::NORMAL_UPDATE_TENANT_CONFIG_OPT update palf.
if (opt == UpdateTenantConfigOpt::NORMAL_UPDATE_TENANT_CONFIG_OPT) {
if (OB_FAIL(log_service->update_log_disk_usage_limit_size(expected_log_disk_size))) {
LOG_WARN("failed to update_log_disk_usage_limit_size", K(ret), K(tenant_id),
K(expected_log_disk_size));
} else {
LOG_INFO("update_tenant_log_disk_size success", K(ret), K(tenant_id), K(opt),
K(expected_log_disk_size));
}
}
if (false == is_virtual_tenant_id(tenant_id) && OB_FAIL(ret)) {
GCTX.log_block_mgr_->abort_update_tenant(expected_log_disk_size, old_log_disk_size);
}
@ -1382,6 +1403,11 @@ int ObMultiTenant::remove_tenant(const uint64_t tenant_id, bool &try_clock_succ)
KP(removed_tenant), K(removed_tenant->lock_));
removed_tenant->lock_.ld_.print();
} else {
const share::ObUnitInfoGetter::ObTenantConfig &config = removed_tenant->get_unit();
const int64_t log_disk_size = config.config_.log_disk_size();
if (!removed_tenant->is_hidden() && !is_virtual_tenant_id(tenant_id)) {
GCTX.log_block_mgr_->remove_tenant(log_disk_size);
}
ObTenant *removed_tenant_tmp = nullptr;
SpinWLockGuard guard(lock_);
@ -1395,7 +1421,6 @@ int ObMultiTenant::remove_tenant(const uint64_t tenant_id, bool &try_clock_succ)
}
if (OB_SUCC(ret)) {
GCTX.log_block_mgr_->remove_tenant();
removed_tenant->destroy();
ob_delete(removed_tenant);
LOG_INFO("remove tenant success", K(tenant_id));
@ -1594,7 +1619,7 @@ int ObMultiTenant::convert_real_to_hidden_sys_tenant()
} else if (OB_FAIL(bucket_lock_.try_wrlock(bucket_lock_idx = get_tenant_lock_bucket_idx(tenant_id)))) {
LOG_WARN("fail to try_wrlock for delete tenant", K(ret), K(tenant_id), K(bucket_lock_idx));
} else if (FALSE_IT(lock_succ = true)) {
} else if (OB_FAIL(update_tenant_unit_no_lock(tenant_meta.unit_))) {
} else if (OB_FAIL(update_tenant_unit_no_lock(tenant_meta.unit_, UpdateTenantConfigOpt::CONVERT_REAL_TO_HIDDEN_SYS_TENANT))) {
LOG_WARN("fail to update_tenant_unit_no_lock", K(ret), K(tenant_meta));
} else {
ObTenantSwitchGuard guard(tenant);

17
src/observer/omt/ob_multi_tenant.h
View File

@ -75,6 +75,12 @@ typedef common::ObSortedVector<ObTenant*> TenantList;
typedef TenantList::iterator TenantIterator;
typedef common::ObVector<uint64_t> TenantIdList;
enum class UpdateTenantConfigOpt {
CONVERT_HIDDEN_TO_REAL_SYS_TENANT = 0,
CONVERT_REAL_TO_HIDDEN_SYS_TENANT = 1,
NORMAL_UPDATE_TENANT_CONFIG_OPT = 2
};
// This is the entry class of OMT module.
class ObMultiTenant
: public share::ObThreadPool
@ -112,7 +118,8 @@ public:
int update_tenant_cpu(const uint64_t tenant_id, const double min_cpu, const double max_cpu);
int update_tenant_memory(const uint64_t tenant_id, const int64_t mem_limit, int64_t &allowed_mem_limit);
int update_tenant_log_disk_size(const uint64_t tenant_id,
const int64_t expected_log_disk_size);
const int64_t expected_log_disk_size,
const UpdateTenantConfigOpt &opt);
int modify_tenant_io(const uint64_t tenant_id, const share::ObUnitConfig &unit_config);
int update_tenant_config(uint64_t tenant_id);
int update_palf_config();
@ -178,15 +185,17 @@ protected:
int create_virtual_tenants();
int remove_tenant(const uint64_t tenant_id, bool &lock_succ);
uint32_t get_tenant_lock_bucket_idx(const uint64_t tenant_id);
int update_tenant_unit_no_lock(const share::ObUnitInfoGetter::ObTenantConfig &unit);
int update_tenant_unit_no_lock(const share::ObUnitInfoGetter::ObTenantConfig &unit,
const UpdateTenantConfigOpt &opt);
protected:
static const int DEL_TRY_TIMES = 30;
enum class ObTenantCreateStep {
STEP_BEGIN = 0, // begin
STEP_CTX_MEM_CONFIG_SETTED = 1, // set_tenant_ctx_idle succ
STEP_TENANT_NEWED = 2, // new tenant succ
STEP_WRITE_PREPARE_SLOG = 3, // write_prepare_create_tenant_slog succ
STEP_LOG_DISK_SIZE_PINNED = 2, // pin log disk size succ
STEP_TENANT_NEWED = 3, // new tenant succ
STEP_WRITE_PREPARE_SLOG = 4, // write_prepare_create_tenant_slog succ
STEP_FINISH,
};

2
src/observer/omt/ob_tenant.cpp
View File

@ -728,7 +728,7 @@ int ObTenant::construct_mtl_init_ctx(const ObTenantMeta &meta, share::ObTenantMo
bool ObTenant::is_hidden()
{
TCRLockGuard guard(meta_lock_);
return tenant_meta_.super_block_.is_hidden_;
return tenant_meta_.is_hidden();
}
ObTenantCreateStatus ObTenant::get_create_status()

5
src/observer/omt/ob_tenant_meta.h
View File

@ -51,6 +51,11 @@ public:
int build(const share::ObUnitInfoGetter::ObTenantConfig &unit,
const storage::ObTenantSuperBlock &super_block);
bool is_hidden() const
{
return super_block_.is_hidden_;
}
TO_STRING_KV(K_(unit), K_(super_block), K_(create_status));

6
unittest/logservice/test_server_log_block_mgr.cpp
View File

@ -130,7 +130,7 @@ public:
if (OB_SUCC(ret)) {
map_[key] = size;
} else {
this->log_block_mgr_.abort_create_tenant(size);
//this->log_block_mgr_.abort_create_tenant(size);
}
}
return ret;
@ -141,8 +141,9 @@ public:
if (map_.end() == map_.find(key)) {
ret = OB_ENTRY_NOT_EXIST;
} else {
int64 size = map_[key];
map_.erase(key);
this->log_block_mgr_.remove_tenant();
this->log_block_mgr_.remove_tenant(size);
}
return ret;
};
@ -156,7 +157,6 @@ public:
ret = this->log_block_mgr_.update_tenant(old_size, new_size);
if (OB_SUCCESS != ret) {
map_[key] = old_size;
this->log_block_mgr_.abort_update_tenant(new_size, old_size);
} else {
map_[key] = new_size;
}