diff --git a/src/logservice/ob_server_log_block_mgr.cpp b/src/logservice/ob_server_log_block_mgr.cpp index 6d29eb29c7..61e306c460 100644 --- a/src/logservice/ob_server_log_block_mgr.cpp +++ b/src/logservice/ob_server_log_block_mgr.cpp @@ -301,11 +301,13 @@ int ObServerLogBlockMgr::create_tenant(const int64_t log_disk_size) { int ret = OB_SUCCESS; ObSpinLockGuard guard(resize_lock_); - if ((min_log_disk_size_for_all_tenants_ += log_disk_size) > get_total_size_guarded_by_lock_()) { + int64_t tmp_log_disk_size = min_log_disk_size_for_all_tenants_; + if ((tmp_log_disk_size += log_disk_size) > get_total_size_guarded_by_lock_()) { ret = OB_MACHINE_RESOURCE_NOT_ENOUGH; CLOG_LOG(ERROR, "ObServerLogBlockMGR can not hold any new tenants", K(ret), KPC(this), K(log_disk_size)); } else { + min_log_disk_size_for_all_tenants_ = tmp_log_disk_size; CLOG_LOG(INFO, "ObServerLogBlockMGR create_tenant success", KPC(this), K(log_disk_size)); } return ret; @@ -324,14 +326,15 @@ int ObServerLogBlockMgr::update_tenant(const int64_t old_log_disk_size, const in { int ret = OB_SUCCESS; ObSpinLockGuard guard(resize_lock_); - min_log_disk_size_for_all_tenants_ -= old_log_disk_size; - if (old_log_disk_size == new_log_disk_size) { - } else if ((min_log_disk_size_for_all_tenants_+=new_log_disk_size) > get_total_size_guarded_by_lock_()) { + int64_t tmp_log_disk_size = min_log_disk_size_for_all_tenants_; + tmp_log_disk_size -= old_log_disk_size; + if ((tmp_log_disk_size +=new_log_disk_size) > get_total_size_guarded_by_lock_()) { ret = OB_MACHINE_RESOURCE_NOT_ENOUGH; CLOG_LOG(ERROR, "ObServerLogBlockMGR can not hold any new tenants", K(ret), KPC(this), K(old_log_disk_size), K(new_log_disk_size)); } else { - CLOG_LOG(INFO, "ObServerLogBlockMGR update_tenant success", K(ret), KPC(this), K(old_log_disk_size), K(new_log_disk_size)); + min_log_disk_size_for_all_tenants_ = tmp_log_disk_size; + CLOG_LOG(INFO, "ObServerLogBlockMGR update_tenant success", KPC(this), K(old_log_disk_size), K(new_log_disk_size)); } return ret; } @@ -346,15 +349,17 @@ void ObServerLogBlockMgr::abort_update_tenant(const int64_t old_log_disk_size, c CLOG_LOG(INFO, "ObServerLogBlockMGR abort_update_tenant success", KPC(this), K(old_log_disk_size), K(new_log_disk_size)); } -int ObServerLogBlockMgr::remove_tenant() +int ObServerLogBlockMgr::remove_tenant(const int64_t log_disk_size) { int ret = OB_SUCCESS; ObSpinLockGuard guard(resize_lock_); - min_log_disk_size_for_all_tenants_ = 0; - if (OB_FAIL(get_tenants_log_disk_size_func_(min_log_disk_size_for_all_tenants_))) { - CLOG_LOG(ERROR, "get_tenants_log_disk_size_func_ failed", K(ret), KPC(this)); + if (min_log_disk_size_for_all_tenants_ - log_disk_size < 0) { + ret = OB_ERR_UNEXPECTED; + CLOG_LOG(ERROR, "unexpected error, min_log_disk_size_for_all_tenants_ is small than zero", + K(ret), KPC(this), K(log_disk_size)); } else { - CLOG_LOG(INFO, "remove_tenant success", K(ret), KPC(this)); + min_log_disk_size_for_all_tenants_ -= log_disk_size; + CLOG_LOG(INFO, "remove tenant from ObServerLogBlockMGR success", KPC(this), K(log_disk_size)); } return ret; } diff --git a/src/logservice/ob_server_log_block_mgr.h b/src/logservice/ob_server_log_block_mgr.h index ec306290e2..ee6ff1eda2 100644 --- a/src/logservice/ob_server_log_block_mgr.h +++ b/src/logservice/ob_server_log_block_mgr.h @@ -198,7 +198,7 @@ public: // @brief after 'del_tenant' in ObMultiTenant success, need remove it from ObServerLogBlockMgr // NB: accurately, when tenant not exist in 'omt_', we can remove it from ObServerLogBlockMgr - int remove_tenant(); + int remove_tenant(const int64_t log_disk_size); TO_STRING_KV("dir:", log_pool_path_, K_(dir_fd), K_(meta_fd), K_(log_pool_meta), diff --git a/src/observer/omt/ob_multi_tenant.cpp b/src/observer/omt/ob_multi_tenant.cpp index fa3e2d29c2..58d9c6da02 100644 --- a/src/observer/omt/ob_multi_tenant.cpp +++ b/src/observer/omt/ob_multi_tenant.cpp @@ -523,6 +523,8 @@ int ObMultiTenant::create_hidden_sys_tenant() LOG_ERROR("fail to construct meta", K(ret)); } else if (OB_FAIL(create_tenant(meta, true/* write_slog*/))) { LOG_ERROR("create hidden sys tenant failed", K(ret)); + } else { + LOG_INFO("create hidden sys tenant success", K(ret), K(meta)); } return ret; @@ -658,10 +660,10 @@ int ObMultiTenant::convert_hidden_to_real_sys_tenant(const ObUnitInfoGetter::ObT LOG_WARN("must be hidden sys tenant", K(ret)); } else if (FALSE_IT(new_super_block = tenant->get_super_block())) { } else if (FALSE_IT(new_super_block.is_hidden_ = false)) { - } else if (OB_FAIL(update_tenant_unit_no_lock(unit))) { + } else if (OB_FAIL(update_tenant_unit_no_lock(unit, UpdateTenantConfigOpt::CONVERT_HIDDEN_TO_REAL_SYS_TENANT))) { LOG_WARN("fail to update_tenant_unit_no_lock", K(ret), K(unit)); - } else if (ObServerCheckpointSlogHandler::get_instance() - .write_tenant_super_block_slog(new_super_block)) { + } else if (OB_FAIL(ObServerCheckpointSlogHandler::get_instance() + .write_tenant_super_block_slog(new_super_block))) { LOG_WARN("fail to write_tenant_super_block_slog", K(ret), K(new_super_block)); } else { tenant->set_tenant_super_block(new_super_block); @@ -692,6 +694,7 @@ int ObMultiTenant::create_tenant(const ObTenantMeta &meta, bool write_slog, cons int64_t bucket_lock_idx = -1; const int64_t log_disk_size = meta.unit_.config_.log_disk_size(); int64_t lock_timeout_ts = abs_timeout_us - 5000000; // reserve 5s for creating tenant + bool is_hidden_sys = meta.is_hidden(); if (IS_NOT_INIT) { ret = OB_NOT_INIT; @@ -739,6 +742,20 @@ int ObMultiTenant::create_tenant(const ObTenantMeta &meta, bool write_slog, cons create_step = ObTenantCreateStep::STEP_CTX_MEM_CONFIG_SETTED; // step1 } } + + if (OB_SUCC(ret)) { + if (!is_virtual_tenant_id(tenant_id) + && !is_hidden_sys + && OB_FAIL(GCTX.log_block_mgr_->create_tenant(log_disk_size))) { + LOG_ERROR("create_tenatn in ObServerLogBlockMgr failed", KR(ret)); + } + // if create_tenant in ObServerLogBlockMGR success, the log disk size need by this tenant has been pinned, + // otherwise, the assigned log disk size of ObServerLogBlockMGR is origin. + if (OB_SUCC(ret)) { + create_step = ObTenantCreateStep::STEP_LOG_DISK_SIZE_PINNED; + } + } + if (OB_FAIL(ret)) { // do nothing } else if (OB_ISNULL(GCTX.cgroup_ctrl_)) { @@ -748,15 +765,14 @@ int ObMultiTenant::create_tenant(const ObTenantMeta &meta, bool write_slog, cons ObTenant, ObModIds::OMT, tenant_id, GCONF.workers_per_cpu_quota.get_value(), *GCTX.cgroup_ctrl_))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("new tenant fail", K(ret)); - } else if (FALSE_IT(create_step = ObTenantCreateStep::STEP_TENANT_NEWED)) { //step2 - + } else if (FALSE_IT(create_step = ObTenantCreateStep::STEP_TENANT_NEWED)) { //step3 } else if (OB_FAIL(tenant->init_ctx())) { LOG_WARN("init ctx fail", K(tenant_id), K(ret)); } else if (write_slog) { if (OB_FAIL(write_create_tenant_prepare_slog(meta))) { LOG_ERROR("fail to write create tenant prepare slog", K(ret)); } else { - create_step = ObTenantCreateStep::STEP_WRITE_PREPARE_SLOG; // step3 + create_step = ObTenantCreateStep::STEP_WRITE_PREPARE_SLOG; // step4 } } @@ -791,7 +807,7 @@ int ObMultiTenant::create_tenant(const ObTenantMeta &meta, bool write_slog, cons LOG_ERROR("fail to write create tenant commit slog", K(ret), K(tenant_id)); } else { tenant->set_create_status(ObTenantCreateStatus::CREATE_COMMIT); - create_step = ObTenantCreateStep::STEP_FINISH; // step4 + create_step = ObTenantCreateStep::STEP_FINISH; // step5 } } @@ -802,19 +818,9 @@ int ObMultiTenant::create_tenant(const ObTenantMeta &meta, bool write_slog, cons if (OB_SUCC(get_tenant_unsafe(tenant_id, tmp_tenant))) { ret = OB_TENANT_EXIST; LOG_ERROR("tenant exist", K(ret), K(tenant_id)); - // NB: hold log disk firstly, guarded by lock_ to avoid create tenant concurrently. - } else if (false == is_virtual_tenant_id(tenant_id) && OB_FAIL(GCTX.log_block_mgr_->create_tenant(log_disk_size))) { - LOG_WARN("create_tenant in ObServerLogBlockMgr failed", K(ret), K(meta)); } else if (OB_FAIL(tenants_.insert(tenant, iter, compare_tenant))) { LOG_ERROR("fail to insert tenant", K(ret), K(tenant_id)); } - // Need ensure that after insert tenant into tenants_ success, means create tenant success. - // NB: - // 1. need call ObServerLogBlockMgr::abort_create_tenant after create tenant failed. - // 2. need guarded by lock, avoid concurrent with create_tenant or del_tenant - if (false == is_valid_tenant_id(tenant_id) && OB_FAIL(ret)) { - GCTX.log_block_mgr_->abort_create_tenant(log_disk_size); - } } // TODO: @lingyang 预期不能失败 if (!is_virtual_tenant_id(tenant_id) && OB_TMP_FAIL(update_tenant_config(tenant_id))) { @@ -854,6 +860,13 @@ int ObMultiTenant::create_tenant(const ObTenantMeta &meta, bool write_slog, cons } } while (OB_SUCCESS != tmp_ret); + do { + tmp_ret = OB_SUCCESS; + if (create_step >= ObTenantCreateStep::STEP_LOG_DISK_SIZE_PINNED) { + GCTX.log_block_mgr_->abort_create_tenant(log_disk_size); + } + } while (OB_SUCCESS != tmp_ret); + if (create_step >= ObTenantCreateStep::STEP_WRITE_PREPARE_SLOG) { if (OB_SUCCESS != (tmp_ret = write_create_tenant_abort_slog(tenant_id))) { LOG_ERROR("fail to write create tenant abort slog", K(tmp_ret)); @@ -869,7 +882,8 @@ int ObMultiTenant::create_tenant(const ObTenantMeta &meta, bool write_slog, cons return ret; } -int ObMultiTenant::update_tenant_unit_no_lock(const ObUnitInfoGetter::ObTenantConfig &unit) +int ObMultiTenant::update_tenant_unit_no_lock(const ObUnitInfoGetter::ObTenantConfig &unit, + const UpdateTenantConfigOpt &opt) { int ret = OB_SUCCESS; @@ -878,6 +892,9 @@ int ObMultiTenant::update_tenant_unit_no_lock(const ObUnitInfoGetter::ObTenantCo const double max_cpu = static_cast(unit.config_.max_cpu()); const uint64_t tenant_id = unit.tenant_id_; int64_t allowed_mem_limit = 0; + // when opt is UpdateTenantConfigOpt::CONVERT_REAL_TO_HIDDEN_SYS_TENANT, the new_log_disk_size should set to zero. + const int64_t new_log_disk_size = + opt == UpdateTenantConfigOpt::CONVERT_REAL_TO_HIDDEN_SYS_TENANT ? 0 : unit.config_.log_disk_size(); if (IS_NOT_INIT) { ret = OB_NOT_INIT; @@ -895,7 +912,7 @@ int ObMultiTenant::update_tenant_unit_no_lock(const ObUnitInfoGetter::ObTenantCo LOG_WARN("fail to update_tenant_freezer_mem_limit", K(ret), K(tenant_id)); } else if (OB_FAIL(tenant->update_thread_cnt(max_cpu))) { LOG_WARN("fail to update mtl module thread_cnt", K(ret), K(tenant_id)); - } else if (OB_FAIL(update_tenant_log_disk_size(tenant_id, unit.config_.log_disk_size()))) { + } else if (OB_FAIL(update_tenant_log_disk_size(tenant_id, new_log_disk_size, opt))) { LOG_WARN("fail to update tenant log disk size", K(ret), K(tenant_id)); } else if (FALSE_IT(tenant->update_memory_size(unit.config_.memory_size()))) { // unreachable @@ -926,7 +943,7 @@ int ObMultiTenant::update_tenant_unit(const ObUnitInfoGetter::ObTenantConfig &un } else if (OB_FAIL(bucket_lock_.wrlock(bucket_lock_idx = get_tenant_lock_bucket_idx(tenant_id)))) { LOG_WARN("fail to try_wrlock for update tenant unit", K(ret), K(tenant_id), K(bucket_lock_idx)); } else if (FALSE_IT(lock_succ = true)) { - } else if (OB_FAIL(update_tenant_unit_no_lock(unit))) { + } else if (OB_FAIL(update_tenant_unit_no_lock(unit, UpdateTenantConfigOpt::NORMAL_UPDATE_TENANT_CONFIG_OPT))) { LOG_WARN("fail to update_tenant_unit_no_lock", K(ret), K(unit)); } @@ -976,7 +993,8 @@ int ObMultiTenant::update_tenant_memory(const uint64_t tenant_id, const int64_t } int ObMultiTenant::update_tenant_log_disk_size(const uint64_t tenant_id, - const int64_t expected_log_disk_size) + const int64_t expected_log_disk_size, + const UpdateTenantConfigOpt &opt) { int ret = OB_SUCCESS; MAKE_TENANT_SWITCH_SCOPE_GUARD(guard); @@ -986,23 +1004,26 @@ int ObMultiTenant::update_tenant_log_disk_size(const uint64_t tenant_id, ObLogService *log_service = MTL(ObLogService *); if (OB_ISNULL(log_service)) { ret = OB_ERR_UNEXPECTED; - } else if (OB_FAIL(log_service->get_palf_disk_usage(unused_log_disk_size, old_log_disk_size))) { + // when opt is UpdateTenantConfigOpt::CONVERT_HIDDEN_TO_REAL_SYS_TENANT, the old_log_disk_size is zero. + } else if (opt != UpdateTenantConfigOpt::CONVERT_HIDDEN_TO_REAL_SYS_TENANT + && OB_FAIL(log_service->get_palf_disk_usage(unused_log_disk_size, old_log_disk_size))) { LOG_WARN("failed to get_palf_disk_usage", K(ret), K(unused_log_disk_size), K(old_log_disk_size), K(expected_log_disk_size)); - } else if (old_log_disk_size == expected_log_disk_size) { - LOG_INFO("no need to update tenant log disk size", K(tenant_id), K(old_log_disk_size), K(expected_log_disk_size)); - } else if(false == is_valid_tenant_id(tenant_id) - && OB_FAIL(GCTX.log_block_mgr_->update_tenant(old_log_disk_size, expected_log_disk_size))) { + } else if(OB_FAIL(GCTX.log_block_mgr_->update_tenant(old_log_disk_size, expected_log_disk_size))) { LOG_WARN("failed to update_tenant in ObServerLogBlockMgr", K(ret), K(old_log_disk_size), K(expected_log_disk_size)); } else { - if (OB_FAIL(log_service->update_log_disk_usage_limit_size(expected_log_disk_size))) { - LOG_WARN("failed to update_log_disk_usage_limit_size", K(ret), K(tenant_id), - K(expected_log_disk_size)); - } else { - LOG_INFO("update_tenant_log_disk_size success", K(ret), K(tenant_id), - K(expected_log_disk_size)); + // Only when opt is UpdateTenantConfigOpt::NORMAL_UPDATE_TENANT_CONFIG_OPT update palf. + if (opt == UpdateTenantConfigOpt::NORMAL_UPDATE_TENANT_CONFIG_OPT) { + if (OB_FAIL(log_service->update_log_disk_usage_limit_size(expected_log_disk_size))) { + LOG_WARN("failed to update_log_disk_usage_limit_size", K(ret), K(tenant_id), + K(expected_log_disk_size)); + } else { + LOG_INFO("update_tenant_log_disk_size success", K(ret), K(tenant_id), K(opt), + K(expected_log_disk_size)); + } } + if (false == is_virtual_tenant_id(tenant_id) && OB_FAIL(ret)) { GCTX.log_block_mgr_->abort_update_tenant(expected_log_disk_size, old_log_disk_size); } @@ -1382,6 +1403,11 @@ int ObMultiTenant::remove_tenant(const uint64_t tenant_id, bool &try_clock_succ) KP(removed_tenant), K(removed_tenant->lock_)); removed_tenant->lock_.ld_.print(); } else { + const share::ObUnitInfoGetter::ObTenantConfig &config = removed_tenant->get_unit(); + const int64_t log_disk_size = config.config_.log_disk_size(); + if (!removed_tenant->is_hidden() && !is_virtual_tenant_id(tenant_id)) { + GCTX.log_block_mgr_->remove_tenant(log_disk_size); + } ObTenant *removed_tenant_tmp = nullptr; SpinWLockGuard guard(lock_); @@ -1395,7 +1421,6 @@ int ObMultiTenant::remove_tenant(const uint64_t tenant_id, bool &try_clock_succ) } if (OB_SUCC(ret)) { - GCTX.log_block_mgr_->remove_tenant(); removed_tenant->destroy(); ob_delete(removed_tenant); LOG_INFO("remove tenant success", K(tenant_id)); @@ -1594,7 +1619,7 @@ int ObMultiTenant::convert_real_to_hidden_sys_tenant() } else if (OB_FAIL(bucket_lock_.try_wrlock(bucket_lock_idx = get_tenant_lock_bucket_idx(tenant_id)))) { LOG_WARN("fail to try_wrlock for delete tenant", K(ret), K(tenant_id), K(bucket_lock_idx)); } else if (FALSE_IT(lock_succ = true)) { - } else if (OB_FAIL(update_tenant_unit_no_lock(tenant_meta.unit_))) { + } else if (OB_FAIL(update_tenant_unit_no_lock(tenant_meta.unit_, UpdateTenantConfigOpt::CONVERT_REAL_TO_HIDDEN_SYS_TENANT))) { LOG_WARN("fail to update_tenant_unit_no_lock", K(ret), K(tenant_meta)); } else { ObTenantSwitchGuard guard(tenant); diff --git a/src/observer/omt/ob_multi_tenant.h b/src/observer/omt/ob_multi_tenant.h index 7228451edd..fd18bfc76c 100644 --- a/src/observer/omt/ob_multi_tenant.h +++ b/src/observer/omt/ob_multi_tenant.h @@ -75,6 +75,12 @@ typedef common::ObSortedVector TenantList; typedef TenantList::iterator TenantIterator; typedef common::ObVector TenantIdList; +enum class UpdateTenantConfigOpt { + CONVERT_HIDDEN_TO_REAL_SYS_TENANT = 0, + CONVERT_REAL_TO_HIDDEN_SYS_TENANT = 1, + NORMAL_UPDATE_TENANT_CONFIG_OPT = 2 +}; + // This is the entry class of OMT module. class ObMultiTenant : public share::ObThreadPool @@ -112,7 +118,8 @@ public: int update_tenant_cpu(const uint64_t tenant_id, const double min_cpu, const double max_cpu); int update_tenant_memory(const uint64_t tenant_id, const int64_t mem_limit, int64_t &allowed_mem_limit); int update_tenant_log_disk_size(const uint64_t tenant_id, - const int64_t expected_log_disk_size); + const int64_t expected_log_disk_size, + const UpdateTenantConfigOpt &opt); int modify_tenant_io(const uint64_t tenant_id, const share::ObUnitConfig &unit_config); int update_tenant_config(uint64_t tenant_id); int update_palf_config(); @@ -178,15 +185,17 @@ protected: int create_virtual_tenants(); int remove_tenant(const uint64_t tenant_id, bool &lock_succ); uint32_t get_tenant_lock_bucket_idx(const uint64_t tenant_id); - int update_tenant_unit_no_lock(const share::ObUnitInfoGetter::ObTenantConfig &unit); + int update_tenant_unit_no_lock(const share::ObUnitInfoGetter::ObTenantConfig &unit, + const UpdateTenantConfigOpt &opt); protected: static const int DEL_TRY_TIMES = 30; enum class ObTenantCreateStep { STEP_BEGIN = 0, // begin STEP_CTX_MEM_CONFIG_SETTED = 1, // set_tenant_ctx_idle succ - STEP_TENANT_NEWED = 2, // new tenant succ - STEP_WRITE_PREPARE_SLOG = 3, // write_prepare_create_tenant_slog succ + STEP_LOG_DISK_SIZE_PINNED = 2, // pin log disk size succ + STEP_TENANT_NEWED = 3, // new tenant succ + STEP_WRITE_PREPARE_SLOG = 4, // write_prepare_create_tenant_slog succ STEP_FINISH, }; diff --git a/src/observer/omt/ob_tenant.cpp b/src/observer/omt/ob_tenant.cpp index f7ac370e3f..f2102c9594 100644 --- a/src/observer/omt/ob_tenant.cpp +++ b/src/observer/omt/ob_tenant.cpp @@ -728,7 +728,7 @@ int ObTenant::construct_mtl_init_ctx(const ObTenantMeta &meta, share::ObTenantMo bool ObTenant::is_hidden() { TCRLockGuard guard(meta_lock_); - return tenant_meta_.super_block_.is_hidden_; + return tenant_meta_.is_hidden(); } ObTenantCreateStatus ObTenant::get_create_status() diff --git a/src/observer/omt/ob_tenant_meta.h b/src/observer/omt/ob_tenant_meta.h index f2156a2f11..34e1a0b608 100644 --- a/src/observer/omt/ob_tenant_meta.h +++ b/src/observer/omt/ob_tenant_meta.h @@ -51,6 +51,11 @@ public: int build(const share::ObUnitInfoGetter::ObTenantConfig &unit, const storage::ObTenantSuperBlock &super_block); + bool is_hidden() const + { + return super_block_.is_hidden_; + } + TO_STRING_KV(K_(unit), K_(super_block), K_(create_status)); diff --git a/unittest/logservice/test_server_log_block_mgr.cpp b/unittest/logservice/test_server_log_block_mgr.cpp index ff319f25d0..236f82b95d 100644 --- a/unittest/logservice/test_server_log_block_mgr.cpp +++ b/unittest/logservice/test_server_log_block_mgr.cpp @@ -130,7 +130,7 @@ public: if (OB_SUCC(ret)) { map_[key] = size; } else { - this->log_block_mgr_.abort_create_tenant(size); + //this->log_block_mgr_.abort_create_tenant(size); } } return ret; @@ -141,8 +141,9 @@ public: if (map_.end() == map_.find(key)) { ret = OB_ENTRY_NOT_EXIST; } else { + int64 size = map_[key]; map_.erase(key); - this->log_block_mgr_.remove_tenant(); + this->log_block_mgr_.remove_tenant(size); } return ret; }; @@ -156,7 +157,6 @@ public: ret = this->log_block_mgr_.update_tenant(old_size, new_size); if (OB_SUCCESS != ret) { map_[key] = old_size; - this->log_block_mgr_.abort_update_tenant(new_size, old_size); } else { map_[key] = new_size; }