From b28397d91ff8e4203700dfe215d70b0bec885a24 Mon Sep 17 00:00:00 2001 From: HaHaJeff Date: Mon, 8 May 2023 09:11:54 +0000 Subject: [PATCH] fixed query LOG_DISK_IN_USE from gv$ob_servers may be negative. --- .../test_observer_expand_shrink.cpp | 10 +++---- src/logservice/ob_server_log_block_mgr.cpp | 30 +++++++++++++++---- src/logservice/ob_server_log_block_mgr.h | 14 ++++++--- src/observer/ob_server.cpp | 12 ++++---- src/observer/ob_service.cpp | 4 +-- .../virtual_table/ob_all_virtual_server.cpp | 6 ++-- src/storage/ob_disk_usage_reporter.cpp | 6 ++-- .../logservice/test_server_log_block_mgr.cpp | 8 +++++ 8 files changed, 61 insertions(+), 29 deletions(-) diff --git a/mittest/simple_server/test_observer_expand_shrink.cpp b/mittest/simple_server/test_observer_expand_shrink.cpp index 31aa7dfb3a..384225f281 100644 --- a/mittest/simple_server/test_observer_expand_shrink.cpp +++ b/mittest/simple_server/test_observer_expand_shrink.cpp @@ -53,17 +53,17 @@ TEST_F(ObserverExpandShink, observer_start) TEST_F(ObserverExpandShink, basic_func) { omt::ObTenantNodeBalancer::get_instance().refresh_interval_ = 1 * 1000 * 1000; - int64_t origin_server_log_free_size, origin_server_log_total_size; - EXPECT_EQ(OB_SUCCESS, GCTX.log_block_mgr_->get_disk_usage(origin_server_log_free_size, origin_server_log_total_size)); + int64_t origin_server_in_use_size, origin_server_log_total_size; + EXPECT_EQ(OB_SUCCESS, GCTX.log_block_mgr_->get_disk_usage(origin_server_in_use_size, origin_server_log_total_size)); GCONF.log_disk_size = GCTX.log_block_mgr_->lower_align_(2 * origin_server_log_total_size); sleep(6); - int64_t new_server_log_free_size, new_server_log_total_size; - EXPECT_EQ(OB_SUCCESS, GCTX.log_block_mgr_->get_disk_usage(new_server_log_free_size, new_server_log_total_size)); + int64_t new_server_in_use_size, new_server_log_total_size; + EXPECT_EQ(OB_SUCCESS, GCTX.log_block_mgr_->get_disk_usage(new_server_in_use_size, new_server_log_total_size)); EXPECT_EQ(new_server_log_total_size, 2 * origin_server_log_total_size); LOG_INFO("first resize success"); GCONF.log_disk_size = 0; sleep(3); - EXPECT_EQ(OB_SUCCESS, GCTX.log_block_mgr_->get_disk_usage(new_server_log_free_size, new_server_log_total_size)); + EXPECT_EQ(OB_SUCCESS, GCTX.log_block_mgr_->get_disk_usage(new_server_in_use_size, new_server_log_total_size)); EXPECT_NE(new_server_log_total_size, 0); LOG_INFO("second resize success"); diff --git a/src/logservice/ob_server_log_block_mgr.cpp b/src/logservice/ob_server_log_block_mgr.cpp index c20e3e088c..e515f495b3 100644 --- a/src/logservice/ob_server_log_block_mgr.cpp +++ b/src/logservice/ob_server_log_block_mgr.cpp @@ -81,6 +81,7 @@ ObServerLogBlockMgr::ObServerLogBlockMgr() min_block_id_(0), max_block_id_(0), min_log_disk_size_for_all_tenants_(0), + block_cnt_in_use_(0), is_inited_(false) { memset(log_pool_path_, '\0', OB_MAX_FILE_NAME_LENGTH); @@ -122,6 +123,7 @@ void ObServerLogBlockMgr::destroy() { CLOG_LOG_RET(WARN, OB_SUCCESS, "ObServerLogBlockMgr destroy", KPC(this)); is_inited_ = false; + block_cnt_in_use_ = 0; min_log_disk_size_for_all_tenants_ = 0; max_block_id_ = 0; min_block_id_ = 0; @@ -211,7 +213,7 @@ int ObServerLogBlockMgr::resize_(const int64_t new_size_byte) return ret; } -int ObServerLogBlockMgr::get_disk_usage(int64_t &free_size_byte, int64_t &total_size_byte) +int ObServerLogBlockMgr::get_disk_usage(int64_t &in_use_size_byte, int64_t &total_size_byte) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { @@ -219,7 +221,7 @@ int ObServerLogBlockMgr::get_disk_usage(int64_t &free_size_byte, int64_t &total_ CLOG_LOG(ERROR, "ObServerLogBlockMgr has not inited", K(ret), KPC(this)); } else { total_size_byte = get_total_size_guarded_by_lock_(); - free_size_byte = get_free_size_guarded_by_lock_(); + in_use_size_byte = get_in_use_size_guarded_by_lock_(); } return ret; } @@ -239,7 +241,7 @@ int ObServerLogBlockMgr::create_block_at(const FileDesc &dest_dir_fd, ret = OB_INVALID_ARGUMENT; CLOG_LOG(ERROR, "Invalid argument", K(ret), KPC(this), K(dest_dir_fd), K(dest_block_path), K(block_size)); - } else if (OB_FAIL(get_and_inc_min_block_id_guarded_by_lock_(src_block_id))) { + } else if (OB_FAIL(get_and_inc_min_block_id_guarded_by_lock_(src_block_id, true))) { CLOG_LOG(WARN, "get_and_inc_min_block_id_guarded_by_lock_ failed", K(ret), KPC(this), K(dest_dir_fd), K(dest_block_path)); } else if (OB_FAIL(block_id_to_string(src_block_id, src_block_path, OB_MAX_FILE_NAME_LENGTH))) { @@ -277,7 +279,7 @@ int ObServerLogBlockMgr::remove_block_at(const FileDesc &src_dir_fd, if (IS_NOT_INIT) { ret = OB_NOT_INIT; CLOG_LOG(ERROR, "ObServerLogBlockMGR has not inited", K(ret), KPC(this)); - } else if (OB_FAIL(get_and_inc_max_block_id_guarded_by_lock_(dest_block_id))) { + } else if (OB_FAIL(get_and_inc_max_block_id_guarded_by_lock_(dest_block_id, true))) { CLOG_LOG(ERROR, "get_and_inc_max_block_id_guarded_by_lock_ failed", K(ret), KPC(this), K(src_dir_fd), K(src_block_path)); } else if (OB_FAIL(block_id_to_string(dest_block_id, dest_block_path, OB_MAX_FILE_NAME_LENGTH))) { @@ -488,6 +490,7 @@ int ObServerLogBlockMgr::do_load_(const char *log_disk_path) CLOG_LOG(ERROR, "check_log_pool_whehter_is_integrity_ failed, unexpected error", K(ret), KPC(this), K(log_disk_path), K(has_allocated_block_cnt)); } else { + block_cnt_in_use_ = has_allocated_block_cnt; CLOG_LOG(INFO, "do_load_ success", K(ret), KPC(this), K(time_guard)); } return ret; @@ -721,6 +724,12 @@ int64_t ObServerLogBlockMgr::get_free_size_guarded_by_lock_() return BLOCK_SIZE * (max_block_id_ - min_block_id_); } +int64_t ObServerLogBlockMgr::get_in_use_size_guarded_by_lock_() +{ + RLockGuard guard(block_id_range_lock_); + return block_cnt_in_use_*BLOCK_SIZE; +} + int ObServerLogBlockMgr::update_log_pool_meta_guarded_by_lock_(const LogPoolMeta &meta) { int ret = OB_SUCCESS; @@ -752,7 +761,8 @@ ObServerLogBlockMgr::get_log_pool_meta_guarded_by_lock_() const } int ObServerLogBlockMgr::get_and_inc_max_block_id_guarded_by_lock_( - block_id_t &out_block_id) + block_id_t &out_block_id, + const bool remove_block) { int ret = OB_SUCCESS; WLockGuard guard(block_id_range_lock_); @@ -763,12 +773,17 @@ int ObServerLogBlockMgr::get_and_inc_max_block_id_guarded_by_lock_( } else { // max_block_id_ is exclusive range out_block_id = max_block_id_++; + if (remove_block) { + block_cnt_in_use_--; + } + OB_ASSERT(block_cnt_in_use_ >= 0); } return ret; } int ObServerLogBlockMgr::get_and_inc_min_block_id_guarded_by_lock_( - block_id_t &out_block_id) + block_id_t &out_block_id, + const bool create_block) { int ret = OB_SUCCESS; WLockGuard guard(block_id_range_lock_); @@ -781,6 +796,9 @@ int ObServerLogBlockMgr::get_and_inc_min_block_id_guarded_by_lock_( K(ret), KPC(this)); } else { out_block_id = min_block_id_++; + if (create_block) { + block_cnt_in_use_++; + } } return ret; } diff --git a/src/logservice/ob_server_log_block_mgr.h b/src/logservice/ob_server_log_block_mgr.h index f4a6f36b52..f7d88257aa 100644 --- a/src/logservice/ob_server_log_block_mgr.h +++ b/src/logservice/ob_server_log_block_mgr.h @@ -153,9 +153,9 @@ public: int try_resize(); // @brief get current disk usage. - // @param[out] current available diskspace. + // @param[out] current in used diskspace. // @param[out] current total diskspace. - int get_disk_usage(int64_t &free_size_byte, int64_t &total_size_byte); + int get_disk_usage(int64_t &in_use_size_byte, int64_t &total_size_byte); // @brief allocate a new block, and move it to the specified directory with the specified // name. @@ -238,8 +238,11 @@ private: const LogPoolMeta &get_log_pool_meta_guarded_by_lock_() const; int64_t get_total_size_guarded_by_lock_(); int64_t get_free_size_guarded_by_lock_(); - int get_and_inc_max_block_id_guarded_by_lock_(palf::block_id_t &out_block_id); - int get_and_inc_min_block_id_guarded_by_lock_(palf::block_id_t &out_block_id); + int64_t get_in_use_size_guarded_by_lock_(); + int get_and_inc_max_block_id_guarded_by_lock_(palf::block_id_t &out_block_id, + const bool remove_block=false); + int get_and_inc_min_block_id_guarded_by_lock_(palf::block_id_t &out_block_id, + const bool create_block=false); int move_block_not_guarded_by_lock_(const palf::FileDesc &dest_dir_fd, const char *dest_block_path, const palf::FileDesc &src_dir_fd, @@ -323,6 +326,9 @@ private: // 'min_log_disk_size_for_all_tenants_' is 0. int64_t min_log_disk_size_for_all_tenants_; GetTenantsLogDiskSize get_tenants_log_disk_size_func_; + // NB: in progress of expanding, the free size byte calcuated by BLOCK_SIZE * (max_block_id_ - min_block_id_) may be greater than + // curr_total_size_, if we calcuated log disk in use by curr_total_size_ - 'free size byte', the resule may be negative. + int64_t block_cnt_in_use_; bool is_inited_; private: diff --git a/src/observer/ob_server.cpp b/src/observer/ob_server.cpp index 3a4ccb457e..66c39d4207 100644 --- a/src/observer/ob_server.cpp +++ b/src/observer/ob_server.cpp @@ -1831,9 +1831,9 @@ int ObServer::init_io() LOG_ERROR("cal_all_part_disk_size failed", KR(ret)); } else { if (log_block_mgr_.is_reserved()) { - int64_t clog_pool_free_size = 0; + int64_t clog_pool_in_use = 0; int64_t clog_pool_total_size = 0; - if (OB_FAIL(log_block_mgr_.get_disk_usage(clog_pool_free_size, clog_pool_total_size))) { + if (OB_FAIL(log_block_mgr_.get_disk_usage(clog_pool_in_use, clog_pool_total_size))) { LOG_ERROR("get clog disk size failed", KR(ret)); } else { log_disk_size = clog_pool_total_size; @@ -2266,19 +2266,19 @@ int ObServer::init_storage() if (OB_SUCC(ret)) { int64_t total_log_disk_size = 0; - int64_t free_log_disk_size = 0; + int64_t log_disk_in_use = 0; // Check if the clog directory is empty if (OB_FAIL(log_block_mgr_.get_disk_usage( - free_log_disk_size, total_log_disk_size))) { + log_disk_in_use, total_log_disk_size))) { LOG_ERROR("ObServerLogBlockMgr get_disk_usage failed", K(ret)); - } else if (0 == total_log_disk_size-free_log_disk_size + } else if (0 == log_disk_in_use && OB_FAIL(logservice::ObServerLogBlockMgr::check_clog_directory_is_empty( OB_FILE_SYSTEM_ROUTER.get_clog_dir(), clogdir_is_empty))) { LOG_ERROR("is_dir_empty fail", K(ret)); } else if (clogdir_is_empty) { LOG_INFO("clog dir is empty"); } else { - clogdir_is_empty = total_log_disk_size == free_log_disk_size; + clogdir_is_empty = log_disk_in_use == 0; } } diff --git a/src/observer/ob_service.cpp b/src/observer/ob_service.cpp index cfdb5aa261..923805c7ac 100644 --- a/src/observer/ob_service.cpp +++ b/src/observer/ob_service.cpp @@ -1560,7 +1560,7 @@ int ObService::get_server_resource_info(share::ObServerResourceInfo &resource_in { int ret = OB_SUCCESS; omt::ObTenantNodeBalancer::ServerResource svr_res_assigned; - int64_t clog_free_size_byte = 0; + int64_t clog_in_use_size_byte = 0; int64_t clog_total_size_byte = 0; logservice::ObServerLogBlockMgr *log_block_mgr = GCTX.log_block_mgr_; resource_info.reset(); @@ -1574,7 +1574,7 @@ int ObService::get_server_resource_info(share::ObServerResourceInfo &resource_in LOG_WARN("log_block_mgr is null", KR(ret), K(GCTX.log_block_mgr_)); } else if (OB_FAIL(omt::ObTenantNodeBalancer::get_instance().get_server_allocated_resource(svr_res_assigned))) { LOG_WARN("fail to get server allocated resource", KR(ret)); - } else if (OB_FAIL(log_block_mgr->get_disk_usage(clog_free_size_byte, clog_total_size_byte))) { + } else if (OB_FAIL(log_block_mgr->get_disk_usage(clog_in_use_size_byte, clog_total_size_byte))) { LOG_WARN("Failed to get clog stat ", KR(ret)); } else if (OB_FAIL(SLOGGERMGR.get_reserved_size(reserved_size))) { LOG_WARN("Failed to get reserved size ", KR(ret)); diff --git a/src/observer/virtual_table/ob_all_virtual_server.cpp b/src/observer/virtual_table/ob_all_virtual_server.cpp index 32fffbc940..838586f6bf 100644 --- a/src/observer/virtual_table/ob_all_virtual_server.cpp +++ b/src/observer/virtual_table/ob_all_virtual_server.cpp @@ -59,7 +59,7 @@ int ObAllVirtualServer::inner_get_next_row(ObNewRow *&row) int64_t data_disk_abnormal_time = 0; logservice::ObServerLogBlockMgr *log_block_mgr = GCTX.log_block_mgr_; - int64_t clog_free_size_byte = 0; + int64_t clog_in_use_size_byte = 0; int64_t clog_total_size_byte = 0; int64_t reserved_size = 4 * 1024 * 1024 * 1024L; // default RESERVED_DISK_SIZE -> 4G @@ -74,7 +74,7 @@ int ObAllVirtualServer::inner_get_next_row(ObNewRow *&row) } else if (OB_ISNULL(GCTX.omt_) || OB_ISNULL(log_block_mgr)) { ret = OB_ERR_UNEXPECTED; SERVER_LOG(ERROR, "omt is NULL", KR(ret), K(GCTX.omt_), K(log_block_mgr)); - } else if (OB_FAIL(log_block_mgr->get_disk_usage(clog_free_size_byte, clog_total_size_byte))) { + } else if (OB_FAIL(log_block_mgr->get_disk_usage(clog_in_use_size_byte, clog_total_size_byte))) { SERVER_LOG(ERROR, "Failed to get clog stat ", KR(ret)); } else if (OB_FAIL(ObIOManager::get_instance().get_device_health_status(dhs, data_disk_abnormal_time))) { @@ -98,7 +98,7 @@ int ObAllVirtualServer::inner_get_next_row(ObNewRow *&row) const int64_t log_disk_capacity = clog_total_size_byte; const int64_t data_disk_in_use = OB_SERVER_BLOCK_MGR.get_used_macro_block_count() * OB_SERVER_BLOCK_MGR.get_macro_block_size(); - const int64_t clog_disk_in_use = clog_total_size_byte - clog_free_size_byte; + const int64_t clog_disk_in_use = clog_in_use_size_byte; const char *data_disk_health_status = device_health_status_to_str(dhs); const int64_t ssl_cert_expired_time = GCTX.ssl_key_expired_time_; diff --git a/src/storage/ob_disk_usage_reporter.cpp b/src/storage/ob_disk_usage_reporter.cpp index 1ee95572c1..c024d3f991 100644 --- a/src/storage/ob_disk_usage_reporter.cpp +++ b/src/storage/ob_disk_usage_reporter.cpp @@ -384,18 +384,18 @@ int ObDiskUsageReportTask::count_server_clog() ObDiskUsageReportKey report_key; logservice::ObServerLogBlockMgr *log_block_mgr = GCTX.log_block_mgr_; - int64_t clog_free_size_byte = 0; + int64_t clog_in_use_size_byte = 0; int64_t clog_total_size_byte = 0; if (OB_ISNULL(log_block_mgr)) { ret = OB_NOT_INIT; SERVER_LOG(ERROR, "log_block_mgr is null", KR(ret), K(GCTX.log_block_mgr_)); - } else if (OB_FAIL(log_block_mgr->get_disk_usage(clog_free_size_byte, clog_total_size_byte))) { + } else if (OB_FAIL(log_block_mgr->get_disk_usage(clog_in_use_size_byte, clog_total_size_byte))) { STORAGE_LOG(ERROR, "Failed to get clog stat ", KR(ret)); } else { report_key.file_type_ = ObDiskReportFileType::OB_DISK_REPORT_TENANT_CLOG_DATA; report_key.tenant_id_ = OB_SERVER_TENANT_ID; - int64_t clog_space = clog_total_size_byte - clog_free_size_byte; + int64_t clog_space = clog_in_use_size_byte; if (OB_FAIL(result_map_.set_refactored(report_key, clog_space, 1))) { STORAGE_LOG(WARN, "failed to set result_map_", K(ret), K(report_key), K(clog_space)); } diff --git a/unittest/logservice/test_server_log_block_mgr.cpp b/unittest/logservice/test_server_log_block_mgr.cpp index d876efa8e1..ce780c3017 100644 --- a/unittest/logservice/test_server_log_block_mgr.cpp +++ b/unittest/logservice/test_server_log_block_mgr.cpp @@ -279,7 +279,12 @@ TEST_F(TestServerLogBlockMgr, basic_func) const int64_t ls_id = 1; EXPECT_EQ(OB_SUCCESS, create_new_blocks_at(ls_id, tenant_ls_fd_map_[ls_id], 0, 10)); + int64_t in_use_size_byte, total_size_byte; + EXPECT_EQ(OB_SUCCESS, log_block_mgr_.get_disk_usage(in_use_size_byte, total_size_byte)); + EXPECT_EQ(10*ObServerLogBlockMgr::BLOCK_SIZE, in_use_size_byte); EXPECT_EQ(OB_SUCCESS, delete_blocks_at(ls_id, tenant_ls_fd_map_[ls_id], 0, 10)); + EXPECT_EQ(OB_SUCCESS, log_block_mgr_.get_disk_usage(in_use_size_byte, total_size_byte)); + EXPECT_EQ(0, in_use_size_byte); } TEST_F(TestServerLogBlockMgr, restart_for_empty_log_disk) @@ -287,9 +292,12 @@ TEST_F(TestServerLogBlockMgr, restart_for_empty_log_disk) log_block_mgr_.destroy(); const int64_t reserved_size = 2 * ObServerLogBlockMgr::GB; const int64_t aligned_reserved_size = log_block_mgr_.lower_align_(reserved_size); + int64_t in_use_size_byte, total_size_byte; EXPECT_EQ(OB_SUCCESS, log_block_mgr_.init(log_pool_base_path_)); EXPECT_EQ(aligned_reserved_size, log_block_mgr_.log_pool_meta_.curr_total_size_); EXPECT_EQ(aligned_reserved_size, log_block_mgr_.log_pool_meta_.next_total_size_); + EXPECT_EQ(OB_SUCCESS, log_block_mgr_.get_disk_usage(in_use_size_byte, total_size_byte)); + EXPECT_EQ(0, in_use_size_byte); EXPECT_EQ(0, log_block_mgr_.log_pool_meta_.status_); }