diff --git a/deps/oblib/src/common/storage/ob_io_device.h b/deps/oblib/src/common/storage/ob_io_device.h index b9bcffd6cb..a066792e44 100644 --- a/deps/oblib/src/common/storage/ob_io_device.h +++ b/deps/oblib/src/common/storage/ob_io_device.h @@ -438,6 +438,7 @@ public: virtual int64_t get_max_block_count(int64_t reserved_size) const = 0; virtual int64_t get_reserved_block_count() const = 0; virtual int check_space_full(const int64_t required_size) const = 0; + virtual int check_write_limited() const = 0; public: ObStorageType device_type_; diff --git a/deps/oblib/src/lib/restore/ob_object_device.cpp b/deps/oblib/src/lib/restore/ob_object_device.cpp index b816e19e3d..b537e77dc2 100644 --- a/deps/oblib/src/lib/restore/ob_object_device.cpp +++ b/deps/oblib/src/lib/restore/ob_object_device.cpp @@ -1011,6 +1011,12 @@ int ObObjectDevice::check_space_full(const int64_t required_size) const return OB_NOT_SUPPORTED; } +int ObObjectDevice::check_write_limited() const +{ + OB_LOG_RET(WARN, OB_NOT_SUPPORTED, "check_write_limited is not support in object device !", K(device_type_)); + return OB_NOT_SUPPORTED; +} + int ObObjectDevice::fdatasync(const ObIOFd &fd) { UNUSED(fd); diff --git a/deps/oblib/src/lib/restore/ob_object_device.h b/deps/oblib/src/lib/restore/ob_object_device.h index 736bcb334c..5f09caa978 100644 --- a/deps/oblib/src/lib/restore/ob_object_device.h +++ b/deps/oblib/src/lib/restore/ob_object_device.h @@ -195,10 +195,11 @@ private: virtual int64_t get_max_block_count(int64_t reserved_size) const override; virtual int64_t get_reserved_block_count() const override; virtual int check_space_full(const int64_t required_size) const override; + virtual int check_write_limited() const override; }; } } -#endif \ No newline at end of file +#endif diff --git a/src/logservice/leader_coordinator/ob_failure_detector.cpp b/src/logservice/leader_coordinator/ob_failure_detector.cpp index 50288bb4d1..19a2c9e2e8 100644 --- a/src/logservice/leader_coordinator/ob_failure_detector.cpp +++ b/src/logservice/leader_coordinator/ob_failure_detector.cpp @@ -30,6 +30,7 @@ #include "logservice/ob_log_service.h" #include "observer/ob_server_event_history_table_operator.h" #include "storage/slog/ob_storage_logger.h" +#include "storage/tx_storage/ob_tenant_freezer.h" #include "share/schema/ob_multi_version_schema_service.h" namespace oceanbase @@ -48,6 +49,7 @@ ObFailureDetector::ObFailureDetector() has_add_data_disk_hang_event_(false), has_add_clog_full_event_(false), has_schema_error_(false), + has_add_disk_full_event_(false), lock_(common::ObLatchIds::ELECTION_LOCK) { COORDINATOR_LOG(INFO, "ObFailureDetector constructed"); @@ -126,6 +128,7 @@ void ObFailureDetector::destroy() has_add_data_disk_hang_event_ = false; has_add_clog_full_event_ = false; has_schema_error_ = false; + has_add_disk_full_event_ = false; COORDINATOR_LOG(INFO, "ObFailureDetector mtl destroy"); } @@ -166,6 +169,8 @@ void ObFailureDetector::detect_failure() detect_palf_disk_full_(); // schema refreshed check detect_schema_not_refreshed_(); + // data disk full check + detect_data_disk_full_(); } int ObFailureDetector::add_failure_event(const FailureEvent &event) @@ -471,6 +476,48 @@ void ObFailureDetector::detect_schema_not_refreshed_() } } +void ObFailureDetector::detect_data_disk_full_() +{ + LC_TIME_GUARD(1_s); + int ret = OB_SUCCESS; + const int64_t now = ObTimeUtility::current_time(); + bool is_disk_enough = true; + FailureEvent data_disk_full_event(FailureType::RESOURCE_NOT_ENOUGH, FailureModule::STORAGE, FailureLevel::NOTICE); + if (OB_FAIL(data_disk_full_event.set_info("data disk almost full event"))) { + COORDINATOR_LOG(ERROR, "data_disk_full_event set_info failed", K(ret)); + } else if (OB_FAIL(THE_IO_DEVICE->check_write_limited()) && + OB_SERVER_OUTOF_DISK_SPACE != ret) { + COORDINATOR_LOG(WARN, "check space full failed", K(ret)); + } else if (OB_SERVER_OUTOF_DISK_SPACE == ret) { + is_disk_enough = false; + ret = OB_SUCCESS; + } else { + // do nothing + } + + if (OB_FAIL(ret)) { + } else if (false == ATOMIC_LOAD(&has_add_disk_full_event_)) { + if (is_disk_enough) { + // data disk is not full, skip. + } else if (OB_FAIL(add_failure_event(data_disk_full_event))) { + COORDINATOR_LOG(ERROR, "add_failure_event failed", K(ret), K(data_disk_full_event)); + } else { + ATOMIC_SET(&has_add_disk_full_event_, true); + LOG_DBA_ERROR(OB_USER_OUTOF_DATA_DISK_SPACE, "msg", "data disk is full, add failure event", + K(data_disk_full_event), K(now)); + } + } else { + if (!is_disk_enough) { + // data disk is still full, cannot remove failure_event. + } else if (OB_FAIL(remove_failure_event(data_disk_full_event))) { + COORDINATOR_LOG(ERROR, "remove_failure_event failed", K(ret), K(data_disk_full_event)); + } else { + ATOMIC_SET(&has_add_disk_full_event_, false); + COORDINATOR_LOG(INFO, "data disk has left space, remove failure event", K(ret), K(data_disk_full_event)); + } + } +} + int ObFailureDetector::FailureEventWithRecoverOp::init(const FailureEvent &event, const ObFunction &recover_detect_operation) { diff --git a/src/logservice/leader_coordinator/ob_failure_detector.h b/src/logservice/leader_coordinator/ob_failure_detector.h index 3f47a39c0d..ba78450982 100644 --- a/src/logservice/leader_coordinator/ob_failure_detector.h +++ b/src/logservice/leader_coordinator/ob_failure_detector.h @@ -96,6 +96,10 @@ public: bool is_clog_disk_has_full_error(); bool is_data_disk_has_fatal_error(); bool is_schema_not_refreshed(); + bool is_data_disk_full() const + { + return has_add_disk_full_event_; + } private: bool check_is_running_() const { return is_running_; } int insert_event_to_table_(const FailureEvent &event, const ObFunction &recover_operation, ObString info); @@ -103,6 +107,7 @@ private: void detect_data_disk_io_failure_(); void detect_palf_disk_full_(); void detect_schema_not_refreshed_(); + void detect_data_disk_full_(); private: struct FailureEventWithRecoverOp { int init(const FailureEvent &event, const ObFunction &recover_detect_operation); @@ -121,6 +126,7 @@ private: bool has_add_data_disk_hang_event_; bool has_add_clog_full_event_; bool has_schema_error_; + bool has_add_disk_full_event_; ObSpinLock lock_; }; diff --git a/src/rootserver/ob_root_service.cpp b/src/rootserver/ob_root_service.cpp index 150521e594..03840f1e9d 100755 --- a/src/rootserver/ob_root_service.cpp +++ b/src/rootserver/ob_root_service.cpp @@ -10000,6 +10000,10 @@ int ObRootService::set_config_pre_hook(obrpc::ObAdminSetConfigArg &arg) ret = check_tx_share_memory_limit_(*item); } else if (0 == STRCMP(item->name_.ptr(), MEMSTORE_LIMIT_PERCENTAGE)) { ret = check_memstore_limit_(*item); + } else if (0 == STRCMP(item->name_.ptr(), DATA_DISK_WRITE_LIMIT_PERCENTAGE)) { + ret = check_data_disk_write_limit_(*item); + } else if (0 == STRCMP(item->name_.ptr(), DATA_DISK_USAGE_LIMIT_PERCENTAGE)) { + ret = check_data_disk_usage_limit_(*item); } else if (0 == STRCMP(item->name_.ptr(), TENANT_MEMSTORE_LIMIT_PERCENTAGE)) { ret = check_tenant_memstore_limit_(*item); } else if (0 == STRCMP(item->name_.ptr(), _TX_DATA_MEMORY_LIMIT_PERCENTAGE)) { @@ -10194,6 +10198,52 @@ int ObRootService::check_write_throttle_trigger_percentage(obrpc::ObAdminSetConf return ret; } +int ObRootService::check_data_disk_write_limit_(obrpc::ObAdminSetConfigItem &item) +{ + int ret = OB_SUCCESS; + bool is_valid = false; + int64_t value = ObConfigIntParser::get(item.value_.ptr(), is_valid); + const char *warn_log = "cluster config data_disk_write_limit_percentage. " + "It should greater than or equal with data_disk_usage_limit_percentage"; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not inited", KR(ret)); + } else if (!is_valid) { + // invalid argument + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(value)); + } else if (value == 0) { + // does not need check data disk write limit percentage + } else if (value < GCONF.data_disk_usage_limit_percentage) { + ret = OB_INVALID_ARGUMENT; + LOG_USER_ERROR(OB_INVALID_ARGUMENT, warn_log); + } + return ret; +} + +int ObRootService::check_data_disk_usage_limit_(obrpc::ObAdminSetConfigItem &item) +{ + int ret = OB_SUCCESS; + bool is_valid = false; + int64_t value = ObConfigIntParser::get(item.value_.ptr(), is_valid); + const char *warn_log = "cluster config data_disk_usage_limit_percentage. " + "It should less than or equal with data_disk_write_limit_percentage"; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not inited", KR(ret)); + } else if (!is_valid) { + // invalid argument + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(value)); + } else if (0 == GCONF.data_disk_write_limit_percentage) { + // does not need check data disk write limit percentage + } else if (value > GCONF.data_disk_write_limit_percentage) { + ret = OB_INVALID_ARGUMENT; + LOG_USER_ERROR(OB_INVALID_ARGUMENT, warn_log); + } + return ret; +} + #undef CHECK_TENANTS_CONFIG_WITH_FUNC #undef CHECK_CLUSTER_CONFIG_WITH_FUNC diff --git a/src/rootserver/ob_root_service.h b/src/rootserver/ob_root_service.h index 4939f9fe09..c1c3e8e265 100644 --- a/src/rootserver/ob_root_service.h +++ b/src/rootserver/ob_root_service.h @@ -909,6 +909,8 @@ private: int check_mds_memory_limit_(obrpc::ObAdminSetConfigItem &item); int check_freeze_trigger_percentage_(obrpc::ObAdminSetConfigItem &item); int check_write_throttle_trigger_percentage(obrpc::ObAdminSetConfigItem &item); + int check_data_disk_write_limit_(obrpc::ObAdminSetConfigItem &item); + int check_data_disk_usage_limit_(obrpc::ObAdminSetConfigItem &item); private: static const int64_t OB_MAX_CLUSTER_REPLICA_COUNT = 10000000; static const int64_t OB_ROOT_SERVICE_START_FAIL_COUNT_UPPER_LIMIT = 5; diff --git a/src/share/config/ob_server_config.h b/src/share/config/ob_server_config.h index 3a0f683181..67841dea84 100644 --- a/src/share/config/ob_server_config.h +++ b/src/share/config/ob_server_config.h @@ -54,6 +54,8 @@ const char* const CLUSTER_ID = "cluster_id"; const char* const CLUSTER_NAME = "cluster"; const char* const FREEZE_TRIGGER_PERCENTAGE = "freeze_trigger_percentage"; const char* const WRITING_THROTTLEIUNG_TRIGGER_PERCENTAGE = "writing_throttling_trigger_percentage"; +const char* const DATA_DISK_WRITE_LIMIT_PERCENTAGE = "data_disk_write_limit_percentage"; +const char* const DATA_DISK_USAGE_LIMIT_PERCENTAGE = "data_disk_usage_limit_percentage"; const char* const _TX_SHARE_MEMORY_LIMIT_PERCENTAGE = "_tx_share_memory_limit_percentage"; const char* const MEMSTORE_LIMIT_PERCENTAGE = "memstore_limit_percentage"; const char* const TENANT_MEMSTORE_LIMIT_PERCENTAGE = "_memstore_limit_percentage"; diff --git a/src/share/ob_local_device.cpp b/src/share/ob_local_device.cpp index 9d2026c781..8120ba5acc 100644 --- a/src/share/ob_local_device.cpp +++ b/src/share/ob_local_device.cpp @@ -1383,6 +1383,55 @@ int64_t ObLocalDevice::get_max_block_size(int64_t reserved_size) const } int ObLocalDevice::check_space_full(const int64_t required_size) const +{ + int ret = OB_SUCCESS; + int64_t used_percent = 0; + const int64_t NO_LIMIT_PERCENT = 100; + + if (OB_UNLIKELY(!is_marked_)) { + ret = OB_NOT_INIT; + SHARE_LOG(WARN, "The ObLocalDevice has not been marked", K(ret)); + } else if (OB_FAIL(get_data_disk_used_percentage_(required_size, + used_percent))) { + SHARE_LOG(WARN, "Fail to get disk used percentage", K(ret)); + } else { + if (GCONF.data_disk_usage_limit_percentage != NO_LIMIT_PERCENT + && used_percent >= GCONF.data_disk_usage_limit_percentage) { + ret = OB_SERVER_OUTOF_DISK_SPACE; + if (REACH_TIME_INTERVAL(24 * 3600LL * 1000 * 1000 /* 24h */)) { + LOG_DBA_ERROR(OB_SERVER_OUTOF_DISK_SPACE, "msg", "disk is almost full", K(ret), K(required_size), K(used_percent)); + } + } + } + return ret; +} + +int ObLocalDevice::check_write_limited() const +{ + int ret = OB_SUCCESS; + int64_t used_percent = 0; + const int64_t required_size = 0; + const int64_t limit_percent = GCONF.data_disk_write_limit_percentage; + + if (OB_UNLIKELY(!is_marked_)) { + ret = OB_NOT_INIT; + SHARE_LOG(WARN, "The ObLocalDevice has not been marked", K(ret)); + } else if (OB_FAIL(get_data_disk_used_percentage_(required_size, + used_percent))) { + SHARE_LOG(WARN, "Fail to get disk used percentage", K(ret)); + } else if (limit_percent != 0 && used_percent >= limit_percent) { + ret = OB_SERVER_OUTOF_DISK_SPACE; + if (REACH_TIME_INTERVAL(60 * 1000 * 1000 /* 1min */)) { + SHARE_LOG(WARN, "disk is full, user write should be stopped", K(ret), K(used_percent), + K(limit_percent)); + } + } + return ret; +} + +int ObLocalDevice::get_data_disk_used_percentage_( + const int64_t required_size, + int64_t &percent) const { int ret = OB_SUCCESS; int64_t reserved_size = 4 * 1024 * 1024 * 1024L; // default RESERVED_DISK_SIZE -> 4G @@ -1401,18 +1450,9 @@ int ObLocalDevice::check_space_full(const int64_t required_size) const if (max_block_cnt > total_block_cnt_) { // auto extend is on actual_free_block_cnt = max_block_cnt - total_block_cnt_ + free_block_cnt_; } - const int64_t NO_LIMIT_PERCENT = 100; const int64_t required_count = required_size / block_size_; const int64_t free_count = actual_free_block_cnt - required_count; - const int64_t used_percent = 100 - 100 * free_count / total_block_cnt_; - if (GCONF.data_disk_usage_limit_percentage != NO_LIMIT_PERCENT - && used_percent >= GCONF.data_disk_usage_limit_percentage) { - ret = OB_SERVER_OUTOF_DISK_SPACE; - if (REACH_TIME_INTERVAL(24 * 3600LL * 1000 * 1000 /* 24h */)) { - LOG_DBA_ERROR(OB_SERVER_OUTOF_DISK_SPACE, "msg", "disk is almost full", K(ret), K(required_size), - K(required_count), K(free_count), K(used_percent)); - } - } + percent = 100 - 100 * free_count / total_block_cnt_; } return ret; } diff --git a/src/share/ob_local_device.h b/src/share/ob_local_device.h index 42cf93905d..8eeb0f784d 100644 --- a/src/share/ob_local_device.h +++ b/src/share/ob_local_device.h @@ -176,11 +176,15 @@ public: virtual int64_t get_max_block_count(int64_t reserved_size) const override; virtual int64_t get_reserved_block_count() const override; virtual int check_space_full(const int64_t required_size) const override; + virtual int check_write_limited() const override; public: static const int64_t RESERVED_BLOCK_INDEX = 2; // the first 2 blocks is used for super block private: + int get_data_disk_used_percentage_( + const int64_t required_size, + int64_t &percent) const; int get_block_file_size( const char *sstable_dir, const int64_t reserved_size, diff --git a/src/storage/tx_storage/ob_access_service.cpp b/src/storage/tx_storage/ob_access_service.cpp index d38fd62499..b4415725f0 100644 --- a/src/storage/tx_storage/ob_access_service.cpp +++ b/src/storage/tx_storage/ob_access_service.cpp @@ -14,6 +14,7 @@ #include "lib/ob_errno.h" #include "lib/objectpool/ob_server_object_pool.h" +#include "logservice/leader_coordinator/ob_failure_detector.h" #include "share/ob_ls_id.h" #include "storage/ob_query_iterator_factory.h" #include "storage/access/ob_table_scan_iterator.h" @@ -29,6 +30,7 @@ namespace oceanbase { using namespace common; using namespace share; +using namespace logservice::coordinator; namespace storage { @@ -99,6 +101,24 @@ int ObAccessService::check_tenant_out_of_memstore_limit_(bool &is_out_of_mem) return ret; } +int ObAccessService::check_data_disk_full_( + const share::ObLSID &ls_id, + bool &is_full) +{ + int ret = OB_SUCCESS; + const uint64_t tenant_id = MTL_ID(); + ObFailureDetector* detector = MTL(ObFailureDetector*); + if (!is_user_tenant(tenant_id) || ls_id.is_sys_ls()) { + is_full = false; + } else if (OB_ISNULL(detector)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("mtl module detector is null", K(ret), KP(detector)); + } else { + is_full = detector->is_data_disk_full(); + } + return ret; +} + int ObAccessService::pre_check_lock( const share::ObLSID &ls_id, transaction::ObTxDesc &tx_desc, @@ -584,6 +604,7 @@ int ObAccessService::check_write_allowed_( { int ret = OB_SUCCESS; bool is_out_of_mem = false; + bool is_disk_full = false; ObLS *ls = nullptr; ObLockID lock_id; ObLockParam lock_param; @@ -598,6 +619,11 @@ int ObAccessService::check_write_allowed_( } else if (is_out_of_mem && !tablet_id.is_inner_tablet()) { ret = OB_TENANT_OUT_OF_MEM; LOG_WARN("this tenant is already out of memstore limit", K(ret), K_(tenant_id)); + } else if (OB_FAIL(check_data_disk_full_(ls_id, is_disk_full))) { + LOG_WARN("fail to check data disk full", K(ret)); + } else if (is_disk_full) { + ret = OB_USER_OUTOF_DATA_DISK_SPACE; + LOG_WARN("data disk full, you should not do io now", K(ret)); } else if (OB_FAIL(get_write_store_ctx_guard_(ls_id, abs_timeout_ts, tx_desc, diff --git a/src/storage/tx_storage/ob_access_service.h b/src/storage/tx_storage/ob_access_service.h index fe994cd525..e64f1c7f9c 100644 --- a/src/storage/tx_storage/ob_access_service.h +++ b/src/storage/tx_storage/ob_access_service.h @@ -202,7 +202,9 @@ public: common::ObIArray &cg_micro_cnt_arr) const; protected: int check_tenant_out_of_memstore_limit_(bool &is_out_of_mem); - + int check_data_disk_full_( + const share::ObLSID &ls_id, + bool &is_full); int get_write_store_ctx_guard_( const share::ObLSID &ls_id, const int64_t timeout, diff --git a/src/storage/tx_storage/ob_tenant_freezer.cpp b/src/storage/tx_storage/ob_tenant_freezer.cpp index 4c52f3d9e7..c28def52ad 100644 --- a/src/storage/tx_storage/ob_tenant_freezer.cpp +++ b/src/storage/tx_storage/ob_tenant_freezer.cpp @@ -970,7 +970,51 @@ bool ObTenantFreezer::is_replay_pending_log_too_large(const int64_t pending_size return bool_ret; } -int ObTenantFreezer::get_tenant_memstore_cond( +int ObTenantFreezer::get_tenant_memstore_used(int64_t &total_memstore_used, + const bool force_refresh) +{ + int ret = OB_SUCCESS; + int64_t unused_active_memstore_used = 0; + int64_t unused_memstore_freeze_trigger = 0; + int64_t unused_memstore_limit = 0; + int64_t unused_freeze_cnt = 0; + if (!is_inited_) { + ret = OB_NOT_INIT; + LOG_WARN("[TenantFreezer] tenant manager not init", KR(ret)); + } else if (OB_FAIL(get_tenant_memstore_cond_(unused_active_memstore_used, + total_memstore_used, + unused_memstore_freeze_trigger, + unused_memstore_limit, + unused_freeze_cnt, + force_refresh))) { + LOG_WARN("get tenant memstore used failed", K(ret)); + } + return ret; +} + +int ObTenantFreezer::get_tenant_memstore_cond(int64_t &active_memstore_used, + int64_t &total_memstore_used, + int64_t &memstore_freeze_trigger, + int64_t &memstore_limit, + int64_t &freeze_cnt, + const bool force_refresh) +{ + int ret = OB_SUCCESS; + if (!is_inited_) { + ret = OB_NOT_INIT; + LOG_WARN("[TenantFreezer] tenant manager not init", KR(ret)); + } else if (OB_FAIL(get_tenant_memstore_cond_(active_memstore_used, + total_memstore_used, + memstore_freeze_trigger, + memstore_limit, + freeze_cnt, + force_refresh))) { + LOG_WARN("get tenant memstore used failed", K(ret)); + } + return ret; +} + +int ObTenantFreezer::get_tenant_memstore_cond_( int64_t &active_memstore_used, int64_t &total_memstore_used, int64_t &memstore_freeze_trigger, @@ -994,11 +1038,8 @@ int ObTenantFreezer::get_tenant_memstore_cond( memstore_freeze_trigger = 0; memstore_limit = 0; - if (!is_inited_) { - ret = OB_NOT_INIT; - LOG_WARN("[TenantFreezer] tenant manager not init", KR(ret)); - } else if (!force_refresh && - current_time - last_refresh_timestamp < MEMSTORE_USED_CACHE_REFRESH_INTERVAL) { + if (!force_refresh && + current_time - last_refresh_timestamp < MEMSTORE_USED_CACHE_REFRESH_INTERVAL) { active_memstore_used = last_active_memstore_used; total_memstore_used = last_total_memstore_used; memstore_freeze_trigger = last_memstore_freeze_trigger; diff --git a/src/storage/tx_storage/ob_tenant_freezer.h b/src/storage/tx_storage/ob_tenant_freezer.h index e15e1fcbea..25e8b52368 100644 --- a/src/storage/tx_storage/ob_tenant_freezer.h +++ b/src/storage/tx_storage/ob_tenant_freezer.h @@ -184,6 +184,9 @@ public: int64_t &memstore_limit, int64_t &freeze_cnt, const bool force_refresh = true); + // get the tenant memstore used + int get_tenant_memstore_used(int64_t &total_memstore_used, + const bool force_refresh = true); // get the tenant memstore limit. int get_tenant_memstore_limit(int64_t &mem_limit); // get the memstore limit percentage @@ -225,6 +228,12 @@ public: void get_freezer_stat_from_history(int64_t pos, ObTenantFreezerStat& stat); private: + int get_tenant_memstore_cond_(int64_t &active_memstore_used, + int64_t &total_memstore_used, + int64_t &memstore_freeze_trigger, + int64_t &memstore_limit, + int64_t &freeze_cnt, + const bool force_refresh = true); int check_memstore_full_(bool &last_result, int64_t &last_check_timestamp, bool &is_out_of_mem,