diff --git a/src/share/allocator/ob_mds_allocator.cpp b/src/share/allocator/ob_mds_allocator.cpp index 166dfa4162..7f62fb5c93 100644 --- a/src/share/allocator/ob_mds_allocator.cpp +++ b/src/share/allocator/ob_mds_allocator.cpp @@ -174,6 +174,11 @@ void ObTenantBufferCtxAllocator::free(void *ptr) ObMdsThrottleGuard::ObMdsThrottleGuard(const bool for_replay, const int64_t abs_expire_time) : for_replay_(for_replay), abs_expire_time_(abs_expire_time) { throttle_tool_ = &(MTL(ObSharedMemAllocMgr *)->share_resource_throttle_tool()); + if (0 == abs_expire_time) { + abs_expire_time_ = + ObClockGenerator::getClock() + ObThrottleUnit::DEFAULT_MAX_THROTTLE_TIME; + } + share::mds_throttled_alloc() = 0; } ObMdsThrottleGuard::~ObMdsThrottleGuard() @@ -184,23 +189,16 @@ ObMdsThrottleGuard::~ObMdsThrottleGuard() if (OB_ISNULL(throttle_tool_)) { MDS_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "throttle tool is unexpected nullptr", KP(throttle_tool_)); } else if (throttle_tool_->is_throttling(share_ti_guard, module_ti_guard)) { - if (MTL(ObTenantFreezer *)->exist_ls_freezing()) { + (void)TxShareMemThrottleUtil::do_throttle( + for_replay_, abs_expire_time_, *throttle_tool_, share_ti_guard, module_ti_guard); + + if (throttle_tool_->still_throttling(share_ti_guard, module_ti_guard)) { (void)throttle_tool_->skip_throttle( share::mds_throttled_alloc(), share_ti_guard, module_ti_guard); if (module_ti_guard.is_valid()) { module_ti_guard.throttle_info()->reset(); } - } else { - uint64_t timeout = 10000; // 10s - int64_t left_interval = abs_expire_time_ - ObClockGenerator::getClock(); - common::ObWaitEventGuard wait_guard( - common::ObWaitEventIds::MEMSTORE_MEM_PAGE_ALLOC_WAIT, timeout, 0, 0, left_interval); - (void)throttle_tool_->do_throttle(abs_expire_time_); - - if (for_replay_) { - get_replay_is_writing_throttling() = true; - } } // reset mds throttled alloc size diff --git a/src/share/allocator/ob_shared_memory_allocator_mgr.h b/src/share/allocator/ob_shared_memory_allocator_mgr.h index 5502edb15e..b4d7d1d596 100644 --- a/src/share/allocator/ob_shared_memory_allocator_mgr.h +++ b/src/share/allocator/ob_shared_memory_allocator_mgr.h @@ -17,6 +17,8 @@ #include "share/allocator/ob_tx_data_allocator.h" #include "share/allocator/ob_mds_allocator.h" #include "share/throttle/ob_share_resource_throttle_tool.h" +#include "share/rc/ob_tenant_base.h" +#include "storage/tx_storage/ob_tenant_freezer.h" namespace oceanbase { namespace share { @@ -78,6 +80,78 @@ private: ObTenantMdsAllocator mds_allocator_; }; +class TxShareMemThrottleUtil +{ +public: + static const int64_t SLEEP_INTERVAL_PER_TIME = 20 * 1000; // 20ms; + template + static int do_throttle(const bool for_replay, + const int64_t abs_expire_time, + TxShareThrottleTool &throttle_tool, + ObThrottleInfoGuard &share_ti_guard, + ObThrottleInfoGuard &module_ti_guard) + { + int ret = OB_SUCCESS; + bool has_printed_lbt = false; + int64_t sleep_time = 0; + int64_t left_interval = share::ObThrottleUnit::DEFAULT_MAX_THROTTLE_TIME; + + if (!for_replay) { + left_interval = min(left_interval, abs_expire_time - ObClockGenerator::getClock()); + } + + uint64_t timeout = 10000; // 10s + common::ObWaitEventGuard wait_guard( + common::ObWaitEventIds::MEMSTORE_MEM_PAGE_ALLOC_WAIT, timeout, 0, 0, left_interval); + + while (throttle_tool.still_throttling(share_ti_guard, module_ti_guard) && + (left_interval > 0)) { + int64_t expected_wait_time = 0; + if (for_replay && MTL(ObTenantFreezer *)->exist_ls_freezing()) { + // skip throttle if ls freeze exists + break; + } else if ((expected_wait_time = + throttle_tool.expected_wait_time(share_ti_guard, module_ti_guard)) <= 0) { + if (expected_wait_time < 0) { + SHARE_LOG(ERROR, + "expected wait time should not smaller than 0", + K(expected_wait_time), + KPC(share_ti_guard.throttle_info()), + KPC(module_ti_guard.throttle_info()), + K(clock), + K(left_interval)); + } + break; + } + + // do sleep when expected_wait_time and left_interval are not equal to 0 + int64_t sleep_interval = min(SLEEP_INTERVAL_PER_TIME, expected_wait_time); + if (0 < sleep_interval) { + sleep_time += sleep_interval; + left_interval -= sleep_interval; + ::usleep(sleep_interval); + } + + PrintThrottleUtil::pirnt_throttle_info(ret, + ALLOCATOR::throttle_unit_name(), + sleep_time, + left_interval, + expected_wait_time, + abs_expire_time, + share_ti_guard, + module_ti_guard, + has_printed_lbt); + } + PrintThrottleUtil::print_throttle_statistic(ret, ALLOCATOR::throttle_unit_name(), sleep_time); + + if (for_replay && sleep_time > 0) { + // avoid print replay_timeout + get_replay_is_writing_throttling() = true; + } + return ret; + } +}; + } // namespace share } // namespace oceanbase diff --git a/src/share/allocator/ob_tx_data_allocator.cpp b/src/share/allocator/ob_tx_data_allocator.cpp index d7e6da82ce..45b401f009 100644 --- a/src/share/allocator/ob_tx_data_allocator.cpp +++ b/src/share/allocator/ob_tx_data_allocator.cpp @@ -111,6 +111,11 @@ ObTxDataThrottleGuard::ObTxDataThrottleGuard(const bool for_replay, const int64_ : for_replay_(for_replay), abs_expire_time_(abs_expire_time) { throttle_tool_ = &(MTL(ObSharedMemAllocMgr *)->share_resource_throttle_tool()); + if (0 == abs_expire_time) { + abs_expire_time_ = + ObClockGenerator::getClock() + ObThrottleUnit::DEFAULT_MAX_THROTTLE_TIME; + } + share::tx_data_throttled_alloc() = 0; } ObTxDataThrottleGuard::~ObTxDataThrottleGuard() @@ -121,23 +126,16 @@ ObTxDataThrottleGuard::~ObTxDataThrottleGuard() if (OB_ISNULL(throttle_tool_)) { MDS_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "throttle tool is unexpected nullptr", KP(throttle_tool_)); } else if (throttle_tool_->is_throttling(share_ti_guard, module_ti_guard)) { - if (MTL(ObTenantFreezer *)->exist_ls_freezing()) { + (void)TxShareMemThrottleUtil::do_throttle( + for_replay_, abs_expire_time_, *throttle_tool_, share_ti_guard, module_ti_guard); + + if (throttle_tool_->still_throttling(share_ti_guard, module_ti_guard)) { (void)throttle_tool_->skip_throttle( share::tx_data_throttled_alloc(), share_ti_guard, module_ti_guard); if (module_ti_guard.is_valid()) { module_ti_guard.throttle_info()->reset(); } - } else { - uint64_t timeout = 10000; // 10s - int64_t left_interval = abs_expire_time_ - ObClockGenerator::getClock(); - common::ObWaitEventGuard wait_guard( - common::ObWaitEventIds::MEMSTORE_MEM_PAGE_ALLOC_WAIT, timeout, 0, 0, left_interval); - (void)throttle_tool_->do_throttle(abs_expire_time_); - - if (for_replay_) { - get_replay_is_writing_throttling() = true; - } } // reset tx data throttled alloc size diff --git a/src/share/throttle/ob_share_resource_throttle_tool.h b/src/share/throttle/ob_share_resource_throttle_tool.h index d61325a6e0..6d704bc810 100644 --- a/src/share/throttle/ob_share_resource_throttle_tool.h +++ b/src/share/throttle/ob_share_resource_throttle_tool.h @@ -41,7 +41,7 @@ template class ObShareResourceThrottleTool { private: - static const uint32_t DEFAULT_THROTTLE_SLEEP_INTERVAL = 20 * 1000; + static const int64_t DEFAULT_THROTTLE_SLEEP_INTERVAL = 20 * 1000; private: // Functors for ShareResourceThrottleTool struct SumModuleHoldResourceFunctor { int64_t sum_; diff --git a/src/share/throttle/ob_share_resource_throttle_tool.ipp b/src/share/throttle/ob_share_resource_throttle_tool.ipp index effee3b6e5..4852d45c3a 100644 --- a/src/share/throttle/ob_share_resource_throttle_tool.ipp +++ b/src/share/throttle/ob_share_resource_throttle_tool.ipp @@ -190,43 +190,6 @@ int64_t ObShareResourceThrottleTool::expected_wait_time( return expected_wait_time; } -#define PRINT_THROTTLE_WARN \ - do { \ - const int64_t WARN_LOG_INTERVAL = 1LL * 60L * 1000L * 1000L /* one minute */; \ - if (sleep_time > (WARN_LOG_INTERVAL) && TC_REACH_TIME_INTERVAL(WARN_LOG_INTERVAL)) { \ - SHARE_LOG(WARN, \ - "[Throttling] Attention!! Sleep More Than One Minute!!", \ - "Throttle Unit Name", \ - ALLOCATOR::throttle_unit_name(), \ - K(sleep_time), \ - K(left_interval), \ - K(expected_wait_t), \ - K(abs_expire_time)); \ - if (!has_printed_lbt) { \ - has_printed_lbt = true; \ - oceanbase::share::ObTaskController::get().allow_next_syslog(); \ - SHARE_LOG(INFO, \ - "[Throttling] (report write throttle info) LBT Info", \ - "Throttle Unit Name", \ - ALLOCATOR::throttle_unit_name(), \ - K(lbt())); \ - } \ - } \ - } while (0) - -#define PRINT_THROTTLE_STATISTIC \ - do { \ - const int64_t MEMSTORE_THROTTLE_LOG_INTERVAL = 1L * 1000L * 1000L; /*one seconds*/ \ - if (sleep_time > 0 && REACH_TIME_INTERVAL(MEMSTORE_THROTTLE_LOG_INTERVAL)) { \ - SHARE_LOG(INFO, \ - "[Throttling] (report write throttle info) Time Info", \ - "Throttle Unit Name", \ - ALLOCATOR::throttle_unit_name(), \ - "Throttle Sleep Time(us)", \ - sleep_time); \ - } \ - } while (0); - template template void ObShareResourceThrottleTool::do_throttle(const int64_t abs_expire_time) @@ -256,20 +219,27 @@ void ObShareResourceThrottleTool::do_throttle(const int6 module_ti_guard.throttle_info()->reset(); } } else { - uint32_t sleep_interval = min(DEFAULT_THROTTLE_SLEEP_INTERVAL, (uint32_t)expected_wait_t); - sleep_time += sleep_interval; - left_interval -= sleep_interval; - ::usleep(sleep_interval); + int64_t sleep_interval = min(DEFAULT_THROTTLE_SLEEP_INTERVAL, expected_wait_t); + if (0 < sleep_interval) { + sleep_time += sleep_interval; + left_interval -= sleep_interval; + ::usleep(sleep_interval); + } } - PRINT_THROTTLE_WARN; + PrintThrottleUtil::pirnt_throttle_info(ret, + ALLOCATOR::throttle_unit_name(), + sleep_time, + left_interval, + expected_wait_t, + abs_expire_time, + share_ti_guard, + module_ti_guard, + has_printed_lbt); } - PRINT_THROTTLE_STATISTIC; + PrintThrottleUtil::print_throttle_statistic(ret, ALLOCATOR::throttle_unit_name(), sleep_time); } } -#undef PRINT_THROTTLE_WARN -#undef PRINT_THROTTLE_STATISTIC - template template void ObShareResourceThrottleTool::skip_throttle(const int64_t skip_size) @@ -318,8 +288,6 @@ void ObShareResourceThrottleTool::update_throttle_config } #undef ACQUIRE_THROTTLE_UNIT -#undef ACQUIRE_ALLOCATOR -#undef FUNC_PREFIX } // namespace share } // namespace oceanbase diff --git a/src/share/throttle/ob_share_throttle_define.cpp b/src/share/throttle/ob_share_throttle_define.cpp index fc9425ec3a..9c4e80b1f8 100644 --- a/src/share/throttle/ob_share_throttle_define.cpp +++ b/src/share/throttle/ob_share_throttle_define.cpp @@ -12,10 +12,12 @@ #include "ob_share_throttle_define.h" +#include "share/throttle/ob_throttle_info.h" #include "observer/omt/ob_tenant_config_mgr.h" #include "lib/alloc/alloc_func.h" #include "storage/tx_storage/ob_tenant_freezer.h" + namespace oceanbase { namespace share { @@ -113,5 +115,56 @@ void FakeAllocatorForTxShare::adaptive_update_limit(const int64_t tenant_id, } } +void PrintThrottleUtil::pirnt_throttle_info(const int err_code, + const char *throttle_unit_name, + const int64_t sleep_time, + const int64_t left_interval, + const int64_t expected_wait_time, + const int64_t abs_expire_time, + const ObThrottleInfoGuard &share_ti_guard, + const ObThrottleInfoGuard &module_ti_guard, + bool &has_printed_lbt) +{ + int ret = err_code; + const int64_t WARN_LOG_INTERVAL = 1LL * 60L * 1000L * 1000L /* one minute */; + if (sleep_time > (WARN_LOG_INTERVAL) && TC_REACH_TIME_INTERVAL(WARN_LOG_INTERVAL)) { + SHARE_LOG(WARN, + "[Throttling] Attention!! Sleep More Than One Minute!!", + "Throttle Unit Name", + throttle_unit_name, + K(sleep_time), + K(left_interval), + K(expected_wait_time), + K(abs_expire_time), + KPC(share_ti_guard.throttle_info()), + KPC(module_ti_guard.throttle_info())); + if (!has_printed_lbt) { + has_printed_lbt = true; + oceanbase::share::ObTaskController::get().allow_next_syslog(); + SHARE_LOG(INFO, + "[Throttling] (report write throttle info) LBT Info", + "Throttle Unit Name", + throttle_unit_name, + K(lbt())); + } + } +} + +void PrintThrottleUtil::print_throttle_statistic(const int err_code, + const char *throttle_unit_name, + const int64_t sleep_time) +{ + int ret = err_code; + const int64_t THROTTLE_LOG_INTERVAL = 1L * 1000L * 1000L; /*one seconds*/ + if (sleep_time > 0 && REACH_TIME_INTERVAL(THROTTLE_LOG_INTERVAL)) { + SHARE_LOG(INFO, + "[Throttling] (report write throttle info) Time Info", + "Throttle Unit Name", + throttle_unit_name, + "Throttle Sleep Time(us)", + sleep_time); + } +} + } // namespace share } // namespace oceanbase diff --git a/src/share/throttle/ob_share_throttle_define.h b/src/share/throttle/ob_share_throttle_define.h index 9353cf66df..31b2421b40 100644 --- a/src/share/throttle/ob_share_throttle_define.h +++ b/src/share/throttle/ob_share_throttle_define.h @@ -77,6 +77,26 @@ namespace share { // ObTenantMdsAllocator>; DEFINE_SHARE_THROTTLE(TxShare, ObMemstoreAllocator, ObTenantTxDataAllocator, ObTenantMdsAllocator) + +class ObThrottleInfoGuard; + +class PrintThrottleUtil { +public: + static void pirnt_throttle_info(const int err_code, + const char *throttle_unit_name, + const int64_t sleep_time, + const int64_t left_interval, + const int64_t expected_wait_time, + const int64_t abs_expire_time, + const ObThrottleInfoGuard &share_ti_guard, + const ObThrottleInfoGuard &module_ti_guard, + bool &has_printed_lbt); + + static void print_throttle_statistic(const int err_code, + const char *throttle_unit_name, + const int64_t sleep_time); +}; + } // namespace share } // namespace oceanbase diff --git a/src/share/throttle/ob_throttle_info.h b/src/share/throttle/ob_throttle_info.h index 97cb42bfef..b0a8aba99f 100644 --- a/src/share/throttle/ob_throttle_info.h +++ b/src/share/throttle/ob_throttle_info.h @@ -130,6 +130,7 @@ struct ObThrottleInfoGuard } ObThrottleInfo *throttle_info() { return throttle_info_; } + const ObThrottleInfo *throttle_info() const { return throttle_info_; } TO_STRING_KV(KP(throttle_info_), KP(throttle_info_map_)); }; diff --git a/src/share/throttle/ob_throttle_unit.ipp b/src/share/throttle/ob_throttle_unit.ipp index ddd0a98cb8..902de64f80 100644 --- a/src/share/throttle/ob_throttle_unit.ipp +++ b/src/share/throttle/ob_throttle_unit.ipp @@ -288,7 +288,7 @@ void ObThrottleUnit::advance_clock(const int64_t holding_size) const int64_t clock = ATOMIC_LOAD(&clock_); const int64_t cur_seq = ATOMIC_LOAD(&sequence_num_); ATOMIC_SET(&clock_, min(cur_seq, clock + avaliable_resource)); - if (REACH_TIME_INTERVAL(1LL * 1000LL * 1000LL/* 1 second */)) { + if (REACH_TIME_INTERVAL(10LL * 1000LL * 1000LL/* 10 seconds */)) { SHARE_LOG(INFO, "[Throttling] Advance Clock", K(avaliable_resource), diff --git a/src/storage/ob_storage_table_guard.cpp b/src/storage/ob_storage_table_guard.cpp index 30498bfb8b..158588f900 100644 --- a/src/storage/ob_storage_table_guard.cpp +++ b/src/storage/ob_storage_table_guard.cpp @@ -71,7 +71,8 @@ void ObStorageTableGuard::throttle_if_needed_() // only do throttle on active memtable if (OB_NOT_NULL(memtable_) && memtable_->is_active_memtable()) { reset(); - (void)do_throttle_(throttle_tool, share_ti_guard, module_ti_guard); + (void)TxShareMemThrottleUtil::do_throttle( + for_replay_, store_ctx_.timeout_, throttle_tool, share_ti_guard, module_ti_guard); } // if throttle is skipped due to some reasons, advance clock by call skip_throttle() and clean throttle status @@ -88,84 +89,6 @@ void ObStorageTableGuard::throttle_if_needed_() } } -#define PRINT_THROTTLE_WARN \ - do { \ - const int64_t WARN_LOG_INTERVAL = 60L * 1000L * 1000L /* one minute */; \ - if (sleep_time > (WARN_LOG_INTERVAL) && TC_REACH_TIME_INTERVAL(WARN_LOG_INTERVAL)) { \ - SHARE_LOG(WARN, \ - "[Throttling] Attention!! Sleep More Than One Minute!!", \ - K(sleep_time), \ - K(left_interval), \ - K(expected_wait_time)); \ - } \ - } while (0) - -#define PRINT_THROTTLE_STATISTIC \ - do { \ - const int64_t MEMSTORE_THROTTLE_LOG_INTERVAL = 1L * 1000L * 1000L; /*one seconds*/ \ - if (sleep_time > 0 && REACH_TIME_INTERVAL(MEMSTORE_THROTTLE_LOG_INTERVAL)) { \ - SHARE_LOG(INFO, \ - "[Throttling] (report write throttle info) Time Info", \ - "Throttle Unit Name", \ - ObMemstoreAllocator::throttle_unit_name(), \ - "Throttle Sleep Time(us)", \ - sleep_time); \ - } \ - } while (0); - -void ObStorageTableGuard::do_throttle_(TxShareThrottleTool &throttle_tool, - ObThrottleInfoGuard &share_ti_guard, - ObThrottleInfoGuard &module_ti_guard) -{ - int ret = OB_SUCCESS; - int64_t sleep_time = 0; - int64_t left_interval = share::ObThrottleUnit::DEFAULT_MAX_THROTTLE_TIME; - - if (!for_replay_) { - left_interval = min(left_interval, store_ctx_.timeout_ - ObClockGenerator::getClock()); - } - - uint64_t timeout = 10000; // 10s - common::ObWaitEventGuard wait_guard( - common::ObWaitEventIds::MEMSTORE_MEM_PAGE_ALLOC_WAIT, timeout, 0, 0, left_interval); - - while (throttle_tool.still_throttling(share_ti_guard, module_ti_guard) && (left_interval > 0)) { - int64_t expected_wait_time = 0; - if (for_replay_ && MTL(ObTenantFreezer *)->exist_ls_freezing()) { - // skip throttle if ls freeze exists - break; - } else if ((expected_wait_time = - throttle_tool.expected_wait_time(share_ti_guard, module_ti_guard)) <= 0) { - if (expected_wait_time < 0) { - LOG_ERROR("expected wait time should not smaller than 0", - K(expected_wait_time), - KPC(share_ti_guard.throttle_info()), - KPC(module_ti_guard.throttle_info()), - K(clock), - K(left_interval)); - } - break; - } - - // do sleep when expected_wait_time and left_interval are not equal to 0 - int64_t sleep_interval = min(SLEEP_INTERVAL_PER_TIME, expected_wait_time); - ::usleep(sleep_interval); - sleep_time += sleep_interval; - left_interval -= sleep_interval; - - PRINT_THROTTLE_WARN; - } - PRINT_THROTTLE_STATISTIC; - - if (for_replay_ && sleep_time > 0) { - // avoid print replay_timeout - get_replay_is_writing_throttling() = true; - } -} - -#undef PRINT_THROTTLE_WARN -#undef PRINT_THROTTLE_STATISTIC - int ObStorageTableGuard::refresh_and_protect_table(ObRelativeTable &relative_table) { int ret = OB_SUCCESS; diff --git a/src/storage/ob_storage_table_guard.h b/src/storage/ob_storage_table_guard.h index 19a270a012..15ede47791 100644 --- a/src/storage/ob_storage_table_guard.h +++ b/src/storage/ob_storage_table_guard.h @@ -66,9 +66,6 @@ private: bool need_to_refresh_table(ObTableStoreIterator &iter); void check_if_need_log_(bool &need_log, bool &need_log_error); void throttle_if_needed_(); - void do_throttle_(share::TxShareThrottleTool &throttle_tool, - share::ObThrottleInfoGuard &share_ti_guard, - share::ObThrottleInfoGuard &module_ti_guard); private: static const int64_t LOG_INTERVAL_US = 10 * 1000 * 1000; // 10s