hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[FEAT MERGE] Implement Resource Throttle

Browse Source
This commit is contained in:
ZenoWang
2023-12-12 15:12:41 +00:00
committed by ob-robot
parent 4235388eaf
commit 56e0ec6dcf
93 changed files with 3353 additions and 1667 deletions
Download Patch File Download Diff File Expand all files Collapse all files

211
src/storage/ob_storage_table_guard.cpp
View File

@ -13,8 +13,9 @@
#define USING_LOG_PREFIX STORAGE
#include "storage/ob_storage_table_guard.h"
#include "share/allocator/ob_memstore_allocator_mgr.h"
#include "share/allocator/ob_shared_memory_allocator_mgr.h"
#include "share/throttle/ob_throttle_common.h"
#include "share/throttle/ob_share_throttle_define.h"
#include "storage/memtable/ob_memtable.h"
#include "storage/ob_i_table.h"
#include "storage/ob_relative_table.h"
@ -47,116 +48,121 @@ ObStorageTableGuard::ObStorageTableGuard(
for_multi_source_data_(for_multi_source_data)
{
init_ts_ = ObTimeUtility::current_time();
get_thread_alloc_stat() = 0;
share::memstore_throttled_alloc() = 0;
}
ObStorageTableGuard::~ObStorageTableGuard()
{
bool &need_speed_limit = tl_need_speed_limit();
ObThrottleStat &stat = get_throttle_stat();
int64_t total_expected_wait_us = 0;
int64_t user_timeout_skip_us = 0;
int64_t frozen_memtable_skip_us = 0;
int64_t replay_frozen_skip_us = 0;
int64_t from_user_skip_us = 0; // does not used now
if (need_control_mem_ && need_speed_limit) {
bool need_sleep = true;
int64_t left_interval = INT64_MAX;
if (!for_replay_) {
left_interval = min(left_interval, store_ctx_.timeout_ - ObTimeUtility::current_time());
}
if (NULL != memtable_) {
need_sleep = memtable_->is_active_memtable();
}
uint64_t timeout = 10000;//10s
common::ObWaitEventGuard wait_guard(common::ObWaitEventIds::MEMSTORE_MEM_PAGE_ALLOC_WAIT, timeout, 0, 0, left_interval);
reset();
int ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS;
bool has_sleep = false;
int64_t sleep_time = 0;
int time = 0;
const int64_t &seq = get_seq();
int64_t clock = 0;
ObGMemstoreAllocator* memstore_allocator = NULL;
if (OB_SUCCESS != (tmp_ret = ObMemstoreAllocatorMgr::get_instance().get_tenant_memstore_allocator(
MTL_ID(), memstore_allocator))) {
} else if (OB_ISNULL(memstore_allocator)) {
LOG_WARN_RET(OB_ALLOCATE_MEMORY_FAILED, "get_tenant_mutil_allocator failed", K(store_ctx_.tablet_id_), K(tmp_ret));
} else {
clock = memstore_allocator->get_clock();
total_expected_wait_us = memstore_allocator->expected_wait_time(seq);
user_timeout_skip_us = max(0, total_expected_wait_us - left_interval);
frozen_memtable_skip_us = need_sleep ? 0 : max(0, total_expected_wait_us - user_timeout_skip_us);
while (need_sleep &&
!memstore_allocator->check_clock_over_seq(seq) &&
(left_interval > 0)) {
if (for_replay_) {
if(MTL(ObTenantFreezer *)->exist_ls_freezing()) {
replay_frozen_skip_us = max(0, total_expected_wait_us - user_timeout_skip_us - sleep_time);
break;
}
}
int64_t expected_wait_time = memstore_allocator->expected_wait_time(seq);
if (expected_wait_time < 0) {
LOG_ERROR("expected wait time should not smaller than 0", K(expected_wait_time), K(seq), K(clock), K(left_interval));
}
if (expected_wait_time <= 0) {
break;
}
int64_t sleep_interval = min(min(left_interval, SLEEP_INTERVAL_PER_TIME), expected_wait_time);
// don't use ob_usleep, as we are already in the scope of 'wait_guard'
if (sleep_interval < 0) {
LOG_ERROR("sleep interval should not smaller than 0", K(expected_wait_time), K(seq), K(clock), K(left_interval));
}
if (sleep_interval > 10 * 60 * 1000 * 1000L) {
LOG_WARN("sleep interval greater than 10 minutes, pay attention", K(expected_wait_time), K(seq), K(clock), K(left_interval));
}
if (sleep_interval <= 0) {
break;
}
::usleep(sleep_interval);
sleep_time += sleep_interval;
time++;
left_interval -= sleep_interval;
has_sleep = true;
need_sleep = memstore_allocator->need_do_writing_throttle();
}
const int64_t finish_clock = memstore_allocator->get_clock();
if (finish_clock < seq) { // we has skip some time, need make the clock skip too.
const int64_t skip_clock = MIN(seq - finish_clock, get_thread_alloc_stat());
memstore_allocator->skip_clock(skip_clock);
}
}
if (REACH_TIME_INTERVAL(100 * 1000L) &&
sleep_time > 0) {
int64_t cost_time = ObTimeUtility::current_time() - init_ts_;
LOG_INFO("throttle situation", K(sleep_time), K(clock), K(time), K(seq), K(for_replay_), K(cost_time));
}
if (for_replay_ && has_sleep) {
// avoid print replay_timeout
get_replay_is_writing_throttling() = true;
}
}
(void)throttle_if_needed_();
reset();
stat.update(total_expected_wait_us,
from_user_skip_us,
user_timeout_skip_us,
frozen_memtable_skip_us,
replay_frozen_skip_us);
const bool last_throttle_status = stat.last_throttle_status;
const int64_t last_print_log_time = stat.last_log_timestamp;
if (stat.need_log(need_speed_limit)) {
LOG_INFO("throttle statics", K(need_speed_limit), K(last_throttle_status), K(last_print_log_time), K(stat));
if (!need_speed_limit && last_throttle_status) {
stat.reset();
}
void ObStorageTableGuard::throttle_if_needed_()
{
int ret = OB_SUCCESS;
if (!need_control_mem_) {
// skip throttle
} else {
TxShareThrottleTool &throttle_tool = MTL(ObSharedMemAllocMgr *)->share_resource_throttle_tool();
ObThrottleInfoGuard share_ti_guard;
ObThrottleInfoGuard module_ti_guard;
int64_t thread_idx = common::get_itid();
if (throttle_tool.is_throttling<ObMemstoreAllocator>(share_ti_guard, module_ti_guard)) {
// only do throttle on active memtable
if (OB_NOT_NULL(memtable_) && memtable_->is_active_memtable()) {
reset();
(void)do_throttle_(throttle_tool, share_ti_guard, module_ti_guard);
}
// if throttle is skipped due to some reasons, advance clock by call skip_throttle() and clean throttle status
// record in throttle info
if (throttle_tool.still_throttling<ObMemstoreAllocator>(share_ti_guard, module_ti_guard)){
int64_t skip_size = share::memstore_throttled_alloc();
(void)throttle_tool.skip_throttle<ObMemstoreAllocator>(skip_size, share_ti_guard, module_ti_guard);
if (OB_NOT_NULL(module_ti_guard.throttle_info())) {
module_ti_guard.throttle_info()->reset();
}
}
}
}
}
#define PRINT_THROTTLE_WARN \
do { \
const int64_t WARN_LOG_INTERVAL = 60L * 1000L * 1000L /* one minute */; \
if (sleep_time > (WARN_LOG_INTERVAL) && TC_REACH_TIME_INTERVAL(WARN_LOG_INTERVAL)) { \
SHARE_LOG(WARN, \
"[Throttling] Attention!! Sleep More Than One Minute!!", \
K(sleep_time), \
K(left_interval), \
K(expected_wait_time)); \
} \
} while (0)
#define PRINT_THROTTLE_STATISTIC \
do { \
const int64_t MEMSTORE_THROTTLE_LOG_INTERVAL = 1L * 1000L * 1000L; /*one seconds*/ \
if (sleep_time > 0 && REACH_TIME_INTERVAL(MEMSTORE_THROTTLE_LOG_INTERVAL)) { \
SHARE_LOG(INFO, \
"[Throttling] Time Info", \
"Throttle Unit Name", \
ObMemstoreAllocator::throttle_unit_name(), \
"Throttle Sleep Time(us)", \
sleep_time); \
} \
} while (0);
void ObStorageTableGuard::do_throttle_(TxShareThrottleTool &throttle_tool,
ObThrottleInfoGuard &share_ti_guard,
ObThrottleInfoGuard &module_ti_guard)
{
int ret = OB_SUCCESS;
int64_t sleep_time = 0;
int64_t left_interval = INT64_MAX;
if (!for_replay_) {
left_interval = min(left_interval, store_ctx_.timeout_ - ObClockGenerator::getCurrentTime());
}
while (throttle_tool.still_throttling<ObMemstoreAllocator>(share_ti_guard, module_ti_guard) && (left_interval > 0)) {
int64_t expected_wait_time = 0;
if (for_replay_ && MTL(ObTenantFreezer *)->exist_ls_freezing()) {
// skip throttle if ls freeze exists
break;
} else if ((expected_wait_time =
throttle_tool.expected_wait_time<ObMemstoreAllocator>(share_ti_guard, module_ti_guard)) <= 0) {
if (expected_wait_time < 0) {
LOG_ERROR("expected wait time should not smaller than 0",
K(expected_wait_time),
KPC(share_ti_guard.throttle_info()),
KPC(module_ti_guard.throttle_info()),
K(clock),
K(left_interval));
}
break;
}
// do sleep when expected_wait_time and left_interval are not equal to 0
int64_t sleep_interval = min(SLEEP_INTERVAL_PER_TIME, expected_wait_time);
::usleep(sleep_interval);
sleep_time += sleep_interval;
left_interval -= sleep_interval;
PRINT_THROTTLE_WARN;
}
PRINT_THROTTLE_STATISTIC;
if (for_replay_ && sleep_time > 0) {
// avoid print replay_timeout
get_replay_is_writing_throttling() = true;
}
}
#undef PRINT_THROTTLE_WARN
#undef PRINT_THROTTLE_STATISTIC
int ObStorageTableGuard::refresh_and_protect_table(ObRelativeTable &relative_table)
{
int ret = OB_SUCCESS;
@ -264,6 +270,7 @@ void ObStorageTableGuard::reset()
memtable_->dec_write_ref();
memtable_ = NULL;
}
share::memstore_throttled_alloc() = 0;
}
void ObStorageTableGuard::double_check_inc_write_ref(