hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

BUGFIX: add some log for write throttle

Browse Source
This commit is contained in:
obdev
2023-07-20 03:18:01 +00:00
committed by ob-robot
parent 5fb975ae7c
commit e124ced9f9
2 changed files with 31 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
src/share/allocator/ob_fifo_arena.cpp
View File

@ -81,10 +81,13 @@ int ObFifoArena::ObWriteThrottleInfo::check_and_calc_decay_factor(int64_t memsto
alloc_duration_ = alloc_duration; alloc_duration_ = alloc_duration;
int64_t available_mem = (100 - trigger_percentage_) * memstore_threshold_ / 100; int64_t available_mem = (100 - trigger_percentage_) * memstore_threshold_ / 100;
double N = static_cast<double>(available_mem) / static_cast<double>(MEM_SLICE_SIZE); double N = static_cast<double>(available_mem) / static_cast<double>(MEM_SLICE_SIZE);
decay_factor_ = (static_cast<double>(alloc_duration) - N * static_cast<double>(MIN_INTERVAL))/ static_cast<double>((((N*(N+1)*N*(N+1)))/4)); double decay_factor = (static_cast<double>(alloc_duration) - N * static_cast<double>(MIN_INTERVAL))/ static_cast<double>((((N*(N+1)*N*(N+1)))/4));
decay_factor_ = decay_factor_ < 0 ? 0 : decay_factor_; decay_factor_ = decay_factor < 0 ? 0 : decay_factor;
COMMON_LOG(INFO, "recalculate decay factor", K(memstore_threshold_), K(trigger_percentage_), COMMON_LOG(INFO, "recalculate decay factor", K(memstore_threshold_), K(trigger_percentage_),
K(decay_factor_), K(alloc_duration), K(available_mem), K(N)); K(decay_factor_), K(alloc_duration), K(available_mem), K(N));
if (decay_factor < 0) {
LOG_ERROR("decay factor is smaller than 0", K(decay_factor), K(alloc_duration), K(N));
}
} }
return ret; return ret;
} }
@ -378,6 +381,7 @@ int64_t ObFifoArena::calc_mem_limit(const int64_t cur_mem_hold, const int64_t tr
int ret = OB_SUCCESS; int ret = OB_SUCCESS;
int64_t mem_can_be_assigned = 0; int64_t mem_can_be_assigned = 0;
const double decay_factor = throttle_info_.decay_factor_;
int64_t init_seq = 0; int64_t init_seq = 0;
int64_t init_page_left_size = 0; int64_t init_page_left_size = 0;
double init_page_left_interval = 0; double init_page_left_interval = 0;
@ -391,21 +395,21 @@ int64_t ObFifoArena::calc_mem_limit(const int64_t cur_mem_hold, const int64_t tr
// there is no speed limit now // there is no speed limit now
// we can get all the memory before speed limit // we can get all the memory before speed limit
mem_can_be_assigned = trigger_mem_limit - cur_mem_hold; mem_can_be_assigned = trigger_mem_limit - cur_mem_hold;
} else if (throttle_info_.decay_factor_ <= 0) { } else if (decay_factor <= 0) {
mem_can_be_assigned = 0; mem_can_be_assigned = 0;
LOG_WARN("we should limit speed, but the decay factor not calculate now", K(cur_mem_hold), K(trigger_mem_limit), K(dt)); LOG_WARN("we should limit speed, but the decay factor not calculate now", K(cur_mem_hold), K(trigger_mem_limit), K(dt));
} else { } else {
init_seq = ((cur_mem_hold - trigger_mem_limit) + MEM_SLICE_SIZE - 1) / (MEM_SLICE_SIZE); init_seq = ((cur_mem_hold - trigger_mem_limit) + MEM_SLICE_SIZE - 1) / (MEM_SLICE_SIZE);
init_page_left_size = MEM_SLICE_SIZE - (cur_mem_hold - trigger_mem_limit) % MEM_SLICE_SIZE; init_page_left_size = MEM_SLICE_SIZE - (cur_mem_hold - trigger_mem_limit) % MEM_SLICE_SIZE;
init_page_left_interval = (1.0 * throttle_info_.decay_factor_ * pow(init_seq, 3) * init_page_left_interval = (1.0 * decay_factor * pow(init_seq, 3) *
init_page_left_size / MEM_SLICE_SIZE); init_page_left_size / MEM_SLICE_SIZE);
past_interval = throttle_info_.decay_factor_ * pow(init_seq, 2) * pow(init_seq + 1, 2) / 4; past_interval = decay_factor * pow(init_seq, 2) * pow(init_seq + 1, 2) / 4;
// there is speed limit // there is speed limit
if (init_page_left_interval > dt) { if (init_page_left_interval > dt) {
last_page_interval = throttle_info_.decay_factor_ * pow(init_seq, 3); last_page_interval = decay_factor * pow(init_seq, 3);
mem_can_be_assigned = dt / last_page_interval * MEM_SLICE_SIZE; mem_can_be_assigned = dt / last_page_interval * MEM_SLICE_SIZE;
} else { } else {
mid_result = 4.0 * (dt + past_interval - init_page_left_interval) / throttle_info_.decay_factor_; mid_result = 4.0 * (dt + past_interval - init_page_left_interval) / decay_factor;
approx_max_chunk_seq = pow(mid_result, 0.25); approx_max_chunk_seq = pow(mid_result, 0.25);
max_seq = floor(approx_max_chunk_seq); max_seq = floor(approx_max_chunk_seq);
for (int i = 0; i < 2; i++) { for (int i = 0; i < 2; i++) {
@ -413,10 +417,10 @@ int64_t ObFifoArena::calc_mem_limit(const int64_t cur_mem_hold, const int64_t tr
max_seq = max_seq + 1; max_seq = max_seq + 1;
} }
} }
accumulate_interval = pow(max_seq, 2) * pow(max_seq + 1, 2) * throttle_info_.decay_factor_ / 4 - past_interval + init_page_left_interval; accumulate_interval = pow(max_seq, 2) * pow(max_seq + 1, 2) * decay_factor / 4 - past_interval + init_page_left_interval;
mem_can_be_assigned = init_page_left_size + (max_seq - init_seq) * MEM_SLICE_SIZE; mem_can_be_assigned = init_page_left_size + (max_seq - init_seq) * MEM_SLICE_SIZE;
if (accumulate_interval > dt) { if (accumulate_interval > dt) {
last_page_interval = throttle_info_.decay_factor_ * pow(max_seq, 3); last_page_interval = decay_factor * pow(max_seq, 3);
mem_can_be_assigned -= (accumulate_interval - dt) / last_page_interval * MEM_SLICE_SIZE; mem_can_be_assigned -= (accumulate_interval - dt) / last_page_interval * MEM_SLICE_SIZE;
} }
} }
@ -426,8 +430,9 @@ int64_t ObFifoArena::calc_mem_limit(const int64_t cur_mem_hold, const int64_t tr
LOG_ERROR("unexpected result", K(max_seq), K(mid_result)); LOG_ERROR("unexpected result", K(max_seq), K(mid_result));
} }
} }
if (mem_can_be_assigned == 0) { // defensive code
LOG_WARN("we can not get memory now", K(mem_can_be_assigned), K(throttle_info_.decay_factor_), K(cur_mem_hold), K(trigger_mem_limit), K(dt)); if (mem_can_be_assigned <= 0) {
LOG_WARN("we can not get memory now", K(mem_can_be_assigned), K(decay_factor), K(cur_mem_hold), K(trigger_mem_limit), K(dt));
} }
return mem_can_be_assigned; return mem_can_be_assigned;
} }

20
src/storage/ob_storage_table_guard.cpp
View File

@ -64,6 +64,7 @@ ObStorageTableGuard::~ObStorageTableGuard()
common::ObWaitEventGuard wait_guard(common::ObWaitEventIds::MEMSTORE_MEM_PAGE_ALLOC_WAIT, timeout, 0, 0, left_interval); common::ObWaitEventGuard wait_guard(common::ObWaitEventIds::MEMSTORE_MEM_PAGE_ALLOC_WAIT, timeout, 0, 0, left_interval);
reset(); reset();
int ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS;
bool has_sleep = false; bool has_sleep = false;
int64_t sleep_time = 0; int64_t sleep_time = 0;
@ -86,15 +87,24 @@ ObStorageTableGuard::~ObStorageTableGuard()
break; break;
} }
} }
//because left_interval and SLEEP_INTERVAL_PER_TIME both are greater than
//zero, so it's safe to convert to uint32_t, be careful with comparation between int and uint
int64_t expected_wait_time = memstore_allocator->expected_wait_time(seq); int64_t expected_wait_time = memstore_allocator->expected_wait_time(seq);
if (expected_wait_time == 0) { if (expected_wait_time < 0) {
LOG_ERROR("expected wait time should not smaller than 0", K(expected_wait_time), K(seq), K(clock), K(left_interval));
}
if (expected_wait_time <= 0) {
break; break;
} }
uint32_t sleep_interval = int64_t sleep_interval = min(min(left_interval, SLEEP_INTERVAL_PER_TIME), expected_wait_time);
static_cast<uint32_t>(min(min(left_interval, SLEEP_INTERVAL_PER_TIME), expected_wait_time));
// don't use ob_usleep, as we are already in the scope of 'wait_guard' // don't use ob_usleep, as we are already in the scope of 'wait_guard'
if (sleep_interval < 0) {
LOG_ERROR("sleep interval should not smaller than 0", K(expected_wait_time), K(seq), K(clock), K(left_interval));
}
if (sleep_interval > 10 * 60 * 1000 * 1000L) {
LOG_WARN("sleep interval greater than 10 minutes, pay attention", K(expected_wait_time), K(seq), K(clock), K(left_interval));
}
if (sleep_interval <= 0) {
break;
}
::usleep(sleep_interval); ::usleep(sleep_interval);
sleep_time += sleep_interval; sleep_time += sleep_interval;
time++; time++;