hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

patch 4.0

Browse Source
This commit is contained in:
wangzelin.wzl
2022-10-24 10:34:53 +08:00
parent 4ad6e00ec3
commit 93a1074b0c
10533 changed files with 2588271 additions and 2299373 deletions
Download Patch File Download Diff File Expand all files Collapse all files

163
src/share/allocator/ob_fifo_arena.cpp
View File

@ -16,26 +16,21 @@
#include "share/ob_tenant_mgr.h"
#include "observer/omt/ob_tenant_config_mgr.h"
#include "lib/alloc/alloc_struct.h"
#include "lib/stat/ob_diagnose_info.h"
using namespace oceanbase::lib;
using namespace oceanbase::omt;
namespace oceanbase {
namespace common {
#ifndef myassert
#define myassert(x) \
if (!x) { \
ob_abort(); \
}
#endif
namespace oceanbase
{
namespace common
{
#define myassert(x) if (!x) { ob_abort(); }
int64_t ObFifoArena::total_hold_ = 0;
int64_t ObFifoArena::Page::get_actual_hold_size()
{
// every time of alloc_page, ruturn a chunk actually
AObject* obj = reinterpret_cast<AObject*>((char*)(this) - AOBJECT_HEADER_SIZE);
abort_unless(NULL != obj);
int64_t actual_size = obj->hold(AllocHelper::cells_per_block(obj->block()->ablock_size_));
return actual_size;
//every time of alloc_page, ruturn a chunk actually
return ObTenantCtxAllocator::get_obj_hold(this);
}
void ObFifoArena::ObWriteThrottleInfo::reset()
@ -64,28 +59,23 @@ void ObFifoArena::ObWriteThrottleInfo::record_limit_event(int64_t interval)
ATOMIC_FAA(&total_throttled_time_, interval);
}
int ObFifoArena::ObWriteThrottleInfo::check_and_calc_decay_factor(
int64_t memstore_threshold, int64_t trigger_percentage, int64_t alloc_duration)
int ObFifoArena::ObWriteThrottleInfo::check_and_calc_decay_factor(int64_t memstore_threshold,
int64_t trigger_percentage,
int64_t alloc_duration)
{
int ret = OB_SUCCESS;
if (memstore_threshold != memstore_threshold_ || trigger_percentage != trigger_percentage_ ||
alloc_duration != alloc_duration_) {
if (memstore_threshold != memstore_threshold_
|| trigger_percentage != trigger_percentage_
|| alloc_duration != alloc_duration_) {
memstore_threshold_ = memstore_threshold;
trigger_percentage_ = trigger_percentage;
alloc_duration_ = alloc_duration;
int64_t available_mem = (100 - trigger_percentage_) * memstore_threshold_ / 100;
double N = static_cast<double>(available_mem) / static_cast<double>(MEM_SLICE_SIZE);
decay_factor_ = (static_cast<double>(alloc_duration) - N * static_cast<double>(MIN_INTERVAL)) /
static_cast<double>((((N * (N + 1) * N * (N + 1))) / 4));
double N = static_cast<double>(available_mem) / static_cast<double>(MEM_SLICE_SIZE);
decay_factor_ = (static_cast<double>(alloc_duration) - N * static_cast<double>(MIN_INTERVAL))/ static_cast<double>((((N*(N+1)*N*(N+1)))/4));
decay_factor_ = decay_factor_ < 0 ? 0 : decay_factor_;
COMMON_LOG(INFO,
"recalculate decay factor",
K(memstore_threshold_),
K(trigger_percentage_),
K(decay_factor_),
K(alloc_duration),
K(available_mem),
K(N));
COMMON_LOG(INFO, "recalculate decay factor", K(memstore_threshold_), K(trigger_percentage_),
K(decay_factor_), K(alloc_duration), K(available_mem), K(N));
}
return ret;
}
@ -93,7 +83,7 @@ int ObFifoArena::ObWriteThrottleInfo::check_and_calc_decay_factor(
int ObFifoArena::init(uint64_t tenant_id)
{
int ret = OB_SUCCESS;
lib::ObMallocAllocator* allocator = lib::ObMallocAllocator::get_instance();
lib::ObMallocAllocator *allocator = lib::ObMallocAllocator::get_instance();
uint64_t ctx_id = ObCtxIds::MEMSTORE_CTX_ID;
if (OB_ISNULL(allocator)) {
ret = OB_INIT_FAIL;
@ -139,7 +129,7 @@ void ObFifoArena::update_nway_per_group(int64_t nway)
void ObFifoArena::shrink_cached_page(int64_t nway)
{
for (int64_t i = 0; i < MAX_CACHED_PAGE_COUNT; i++) {
for(int64_t i = 0; i < MAX_CACHED_PAGE_COUNT; i++) {
if ((i % Handle::MAX_NWAY) >= nway) {
Page** paddr = cur_pages_ + i;
Page* page = NULL;
@ -162,11 +152,11 @@ void* ObFifoArena::alloc(int64_t adv_idx, Handle& handle, int64_t size)
int ret = OB_SUCCESS;
void* ptr = NULL;
int64_t rsize = size + sizeof(Page) + sizeof(Ref);
speed_limit(ATOMIC_LOAD(&hold_), size);
CriticalGuard(get_qs());
int64_t way_id = get_way_id();
int64_t idx = get_idx(adv_idx, way_id);
Page** paddr = cur_pages_ + idx;
speed_limit(ATOMIC_LOAD(&hold_), size);
if (adv_idx < 0 || size < 0) {
COMMON_LOG(INFO, "invalid argument", K(adv_idx), K(size));
ret = OB_INVALID_ARGUMENT;
@ -227,14 +217,14 @@ void ObFifoArena::release_ref(Ref* ref)
void ObFifoArena::free(Handle& handle)
{
bool wait_qs_done = false;
for (int i = 0; i < Handle::MAX_NWAY; i++) {
for(int i = 0; i < Handle::MAX_NWAY; i++) {
Ref* ref = NULL;
Ref* next_ref = handle.ref_[i];
if (NULL != next_ref && !wait_qs_done) {
WaitQuiescent(get_qs());
wait_qs_done = true;
}
while (NULL != (ref = next_ref)) {
while(NULL != (ref = next_ref)) {
next_ref = ref->next_;
release_ref(ref);
}
@ -281,13 +271,12 @@ void ObFifoArena::destroy_page(Page* page)
}
}
bool ObFifoArena::need_do_writing_throttle()
bool ObFifoArena::need_do_writing_throttle() const
{
int64_t trigger_percentage = get_writing_throttling_trigger_percentage_();
int64_t trigger_mem_limit = lastest_memstore_threshold_ * trigger_percentage / 100;
int64_t trigger_mem_remain = lastest_memstore_threshold_ * (100 - trigger_percentage) / 100;
int64_t cur_mem_hold = ATOMIC_LOAD(&hold_);
bool need_do_writing_throttle = (cur_mem_hold > trigger_mem_limit) || (get_tenant_memory_remain_() < trigger_mem_remain);
bool need_do_writing_throttle = cur_mem_hold > trigger_mem_limit;
return need_do_writing_throttle;
}
@ -296,37 +285,22 @@ void ObFifoArena::speed_limit(int64_t cur_mem_hold, int64_t alloc_size)
int ret = OB_SUCCESS;
int64_t trigger_percentage = get_writing_throttling_trigger_percentage_();
int64_t trigger_mem_limit = 0;
int64_t trigger_mem_remain = 0;
bool is_memstore_overused = false;
if (trigger_percentage < 100) {
const int64_t tenant_memory_remain = get_tenant_memory_remain_();
if (OB_UNLIKELY(
cur_mem_hold < 0 || alloc_size <= 0 || lastest_memstore_threshold_ <= 0 || trigger_percentage <= 0)) {
COMMON_LOG(ERROR,
"invalid arguments",
K(cur_mem_hold),
K(alloc_size),
K(lastest_memstore_threshold_),
K(trigger_percentage));
} else if ((is_memstore_overused = (cur_mem_hold > (trigger_mem_limit = lastest_memstore_threshold_ * trigger_percentage / 100)))
|| (tenant_memory_remain < (trigger_mem_remain = lastest_memstore_threshold_ - trigger_mem_limit))) {
if (OB_UNLIKELY(cur_mem_hold < 0 || alloc_size <= 0 || lastest_memstore_threshold_ <= 0 || trigger_percentage <= 0)) {
COMMON_LOG(ERROR, "invalid arguments", K(cur_mem_hold), K(alloc_size), K(lastest_memstore_threshold_), K(trigger_percentage));
} else if (cur_mem_hold > (trigger_mem_limit = lastest_memstore_threshold_ * trigger_percentage / 100)) {
int64_t alloc_duration = get_writing_throttling_maximum_duration_();
if (OB_FAIL(throttle_info_.check_and_calc_decay_factor(
lastest_memstore_threshold_, trigger_percentage, alloc_duration))) {
if (OB_FAIL(throttle_info_.check_and_calc_decay_factor(lastest_memstore_threshold_, trigger_percentage, alloc_duration))) {
COMMON_LOG(WARN, "failed to check_and_calc_decay_factor", K(cur_mem_hold), K(alloc_size), K(throttle_info_));
} else {
const int64_t mem_overused = MAX(cur_mem_hold - trigger_mem_limit, trigger_mem_remain - tenant_memory_remain);
int64_t throttling_interval = get_throttling_interval(mem_overused, alloc_size, is_memstore_overused);
int64_t throttling_interval = get_throttling_interval(cur_mem_hold, alloc_size, trigger_mem_limit);
int64_t cur_ts = ObTimeUtility::current_time();
int64_t new_base_ts = ATOMIC_AAF(&last_base_ts_, throttling_interval);
int64_t sleep_interval = new_base_ts - cur_ts;
if (sleep_interval > 0) {
ObWaitEventGuard wait_guard(
ObWaitEventIds::MEMSTORE_MEM_PAGE_ALLOC_INFO, throttling_interval, cur_mem_hold, sleep_interval, cur_ts);
usleep(1); // Here sleep 1us is to let the wait guard recognize this statistic
// The playback of a single log may allocate 2M blocks multiple times
uint32_t final_sleep_interval = static_cast<uint32_t>(
MIN((get_writing_throttling_sleep_interval() + sleep_interval - 1), MAX_WAIT_INTERVAL));
//The playback of a single log may allocate 2M blocks multiple times
uint32_t final_sleep_interval =
static_cast<uint32_t>(MIN((get_writing_throttling_sleep_interval() + sleep_interval - 1), MAX_WAIT_INTERVAL));
get_writing_throttling_sleep_interval() = final_sleep_interval;
throttle_info_.record_limit_event(sleep_interval - 1);
} else {
@ -335,46 +309,30 @@ void ObFifoArena::speed_limit(int64_t cur_mem_hold, int64_t alloc_size)
last_reclaimed_ = ATOMIC_LOAD(&reclaimed_);
}
if (REACH_TIME_INTERVAL(1 * 1000 * 1000L)) {
COMMON_LOG(INFO,
"report write throttle info",
K(alloc_size),
K(throttling_interval),
K(attr_),
"freed memory(MB):",
(ATOMIC_LOAD(&reclaimed_) - last_reclaimed_) / 1024 / 1024,
"last_base_ts",
ATOMIC_LOAD(&last_base_ts_),
K(cur_mem_hold),
K(trigger_mem_limit),
K(trigger_mem_remain),
K(tenant_memory_remain),
K(throttle_info_),
K(mem_overused));
COMMON_LOG(INFO, "report write throttle info", K(alloc_size), K(throttling_interval), K(attr_),
"freed memory(MB):" , (ATOMIC_LOAD(&reclaimed_) - last_reclaimed_) / 1024 / 1024,
"last_base_ts", ATOMIC_LOAD(&last_base_ts_),
K(cur_mem_hold), K(throttle_info_));
}
}
} else { /*do nothing*/
}
} else {/*do nothing*/}
}
}
int64_t ObFifoArena::get_throttling_interval(const int64_t mem_overused,
const int64_t alloc_size,
const bool is_memstore_overused)
int64_t ObFifoArena::get_throttling_interval(int64_t cur_mem_hold,
int64_t alloc_size,
int64_t trigger_mem_limit)
{
constexpr int64_t MIN_INTERVAL_PER_ALLOC = 20;
int64_t chunk_cnt = ((alloc_size + MEM_SLICE_SIZE - 1) / (MEM_SLICE_SIZE));
int64_t chunk_seq = (mem_overused + alloc_size + MEM_SLICE_SIZE - 1)/ (MEM_SLICE_SIZE);
int64_t chunk_seq = ((cur_mem_hold - trigger_mem_limit) + MEM_SLICE_SIZE - 1)/ (MEM_SLICE_SIZE);
int64_t ret_interval = 0;
double cur_chunk_seq = 1.0;
for (int64_t i = 0; i < chunk_cnt && cur_chunk_seq > 0.0; ++i) {
cur_chunk_seq = static_cast<double>(chunk_seq - i);
ret_interval += static_cast<int64_t>(throttle_info_.decay_factor_ * cur_chunk_seq * cur_chunk_seq * cur_chunk_seq);
}
ret_interval = ret_interval + MIN_INTERVAL;
const int64_t calc_sleep_interval = alloc_size * ret_interval / MEM_SLICE_SIZE;
const int64_t actual_sleep_interval = is_memstore_overused ? calc_sleep_interval
: MAX(TENANT_MEMORY_EXHAUSTION_FACTOR * calc_sleep_interval, MIN_SLEEP_INTERVAL_WITH_TENANT_MEMORY_EXHAUSTION);
return actual_sleep_interval;
return alloc_size * ret_interval / MEM_SLICE_SIZE + MIN_INTERVAL_PER_ALLOC;
}
void ObFifoArena::set_memstore_threshold(int64_t memstore_threshold)
@ -382,10 +340,18 @@ void ObFifoArena::set_memstore_threshold(int64_t memstore_threshold)
ATOMIC_STORE(&lastest_memstore_threshold_, memstore_threshold);
}
template<int64_t N>
struct INTEGER_WRAPPER
{
INTEGER_WRAPPER() : v_(N) {}
int64_t v_;
};
int64_t ObFifoArena::get_writing_throttling_trigger_percentage_() const
{
static thread_local int64_t trigger_percentage = DEFAULT_TRIGGER_PERCENTAGE;
if (TC_REACH_TIME_INTERVAL(1 * 1000 * 1000)) { // 1s
RLOCAL(INTEGER_WRAPPER<DEFAULT_TRIGGER_PERCENTAGE>, wrapper);
int64_t &trigger_percentage = (&wrapper)->v_;
if (TC_REACH_TIME_INTERVAL(5 * 1000 * 1000)) { // 5s
omt::ObTenantConfigGuard tenant_config(TENANT_CONF(attr_.tenant_id_));
if (!tenant_config.is_valid()) {
COMMON_LOG(INFO, "failed to get tenant config", K(attr_));
@ -398,11 +364,12 @@ int64_t ObFifoArena::get_writing_throttling_trigger_percentage_() const
int64_t ObFifoArena::get_writing_throttling_maximum_duration_() const
{
static thread_local int64_t duration = DEFAULT_DURATION;
if (TC_REACH_TIME_INTERVAL(1 * 1000 * 1000)) { // 1s
RLOCAL(INTEGER_WRAPPER<DEFAULT_DURATION>, wrapper);
int64_t &duration = (&wrapper)->v_;
if (TC_REACH_TIME_INTERVAL(1 * 1000 * 1000)) { // 1s
omt::ObTenantConfigGuard tenant_config(TENANT_CONF(attr_.tenant_id_));
if (!tenant_config.is_valid()) {
// keep default
//keep default
COMMON_LOG(INFO, "failed to get tenant config", K(attr_));
} else {
duration = tenant_config->writing_throttling_maximum_duration;
@ -411,17 +378,5 @@ int64_t ObFifoArena::get_writing_throttling_maximum_duration_() const
return duration;
}
int64_t ObFifoArena::get_tenant_memory_remain_()
{
const uint64_t tenant_id = attr_.tenant_id_;
int64_t cur_time = ObTimeUtility::current_time();
int64_t old_time = ATOMIC_LOAD(&last_query_time_);
if (OB_UNLIKELY((QUERY_MEM_INTERVAL + old_time) < cur_time)
&& old_time == ATOMIC_CAS(&last_query_time_, old_time, cur_time)) {
ATOMIC_STORE(&remain_, lib::get_tenant_memory_remain(tenant_id));
}
return ATOMIC_LOAD(&remain_);
}
}; // namespace common
}; // end namespace oceanbase
}; // end namespace allocator
}; // end namespace oceanbase