hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[CP][FEAT MERGE] implement of log storage compression

Browse Source
This commit is contained in:
yyy-hust
2024-04-07 06:45:18 +00:00
committed by ob-robot
parent e007f676a5
commit 784d2231f6
50 changed files with 1173 additions and 284 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
src/share/allocator/ob_tenant_mutil_allocator.cpp
View File

@ -38,9 +38,8 @@ ObTenantMutilAllocator::ObTenantMutilAllocator(uint64_t tenant_id)
LOG_IO_FLASHBACK_TASK_SIZE(sizeof(palf::LogIOFlashbackTask)),
LOG_IO_PURGE_THROTTLING_TASK_SIZE(sizeof(palf::LogIOPurgeThrottlingTask)),
clog_blk_alloc_(),
common_blk_alloc_(),
unlimited_blk_alloc_(),
replay_log_task_blk_alloc_(REPLAY_MEM_LIMIT_THRESHOLD),
clog_compressing_blk_alloc_(CLOG_COMPRESSION_MEM_LIMIT_THRESHOLD),
clog_ge_alloc_(ObMemAttr(tenant_id, ObModIds::OB_CLOG_GE), ObVSliceAlloc::DEFAULT_BLOCK_SIZE, clog_blk_alloc_),
log_handle_submit_task_alloc_(LOG_HANDLE_SUBMIT_TASK_SIZE, ObMemAttr(tenant_id, "HandleSubmit"), choose_blk_size(LOG_HANDLE_SUBMIT_TASK_SIZE), clog_blk_alloc_, this),
log_io_flush_log_task_alloc_(LOG_IO_FLUSH_LOG_TASK_SIZE, ObMemAttr(tenant_id, "FlushLog"), choose_blk_size(LOG_IO_FLUSH_LOG_TASK_SIZE), clog_blk_alloc_, this),
@ -50,7 +49,8 @@ ObTenantMutilAllocator::ObTenantMutilAllocator(uint64_t tenant_id)
palf_fetch_log_task_alloc_(PALF_FETCH_LOG_TASK_SIZE, ObMemAttr(tenant_id, ObModIds::OB_FETCH_LOG_TASK), choose_blk_size(PALF_FETCH_LOG_TASK_SIZE), clog_blk_alloc_, this),
replay_log_task_alloc_(ObMemAttr(tenant_id, ObModIds::OB_LOG_REPLAY_TASK), common::OB_MALLOC_BIG_BLOCK_SIZE, replay_log_task_blk_alloc_),
log_io_flashback_task_alloc_(LOG_IO_FLASHBACK_TASK_SIZE, ObMemAttr(tenant_id, "Flashback"), choose_blk_size(LOG_IO_FLASHBACK_TASK_SIZE), clog_blk_alloc_, this),
log_io_purge_throttling_task_alloc_(LOG_IO_PURGE_THROTTLING_TASK_SIZE, ObMemAttr(tenant_id, "PurgeThrottle"), choose_blk_size(LOG_IO_PURGE_THROTTLING_TASK_SIZE), clog_blk_alloc_, this)
log_io_purge_throttling_task_alloc_(LOG_IO_PURGE_THROTTLING_TASK_SIZE, ObMemAttr(tenant_id, "PurgeThrottle"), choose_blk_size(LOG_IO_PURGE_THROTTLING_TASK_SIZE), clog_blk_alloc_, this),
clog_compression_buf_alloc_(ObMemAttr(tenant_id, "LogComBuf"), common::OB_MALLOC_BIG_BLOCK_SIZE, clog_compressing_blk_alloc_)
{
// set_nway according to tenant's max_cpu
double min_cpu = 0;
@ -83,6 +83,7 @@ void ObTenantMutilAllocator::destroy()
log_io_purge_throttling_task_alloc_.destroy();
palf_fetch_log_task_alloc_.destroy();
replay_log_task_alloc_.destroy();
clog_compression_buf_alloc_.destroy();
}
int ObTenantMutilAllocator::choose_blk_size(int obj_size)
@ -111,6 +112,7 @@ void ObTenantMutilAllocator::try_purge()
log_io_purge_throttling_task_alloc_.purge_extra_cached_block(0);
palf_fetch_log_task_alloc_.purge_extra_cached_block(0);
replay_log_task_alloc_.purge_extra_cached_block(0);
clog_compression_buf_alloc_.purge_extra_cached_block(0);
}
void *ObTenantMutilAllocator::ge_alloc(const int64_t size)
@ -340,12 +342,12 @@ void ObTenantMutilAllocator::set_limit(const int64_t total_limit)
ATOMIC_STORE(&total_limit_, total_limit);
const int64_t clog_limit = total_limit / 100 * CLOG_MEM_LIMIT_PERCENT;
const int64_t replay_limit = std::min(total_limit / 100 * REPLAY_MEM_LIMIT_PERCENT, REPLAY_MEM_LIMIT_THRESHOLD);
const int64_t common_limit = total_limit - (clog_limit + replay_limit);
const int64_t clog_compress_limit = std::min(total_limit / 100 * CLOG_COMPRESSION_MEM_LIMIT_PERCENT, CLOG_COMPRESSION_MEM_LIMIT_THRESHOLD);
clog_blk_alloc_.set_limit(clog_limit);
common_blk_alloc_.set_limit(common_limit);
replay_log_task_alloc_.set_limit(replay_limit);
clog_compressing_blk_alloc_.set_limit(clog_compress_limit);
OB_LOG(INFO, "ObTenantMutilAllocator set tenant mem limit finished", K(tenant_id_), K(total_limit), K(clog_limit),
K(replay_limit), K(common_limit));
K(replay_limit), K(clog_compress_limit));
}
}
@ -356,8 +358,31 @@ int64_t ObTenantMutilAllocator::get_limit() const
int64_t ObTenantMutilAllocator::get_hold() const
{
return clog_blk_alloc_.hold() + common_blk_alloc_.hold()
+ replay_log_task_blk_alloc_.hold();
return clog_blk_alloc_.hold() + replay_log_task_blk_alloc_.hold() + clog_compressing_blk_alloc_.hold();
}
void *ObTenantMutilAllocator::alloc_append_compression_buf(const int64_t size)
{
return clog_compression_buf_alloc_.alloc(size);
}
void ObTenantMutilAllocator::free_append_compression_buf(void *ptr)
{
if (OB_LIKELY(NULL != ptr)) {
clog_compression_buf_alloc_.free(ptr);
}
}
void *ObTenantMutilAllocator::alloc_replay_decompression_buf(const int64_t size)
{
return replay_log_task_alloc_.alloc(size);
}
void ObTenantMutilAllocator::free_replay_decompression_buf(void *ptr)
{
if (OB_LIKELY(NULL != ptr)) {
replay_log_task_alloc_.free(ptr);
}
}
#define SLICE_FREE_OBJ(name, cls) \

23
src/share/allocator/ob_tenant_mutil_allocator.h
View File

@ -75,8 +75,14 @@ public:
virtual void free_log_io_flashback_task(palf::LogIOFlashbackTask *ptr) = 0;
virtual palf::LogIOPurgeThrottlingTask *alloc_log_io_purge_throttling_task(const int64_t palf_id, const int64_t palf_epoch) = 0;
virtual void free_log_io_purge_throttling_task(palf::LogIOPurgeThrottlingTask *ptr) = 0;
virtual void *alloc_append_compression_buf(const int64_t size) = 0;
virtual void free_append_compression_buf(void *ptr) = 0;
virtual void *alloc_replay_decompression_buf(const int64_t size) = 0;
virtual void free_replay_decompression_buf(void *ptr) = 0;
virtual ObIAllocator *get_replay_decompression_allocator() = 0;
TO_STRING_KV(K_(flying_log_task), K_(flying_meta_task));
protected:
int64_t flying_log_task_;
int64_t flying_log_handle_submit_task_;
@ -93,8 +99,12 @@ public:
const int64_t REPLAY_MEM_LIMIT_PERCENT = 5;
// The memory limit of replay engine
const int64_t REPLAY_MEM_LIMIT_THRESHOLD = 512 * 1024 * 1024ll;
// The memory percent of clog compression
const int64_t CLOG_COMPRESSION_MEM_LIMIT_PERCENT = 3;
// The memory limit of clog compression
const int64_t CLOG_COMPRESSION_MEM_LIMIT_THRESHOLD = 128 * 1024 * 1024L;
// The memory percent of replay engine for inner_table
const int64_t INNER_TABLE_REPLAY_MEM_PERCENT = 20;
static int choose_blk_size(int obj_size);
public:
@ -140,6 +150,13 @@ public:
void free_log_io_flashback_task(palf::LogIOFlashbackTask *ptr);
palf::LogIOPurgeThrottlingTask *alloc_log_io_purge_throttling_task(const int64_t palf_id, const int64_t palf_epoch);
void free_log_io_purge_throttling_task(palf::LogIOPurgeThrottlingTask *ptr);
void *alloc_append_compression_buf(const int64_t size);
void free_append_compression_buf(void *ptr);
//alloc buf from replay_log_task_alloc
void *alloc_replay_decompression_buf(const int64_t size);
void free_replay_decompression_buf(void *ptr);
ObIAllocator *get_replay_decompression_allocator() {return &replay_log_task_alloc_;}
private:
uint64_t tenant_id_ CACHE_ALIGNED;
int64_t total_limit_;
@ -153,9 +170,8 @@ private:
const int LOG_IO_FLASHBACK_TASK_SIZE;
const int LOG_IO_PURGE_THROTTLING_TASK_SIZE;
ObBlockAllocMgr clog_blk_alloc_;
ObBlockAllocMgr common_blk_alloc_;
ObBlockAllocMgr unlimited_blk_alloc_;
ObBlockAllocMgr replay_log_task_blk_alloc_;
ObBlockAllocMgr clog_compressing_blk_alloc_;
ObVSliceAlloc clog_ge_alloc_;
ObSliceAlloc log_handle_submit_task_alloc_;
ObSliceAlloc log_io_flush_log_task_alloc_;
@ -166,6 +182,7 @@ private:
ObVSliceAlloc replay_log_task_alloc_;
ObSliceAlloc log_io_flashback_task_alloc_;
ObSliceAlloc log_io_purge_throttling_task_alloc_;
ObVSliceAlloc clog_compression_buf_alloc_;
};
} // end of namespace common