hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[CP] [FEAT MERGE] chunk_mgr_opt

Browse Source
This commit is contained in:
tushicheng
2024-03-14 09:46:12 +00:00
committed by ob-robot
parent d6e0624219
commit 668b1289ad
8 changed files with 278 additions and 378 deletions
Download Patch File Download Diff File Expand all files Collapse all files

262
deps/oblib/unittest/lib/alloc/test_chunk_mgr.cpp vendored
View File

@ -31,9 +31,7 @@ class TestChunkMgr
{
public:
TestChunkMgr()
{
large_free_list_.set_max_chunk_cache_size(AChunkList::DEFAULT_MAX_CHUNK_CACHE_SIZE);
}
{}
virtual void SetUp()
{
}
@ -55,43 +53,45 @@ TEST_F(TestChunkMgr, NormalChunk)
{
int NORMAL_SIZE = OB_MALLOC_BIG_BLOCK_SIZE;
int LARGE_SIZE = INTACT_ACHUNK_SIZE + 100;
int normal_hold = 0;
int normal_hold = AChunkMgr::hold(NORMAL_SIZE);
int large_hold = AChunkMgr::hold(LARGE_SIZE);
{
int64_t hold = 0;
AChunk *chunks[1024] = {};
for (int i = 0; i < 1024; i++) {
AChunk *chunks[8] = {};
// direct alloc 2M
for (int i = 0; i < 8; ++i) {
chunks[i] = alloc_chunk(NORMAL_SIZE);
normal_hold = chunks[i]->hold();
hold += normal_hold;
}
set_max_chunk_cache_size(hold - normal_hold);
for (int i = 0; i < 1024; i++) {
// create 2M-cache
for (int i = 0; i < 8; ++i) {
free_chunk(chunks[i]);
chunks[i] = NULL;
}
EXPECT_EQ(1023, free_list_.get_pushes());
EXPECT_EQ(0, free_list_.get_pops());
EXPECT_EQ(0, large_free_list_.get_pushes());
EXPECT_EQ(0, large_free_list_.get_pops());
hold -= normal_hold;
// alloc chunk from 2M-cache
for (int i = 0; i < 8; ++i) {
chunks[i] = alloc_chunk(NORMAL_SIZE);
}
EXPECT_EQ(8, slots_[0]->get_pushes());
EXPECT_EQ(8, slots_[0]->get_pops());
EXPECT_EQ(0, slots_[1]->get_pushes());
EXPECT_EQ(0, slots_[1]->get_pops());
EXPECT_EQ(hold, hold_);
// direct alloc 4M
// alloc chunk by wash 4M-cache
{
auto *chunk = alloc_chunk(LARGE_SIZE);
AChunk *chunk = alloc_chunk(LARGE_SIZE);
EXPECT_TRUE(NULL != chunk);
EXPECT_EQ(0, free_list_.get_pops());
EXPECT_EQ(0, large_free_list_.get_pops());
hold += chunk->hold();
hold += large_hold;
free_chunk(chunk);
EXPECT_EQ(hold, hold_);
}
// wash alloc
{
set_limit(hold);
auto *chunk = alloc_chunk(LARGE_SIZE);
chunk = alloc_chunk(NORMAL_SIZE);
EXPECT_TRUE(NULL != chunk);
EXPECT_EQ(2, free_list_.get_pops());
hold = hold - normal_hold * 2 + chunk->hold();
hold -= (large_hold - normal_hold);
EXPECT_EQ(8, slots_[0]->get_pushes());
EXPECT_EQ(8, slots_[0]->get_pops());
EXPECT_EQ(1, slots_[1]->get_pushes());
EXPECT_EQ(1, slots_[1]->get_pops());
EXPECT_EQ(hold, hold_);
}
}
@ -101,43 +101,56 @@ TEST_F(TestChunkMgr, LargeChunk)
{
int NORMAL_SIZE = OB_MALLOC_BIG_BLOCK_SIZE;
int LARGE_SIZE = INTACT_ACHUNK_SIZE + 100;
int large_hold = 0;
int BIG_SIZE = INTACT_ACHUNK_SIZE * 2 + 100;
int normal_hold = AChunkMgr::hold(NORMAL_SIZE);
int large_hold = AChunkMgr::hold(LARGE_SIZE);
int big_hold = AChunkMgr::hold(BIG_SIZE);
{
int64_t hold = 0;
AChunk *chunks[1024] = {};
for (int i = 0; i < 1024; i++) {
AChunk *chunks[8] = {};
// direct alloc 4M
for (int i = 0; i < 8; ++i) {
chunks[i] = alloc_chunk(LARGE_SIZE);
large_hold = chunks[i]->hold();
hold += large_hold;
}
set_max_large_chunk_cache_size(hold - large_hold);
for (int i = 0; i < 1024; i++) {
// create 4M-cache
for (int i = 0; i < 8; ++i) {
free_chunk(chunks[i]);
}
EXPECT_EQ(1023, large_free_list_.get_pushes());
EXPECT_EQ(0, large_free_list_.get_pops());
EXPECT_EQ(0, free_list_.get_pushes());
EXPECT_EQ(0, free_list_.get_pops());
hold -= large_hold;
// alloc chunk from self-cache(4M-cache)
for (int i = 0; i < 8; ++i) {
chunks[i] = alloc_chunk(LARGE_SIZE);
}
EXPECT_EQ(8, slots_[1]->get_pushes());
EXPECT_EQ(8, slots_[1]->get_pops());
EXPECT_EQ(hold, hold_);
// direct alloc 2M
// alloc chunk by wash 6M-cache and 2M-cache
{
auto *chunk = alloc_chunk(NORMAL_SIZE);
AChunk *chunk = alloc_chunk(BIG_SIZE);
EXPECT_TRUE(NULL != chunk);
EXPECT_EQ(0, free_list_.get_pops());
EXPECT_EQ(0, large_free_list_.get_pops());
hold += chunk->hold();
hold += big_hold;
free_chunk(chunk);
chunk = alloc_chunk(NORMAL_SIZE);
EXPECT_TRUE(NULL != chunk);
hold += normal_hold;
free_chunk(chunk);
EXPECT_EQ(1, slots_[0]->get_pushes());
EXPECT_EQ(0, slots_[0]->get_pops());
EXPECT_EQ(1, slots_[2]->get_pushes());
EXPECT_EQ(0, slots_[2]->get_pops());
EXPECT_EQ(hold, hold_);
}
// wash alloc
{
set_limit(hold);
auto *chunk = alloc_chunk(NORMAL_SIZE);
chunk = alloc_chunk(LARGE_SIZE);
hold += (large_hold - big_hold);
EXPECT_TRUE(NULL != chunk);
EXPECT_EQ(1, large_free_list_.get_pops());
hold = hold - large_hold + chunk->hold();
chunk = alloc_chunk(LARGE_SIZE);
hold += (large_hold - normal_hold);
EXPECT_TRUE(NULL != chunk);
EXPECT_EQ(1, slots_[0]->get_pushes());
EXPECT_EQ(1, slots_[0]->get_pops());
EXPECT_EQ(1, slots_[2]->get_pushes());
EXPECT_EQ(1, slots_[2]->get_pops());
EXPECT_EQ(hold, hold_);
}
}
@ -147,55 +160,34 @@ TEST_F(TestChunkMgr, HugeChunk)
{
int NORMAL_SIZE = OB_MALLOC_BIG_BLOCK_SIZE;
int LARGE_SIZE = INTACT_ACHUNK_SIZE + 100;
int HUGE_SIZE = INTACT_ACHUNK_SIZE * 3;
set_limit(20L<<30);
int huge_hold = 0;
int HUGE_SIZE = INTACT_ACHUNK_SIZE * 10 + 100;
{
int64_t hold = 0;
{
int64_t temp_hold = 0;
AChunk *chunks[1024] = {};
for (int i = 0; i < 1024; i++) {
chunks[i] = alloc_chunk(LARGE_SIZE);
hold += chunks[i]->hold();
temp_hold += chunks[i]->hold();
AChunk *chunks[8][2] = {};
for (int i = 0; i < 8; ++i) {
chunks[i][0] = alloc_chunk(NORMAL_SIZE);
chunks[i][1] = alloc_chunk(LARGE_SIZE);
hold += chunks[i][0]->hold();
hold += chunks[i][1]->hold();
}
set_max_large_chunk_cache_size(temp_hold);
for (int i = 0; i < 1024; i++) {
free_chunk(chunks[i]);
for (int i = 0; i < 8; ++i) {
free_chunk(chunks[i][0]);
free_chunk(chunks[i][1]);
}
EXPECT_EQ(1024, large_free_list_.get_pushes());
EXPECT_EQ(0, large_free_list_.get_pops());
EXPECT_EQ(0, free_list_.get_pushes());
EXPECT_EQ(0, free_list_.get_pops());
EXPECT_EQ(hold, hold_);
}
{
int64_t temp_hold = 0;
AChunk *chunks[1024] = {};
for (int i = 0; i < 1024; i++) {
chunks[i] = alloc_chunk(NORMAL_SIZE);
hold += chunks[i]->hold();
temp_hold += chunks[i]->hold();
}
set_max_chunk_cache_size(temp_hold);
for (int i = 0; i < 1024; i++) {
free_chunk(chunks[i]);
}
EXPECT_EQ(1024, free_list_.get_pushes());
EXPECT_EQ(0, free_list_.get_pops());
EXPECT_EQ(1024, large_free_list_.get_pushes());
EXPECT_EQ(0, large_free_list_.get_pops());
EXPECT_EQ(8, slots_[0]->get_pushes());
EXPECT_EQ(0, slots_[0]->get_pops());
EXPECT_EQ(8, slots_[1]->get_pushes());
EXPECT_EQ(0, slots_[1]->get_pops());
EXPECT_EQ(hold, hold_);
}
// direct alloc huge
{
auto *chunk = alloc_chunk(HUGE_SIZE);
AChunk *chunk = alloc_chunk(HUGE_SIZE);
EXPECT_TRUE(NULL != chunk);
EXPECT_EQ(0, free_list_.get_pops());
EXPECT_EQ(0, large_free_list_.get_pops());
EXPECT_EQ(0, slots_[0]->get_pops());
EXPECT_EQ(0, slots_[1]->get_pops());
hold += chunk->hold();
EXPECT_EQ(hold, hold_);
}
@ -203,14 +195,14 @@ TEST_F(TestChunkMgr, HugeChunk)
// wash alloc
{
set_limit(hold);
auto *chunk = alloc_chunk(free_list_.hold() - 100);
AChunk *chunk = alloc_chunk(HUGE_SIZE);
EXPECT_TRUE(NULL != chunk);
EXPECT_EQ(0, free_list_.hold());
EXPECT_NE(0, large_free_list_.hold());
EXPECT_NE(0, slots_[0]->hold());
EXPECT_EQ(0, slots_[1]->hold());
chunk = alloc_chunk(large_free_list_.hold() - 100);
chunk = alloc_chunk(slots_[0]->hold());
EXPECT_TRUE(NULL != chunk);
EXPECT_EQ(0, large_free_list_.hold());
EXPECT_EQ(0, slots_[0]->hold());
}
}
}
@ -222,14 +214,14 @@ TEST_F(TestChunkMgr, BorderCase_advise_shrink)
auto *chunk = alloc_chunk(LARGE_SIZE);
// pollute chunk
memset(chunk->data_, 0xaa, chunk->hold());
EXPECT_EQ(0, large_free_list_.get_pushes());
EXPECT_EQ(0, slots_[1]->get_pushes());
int64_t orig_chunk_hold = chunk->hold();
int64_t orig_hold = hold_;
free_chunk(chunk);
EXPECT_EQ(1, large_free_list_.get_pushes());
EXPECT_EQ(0, large_free_list_.get_pops());
EXPECT_EQ(1, slots_[1]->get_pushes());
EXPECT_EQ(0, slots_[1]->get_pops());
chunk = alloc_chunk(LARGE_SIZE - ps * 3);
EXPECT_EQ(1, large_free_list_.get_pops());
EXPECT_EQ(1, slots_[1]->get_pops());
EXPECT_EQ(madvise_len_, ps * 3);
EXPECT_FALSE(0 == chunk->data_[0] && 0 == memcmp(chunk->data_, chunk->data_ + 1, chunk->hold() - 1));
EXPECT_EQ(orig_chunk_hold - chunk->hold(), orig_hold - hold_);
@ -242,14 +234,14 @@ TEST_F(TestChunkMgr, BorderCase_advise_expand)
auto *chunk = alloc_chunk(LARGE_SIZE);
// pollute chunk
memset(chunk->data_, 0xaa, chunk->hold());
EXPECT_EQ(0, large_free_list_.get_pushes());
EXPECT_EQ(0, slots_[1]->get_pushes());
int64_t orig_chunk_hold = chunk->hold();
int64_t orig_hold = hold_;
free_chunk(chunk);
EXPECT_EQ(1, large_free_list_.get_pushes());
EXPECT_EQ(0, large_free_list_.get_pops());
EXPECT_EQ(1, slots_[1]->get_pushes());
EXPECT_EQ(0, slots_[1]->get_pops());
chunk = alloc_chunk(LARGE_SIZE + ps * 3);
EXPECT_EQ(1, large_free_list_.get_pops());
EXPECT_EQ(1, slots_[1]->get_pops());
EXPECT_FALSE(0 == chunk->data_[0] && 0 == memcmp(chunk->data_, chunk->data_ + 1, chunk->hold() - 1));
EXPECT_EQ(orig_chunk_hold - (int64_t)chunk->hold(), orig_hold - hold_);
}
@ -261,16 +253,16 @@ TEST_F(TestChunkMgr, BorderCase_advise_fail)
auto *chunk = alloc_chunk(LARGE_SIZE);
// pollute chunk
memset(chunk->data_, 0xaa, chunk->hold());
EXPECT_EQ(0, large_free_list_.get_pushes());
EXPECT_EQ(0, slots_[1]->get_pushes());
int64_t orig_chunk_hold = chunk->hold();
int64_t orig_hold = hold_;
free_chunk(chunk);
EXPECT_EQ(1, large_free_list_.get_pushes());
EXPECT_EQ(0, large_free_list_.get_pops());
EXPECT_EQ(1, slots_[1]->get_pushes());
EXPECT_EQ(0, slots_[1]->get_pops());
need_fail_ = true;
chunk = alloc_chunk(LARGE_SIZE - ps * 3);
EXPECT_EQ(1, large_free_list_.get_pushes());
EXPECT_EQ(1, large_free_list_.get_pops());
EXPECT_EQ(1, slots_[1]->get_pushes());
EXPECT_EQ(1, slots_[1]->get_pops());
// check remap happened
EXPECT_TRUE(0 == chunk->data_[0] && 0 == memcmp(chunk->data_, chunk->data_ + 1, chunk->hold() - 1));
EXPECT_EQ(orig_chunk_hold - (int64_t)chunk->hold(), orig_hold - hold_);
@ -286,17 +278,17 @@ TEST_F(TestChunkMgr, alloc_co_chunk)
chunk = alloc_chunk(LARGE_SIZE);
free_chunk(chunk);
}
EXPECT_EQ(1, large_free_list_.get_pushes());
EXPECT_EQ(0, large_free_list_.get_pops());
EXPECT_EQ(1, free_list_.get_pushes());
EXPECT_EQ(0, free_list_.get_pops());
EXPECT_EQ(1, slots_[1]->get_pushes());
EXPECT_EQ(0, slots_[1]->get_pops());
EXPECT_EQ(1, slots_[0]->get_pushes());
EXPECT_EQ(0, slots_[0]->get_pops());
set_limit(hold_);
auto *chunk = alloc_co_chunk(NORMAL_SIZE);
EXPECT_TRUE(chunk != NULL);
EXPECT_EQ(1, free_list_.get_pops());
EXPECT_EQ(1, slots_[1]->get_pops());
chunk = alloc_co_chunk(NORMAL_SIZE);
EXPECT_TRUE(chunk != NULL);
EXPECT_EQ(1, large_free_list_.get_pops());
EXPECT_EQ(1, slots_[0]->get_pops());
}
TEST_F(TestChunkMgr, FreeListBasic)
@ -304,51 +296,29 @@ TEST_F(TestChunkMgr, FreeListBasic)
{
AChunk *chunk = alloc_chunk(0);
free_chunk(chunk);
EXPECT_EQ(1, free_list_.get_pushes());
EXPECT_EQ(1, slots_[0]->get_pushes());
}
{
AChunk *chunk = alloc_chunk(0);
free_chunk(chunk);
EXPECT_EQ(2, free_list_.get_pushes());
EXPECT_EQ(1, free_list_.get_pops());
EXPECT_EQ(2, slots_[0]->get_pushes());
EXPECT_EQ(1, slots_[0]->get_pops());
}
{
AChunk *chunk = alloc_chunk(OB_MALLOC_BIG_BLOCK_SIZE);
free_chunk(chunk);
EXPECT_EQ(3, free_list_.get_pushes());
EXPECT_EQ(2, free_list_.get_pops());
EXPECT_EQ(3, slots_[0]->get_pushes());
EXPECT_EQ(2, slots_[0]->get_pops());
}
}
TEST_F(TestChunkMgr, FreeListManyChunk)
{
AChunk *chunks[1024] = {};
for (int i = 0; i < 1024; i++) {
chunks[i] = alloc_chunk(OB_MALLOC_BIG_BLOCK_SIZE);
}
for (int i = 0; i < 1024; i++) {
free_chunk(chunks[i]);
}
EXPECT_EQ(AChunkList::DEFAULT_MAX_CHUNK_CACHE_SIZE/INTACT_ACHUNK_SIZE, free_list_.get_pushes());
EXPECT_EQ(0, free_list_.get_pops());
for (int i = 0; i < 1024; i++) {
chunks[i] = alloc_chunk(OB_MALLOC_BIG_BLOCK_SIZE);
}
for (int i = 0; i < 1024; i++) {
free_chunk(chunks[i]);
}
EXPECT_EQ(2* AChunkList::DEFAULT_MAX_CHUNK_CACHE_SIZE/INTACT_ACHUNK_SIZE, free_list_.get_pushes());
EXPECT_EQ(AChunkList::DEFAULT_MAX_CHUNK_CACHE_SIZE/INTACT_ACHUNK_SIZE, free_list_.get_pops());
}
TEST_F(TestChunkMgr, sync_wash)
{
set_limit(1LL<<30);
int NORMAL_SIZE = OB_MALLOC_BIG_BLOCK_SIZE;
int LARGE_SIZE = INTACT_ACHUNK_SIZE + 100;
free_list_.set_max_chunk_cache_size(1LL<<30);
large_free_list_.set_max_chunk_cache_size(1LL<<30);
slots_[0]->set_max_chunk_cache_size(1LL<<30);
slots_[1]->set_max_chunk_cache_size(1LL<<30);
AChunk *chunks[16][2] = {};
for (int i = 0; i < 16; ++i) {
chunks[i][0] = alloc_chunk(NORMAL_SIZE);
@ -360,13 +330,13 @@ TEST_F(TestChunkMgr, sync_wash)
chunks[i][j] = NULL;
}
}
int64_t hold = free_list_.hold() + large_free_list_.hold();
int64_t hold = get_freelist_hold();
EXPECT_EQ(hold, hold_);
EXPECT_EQ(16, free_list_.count());
EXPECT_EQ(16, large_free_list_.count());
EXPECT_EQ(16, slots_[0]->count());
EXPECT_EQ(16, slots_[1]->count());
int64_t washed_size = sync_wash();
EXPECT_EQ(hold, washed_size);
EXPECT_EQ(0, hold_);
EXPECT_EQ(0, free_list_.count());
EXPECT_EQ(0, large_free_list_.count());
EXPECT_EQ(0, slots_[0]->count());
EXPECT_EQ(0, slots_[1]->count());
}