hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[TMP.FILE] fix block leak and limit disk for tmp file

Browse Source
This commit is contained in:
Tyshawn 2021-12-20 20:04:31 +08:00 committed by LINxiansheng
parent ea5bd71d32
commit 18de895dc8
4 changed files with 86 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/storage/blocksstable/ob_tmp_file_cache.cpp
View File

@ -1057,6 +1057,8 @@ int ObTmpTenantMemBlockManager::write_io(const ObTmpBlockIOInfo& io_info, ObMacr
full_meta.schema_ = &macro_schema;
if (OB_FAIL(build_macro_meta(io_info.tenant_id_, full_meta))) {
STORAGE_LOG(WARN, "fail to build macro meta", K(ret));
} else if (OB_FAIL(OB_STORE_FILE.check_disk_full(OB_FILE_SYSTEM.get_macro_block_size()))) {
STORAGE_LOG(WARN, "fail to check space full", K(ret));
} else {
write_info.io_desc_ = io_info.io_desc_;
write_info.meta_ = full_meta;

17
src/storage/blocksstable/ob_tmp_file_store.cpp
View File

@ -104,12 +104,17 @@ int ObTmpFilePageBuddy::alloc_all_pages()
return ret;
}
int ObTmpFilePageBuddy::alloc(const int32_t page_nums, int32_t& start_page_id)
int ObTmpFilePageBuddy::alloc(const int32_t page_nums,
int32_t &start_page_id,
int32_t &alloced_page_nums)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "ObTmpFilePageBuddy has not been inited", K(ret));
} else if (OB_UNLIKELY(page_nums <= 0)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid argument", K(ret), K(page_nums));
} else {
int32_t index = std::ceil(std::log(page_nums) / std::log(2));
bool is_alloced = false;
@ -127,6 +132,7 @@ int ObTmpFilePageBuddy::alloc(const int32_t page_nums, int32_t& start_page_id)
free_area_[static_cast<int32_t>(std::log(tmp->page_nums_) / std::log(2))] = area;
}
start_page_id = tmp->start_page_id_;
alloced_page_nums = std::pow(2, index);
is_alloced = true;
tmp->~ObTmpFileArea();
}
@ -456,17 +462,18 @@ int ObTmpMacroBlock::alloc(const int32_t page_nums, ObTmpFileExtent& extent)
{
int ret = OB_SUCCESS;
int32_t start_page_id = extent.get_start_page_id();
int32_t alloced_page_nums = 0;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "ObTmpMacroBlock has not been inited", K(ret));
} else if (OB_FAIL(page_buddy_.alloc(page_nums, start_page_id))) {
} else if (OB_FAIL(page_buddy_.alloc(page_nums, start_page_id, alloced_page_nums))) {
STORAGE_LOG(WARN, "Fail to allocate the tmp extent", K(ret), K_(block_id), K_(page_buddy));
} else {
extent.set_block_id(block_id_);
extent.set_page_nums(page_nums);
extent.set_page_nums(alloced_page_nums);
extent.set_start_page_id(start_page_id);
extent.alloced();
free_page_nums_ -= page_nums;
free_page_nums_ -= alloced_page_nums;
if (OB_FAIL(using_extents_.push_back(&extent))) {
STORAGE_LOG(WARN, "Fail to push back into using_extexts", K(ret));
page_buddy_.free(extent.get_start_page_id(), extent.get_page_nums());
@ -811,7 +818,7 @@ int ObTmpTenantFileStore::alloc(
STORAGE_LOG(WARN, "fail to alloc tmp extent", K(ret));
int tmp_ret = OB_SUCCESS;
if (free_blocks.count() > 0) {
for (int32_t i = free_blocks.count() - 1; OB_SUCC(tmp_ret) && i >= 0; i--) {
for (int32_t i = free_blocks.count() - 1; OB_SUCCESS == tmp_ret && i >= 0; i--) {
if (free_blocks.at(i)->is_empty()) {
if (OB_SUCCESS != (tmp_ret = free_macro_block(free_blocks.at(i)))) {
STORAGE_LOG(WARN, "fail to free tmp macro block", K(tmp_ret));

2
src/storage/blocksstable/ob_tmp_file_store.h
View File

@ -54,7 +54,7 @@ public:
int init(common::ObIAllocator& allocator);
void destroy();
int alloc_all_pages();
int alloc(const int32_t page_nums, int32_t& start_page_id);
int alloc(const int32_t page_nums, int32_t &start_page_id, int32_t &alloced_page_nums);
void free(const int32_t start_page_id, const int32_t page_nums);
OB_INLINE int64_t get_max_cont_page_nums() const
{

78
unittest/storage/blocksstable/test_tmp_file.cpp
View File

@ -1203,8 +1203,9 @@ TEST_F(TestTmpFile, test_single_dir_two_file)
{
int ret = OB_SUCCESS;
int64_t dir = -1;
int64_t fd = -1;
const int64_t macro_block_size = OB_FILE_SYSTEM.get_macro_block_size();
int64_t fd_1 = -1;
int64_t fd_2 = -1;
const int64_t macro_block_size = 64 * 1024;
ObTmpFileIOInfo io_info1;
ObTmpFileIOInfo io_info2;
ObTmpFileIOHandle handle1;
@ -1218,18 +1219,18 @@ TEST_F(TestTmpFile, test_single_dir_two_file)
}
char* read_buf = new char[macro_block_size + 256];
ret = ObTmpFileManager::get_instance().open(fd, dir);
ret = ObTmpFileManager::get_instance().open(fd_1, dir);
ASSERT_EQ(OB_SUCCESS, ret);
io_info1.fd_ = fd;
io_info1.fd_ = fd_1;
io_info1.tenant_id_ = 1;
io_info1.io_desc_.category_ = USER_IO;
io_info1.io_desc_.wait_event_no_ = 2;
io_info1.buf_ = write_buf;
io_info1.size_ = macro_block_size + 256;
ret = ObTmpFileManager::get_instance().open(fd, dir);
ret = ObTmpFileManager::get_instance().open(fd_2, dir);
ASSERT_EQ(OB_SUCCESS, ret);
io_info2.fd_ = fd;
io_info2.fd_ = fd_2;
io_info2.tenant_id_ = 1;
io_info2.io_desc_.category_ = USER_IO;
io_info2.io_desc_.wait_event_no_ = 2;
@ -1265,7 +1266,13 @@ TEST_F(TestTmpFile, test_single_dir_two_file)
cmp = memcmp(handle2.get_buffer(), write_buf, macro_block_size + 256);
ASSERT_EQ(0, cmp);
ObTmpFileManager::get_instance().remove(fd);
ObTmpTenantFileStore *store = NULL;
OB_TMP_FILE_STORE.get_store(1, store);
store->print_block_usage();
ObTmpFileManager::get_instance().remove(fd_1);
ObTmpFileManager::get_instance().remove(fd_2);
OB_TMP_FILE_STORE.get_store(1, store);
store->print_block_usage();
}
/*TEST_F(TestTmpFile, test_iter_end)
@ -1574,6 +1581,63 @@ TEST_F(TestTmpFile, test_sql_workload)
ObTmpFileManager::get_instance().remove(fd);
}
TEST_F(TestTmpFile, test_page_buddy)
{
int ret = OB_SUCCESS;
ObArenaAllocator allocator;
ObTmpFilePageBuddy page_buddy_1;
ret = page_buddy_1.init(allocator);
ASSERT_EQ(OB_SUCCESS, ret);
int32_t page_nums = 64;
int32_t alloced_page_nums = 64;
int32_t start_page_id = -1;
ret = page_buddy_1.alloc(page_nums, start_page_id, alloced_page_nums);
ASSERT_EQ(OB_SUCCESS, ret);
int32_t start_page_id_2 = -1;
ret = page_buddy_1.alloc(page_nums, start_page_id_2, alloced_page_nums);
ASSERT_EQ(OB_SUCCESS, ret);
page_buddy_1.free(start_page_id + 63, page_nums - 63);
page_buddy_1.free(start_page_id_2 + 1, page_nums - 1);
page_nums = 63;
page_buddy_1.free(start_page_id, page_nums);
page_nums = 1;
page_buddy_1.free(start_page_id_2, page_nums);
STORAGE_LOG(INFO, "page buddy", K(page_buddy_1));
ASSERT_EQ(true, page_buddy_1.is_empty());
ObTmpFilePageBuddy page_buddy_2;
ret = page_buddy_2.init(allocator);
start_page_id = 0;
ret = page_buddy_2.alloc_all_pages();
ASSERT_EQ(OB_SUCCESS, ret);
int32_t free_nums = 511 - 129;
page_buddy_2.free(start_page_id + 129, free_nums);
free_nums = 127;
page_buddy_2.free(start_page_id + 2, free_nums);
free_nums = 2;
page_buddy_2.free(start_page_id, free_nums);
STORAGE_LOG(INFO, "page buddy", K(page_buddy_2));
ASSERT_EQ(true, page_buddy_2.is_empty());
for (int32_t i = 1; i < 129; i++) {
ObTmpFilePageBuddy page_buddy_3;
int32_t page_num_2 = i;
ret = page_buddy_3.init(allocator);
ASSERT_EQ(OB_SUCCESS, ret);
ret = page_buddy_3.alloc(page_num_2, start_page_id, alloced_page_nums);
ASSERT_EQ(OB_SUCCESS, ret);
page_buddy_3.free(start_page_id, alloced_page_nums);
STORAGE_LOG(INFO, "page buddy", K(page_buddy_3));
ASSERT_EQ(true, page_buddy_3.is_empty());
STORAGE_LOG(INFO, "page buddy", K(page_buddy_3));
}
}
} // end namespace unittest
} // end namespace oceanbase