hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[FEAT MERGE] shared_nothing_tmp_file

Browse Source 
Co-authored-by: lalalafeier <wufeifei2023@163.com>
Co-authored-by: wanyue-wy <345657357@qq.com>
Co-authored-by: simonjoylet <simonjoylet@gmail.com>
This commit is contained in:
obdev
2024-08-15 12:12:57 +00:00
committed by ob-robot
parent c189ffd722
commit a897079037
98 changed files with 25410 additions and 9474 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
unittest/storage/backup/test_backup_index_merger.cpp
View File

@ -28,6 +28,7 @@
#include "test_backup.h"
#include "test_backup_include.h"
#include "storage/blocksstable/ob_logic_macro_id.h"
#include "mtlenv/mock_tenant_module_env.h"
using namespace testing;
using namespace oceanbase;
@ -317,12 +318,11 @@ void TestBackupIndexMerger::SetUp()
}
// set observer memory limit
CHUNK_MGR.set_limit(8L * 1024L * 1024L * 1024L);
ret = ObTmpFileManager::get_instance().init();
if (OB_INIT_TWICE == ret) {
ret = OB_SUCCESS;
} else {
ASSERT_EQ(OB_SUCCESS, ret);
}
ASSERT_EQ(OB_SUCCESS, common::ObClockGenerator::init());
ASSERT_EQ(OB_SUCCESS, tmp_file::ObTmpBlockCache::get_instance().init("tmp_block_cache", 1));
ASSERT_EQ(OB_SUCCESS, tmp_file::ObTmpPageCache::get_instance().init("tmp_page_cache", 1));
static ObTenantBase tenant_ctx(OB_SYS_TENANT_ID);
ObTenantEnv::set_tenant(&tenant_ctx);
ObTenantIOManager *io_service = nullptr;
@ -330,14 +330,24 @@ void TestBackupIndexMerger::SetUp()
EXPECT_EQ(OB_SUCCESS, ObTenantIOManager::mtl_init(io_service));
EXPECT_EQ(OB_SUCCESS, io_service->start());
tenant_ctx.set(io_service);
tmp_file::ObTenantTmpFileManager *tf_mgr = nullptr;
EXPECT_EQ(OB_SUCCESS, mtl_new_default(tf_mgr));
EXPECT_EQ(OB_SUCCESS, tmp_file::ObTenantTmpFileManager::mtl_init(tf_mgr));
tf_mgr->page_cache_controller_.write_buffer_pool_.default_wbp_memory_limit_ = 40*1024*1024;
EXPECT_EQ(OB_SUCCESS, tf_mgr->start());
tenant_ctx.set(tf_mgr);
ObTenantEnv::set_tenant(&tenant_ctx);
inner_init_();
}
void TestBackupIndexMerger::TearDown()
{
ObTmpFileManager::get_instance().destroy();
tmp_file::ObTmpBlockCache::get_instance().destroy();
tmp_file::ObTmpPageCache::get_instance().destroy();
ObKVGlobalCache::get_instance().destroy();
common::ObClockGenerator::destroy();
}
void TestBackupIndexMerger::inner_init_()

19
unittest/storage/backup/test_backup_iterator.cpp
View File

@ -24,6 +24,7 @@
#include "test_backup_include.h"
#include "share/backup/ob_backup_io_adapter.h"
#include "lib/string/ob_string.h"
#include "mtlenv/mock_tenant_module_env.h"
using namespace oceanbase;
using namespace oceanbase::common;
@ -111,7 +112,11 @@ void TestBackupIndexIterator::SetUp()
}
// set observer memory limit
CHUNK_MGR.set_limit(8L * 1024L * 1024L * 1024L);
ret = ObTmpFileManager::get_instance().init();
ASSERT_EQ(OB_SUCCESS, common::ObClockGenerator::init());
ASSERT_EQ(OB_SUCCESS, tmp_file::ObTmpBlockCache::get_instance().init("tmp_block_cache", 1));
ASSERT_EQ(OB_SUCCESS, tmp_file::ObTmpPageCache::get_instance().init("tmp_page_cache", 1));
if (OB_INIT_TWICE == ret) {
ret = OB_SUCCESS;
} else {
@ -124,6 +129,14 @@ void TestBackupIndexIterator::SetUp()
EXPECT_EQ(OB_SUCCESS, ObTenantIOManager::mtl_init(io_service));
EXPECT_EQ(OB_SUCCESS, io_service->start());
tenant_ctx.set(io_service);
tmp_file::ObTenantTmpFileManager *tf_mgr = nullptr;
EXPECT_EQ(OB_SUCCESS, mtl_new_default(tf_mgr));
EXPECT_EQ(OB_SUCCESS, tmp_file::ObTenantTmpFileManager::mtl_init(tf_mgr));
tf_mgr->page_cache_controller_.write_buffer_pool_.default_wbp_memory_limit_ = 40*1024*1024;
EXPECT_EQ(OB_SUCCESS, tf_mgr->start());
tenant_ctx.set(tf_mgr);
ObTenantEnv::set_tenant(&tenant_ctx);
inner_init_();
ASSERT_EQ(OB_SUCCESS, bandwidth_throttle_.init(1024 * 1024 * 60));
@ -131,8 +144,10 @@ void TestBackupIndexIterator::SetUp()
void TestBackupIndexIterator::TearDown()
{
ObTmpFileManager::get_instance().destroy();
tmp_file::ObTmpBlockCache::get_instance().destroy();
tmp_file::ObTmpPageCache::get_instance().destroy();
ObKVGlobalCache::get_instance().destroy();
common::ObClockGenerator::destroy();
}
void TestBackupIndexIterator::inner_init_()

20
unittest/storage/backup/test_backup_macro_block_index_merger.cpp
View File

@ -15,6 +15,7 @@
#define private public
#define protected public
#include "observer/omt/ob_tenant_mtl_helper.h"
#include "storage/backup/ob_backup_data_struct.h"
#include "storage/backup/ob_backup_index_merger.h"
#include "storage/backup/ob_backup_index_store.h"
@ -190,12 +191,8 @@ void TestBackupMacroIndexMerger::SetUp()
}
// set observer memory limit
CHUNK_MGR.set_limit(8L * 1024L * 1024L * 1024L);
ret = ObTmpFileManager::get_instance().init();
if (OB_INIT_TWICE == ret) {
ret = OB_SUCCESS;
} else {
ASSERT_EQ(OB_SUCCESS, ret);
}
ASSERT_EQ(OB_SUCCESS, tmp_file::ObTmpPageCache::get_instance().init("tmp_page_cache", 1));
static ObTenantBase tenant_ctx(OB_SYS_TENANT_ID);
ObTenantEnv::set_tenant(&tenant_ctx);
ObTenantIOManager *io_service = nullptr;
@ -203,14 +200,23 @@ void TestBackupMacroIndexMerger::SetUp()
EXPECT_EQ(OB_SUCCESS, ObTenantIOManager::mtl_init(io_service));
EXPECT_EQ(OB_SUCCESS, io_service->start());
tenant_ctx.set(io_service);
tmp_file::ObTenantTmpFileManager *tf_mgr = nullptr;
EXPECT_EQ(OB_SUCCESS, mtl_new_default(tf_mgr));
EXPECT_EQ(OB_SUCCESS, tmp_file::ObTenantTmpFileManager::mtl_init(tf_mgr));
tf_mgr->page_cache_controller_.write_buffer_pool_.default_wbp_memory_limit_ = 40*1024*1024;
EXPECT_EQ(OB_SUCCESS, tf_mgr->start());
tenant_ctx.set(tf_mgr);
ObTenantEnv::set_tenant(&tenant_ctx);
inner_init_();
}
void TestBackupMacroIndexMerger::TearDown()
{
ObTmpFileManager::get_instance().destroy();
tmp_file::ObTmpPageCache::get_instance().destroy();
ObKVGlobalCache::get_instance().destroy();
common::ObClockGenerator::destroy();
}
void TestBackupMacroIndexMerger::clean_env_()

19
unittest/storage/backup/test_backup_tmp_file.cpp
View File

@ -20,6 +20,7 @@
#include "storage/backup/ob_backup_data_struct.h"
#include "storage/blocksstable/ob_data_file_prepare.h"
#include "test_backup.h"
#include "mtlenv/mock_tenant_module_env.h"
using namespace oceanbase;
using namespace oceanbase::common;
@ -66,8 +67,10 @@ void TestBackupTmpFile::SetUp()
}
// set observer memory limit
CHUNK_MGR.set_limit(8L * 1024L * 1024L * 1024L);
ret = ObTmpFileManager::get_instance().init();
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(OB_SUCCESS, common::ObClockGenerator::init());
ASSERT_EQ(OB_SUCCESS, tmp_file::ObTmpBlockCache::get_instance().init("tmp_block_cache", 1));
ASSERT_EQ(OB_SUCCESS, tmp_file::ObTmpPageCache::get_instance().init("tmp_page_cache", 1));
static ObTenantBase tenant_ctx(OB_SYS_TENANT_ID);
ObTenantEnv::set_tenant(&tenant_ctx);
ObTenantIOManager *io_service = nullptr;
@ -75,14 +78,24 @@ void TestBackupTmpFile::SetUp()
EXPECT_EQ(OB_SUCCESS, ObTenantIOManager::mtl_init(io_service));
EXPECT_EQ(OB_SUCCESS, io_service->start());
tenant_ctx.set(io_service);
tmp_file::ObTenantTmpFileManager *tf_mgr = nullptr;
EXPECT_EQ(OB_SUCCESS, mtl_new_default(tf_mgr));
EXPECT_EQ(OB_SUCCESS, tmp_file::ObTenantTmpFileManager::mtl_init(tf_mgr));
tf_mgr->page_cache_controller_.write_buffer_pool_.default_wbp_memory_limit_ = 40*1024*1024;
EXPECT_EQ(OB_SUCCESS, tf_mgr->start());
tenant_ctx.set(tf_mgr);
ObTenantEnv::set_tenant(&tenant_ctx);
}
void TestBackupTmpFile::TearDown()
{
ObTmpFileManager::get_instance().destroy();
tmp_file::ObTmpBlockCache::get_instance().destroy();
tmp_file::ObTmpPageCache::get_instance().destroy();
ObKVGlobalCache::get_instance().destroy();
TestDataFilePrepare::TearDown();
common::ObClockGenerator::destroy();
}
static void make_meta_index(backup::ObBackupMetaIndex &meta_index)

20
unittest/storage/backup/test_backup_utils.cpp
View File

@ -23,6 +23,7 @@
#include "storage/blocksstable/ob_logic_macro_id.h"
#include "storage/ob_parallel_external_sort.h"
#include "storage/blocksstable/ob_data_file_prepare.h"
#include "mtlenv/mock_tenant_module_env.h"
#include "storage/backup/ob_backup_ctx.h"
using namespace oceanbase;
@ -261,7 +262,10 @@ void TestBackupExternalSort::SetUp()
{
TestDataFilePrepare::SetUp();
EXPECT_EQ(OB_SUCCESS, init_tenant_mgr());
EXPECT_EQ(OB_SUCCESS, ObTmpFileManager::get_instance().init());
ASSERT_EQ(OB_SUCCESS, common::ObClockGenerator::init());
ASSERT_EQ(OB_SUCCESS, tmp_file::ObTmpBlockCache::get_instance().init("tmp_block_cache", 1));
ASSERT_EQ(OB_SUCCESS, tmp_file::ObTmpPageCache::get_instance().init("tmp_page_cache", 1));
static ObTenantBase tenant_ctx(OB_SYS_TENANT_ID);
ObTenantEnv::set_tenant(&tenant_ctx);
ObTenantIOManager *io_service = nullptr;
@ -269,14 +273,26 @@ void TestBackupExternalSort::SetUp()
EXPECT_EQ(OB_SUCCESS, ObTenantIOManager::mtl_init(io_service));
EXPECT_EQ(OB_SUCCESS, io_service->start());
tenant_ctx.set(io_service);
tmp_file::ObTenantTmpFileManager *tf_mgr = nullptr;
EXPECT_EQ(OB_SUCCESS, mtl_new_default(tf_mgr));
EXPECT_EQ(OB_SUCCESS, tmp_file::ObTenantTmpFileManager::mtl_init(tf_mgr));
tf_mgr->page_cache_controller_.write_buffer_pool_.default_wbp_memory_limit_ = 40*1024*1024;
EXPECT_EQ(OB_SUCCESS, tf_mgr->start());
tenant_ctx.set(tf_mgr);
ObTenantEnv::set_tenant(&tenant_ctx);
}
void TestBackupExternalSort::TearDown()
{
allocator_.reuse();
ObTmpFileManager::get_instance().destroy();
tmp_file::ObTmpBlockCache::get_instance().destroy();
tmp_file::ObTmpPageCache::get_instance().destroy();
TestDataFilePrepare::TearDown();
common::ObClockGenerator::destroy();
destroy_tenant_mgr();
}