hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[FEAT MERGE] add direct load function

Browse Source 
Co-authored-by: Monk-Liu <1152761042@qq.com>
Co-authored-by: saltonz <saltonzh@gmail.com>
Co-authored-by: yongshige <598633031@qq.com>
This commit is contained in:
obdev
2023-01-28 18:08:50 +08:00
committed by ob-robot
parent f27d2efc83
commit 81d28c0295
384 changed files with 55860 additions and 1239 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
unittest/storage/CMakeLists.txt
View File

@ -20,6 +20,7 @@ add_subdirectory(mockcontainer)
add_subdirectory(transaction)
add_subdirectory(tx)
add_subdirectory(blocksstable)
add_subdirectory(direct_load)
add_subdirectory(utl_file)
add_subdirectory(tx_table)

2
unittest/storage/backup/test_backup_index_merger.cpp
View File

@ -284,7 +284,7 @@ protected:
};
TestBackupIndexMerger::TestBackupIndexMerger()
: TestDataFilePrepare(&getter, "TestBackupIndexMerger"),
: TestDataFilePrepare(&getter, "TestBackupIndexMerger", OB_DEFAULT_MACRO_BLOCK_SIZE, 800),
tenant_base_(500),
job_desc_(),
backup_dest_(),

4
unittest/storage/backup/test_backup_iterator.cpp
View File

@ -73,7 +73,7 @@ protected:
TestBackupIndexIterator::TestBackupIndexIterator()
: TestDataFilePrepare(&getter, "TestBackupIndexIterator"),
: TestDataFilePrepare(&getter, "TestBackupIndexIterator", OB_DEFAULT_MACRO_BLOCK_SIZE, 500),
tenant_base_(500),
job_desc_(),
backup_dest_(),
@ -344,4 +344,4 @@ int main(int argc, char **argv)
OB_LOGGER.set_log_level("INFO");
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
}

266
unittest/storage/blocksstable/test_tmp_file.cpp
View File

@ -14,6 +14,8 @@
#define protected public
#define private public
#include "storage/blocksstable/ob_tmp_file.h"
#include "storage/blocksstable/ob_tmp_file_store.h"
#include "storage/blocksstable/ob_tmp_file_cache.h"
#include "ob_row_generate.h"
#include "ob_data_file_prepare.h"
#include "share/ob_simple_mem_limit_getter.h"
@ -80,6 +82,7 @@ class TestTmpFileStress : public share::ObThreadPool
{
public:
TestTmpFileStress();
TestTmpFileStress(ObTenantBase *tenant_ctx);
virtual ~TestTmpFileStress();
int init(const int fd, const bool is_write, const int64_t thread_cnt, ObTableSchema *table_schema,
const bool is_plain_data, const bool is_big_file);
@ -103,6 +106,7 @@ private:
bool is_big_file_;
ObTableSchema *table_schema_;
bool is_plain_;
ObTenantBase *tenant_ctx_;
};
TestTmpFileStress::TestTmpFileStress()
@ -111,6 +115,13 @@ TestTmpFileStress::TestTmpFileStress()
{
}
TestTmpFileStress::TestTmpFileStress(ObTenantBase *tenant_ctx)
: thread_cnt_(0), size_(OB_SERVER_BLOCK_MGR.get_macro_block_size()), fd_(0),
is_write_(false), is_big_file_(false), table_schema_(NULL), is_plain_(false),
tenant_ctx_(tenant_ctx)
{
}
TestTmpFileStress::~TestTmpFileStress()
{
}
@ -340,8 +351,7 @@ void TestTmpFileStress::read_plain_data(const char *read_buf, const int64_t macr
void TestTmpFileStress::run1()
{
ObTenantBase tenant_ctx(1);
ObTenantEnv::set_tenant(&tenant_ctx);
ObTenantEnv::set_tenant(tenant_ctx_);
if (is_plain_) {
char *buf = NULL;
write_plain_data(buf, size_);
@ -359,6 +369,7 @@ class TestMultiTmpFileStress : public share::ObThreadPool
{
public:
TestMultiTmpFileStress();
TestMultiTmpFileStress(ObTenantBase *tenant_ctx);
virtual ~TestMultiTmpFileStress();
int init(const int64_t file_cnt, const int64_t dir_id, const int64_t thread_cnt,
ObTableSchema *table_schema, const bool is_plain_data, const bool is_big_file);
@ -373,6 +384,7 @@ private:
ObTableSchema *table_schema_;
bool is_big_file_;
bool is_plain_data_;
ObTenantBase *tenant_ctx_;
};
TestMultiTmpFileStress::TestMultiTmpFileStress()
@ -384,6 +396,16 @@ TestMultiTmpFileStress::TestMultiTmpFileStress()
is_plain_data_(false)
{
}
TestMultiTmpFileStress::TestMultiTmpFileStress(ObTenantBase *tenant_ctx)
: file_cnt_(0),
dir_id_(-1),
thread_cnt_perf_file_(0),
table_schema_(NULL),
is_big_file_(false),
is_plain_data_(false),
tenant_ctx_(tenant_ctx)
{
}
TestMultiTmpFileStress::~TestMultiTmpFileStress()
{
@ -417,7 +439,7 @@ void TestMultiTmpFileStress::run_plain_case()
{
int ret = OB_SUCCESS;
int64_t fd = 0;
TestTmpFileStress test;
TestTmpFileStress test(tenant_ctx_);
ret = ObTmpFileManager::get_instance().open(fd, dir_id_);
ASSERT_EQ(OB_SUCCESS, ret);
ret = test.init(fd, true, thread_cnt_perf_file_, table_schema_, is_plain_data_, is_big_file_);
@ -433,8 +455,8 @@ void TestMultiTmpFileStress::run_normal_case()
int ret = OB_SUCCESS;
int64_t fd = 0;
const int64_t timeout_ms = 50000;
TestTmpFileStress test_write;
TestTmpFileStress test_read;
TestTmpFileStress test_write(tenant_ctx_);
TestTmpFileStress test_read(tenant_ctx_);
ret = ObTmpFileManager::get_instance().open(fd, dir_id_);
ASSERT_EQ(OB_SUCCESS, ret);
STORAGE_LOG(INFO, "open file success", K(fd));
@ -454,6 +476,7 @@ void TestMultiTmpFileStress::run_normal_case()
void TestMultiTmpFileStress::run1()
{
ObTenantEnv::set_tenant(tenant_ctx_);
if (is_plain_data_) {
run_plain_case();
} else {
@ -557,6 +580,7 @@ void TestTmpFile::TearDown()
table_schema_.reset();
ObTmpFileManager::get_instance().destroy();
ObKVGlobalCache::get_instance().destroy();
ObTmpFileStore::get_instance().destroy();
TestDataFilePrepare::TearDown();
}
@ -645,7 +669,7 @@ TEST_F(TestTmpFile, test_multi_small_file_single_thread_read_write)
const int64_t file_cnt = 4;
const bool is_plain_data = false;
const bool is_big_file = false;
TestMultiTmpFileStress test;
TestMultiTmpFileStress test(MTL_CTX());
int64_t dir = -1;
ret = ObTmpFileManager::get_instance().alloc_dir(dir);
ASSERT_EQ(OB_SUCCESS, ret);
@ -674,7 +698,7 @@ TEST_F(TestTmpFile, test_multi_small_file_multi_thread_read_write )
const int64_t file_cnt = 4;
const bool is_plain_data = false;
const bool is_big_file = false;
TestMultiTmpFileStress test;
TestMultiTmpFileStress test(MTL_CTX());
int64_t dir = -1;
ret = ObTmpFileManager::get_instance().alloc_dir(dir);
ASSERT_EQ(OB_SUCCESS, ret);
@ -782,7 +806,7 @@ TEST_F(TestTmpFile, test_single_file_single_thread_read_write)
const int64_t file_cnt = 1;
const bool is_plain_data = false;
const bool is_big_file = true;
TestMultiTmpFileStress test;
TestMultiTmpFileStress test(MTL_CTX());
int64_t dir = -1;
ret = ObTmpFileManager::get_instance().alloc_dir(dir);
ASSERT_EQ(OB_SUCCESS, ret);
@ -895,8 +919,8 @@ TEST_F(TestTmpFile, test_100_small_files)
int64_t fd = 0;
int count = 100;
const int64_t timeout_ms = 50000;
TestTmpFileStress test_write;
TestTmpFileStress test_read;
TestTmpFileStress test_write(MTL_CTX());
TestTmpFileStress test_read(MTL_CTX());
ret = ObTmpFileManager::get_instance().alloc_dir(dir);
ASSERT_EQ(OB_SUCCESS, ret);
while (count--) {
@ -937,7 +961,7 @@ TEST_F(TestTmpFile, test_single_file_multi_thread_read_write)
const int64_t file_cnt = 1;
const bool is_plain_data = false;
const bool is_big_file = true;
TestMultiTmpFileStress test;
TestMultiTmpFileStress test(MTL_CTX());
int64_t dir = -1;
ret = ObTmpFileManager::get_instance().alloc_dir(dir);
ASSERT_EQ(OB_SUCCESS, ret);
@ -967,7 +991,7 @@ TEST_F(TestTmpFile, test_multi_file_single_thread_read_write)
const int64_t file_cnt = 4;
const bool is_plain_data = false;
const bool is_big_file = true;
TestMultiTmpFileStress test;
TestMultiTmpFileStress test(MTL_CTX());
int64_t dir = -1;
ret = ObTmpFileManager::get_instance().alloc_dir(dir);
ASSERT_EQ(OB_SUCCESS, ret);
@ -997,7 +1021,7 @@ TEST_F(TestTmpFile, test_multi_file_multi_thread_read_write)
const int64_t file_cnt = 4;
const bool is_plain_data = false;
const bool is_big_file = true;
TestMultiTmpFileStress test;
TestMultiTmpFileStress test(MTL_CTX());
int64_t dir = -1;
ret = ObTmpFileManager::get_instance().alloc_dir(dir);
ASSERT_EQ(OB_SUCCESS, ret);
@ -1027,7 +1051,7 @@ TEST_F(TestTmpFile, test_write_not_macro_size)
const int64_t file_cnt = 1;
const bool is_plain_data = true;
const bool is_big_file = true;
TestMultiTmpFileStress test;
TestMultiTmpFileStress test(MTL_CTX());
int64_t dir = -1;
ret = ObTmpFileManager::get_instance().alloc_dir(dir);
ASSERT_EQ(OB_SUCCESS, ret);
@ -1383,7 +1407,7 @@ TEST_F(TestTmpFile, test_single_dir_multi_file)
const int64_t file_cnt = 4;
const bool is_plain_data = false;
const bool is_big_file = false;
TestMultiTmpFileStress test;
TestMultiTmpFileStress test(MTL_CTX());
int64_t dir = -1;
ret = ObTmpFileManager::get_instance().alloc_dir(dir);
ASSERT_EQ(OB_SUCCESS, ret);
@ -1413,7 +1437,7 @@ TEST_F(TestTmpFile, test_drop_tenant_file)
const int64_t file_cnt = 4;
const bool is_plain_data = false;
const bool is_big_file = true;
TestMultiTmpFileStress test;
TestMultiTmpFileStress test(MTL_CTX());
int64_t dir = -1;
ret = ObTmpFileManager::get_instance().alloc_dir(dir);
ASSERT_EQ(OB_SUCCESS, ret);
@ -1434,8 +1458,8 @@ TEST_F(TestTmpFile, test_drop_tenant_file)
int64_t fd = 0;
int count = 100;
const int64_t timeout_ms = 50000;
TestTmpFileStress test_write;
TestTmpFileStress test_read;
TestTmpFileStress test_write(MTL_CTX());
TestTmpFileStress test_read(MTL_CTX());
ret = ObTmpFileManager::get_instance().alloc_dir(dir);
ASSERT_EQ(OB_SUCCESS, ret);
while (count--) {
@ -1623,15 +1647,15 @@ TEST_F(TestTmpFile, test_page_buddy)
ret = page_buddy_1.init(allocator);
ASSERT_EQ(OB_SUCCESS, ret);
int32_t page_nums = 64;
int32_t alloced_page_nums = 64;
int32_t start_page_id = -1;
uint8_t page_nums = 64;
uint8_t alloced_page_nums = 64;
uint8_t start_page_id = 255;
ASSERT_EQ(true, page_buddy_1.is_empty());
ret = page_buddy_1.alloc(page_nums, start_page_id, alloced_page_nums);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(false, page_buddy_1.is_empty());
int32_t start_page_id_2 = -1;
uint8_t start_page_id_2 = 255;
ret = page_buddy_1.alloc(page_nums, start_page_id_2, alloced_page_nums);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(false, page_buddy_1.is_empty());
@ -1691,6 +1715,204 @@ TEST_F(TestTmpFile, test_page_buddy)
ASSERT_EQ(false, page_buddy_4.is_empty());
}
TEST_F(TestTmpFile, test_tmp_file_sync)
{
int ret = OB_SUCCESS;
int64_t dir = -1;
int64_t fd = -1;
ObTmpFileIOInfo io_info;
ObTmpFileIOHandle handle;
ret = ObTmpFileManager::get_instance().alloc_dir(dir);
ASSERT_EQ(OB_SUCCESS, ret);
ret = ObTmpFileManager::get_instance().open(fd, dir);
ASSERT_EQ(OB_SUCCESS, ret);
int64_t write_size = 16*1024;
char *write_buf = (char *)malloc(write_size);
for (int64_t i = 0; i < write_size; ++i) {
write_buf[i] = static_cast<char>(i % 256);
}
io_info.fd_ = fd;
io_info.tenant_id_ = 1;
io_info.io_desc_.set_group_id(THIS_WORKER.get_group_id());
io_info.io_desc_.set_wait_event(2);
io_info.buf_ = write_buf;
io_info.size_ = write_size;
const int64_t timeout_ms = 5000;
int64_t write_time = ObTimeUtility::current_time();
ret = ObTmpFileManager::get_instance().write(io_info, timeout_ms);
write_time = ObTimeUtility::current_time() - write_time;
ASSERT_EQ(OB_SUCCESS, ret);
free(write_buf);
STORAGE_LOG(INFO, "test_tmp_file_sync");
STORAGE_LOG(INFO, "io time", K(write_time));
ObTmpTenantFileStoreHandle store_handle;
OB_TMP_FILE_STORE.get_store(1, store_handle);
ASSERT_EQ(1, store_handle.get_tenant_store()->tmp_mem_block_manager_.t_mblk_map_.size());
ObTmpFileManager::get_instance().sync(fd, 5000);
ASSERT_EQ(0, store_handle.get_tenant_store()->tmp_mem_block_manager_.t_mblk_map_.size());
store_handle.get_tenant_store()->print_block_usage();
ObMallocAllocator::get_instance()->print_tenant_memory_usage(1);
ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(1);
ObMallocAllocator::get_instance()->print_tenant_memory_usage(500);
ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(500);
ObTmpFileManager::get_instance().remove(fd);
}
TEST_F(TestTmpFile, test_tmp_file_sync_same_block)
{
int ret = OB_SUCCESS;
int64_t dir = -1;
int64_t fd1, fd2 = -1;
const int64_t timeout_ms = 5000;
ObTmpFileIOHandle handle;
ObTmpFileIOInfo io_info;
io_info.tenant_id_ = 1;
io_info.io_desc_.set_group_id(THIS_WORKER.get_group_id());
io_info.io_desc_.set_wait_event(2);
int64_t write_size = 16 *1024;
char *write_buf = (char *)malloc(write_size);
for (int64_t i = 0; i < write_size; ++i) {
write_buf[i] = static_cast<char>(i % 256);
}
io_info.buf_ = write_buf;
io_info.size_ = write_size;
ret = ObTmpFileManager::get_instance().alloc_dir(dir);
ASSERT_EQ(OB_SUCCESS, ret);
ret = ObTmpFileManager::get_instance().open(fd1, dir);
ASSERT_EQ(OB_SUCCESS, ret);
io_info.fd_ = fd1;
int64_t write_time = ObTimeUtility::current_time();
ret = ObTmpFileManager::get_instance().write(io_info, timeout_ms);
write_time = ObTimeUtility::current_time() - write_time;
ASSERT_EQ(OB_SUCCESS, ret);
ret = ObTmpFileManager::get_instance().open(fd2, dir);
ASSERT_EQ(OB_SUCCESS, ret);
io_info.fd_ = fd2;
write_time = ObTimeUtility::current_time();
ret = ObTmpFileManager::get_instance().write(io_info, timeout_ms);
write_time = ObTimeUtility::current_time() - write_time;
ASSERT_EQ(OB_SUCCESS, ret);
free(write_buf);
STORAGE_LOG(INFO, "test_tmp_file_sync_same_block");
STORAGE_LOG(INFO, "io time", K(write_time));
ObTmpTenantFileStoreHandle store_handle;
OB_TMP_FILE_STORE.get_store(1, store_handle);
ASSERT_EQ(1, store_handle.get_tenant_store()->tmp_mem_block_manager_.t_mblk_map_.size());
ObTmpFileManager::get_instance().sync(fd1, 5000);
ASSERT_EQ(1, store_handle.get_tenant_store()->tmp_mem_block_manager_.t_mblk_map_.size());
ObTmpFileManager::get_instance().sync(fd2, 5000);
ASSERT_EQ(0, store_handle.get_tenant_store()->tmp_mem_block_manager_.t_mblk_map_.size());
store_handle.get_tenant_store()->print_block_usage();
ObMallocAllocator::get_instance()->print_tenant_memory_usage(1);
ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(1);
ObMallocAllocator::get_instance()->print_tenant_memory_usage(500);
ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(500);
ObTmpFileManager::get_instance().remove(fd1);
ObTmpFileManager::get_instance().remove(fd2);
}
TEST_F(TestTmpFile, test_tmp_file_wash)
{
int ret = OB_SUCCESS;
const int64_t timeout_ms = 5000;
int count = 64;
int64_t dir = -1;
int64_t fd = -1;
ObTmpFileIOHandle handle;
ObTmpFileIOInfo io_info;
io_info.tenant_id_ = 1;
io_info.io_desc_.set_group_id(THIS_WORKER.get_group_id());
io_info.io_desc_.set_wait_event(2);
int64_t write_size = 1024 *1024;
char *write_buf = (char *)malloc(write_size);
for (int64_t i = 0; i < write_size; ++i) {
write_buf[i] = static_cast<char>(i % 256);
}
io_info.buf_ = write_buf;
io_info.size_ = write_size;
for(int64_t i=0; i<count;i++){
ret = ObTmpFileManager::get_instance().alloc_dir(dir);
ASSERT_EQ(OB_SUCCESS, ret);
ret = ObTmpFileManager::get_instance().open(fd, dir);
ASSERT_EQ(OB_SUCCESS, ret);
io_info.fd_ = fd;
ret = ObTmpFileManager::get_instance().write(io_info, timeout_ms);
ASSERT_EQ(OB_SUCCESS, ret);
if(i==count/2){
ret = ObTmpFileManager::get_instance().write(io_info, timeout_ms);
ASSERT_EQ(OB_SUCCESS, ret);
}
}
free(write_buf);
STORAGE_LOG(INFO, "test_tmp_file_wash");
ObTmpTenantFileStoreHandle store_handle;
OB_TMP_FILE_STORE.get_store(1, store_handle);
int64_t oldest_id = -1;
int64_t oldest_time = INT64_MAX;
int64_t newest_time = -1;
int64_t newest_id = -1;
int64_t used_up_id = -1;
ObTmpTenantMemBlockManager::TmpMacroBlockMap::iterator iter;
for(iter = store_handle.get_tenant_store()->tmp_mem_block_manager_.t_mblk_map_.begin();
iter != store_handle.get_tenant_store()->tmp_mem_block_manager_.t_mblk_map_.end(); ++iter){
int64_t alloc_time = iter->second->get_alloc_time();
if (alloc_time < oldest_time){
oldest_id = iter->first;
oldest_time = alloc_time;
}
if (alloc_time > newest_time){
newest_id = iter->first;
newest_time = alloc_time;
}
if(iter->second->get_free_page_nums()==0){
used_up_id = iter->first;
}
}
ObTmpMacroBlock* wash_block;
ret = store_handle.get_tenant_store()->tmp_mem_block_manager_.t_mblk_map_.get_refactored(newest_id, wash_block);
ASSERT_EQ(OB_SUCCESS, ret);
wash_block->alloc_time_ = wash_block->alloc_time_ - 60 * 1000000L;
ObArray<ObTmpMacroBlock*> free_blocks;
ASSERT_EQ(64, store_handle.get_tenant_store()->tmp_mem_block_manager_.t_mblk_map_.size());
store_handle.get_tenant_store()->tmp_mem_block_manager_.wash(3,free_blocks);
ret = store_handle.get_tenant_store()->tmp_mem_block_manager_.t_mblk_map_.get_refactored(oldest_id, wash_block);
ASSERT_NE(OB_SUCCESS, ret);
ret = store_handle.get_tenant_store()->tmp_mem_block_manager_.t_mblk_map_.get_refactored(newest_id, wash_block);
ASSERT_NE(OB_SUCCESS, ret);
ret = store_handle.get_tenant_store()->tmp_mem_block_manager_.t_mblk_map_.get_refactored(used_up_id, wash_block);
ASSERT_NE(OB_SUCCESS, ret);
ASSERT_EQ(61, store_handle.get_tenant_store()->tmp_mem_block_manager_.t_mblk_map_.size());
store_handle.get_tenant_store()->print_block_usage();
ObMallocAllocator::get_instance()->print_tenant_memory_usage(1);
ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(1);
ObMallocAllocator::get_instance()->print_tenant_memory_usage(500);
ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(500);
count = 64;
while (count--) {
ret = ObTmpFileManager::get_instance().remove(count);
ASSERT_EQ(OB_SUCCESS, ret);
}
}
} // end namespace unittest
} // end namespace oceanbase

2
unittest/storage/direct_load/CMakeLists.txt Normal file
View File

@ -0,0 +1,2 @@
storage_unittest(test_direct_load_index_block_writer)
storage_unittest(test_direct_load_data_block_writer)

1529
unittest/storage/direct_load/test_direct_load_data_block_writer.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

196
unittest/storage/direct_load/test_direct_load_index_block_writer.cpp Normal file
View File

@ -0,0 +1,196 @@
// Copyright 2014-2014 Alibaba Inc. All Rights Reserved.
// Author:
// yiren.ly <yiren.ly@oceanbase.com>
//
// This file defines test_direct_load_data_block_writer.cpp
#include <gtest/gtest.h>
#define private public
#define protected public
#include <cstdlib>
#include <ctime>
#include "../unittest/storage/blocksstable/ob_data_file_prepare.h"
#include "../unittest/storage/blocksstable/ob_row_generate.h"
#include "observer/table_load/ob_table_load_partition_location.h"
#include "share/ob_simple_mem_limit_getter.h"
#include "storage/blocksstable/ob_tmp_file.h"
#include "storage/direct_load/ob_direct_load_tmp_file.h"
#include "storage/direct_load/ob_direct_load_sstable_scanner.h"
#include "storage/ob_i_store.h"
namespace oceanbase
{
using namespace common;
using namespace blocksstable;
using namespace storage;
using namespace share::schema;
static ObSimpleMemLimitGetter getter;
namespace unittest
{
class TestIndexBlockWriter : public TestDataFilePrepare
{
public:
static const int64_t rowkey_column_count = 2;
// Every ObObjType from ObTinyIntType to ObHexStringType inclusive.
// Skip ObNullType and ObExtendType because for external usage, a column type
// can't be NULL or NOP.
static const int64_t column_num = ObHexStringType;
static const int64_t macro_block_size = 2L * 8 * 1024L;
static const int64_t SNAPSHOT_VERSION = 2;
public:
TestIndexBlockWriter() : TestDataFilePrepare(&getter, "TestIndexBlockWriter", 2 * 1024 * 1024, 2048){};
virtual void SetUp();
virtual void TearDown();
static void SetUpTestCase() {}
static void TearDownTestCase() {}
void test_alloc(char *&ptr, const int64_t size);
private:
void prepare_schema();
protected:
ObTableSchema table_schema_;
ObRowGenerate row_generate_;
ObDirectLoadTmpFileManager *file_mgr_;
ObArenaAllocator allocator_;
};
void TestIndexBlockWriter::test_alloc(char *&ptr, const int64_t size)
{
ptr = reinterpret_cast<char *>(allocator_.alloc(size));
ASSERT_TRUE(nullptr != ptr);
}
void TestIndexBlockWriter::prepare_schema()
{
ObColumnSchemaV2 column;
int64_t table_id = 3001;
int64_t micro_block_size = 16 * 1024;
// init table schema
table_schema_.reset();
ASSERT_EQ(OB_SUCCESS, table_schema_.set_table_name("test_macro_file"));
table_schema_.set_tenant_id(1);
table_schema_.set_tablegroup_id(1);
table_schema_.set_database_id(1);
table_schema_.set_table_id(table_id);
table_schema_.set_rowkey_column_num(rowkey_column_count);
table_schema_.set_max_used_column_id(column_num);
table_schema_.set_block_size(micro_block_size);
table_schema_.set_compress_func_name("none");
// init column
char name[OB_MAX_FILE_NAME_LENGTH];
memset(name, 0, sizeof(name));
for (int64_t i = 0; i < column_num; ++i) {
ObObjType obj_type = static_cast<ObObjType>(i + 1);
column.reset();
column.set_table_id(table_id);
column.set_column_id(i + OB_APP_MIN_COLUMN_ID);
sprintf(name, "test%020ld", i);
ASSERT_EQ(OB_SUCCESS, column.set_column_name(name));
column.set_data_type(obj_type);
if (obj_type == common::ObIntType) {
column.set_rowkey_position(1);
} else if (obj_type == common::ObNumberType) {
column.set_rowkey_position(2);
} else {
column.set_rowkey_position(0);
}
column.set_collation_type(ObCollationType::CS_TYPE_UTF8MB4_GENERAL_CI);
ASSERT_EQ(OB_SUCCESS, table_schema_.add_column(column));
}
ObTmpFileManager::get_instance().destroy();
}
void TestIndexBlockWriter::SetUp()
{
int ret = OB_SUCCESS;
// init file
const int64_t bucket_num = 1024;
const int64_t max_cache_size = 1024 * 1024 * 1024;
const int64_t block_size = common::OB_MALLOC_BIG_BLOCK_SIZE;
TestDataFilePrepare::SetUp();
prepare_schema();
file_mgr_ = OB_NEWx(ObDirectLoadTmpFileManager, (&allocator_));
ret = file_mgr_->init(table_schema_.get_tenant_id());
ASSERT_EQ(OB_SUCCESS, ret);
// init ObRowGenerate
ASSERT_EQ(OB_SUCCESS, row_generate_.init(table_schema_));
ret = getter.add_tenant(1, 8L * 1024L * 1024L, 2L * 1024L * 1024L * 1024L);
ASSERT_EQ(OB_SUCCESS, ret);
ret = ObKVGlobalCache::get_instance().init(&getter, bucket_num, max_cache_size, block_size);
if (OB_INIT_TWICE == ret) {
ret = OB_SUCCESS;
} else {
ASSERT_EQ(OB_SUCCESS, ret);
}
// set observer memory limit
CHUNK_MGR.set_limit(8L * 1024L * 1024L * 1024L);
ret = ObTmpFileManager::get_instance().init();
ASSERT_EQ(OB_SUCCESS, ret);
static ObTenantBase tenant_ctx(1);
ObTenantEnv::set_tenant(&tenant_ctx);
ObTenantIOManager *io_service = nullptr;
EXPECT_EQ(OB_SUCCESS, ObTenantIOManager::mtl_init(io_service));
}
void TestIndexBlockWriter::TearDown()
{
file_mgr_->~ObDirectLoadTmpFileManager();
table_schema_.reset();
ObTmpFileManager::get_instance().destroy();
ObKVGlobalCache::get_instance().destroy();
TestDataFilePrepare::TearDown();
}
TEST_F(TestIndexBlockWriter, test_write_and_read)
{
int ret = OB_SUCCESS;
ObArray<int64_t> array;
int64_t sum = 0;
ObDirectLoadIndexBlockWriter index_block_writer;
ObDirectLoadIndexBlockReader index_block_reader;
uint64_t tenant_id = table_schema_.get_tenant_id();
ObDirectLoadTmpFileHandle file_handle;
const int64_t index_block_size = DIO_ALIGN_SIZE;
int64_t dir_id = -1;
ret = file_mgr_->alloc_dir(dir_id);
ASSERT_EQ(OB_SUCCESS, ret);
ret= file_mgr_->alloc_file(dir_id, file_handle);
ASSERT_EQ(OB_SUCCESS, ret);
// init index block writer
ret = index_block_writer.init(tenant_id, DIO_ALIGN_SIZE, file_handle);
ASSERT_EQ(OB_SUCCESS, ret);
for (int64_t i = 0; i < 1000; ++i) {
ObDirectLoadIndexBlockItem item;
sum += rand() % 1024;
item.end_offset_ = sum;
array.push_back(item.end_offset_);
ret = index_block_writer.append_row(100, item);
ASSERT_EQ(OB_SUCCESS, ret);
}
ret = index_block_writer.close();
ASSERT_EQ(OB_SUCCESS, ret);
ret = index_block_reader.init(1, DIO_ALIGN_SIZE, file_handle);
ASSERT_EQ(OB_SUCCESS, ret);
for (int64_t i = 0; i < 1000; ++i) {
ObDirectLoadIndexInfo info;
ret = index_block_reader.get_index_info(i, info);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_TRUE(array.at(i) == (info.offset_ + info.size_));
}
}
} // end namespace unittest
} // end namespace oceanbase
int main(int argc, char **argv)
{
system("rm -rf test_direct_load_index_block_writer.log");
OB_LOGGER.set_file_name("test_direct_load_index_block_writer.log", true, true);
oceanbase::common::ObLogger::get_logger().set_log_level("INFO");
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}