hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[FEAT MERGE] merge quick restore

Browse Source 
Co-authored-by: LoLolobster <949673574@qq.com>
Co-authored-by: JiahuaChen <garfieldjia@qq.com>
Co-authored-by: oceanoverflow <oceanoverflow@gmail.com>
This commit is contained in:
wxhwang
2024-08-02 07:23:13 +00:00
committed by ob-robot
parent bb08bad69e
commit 1a7b85c347
281 changed files with 25802 additions and 4677 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
unittest/storage/backup/test_backup.h
View File

@ -54,19 +54,6 @@ static void make_random_logic_id(blocksstable::ObLogicMacroBlockId &logic_id)
logic_id.logic_version_ = random(1, 10000000);
}
static void make_random_backup_physical_id(ObBackupPhysicalID &physical_id)
{
physical_id.ls_id_ = random(1, ObBackupPhysicalID::MAX_BACKUP_LS_ID);
physical_id.type_ = 0;
physical_id.turn_id_ = random(1, ObBackupPhysicalID::MAX_BACKUP_TURN_ID);
physical_id.retry_id_ = random(0, ObBackupPhysicalID::MAX_BACKUP_RETRY_ID);
physical_id.file_id_ = random(0, ObBackupPhysicalID::MAX_BACKUP_FILE_ID);
physical_id.reserved_ = 0;
physical_id.backup_set_id_ = random(1, ObBackupPhysicalID::MAX_BACKUP_SET_ID);
physical_id.offset_ = random(1, ObBackupPhysicalID::MAX_BACKUP_FILE_SIZE);
physical_id.length_ = random(1, ObBackupPhysicalID::MAX_BACKUP_BLOCK_SIZE);
}
static void make_random_backup_set_id(int64_t &backup_set_id)
{
backup_set_id = random(1, 4096);
@ -110,12 +97,6 @@ static void make_random_table_key(storage::ObITable::TableKey &table_key)
table_key.scn_range_.end_scn_.convert_for_gts(100);
}
static void make_random_pair(ObBackupMacroBlockIDPair &pair)
{
make_random_logic_id(pair.logic_id_);
make_random_backup_physical_id(pair.physical_id_);
}
static void make_random_cache_key(ObBackupIndexCacheKey &cache_key)
{
ObBackupBlockDesc block_desc;

2
unittest/storage/backup/test_backup_ctx.cpp
View File

@ -154,7 +154,7 @@ void TestBackupCtx::inner_init_()
job_desc_.task_id_ = 1;
backup_set_desc_.backup_set_id_ = 1;
backup_set_desc_.backup_type_.type_ = ObBackupType::FULL_BACKUP;
backup_data_type_.set_major_data_backup();
backup_data_type_.set_user_data_backup();
incarnation_ = 1;
tenant_id_ = 1002;
ls_id_ = ObLSID(1001);

188
unittest/storage/backup/test_backup_data_path.cpp Normal file
View File

@ -0,0 +1,188 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX STORAGE
#include <gtest/gtest.h>
#define private public
#define protected public
#include "share/backup/ob_backup_path.h"
using namespace oceanbase;
using namespace oceanbase::common;
using namespace oceanbase::share;
using namespace oceanbase::blocksstable;
namespace oceanbase
{
namespace backup
{
class TestBackupPath : public ::testing::Test {
public:
TestBackupPath();
virtual ~TestBackupPath();
virtual void SetUp();
virtual void TearDown();
private:
void inner_init_();
protected:
ObBackupDest backup_dest_;
int64_t task_id_;
int64_t incarnation_;
int64_t tenant_id_;
int64_t dest_id_;
ObBackupSetDesc backup_set_desc_;
share::ObLSID ls_id_;
ObBackupDataType backup_data_type_;
int64_t turn_id_;
int64_t retry_id_;
int64_t file_id_;
common::ObInOutBandwidthThrottle throttle_;
char test_dir_[OB_MAX_URI_LENGTH];
char test_dir_uri_[OB_MAX_URI_LENGTH];
DISALLOW_COPY_AND_ASSIGN(TestBackupPath);
};
TestBackupPath::TestBackupPath()
: backup_dest_(),
incarnation_(),
tenant_id_(OB_INVALID_ID),
dest_id_(0),
backup_set_desc_(),
ls_id_(),
backup_data_type_(),
turn_id_(-1),
retry_id_(-1),
file_id_(-1),
test_dir_(""),
test_dir_uri_("")
{}
TestBackupPath::~TestBackupPath()
{}
void TestBackupPath::SetUp()
{
inner_init_();
}
void TestBackupPath::inner_init_()
{
int ret = OB_SUCCESS;
ret = databuff_printf(test_dir_, sizeof(test_dir_), "%s", "obbackup");
EXPECT_EQ(OB_SUCCESS, ret);
ret = databuff_printf(test_dir_uri_, sizeof(test_dir_uri_), "file:///%s", test_dir_);
EXPECT_EQ(OB_SUCCESS, ret);
ret = backup_dest_.set(test_dir_uri_);
EXPECT_EQ(OB_SUCCESS, ret);
task_id_ = 1;
backup_set_desc_.backup_set_id_ = 1;
backup_set_desc_.backup_type_.type_ = ObBackupType::FULL_BACKUP;
backup_data_type_.set_user_data_backup();
incarnation_ = 1;
tenant_id_ = 1;
dest_id_ = 1;
ls_id_ = ObLSID(1001);
turn_id_ = 1;
retry_id_ = 0;
file_id_ = 0;
}
void TestBackupPath::TearDown()
{
}
// TODO(yangyi.yyy): test backup tenant path
TEST_F(TestBackupPath, get_ls_macro_block_index_backup_path)
{
int ret = OB_SUCCESS;
share::ObBackupPath backup_path1;
ret = ObBackupPathUtilV_4_3_2::get_ls_macro_block_index_backup_path(
backup_dest_, backup_set_desc_, ls_id_, backup_data_type_, turn_id_, retry_id_, backup_path1);
EXPECT_EQ(OB_SUCCESS, ret);
const char *str = "file:///obbackup/backup_set_1_full/logstream_1001/user_data_turn_1_retry_0/macro_block_index.obbak";
int cmp_result = std::strcmp(backup_path1.get_ptr(), str);
EXPECT_EQ(cmp_result, 0);
}
TEST_F(TestBackupPath, get_tenant_macro_block_index_backup_path)
{
int ret = OB_SUCCESS;
share::ObBackupPath backup_path1;
backup_data_type_.set_major_data_backup();
ret = ObBackupPathUtilV_4_3_2::get_tenant_macro_block_index_backup_path(
backup_dest_, backup_set_desc_, backup_data_type_, turn_id_, retry_id_, backup_path1);
EXPECT_EQ(OB_SUCCESS, ret);
const char *str = "file:///obbackup/backup_set_1_full/infos/major_data_info_turn_1/tenant_major_data_macro_block_index.0.obbak";
int cmp_result = std::strcmp(backup_path1.get_ptr(), str);
EXPECT_EQ(cmp_result, 0);
}
TEST_F(TestBackupPath, get_intermediate_layer_index_backup_path)
{
int ret = OB_SUCCESS;
share::ObBackupPath backup_path1;
ObBackupIntermediateTreeType index_tree_type = ObBackupIntermediateTreeType::BACKUP_INDEX_TREE;
ret = ObBackupPathUtilV_4_3_2::get_intermediate_layer_index_backup_path(
backup_dest_, backup_set_desc_, ls_id_, backup_data_type_, turn_id_, retry_id_, file_id_, index_tree_type, backup_path1);
EXPECT_EQ(OB_SUCCESS, ret);
const char *str1 = "file:///obbackup/backup_set_1_full/logstream_1001/user_data_turn_1_retry_0/index_tree.0.obbak";
int cmp_result = std::strcmp(backup_path1.get_ptr(), str1);
EXPECT_EQ(cmp_result, 0);
share::ObBackupPath backup_path2;
ObBackupIntermediateTreeType meta_tree_type = ObBackupIntermediateTreeType::BACKUP_META_TREE;
ret = ObBackupPathUtilV_4_3_2::get_intermediate_layer_index_backup_path(
backup_dest_, backup_set_desc_, ls_id_, backup_data_type_, turn_id_, retry_id_, file_id_, meta_tree_type, backup_path2);
EXPECT_EQ(OB_SUCCESS, ret);
const char *str2 = "file:///obbackup/backup_set_1_full/logstream_1001/user_data_turn_1_retry_0/meta_tree.0.obbak";
cmp_result = std::strcmp(backup_path2.get_ptr(), str2);
EXPECT_EQ(cmp_result, 0);
}
}
}
int main(int argc, char **argv)
{
system("rm -f test_backup_path.log*");
ObLogger &logger = ObLogger::get_logger();
logger.set_file_name("test_backup_path.log", true);
logger.set_log_level("info");
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

39
unittest/storage/backup/test_backup_data_struct.cpp
View File

@ -26,27 +26,6 @@ using namespace oceanbase::backup;
namespace oceanbase {
namespace backup {
static int make_random_mapping_meta(ObBackupMacroBlockIDMappingsMeta &mapping_meta)
{
int ret = OB_SUCCESS;
const int64_t sstable_count = random(1, MAX_SSTABLE_CNT_IN_STORAGE);
mapping_meta.version_ = ObBackupMacroBlockIDMappingsMeta::MAPPING_META_VERSION_V1;
ret = mapping_meta.prepare_id_mappings(sstable_count);
EXPECT_EQ(OB_SUCCESS, ret);
for (int64_t i = 0; OB_SUCC(ret) && i < sstable_count; ++i) {
ObBackupMacroBlockIDMapping &mapping = *mapping_meta.id_map_list_[i];
make_random_table_key(mapping.table_key_);
const int64_t pair_count = random(1, 10000);
for (int64_t j = 0; OB_SUCC(ret) && j < pair_count; ++j) {
ObBackupMacroBlockIDPair pair;
make_random_pair(pair);
ret = mapping.id_pair_list_.push_back(pair);
EXPECT_EQ(OB_SUCCESS, ret);
}
}
return ret;
}
static bool meta_is_equal(const ObBackupMacroBlockIDMappingsMeta &lhs, const ObBackupMacroBlockIDMappingsMeta &rhs)
{
bool bret = true;
@ -64,8 +43,8 @@ static bool meta_is_equal(const ObBackupMacroBlockIDMappingsMeta &lhs, const ObB
break;
} else {
for (int64_t j = 0; j < lhs_map.id_pair_list_.count(); ++j) {
const ObBackupMacroBlockIDPair &lhs_pair = lhs_map.id_pair_list_.at(j);
const ObBackupMacroBlockIDPair &rhs_pair = rhs_map.id_pair_list_.at(j);
const ObCompatBackupMacroBlockIDPair &lhs_pair = lhs_map.id_pair_list_.at(j);
const ObCompatBackupMacroBlockIDPair &rhs_pair = rhs_map.id_pair_list_.at(j);
if (lhs_pair.logic_id_ != rhs_pair.logic_id_ || lhs_pair.physical_id_ != rhs_pair.physical_id_) {
bret = false;
break;
@ -111,20 +90,6 @@ TEST(TestBackupDataStruct, BackupPhysicalID)
ASSERT_EQ(macro_index, actual_macro_index);
}
TEST(TestBackupDataStruct, BackupMacroBlockIDMappingsMeta)
{
ObBackupMacroBlockIDMappingsMeta write_meta;
ObBackupMacroBlockIDMappingsMeta read_meta;
make_random_mapping_meta(write_meta);
const int64_t buf_len = write_meta.get_serialize_size();
char *buf = static_cast<char *>(malloc(buf_len));
int64_t pos = 0;
write_meta.serialize(buf, buf_len, pos);
pos = 0;
read_meta.deserialize(buf, buf_len, pos);
ASSERT_TRUE(meta_is_equal(write_meta, read_meta));
}
TEST(TestBackupDataStruct, BackupMetaKeyCompare)
{
ObBackupMetaKey lhs_key;

113
unittest/storage/backup/test_backup_device_macro_block_id.cpp Normal file
View File

@ -0,0 +1,113 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX STORAGE
#include <gtest/gtest.h>
#define private public
#define protected public
#include "storage/backup/ob_backup_data_struct.h"
#include "storage/blocksstable/ob_macro_block_id.h"
#include "common/storage/ob_io_device.h"
using namespace oceanbase;
using namespace oceanbase::common;
using namespace oceanbase::blocksstable;
using namespace oceanbase::storage;
namespace oceanbase
{
namespace backup
{
TEST(ObBackupDeviceMacroBlockId, basic) {
ObBackupDeviceMacroBlockId id;
// Test default constructor
id = ObBackupDeviceMacroBlockId();
EXPECT_FALSE(id.is_valid());
// Test set() and getters
int64_t backup_set_id = 1;
int64_t ls_id = 2;
int64_t data_type = 3;
int64_t turn_id = 4;
int64_t retry_id = 5;
int64_t file_id = 6;
int64_t offset = 4096;
int64_t length = 4096;
EXPECT_EQ(OB_SUCCESS,
id.set(backup_set_id, ls_id, data_type, turn_id, retry_id, file_id, offset, length, ObBackupDeviceMacroBlockId::INDEX_TREE_BLOCK));
EXPECT_EQ(backup_set_id, id.get_backupset_id());
EXPECT_EQ(data_type, id.get_data_type());
EXPECT_EQ(turn_id, id.get_turn_id());
EXPECT_EQ(retry_id, id.get_retry_id());
EXPECT_EQ(file_id, id.get_file_id());
EXPECT_EQ(offset, id.get_offset());
EXPECT_EQ(length, id.get_length());
EXPECT_EQ(static_cast<uint64_t>(blocksstable::ObMacroBlockIdMode::ID_MODE_BACKUP), id.get_id_mode());
EXPECT_TRUE(id.is_valid());
EXPECT_TRUE(ObBackupDeviceMacroBlockId::is_backup_block_file(id.first_id()));
// Test reset()
id.reset();
EXPECT_FALSE(id.is_valid());
EXPECT_FALSE(ObBackupDeviceMacroBlockId::is_backup_block_file(id.first_id()));
blocksstable::MacroBlockId macro_block_id(id.first_id(), id.second_id(), id.third_id());
EXPECT_TRUE(macro_block_id.is_valid());
// fd={first_id:-1, second_id:-1, third_id:-1, fourth_id:0, fifth_id:1, device_handle:null}
// max_file_size=4294967296, io_fd={first_id:-1, second_id:-1, third_id:-1, fourth_id:1369094286720630784, fifth_id:1, device_handle:null})
// info={tenant_id_:1002, fd_:{first_id:0, second_id:-1, third_id:0, fourth_id:-1, fifth_id:-1, device_handle:null},
// ObIOFd io_fd(NULL, 0, -1, 0, -1, -1);
// EXPECT_TRUE(io_fd.is_valid());
// [2023-08-14 15:55:33.678613] INFO [STORAGE] TestBody (test_backup_device_macro_block_id.cpp:84) [41402][][T0][Y0-0000000000000000-0-0] [lt=0] YYY DEBUG(macro_id={backup_set_id:2, dir_id:1, turn_id:1, retry_id:0, file_id:0, offset:1, length:512})
// first_id:69206017, second_id:1001, third_id:2233785415192543234
// {first_id:2097153, second_id:1001, third_id:2233785415192543234
ObBackupDeviceMacroBlockId macro_id;
macro_id.first_id_ = 67133441;
macro_id.second_id_ = 1;
macro_id.third_id_ = 2161727821171392513;
ObArray<int64_t> array;
array.prepare_allocate(1);
array.at(0) = 1;
ObBackupDeviceMacroBlockId macro_id_1;
macro_id_1.first_id_ = 4557655548891137;
macro_id_1.second_id_ = 1003;
macro_id_1.third_id_ = 68769808390;
const int64_t offset_1 = macro_id_1.offset_ * 4096;
EXPECT_EQ(offset_1, -995655680);
const int64_t offset_2 = macro_id_1.offset_ * DIO_READ_ALIGN_SIZE;
EXPECT_EQ(offset_2, 3299311616);
}
}
}
int main(int argc, char **argv)
{
system("rm -f test_backup_device_macro_block_id.log*");
ObLogger &logger = ObLogger::get_logger();
logger.set_file_name("test_backup_device_macro_block_id.log", true);
logger.set_log_level("info");
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

138
unittest/storage/backup/test_backup_device_wrapper.cpp Normal file
View File

@ -0,0 +1,138 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX STORAGE
#include <gtest/gtest.h>
#include "storage/backup/ob_backup_device_wrapper.h"
#include "share/backup/ob_backup_io_adapter.h"
#include "lib/restore/ob_object_device.h"
#include "common/storage/ob_device_common.h"
#include "share/ob_ls_id.h"
#define private public
#define protected public
using namespace oceanbase;
using namespace oceanbase::common;
using namespace oceanbase::share;
using namespace oceanbase::backup;
#define OK(ass) ASSERT_EQ(OB_SUCCESS, (ass))
namespace oceanbase
{
namespace backup
{
TEST(TestBackupDeviceWrapper, test_alloc_block) {
char test_dir[OB_MAX_URI_LENGTH];
char test_dir_uri[OB_MAX_URI_LENGTH];
char uri[OB_MAX_URI_LENGTH];
OK(databuff_printf(test_dir, sizeof(test_dir), "%s/test_backup_device_wrapper_dir", get_current_dir_name()));
OK(databuff_printf(test_dir_uri, sizeof(test_dir_uri), "file://%s", test_dir));
OK(databuff_printf(uri, sizeof(uri), "file://%s/test_file", test_dir));
ObBackupStorageInfo storage_info;
storage_info.device_type_ = ObStorageType::OB_STORAGE_FILE;
const int64_t backup_set_id = 1;
const share::ObLSID ls_id(1001);
share::ObBackupDataType backup_data_type;
backup_data_type.set_user_data_backup();
const int64_t turn_id = 1;
const int64_t retry_id = 0;
const int64_t file_id = 0;
const ObBackupDeviceMacroBlockId::BlockType block_type = ObBackupDeviceMacroBlockId::INDEX_TREE_BLOCK;
static const int64_t opt_cnt = BACKUP_WRAPPER_DEVICE_OPT_NUM;
ObIODOpts io_d_opts_write;
ObIODOpt write_opts[opt_cnt];
io_d_opts_write.opts_ = write_opts;
io_d_opts_write.opt_cnt_ = opt_cnt;
io_d_opts_write.opts_[0].set("storage_info", "");
OK(ObBackupWrapperIODevice::setup_io_opts_for_backup_device(
backup_set_id, ls_id, backup_data_type, turn_id, retry_id, file_id, block_type, OB_STORAGE_ACCESS_RANDOMWRITER, &io_d_opts_write));
ObBackupWrapperIODevice wrapper_io_write;
ObBackupIoAdapter util;
OK(util.mkdir(test_dir_uri, &storage_info));
ObIOFd write_io_fd;
OK(wrapper_io_write.open(uri, -1, 0, write_io_fd, &io_d_opts_write));
// Allocate a block
ObIOFd block_id;
OK(wrapper_io_write.alloc_block(&io_d_opts_write, block_id));
write_io_fd.first_id_ = block_id.first_id_;
write_io_fd.second_id_ = block_id.second_id_;
write_io_fd.third_id_ = block_id.third_id_;
const int64_t size = 100;
char write_buf[size];
memset(write_buf, 1, sizeof(write_buf));
int64_t write_size = 0;
OK(wrapper_io_write.pwrite(write_io_fd, 0, size, write_buf, write_size));
EXPECT_EQ(write_size, size);
ObIODOpts io_d_opts_read;
ObIODOpt read_opts[opt_cnt];
io_d_opts_read.opts_ = read_opts;
io_d_opts_read.opt_cnt_ = opt_cnt;
io_d_opts_read.opts_[0].set("storage_info", "");
OK(ObBackupWrapperIODevice::setup_io_opts_for_backup_device(
backup_set_id, ls_id, backup_data_type, turn_id, retry_id, file_id, block_type, OB_STORAGE_ACCESS_READER, &io_d_opts_read));
ObBackupWrapperIODevice wrapper_io_read;
ObIOFd read_io_fd;
OK(wrapper_io_read.open(uri, -1, 0, read_io_fd, &io_d_opts_read));
read_io_fd.first_id_ = block_id.first_id_;
read_io_fd.second_id_ = block_id.second_id_;
read_io_fd.third_id_ = block_id.third_id_;
char read_buf[size];
int64_t read_size = 0;
OK(wrapper_io_read.pread(read_io_fd, 0, size, read_buf, read_size, NULL));
EXPECT_EQ(write_size, read_size);
// 比较read_buf和write_buf的内容
bool is_same = true;
for (int64_t i = 0; i < size; ++i) {
if (read_buf[i] != write_buf[i]) {
is_same = false;
break;
}
}
EXPECT_TRUE(is_same);
}
}
}
int main(int argc, char **argv)
{
system("rm -f test_backup_device_wrapper.log*");
ObLogger &logger = ObLogger::get_logger();
logger.set_file_name("test_backup_device_wrapper.log", true);
logger.set_log_level("info");
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

212
unittest/storage/backup/test_backup_include.h
View File

@ -24,164 +24,6 @@ namespace backup
int64_t max_tablet_id = 0;
/* ObFakeBackupMacroBlockIndexIterator */
class ObFakeBackupMacroBlockIndexIterator : public ObIMacroBlockIndexIterator {
public:
ObFakeBackupMacroBlockIndexIterator();
virtual ~ObFakeBackupMacroBlockIndexIterator();
int init(const int64_t task_id, const share::ObBackupDest &backup_dest, const uint64_t tenant_id,
const share::ObBackupSetDesc &backup_set_desc, const share::ObLSID &ls_id,
const share::ObBackupDataType &backup_data_type, const int64_t turn_id, const int64_t retry_id,
const int64_t file_count = 80, const int64_t item_count = 1024);
virtual int next() override;
virtual bool is_iter_end() const override;
virtual int get_cur_index(ObBackupMacroRangeIndex &index) override;
virtual ObBackupIndexIteratorType get_type() const override
{
return BACKUP_MACRO_BLOCK_INDEX_ITERATOR;
}
private:
int generate_random_macro_range_index_(
const int64_t backup_set_id, const share::ObLSID &ls_id, const int64_t turn_id, const int64_t retry_id);
void make_random_macro_range_index_(const int64_t tablet_id, const int64_t backup_set_id, const share::ObLSID &ls_id,
const int64_t turn_id, const int64_t retry_id, const int64_t file_id, ObBackupMacroRangeIndex &range_index);
private:
int64_t cur_idx_;
int64_t cur_tablet_id_;
int64_t random_count_;
int64_t file_count_;
int64_t per_file_item_count_;
common::ObArray<ObBackupMacroRangeIndex> cur_index_list_;
DISALLOW_COPY_AND_ASSIGN(ObFakeBackupMacroBlockIndexIterator);
};
ObFakeBackupMacroBlockIndexIterator::ObFakeBackupMacroBlockIndexIterator()
: cur_idx_(0),
cur_tablet_id_(0),
random_count_(0),
file_count_(0),
per_file_item_count_(0),
cur_index_list_()
{}
ObFakeBackupMacroBlockIndexIterator::~ObFakeBackupMacroBlockIndexIterator()
{}
int ObFakeBackupMacroBlockIndexIterator::init(const int64_t task_id, const share::ObBackupDest &backup_dest,
const uint64_t tenant_id, const share::ObBackupSetDesc &backup_set_desc, const share::ObLSID &ls_id,
const share::ObBackupDataType &backup_data_type, const int64_t turn_id, const int64_t retry_id,
const int64_t file_count, const int64_t per_file_item_count)
{
int ret = OB_SUCCESS;
max_tablet_id = 0;
task_id_ = task_id;
backup_dest_.deep_copy(backup_dest);
tenant_id_ = tenant_id;
backup_set_desc_ = backup_set_desc;
ls_id_ = ls_id;
backup_data_type_ = backup_data_type;
turn_id_ = turn_id;
retry_id_ = retry_id;
cur_idx_ = 0;
cur_file_id_ = -1;
file_count_ = file_count;
per_file_item_count_ = per_file_item_count;
generate_random_macro_range_index_(backup_set_desc.backup_set_id_, ls_id, turn_id, retry_id);
is_inited_ = true;
return ret;
}
int ObFakeBackupMacroBlockIndexIterator::next()
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("iterator do not init", K(ret));
} else if (cur_idx_ >= random_count_) {
ret = OB_ITER_END;
LOG_WARN("iter end", K(ret), K_(cur_idx), K_(cur_index_list));
} else {
cur_idx_++;
}
return ret;
}
bool ObFakeBackupMacroBlockIndexIterator::is_iter_end() const
{
bool bret = false;
bret = cur_idx_ >= cur_index_list_.count() || -1 == cur_idx_;
return bret;
}
int ObFakeBackupMacroBlockIndexIterator::get_cur_index(ObBackupMacroRangeIndex &range_index)
{
int ret = OB_SUCCESS;
range_index.reset();
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("iterator do not init", K(ret));
} else if (cur_idx_ < 0) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("current index is not valid",
K(ret),
K_(tenant_id),
K_(ls_id),
K_(backup_data_type),
K(cur_idx_),
K_(cur_file_id),
K(file_id_list_));
} else if (cur_idx_ >= cur_index_list_.count()) {
ret = OB_ITER_END;
LOG_WARN("iterator is end", K(ret), KPC(this));
} else {
range_index = cur_index_list_.at(cur_idx_);
}
return ret;
}
int ObFakeBackupMacroBlockIndexIterator::generate_random_macro_range_index_(
const int64_t backup_set_id, const share::ObLSID &ls_id, const int64_t turn_id, const int64_t retry_id)
{
int ret = OB_SUCCESS;
ObBackupMacroRangeIndex range_index;
const int64_t file_count = file_count_;
for (int64_t i = 0; OB_SUCC(ret) && i < file_count; ++i) {
const int64_t item_count = per_file_item_count_;
const int64_t file_id = i;
for (int64_t j = 0; OB_SUCC(ret) && j < item_count; ++j) {
cur_tablet_id_++;
range_index.reset();
make_random_macro_range_index_(cur_tablet_id_, backup_set_id, ls_id, turn_id, retry_id, file_id, range_index);
ret = cur_index_list_.push_back(range_index);
EXPECT_EQ(OB_SUCCESS, ret);
random_count_++;
max_tablet_id = std::max(max_tablet_id, cur_tablet_id_);
}
}
return ret;
}
void ObFakeBackupMacroBlockIndexIterator::make_random_macro_range_index_(const int64_t tablet_id,
const int64_t backup_set_id, const share::ObLSID &ls_id, const int64_t turn_id, const int64_t retry_id,
const int64_t file_id, ObBackupMacroRangeIndex &range_index)
{
range_index.start_key_ = blocksstable::ObLogicMacroBlockId(1, 1, tablet_id);
range_index.end_key_ = blocksstable::ObLogicMacroBlockId(1, 1, tablet_id);
range_index.backup_set_id_ = backup_set_id;
range_index.ls_id_ = ls_id;
range_index.turn_id_ = turn_id;
range_index.retry_id_ = retry_id;
range_index.file_id_ = file_id;
make_random_offset(range_index.offset_);
make_random_offset(range_index.length_);
}
/* ObFakeBackupMacroIndexMerger */
class ObFakeBackupMacroIndexMerger : public ObBackupMacroBlockIndexMerger {
@ -207,59 +49,5 @@ private:
DISALLOW_COPY_AND_ASSIGN(ObFakeBackupMacroIndexMerger);
};
ObFakeBackupMacroIndexMerger::ObFakeBackupMacroIndexMerger()
{}
ObFakeBackupMacroIndexMerger::~ObFakeBackupMacroIndexMerger()
{}
int ObFakeBackupMacroIndexMerger::get_all_retries_(const int64_t task_id, const uint64_t tenant_id,
const share::ObBackupDataType &backup_data_type, const share::ObLSID &ls_id, common::ObISQLClient &sql_proxy,
common::ObIArray<ObBackupRetryDesc> &retry_list)
{
int ret = OB_SUCCESS;
UNUSEDx(task_id, tenant_id, backup_data_type, sql_proxy);
retry_list.reset();
ObBackupRetryDesc desc;
desc.ls_id_ = ls_id;
desc.turn_id_ = 1;
desc.retry_id_ = 0;
desc.last_file_id_ = 10;
if (OB_FAIL(retry_list.push_back(desc))) {
LOG_WARN("failed to push back", K(ret), K(desc));
} else {
LOG_INFO("fake get all retries", K(retry_list));
}
return ret;
}
int ObFakeBackupMacroIndexMerger::alloc_merge_iter_(const bool tenant_level, const ObBackupIndexMergeParam &merge_param,
const ObBackupRetryDesc &retry_desc, ObIMacroBlockIndexIterator *&iter)
{
int ret = OB_SUCCESS;
ObFakeBackupMacroBlockIndexIterator *tmp_iter = NULL;
if (!retry_desc.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("get invalid args", K(ret));
} else if (OB_ISNULL(tmp_iter = OB_NEW(ObFakeBackupMacroBlockIndexIterator, ObModIds::BACKUP))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to alloc iterator", K(ret));
} else if (OB_FAIL(tmp_iter->init(merge_param.task_id_,
merge_param.backup_dest_,
merge_param.tenant_id_,
merge_param.backup_set_desc_,
retry_desc.ls_id_,
merge_param.backup_data_type_,
retry_desc.turn_id_,
retry_desc.retry_id_,
file_count_,
per_file_item_count_))) {
LOG_WARN("failed to init meta index iterator", K(ret), K(merge_param));
} else {
iter = tmp_iter;
}
return ret;
}
}
}

89
unittest/storage/backup/test_backup_index_compressor.cpp Normal file
View File

@ -0,0 +1,89 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX STORAGE
#include <gtest/gtest.h>
#define private public
#define protected public
#include "storage/backup/ob_backup_index_compressor.h"
#include "lib/allocator/page_arena.h"
#include <cstdlib>
#include <ctime>
using namespace oceanbase;
using namespace oceanbase::common;
using namespace oceanbase::share;
using namespace oceanbase::backup;
using namespace oceanbase::blocksstable;
namespace oceanbase
{
namespace backup
{
TEST(ObBackupIndexBlockCompressorTest, Compress)
{
int ret = OB_SUCCESS;
ObBackupIndexBlockCompressor compressor;
const int64_t block_size = 16 * 1024;
const ObCompressorType compressor_type = ObCompressorType::ZSTD_COMPRESSOR;
ret = compressor.init(block_size, compressor_type);
EXPECT_EQ(OB_SUCCESS, ret);
int buffer_size = 16 * 1024;
char buffer[buffer_size];
for (int i = 0; i < buffer_size; ++i) {
buffer[i] = std::rand() % 8;
}
const char* in = buffer;
int64_t in_size = buffer_size;
const char* out = NULL;
int64_t out_size = 0;
ret = compressor.compress(in, in_size, out, out_size);
EXPECT_EQ(OB_SUCCESS, ret);
ObBackupIndexBlockCompressor decompressor;
ret = decompressor.init(block_size, compressor_type);
EXPECT_EQ(OB_SUCCESS, ret);
const char *decomp_out = NULL;
int64_t decomp_size = 0;
ret = decompressor.decompress(out, out_size, block_size, decomp_out, decomp_size);
EXPECT_EQ(OB_SUCCESS, ret);
EXPECT_EQ(in_size, decomp_size);
EXPECT_LE(out_size, in_size);
LOG_INFO("compress info", K(in_size), K(out_size), K(decomp_size));
}
}
}
int main(int argc, char **argv)
{
system("rm -f test_backup_index_compressor.log*");
ObLogger &logger = ObLogger::get_logger();
logger.set_file_name("test_backup_index_compressor.log", true);
logger.set_log_level("info");
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

37
unittest/storage/backup/test_backup_index_merger.cpp
View File

@ -61,8 +61,7 @@ public:
virtual ~ObFakeBackupMetaIndexIterator();
int init(const int64_t task_id, const share::ObBackupDest &backup_dest, const uint64_t tenant_id,
const share::ObBackupSetDesc &backup_set_desc, const share::ObLSID &ls_id,
const share::ObBackupDataType &backup_data_type, const int64_t turn_id, const int64_t retry_id,
const bool is_sec_meta);
const share::ObBackupDataType &backup_data_type, const int64_t turn_id, const int64_t retry_id);
virtual int next() override;
virtual bool is_iter_end() const override;
@ -87,8 +86,7 @@ ObFakeBackupMetaIndexIterator::~ObFakeBackupMetaIndexIterator()
int ObFakeBackupMetaIndexIterator::init(const int64_t task_id, const share::ObBackupDest &backup_dest,
const uint64_t tenant_id, const share::ObBackupSetDesc &backup_set_desc, const share::ObLSID &ls_id,
const share::ObBackupDataType &backup_data_type, const int64_t turn_id, const int64_t retry_id,
const bool is_sec_meta)
const share::ObBackupDataType &backup_data_type, const int64_t turn_id, const int64_t retry_id)
{
int ret = OB_SUCCESS;
max_tablet_id = 0;
@ -102,7 +100,6 @@ int ObFakeBackupMetaIndexIterator::init(const int64_t task_id, const share::ObBa
retry_id_ = retry_id;
cur_idx_ = 0;
cur_file_id_ = -1;
is_sec_meta_ = is_sec_meta;
generate_random_meta_index_(backup_set_desc.backup_set_id_, ls_id, turn_id, retry_id);
is_inited_ = true;
return ret;
@ -184,7 +181,7 @@ private:
const share::ObBackupDataType &backup_data_type, const share::ObLSID &ls_id, common::ObISQLClient &sql_proxy,
common::ObIArray<ObBackupRetryDesc> &retry_list) override;
virtual int alloc_merge_iter_(const ObBackupIndexMergeParam &merge_param, const ObBackupRetryDesc &retry_desc,
const bool is_sec_meta, ObBackupMetaIndexIterator *&iter) override;
ObBackupMetaIndexIterator *&iter) override;
private:
DISALLOW_COPY_AND_ASSIGN(ObFakeBackupMetaIndexMerger);
@ -217,7 +214,7 @@ int ObFakeBackupMetaIndexMerger::get_all_retries_(const int64_t task_id, const u
}
int ObFakeBackupMetaIndexMerger::alloc_merge_iter_(const ObBackupIndexMergeParam &merge_param,
const ObBackupRetryDesc &retry_desc, const bool is_sec_meta, ObBackupMetaIndexIterator *&iter)
const ObBackupRetryDesc &retry_desc, ObBackupMetaIndexIterator *&iter)
{
int ret = OB_SUCCESS;
ObFakeBackupMetaIndexIterator *tmp_iter = NULL;
@ -234,8 +231,7 @@ int ObFakeBackupMetaIndexMerger::alloc_merge_iter_(const ObBackupIndexMergeParam
retry_desc.ls_id_,
merge_param.backup_data_type_,
retry_desc.turn_id_,
retry_desc.retry_id_,
is_sec_meta))) {
retry_desc.retry_id_))) {
LOG_WARN("failed to init meta index iterator", K(ret), K(merge_param));
} else {
iter = tmp_iter;
@ -361,7 +357,7 @@ void TestBackupIndexMerger::inner_init_()
job_desc_.task_id_ = 1;
backup_set_desc_.backup_set_id_ = 1;
backup_set_desc_.backup_type_.type_ = ObBackupType::FULL_BACKUP;
backup_data_type_.set_major_data_backup();
backup_data_type_.set_user_data_backup();
incarnation_ = 1;
tenant_id_ = 1;
dest_id_ = 1;
@ -384,8 +380,7 @@ void TestBackupIndexMerger::fake_init_meta_index_merger_(ObFakeBackupMetaIndexMe
common::ObMySQLProxy sql_proxy;
ObBackupIndexMergeParam merge_param;
build_backup_index_merge_param_(merge_param);
const bool is_sec_meta = false;
ret = merger.init(merge_param, is_sec_meta, sql_proxy, throttle_);
ret = merger.init(merge_param, sql_proxy, throttle_);
EXPECT_EQ(OB_SUCCESS, ret);
}
@ -436,6 +431,7 @@ void TestBackupIndexMerger::build_backup_index_merge_param_(ObBackupIndexMergePa
merge_param.ls_id_ = ls_id_;
merge_param.turn_id_ = turn_id_;
merge_param.retry_id_ = retry_id_;
merge_param.compressor_type_ = ObCompressorType::ZSTD_1_3_8_COMPRESSOR;
}
void TestBackupIndexMerger::build_backup_index_store_param_(ObBackupIndexStoreParam &store_param)
@ -498,21 +494,6 @@ TEST_F(TestBackupIndexMerger, test_backup_meta_index_merger)
iterate_meta_index_store_(start_id, end_id, index_store);
}
TEST_F(TestBackupIndexMerger, test_backup_macro_index_merger)
{
int ret = OB_SUCCESS;
clean_env_();
ObFakeBackupMacroIndexMerger merger;
fake_init_macro_index_merger_(merger);
ret = merger.merge_index();
EXPECT_EQ(OB_SUCCESS, ret);
ObMockBackupMacroBlockIndexStore index_store;
init_backup_macro_index_store_(index_store);
const int64_t start_id = 1;
const int64_t end_id = max_tablet_id;
iterate_macro_index_store_(start_id, end_id, index_store);
}
TEST_F(TestBackupIndexMerger, test_backup_meta_index_same_meta_index_different_retry)
{
ObBackupMetaIndex meta_index1;
@ -521,7 +502,6 @@ TEST_F(TestBackupIndexMerger, test_backup_meta_index_same_meta_index_different_r
meta_index1.turn_id_ = 1;
meta_index1.retry_id_ = 0;
ObBackupMetaIndexIterator iter1;
iter1.is_sec_meta_ = false;
iter1.cur_idx_ = 0;
iter1.cur_file_id_ = 0;
iter1.is_inited_ = true;
@ -534,7 +514,6 @@ TEST_F(TestBackupIndexMerger, test_backup_meta_index_same_meta_index_different_r
meta_index2.turn_id_ = 1;
meta_index2.retry_id_ = 1;
ObBackupMetaIndexIterator iter2;
iter2.is_sec_meta_ = false;
iter2.cur_idx_ = 0;
iter2.cur_file_id_ = 0;
iter2.is_inited_ = true;

87
unittest/storage/backup/test_backup_iterator.cpp
View File

@ -152,7 +152,7 @@ void TestBackupIndexIterator::inner_init_()
job_desc_.task_id_ = 1;
backup_set_desc_.backup_set_id_ = 1;
backup_set_desc_.backup_type_.type_ = ObBackupType::FULL_BACKUP;
backup_data_type_.set_major_data_backup();
backup_data_type_.set_user_data_backup();
incarnation_ = 1;
tenant_id_ = 1;
dest_id_ = 1;
@ -255,91 +255,6 @@ TEST_F(TestBackupIndexIterator, test_extract_backup_file_id)
ASSERT_EQ(10, tmp_file_id);
}
TEST_F(TestBackupIndexIterator, test_backup_macro_range_index_iterator_921_KB)
{
int ret = OB_SUCCESS;
clean_env_();
ObFakeBackupMacroIndexMerger merger;
const int64_t file_count = 20;
const int64_t per_file_item_count = 1024;
fake_init_macro_index_merger_(file_count, per_file_item_count, merger);
ret = merger.merge_index();
ASSERT_EQ(OB_SUCCESS, ret);
const int64_t start_id = 1;
const int64_t end_id = file_count * per_file_item_count;
ObBackupMacroRangeIndexIterator range_index_iterator;
init_macro_range_index_iterator(range_index_iterator);
iterate_macro_range_index_iterator(start_id, end_id, range_index_iterator);
}
TEST_F(TestBackupIndexIterator, test_backup_macro_range_index_iterator_1_8_MB)
{
int ret = OB_SUCCESS;
clean_env_();
ObFakeBackupMacroIndexMerger merger;
const int64_t file_count = 40;
const int64_t per_file_item_count = 1024;
fake_init_macro_index_merger_(file_count, per_file_item_count, merger);
ret = merger.merge_index();
ASSERT_EQ(OB_SUCCESS, ret);
const int64_t start_id = 1;
const int64_t end_id = file_count * per_file_item_count;
ObBackupMacroRangeIndexIterator range_index_iterator;
init_macro_range_index_iterator(range_index_iterator);
iterate_macro_range_index_iterator(start_id, end_id, range_index_iterator);
}
TEST_F(TestBackupIndexIterator, test_backup_macro_range_index_iterator_2_8_MB)
{
int ret = OB_SUCCESS;
clean_env_();
ObFakeBackupMacroIndexMerger merger;
const int64_t file_count = 60;
const int64_t per_file_item_count = 1024;
fake_init_macro_index_merger_(file_count, per_file_item_count, merger);
ret = merger.merge_index();
ASSERT_EQ(OB_SUCCESS, ret);
const int64_t start_id = 1;
const int64_t end_id = file_count * per_file_item_count;
ObBackupMacroRangeIndexIterator range_index_iterator;
init_macro_range_index_iterator(range_index_iterator);
iterate_macro_range_index_iterator(start_id, end_id, range_index_iterator);
}
TEST_F(TestBackupIndexIterator, test_backup_macro_range_index_iterator_3_7_MB)
{
int ret = OB_SUCCESS;
clean_env_();
ObFakeBackupMacroIndexMerger merger;
const int64_t file_count = 80;
const int64_t per_file_item_count = 1024;
fake_init_macro_index_merger_(file_count, per_file_item_count, merger);
ret = merger.merge_index();
ASSERT_EQ(OB_SUCCESS, ret);
const int64_t start_id = 1;
const int64_t end_id = file_count * per_file_item_count;
ObBackupMacroRangeIndexIterator range_index_iterator;
init_macro_range_index_iterator(range_index_iterator);
iterate_macro_range_index_iterator(start_id, end_id, range_index_iterator);
}
TEST_F(TestBackupIndexIterator, test_backup_macro_range_index_iterator_4_6_MB)
{
int ret = OB_SUCCESS;
clean_env_();
ObFakeBackupMacroIndexMerger merger;
const int64_t file_count = 100;
const int64_t per_file_item_count = 1024;
fake_init_macro_index_merger_(file_count, per_file_item_count, merger);
ret = merger.merge_index();
ASSERT_EQ(OB_SUCCESS, ret);
const int64_t start_id = 1;
const int64_t end_id = file_count * per_file_item_count;
ObBackupMacroRangeIndexIterator range_index_iterator;
init_macro_range_index_iterator(range_index_iterator);
iterate_macro_range_index_iterator(start_id, end_id, range_index_iterator);
}
} // namespace backup
} // namespace oceanbase

354
unittest/storage/backup/test_backup_macro_block_index_merger.cpp Normal file
View File

@ -0,0 +1,354 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX STORAGE
#include <gtest/gtest.h>
#define private public
#define protected public
#include "storage/backup/ob_backup_data_struct.h"
#include "storage/backup/ob_backup_index_merger.h"
#include "storage/backup/ob_backup_index_store.h"
#include "storage/backup/ob_backup_factory.h"
#include "storage/blocksstable/ob_data_file_prepare.h"
namespace oceanbase
{
namespace backup
{
static ObSimpleMemLimitGetter getter;
class FakeMacroBlockIndexIterator : public ObIMacroBlockIndexIterator
{
public:
FakeMacroBlockIndexIterator() {}
virtual ~FakeMacroBlockIndexIterator() {}
virtual ObBackupIndexIteratorType get_type() const
{
return BACKUP_ORDERED_MACRO_BLOCK_INDEX_ITERATOR;
}
virtual int next()
{
cur_idx_++;
return OB_SUCCESS;
}
virtual bool is_iter_end() const
{
return cur_idx_ > total_count_;
}
virtual int get_cur_index(ObBackupMacroRangeIndex &index) override
{
return OB_NOT_SUPPORTED;
}
virtual int get_cur_index(ObBackupMacroBlockIndex &index) override
{
index.logic_id_.tablet_id_ = 200001;
index.logic_id_.logic_version_ = cur_idx_;
index.logic_id_.data_seq_ = cur_idx_;
index.backup_set_id_ = 1;
index.ls_id_ = ObLSID(1001);
index.turn_id_ = 1;
index.retry_id_ = 0;
index.file_id_ = cur_idx_ % 2048;
index.offset_ = cur_idx_ * 4096;
index.length_ = (cur_idx_) * (cur_idx_ + 1) * 4096;
index.reusable_ = true;
return OB_SUCCESS;
}
private:
int64_t cur_idx_ = 1;
int64_t total_count_ = 10000;
};
class FakeUnorderedMacroBlockIndexMerger : public ObBackupUnorderdMacroBlockIndexMerger {
public:
virtual int get_all_retries_(const int64_t task_id, const uint64_t tenant_id, const share::ObBackupDataType &backup_data_type,
const share::ObLSID &ls_id, common::ObISQLClient &sql_proxy, common::ObIArray<ObBackupRetryDesc> &retry_list) override
{
int ret = OB_SUCCESS;
UNUSEDx(task_id, tenant_id, backup_data_type, sql_proxy);
retry_list.reset();
ObBackupRetryDesc desc;
desc.ls_id_ = ls_id;
desc.turn_id_ = 1;
desc.retry_id_ = 0;
desc.last_file_id_ = 10;
if (OB_FAIL(retry_list.push_back(desc))) {
LOG_WARN("failed to push back", K(ret), K(desc));
} else {
LOG_INFO("fake get all retries", K(retry_list));
}
return ret;
}
virtual int get_prev_tenant_index_retry_id_(const ObBackupIndexMergeParam &merge_param,
const share::ObBackupSetDesc &prev_backup_set_desc, const int64_t prev_turn_id, int64_t &retry_id) override
{
retry_id = 0;
return OB_SUCCESS;
}
virtual int prepare_macro_block_iterators_(
const ObBackupIndexMergeParam &merge_param, const common::ObIArray<ObBackupRetryDesc> &retry_list,
common::ObISQLClient &sql_proxy, common::ObIArray<ObIMacroBlockIndexIterator *> &iterators) override
{
int ret = OB_SUCCESS;
const int64_t task_id = merge_param.task_id_;
const ObBackupDest &backup_dest = merge_param.backup_dest_;
const ObBackupSetDesc &backup_set_desc = merge_param.backup_set_desc_;
const uint64_t tenant_id = merge_param.tenant_id_;
const ObBackupDataType &backup_data_type = merge_param.backup_data_type_;
const share::ObLSID &ls_id = ObLSID(1001);
const int64_t turn_id = 1;
const int64_t retry_id = 0;
const ObBackupIndexIteratorType type = BACKUP_ORDERED_MACRO_BLOCK_INDEX_ITERATOR;
FakeMacroBlockIndexIterator *tmp_iter = new FakeMacroBlockIndexIterator;
if (OB_FAIL(iterators.push_back(tmp_iter))) {
LOG_WARN("failed to push back", K(ret));
}
return ret;
}
};
class TestBackupMacroIndexMerger : public TestDataFilePrepare {
public:
TestBackupMacroIndexMerger();
virtual ~TestBackupMacroIndexMerger();
virtual void SetUp();
virtual void TearDown();
private:
void inner_init_();
void clean_env_();
void fake_init_macro_index_merger_(FakeUnorderedMacroBlockIndexMerger &merger);
void build_backup_index_merge_param_(ObBackupIndexMergeParam &merge_param);
void build_backup_index_store_param_(ObBackupIndexStoreParam &store_param);
void init_backup_macro_index_store_(ObBackupOrderedMacroBlockIndexStore &macro_index_store);
protected:
ObTenantBase tenant_base_;
ObBackupJobDesc job_desc_;
ObBackupDest backup_dest_;
int64_t task_id_;
int64_t tenant_id_;
int64_t dest_id_;
ObBackupSetDesc backup_set_desc_;
share::ObLSID ls_id_;
ObBackupDataType backup_data_type_;
int64_t turn_id_;
int64_t retry_id_;
common::ObInOutBandwidthThrottle throttle_;
char test_dir_[OB_MAX_URI_LENGTH];
char test_dir_uri_[OB_MAX_URI_LENGTH];
DISALLOW_COPY_AND_ASSIGN(TestBackupMacroIndexMerger);
};
TestBackupMacroIndexMerger::TestBackupMacroIndexMerger()
: TestDataFilePrepare(&getter, "TestBackupMacroIndexMerger", OB_DEFAULT_MACRO_BLOCK_SIZE, 800),
tenant_base_(500),
job_desc_(),
backup_dest_(),
tenant_id_(OB_INVALID_ID),
dest_id_(0),
backup_set_desc_(),
ls_id_(),
backup_data_type_(),
turn_id_(-1),
retry_id_(-1),
test_dir_(""),
test_dir_uri_("")
{}
TestBackupMacroIndexMerger::~TestBackupMacroIndexMerger()
{}
void TestBackupMacroIndexMerger::SetUp()
{
int ret = OB_SUCCESS;
const int64_t bucket_num = 1024;
const int64_t max_cache_size = 1024 * 1024 * 1024;
const int64_t block_size = common::OB_MALLOC_BIG_BLOCK_SIZE;
TestDataFilePrepare::SetUp();
ret = getter.add_tenant(1, 8L * 1024L * 1024L, 2L * 1024L * 1024L * 1024L);
EXPECT_EQ(OB_SUCCESS, ret);
ret = ObKVGlobalCache::get_instance().init(&getter, bucket_num, max_cache_size, block_size);
if (OB_INIT_TWICE == ret) {
ret = OB_SUCCESS;
} else {
EXPECT_EQ(OB_SUCCESS, ret);
}
// set observer memory limit
CHUNK_MGR.set_limit(8L * 1024L * 1024L * 1024L);
ret = ObTmpFileManager::get_instance().init();
if (OB_INIT_TWICE == ret) {
ret = OB_SUCCESS;
} else {
ASSERT_EQ(OB_SUCCESS, ret);
}
static ObTenantBase tenant_ctx(OB_SYS_TENANT_ID);
ObTenantEnv::set_tenant(&tenant_ctx);
ObTenantIOManager *io_service = nullptr;
EXPECT_EQ(OB_SUCCESS, ObTenantIOManager::mtl_new(io_service));
EXPECT_EQ(OB_SUCCESS, ObTenantIOManager::mtl_init(io_service));
EXPECT_EQ(OB_SUCCESS, io_service->start());
tenant_ctx.set(io_service);
ObTenantEnv::set_tenant(&tenant_ctx);
inner_init_();
}
void TestBackupMacroIndexMerger::TearDown()
{
ObTmpFileManager::get_instance().destroy();
ObKVGlobalCache::get_instance().destroy();
}
void TestBackupMacroIndexMerger::clean_env_()
{
system((std::string("rm -rf ") + test_dir_ + std::string("*")).c_str());
}
void TestBackupMacroIndexMerger::inner_init_()
{
int ret = OB_SUCCESS;
ObBackupIoAdapter util;
ret = databuff_printf(test_dir_, sizeof(test_dir_), "%s/test_backup_macro_block_index_merger_dir", get_current_dir_name());
EXPECT_EQ(OB_SUCCESS, ret);
ret = databuff_printf(test_dir_uri_, sizeof(test_dir_uri_), "file://%s", test_dir_);
EXPECT_EQ(OB_SUCCESS, ret);
ret = backup_dest_.set(test_dir_uri_);
EXPECT_EQ(OB_SUCCESS, ret);
EXPECT_EQ(OB_SUCCESS, util.mkdir(test_dir_uri_, backup_dest_.get_storage_info()));
task_id_ = 1;
job_desc_.job_id_ = 1;
job_desc_.task_id_ = 1;
backup_set_desc_.backup_set_id_ = 1;
backup_set_desc_.backup_type_.type_ = ObBackupType::FULL_BACKUP;
backup_data_type_.set_user_data_backup();
tenant_id_ = 1;
dest_id_ = 1;
ls_id_ = ObLSID(1001);
turn_id_ = 1;
retry_id_ = 0;
ret = throttle_.init(10);
EXPECT_EQ(OB_SUCCESS, ret);
ret = OB_BACKUP_INDEX_CACHE.init();
if (OB_INIT_TWICE == ret) {
ret = OB_SUCCESS;
} else {
EXPECT_EQ(OB_SUCCESS, ret);
}
}
void TestBackupMacroIndexMerger::build_backup_index_merge_param_(ObBackupIndexMergeParam &merge_param)
{
merge_param.task_id_ = task_id_;
merge_param.backup_dest_.deep_copy(backup_dest_);
merge_param.tenant_id_ = tenant_id_;
merge_param.dest_id_ = dest_id_;
merge_param.backup_set_desc_ = backup_set_desc_;
merge_param.backup_data_type_ = backup_data_type_;
merge_param.index_level_ = ObBackupIndexLevel::BACKUP_INDEX_LEVEL_LOG_STREAM;
merge_param.ls_id_ = ls_id_;
merge_param.turn_id_ = turn_id_;
merge_param.retry_id_ = retry_id_;
merge_param.compressor_type_ = ObCompressorType::ZSTD_1_3_8_COMPRESSOR;
}
void TestBackupMacroIndexMerger::build_backup_index_store_param_(ObBackupIndexStoreParam &store_param)
{
store_param.index_level_ = ObBackupIndexLevel::BACKUP_INDEX_LEVEL_LOG_STREAM;
store_param.tenant_id_ = tenant_id_;
store_param.backup_set_id_ = backup_set_desc_.backup_set_id_;
store_param.ls_id_ = ls_id_;
store_param.is_tenant_level_ = false;
store_param.backup_data_type_ = backup_data_type_;
store_param.turn_id_ = turn_id_;
store_param.retry_id_ = retry_id_;
}
void TestBackupMacroIndexMerger::fake_init_macro_index_merger_(FakeUnorderedMacroBlockIndexMerger &merger)
{
int ret = OB_SUCCESS;
common::ObMySQLProxy sql_proxy;
ObBackupIndexMergeParam merge_param;
build_backup_index_merge_param_(merge_param);
ret = merger.init(merge_param, sql_proxy, throttle_);
EXPECT_EQ(OB_SUCCESS, ret);
}
void TestBackupMacroIndexMerger::init_backup_macro_index_store_(ObBackupOrderedMacroBlockIndexStore &macro_index_store)
{
int ret = OB_SUCCESS;
ObBackupRestoreMode mode = ObBackupRestoreMode::BACKUP_MODE;
ObBackupIndexStoreParam store_param;
build_backup_index_store_param_(store_param);
ret = macro_index_store.init(mode, store_param, backup_dest_, backup_set_desc_, OB_BACKUP_INDEX_CACHE);
EXPECT_EQ(OB_SUCCESS, ret);
}
TEST_F(TestBackupMacroIndexMerger, test_backup_macro_block_index_merger)
{
int ret = OB_SUCCESS;
clean_env_();
FakeUnorderedMacroBlockIndexMerger merger;
fake_init_macro_index_merger_(merger);
ret = merger.merge_index();
EXPECT_EQ(OB_SUCCESS, ret);
}
TEST_F(TestBackupMacroIndexMerger, test_backup_macro_block_index_store)
{
int ret = OB_SUCCESS;
ObBackupOrderedMacroBlockIndexStore macro_index_store;
init_backup_macro_index_store_(macro_index_store);
const int64_t upper_bound = 9999;
for (int64_t i = 1; i <= upper_bound; ++i) {
ObLogicMacroBlockId logic_id;
logic_id.tablet_id_ = 200001;
logic_id.logic_version_ = i;
logic_id.data_seq_ = i;
ObBackupMacroBlockIndex macro_index;
ret = macro_index_store.get_macro_block_index(logic_id, macro_index);
EXPECT_EQ(OB_SUCCESS, ret);
}
}
TEST_F(TestBackupMacroIndexMerger, test_backup_macro_block_index_iterator)
{
int ret = OB_SUCCESS;
ObBackupOrderedMacroBlockIndexIterator iter;
ret = iter.init(task_id_, backup_dest_, tenant_id_, backup_set_desc_, ls_id_, backup_data_type_, turn_id_, retry_id_);
EXPECT_EQ(OB_SUCCESS, ret);
ObBackupMacroBlockIndex macro_block_index;
while (OB_SUCC(ret) && !iter.is_iter_end()) {
ret = iter.get_cur_index(macro_block_index);
EXPECT_EQ(OB_SUCCESS, ret);
ret = iter.next();
EXPECT_EQ(OB_SUCCESS, ret);
}
}
}
}
int main(int argc, char **argv)
{
system("rm -f test_backup_macro_block_index_merger.log*");
ObLogger &logger = ObLogger::get_logger();
logger.set_file_name("test_backup_macro_block_index_merger.log", true);
logger.set_log_level("info");
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

339
unittest/storage/backup/test_backup_sync_io_mock_async_io.cpp Normal file
View File

@ -0,0 +1,339 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX STORAGE
#include <gtest/gtest.h>
#include "storage/backup/ob_backup_data_struct.h"
#include "storage/blocksstable/ob_block_manager.h"
#include "storage/backup/ob_backup_device_wrapper.h"
#include "lib/allocator/page_arena.h"
#include "share/io/ob_io_define.h"
#include "share/io/ob_io_manager.h"
#include "share/backup/ob_backup_io_adapter.h"
#include "share/ob_local_device.h"
#define private public
#define protected public
using namespace oceanbase;
using namespace oceanbase::common;
using namespace oceanbase::share;
using namespace oceanbase::backup;
using namespace oceanbase::blocksstable;
#define OK(ass) EXPECT_EQ(OB_SUCCESS, (ass))
namespace oceanbase
{
namespace backup
{
static const int64_t backup_set_id = 1;
static const uint64_t tenant_id = OB_SYS_TENANT_ID;
static const int64_t ls_id = 1001;
static const int64_t turn_id = 1;
static const int64_t retry_id = 0;
static const int64_t file_id = 1024;
static const int64_t offset = 4096;
static const int64_t length = 4096;
static ObBackupDataType backup_data_type;
#define TEST_ROOT_DIR "io_test"
#define TEST_DATA_DIR TEST_ROOT_DIR "/data_dir"
#define TEST_SSTABLE_DIR TEST_DATA_DIR "/sstable"
int init_device(const int64_t media_id, ObLocalDevice &device)
{
int ret = OB_SUCCESS;
const int64_t IO_OPT_COUNT = 6;
const int64_t block_size = 1024L * 1024L * 2L; // 2MB
const int64_t data_disk_size = 1024L * 1024L * 1024L; // 1GB
const int64_t data_disk_percentage = 50L;
ObIODOpt io_opts[IO_OPT_COUNT];
io_opts[0].key_ = "data_dir"; io_opts[0].value_.value_str = TEST_DATA_DIR;
io_opts[1].key_ = "sstable_dir"; io_opts[1].value_.value_str = TEST_SSTABLE_DIR;
io_opts[2].key_ = "block_size"; io_opts[2].value_.value_int64 = block_size;
io_opts[3].key_ = "datafile_disk_percentage"; io_opts[3].value_.value_int64 = data_disk_percentage;
io_opts[4].key_ = "datafile_size"; io_opts[4].value_.value_int64 = data_disk_size;
io_opts[5].key_ = "media_id"; io_opts[5].value_.value_int64 = media_id;
ObIODOpts init_opts;
init_opts.opts_ = io_opts;
init_opts.opt_cnt_ = IO_OPT_COUNT;
bool need_format = false;
if (OB_FAIL(device.init(init_opts))) {
LOG_WARN("init device failed", K(ret));
} else {
int64_t reserved_size = 0;
ObIODOpts opts_start;
ObIODOpt opt_start;
opts_start.opts_ = &(opt_start);
opts_start.opt_cnt_ = 1;
opt_start.set("reserved size", reserved_size);
if (OB_FAIL(device.start(opts_start))) {
LOG_WARN("start device failed", K(ret));
}
}
return ret;
}
class TestBackupMockAsyncIO : public ::testing::Test
{
public:
static void SetUpTestCase()
{
// prepare test directory and file
system("mkdir -p " TEST_DATA_DIR);
system("mkdir -p " TEST_SSTABLE_DIR);
// init io device
static oceanbase::share::ObLocalDevice local_device;
OK(init_device(0, local_device));
THE_IO_DEVICE = &local_device;
}
static void TearDownTestCase()
{
THE_IO_DEVICE->destroy();
}
virtual void SetUp()
{
ObIOManager::get_instance().destroy();
const int64_t memory_limit = 10L * 1024L * 1024L * 1024L; // 10GB
OK(ObIOManager::get_instance().init(memory_limit));
OK(ObIOManager::get_instance().start());
// add io device
OK(OB_IO_MANAGER.add_device_channel(THE_IO_DEVICE, 16, 2, 1024));
static ObTenantBase tenant_ctx(OB_SYS_TENANT_ID);
ObTenantEnv::set_tenant(&tenant_ctx);
ObTenantIOManager *io_service = nullptr;
OK(ObTenantIOManager::mtl_new(io_service));
OK(ObTenantIOManager::mtl_init(io_service));
OK(io_service->start());
tenant_ctx.set(io_service);
ObTenantEnv::set_tenant(&tenant_ctx);
ObMallocAllocator::get_instance()->set_tenant_limit(1, 8L * 1024L * 1024L * 1024L /* 8 GB */);
}
virtual void TearDown()
{
ObIOManager::get_instance().stop();
ObIOManager::get_instance().destroy();
}
};
int prepare_backup_device_macro_block_id(ObBackupDeviceMacroBlockId &macro_id)
{
int ret = OB_SUCCESS;
macro_id.turn_id_ = turn_id;
macro_id.length_ = offset;
macro_id.offset_ = length;
macro_id.reserved_ = 0;
macro_id.ls_id_ = ls_id;
macro_id.backup_set_id_ = backup_set_id;
macro_id.data_type_ = 1;
macro_id.retry_id_ = retry_id;
macro_id.file_id_ = file_id;
macro_id.block_type_ = ObBackupDeviceMacroBlockId::INDEX_TREE_BLOCK;
macro_id.id_mode_ = static_cast<uint64_t>(blocksstable::ObMacroBlockIdMode::ID_MODE_BACKUP);
macro_id.version_ = ObBackupDeviceMacroBlockId::BACKUP_MACRO_BLOCK_ID_VERSION;
return ret;
}
int prepare_macro_block_write_info(
const int64_t offset,
const char *buf, const int64_t length,
ObMacroBlockWriteInfo &write_info)
{
int ret = OB_SUCCESS;
write_info.io_desc_.set_wait_event(ObWaitEventIds::DB_FILE_COMPACT_WRITE);
write_info.offset_ = offset;
write_info.buffer_ = buf;
write_info.size_ = length;
return ret;
}
int prepare_macro_block_read_info(
const ObBackupDeviceMacroBlockId &backup_macro_id,
const int64_t offset, const int64_t size,
ObMacroBlockReadInfo &read_info)
{
int ret = OB_SUCCESS;
MacroBlockId macro_block_id(backup_macro_id.first_id(),
backup_macro_id.second_id(),
backup_macro_id.third_id());
read_info.macro_block_id_ = macro_block_id;
read_info.offset_ = offset;
read_info.size_ = size;
read_info.io_desc_.set_mode(ObIOMode::READ);
read_info.io_desc_.set_wait_event(ObWaitEventIds::DB_FILE_DATA_READ);
// read_info.io_desc_.set_resource_group_id(ObIOModule::ROOT_BLOCK_IO);
return ret;
}
int prepare_backup_device_handle(const ObStorageAccessType &access_type,
ObBackupWrapperIODevice &device, ObIOFd &io_fd)
{
int ret = OB_SUCCESS;
char test_dir[OB_MAX_URI_LENGTH];
char test_dir_uri[OB_MAX_URI_LENGTH];
char uri[OB_MAX_URI_LENGTH];
databuff_printf(test_dir, sizeof(test_dir), "%s/test_backup_async_io_dir", get_current_dir_name());
databuff_printf(test_dir_uri, sizeof(test_dir_uri), "file://%s", test_dir);
databuff_printf(uri, sizeof(uri), "file://%s/test_file", test_dir);
ObBackupStorageInfo storage_info;
storage_info.device_type_ = ObStorageType::OB_STORAGE_FILE;
static const int64_t opt_cnt = BACKUP_WRAPPER_DEVICE_OPT_NUM;
ObIODOpts io_d_opts;
ObIODOpt opts[opt_cnt];
io_d_opts.opts_ = opts;
io_d_opts.opt_cnt_ = opt_cnt;
backup_data_type.set_user_data_backup();
io_d_opts.opts_[0].set("storage_info", "");
const ObBackupDeviceMacroBlockId::BlockType block_type = ObBackupDeviceMacroBlockId::DATA_BLOCK;
ret = ObBackupWrapperIODevice::setup_io_opts_for_backup_device(
backup_set_id, ObLSID(ls_id), backup_data_type, turn_id, retry_id, file_id, block_type, access_type, &io_d_opts);
EXPECT_EQ(OB_SUCCESS, ret);
ObBackupIoAdapter util;
ret = util.mkdir(test_dir_uri, &storage_info);
EXPECT_EQ(OB_SUCCESS, ret);
ret = device.open(uri, -1, 0, io_fd, &io_d_opts);
EXPECT_EQ(OB_SUCCESS, ret);
return ret;
}
int prepare_write_io_info(const ObBackupDeviceMacroBlockId &macro_id,
const ObMacroBlockWriteInfo &write_info, ObIODevice *device_handle, ObIOInfo &write_io_info)
{
int ret = OB_SUCCESS;
write_io_info.tenant_id_ = tenant_id;
write_io_info.fd_.device_handle_ = device_handle;
write_io_info.fd_.first_id_ = macro_id.first_id();
write_io_info.fd_.second_id_ = macro_id.second_id();
write_io_info.fd_.third_id_ = macro_id.third_id();
write_io_info.offset_ = write_info.offset_;
write_io_info.size_ = write_info.size_;
write_io_info.buf_ = write_info.buffer_;
write_io_info.flag_ = write_info.io_desc_;
if (ObBackupDeviceMacroBlockId::is_backup_block_file(macro_id.first_id())) {
write_io_info.fd_.fd_id_ = static_cast<ObBackupWrapperIODevice *>(device_handle)->simulated_fd_id();
write_io_info.fd_.slot_version_ = static_cast<ObBackupWrapperIODevice *>(device_handle)->simulated_slot_version();
}
write_io_info.flag_.set_resource_group_id(THIS_WORKER.get_group_id());
write_io_info.flag_.set_write();
return ret;
}
int prepare_read_io_info(const ObBackupDeviceMacroBlockId &macro_id,
const ObMacroBlockReadInfo &read_info, ObIODevice *device_handle, ObIOInfo &read_io_info)
{
int ret = OB_SUCCESS;
read_io_info.tenant_id_ = tenant_id;
read_io_info.fd_.device_handle_ = device_handle;
read_io_info.fd_.first_id_ = macro_id.first_id();
read_io_info.fd_.second_id_ = macro_id.second_id();
read_io_info.fd_.third_id_ = macro_id.third_id();
read_io_info.offset_ = read_info.offset_;
read_io_info.size_ = static_cast<int32_t>(read_info.size_);
read_io_info.flag_ = read_info.io_desc_;
char *buf = NULL;
if (ObBackupDeviceMacroBlockId::is_backup_block_file(macro_id.first_id())) {
read_io_info.fd_.fd_id_ = static_cast<ObBackupWrapperIODevice *>(device_handle)->simulated_fd_id();
read_io_info.fd_.slot_version_ = static_cast<ObBackupWrapperIODevice *>(device_handle)->simulated_slot_version();
if (OB_FAIL(static_cast<ObBackupWrapperIODevice *>(device_handle)->alloc_mem_block(
read_io_info.size_, buf))) {
LOG_WARN("failed to alloc mem block", K(ret));
} else {
read_io_info.buf_ = buf;
}
}
read_io_info.flag_.set_resource_group_id(THIS_WORKER.get_group_id());
read_io_info.flag_.set_read();
return ret;
}
TEST_F(TestBackupMockAsyncIO, test_sync_io_mock_async_io)
{
ObBackupDeviceMacroBlockId macro_id;
char write_buf[length];
memset(write_buf, 1, sizeof(write_buf));
OK(prepare_backup_device_macro_block_id(macro_id));
ObMacroBlockWriteInfo write_info;
ObBackupWrapperIODevice write_device_handle;
ObIOInfo write_io_info;
ObIOHandle write_io_handle;
ObIOFd write_io_fd;
const ObStorageAccessType writer_access_type = OB_STORAGE_ACCESS_RANDOMWRITER;
EXPECT_TRUE(ObBackupDeviceMacroBlockId::is_backup_block_file(macro_id.first_id()));
// mock write
OK(prepare_macro_block_write_info(offset, write_buf, length, write_info));
OK(prepare_backup_device_handle(writer_access_type, write_device_handle, write_io_fd));
OK(prepare_write_io_info(macro_id, write_info, &write_device_handle, write_io_info));
OK(ObIOManager::get_instance().aio_write(write_io_info, write_io_handle));
OK(write_io_handle.wait());
// TODO(yanfeng): read need MTL, deal with this later
// ObMacroBlockReadInfo read_info;
// ObBackupWrapperIODevice read_device_handle;
// ObIOInfo read_io_info;
// ObIOHandle read_io_handle;
// ObIOFd read_io_fd;
// const ObStorageAccessType read_access_type = OB_STORAGE_ACCESS_READER;
// char *read_buf = NULL;
// // mock read
// OK(prepare_macro_block_read_info(macro_id, offset, length, read_info));
// OK(prepare_backup_device_handle(read_access_type, read_device_handle, read_io_fd));
// OK(prepare_read_io_info(macro_id, read_info, &read_device_handle, read_io_info));
// OK(ObIOManager::get_instance().aio_read(read_io_info, read_io_handle));
// OK(read_io_handle.wait(wait_timeout));
// // 比较read_buf和write_buf的内容
// bool is_same = true;
// for (int64_t i = 0; i < length; ++i) {
// if (read_io_info.buf_[i] != write_buf[i]) {
// is_same = false;
// break;
// }
// }
// EXPECT_TRUE(is_same);
}
}
}
int main(int argc, char **argv)
{
system("rm -f test_backup_sync_io_mock_async_io.log*");
ObLogger &logger = ObLogger::get_logger();
logger.set_file_name("test_backup_sync_io_mock_async_io.log", true);
logger.set_log_level("info");
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

136
unittest/storage/backup/test_backup_tablet_reorganize_helper.cpp Normal file
View File

@ -0,0 +1,136 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX STORAGE
#include <gtest/gtest.h>
#include "lib/container/ob_iarray.h"
#include "lib/container/ob_array.h"
#include "common/ob_tablet_id.h"
#include <algorithm>
#define private public
#define protected public
#include "share/backup/ob_backup_tablet_reorganize_helper.h"
using namespace oceanbase;
using namespace oceanbase::common;
using namespace oceanbase::share;
namespace oceanbase
{
namespace backup
{
void set_tablet_info(const int64_t src_tablet_id, const int64_t dest_tablet_id, ObTabletReorganizeInfo &info)
{
info.src_tablet_id_ = src_tablet_id;
info.dest_tablet_id_ = dest_tablet_id;
info.tenant_id_ = 1002;
info.ls_id_ = ObLSID(1001);
}
/*
1
/ \
2 3
/ \
4 5
*/
TEST(TestBackupReorganizeHelper, test1)
{
int ret = OB_SUCCESS;
common::ObTabletID tablet_id(1);
common::ObArray<ObTabletReorganizeInfo> infos;
ObTabletReorganizeInfo info;
set_tablet_info(1, 2, info);
ret = infos.push_back(info);
EXPECT_EQ(OB_SUCCESS, ret);
set_tablet_info(1, 3, info);
ret = infos.push_back(info);
EXPECT_EQ(OB_SUCCESS, ret);
set_tablet_info(3, 4, info);
ret = infos.push_back(info);
EXPECT_EQ(OB_SUCCESS, ret);
set_tablet_info(3, 5, info);
ret = infos.push_back(info);
EXPECT_EQ(OB_SUCCESS, ret);
common::ObArray<common::ObTabletID> tablet_list;
ret = ObBackupTabletReorganizeHelper::get_leaf_children_from_history(1002, infos, tablet_id, tablet_list);
EXPECT_EQ(OB_SUCCESS, ret);
}
/*
1
/ \
2 3
/ \
4 5
/ \
6 7
*/
TEST(TestBackupReorganizeHelper, test2)
{
int ret = OB_SUCCESS;
common::ObTabletID tablet_id(1);
ObArray<ObTabletReorganizeInfo> infos;
ObTabletReorganizeInfo info;
set_tablet_info(1, 2, info);
ret = infos.push_back(info);
EXPECT_EQ(OB_SUCCESS, ret);
set_tablet_info(1, 3, info);
ret = infos.push_back(info);
EXPECT_EQ(OB_SUCCESS, ret);
set_tablet_info(2, 4, info);
ret = infos.push_back(info);
EXPECT_EQ(OB_SUCCESS, ret);
set_tablet_info(2, 5, info);
ret = infos.push_back(info);
EXPECT_EQ(OB_SUCCESS, ret);
set_tablet_info(4, 6, info);
ret = infos.push_back(info);
EXPECT_EQ(OB_SUCCESS, ret);
set_tablet_info(4, 7, info);
ret = infos.push_back(info);
EXPECT_EQ(OB_SUCCESS, ret);
ObArray<ObTabletID> tablet_list;
ret = ObBackupTabletReorganizeHelper::get_leaf_children_from_history(1002, infos, tablet_id, tablet_list);
EXPECT_EQ(OB_SUCCESS, ret);
}
}
}
int main(int argc, char **argv)
{
system("rm -f test_backup_tablet_reorganize_helper.log*");
ObLogger &logger = ObLogger::get_logger();
logger.set_file_name("test_backup_tablet_reorganize_helper.log", true);
logger.set_log_level("info");
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

99
unittest/storage/backup/test_backup_utils.cpp
View File

@ -117,6 +117,8 @@ int ObFakeBackupTabletProvider::get_next_batch_items(common::ObIArray<ObBackupPr
int ret = OB_SUCCESS;
items.reset();
lib::ObMutexGuard guard(mutex_);
ObBackupDataType backup_data_type;
backup_data_type.set_user_data_backup();
if (supplied_item_count_ >= total_item_count_) {
ret = OB_ITER_END;
} else {
@ -131,7 +133,7 @@ int ObFakeBackupTabletProvider::get_next_batch_items(common::ObIArray<ObBackupPr
ObBackupMacroBlockId backup_macro_id;
backup_macro_id.macro_block_id_ = fake_macro_block_id;
backup_macro_id.logic_id_ = fake_logic_id;
if (OB_FAIL(item.set(PROVIDER_ITEM_TABLET_META, backup_macro_id, fake_table_key, tablet_id))) {
if (OB_FAIL(item.set(PROVIDER_ITEM_TABLET_AND_SSTABLE_META, backup_data_type, backup_macro_id, fake_table_key, tablet_id))) {
LOG_WARN("failed to set item", K(ret));
} else if (OB_FAIL(items.push_back(item))) {
LOG_WARN("failed to push back", K(ret), K(item));
@ -231,61 +233,6 @@ void make_macro_block_id_array(const int64_t tablet_id, const int64_t logic_vers
}
}
TEST(TestBackupUtils, test_check_macro_block_reuse)
{
int ret = OB_SUCCESS;
ObBackupTabletProvider provider;
bool need_skip = false;
const int64_t tablet_id = 200001;
const int64_t logic_version_1 = 100;
common::ObArray<int64_t> data_seq_list;
ret = data_seq_list.push_back(0);
ASSERT_EQ(OB_SUCCESS, ret);
ret = data_seq_list.push_back(1);
ASSERT_EQ(OB_SUCCESS, ret);
ret = data_seq_list.push_back(2);
ASSERT_EQ(OB_SUCCESS, ret);
ret = data_seq_list.push_back(3);
ASSERT_EQ(OB_SUCCESS, ret);
ret = data_seq_list.push_back(5);
ASSERT_EQ(OB_SUCCESS, ret);
common::ObArray<ObBackupMacroBlockIDPair> id_pair_list_1;
make_macro_block_id_array(tablet_id, logic_version_1, data_seq_list, id_pair_list_1);
blocksstable::ObLogicMacroBlockId logic_id_1(0/*data_seq*/, 100/*logic_version*/, tablet_id);
provider.inner_check_macro_block_need_skip_(logic_id_1, id_pair_list_1, need_skip);
ASSERT_TRUE(need_skip);
logic_id_1 = ObLogicMacroBlockId(1/*data_seq*/, 100/*logic_version*/, tablet_id);
provider.inner_check_macro_block_need_skip_(logic_id_1, id_pair_list_1, need_skip);
ASSERT_TRUE(need_skip);
logic_id_1 = ObLogicMacroBlockId(2/*data_seq*/, 100/*logic_version*/, tablet_id);
provider.inner_check_macro_block_need_skip_(logic_id_1, id_pair_list_1, need_skip);
ASSERT_TRUE(need_skip);
logic_id_1 = ObLogicMacroBlockId(3/*data_seq*/, 100/*logic_version*/, tablet_id);
provider.inner_check_macro_block_need_skip_(logic_id_1, id_pair_list_1, need_skip);
ASSERT_TRUE(need_skip);
logic_id_1 = ObLogicMacroBlockId(4/*data_seq*/, 100/*logic_version*/, tablet_id);
provider.inner_check_macro_block_need_skip_(logic_id_1, id_pair_list_1, need_skip);
ASSERT_FALSE(need_skip);
logic_id_1 = ObLogicMacroBlockId(5/*data_seq*/, 100/*logic_version*/, tablet_id);
provider.inner_check_macro_block_need_skip_(logic_id_1, id_pair_list_1, need_skip);
ASSERT_TRUE(need_skip);
logic_id_1 = ObLogicMacroBlockId(6/*data_seq*/, 100/*logic_version*/, tablet_id);
provider.inner_check_macro_block_need_skip_(logic_id_1, id_pair_list_1, need_skip);
ASSERT_FALSE(need_skip);
logic_id_1 = ObLogicMacroBlockId(0/*data_seq*/, 200/*logic_version*/, tablet_id);
provider.inner_check_macro_block_need_skip_(logic_id_1, id_pair_list_1, need_skip);
ASSERT_FALSE(need_skip);
}
class TestBackupExternalSort : public blocksstable::TestDataFilePrepare
{
public:
@ -368,6 +315,8 @@ int TestBackupExternalSort::generate_items(const int64_t min_count, common::ObIA
int ret = OB_SUCCESS;
ObBackupProviderItem *item = NULL;
items.reset();
ObBackupDataType backup_data_type;
backup_data_type.set_user_data_backup();
if (min_count < 0) {
ret = OB_INVALID_ARGUMENT;
COMMON_LOG(WARN, "invalid argument", K(ret), K(min_count));
@ -375,7 +324,7 @@ int TestBackupExternalSort::generate_items(const int64_t min_count, common::ObIA
void *buf = NULL;
ObTabletID tablet_id;
for (int64_t i = 0; OB_SUCC(ret) && i < min_count; ++i) {
ObBackupProviderItemType item_type = PROVIDER_ITEM_TABLET_META;
ObBackupProviderItemType item_type = PROVIDER_ITEM_TABLET_AND_SSTABLE_META;
make_random_tablet_id(tablet_id);
if (OB_ISNULL(buf = allocator.alloc(sizeof(ObBackupProviderItem)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
@ -383,7 +332,7 @@ int TestBackupExternalSort::generate_items(const int64_t min_count, common::ObIA
} else if (OB_ISNULL(item = new (buf) ObBackupProviderItem())) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to placement new item", K(ret));
} else if (OB_FAIL(item->set_with_fake(item_type, tablet_id))) {
} else if (OB_FAIL(item->set_with_fake(item_type, tablet_id, backup_data_type))) {
LOG_WARN("failed to set item", K(ret), K(tablet_id));
} else if (OB_FAIL(items.push_back(item))) {
LOG_WARN("failed to push back item", K(ret));
@ -393,40 +342,6 @@ int TestBackupExternalSort::generate_items(const int64_t min_count, common::ObIA
return ret;
}
TEST_F(TestBackupExternalSort, test_sort)
{
int ret = OB_SUCCESS;
int64_t buf_mem_limit = ObExternalSortConstant::MIN_MEMORY_LIMIT;
int64_t file_buf_size = 2 << 20;
int64_t expire_timestamp = 0;
const uint64_t tenant_id = OB_SYS_TENANT_ID;
typedef storage::ObExternalSort<ObBackupProviderItem, ObBackupProviderItemCompare> ExternalSort;
ExternalSort external_sort;
ObBackupProviderItemCompare backup_item_cmp(ret);
ObBackupDataType backup_data_type;
backup_data_type.set_major_data_backup();
backup_item_cmp.set_backup_data_type(backup_data_type);
ObArenaAllocator allocator;
const int64_t min_count = calculate_min_item_count();
ObVector<ObBackupProviderItem *> total_items;
ret = generate_items(min_count, allocator, total_items);
EXPECT_EQ(OB_SUCCESS, ret);
ret = external_sort.init(buf_mem_limit, file_buf_size, expire_timestamp, tenant_id, &backup_item_cmp);
EXPECT_EQ(OB_SUCCESS, ret);
for (int64_t i = 0; OB_SUCC(ret) && i < total_items.size(); ++i) {
const ObBackupProviderItem *item = total_items.at(i);
ret = external_sort.add_item(*item);
EXPECT_EQ(OB_SUCCESS, ret);
}
ret = external_sort.do_sort(true /*final_merge*/);
EXPECT_EQ(OB_SUCCESS, ret);
}
} // namespace backup
} // namespace oceanbase

6
unittest/storage/backup/test_log_stream_backup.cpp
View File

@ -133,7 +133,7 @@ void TestLogStreamBackup::SetUp()
backup_set_desc_.backup_set_id_ = 1;
backup_set_desc_.backup_type_.type_ = ObBackupType::FULL_BACKUP;
backup_set_desc_.backup_date_ = 20211231;
backup_data_type_.set_major_data_backup();
backup_data_type_.set_user_data_backup();
incarnation_ = 1;
tenant_id_ = 1;
ls_id_ = ObLSID(1);
@ -696,7 +696,7 @@ TEST_F(TestLogStreamBackup, test_build_tenant_level_full_index)
ASSERT_EQ(OB_SUCCESS, ret);
ObBackupDataType major_backup_type;
major_backup_type.set_major_data_backup();
major_backup_type.set_user_data_backup();
ret = ObBackupHandler::schedule_build_tenant_level_index_dag(
job_desc_, backup_dest_, tenant_id_, full_backup_set_desc_, turn_id_, major_backup_type);
ASSERT_EQ(OB_SUCCESS, ret);
@ -745,7 +745,7 @@ TEST_F(TestLogStreamBackup, test_build_tenant_level_inc_index)
ASSERT_EQ(OB_SUCCESS, ret);
ObBackupDataType major_backup_type;
major_backup_type.set_major_data_backup();
major_backup_type.set_user_data_backup();
ret = ObBackupHandler::schedule_build_tenant_level_index_dag(
job_desc_, backup_dest_, tenant_id_, inc_backup_set_desc_, turn_id_, major_backup_type);
ASSERT_EQ(OB_SUCCESS, ret);