hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

patch backup piece to 3.1. open source

Browse Source
This commit is contained in:
xuhuleon
2021-09-02 15:26:39 +08:00
committed by wangzelin.wzl
parent 153f16ad8a
commit 62cb470338
314 changed files with 88279 additions and 35466 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
unittest/share/backup/CMakeLists.txt
View File

@ -1,3 +1,4 @@
ob_unittest(test_backup_path)
ob_unittest(test_backup_struct)
ob_unittest(test_tenant_name_mgr)
ob_unittest(test_tenant_name_mgr)
ob_unittest(test_log_archive_backup_info_mgr)

12
unittest/share/backup/test_backup_path.cpp
View File

@ -54,7 +54,7 @@ TEST(ObBackupPathUtil, trim_right_backslash)
TEST(ObBackupPathUtil, base_data_path)
{
ObBackupPath path;
const char* backup_root_path = "oss://root_backup_dir";
const char* backup_root_path = "oss://root_backup_dir?host=xxx&access_id=xxx&access_key=xxx";
const char* cluster_name = "cluster_name";
const uint64_t cluster_id = 1;
const uint64_t incarnation = 1;
@ -94,6 +94,7 @@ TEST(ObBackupPathUtil, base_data_path)
test_backup_path_info.tenant_id_ = tenant_id;
test_backup_path_info.full_backup_set_id_ = full_backup_set_id;
test_backup_path_info.inc_backup_set_id_ = inc_backup_set_id;
test_backup_path_info.compatible_ = OB_BACKUP_COMPATIBLE_VERSION_V1;
LOG_INFO("dump path", K(test_backup_path_info));
ASSERT_EQ(true, test_backup_path_info.is_valid());
@ -170,8 +171,13 @@ TEST(ObBackupPathUtil, base_data_path)
path.reset();
ASSERT_EQ(OB_SUCCESS,
ObBackupPathUtil::get_macro_block_file_path(
test_backup_path_info, table_id, part_id, test_backup_path_info.inc_backup_set_id_, sub_task_id, path));
ObBackupPathUtil::get_macro_block_file_path(test_backup_path_info,
table_id,
part_id,
test_backup_path_info.full_backup_set_id_,
test_backup_path_info.inc_backup_set_id_,
sub_task_id,
path));
expect_path = macro_block_path;
ret = path.get_obstr().compare(expect_path);
LOG_INFO("dump path", K(path), K(expect_path));

415
unittest/share/backup/test_backup_struct.cpp
View File

@ -12,13 +12,67 @@
#define USING_LOG_PREFIX SHARE
#include "share/backup/ob_backup_path.h"
#include <gtest/gtest.h>
#include "lib/container/ob_array_array.h"
#include "lib/container/ob_array_iterator.h"
#define private public
#include "share/backup/ob_backup_path.h"
#include "share/backup/ob_log_archive_backup_info_mgr.h"
#include "share/backup/ob_backup_struct.h"
#undef private
#include <algorithm>
#include "lib/hash/ob_hashmap.h"
using namespace oceanbase;
using namespace common;
using namespace share;
TEST(ObBackupUtils, check_is_tmp_file)
{
bool is_tmp_file = false;
ObString file_name_1 = "file_name.tmp.123456";
ASSERT_EQ(OB_SUCCESS, ObBackupUtils::check_is_tmp_file(file_name_1, is_tmp_file));
ASSERT_TRUE(is_tmp_file);
ObString file_name_2 = "file_name.TMP.123456";
ASSERT_EQ(OB_SUCCESS, ObBackupUtils::check_is_tmp_file(file_name_2, is_tmp_file));
ASSERT_FALSE(is_tmp_file);
ObString file_name_3 = "file_name.tmp";
ASSERT_EQ(OB_SUCCESS, ObBackupUtils::check_is_tmp_file(file_name_3, is_tmp_file));
ASSERT_FALSE(is_tmp_file);
ObString file_name_4 = "file_name.tmp.";
ASSERT_EQ(OB_SUCCESS, ObBackupUtils::check_is_tmp_file(file_name_4, is_tmp_file));
ASSERT_FALSE(is_tmp_file);
ObString file_name_5 = "file_name";
ASSERT_EQ(OB_SUCCESS, ObBackupUtils::check_is_tmp_file(file_name_5, is_tmp_file));
ASSERT_FALSE(is_tmp_file);
ObString file_name_6 = "file_name.123456";
ASSERT_EQ(OB_SUCCESS, ObBackupUtils::check_is_tmp_file(file_name_6, is_tmp_file));
ASSERT_FALSE(is_tmp_file);
}
TEST(ObSimpleBackupSetPath, simple_path_disk)
{
ObSimpleBackupSetPath simple_path;
simple_path.backup_set_id_ = 1;
simple_path.backup_dest_ = "file:///root_backup_dir";
LOG_INFO("simple_path", K(simple_path.get_simple_path()), K(simple_path.get_storage_info()));
ASSERT_EQ(OB_SUCCESS, simple_path.get_simple_path().case_compare("file:///root_backup_dir"));
ASSERT_EQ(OB_SUCCESS, simple_path.get_storage_info().case_compare(""));
}
TEST(ObSimpleBackupSetPath, simple_path_oss)
{
ObSimpleBackupSetPath simple_path;
simple_path.backup_set_id_ = 1;
simple_path.backup_dest_ =
"oss://backup_dir/?host=http://oss-cn-hangzhou-zmf.aliyuncs.com&access_id=111&access_key=222";
LOG_INFO("simple_path", K(simple_path.get_simple_path()), K(simple_path.get_storage_info()));
ASSERT_EQ(OB_SUCCESS, simple_path.get_simple_path().case_compare("oss://backup_dir/"));
ASSERT_EQ(OB_SUCCESS,
simple_path.get_storage_info().case_compare(
"host=http://oss-cn-hangzhou-zmf.aliyuncs.com&access_id=111&access_key=222"));
}
TEST(ObBackupDest, disk)
{
const char* backup_test = "file:///root_backup_dir";
@ -40,7 +94,364 @@ TEST(ObBackupDest, oss)
ASSERT_EQ(0, strcmp(dest.storage_info_, "host=http://oss-cn-hangzhou-zmf.aliyuncs.com&access_id=111&access_key=222"));
}
int main(int argc, char** argv)
int check_backup_dest_opt(const ObBackupDestOpt& opt, int64_t log_archive_checkpoint_interval, int64_t recovery_window,
int64_t piece_switch_interval, int64_t backup_copies, bool auto_delete_obsolete_backup,
bool auto_touch_reserved_backup)
{
int ret = OB_SUCCESS;
if (opt.log_archive_checkpoint_interval_ != log_archive_checkpoint_interval) {
ret = OB_ERR_SYS;
LOG_WARN("not match", K(log_archive_checkpoint_interval), K(opt));
} else if (opt.recovery_window_ != recovery_window) {
ret = OB_ERR_SYS;
LOG_WARN("not match", K(recovery_window), K(opt));
} else if (opt.piece_switch_interval_ != piece_switch_interval) {
ret = OB_ERR_SYS;
LOG_WARN("not match", K(piece_switch_interval), K(opt));
} else if (opt.backup_copies_ != backup_copies) {
ret = OB_ERR_SYS;
LOG_WARN("not match", K(backup_copies), K(opt));
} else if (opt.auto_delete_obsolete_backup_ != auto_delete_obsolete_backup) {
ret = OB_ERR_SYS;
LOG_WARN("not match", K(auto_delete_obsolete_backup), K(opt));
} else if (opt.auto_touch_reserved_backup_ != auto_touch_reserved_backup) {
ret = OB_ERR_SYS;
LOG_WARN("not match", K(auto_touch_reserved_backup), K(opt));
}
return ret;
}
TEST(ObBackupDestOpt, normal)
{
const char* backup_dest_opt_str =
"recovery_window=7d&auto_delete_obsolete_backup=true&backup_copies=2&log_archive_piece_switch_interval=1d";
const char* backup_backup_dest_opt_str = "recovery_window=14d&auto_delete_obsolete_backup=true";
ObBackupDestOpt backup_dest_opt;
ObBackupDestOpt backup_backup_dest_opt;
const bool global_auto_delete = false;
const int64_t global_recovery_window = 0;
const int64_t global_checkpoint_interval = 120 * 1000LL * 1000; // default config 120s
const int64_t global_backup_checkpoint_interval = 0;
ASSERT_EQ(OB_SUCCESS,
backup_dest_opt.init(false /*is_backup_backup*/,
backup_dest_opt_str,
global_auto_delete,
global_recovery_window,
global_checkpoint_interval,
false));
ASSERT_EQ(OB_SUCCESS,
check_backup_dest_opt(backup_dest_opt,
global_checkpoint_interval,
7 * 24 * 3600LL * 1000 * 1000 /*recovery_window*/,
1 * 24 * 3600LL * 1000 * 1000 /*piece_switch_interval*/,
2 /*backup_copies*/,
true /*auto delete*/,
false /*auto touch*/));
ASSERT_EQ(OB_SUCCESS,
backup_backup_dest_opt.init(true /*is_backup_backup*/,
backup_backup_dest_opt_str,
global_auto_delete,
global_recovery_window,
global_backup_checkpoint_interval,
false));
ASSERT_EQ(OB_SUCCESS,
check_backup_dest_opt(backup_backup_dest_opt,
0,
14 * 24 * 3600LL * 1000 * 1000 /*recovery_window*/,
0 /*piece_switch_interval*/,
0 /*backup_copies*/,
true /*auto delete*/,
false /*auto touch*/));
}
TEST(ObBackupDestOpt, backup_dest_all)
{
ObBackupDestOpt backup_dest_opt;
ASSERT_EQ(OB_SUCCESS,
backup_dest_opt.init(false /*is_backup_backup*/,
"log_archive_checkpoint_interval=10m&recovery_window=30d&auto_delete_obsolete_backup=true&log_archive_piece_"
"switch_interval=3d&backup_copies=3",
false /*global_auto_delete*/,
0 /*global_recovery_window*/,
120 * 1000LL * 1000LL /*global_checkpoint_interval*/,
false));
ASSERT_EQ(OB_SUCCESS,
check_backup_dest_opt(backup_dest_opt,
10 * 60 * 1000LL * 1000LL /*global_checkpoint_interval*/,
30 * 24 * 3600LL * 1000 * 1000 /*recovery_window*/,
3 * 24 * 3600LL * 1000 * 1000 /*piece_switch_interval*/,
3 /*backup_copies*/,
true /*auto delete*/,
false /*auto touch*/));
}
TEST(ObBackupDestOpt, backup_dest_empty)
{
ObBackupDestOpt backup_dest_opt;
ASSERT_EQ(OB_SUCCESS,
backup_dest_opt.init(false /*is_backup_backup*/,
"",
false /*global_auto_delete*/,
0 /*global_recovery_window*/,
120 * 1000LL * 1000LL /*global_checkpoint_interval*/,
false));
ASSERT_EQ(OB_SUCCESS,
check_backup_dest_opt(backup_dest_opt,
2 * 60 * 1000LL * 1000LL /*global_checkpoint_interval*/,
0 /*recovery_window*/,
0 /*piece_switch_interval*/,
0 /*backup_copies*/,
0 /*auto delete*/,
false /*auto touch*/));
}
TEST(ObBackupDestOpt, backup_dest_invalid_format)
{
ObBackupDestOpt backup_dest_opt;
// invalid key checkpoint_interval
ASSERT_EQ(OB_INVALID_ARGUMENT,
backup_dest_opt.init(false /*is_backup_backup*/,
"checkpoint_interval=10m&recovery_window=30d&auto_delete_obsolete_backup=true&log_archive_piece_switch_"
"interval=3d&backup_copies=3&auto_update_reserved_backup_timestamp=true",
false /*global_auto_delete*/,
0 /*global_recovery_window*/,
120 * 1000LL * 1000LL /*global_checkpoint_interval*/,
false));
// has blank space
ASSERT_EQ(OB_INVALID_ARGUMENT,
backup_dest_opt.init(false /*is_backup_backup*/,
"log_archive_checkpoint_interval=10m&recovery_window=30d "
"&auto_delete_obsolete_backup=true&log_archive_piece_switch_interval=3d&backup_copies=3&auto_update_reserved_"
"backup_timestamp=true",
false /*global_auto_delete*/,
0 /*global_recovery_window*/,
120 * 1000LL * 1000LL /*global_checkpoint_interval*/,
false));
// recovery_window has no value
ASSERT_EQ(OB_INVALID_ARGUMENT,
backup_dest_opt.init(false /*is_backup_backup*/,
"log_archive_checkpoint_interval=10m&recovery_window=&auto_delete_obsolete_backup=true&log_archive_piece_"
"switch_interval=3d&backup_copies=3&auto_update_reserved_backup_timestamp=true",
false /*global_auto_delete*/,
0 /*global_recovery_window*/,
120 * 1000LL * 1000LL /*global_checkpoint_interval*/,
false));
// recovery_window has invalid value
ASSERT_EQ(OB_INVALID_ARGUMENT,
backup_dest_opt.init(false /*is_backup_backup*/,
"log_archive_checkpoint_interval=10m&recovery_window=a&auto_delete_obsolete_backup=true&log_archive_piece_"
"switch_interval=3d&backup_copies=3&auto_update_reserved_backup_timestamp=true",
false /*global_auto_delete*/,
0 /*global_recovery_window*/,
120 * 1000LL * 1000LL /*global_checkpoint_interval*/,
false));
// auto_delete_obsolete_backup has invalid value
ASSERT_EQ(OB_INVALID_ARGUMENT,
backup_dest_opt.init(false /*is_backup_backup*/,
"log_archive_checkpoint_interval=10m&recovery_window=7d&auto_delete_obsolete_backup=a&log_archive_piece_"
"switch_interval=3d&backup_copies=3&auto_update_reserved_backup_timestamp=true",
false /*global_auto_delete*/,
0 /*global_recovery_window*/,
120 * 1000LL * 1000LL /*global_checkpoint_interval*/,
false));
}
TEST(ObBackupDestOpt, backup_dest_invalid_delete_and_touch)
{
ObBackupDestOpt backup_dest_opt;
ASSERT_EQ(OB_NOT_SUPPORTED,
backup_dest_opt.init(false /*is_backup_backup*/,
"log_archive_checkpoint_interval=10m&recovery_window=30d&auto_delete_obsolete_backup=true&log_archive_piece_"
"switch_interval=3d&backup_copies=3&auto_update_reserved_backup_timestamp=true",
false /*global_auto_delete*/,
0 /*global_recovery_window*/,
120 * 1000LL * 1000LL /*global_checkpoint_interval*/,
false));
}
TEST(ObBackupDestOpt, backup_dest_invalid_copies)
{
ObBackupDestOpt backup_dest_opt;
ASSERT_EQ(OB_NOT_SUPPORTED,
backup_dest_opt.init(false /*is_backup_backup*/,
"log_archive_checkpoint_interval=10m&recovery_window=30d&auto_delete_obsolete_backup=true&log_archive_piece_"
"switch_interval=3d&backup_copies=-1",
false /*global_auto_delete*/,
0 /*global_recovery_window*/,
120 * 1000LL * 1000LL /*global_checkpoint_interval*/,
false));
}
TEST(ObBackupDestOpt, backup_dest_invalid_recovery_window)
{
ObBackupDestOpt backup_dest_opt;
ASSERT_EQ(OB_INVALID_ARGUMENT,
backup_dest_opt.init(false /*is_backup_backup*/,
"log_archive_checkpoint_interval=10m&recovery_window=-1d&log_archive_piece_switch_interval=3d&backup_copies="
"3&auto_update_reserved_backup_timestamp=true",
false /*global_auto_delete*/,
0 /*global_recovery_window*/,
120 * 1000LL * 1000LL /*global_checkpoint_interval*/,
false));
}
TEST(ObBackupDestOpt, backup_dest_invalid_piece_switch)
{
ObBackupDestOpt backup_dest_opt;
ASSERT_EQ(OB_NOT_SUPPORTED,
backup_dest_opt.init(false /*is_backup_backup*/,
"log_archive_checkpoint_interval=10m&recovery_window=30d&log_archive_piece_switch_interval=1s&backup_copies="
"3&auto_update_reserved_backup_timestamp=true",
false /*global_auto_delete*/,
0 /*global_recovery_window*/,
120 * 1000LL * 1000LL /*global_checkpoint_interval*/,
false));
}
TEST(ObBackupDestOpt, backup_backup_dest_invalid_copies)
{
ObBackupDestOpt backup_dest_opt;
ASSERT_EQ(OB_INVALID_ARGUMENT,
backup_dest_opt.init(true /*is_backup_backup*/,
"log_archive_checkpoint_interval=10m&recovery_window=30d&auto_delete_obsolete_backup=true&backup_copies=1",
false /*global_auto_delete*/,
0 /*global_recovery_window*/,
120 * 1000LL * 1000LL /*global_checkpoint_interval*/,
false));
}
TEST(ObBackupDestOpt, backup_backup_dest_invalid_piece_switch)
{
ObBackupDestOpt backup_dest_opt;
ASSERT_EQ(OB_INVALID_ARGUMENT,
backup_dest_opt.init(true /*is_backup_backup*/,
"log_archive_checkpoint_interval=10m&recovery_window=30d&auto_delete_obsolete_backup=true&log_archive_piece_"
"switch_interval=3d&backup_copies=0",
false /*global_auto_delete*/,
0 /*global_recovery_window*/,
120 * 1000LL * 1000LL /*global_checkpoint_interval*/,
false));
}
TEST(ObLogArchiveBackupInfoMgr, log_archive_info_cmp)
{
int ret = OB_SUCCESS;
ObLogArchiveBackupInfoMgr::CompareBackupPieceInfo cmp(ret);
ObBackupPieceInfo piece_info;
ObArray<ObBackupPieceInfo> piece_infos;
ObArray<ObBackupPieceInfo> piece_infos_copy1;
ObArray<ObBackupPieceInfo> piece_infos_copy2;
ObArray<ObBackupPieceInfo> piece_infos_copy3;
ret = piece_info.backup_dest_.assign("file::///oceanbase/");
ASSERT_EQ(OB_SUCCESS, ret);
piece_info.checkpoint_ts_ = 1621652028364222;
piece_info.compatible_ = ObTenantLogArchiveStatus::COMPATIBLE_VERSION_2;
piece_info.create_date_ = 20210523;
piece_info.file_status_ = ObBackupFileStatus::BACKUP_FILE_AVAILABLE;
piece_info.key_.tenant_id_ = 1;
piece_info.max_ts_ = 1621652028364222;
piece_info.start_piece_id_ = 0;
piece_info.start_ts_ = 0;
piece_info.status_ = ObBackupPieceStatus::BACKUP_PIECE_ACTIVE;
//mock array with same incarnation, round, copy id
for (int64_t i = 0; OB_SUCC(ret) && i < 10; ++i) {
piece_info.key_.copy_id_ = 0;
piece_info.key_.incarnation_ = 1;
piece_info.key_.round_id_ = 1;
piece_info.key_.backup_piece_id_ = i;
ret = piece_infos.push_back(piece_info);
ASSERT_EQ(OB_SUCCESS, ret);
}
std::sort(piece_infos.begin(), piece_infos.end(), cmp);
for (int64_t i = 0; OB_SUCC(ret) && i < piece_infos.count(); ++i) {
const ObBackupPieceInfo &tmp_piece_info = piece_infos.at(i);
ASSERT_EQ(tmp_piece_info.key_.backup_piece_id_, i);
}
piece_infos.reset();
for (int64_t i = 0; OB_SUCC(ret) && i < 10; ++i) {
piece_info.key_.copy_id_ = 1;
piece_info.key_.incarnation_ = 1;
piece_info.key_.round_id_ = 1+i;
piece_info.key_.backup_piece_id_ = i;
ret = piece_infos_copy1.push_back(piece_info);
ASSERT_EQ(OB_SUCCESS, ret);
}
for (int64_t i = 0; OB_SUCC(ret) && i < 9; ++i) {
piece_info.key_.copy_id_ = 2;
piece_info.key_.incarnation_ = 1;
piece_info.key_.round_id_ = 1+i;
piece_info.key_.backup_piece_id_ = i;
ret = piece_infos_copy2.push_back(piece_info);
ASSERT_EQ(OB_SUCCESS, ret);
}
for (int64_t i = 0; OB_SUCC(ret) && i < 7; ++i) {
piece_info.key_.copy_id_ = 3;
piece_info.key_.incarnation_ = 1;
piece_info.key_.round_id_ = 1+i;
piece_info.key_.backup_piece_id_ = i;
ret = piece_infos_copy3.push_back(piece_info);
ASSERT_EQ(OB_SUCCESS, ret);
}
//add to piece infos
for (int64_t i = 0; OB_SUCC(ret) && i < piece_infos_copy3.count(); ++i) {
const ObBackupPieceInfo &tmp_piece_info = piece_infos_copy3.at(i);
ret = piece_infos.push_back(tmp_piece_info);
ASSERT_EQ(OB_SUCCESS, ret);
}
for (int64_t i = 0; OB_SUCC(ret) && i < piece_infos_copy1.count(); ++i) {
const ObBackupPieceInfo &tmp_piece_info = piece_infos_copy1.at(i);
ret = piece_infos.push_back(tmp_piece_info);
ASSERT_EQ(OB_SUCCESS, ret);
}
for (int64_t i = 0; OB_SUCC(ret) && i < piece_infos_copy2.count(); ++i) {
const ObBackupPieceInfo &tmp_piece_info = piece_infos_copy2.at(i);
ret = piece_infos.push_back(tmp_piece_info);
ASSERT_EQ(OB_SUCCESS, ret);
}
std::sort(piece_infos.begin(), piece_infos.end(), cmp);
for (int64_t i = 0; OB_SUCC(ret) && i < 10; ++i) {
const ObBackupPieceInfo &tmp_piece_info = piece_infos.at(i);
ASSERT_EQ(tmp_piece_info.key_.backup_piece_id_, i);
ASSERT_EQ(tmp_piece_info.key_.round_id_, i + 1);
ASSERT_EQ(tmp_piece_info.key_.copy_id_, 1);
}
ret = piece_infos.push_back(piece_info);
ASSERT_EQ(OB_SUCCESS, ret);
std::sort(piece_infos.begin(), piece_infos.end(), cmp);
ASSERT_EQ(OB_ERR_UNEXPECTED, ret);
}
int main(int argc, char **argv)
{
OB_LOGGER.set_log_level("INFO");
testing::InitGoogleTest(&argc, argv);

129
unittest/share/backup/test_log_archive_backup_info_mgr.cpp Normal file
View File

@ -0,0 +1,129 @@
// Copyright 2010-2021 OceanBase Inc. All Rights Reserved.
// Author:
// yongle.xh@antfin.com
//
#define USING_LOG_PREFIX SHARE
#include <gtest/gtest.h>
#define private public
#include "share/backup/ob_backup_path.h"
#include "share/backup/ob_log_archive_backup_info_mgr.h"
using namespace oceanbase;
using namespace common;
using namespace share;
void init_piece(const int64_t round_id, const int64_t piece_id, const int64_t copy_id, ObBackupPieceInfo& piece)
{
piece.reset();
piece.key_.incarnation_ = OB_START_INCARNATION;
piece.key_.tenant_id_ = 1;
piece.key_.round_id_ = round_id;
piece.key_.backup_piece_id_ = piece_id;
piece.key_.copy_id_ = copy_id;
piece.create_date_ = 1;
piece.start_ts_ = 1;
piece.checkpoint_ts_ = 2;
piece.max_ts_ = 3;
piece.status_ = ObBackupPieceStatus::BACKUP_PIECE_FROZEN;
piece.file_status_ = ObBackupFileStatus::BACKUP_FILE_AVAILABLE;
piece.backup_dest_.assign("file:///backup");
piece.compatible_ = ObTenantLogArchiveStatus::COMPATIBLE_VERSION_2;
piece.start_piece_id_ = 1;
}
int check_piece(ObArray<share::ObBackupPieceInfo>& result, ObArray<share::ObBackupPieceInfo>& expect)
{
int ret = OB_SUCCESS;
if (result.count() != expect.count()) {
ret = OB_ERR_SYS;
LOG_ERROR("count not match", "result_count", result.count(), "expect_count", expect.count());
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < result.count(); ++i) {
if (result.at(i) != expect.at(i)) {
ret = OB_ERR_SYS;
LOG_ERROR("piece not match", K(ret), K(i));
}
}
}
if (OB_FAIL(ret)) {
for (int64_t i = 0; i < result.count(); ++i) {
LOG_INFO("dump result piece", K(ret), K(i), "result", result.at(i));
}
for (int64_t i = 0; i < expect.count(); ++i) {
LOG_INFO("dump expect piece", K(ret), K(i), "expect", expect.at(i));
}
} else {
for (int64_t i = 0; i < result.count(); ++i) {
LOG_INFO("dump result piece", K(ret), K(i), "result", result.at(i));
}
}
return ret;
}
TEST(ObExternalBackupPieceInfo, update)
{
ObExternalBackupPieceInfo external_info;
share::ObBackupPieceInfo piece1_1;
share::ObBackupPieceInfo piece1_2;
share::ObBackupPieceInfo piece2_1;
ObArray<share::ObBackupPieceInfo> expect_array;
ObArray<share::ObBackupPieceInfo> result_array;
// normal seq
init_piece(1, 1, 0, piece1_1);
init_piece(1, 2, 0, piece1_2);
init_piece(2, 1, 0, piece2_1);
ASSERT_EQ(OB_SUCCESS, external_info.update(piece1_1));
ASSERT_EQ(OB_SUCCESS, external_info.update(piece1_2));
ASSERT_EQ(OB_SUCCESS, external_info.update(piece2_1));
ASSERT_EQ(OB_SUCCESS, external_info.get_piece_array(result_array));
expect_array.reset();
ASSERT_EQ(OB_SUCCESS, expect_array.push_back(piece1_1));
ASSERT_EQ(OB_SUCCESS, expect_array.push_back(piece1_2));
ASSERT_EQ(OB_SUCCESS, expect_array.push_back(piece2_1));
ASSERT_EQ(OB_SUCCESS, check_piece(result_array, expect_array));
// update same copy and piece
init_piece(1, 2, 0, piece1_2);
piece1_2.max_ts_ = 100;
ASSERT_EQ(OB_SUCCESS, external_info.update(piece1_2));
ASSERT_EQ(OB_SUCCESS, external_info.get_piece_array(result_array));
expect_array.reset();
ASSERT_EQ(OB_SUCCESS, expect_array.push_back(piece1_1));
ASSERT_EQ(OB_SUCCESS, expect_array.push_back(piece1_2));
ASSERT_EQ(OB_SUCCESS, expect_array.push_back(piece2_1));
ASSERT_EQ(OB_SUCCESS, check_piece(result_array, expect_array));
// insert 1_3
share::ObBackupPieceInfo piece1_3;
init_piece(1, 3, 0, piece1_3);
ASSERT_EQ(OB_SUCCESS, external_info.update(piece1_3));
ASSERT_EQ(OB_SUCCESS, external_info.get_piece_array(result_array));
expect_array.reset();
ASSERT_EQ(OB_SUCCESS, expect_array.push_back(piece1_1));
ASSERT_EQ(OB_SUCCESS, expect_array.push_back(piece1_2));
ASSERT_EQ(OB_SUCCESS, expect_array.push_back(piece1_3));
ASSERT_EQ(OB_SUCCESS, expect_array.push_back(piece2_1));
ASSERT_EQ(OB_SUCCESS, check_piece(result_array, expect_array));
// insert same piece with diff copy
share::ObBackupPieceInfo piece1_2_1;
init_piece(1, 2, 1, piece1_2_1);
ASSERT_EQ(OB_SUCCESS, external_info.update(piece1_2_1));
ASSERT_EQ(OB_SUCCESS, external_info.get_piece_array(result_array));
expect_array.reset();
ASSERT_EQ(OB_SUCCESS, expect_array.push_back(piece1_1));
ASSERT_EQ(OB_SUCCESS, expect_array.push_back(piece1_2));
ASSERT_EQ(OB_SUCCESS, expect_array.push_back(piece1_2_1));
ASSERT_EQ(OB_SUCCESS, expect_array.push_back(piece1_3));
ASSERT_EQ(OB_SUCCESS, expect_array.push_back(piece2_1));
ASSERT_EQ(OB_SUCCESS, check_piece(result_array, expect_array));
}
int main(int argc, char** argv)
{
OB_LOGGER.set_log_level("INFO");
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

446
unittest/share/backup/test_multi_backup_dest_util.cpp Normal file
View File

@ -0,0 +1,446 @@
// Copyright 2010-2021 OceanBase Inc. All Rights Reserved.
// Author:
// yangyi.yyy@antfin.com
//
#define USING_LOG_PREFIX SHARE
#include <gtest/gtest.h>
#define private public
#include "share/backup/ob_backup_struct.h"
#include "share/backup/ob_multi_backup_dest_util.h"
using namespace oceanbase;
using namespace oceanbase::common;
using namespace oceanbase::share;
void mock_backup_set(const int64_t backup_set_id, const int64_t prev_full_backup_set_id,
const int64_t prev_inc_backup_set_id, const int64_t snapshot_version, const int64_t start_replay_log_ts,
const share::ObBackupType::BackupType type, share::ObBackupSetFileInfo& set_info)
{
set_info.reset();
set_info.backup_set_id_ = backup_set_id;
set_info.prev_full_backup_set_id_ = prev_full_backup_set_id;
set_info.prev_inc_backup_set_id_ = prev_inc_backup_set_id;
set_info.snapshot_version_ = snapshot_version;
set_info.start_replay_log_ts_ = start_replay_log_ts;
set_info.backup_type_.type_ = type;
}
void mock_backup_piece(const int64_t round_id, const int64_t piece_id, const int64_t start_ts,
const int64_t checkpoint_ts, const int64_t max_ts, const share::ObBackupPieceStatus::STATUS& status,
share::ObBackupPieceInfo& piece)
{
piece.reset();
piece.key_.round_id_ = round_id; // check all same round
piece.key_.backup_piece_id_ = piece_id; // check piece is continuous
piece.start_ts_ = start_ts;
piece.checkpoint_ts_ = checkpoint_ts;
piece.max_ts_ = max_ts;
piece.status_ = status; // check all but last is frozen
piece.start_piece_id_ = 1; // check start piece id
}
int mock_only_full_backup_set(const int64_t full_count, common::ObArray<share::ObBackupSetFileInfo>& set_list)
{
int ret = OB_SUCCESS;
set_list.reset();
for (int64_t i = 1; OB_SUCC(ret) && i <= full_count; ++i) {
ObBackupSetFileInfo info;
mock_backup_set(i, 0, 0, 100, 100, ObBackupType::FULL_BACKUP, info);
if (OB_FAIL(set_list.push_back(info))) {
LOG_WARN("failed to push back", KR(ret));
}
}
return ret;
}
int mock_only_inc_backup_set_list(const int64_t inc_count, common::ObArray<share::ObBackupSetFileInfo>& set_list)
{
int ret = OB_SUCCESS;
set_list.reset();
for (int64_t i = 2; OB_SUCC(ret) && i < 2 + inc_count; ++i) {
ObBackupSetFileInfo info;
mock_backup_set(i, 1, 1, 100, 100, ObBackupType::INCREMENTAL_BACKUP, info);
if (OB_FAIL(set_list.push_back(info))) {
LOG_WARN("failed to push back", KR(ret));
}
}
return ret;
}
int mock_normal_full_and_inc_backup_set_list(common::ObArray<share::ObBackupSetFileInfo>& set_list)
{
int ret = OB_SUCCESS;
set_list.reset();
for (int64_t i = 1; OB_SUCC(ret) && i <= 3; ++i) {
ObBackupSetFileInfo info;
if (1 == i) {
mock_backup_set(1, 0, 0, 100, 100, ObBackupType::FULL_BACKUP, info);
} else if (2 == i) {
mock_backup_set(2, 1, 1, 200, 200, ObBackupType::INCREMENTAL_BACKUP, info);
} else if (3 == i) {
mock_backup_set(3, 1, 2, 300, 300, ObBackupType::INCREMENTAL_BACKUP, info);
}
if (OB_FAIL(set_list.push_back(info))) {
LOG_WARN("failed to push back", KR(ret));
}
}
return ret;
}
int mock_duplicate_inc_backup_set_list(common::ObArray<share::ObBackupSetFileInfo>& set_list)
{
int ret = OB_SUCCESS;
set_list.reset();
for (int64_t i = 1; OB_SUCC(ret) && i <= 3; ++i) {
ObBackupSetFileInfo info;
if (1 == i) {
mock_backup_set(1, 0, 0, 100, 100, ObBackupType::FULL_BACKUP, info);
} else if (2 == i) {
mock_backup_set(2, 1, 1, 200, 200, ObBackupType::INCREMENTAL_BACKUP, info);
} else if (3 == i) {
mock_backup_set(2, 1, 1, 300, 300, ObBackupType::INCREMENTAL_BACKUP, info);
}
if (OB_FAIL(set_list.push_back(info))) {
LOG_WARN("failed to push back", KR(ret));
}
}
return ret;
}
int mock_full_and_inc_discontinuous_backup_set_list_with_failed(common::ObArray<share::ObBackupSetFileInfo>& set_list)
{
int ret = OB_SUCCESS;
for (int64_t i = 1; OB_SUCC(ret) && i <= 3; ++i) {
ObBackupSetFileInfo info;
if (1 == i) {
mock_backup_set(1, 0, 0, 100, 100, ObBackupType::FULL_BACKUP, info);
} else if (2 == i) {
mock_backup_set(2, 1, 1, 200, 200, ObBackupType::INCREMENTAL_BACKUP, info);
info.result_ = OB_IO_ERROR;
} else if (3 == i) {
mock_backup_set(3, 1, 1, 300, 300, ObBackupType::INCREMENTAL_BACKUP, info);
}
if (OB_FAIL(set_list.push_back(info))) {
LOG_WARN("failed to push back", KR(ret));
}
}
return ret;
}
int mock_full_and_inc_discontinuous_backup_set_list(common::ObArray<share::ObBackupSetFileInfo>& set_list)
{
int ret = OB_SUCCESS;
set_list.reset();
for (int64_t i = 1; OB_SUCC(ret) && i <= 3; ++i) {
ObBackupSetFileInfo info;
if (1 == i) {
mock_backup_set(1, 0, 0, 100, 100, ObBackupType::FULL_BACKUP, info);
} else if (2 == i) {
continue;
} else if (3 == i) {
mock_backup_set(3, 1, 2, 300, 300, ObBackupType::INCREMENTAL_BACKUP, info);
}
if (OB_FAIL(set_list.push_back(info))) {
LOG_WARN("failed to push back", KR(ret));
}
}
return ret;
}
int mock_normal_round_piece_list(common::ObArray<share::ObBackupPieceInfo>& piece_list)
{
int ret = OB_SUCCESS;
piece_list.reset();
for (int64_t i = 1; OB_SUCC(ret) && i <= 3; ++i) {
ObBackupPieceInfo piece_info;
if (1 == i) {
mock_backup_piece(1, i, 100, 200, 300, ObBackupPieceStatus::BACKUP_PIECE_FROZEN, piece_info);
} else if (2 == i) {
mock_backup_piece(1, i, 300, 400, 500, ObBackupPieceStatus::BACKUP_PIECE_FROZEN, piece_info);
} else if (3 == i) {
mock_backup_piece(1, i, 500, 700, 1000, ObBackupPieceStatus::BACKUP_PIECE_FROZEN, piece_info);
}
if (OB_FAIL(piece_list.push_back(piece_info))) {
LOG_WARN("failed to push back", KR(ret));
}
}
return ret;
}
int mock_discontinuous_piece_list(common::ObArray<share::ObBackupPieceInfo>& piece_list)
{
int ret = OB_SUCCESS;
for (int64_t i = 1; OB_SUCC(ret) && i <= 3; ++i) {
ObBackupPieceInfo piece_info;
piece_info.start_piece_id_ = 1;
if (1 == i) {
mock_backup_piece(1, i, 100, 200, 300, ObBackupPieceStatus::BACKUP_PIECE_FROZEN, piece_info);
} else if (2 == i) {
continue;
} else if (3 == i) {
mock_backup_piece(1, i, 600, 1000, 1100, ObBackupPieceStatus::BACKUP_PIECE_FROZEN, piece_info);
}
if (OB_FAIL(piece_list.push_back(piece_info))) {
LOG_WARN("failed to push back", KR(ret));
}
}
return ret;
}
int mock_duplicate_piece_list(common::ObArray<share::ObBackupPieceInfo>& piece_list)
{
int ret = OB_SUCCESS;
for (int64_t i = 1; OB_SUCC(ret) && i <= 3; ++i) {
ObBackupPieceInfo piece_info;
mock_backup_piece(1, 1, 100, 200, 300, ObBackupPieceStatus::BACKUP_PIECE_FROZEN, piece_info);
if (OB_FAIL(piece_list.push_back(piece_info))) {
LOG_WARN("failed to push back", KR(ret));
}
}
return ret;
}
int mock_mid_freezing_piece_list(common::ObArray<share::ObBackupPieceInfo>& piece_list)
{
int ret = OB_SUCCESS;
piece_list.reset();
for (int64_t i = 1; OB_SUCC(ret) && i <= 3; ++i) {
ObBackupPieceInfo piece_info;
if (1 == i) {
mock_backup_piece(1, 1, 100, 200, 300, ObBackupPieceStatus::BACKUP_PIECE_FROZEN, piece_info);
} else if (2 == i) {
mock_backup_piece(1, 2, 300, 500, 600, ObBackupPieceStatus::BACKUP_PIECE_FREEZING, piece_info);
} else if (3 == i) {
mock_backup_piece(1, 3, 600, 1000, 1100, ObBackupPieceStatus::BACKUP_PIECE_ACTIVE, piece_info);
}
if (OB_FAIL(piece_list.push_back(piece_info))) {
LOG_WARN("failed to push back", KR(ret));
}
}
return ret;
}
int mock_different_round_piece_list(common::ObArray<share::ObBackupPieceInfo>& piece_list)
{
int ret = OB_SUCCESS;
piece_list.reset();
for (int64_t i = 1; OB_SUCC(ret) && i <= 3; ++i) {
ObBackupPieceInfo piece_info;
if (1 == i) {
mock_backup_piece(1, 1, 100, 200, 300, ObBackupPieceStatus::BACKUP_PIECE_FROZEN, piece_info);
} else if (2 == i) {
mock_backup_piece(2, 2, 300, 500, 600, ObBackupPieceStatus::BACKUP_PIECE_FREEZING, piece_info);
} else if (3 == i) {
mock_backup_piece(3, 3, 600, 1000, 1100, ObBackupPieceStatus::BACKUP_PIECE_ACTIVE, piece_info);
}
if (OB_FAIL(piece_list.push_back(piece_info))) {
LOG_WARN("failed to push back", KR(ret));
}
}
return ret;
}
// case1:
// F I I
// 1 2 3
TEST(ObMultiBackupDestUtil, case1)
{
int ret = OB_SUCCESS;
const int64_t restore_timestamp = 600; // change restore timestamp
bool is_complete = false;
ObArray<ObBackupSetFileInfo> set_list;
ObArray<ObBackupPieceInfo> piece_list;
ret = mock_normal_full_and_inc_backup_set_list(set_list);
ASSERT_EQ(OB_SUCCESS, ret);
ret = mock_normal_round_piece_list(piece_list);
ASSERT_EQ(OB_SUCCESS, ret);
ret = ObMultiBackupDestUtil::check_multi_path_is_complete(restore_timestamp, set_list, piece_list, is_complete);
ASSERT_EQ(OB_SUCCESS, ret);
}
// I I
// 1 2 3
TEST(ObMultiBackupDestUtil, case2)
{
int ret = OB_SUCCESS;
const int64_t restore_timestamp = 1000;
bool is_complete = false;
ObArray<ObBackupSetFileInfo> set_list;
ObArray<ObBackupPieceInfo> piece_list;
ret = mock_only_inc_backup_set_list(2, set_list);
ASSERT_EQ(OB_SUCCESS, ret);
ret = mock_normal_round_piece_list(piece_list);
ASSERT_EQ(OB_SUCCESS, ret);
ret = ObMultiBackupDestUtil::check_multi_path_is_complete(restore_timestamp, set_list, piece_list, is_complete);
ASSERT_EQ(OB_INVALID_ARGUMENT, ret);
}
// case 3: missing one incremental backup set
// F X I
// 1 2 3
TEST(ObMultiBackupDestUtil, case3)
{
int ret = OB_SUCCESS;
const int64_t restore_timestamp = 1000;
bool is_complete = false;
ObArray<ObBackupSetFileInfo> set_list;
ObArray<ObBackupPieceInfo> piece_list;
ret = mock_full_and_inc_discontinuous_backup_set_list(set_list);
ASSERT_EQ(OB_SUCCESS, ret);
ret = mock_normal_round_piece_list(piece_list);
ASSERT_EQ(OB_SUCCESS, ret);
ret = ObMultiBackupDestUtil::check_multi_path_is_complete(restore_timestamp, set_list, piece_list, is_complete);
ASSERT_EQ(OB_INVALID_ARGUMENT, ret);
}
// case 4: one incremntal backup set is failed
// F X I
// 1 2 3
TEST(ObMultiBackupDestUtil, case4)
{
int ret = OB_SUCCESS;
const int64_t restore_timestamp = 1000;
bool is_complete = false;
ObArray<ObBackupSetFileInfo> set_list;
ObArray<ObBackupPieceInfo> piece_list;
ret = mock_full_and_inc_discontinuous_backup_set_list_with_failed(set_list);
ASSERT_EQ(OB_SUCCESS, ret);
ret = mock_normal_round_piece_list(piece_list);
ASSERT_EQ(OB_SUCCESS, ret);
ret = ObMultiBackupDestUtil::check_multi_path_is_complete(restore_timestamp, set_list, piece_list, is_complete);
ASSERT_EQ(OB_INVALID_ARGUMENT, ret);
}
// case 5: piece id not continuous
// F
// 1 3
TEST(ObMultiBackupDestUtil, case5)
{
int ret = OB_SUCCESS;
const int64_t restore_timestamp = 1000;
bool is_complete = false;
ObArray<ObBackupSetFileInfo> set_list;
ObArray<ObBackupPieceInfo> piece_list;
ret = mock_only_full_backup_set(1, set_list);
ASSERT_EQ(OB_SUCCESS, ret);
ret = mock_discontinuous_piece_list(piece_list);
ASSERT_EQ(OB_SUCCESS, ret);
ret = ObMultiBackupDestUtil::check_multi_path_is_complete(restore_timestamp, set_list, piece_list, is_complete);
ASSERT_EQ(OB_INVALID_ARGUMENT, ret);
}
// case 6: duplicate inc backup set id
TEST(ObMultiBackupDestUtil, case6)
{
int ret = OB_SUCCESS;
const int64_t restore_timestamp = 700;
bool is_complete = false;
ObArray<ObBackupSetFileInfo> set_list;
ObArray<ObBackupPieceInfo> piece_list;
ret = mock_duplicate_inc_backup_set_list(set_list);
ASSERT_EQ(OB_SUCCESS, ret);
ret = mock_normal_round_piece_list(piece_list);
ASSERT_EQ(OB_SUCCESS, ret);
ret = ObMultiBackupDestUtil::check_multi_path_is_complete(restore_timestamp, set_list, piece_list, is_complete);
ASSERT_EQ(OB_INVALID_ARGUMENT, ret);
}
// case 7: duplicate backup piece id
TEST(ObMultiBackupDestUtil, case7)
{
int ret = OB_SUCCESS;
const int64_t restore_timestamp = 700;
bool is_complete = false;
ObArray<ObBackupSetFileInfo> set_list;
ObArray<ObBackupPieceInfo> piece_list;
ret = mock_normal_full_and_inc_backup_set_list(set_list);
ASSERT_EQ(OB_SUCCESS, ret);
ret = mock_duplicate_piece_list(piece_list);
ASSERT_EQ(OB_SUCCESS, ret);
ret = ObMultiBackupDestUtil::check_multi_path_is_complete(restore_timestamp, set_list, piece_list, is_complete);
ASSERT_EQ(OB_INVALID_ARGUMENT, ret);
}
// case 8: has freezing backup piece
TEST(ObMultiBackupDestUtil, case8)
{
int ret = OB_SUCCESS;
const int64_t restore_timestamp = 700;
bool is_complete = false;
ObArray<ObBackupSetFileInfo> set_list;
ObArray<ObBackupPieceInfo> piece_list;
ret = mock_normal_full_and_inc_backup_set_list(set_list);
ASSERT_EQ(OB_SUCCESS, ret);
ret = mock_mid_freezing_piece_list(piece_list);
ASSERT_EQ(OB_SUCCESS, ret);
ret = ObMultiBackupDestUtil::check_multi_path_is_complete(restore_timestamp, set_list, piece_list, is_complete);
ASSERT_EQ(OB_INVALID_ARGUMENT, ret);
}
// case 9: piece from different round
TEST(ObMultiBackupDestUtil, case9)
{
int ret = OB_SUCCESS;
const int64_t restore_timestamp = 700;
bool is_complete = false;
ObArray<ObBackupSetFileInfo> set_list;
ObArray<ObBackupPieceInfo> piece_list;
ret = mock_normal_full_and_inc_backup_set_list(set_list);
ASSERT_EQ(OB_SUCCESS, ret);
ret = mock_different_round_piece_list(piece_list);
ASSERT_EQ(OB_SUCCESS, ret);
ret = ObMultiBackupDestUtil::check_multi_path_is_complete(restore_timestamp, set_list, piece_list, is_complete);
ASSERT_EQ(OB_INVALID_ARGUMENT, ret);
}
// case 10: restore timestamp too small
TEST(ObMultiBackupDestUtil, case10)
{
int ret = OB_SUCCESS;
const int64_t restore_timestamp = 299;
bool is_complete = false;
ObArray<ObBackupSetFileInfo> set_list;
ObArray<ObBackupPieceInfo> piece_list;
ret = mock_normal_full_and_inc_backup_set_list(set_list);
ASSERT_EQ(OB_SUCCESS, ret);
ret = mock_normal_round_piece_list(piece_list);
ASSERT_EQ(OB_SUCCESS, ret);
ret = ObMultiBackupDestUtil::check_multi_path_is_complete(restore_timestamp, set_list, piece_list, is_complete);
ASSERT_EQ(OB_INVALID_ARGUMENT, ret);
}
// case 11: restore timestamp too large
TEST(ObMultiBackupDestUtil, case11)
{
int ret = OB_SUCCESS;
const int64_t restore_timestamp = 1000;
bool is_complete = false;
ObArray<ObBackupSetFileInfo> set_list;
ObArray<ObBackupPieceInfo> piece_list;
ret = mock_normal_full_and_inc_backup_set_list(set_list);
ASSERT_EQ(OB_SUCCESS, ret);
ret = mock_normal_round_piece_list(piece_list);
ASSERT_EQ(OB_SUCCESS, ret);
ret = ObMultiBackupDestUtil::check_multi_path_is_complete(restore_timestamp, set_list, piece_list, is_complete);
ASSERT_EQ(OB_INVALID_ARGUMENT, ret);
}
int main(int argc, char** argv)
{
OB_LOGGER.set_log_level("INFO");
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}