hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix tenant level backup restore failure

Browse Source
This commit is contained in:
oceanoverflow
2022-03-02 10:34:02 +08:00
committed by LINxiansheng
parent 42d9540ff9
commit 875077c242
4 changed files with 135 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

128
src/share/backup/ob_backup_info_mgr.cpp
View File

@ -1142,60 +1142,25 @@ int ObRestoreBackupInfoUtil::get_restore_backup_info_v2_(
const GetRestoreBackupInfoParam &param, ObRestoreBackupInfo &info)
{
int ret = OB_SUCCESS;
ObFakeBackupLeaseService fake_backup_lease;
ObExternSingleBackupSetInfoMgr backup_info_mgr;
ObLogArchiveBackupInfoMgr log_archive_backup_info_mgr;
ObExternTenantLocalityInfoMgr tenant_locality_info_mgr;
ObExternPGListMgr pg_list_mgr;
ObSimpleBackupSetPath simple_set_path; // largest backup set path
ObSimpleBackupPiecePath simple_piece_path; // smallest backup piece path
ObBackupPath piece_backup_path;
const char *backup_dest = param.backup_dest_;
const char *backup_cluster_name = param.backup_cluster_name_;
const int64_t cluster_id = param.cluster_id_;
const int64_t incarnation = param.incarnation_;
const int64_t restore_timestamp = param.restore_timestamp_;
const char *passwd_array = param.passwd_array_;
const int64_t cluster_version = ObClusterVersion::get_instance().get_cluster_version();
ObExternLogArchiveBackupInfo log_archive_backup_info;
ObTenantLogArchiveStatus log_archive_status;
ObExternTenantLocalityInfo tenant_locality_info;
ObBackupPieceInfo piece_info;
ObBackupSetFileInfo backup_set_info;
if (OB_FAIL(param.get_smallest_backup_piece_path(simple_piece_path))) {
LOG_WARN("failed to get smallest simple backup piece path", K(ret));
} else if (OB_FAIL(param.get_largest_backup_set_path(simple_set_path))) {
LOG_WARN("failed to get smallest largest backup set path", K(ret));
} else if (OB_FAIL(backup_info_mgr.init(simple_set_path, fake_backup_lease))) {
LOG_WARN("failed to init backup info mgr", K(ret), K(simple_set_path));
} else if (OB_FAIL(backup_info_mgr.get_extern_backup_set_file_info(passwd_array, backup_set_info))) {
LOG_WARN("failed to find backup info", K(ret), K(restore_timestamp));
} else if (OB_FAIL(piece_backup_path.init(simple_piece_path.get_simple_path()))) {
LOG_WARN("failed to init piece backup path", K(ret));
} else if (OB_FAIL(piece_backup_path.join(OB_STR_TENANT_CLOG_SINGLE_BACKUP_PIECE_INFO))) {
LOG_WARN("failed to join single backup piece info");
} else if (OB_FAIL(log_archive_backup_info_mgr.read_external_single_backup_piece_info(
piece_backup_path, simple_piece_path.get_storage_info(), piece_info, fake_backup_lease))) {
LOG_WARN("failed to read external single backup piece info", K(ret), K(piece_backup_path), K(simple_piece_path));
} else if (backup_set_info.snapshot_version_ < piece_info.start_ts_) {
ObExternTenantLocalityInfo tenant_locality_info;
bool is_snapshot_restore = false;
if (OB_FAIL(
inner_get_restore_backup_set_info_(param, backup_set_info, tenant_locality_info, info.sys_pg_key_list_))) {
LOG_WARN("failed to inner get restore backup set info", K(ret));
} else if (OB_FAIL(check_is_snapshot_restore_(param, backup_set_info, is_snapshot_restore))) {
LOG_WARN("failed to check is snapshot backup", K(ret), K(backup_set_info));
} else if (!is_snapshot_restore && OB_FAIL(inner_get_restore_backup_piece_info_(param, piece_info))) {
LOG_WARN("failed to inner get restore backup piece info", K(ret));
} else if (!is_snapshot_restore && backup_set_info.snapshot_version_ < piece_info.start_ts_) {
ret = OB_ISOLATED_BACKUP_SET;
LOG_WARN("log archive status is not continues with backup info", K(ret), K(backup_set_info));
} else if (OB_FAIL(tenant_locality_info_mgr.init(simple_set_path, fake_backup_lease))) {
LOG_WARN("failed to init tenant locality info mgr", K(ret), K(simple_set_path));
} else if (OB_FAIL(tenant_locality_info_mgr.get_extern_tenant_locality_info(tenant_locality_info))) {
LOG_WARN("failed to find tenant locality info", K(ret));
} else if (backup_set_info.cluster_version_ > cluster_version) {
ret = OB_NOT_SUPPORTED;
LOG_ERROR("cannot restore from newer cluster to older cluster", K(ret), K(cluster_version), K(backup_set_info));
} else if (OB_FAIL(pg_list_mgr.init(simple_set_path, fake_backup_lease))) {
LOG_WARN("failed to get sys pg list", K(ret), K(backup_set_info), K(simple_set_path));
} else if (OB_FAIL(pg_list_mgr.get_sys_pg_list(info.sys_pg_key_list_))) {
LOG_WARN("failed to get sys pg list", K(ret), K(backup_set_info));
} else if (info.sys_pg_key_list_.empty()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("sys pg key list should not be empty", K(ret), K(backup_set_info));
} else {
info.compat_mode_ = tenant_locality_info.compat_mode_;
info.frozen_data_version_ = backup_set_info.backup_data_version_;
@ -1214,15 +1179,86 @@ int ObRestoreBackupInfoUtil::get_restore_backup_info_v2_(
info.physical_restore_info_.inc_backup_set_id_ = backup_set_info.backup_set_id_;
info.physical_restore_info_.incarnation_ = incarnation;
info.physical_restore_info_.tenant_id_ = tenant_locality_info.tenant_id_;
info.physical_restore_info_.log_archive_round_ = piece_info.key_.round_id_;
info.physical_restore_info_.compatible_ = backup_set_info.compatible_;
info.physical_restore_info_.cluster_version_ = backup_set_info.cluster_version_;
info.physical_restore_info_.backup_date_ = backup_set_info.date_;
if (!is_snapshot_restore) {
info.physical_restore_info_.log_archive_round_ = piece_info.key_.round_id_;
}
FLOG_INFO("get_restore_backup_info", K(info), K(piece_info), K(backup_set_info));
}
return ret;
}
int ObRestoreBackupInfoUtil::check_is_snapshot_restore_(
const GetRestoreBackupInfoParam &param, const ObBackupSetFileInfo &backup_set_info, bool &is_snapshot_restore)
{
int ret = OB_SUCCESS;
is_snapshot_restore = false;
if (param.backup_piece_path_list_.empty() && !param.backup_set_path_list_.empty()) {
is_snapshot_restore = backup_set_info.snapshot_version_ == param.restore_timestamp_;
LOG_INFO("check is snapshot backup", K(backup_set_info.snapshot_version_), K(param.restore_timestamp_));
}
return ret;
}
int ObRestoreBackupInfoUtil::inner_get_restore_backup_set_info_(const GetRestoreBackupInfoParam &param,
ObBackupSetFileInfo &backup_set_info, ObExternTenantLocalityInfo &tenant_locality_info,
common::ObIArray<common::ObPGKey> &sys_pg_key_list)
{
int ret = OB_SUCCESS;
ObFakeBackupLeaseService fake_backup_lease;
ObExternPGListMgr pg_list_mgr;
ObExternSingleBackupSetInfoMgr backup_info_mgr;
ObExternTenantLocalityInfoMgr tenant_locality_info_mgr;
const char *passwd_array = param.passwd_array_;
const int64_t cluster_version = ObClusterVersion::get_instance().get_cluster_version();
ObSimpleBackupSetPath simple_set_path; // largest backup set path
if (OB_FAIL(param.get_largest_backup_set_path(simple_set_path))) {
LOG_WARN("failed to get smallest largest backup set path", K(ret));
} else if (OB_FAIL(backup_info_mgr.init(simple_set_path, fake_backup_lease))) {
LOG_WARN("failed to init backup info mgr", K(ret), K(simple_set_path));
} else if (OB_FAIL(backup_info_mgr.get_extern_backup_set_file_info(passwd_array, backup_set_info))) {
LOG_WARN("failed to find backup info", K(ret), K(passwd_array));
} else if (OB_FAIL(tenant_locality_info_mgr.init(simple_set_path, fake_backup_lease))) {
LOG_WARN("failed to init tenant locality info mgr", K(ret), K(simple_set_path));
} else if (OB_FAIL(tenant_locality_info_mgr.get_extern_tenant_locality_info(tenant_locality_info))) {
LOG_WARN("failed to find tenant locality info", K(ret));
} else if (backup_set_info.cluster_version_ > cluster_version) {
ret = OB_NOT_SUPPORTED;
LOG_ERROR("cannot restore from newer cluster to older cluster", K(ret), K(cluster_version), K(backup_set_info));
} else if (OB_FAIL(pg_list_mgr.init(simple_set_path, fake_backup_lease))) {
LOG_WARN("failed to get sys pg list", K(ret), K(backup_set_info), K(simple_set_path));
} else if (OB_FAIL(pg_list_mgr.get_sys_pg_list(sys_pg_key_list))) {
LOG_WARN("failed to get sys pg list", K(ret), K(backup_set_info));
} else if (sys_pg_key_list.empty()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("sys pg key list should not be empty", K(ret), K(backup_set_info));
}
return ret;
}
int ObRestoreBackupInfoUtil::inner_get_restore_backup_piece_info_(
const GetRestoreBackupInfoParam &param, ObBackupPieceInfo &piece_info)
{
int ret = OB_SUCCESS;
ObFakeBackupLeaseService fake_backup_lease;
ObLogArchiveBackupInfoMgr log_archive_backup_info_mgr;
ObSimpleBackupPiecePath simple_piece_path; // smallest backup piece path
ObBackupPath piece_backup_path;
if (OB_FAIL(param.get_smallest_backup_piece_path(simple_piece_path))) {
LOG_WARN("failed to get smallest simple backup piece path", K(ret));
} else if (OB_FAIL(piece_backup_path.init(simple_piece_path.get_simple_path()))) {
LOG_WARN("failed to init piece backup path", K(ret));
} else if (OB_FAIL(piece_backup_path.join(OB_STR_TENANT_CLOG_SINGLE_BACKUP_PIECE_INFO))) {
LOG_WARN("failed to join single backup piece info");
} else if (OB_FAIL(log_archive_backup_info_mgr.read_external_single_backup_piece_info(
piece_backup_path, simple_piece_path.get_storage_info(), piece_info, fake_backup_lease))) {
LOG_WARN("failed to read external single backup piece info", K(ret), K(piece_backup_path), K(simple_piece_path));
}
return ret;
}
int ObRestoreBackupInfoUtil::get_restore_sys_table_ids(
const ObPhysicalRestoreInfo &info, common::ObIArray<common::ObPartitionKey> &pkey_list)
{

8
src/share/backup/ob_backup_info_mgr.h
View File

@ -45,7 +45,6 @@ struct ObLogArchiveSimpleInfo final {
K_(cur_piece_create_date), K_(is_piece_freezing), K_(prev_piece_id), K_(prev_piece_create_date));
};
// 用于切主后确认pg的checkpoint ts，可能会触发内部表的访问。
class ObLogArchiveInfoMgr final {
public:
static ObLogArchiveInfoMgr &get_instance();
@ -171,6 +170,13 @@ private:
static int get_restore_backup_info_v1_(const GetRestoreBackupInfoParam &param, ObRestoreBackupInfo &info);
// get info from simple path level
static int get_restore_backup_info_v2_(const GetRestoreBackupInfoParam &param, ObRestoreBackupInfo &info);
static int check_is_snapshot_restore_(
const GetRestoreBackupInfoParam &param, const ObBackupSetFileInfo &backup_set_info, bool &is_snapshot_restore);
static int inner_get_restore_backup_set_info_(const GetRestoreBackupInfoParam &param,
ObBackupSetFileInfo &backup_set_info, ObExternTenantLocalityInfo &tenant_locality_info,
common::ObIArray<common::ObPGKey> &sys_pg_keys);
static int inner_get_restore_backup_piece_info_(
const GetRestoreBackupInfoParam &param, ObBackupPieceInfo &piece_info);
};
class ObRestoreFatalErrorReporter : public share::ObThreadPool {

49
src/share/backup/ob_multi_backup_dest_util.cpp
View File

@ -157,6 +157,7 @@ int ObMultiBackupDestUtil::get_multi_backup_path_list(const bool is_preview, con
int64_t snapshot_version = 0;
int64_t start_replay_log_ts = 0;
bool is_compat_path = false;
bool is_snapshot_restore = false;
if (OB_FAIL(get_backup_set_list(is_preview,
cluster_name,
cluster_id,
@ -166,8 +167,11 @@ int ObMultiBackupDestUtil::get_multi_backup_path_list(const bool is_preview, con
set_list,
snapshot_version,
start_replay_log_ts,
is_compat_path))) {
is_compat_path,
is_snapshot_restore))) {
LOG_WARN("failed to get backup set", KR(ret), K(tenant_id), K(restore_timestamp), K(list));
} else if (is_snapshot_restore) {
LOG_INFO("tenant level backup no need get piece list", K(tenant_id));
} else {
if (is_compat_path) {
if (OB_FAIL(get_compat_backup_piece_list(cluster_name,
@ -675,7 +679,7 @@ int ObMultiBackupDestUtil::get_cluster_backup_dest(const ObBackupDest &backup_de
int ObMultiBackupDestUtil::get_backup_set_list(const bool is_preview, const char *cluster_name,
const int64_t cluster_id, const uint64_t tenant_id, const int64_t restore_timestamp,
const common::ObString &backup_dest_str, common::ObArray<ObSimpleBackupSetPath> &list, int64_t &snapshot_version,
int64_t &start_replay_log_ts, bool &is_compat_path)
int64_t &start_replay_log_ts, bool &is_compat_path, bool &is_snapshot_restore)
{
int ret = OB_SUCCESS;
snapshot_version = 0;
@ -703,10 +707,13 @@ int ObMultiBackupDestUtil::get_backup_set_list(const bool is_preview, const char
tmp_list,
snapshot_version,
start_replay_log_ts,
is_compat_path))) {
is_compat_path,
is_snapshot_restore))) {
LOG_WARN("failed to inner get backup set list", KR(ret), K(tenant_id), K(backup_dest));
} else if (OB_FAIL(append(list, tmp_list))) {
LOG_WARN("failed to add array", KR(ret), K(tmp_list));
} else {
LOG_INFO("get backup set list", K(list));
}
return ret;
}
@ -714,7 +721,7 @@ int ObMultiBackupDestUtil::get_backup_set_list(const bool is_preview, const char
int ObMultiBackupDestUtil::do_get_backup_set_list(const bool is_preview, const char *cluster_name,
const int64_t cluster_id, const uint64_t tenant_id, const int64_t restore_timestamp,
const ObBackupDest &backup_dest, common::ObArray<ObSimpleBackupSetPath> &path_list, int64_t &snapshot_version,
int64_t &start_replay_log_ts, bool &is_compat_path)
int64_t &start_replay_log_ts, bool &is_compat_path, bool &is_snapshot_restore)
{
int ret = OB_SUCCESS;
path_list.reset();
@ -744,7 +751,8 @@ int ObMultiBackupDestUtil::do_get_backup_set_list(const bool is_preview, const c
backup_dest,
path_list,
snapshot_version,
start_replay_log_ts))) {
start_replay_log_ts,
is_snapshot_restore))) {
LOG_WARN("failed to do get cluster level backup set list",
KR(ret),
K(is_preview),
@ -768,6 +776,8 @@ int ObMultiBackupDestUtil::do_get_backup_set_list(const bool is_preview, const c
} else {
LOG_WARN("failed to get extern backup set file infos", KR(ret), K(simple_path));
}
} else if (OB_FAIL(check_is_snapshot_restore(info, restore_timestamp, is_snapshot_restore))) {
LOG_WARN("failed to check is snapshot backup", K(ret), K(info));
} else {
simple_path.backup_set_id_ = info.backup_set_id_;
simple_path.copy_id_ = info.copy_id_;
@ -787,7 +797,7 @@ int ObMultiBackupDestUtil::do_get_backup_set_list(const bool is_preview, const c
int ObMultiBackupDestUtil::do_get_backup_set_list_from_cluster_level(const bool is_preview, const char *cluster_name,
const int64_t cluster_id, const uint64_t tenant_id, const int64_t restore_timestamp,
const ObBackupDest &backup_dest, common::ObArray<ObSimpleBackupSetPath> &path_list, int64_t &snapshot_version,
int64_t &start_replay_log_ts)
int64_t &start_replay_log_ts, bool &is_snapshot_restore)
{
int ret = OB_SUCCESS;
ObClusterBackupDest cluster_backup_dest;
@ -823,7 +833,8 @@ int ObMultiBackupDestUtil::do_get_backup_set_list_from_cluster_level(const bool
file_infos,
path_list,
snapshot_version,
start_replay_log_ts))) {
start_replay_log_ts,
is_snapshot_restore))) {
LOG_WARN("failed to do inner get backup set list", KR(ret), K(restore_timestamp));
} else {
std::sort(path_list.begin(), path_list.end(), cmp_set);
@ -869,8 +880,10 @@ int ObMultiBackupDestUtil::do_get_backup_set_list_from_cluster_level(const bool
ObClusterBackupDest cluster_backup_dest;
ObBackupDest tmp_backup_dest;
char tmp_simple_path_str[OB_MAX_BACKUP_DEST_LENGTH] = "";
if (OB_FAIL(cluster_backup_dest.set(
tmp_file_info.backup_dest_.ptr(), cluster_name, cluster_id, OB_START_INCARNATION))) {
if (OB_FAIL(check_is_snapshot_restore(tmp_file_info, restore_timestamp, is_snapshot_restore))) {
LOG_WARN("failed to check is snaptshot backup", K(ret), K(tmp_file_info));
} else if (OB_FAIL(cluster_backup_dest.set(
tmp_file_info.backup_dest_.ptr(), cluster_name, cluster_id, OB_START_INCARNATION))) {
LOG_WARN("failed to set cluster backup dest");
} else if (OB_FAIL(base_data_path_info.set(cluster_backup_dest,
tmp_file_info.tenant_id_,
@ -908,7 +921,8 @@ int ObMultiBackupDestUtil::do_get_backup_set_list_from_cluster_level(const bool
int ObMultiBackupDestUtil::do_inner_get_backup_set_list(const char *cluster_name, const int64_t cluster_id,
const int64_t restore_timestamp, const ObBackupDest &backup_dest, const ObArray<ObBackupSetFileInfo> &file_infos,
common::ObArray<ObSimpleBackupSetPath> &path_list, int64_t &snapshot_version, int64_t &start_replay_log_ts)
common::ObArray<ObSimpleBackupSetPath> &path_list, int64_t &snapshot_version, int64_t &start_replay_log_ts,
bool &is_snapshot_restore)
{
int ret = OB_SUCCESS;
snapshot_version = -1;
@ -939,6 +953,8 @@ int ObMultiBackupDestUtil::do_inner_get_backup_set_list(const char *cluster_name
const ObBackupSetFileInfo &info = file_infos.at(i);
if (OB_SUCCESS != info.result_) {
// do nothing
} else if (OB_FAIL(check_is_snapshot_restore(info, restore_timestamp, is_snapshot_restore))) {
LOG_WARN("failed to check is snaptshot backup", K(ret), K(info), K(restore_timestamp));
} else if (!ObBackupFileStatus::can_show_in_preview(info.file_status_)) {
LOG_INFO("backup set info cannot list in preview", K(info));
} else {
@ -1426,5 +1442,18 @@ int ObMultiBackupDestUtil::check_backup_path_is_backup_backup(const char *cluste
return ret;
}
int ObMultiBackupDestUtil::check_is_snapshot_restore(
const share::ObBackupSetFileInfo &backup_set_info, const int64_t restore_timestamp, bool &is_snapshot_restore)
{
int ret = OB_SUCCESS;
if (!backup_set_info.is_valid() || restore_timestamp <= 0) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("get invalid args", K(ret), K(backup_set_info), K(restore_timestamp));
} else {
is_snapshot_restore = backup_set_info.snapshot_version_ == restore_timestamp;
}
return ret;
}
} // end namespace share
} // end namespace oceanbase

11
src/share/backup/ob_multi_backup_dest_util.h
View File

@ -89,18 +89,19 @@ private:
static int get_backup_set_list(const bool is_preview, const char *cluster_name, const int64_t cluster_id,
const uint64_t tenant_id, const int64_t restore_timestamp, const common::ObString &backup_dest_str,
common::ObArray<ObSimpleBackupSetPath> &path_list, int64_t &snapshot_version, int64_t &start_replay_log_ts,
bool &is_compat_path);
bool &is_compat_path, bool &is_snapshot_restore);
static int do_get_backup_set_list(const bool is_preview, const char *cluster_name, const int64_t cluster_id,
const uint64_t tenant_id, const int64_t restore_timestamp, const ObBackupDest &backup_dest,
common::ObArray<ObSimpleBackupSetPath> &path_list, int64_t &snapshot_version, int64_t &start_replay_log_ts,
bool &is_compat_path);
bool &is_compat_path, bool &is_snapshot_restore);
static int do_get_backup_set_list_from_cluster_level(const bool is_preview, const char *cluster_name,
const int64_t cluster_id, const uint64_t tenant_id, const int64_t restore_timestamp,
const ObBackupDest &backup_dest, common::ObArray<ObSimpleBackupSetPath> &path_list, int64_t &snapshot_version,
int64_t &start_replay_log_ts);
int64_t &start_replay_log_ts, bool &is_snapshot_restore);
static int do_inner_get_backup_set_list(const char *cluster_name, const int64_t cluster_id,
const int64_t restore_timestamp, const ObBackupDest &backup_dest, const ObArray<ObBackupSetFileInfo> &file_infos,
common::ObArray<ObSimpleBackupSetPath> &path_list, int64_t &snapshot_version, int64_t &start_replay_log_ts);
common::ObArray<ObSimpleBackupSetPath> &path_list, int64_t &snapshot_version, int64_t &start_replay_log_ts,
bool &is_snapshot_restore);
static int get_backup_piece_list(const bool is_preview, const char *cluster_name, const int64_t cluster_id,
const uint64_t tenant_id, const int64_t snapshot_version, const int64_t start_replay_log_ts,
const int64_t restore_timestamp, const common::ObString &backup_dest_str,
@ -125,6 +126,8 @@ private:
static int check_backup_path_is_backup_backup(const char *cluster_name, const int64_t cluster_id,
const common::ObString &root_path, const common::ObString &storage_info, const uint64_t tenant_id,
bool &is_backup_backup);
static int check_is_snapshot_restore(
const share::ObBackupSetFileInfo &backup_set_info, const int64_t restore_timestamp, bool &is_snapshot_restore);
private:
DISALLOW_COPY_AND_ASSIGN(ObMultiBackupDestUtil);