hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity
Files
9b31f8aa0310a472edb4ba6b06a52a93b6f003fa
oceanbase/src/rootserver/backup/ob_backup_clean_task_mgr.cpp
wenxingsen 9b31f8aa03 [FEAT MERGE] 开源裁减颗粒度优化 
Co-authored-by: nroskill <nroskill@gmail.com>
Co-authored-by: akaError <lzg020616@163.com>
Co-authored-by: yinyj17 <yinyijun92@gmail.com>
2023-08-15 02:40:26 +00:00

1288 lines
51 KiB
C++
Raw Blame History

/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX RS
#include "ob_backup_clean_task_mgr.h"
#include "ob_backup_clean_ls_task_mgr.h"
#include "storage/ls/ob_ls.h"
#include "storage/backup/ob_backup_data_store.h"
#include "share/backup/ob_backup_path.h"
#include "share/backup/ob_backup_struct.h"
#include "share/backup/ob_backup_clean_struct.h"
#include "share/backup/ob_backup_clean_operator.h"
#include "share/backup/ob_archive_persist_helper.h"
#include "share/backup/ob_backup_clean_util.h"
#include "lib/utility/utility.h"
#include "share/backup/ob_backup_io_adapter.h"
#include "share/backup/ob_archive_path.h"
#include "share/backup/ob_archive_store.h"
#include "share/backup/ob_backup_connectivity.h"
#include "share/ob_debug_sync_point.h"
#include "lib/utility/ob_tracepoint.h"
using namespace oceanbase;
using namespace rootserver;
using namespace share;
//****************************ObBackupCleanTaskMgr**********************
ObBackupCleanTaskMgr::ObBackupCleanTaskMgr()
: is_inited_(false),
tenant_id_(OB_INVALID_TENANT_ID),
backup_set_info_(),
backup_piece_info_(),
task_attr_(),
backup_dest_(),
job_attr_(nullptr),
sql_proxy_(nullptr),
rpc_proxy_(nullptr),
task_scheduler_(nullptr),
backup_service_(nullptr)
{
}
int ObBackupCleanTaskMgr::init(
const uint64_t tenant_id,
const int64_t task_id,
ObBackupCleanJobAttr &job_attr,
common::ObISQLClient &sql_proxy,
obrpc::ObSrvRpcProxy &rpc_proxy,
ObBackupTaskScheduler &task_scheduler,
ObBackupCleanService &backup_service)
{
int ret = OB_SUCCESS;
if (IS_INIT) {
ret = OB_INIT_TWICE;
LOG_WARN("init twice", K(ret));
} else if (OB_FAIL(ObBackupCleanTaskOperator::get_backup_clean_task(sql_proxy, task_id, job_attr.tenant_id_, false/* need_lock */, task_attr_))) {
LOG_WARN("failed to get backup task", K(ret), "job_id", job_attr.job_id_, "tenant_id", job_attr.tenant_id_);
} else if (!task_attr_.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(task_attr_));
} else if (task_attr_.is_delete_backup_set_task()) {
if (OB_FAIL(ObBackupSetFileOperator::get_backup_set_file(sql_proxy, false/* need_lock */, task_attr_.backup_set_id_,
task_attr_.incarnation_id_, task_attr_.tenant_id_, task_attr_.dest_id_, backup_set_info_))) {
LOG_WARN("failed to get backup set file", K(ret), K(task_attr_));
} else if (!backup_set_info_.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(task_attr_));
} else if (OB_FAIL(ObBackupStorageInfoOperator::get_backup_dest(sql_proxy, job_attr.tenant_id_, backup_set_info_.backup_path_, backup_dest_))) {
LOG_WARN("failed to get backup dest", K(ret), K(backup_set_info_));
}
} else if (task_attr_.is_delete_backup_piece_task()) {
ObArchivePersistHelper archive_table_op;
if (OB_FAIL(archive_table_op.init(job_attr.tenant_id_))) {
LOG_WARN("failed to init archive piece attr", K(ret));
} else if (OB_FAIL(archive_table_op.get_piece(sql_proxy, task_attr_.dest_id_, task_attr_.round_id_,
task_attr_.backup_piece_id_, false/* need_lock */, backup_piece_info_))) {
LOG_WARN("failed to get backup piece", K(ret), K(task_attr_));
} else if(!backup_piece_info_.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(task_attr_));
} else if (OB_FAIL(ObBackupStorageInfoOperator::get_backup_dest(sql_proxy, job_attr.tenant_id_, backup_piece_info_.path_, backup_dest_))) {
LOG_WARN("failed to get backup dest", K(ret), K(backup_piece_info_));
}
}
if (OB_SUCC(ret)) {
tenant_id_ = tenant_id;
job_attr_ = &job_attr;
sql_proxy_ = &sql_proxy;
rpc_proxy_ = &rpc_proxy;
task_scheduler_ = &task_scheduler;
backup_service_ = &backup_service;
is_inited_ = true;
}
// TODO(wenjinyu.wjy): integrate sql_proxy and lease_service 4.3
return ret;
}
int ObBackupCleanTaskMgr::advance_task_status_(
common::ObISQLClient &sql_proxy,
const ObBackupCleanStatus &next_status,
const int result,
const int64_t end_ts)
{
int ret = OB_SUCCESS;
if (OB_FAIL(backup_service_->check_leader())) {
LOG_WARN("failed to check leader", K(ret));
} else if (OB_FAIL(ObBackupCleanTaskOperator::advance_task_status(sql_proxy, task_attr_, next_status, result, end_ts))) {
LOG_WARN("failed to advance set status", K(ret), K(task_attr_), K(next_status));
}
return ret;
}
int ObBackupCleanTaskMgr::mark_backup_set_files_deleting_()
{
int ret = OB_SUCCESS;
ObBackupFileStatus::STATUS file_status = ObBackupFileStatus::STATUS::BACKUP_FILE_DELETING;
if (!backup_set_info_.is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("backup set info is valid", K(ret));
} else if (backup_set_info_.file_status_ == file_status) {
// do nothing
} else if (OB_FAIL(ObBackupFileStatus::check_can_change_status(backup_set_info_.file_status_, file_status))) {
LOG_WARN("failed to check can change status", K(ret));
} else if (FALSE_IT(backup_set_info_.file_status_ = file_status)) {
} else if (OB_FAIL(ObBackupSetFileOperator::update_backup_set_file_status(*sql_proxy_, backup_set_info_))) {
LOG_WARN("failed to update backup set file_status", K(ret), K(file_status));
}
return ret;
}
int ObBackupCleanTaskMgr::mark_backup_set_files_deleted_()
{
int ret = OB_SUCCESS;
ObBackupFileStatus::STATUS file_status = ObBackupFileStatus::STATUS::BACKUP_FILE_DELETED;
if (!backup_set_info_.is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("backup set info is valid", K(ret));
} else if (OB_FAIL(ObBackupFileStatus::check_can_change_status(backup_set_info_.file_status_, file_status))) {
LOG_WARN("failed to check can change status", K(ret));
} else if (FALSE_IT(backup_set_info_.file_status_ = file_status)) {
} else if (OB_FAIL(ObBackupSetFileOperator::update_backup_set_file_status(*sql_proxy_, backup_set_info_))) {
LOG_WARN("failed to update backup set file", K(ret));
}
return ret;
}
int ObBackupCleanTaskMgr::mark_backup_piece_files_deleting_()
{
int ret = OB_SUCCESS;
ObArchivePersistHelper archive_table_op;
ObBackupFileStatus::STATUS file_status = ObBackupFileStatus::STATUS::BACKUP_FILE_DELETING;
if (!backup_piece_info_.is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("backup piece info is valid", K(ret));
} else if (backup_piece_info_.file_status_ == file_status) {
// do nothing
} else if (OB_FAIL(archive_table_op.init(task_attr_.tenant_id_))) {
LOG_WARN("failed to init archive piece attr", K(ret));
} else if (OB_FAIL(ObBackupFileStatus::check_can_change_status(backup_piece_info_.file_status_, file_status))) {
LOG_WARN("failed to check can change status", K(ret));
} else if (OB_FAIL(archive_table_op.mark_new_piece_file_status(*sql_proxy_, backup_piece_info_.key_.dest_id_,
backup_piece_info_.key_.round_id_, backup_piece_info_.key_.piece_id_, file_status))) {
LOG_WARN("failed to update backup piece file status", K(ret));
}
return ret;
}
int ObBackupCleanTaskMgr::mark_backup_piece_files_deleted_()
{
int ret = OB_SUCCESS;
ObArchivePersistHelper archive_table_op;
ObBackupFileStatus::STATUS file_status = ObBackupFileStatus::STATUS::BACKUP_FILE_DELETED;
if (!backup_piece_info_.is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("backup piece info is valid", K(ret));
} else if (OB_FAIL(archive_table_op.init(task_attr_.tenant_id_))) {
LOG_WARN("failed to init archive piece attr", K(ret));
} else if (OB_FAIL(ObBackupFileStatus::check_can_change_status(backup_piece_info_.file_status_, file_status))) {
LOG_WARN("failed to check can change status", K(ret));
} else if (FALSE_IT(backup_piece_info_.file_status_ = file_status)) {
} else if (OB_FAIL(archive_table_op.mark_new_piece_file_status(*sql_proxy_, backup_piece_info_.key_.dest_id_,
backup_piece_info_.key_.round_id_, backup_piece_info_.key_.piece_id_, file_status))) {
LOG_WARN("failed to update backup piece file status", K(ret));
}
return ret;
}
int ObBackupCleanTaskMgr::mark_backup_files_deleted_()
{
int ret = OB_SUCCESS;
if (task_attr_.is_delete_backup_set_task()) {
if (OB_FAIL(mark_backup_set_files_deleted_())) {
LOG_WARN("failed to mark backup set files deleted", K(ret));
}
} else if (task_attr_.is_delete_backup_piece_task()) {
if (OB_FAIL(mark_backup_piece_files_deleted_())) {
LOG_WARN("failed to mark backup piece files deleted", K(ret));
}
}
return ret;
}
int ObBackupCleanTaskMgr::mark_backup_files_deleting_()
{
int ret = OB_SUCCESS;
if (task_attr_.is_delete_backup_set_task()) {
if (OB_FAIL(mark_backup_set_files_deleting_())) {
LOG_WARN("failed to mark backup set files deleting", K(ret), K_(backup_set_info));
}
} else if (task_attr_.is_delete_backup_piece_task()) {
if (OB_FAIL(mark_backup_piece_files_deleting_())) {
LOG_WARN("failed to mark backup piece files deleting", K(ret), K_(backup_piece_info));
}
}
return ret;
}
// file:///backup/backup_sets/backup_set_xx_xxx_xxxx_end_xxx.obbak
int ObBackupCleanTaskMgr::try_delete_extern_set_end_file_()
{
int ret = OB_SUCCESS;
ObBackupPath end_path;
ObBackupSetDesc desc;
desc.backup_set_id_ = backup_set_info_.backup_set_id_;
desc.backup_type_ = backup_set_info_.backup_type_;
if (!backup_set_info_.is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("backup set info is valid", K(ret));
} else if (OB_SUCCESS == backup_set_info_.result_) {
if (OB_FAIL(ObBackupPathUtil::get_backup_set_placeholder_end_success_path(
backup_dest_, desc, backup_set_info_.min_restore_scn_, end_path))) {
LOG_WARN("failed to get backup set end placeholder path", K(ret));
}
} else {
if (OB_FAIL(ObBackupPathUtil::get_backup_set_placeholder_end_failed_path(
backup_dest_, desc, backup_set_info_.min_restore_scn_, end_path))) {
LOG_WARN("failed to get backup set end placeholder path", K(ret));
}
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(ObBackupCleanUtil::delete_backup_file(end_path, backup_dest_.get_storage_info()))) {
LOG_WARN("failed to delete backup file", K(ret), K(task_attr_));
}
return ret;
}
int ObBackupCleanTaskMgr::try_delete_extern_piece_end_file_()
{
int ret = OB_SUCCESS;
ObBackupPath end_path;
if (!backup_piece_info_.is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("backup piece info is invalid", K(ret), K_(backup_piece_info));
} else if (OB_FAIL(ObArchivePathUtil::get_piece_end_file_path(
backup_dest_, backup_piece_info_.key_.dest_id_, backup_piece_info_.key_.round_id_,
backup_piece_info_.key_.piece_id_, backup_piece_info_.checkpoint_scn_, end_path))) {
LOG_WARN("failed to get backup set end placeholder path", K(ret), K_(backup_piece_info));
} else if (OB_FAIL(ObBackupCleanUtil::delete_backup_file(end_path, backup_dest_.get_storage_info()))) {
LOG_WARN("failed to delete backup file", K(ret), K_(task_attr), K(end_path));
}
return ret;
}
int ObBackupCleanTaskMgr::try_delete_extern_end_file_()
{
int ret = OB_SUCCESS;
if (task_attr_.is_delete_backup_set_task()) {
if (OB_FAIL(try_delete_extern_set_end_file_())) {
LOG_WARN("failed to try delete extern set end file", K(ret));
}
} else if (task_attr_.is_delete_backup_piece_task()) {
if (OB_FAIL(try_delete_extern_piece_end_file_())) {
LOG_WARN("failed to try delete extern piece end file", K(ret));
}
}
return ret;
}
bool ObBackupCleanTaskMgr::is_can_retry_(const int error)
{
return job_attr_->can_retry_ && job_attr_->retry_count_ < OB_MAX_RETRY_TIMES && ObBackupUtils::is_need_retry_error(error);
}
int ObBackupCleanTaskMgr::prepare_backup_clean_task_()
{
int ret = OB_SUCCESS;
#ifdef ERRSIM
ret = OB_E(EventTable::EN_BACKUP_DELETE_MARK_DELETING) OB_SUCCESS;
#endif
if (OB_FAIL(ret)) {
} else if (OB_FAIL(mark_backup_files_deleting_())) {
LOG_WARN("failed to mark backup file deleting", K(ret));
} else if (OB_FAIL(try_delete_extern_end_file_())) {
LOG_WARN("failed to try delete extern end file", K(ret));
} else if (OB_FAIL(persist_ls_tasks_())) {
LOG_WARN("failed to persist log stream task", K(ret), K(task_attr_));
}
return ret;
}
int ObBackupCleanTaskMgr::process()
{
int ret = OB_SUCCESS;
FLOG_INFO("[BACKUP_CLEAN]schedule backup clean set task", K(task_attr_));
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("tenant backup data scheduler not init", K(ret));
} else {
ObBackupCleanStatus::Status status = task_attr_.status_.status_;
switch (status) {
case ObBackupCleanStatus::Status::INIT: {
if (OB_FAIL(prepare_backup_clean_task_())) {
LOG_WARN("failed to persist backup clean ls task", K(ret), K(task_attr_));
}
break;
}
case ObBackupCleanStatus::Status::DOING: {
DEBUG_SYNC(BACKUP_DELETE_TASK_STATUS_DOING);
if (OB_FAIL(backup_clean_ls_tasks_())) {
LOG_WARN("failed to backup clean ls task", K(ret), K(task_attr_));
}
break;
}
case ObBackupCleanStatus::Status::COMPLETED:
case ObBackupCleanStatus::Status::FAILED:
case ObBackupCleanStatus::Status::CANCELED: {
if (OB_FAIL(do_cleanup())) {
LOG_WARN("failed to move history", K(ret), K(task_attr_));
}
break;
}
case ObBackupCleanStatus::Status::CANCELING: {
if (OB_FAIL(do_cancel_())) {
LOG_WARN("failed to cancel backup", K(ret), K(task_attr_));
}
break;
}
default: {
ret = OB_ERR_SYS;
LOG_ERROR("unknown backup clean status", K(ret), K(task_attr_));
}
}
if (OB_FAIL(ret) && !is_can_retry_(ret)) {
if (ObBackupCleanStatus::Status::INIT == task_attr_.status_.status_
|| ObBackupCleanStatus::Status::DOING == task_attr_.status_.status_) {
int tmp_ret = OB_SUCCESS;
if (OB_SUCCESS != (tmp_ret = deal_failed_task(ret))) {
LOG_WARN("failed to advance task status", K(tmp_ret), K(ret), K(task_attr_.status_));
}
ret = (OB_SUCCESS != tmp_ret) ? tmp_ret : ret;
}
}
}
return ret;
}
int ObBackupCleanTaskMgr::persist_ls_tasks_()
{
int ret = OB_SUCCESS;
ObArray<ObLSID> ls_ids;
ObBackupCleanStatus next_status;
ObMySQLTransaction trans;
next_status.status_ = ObBackupCleanStatus::Status::DOING;
if (OB_FAIL(get_ls_ids_(ls_ids))) {
LOG_WARN("failed to get ls ids", K(ret));
} else if (OB_FAIL(trans.start(sql_proxy_, gen_meta_tenant_id(tenant_id_)))) {
LOG_WARN("failed to start trans", K(ret));
} else {
if (0 != ls_ids.count()) {
ObBackupCleanLSTaskAttr new_ls_attr;
for (int i = 0; OB_SUCC(ret) && i < ls_ids.count(); ++i) {
const ObLSID &ls_id = ls_ids.at(i);
new_ls_attr.reset();
if (OB_FAIL(generate_ls_task_(ls_id, new_ls_attr))) {
LOG_WARN("failed to generate log stream tasks", K(ret), K(ls_ids));
} else if (OB_FAIL(backup_service_->check_leader())) {
LOG_WARN("failed to check leader", K(ret));
} else if (OB_FAIL(ObBackupCleanLSTaskOperator::insert_ls_task(trans, new_ls_attr))) {
LOG_WARN("failed to insert backup log stream task", K(ret), K(new_ls_attr));
}
}
}
if (OB_FAIL(ret)) {
} else if (FALSE_IT(task_attr_.total_ls_count_ = ls_ids.count())) {
} else if (OB_FAIL(advance_task_status_(trans, next_status, task_attr_.result_, task_attr_.end_ts_))) {
LOG_WARN("failed to advance status to DOING", K(ret), K(ls_ids));
} else if (OB_FAIL(ObBackupCleanTaskOperator::update_ls_count(trans, task_attr_, true/*is_total*/))) {
LOG_WARN("failed to update ls task count", K(ret), K(task_attr_.total_ls_count_), K(ls_ids));
}
if (OB_SUCC(ret)) {
if (OB_FAIL(trans.end(true))) {
LOG_WARN("failed to commit trans", K(ret));
}
} else {
int tmp_ret = OB_SUCCESS;
if (OB_SUCCESS != (tmp_ret = trans.end(false))) {
LOG_WARN("failed to roll back status", K(ret), K(tmp_ret));
}
}
}
return ret;
}
int ObBackupCleanTaskMgr::get_set_ls_ids_from_traverse_(ObIArray<ObLSID> &ls_ids)
{
int ret = OB_SUCCESS;
ObBackupPath path;
ObBackupSetDesc desc;
desc.backup_set_id_ = task_attr_.backup_set_id_;
desc.backup_type_ = backup_set_info_.backup_type_;
if (OB_FAIL(ObBackupPathUtil::get_backup_set_dir_path(
backup_dest_, desc, path))) {
LOG_WARN("failed to get tenant data backup set dir path", K(ret), K_(backup_set_info));
} else if (OB_FAIL(get_ls_ids_from_traverse_(path, ls_ids))) {
LOG_WARN("failed to get set ls ids from traverse", K(ret), K(path));
} else {
LOG_INFO("[BACKUP_CLEAN]success get set ls ids from traverse", K(path), K(ls_ids));
}
return ret;
}
int ObBackupCleanTaskMgr::get_set_ls_ids_(ObIArray<ObLSID> &ls_ids)
{
int ret = OB_SUCCESS;
ObBackupDataStore store;
share::ObBackupSetDesc desc;
ObBackupDataLSAttrDesc ls_attr;
desc.backup_set_id_ = backup_set_info_.backup_set_id_;
desc.backup_type_ = backup_set_info_.backup_type_;
if (OB_FAIL(store.init(backup_dest_, desc))) {
LOG_WARN("faield to init backup data extern mgr", K(ret));
} else if (OB_FAIL(store.read_ls_attr_info(ls_attr))) {
if (OB_BACKUP_FILE_NOT_EXIST == ret) {
if (OB_FAIL(get_set_ls_ids_from_traverse_(ls_ids))) {
LOG_WARN("failed to get set ls ids from traverse", K(ret));
}
} else {
LOG_WARN("failed to read log stream info", K(ret));
}
} else if (!ls_attr.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid ls info", K(ret), K(ls_attr));
} else {
for (int i = 0; OB_SUCC(ret) && i < ls_attr.ls_attr_array_.count(); ++i) {
const ObLSID &ls_id = ls_attr.ls_attr_array_.at(i).get_ls_id();
if (!ls_id.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid ls info", K(ret), K(ls_id));
} else if (OB_FAIL(ls_ids.push_back(ls_id))) {
LOG_WARN("failed to push back ls_id", K(ret), K(ls_id));
}
}
}
return ret;
}
int ObBackupCleanTaskMgr::get_piece_ls_ids_from_traverse_(ObIArray<ObLSID> &ls_ids)
{
int ret = OB_SUCCESS;
ObBackupPath path;
if (OB_FAIL(ObArchivePathUtil::get_piece_dir_path(
backup_dest_, backup_piece_info_.key_.dest_id_, backup_piece_info_.key_.round_id_,
backup_piece_info_.key_.piece_id_, path))) {
LOG_WARN("failed to get piece dir path", K(ret), K_(backup_piece_info));
} else if (OB_FAIL(get_ls_ids_from_traverse_(path, ls_ids))) {
LOG_WARN("failed to get ls ids from traverse", K(ret), K(path));
}
return ret;
}
int ObBackupCleanTaskMgr::get_piece_ls_ids_(ObIArray<ObLSID> &ls_ids)
{
int ret = OB_SUCCESS;
ObBackupPath path;
ObArchiveStore store;
ObPieceInfoDesc desc;
if (OB_FAIL(store.init(backup_dest_))) {
LOG_WARN("failed to init store", K(ret), K_(task_attr));
} else if (OB_FAIL(store.read_piece_info(backup_piece_info_.key_.dest_id_, backup_piece_info_.key_.round_id_,
backup_piece_info_.key_.piece_id_, desc))) {
if (OB_BACKUP_FILE_NOT_EXIST == ret) {
if (OB_FAIL(get_piece_ls_ids_from_traverse_(ls_ids))) {
LOG_WARN("failed to get piece ls ids from traverse", K(ret));
}
} else {
LOG_WARN("failed to read piece info", K(ret), K_(task_attr), K(backup_piece_info_));
}
} else if (!desc.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid ls info", K(ret), K(desc));
} else {
for (int i = 0; OB_SUCC(ret) && i < desc.filelist_.count(); ++i) {
const ObLSID &ls_id = desc.filelist_.at(i).ls_id_;
if (!ls_id.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid ls info", K(ret), K(ls_id));
} else if (OB_FAIL(ls_ids.push_back(ls_id))) {
LOG_WARN("failed to push back ls_id", K(ret), K(ls_id));
}
}
}
return ret;
}
int ObBackupCleanTaskMgr::get_ls_ids_(ObIArray<ObLSID> &ls_ids)
{
int ret = OB_SUCCESS;
if (task_attr_.is_delete_backup_set_task()) {
if (OB_FAIL(get_set_ls_ids_(ls_ids))) {
LOG_WARN("failed to get set ls ids", K(ret), K_(task_attr), K_(backup_set_info));
}
} else if (task_attr_.is_delete_backup_piece_task()) {
if (OB_FAIL(get_piece_ls_ids_(ls_ids))) {
LOG_WARN("failed to piece ls ids", K(ret), K_(task_attr), K_(backup_piece_info));
}
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("clean task type is invalid", K(ret), K_(task_attr));
}
return ret;
}
int ObBackupCleanTaskMgr::parse_int_(const char *str, int64_t &val)
{
int ret = OB_SUCCESS;
bool is_valid = true;
val = 0;
if (OB_ISNULL(str)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", K(ret), KP(str));
} else {
char *p_end = NULL;
if ('\0' == str[0]) {
is_valid = false;
} else if (OB_FAIL(ob_strtoll(str, p_end, val))) {
LOG_WARN("failed to get value from string", K(ret), K(str));
} else if ('\0' == *p_end) {
is_valid = true;
} else {
is_valid = false;
LOG_WARN("set int error", K(str), K(is_valid));
}
if (!is_valid) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid int str", K(ret), K(str));
}
}
return ret;
}
int ObBackupCleanTaskMgr::parse_ls_id_(const char *dir_name, int64_t &id_val)
{
int ret = OB_SUCCESS;
id_val = 0;
const char *LOGSTREAM = "logstream";
char *token = NULL;
char *saved_ptr = NULL;
char tmp_name[OB_MAX_URI_LENGTH] = { 0 };
if (OB_ISNULL(dir_name)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", K(ret), KP(dir_name));
} else if (OB_FAIL(databuff_printf(tmp_name, sizeof(tmp_name), "%s", dir_name))) {
LOG_WARN("failed to set dir name", K(ret), KP(dir_name));
} else {
token = tmp_name;
for (char *str = token; OB_SUCC(ret); str = NULL) {
token = ::strtok_r(str, "_", &saved_ptr);
if (NULL == token) {
break;
} else if (0 == strncmp(LOGSTREAM, token, strlen(LOGSTREAM))) {
continue;
} else if (OB_FAIL(parse_int_(token, id_val))){
LOG_WARN("invalid number", K(token), K(id_val));
}
}
}
return ret;
}
int ObBackupCleanTaskMgr::get_ls_ids_from_traverse_(const ObBackupPath &path, ObIArray<ObLSID> &ls_ids)
{
int ret = OB_SUCCESS;
ObBackupIoAdapter util;
ObArray<ObIODirentEntry> d_entrys;
ObDirPrefixEntryNameFilter prefix_op(d_entrys);
char logstream_prefix[OB_BACKUP_DIR_PREFIX_LENGTH] = { 0 };
if (OB_FAIL(databuff_printf(logstream_prefix, sizeof(logstream_prefix), "%s", "logstream_"))) {
LOG_WARN("failed to set log stream prefix", K(ret));
} else if (OB_FAIL(prefix_op.init(logstream_prefix, strlen(logstream_prefix)))) {
LOG_WARN("failed to init dir prefix", K(ret), K(logstream_prefix));
// TODO(wenjinyu.wjy) iterate dir sequentially 4.3
} else if (OB_FAIL(util.list_directories(path.get_obstr(), backup_dest_.get_storage_info(), prefix_op))) {
LOG_WARN("failed to list files", K(ret));
} else {
ObIODirentEntry tmp_entry;
ObLSID ls_id;
for (int64_t i = 0; OB_SUCC(ret) && i < d_entrys.count(); ++i) {
int64_t id_val = 0;
tmp_entry = d_entrys.at(i);
if (OB_ISNULL(tmp_entry.name_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("file name is null", K(ret));
} else if (OB_FAIL(parse_ls_id_(tmp_entry.name_, id_val))) {
LOG_WARN("failed to parse ls id", K(ret));
} else if (FALSE_IT(ls_id = id_val)) {
} else if (OB_FAIL(ls_ids.push_back(ls_id))) {
LOG_WARN("failed to push back ls_ids", K(ret));
}
}
}
return ret;
}
int ObBackupCleanTaskMgr::generate_ls_task_(const share::ObLSID &ls_id, share::ObBackupCleanLSTaskAttr &new_ls_attr)
{
int ret = OB_SUCCESS;
if (!ls_id.is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ls id is invalid", K(ret), K(ls_id));
} else {
new_ls_attr.task_id_ = task_attr_.task_id_;
new_ls_attr.tenant_id_ = task_attr_.tenant_id_;
new_ls_attr.ls_id_ = ls_id;
new_ls_attr.job_id_ = task_attr_.job_id_;
new_ls_attr.backup_set_id_ = task_attr_.backup_set_id_;
new_ls_attr.backup_piece_id_ = task_attr_.backup_piece_id_;
new_ls_attr.round_id_ = backup_piece_info_.key_.round_id_ != 0 ? backup_piece_info_.key_.round_id_ : 0;
new_ls_attr.task_type_ = task_attr_.task_type_;
new_ls_attr.status_.status_ = ObBackupTaskStatus::Status::INIT;
const int64_t current_time = ObTimeUtility::current_time();
new_ls_attr.start_ts_ = current_time;
new_ls_attr.end_ts_ = 0;
new_ls_attr.result_ = OB_SUCCESS;
}
return ret;
}
// file:///obbackup/backup_set_1_full/infos/locality_info.obbak
int ObBackupCleanTaskMgr::delete_tenant_locality_info_()
{
int ret = OB_SUCCESS;
ObBackupPath path;
ObBackupSetDesc desc;
desc.backup_set_id_ = backup_set_info_.backup_set_id_;
desc.backup_type_ = backup_set_info_.backup_type_;
if (OB_FAIL(ObBackupPathUtil::get_locality_info_path(backup_dest_, desc, path))) {
LOG_WARN("failed to get tenant locality path", K(ret));
} else if (OB_FAIL(share::ObBackupCleanUtil::delete_backup_file(path, backup_dest_.get_storage_info()))) {
LOG_WARN("failed to delete backup file", K(ret), K(task_attr_), K(path));
}
return ret;
}
// file:///obbackup/backup_set_1_full/infos/diagnose_info.obbak
int ObBackupCleanTaskMgr::delete_tenant_diagnose_info_()
{
int ret = OB_SUCCESS;
ObBackupPath path;
ObBackupSetDesc desc;
desc.backup_set_id_ = backup_set_info_.backup_set_id_;
desc.backup_type_ = backup_set_info_.backup_type_;
if (OB_FAIL(ObBackupPathUtil::get_diagnose_info_path(backup_dest_, desc, path))) {
LOG_WARN("failed to get tenant locality path", K(ret));
} else if (OB_FAIL(share::ObBackupCleanUtil::delete_backup_file(path, backup_dest_.get_storage_info()))) {
LOG_WARN("failed to delete backup file", K(ret), K(task_attr_), K(path));
}
return ret;
}
// file:///obbackup/backup_set_1_full/infos/meta_info/
int ObBackupCleanTaskMgr::delete_meta_info_dir_()
{
int ret = OB_SUCCESS;
ObBackupPath path;
ObBackupSetDesc desc;
desc.backup_set_id_ = backup_set_info_.backup_set_id_;
desc.backup_type_ = backup_set_info_.backup_type_;
if (OB_FAIL(ObBackupPathUtil::get_tenant_meta_info_dir_path(backup_dest_, desc, path))) {
LOG_WARN("failed to get tenant locality path", K(ret));
} else if (OB_FAIL(share::ObBackupCleanUtil::delete_backup_dir_files(path, backup_dest_.get_storage_info()))) {
LOG_WARN("failed to delete backup file", K(ret), K(task_attr_), K(path));
}
return ret;
}
// file:///obbackup/backup_set_1_full/infos/major_data_info_turn_X/
int ObBackupCleanTaskMgr::delete_major_data_info_dir_()
{
int ret = OB_SUCCESS;
ObBackupPath path;
ObBackupSetDesc desc;
desc.backup_set_id_ = backup_set_info_.backup_set_id_;
desc.backup_type_ = backup_set_info_.backup_type_;
share::ObBackupDataType backup_data_type;
backup_data_type.set_major_data_backup();
if (OB_FAIL(ObBackupPathUtil::get_ls_info_data_info_dir_path(backup_dest_, desc, backup_data_type,
backup_set_info_.major_turn_id_, path))) {
LOG_WARN("failed to get ls info data info", K(ret));
} else if (OB_FAIL(share::ObBackupCleanUtil::delete_backup_dir_files(path, backup_dest_.get_storage_info()))) {
LOG_WARN("failed to delete backup file", K(ret), K(task_attr_), K(path));
}
return ret;
}
// file:///obbackup/backup_set_1_full/infos/minor_data_info_turn_X/
int ObBackupCleanTaskMgr::delete_minor_data_info_dir_()
{
int ret = OB_SUCCESS;
ObBackupPath path;
ObBackupSetDesc desc;
desc.backup_set_id_ = backup_set_info_.backup_set_id_;
desc.backup_type_ = backup_set_info_.backup_type_;
share::ObBackupDataType backup_data_type;
backup_data_type.set_minor_data_backup();
if (OB_FAIL(ObBackupPathUtil::get_ls_info_data_info_dir_path(backup_dest_, desc,
backup_data_type, backup_set_info_.minor_turn_id_, path))) {
LOG_WARN("failed to get ls info data info path", K(ret));
} else if (OB_FAIL(share::ObBackupCleanUtil::delete_backup_dir_files(path, backup_dest_.get_storage_info()))) {
LOG_WARN("failed to delete backup file", K(ret), K(task_attr_), K(path));
}
return ret;
}
// file:///obbackup/backup_set_1_full/single_backup_set_info.obbak
int ObBackupCleanTaskMgr::delete_single_backup_set_info_()
{
int ret = OB_SUCCESS;
ObBackupPath path;
ObBackupSetDesc desc;
desc.backup_set_id_ = backup_set_info_.backup_set_id_;
desc.backup_type_ = backup_set_info_.backup_type_;
if (OB_FAIL(ObBackupPathUtil::get_backup_set_info_path(backup_dest_, desc, path))) {
LOG_WARN("failed to get tenant locality path", K(ret));
} else if (OB_FAIL(share::ObBackupCleanUtil::delete_backup_file(path, backup_dest_.get_storage_info()))) {
LOG_WARN("failed to delete backup file", K(ret), K(task_attr_), K(path));
}
return ret;
}
int ObBackupCleanTaskMgr::delete_data_info_turn_files_(const ObBackupPath &infos_path)
{
int ret = OB_SUCCESS;
ObBackupIoAdapter util;
ObArray<ObIODirentEntry> d_entrys;
const char info_turn_prefix[OB_BACKUP_DIR_PREFIX_LENGTH] = "data_info_turn_";
ObDirPrefixEntryNameFilter prefix_op(d_entrys);
if (OB_FAIL(prefix_op.init(info_turn_prefix, strlen(info_turn_prefix)))) {
LOG_WARN("failed to init dir prefix", K(ret), K(info_turn_prefix));
// TODO(wenjinyu.wjy) iterate dir sequentially 4.3
} else if (OB_FAIL(util.list_directories(infos_path.get_obstr(), backup_dest_.get_storage_info(), prefix_op))) {
LOG_WARN("failed to list directories", K(ret));
} else {
ObIODirentEntry tmp_entry;
ObBackupPath info_path;
for (int64_t i = 0; OB_SUCC(ret) && i < d_entrys.count(); ++i) {
ObIODirentEntry tmp_entry = d_entrys.at(i);
info_path.reset();
if (OB_ISNULL(tmp_entry.name_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("file name is null", K(ret));
} else if (OB_FAIL(info_path.init(infos_path.get_ptr()))) {
LOG_WARN("failed to init major path", K(ret), K(info_path));
} else if (OB_FAIL(info_path.join(tmp_entry.name_, ObBackupFileSuffix::NONE))) {
LOG_WARN("failed to join major path", K(ret));
} else if (OB_FAIL(ObBackupCleanUtil::delete_backup_dir_files(info_path, backup_dest_.get_storage_info()))) {
LOG_WARN("failed to delete backup log stream dir files", K(ret), K(info_path));
}
}
}
return ret;
}
int ObBackupCleanTaskMgr::delete_backup_dir_(const share::ObBackupPath &path)
{
int ret = OB_SUCCESS;
if (OB_FAIL(ObBackupCleanUtil::delete_backup_dir_files(path, backup_dest_.get_storage_info()))) {
LOG_WARN("failed to delete info turn files", K(ret), K(path));
}
return ret;
}
// file:///obbackup/backup_set_1_full/
int ObBackupCleanTaskMgr::delete_backup_set_dir_()
{
int ret = OB_SUCCESS;
ObBackupPath set_path;
ObBackupSetDesc desc;
desc.backup_set_id_ = backup_set_info_.backup_set_id_;
desc.backup_type_ = backup_set_info_.backup_type_;
if (OB_FAIL(ObBackupPathUtil::get_backup_set_dir_path(backup_dest_, desc, set_path))) {
LOG_WARN("failed to get tenant data backup set dir", K(ret), K_(backup_dest), K_(task_attr));
} else if (OB_FAIL(delete_backup_dir_(set_path))) {
LOG_WARN("failed to delete backup dir", K(ret), K(set_path));
}
return ret;
}
// file:///backup/backup_sets/backup_set_xx_xxx_start
int ObBackupCleanTaskMgr::delete_backup_set_start_file_()
{
int ret = OB_SUCCESS;
ObBackupPath start_path;
ObBackupSetDesc desc;
desc.backup_set_id_ = backup_set_info_.backup_set_id_;
desc.backup_type_ = backup_set_info_.backup_type_;
if (OB_FAIL(ObBackupPathUtil::get_backup_set_placeholder_start_path(backup_dest_, desc, start_path))) {
LOG_WARN("failed to get backup set end placeholder path", K(ret));
} else if (OB_FAIL(share::ObBackupCleanUtil::delete_backup_file(start_path, backup_dest_.get_storage_info()))) {
LOG_WARN("failed to delete backup file", K(ret), K(task_attr_), K(start_path));
}
return ret;
}
// file:///obbackup/backup_set_1_full/backup_set_1_full_xxxx_xxxxx
int ObBackupCleanTaskMgr::delete_backup_set_inner_placeholder_()
{
int ret = OB_SUCCESS;
ObBackupPath path;
ObBackupSetDesc desc;
desc.backup_set_id_ = backup_set_info_.backup_set_id_;
desc.backup_type_ = backup_set_info_.backup_type_;
char placeholder_prefix[OB_MAX_BACKUP_CHECK_FILE_NAME_LENGTH] = { 0 };
ObBackupIoAdapter util;
ObBackupPrefixDeleteFileOp prefix_delete_op;
if (OB_FAIL(ObBackupPathUtil::get_backup_set_dir_path(backup_dest_, desc, path))) {
LOG_WARN("failed to get tenant data backup set dir", K(ret), K_(backup_dest));
} else if (OB_FAIL(ObBackupPathUtil::get_backup_set_inner_placeholder_prefix(desc, placeholder_prefix, sizeof(placeholder_prefix)))) {
LOG_WARN("failed to get backup set inner placeholder prefix", K(ret));
} else if (OB_FAIL(prefix_delete_op.init(placeholder_prefix, strlen(placeholder_prefix), path, backup_dest_.get_storage_info()))) {
LOG_WARN("failed to init prefix delete", K(ret), K(placeholder_prefix));
} else if (OB_FAIL(util.list_files(path.get_obstr(), backup_dest_.get_storage_info(), prefix_delete_op))) {
LOG_WARN("failed to list files", K(ret), K(path), K(placeholder_prefix));
}
return ret;
}
// oss://backup/backup_set_xx_xxx_xxxxx/infos/tenant_backup_set_infos
int ObBackupCleanTaskMgr::delete_tenant_backup_set_infos_()
{
int ret = OB_SUCCESS;
ObBackupPath path;
ObBackupSetDesc desc;
desc.backup_set_id_ = backup_set_info_.backup_set_id_;
desc.backup_type_ = backup_set_info_.backup_type_;
if (OB_FAIL(ObBackupPathUtil::get_tenant_backup_set_infos_path(backup_dest_, desc, path))) {
LOG_WARN("failed to get backup set end placeholder path", K(ret));
} else if (OB_FAIL(share::ObBackupCleanUtil::delete_backup_file(path, backup_dest_.get_storage_info()))) {
LOG_WARN("failed to delete backup file", K(ret), K(task_attr_), K(path));
}
return ret;
}
int ObBackupCleanTaskMgr::delete_backup_set_meta_info_files_()
{
int ret = OB_SUCCESS;
ObBackupPath infos_path;
ObBackupSetDesc desc;
desc.backup_set_id_ = backup_set_info_.backup_set_id_;
desc.backup_type_ = backup_set_info_.backup_type_;
if (!backup_set_info_.is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("backup set info is valid", K(ret));
} else if (OB_FAIL(delete_meta_info_dir_())) { // first deleted meta file on infos dir
LOG_WARN("failed to delete ls attr info file", K(ret));
} else if (OB_FAIL(delete_major_data_info_dir_())) {
LOG_WARN("failed to delete major data info file", K(ret));
} else if (OB_FAIL(delete_minor_data_info_dir_())) {
LOG_WARN("failed to delete minor data info file", K(ret));
} else if (OB_FAIL(ObBackupPathUtil::get_ls_info_dir_path(backup_dest_, desc, infos_path))) {
LOG_WARN("failed to get backup log stream info path", K(ret));
} else if (OB_FAIL(delete_data_info_turn_files_(infos_path))) {
LOG_WARN("failed to delete tenant info turn ", K(ret));
} else if (OB_FAIL(delete_tenant_locality_info_())) {
LOG_WARN("failed to delete tenant locality info", K(ret));
} else if (OB_FAIL(delete_tenant_diagnose_info_())) {
LOG_WARN("failed to delete tenant diagnose info", K(ret));
} else if (OB_FAIL(delete_backup_dir_(infos_path))) {
LOG_WARN("failed to delete backup infos dir", K(ret));
} else if (OB_FAIL(delete_single_backup_set_info_())) {
LOG_WARN("failed to delete backup set file", K(ret));
} else if (OB_FAIL(delete_tenant_backup_set_infos_())) {
LOG_WARN("failed to delete tenant backup set infos", K(ret));
} else if (OB_FAIL(delete_backup_set_inner_placeholder_())) {
LOG_WARN("failed to delete backup set inner placeholder", K(ret));
} else if (OB_FAIL(delete_backup_set_dir_())) {
LOG_WARN("failed to delete backup set dir", K(ret));
} else if (OB_FAIL(delete_backup_set_start_file_())) {
LOG_WARN("failed to delete backup set start file", K(ret));
}
return ret;
}
// oss://archive/pieces/piece_[dest_id]_[round_id]_[piece_id]_[create_timestamp]_start
int ObBackupCleanTaskMgr::delete_backup_piece_start_file_()
{
int ret = OB_SUCCESS;
ObBackupPath path;
if (OB_FAIL(ObArchivePathUtil::get_piece_start_file_path(
backup_dest_, backup_piece_info_.key_.dest_id_, backup_piece_info_.key_.round_id_,
backup_piece_info_.key_.piece_id_, backup_piece_info_.start_scn_, path))) {
LOG_WARN("failed to get piece start file path", K(ret));
} else if (OB_FAIL(share::ObBackupCleanUtil::delete_backup_file(path, backup_dest_.get_storage_info()))) {
LOG_WARN("failed to delete backup file", K(ret), K(task_attr_), K(path));
}
return ret;
}
// oss://archive/[dest_id]_[round_id]_[piece_id]/single_piece_info.obarc
int ObBackupCleanTaskMgr::delete_single_piece_info_()
{
int ret = OB_SUCCESS;
ObBackupPath path;
if (OB_FAIL(ObArchivePathUtil::get_single_piece_file_path(
backup_dest_, backup_piece_info_.key_.dest_id_, backup_piece_info_.key_.round_id_,
backup_piece_info_.key_.piece_id_, path))) {
LOG_WARN("failed to get single piece file path", K(ret));
} else if (OB_FAIL(share::ObBackupCleanUtil::delete_backup_file(path, backup_dest_.get_storage_info()))) {
LOG_WARN("failed to delete backup file", K(ret), K(task_attr_), K(path));
}
return ret;
}
// oss://archive/piece_d[dest_id]r[round_id]p[piece_id]/tenant_archive_piece_infos.obarc
int ObBackupCleanTaskMgr::delete_tenant_archive_piece_infos()
{
int ret = OB_SUCCESS;
ObBackupPath path;
if (OB_FAIL(ObArchivePathUtil::get_tenant_archive_piece_infos_file_path(backup_dest_, backup_piece_info_.key_.dest_id_,
backup_piece_info_.key_.round_id_, backup_piece_info_.key_.piece_id_, path))) {
LOG_WARN("failed to get tenant archive piece infos path", K(ret));
} else if (OB_FAIL(share::ObBackupCleanUtil::delete_backup_file(path, backup_dest_.get_storage_info()))) {
LOG_WARN("failed to delete backup file", K(ret), K(task_attr_), K(path));
}
return ret;
}
// oss://archive/[dest_id]_[round_id]_[piece_id]/file_info.obarc
int ObBackupCleanTaskMgr::delete_piece_info_file_()
{
int ret = OB_SUCCESS;
ObBackupPath path;
if (OB_FAIL(ObArchivePathUtil::get_piece_info_file_path(
backup_dest_, backup_piece_info_.key_.dest_id_, backup_piece_info_.key_.round_id_,
backup_piece_info_.key_.piece_id_, path))) {
LOG_WARN("failed to get piece info file path", K(ret));
} else if (OB_FAIL(share::ObBackupCleanUtil::delete_backup_file(path, backup_dest_.get_storage_info()))) {
LOG_WARN("failed to delete backup file", K(ret), K(task_attr_), K(path));
}
return ret;
}
// oss://archive/[dest_id]_[round_id]_[piece_id]/piece_[dest_id]_[round_id]_[piece_id]_[start_ts]_[end_ts]
int ObBackupCleanTaskMgr::delete_piece_inner_placeholder_file_()
{
int ret = OB_SUCCESS;
ObBackupPath path;
if (OB_FAIL(ObArchivePathUtil::get_piece_inner_placeholder_file_path(
backup_dest_, backup_piece_info_.key_.dest_id_, backup_piece_info_.key_.round_id_,
backup_piece_info_.key_.piece_id_, backup_piece_info_.start_scn_, backup_piece_info_.checkpoint_scn_, path))) {
LOG_WARN("failed to get piece inner placeholder file path", K(ret));
} else if (OB_FAIL(share::ObBackupCleanUtil::delete_backup_file(path, backup_dest_.get_storage_info()))) {
LOG_WARN("failed to delete backup file", K(ret), K(task_attr_), K(path));
}
return ret;
}
// oss://archive/[dest_id]_[round_id]_[piece_id]/checkpoint
int ObBackupCleanTaskMgr::delete_piece_checkpoint_file_()
{
int ret = OB_SUCCESS;
ObBackupPath path;
if (OB_FAIL(ObArchivePathUtil::get_piece_checkpoint_dir_path(
backup_dest_, backup_piece_info_.key_.dest_id_, backup_piece_info_.key_.round_id_,
backup_piece_info_.key_.piece_id_, path))) {
LOG_WARN("failed to get piece checkpoint dir path", K(ret));
} else if (OB_FAIL(share::ObBackupCleanUtil::delete_backup_dir_files(path, backup_dest_.get_storage_info()))) {
LOG_WARN("failed to delete backup file", K(ret), K(task_attr_), K(path));
}
return ret;
}
int ObBackupCleanTaskMgr::delete_backup_piece_meta_info_files_()
{
int ret = OB_SUCCESS;
ObBackupPath path;
if (!backup_piece_info_.is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("backup piece info is valid", K(ret));
} else if (OB_FAIL(delete_single_piece_info_())) {
LOG_WARN("failed to delete single piece info file", K(ret), K(backup_piece_info_));
} else if (OB_FAIL(delete_piece_info_file_())) {
LOG_WARN("failed to delete info file", K(ret), K(backup_piece_info_));
} else if (OB_FAIL(delete_tenant_archive_piece_infos())) {
LOG_WARN("failed to delete info file", K(ret), K(backup_piece_info_));
} else if (OB_FAIL(delete_piece_inner_placeholder_file_())) {
LOG_WARN("failed to delete piece inner placeholder file", K(ret), K(backup_piece_info_));
} else if (OB_FAIL(delete_piece_checkpoint_file_())) {
LOG_WARN("failed to delete checkpoint file", K(ret), K(backup_piece_info_));
} else if (OB_FAIL(ObArchivePathUtil::get_piece_dir_path(
backup_dest_, backup_piece_info_.key_.dest_id_, backup_piece_info_.key_.round_id_,
backup_piece_info_.key_.piece_id_, path))) {
LOG_WARN("failed to get tenant backup piece dir path", K(ret));
} else if (OB_FAIL(delete_backup_dir_(path))) {
LOG_WARN("failed to delete backup infos dir", K(ret));
} else if (OB_FAIL(delete_backup_piece_start_file_())) {
LOG_WARN("failed to delete backup set start file", K(ret));
}
return ret;
}
int ObBackupCleanTaskMgr::delete_backup_meta_info_files_()
{
int ret = OB_SUCCESS;
if (task_attr_.is_delete_backup_set_task()) {
if (OB_FAIL(delete_backup_set_meta_info_files_())) {
LOG_WARN("failed to delete set meta info files", K(ret));
}
} else if (task_attr_.is_delete_backup_piece_task()) {
if (OB_FAIL(delete_backup_piece_meta_info_files_())) {
LOG_WARN("failed to delete piece meta info files", K(ret));
}
}
return ret;
}
int ObBackupCleanTaskMgr::backup_clean_ls_tasks_()
{
int ret = OB_SUCCESS;
ObArray<ObBackupCleanLSTaskAttr> ls_tasks;
int64_t finish_cnt = 0;
bool need_do_ls_task = true;
int tmp_ret = OB_SUCCESS;
ObBackupCleanStatus next_status;
if (OB_FAIL(ObBackupCleanLSTaskOperator::get_ls_tasks_from_task_id(*sql_proxy_, task_attr_.task_id_, task_attr_.tenant_id_, false/*update*/, ls_tasks))) {
LOG_WARN("failed to get log stream tasks", K(ret));
} else if (ls_tasks.count() != task_attr_.total_ls_count_) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ls task count error", K(ret), K(ls_tasks.count()), K(task_attr_.total_ls_count_));
} else if (ls_tasks.empty() && task_attr_.total_ls_count_ == 0) {
// do nothing, allow no log stream tasks
} else if (OB_FAIL(do_backup_clean_ls_tasks_(ls_tasks, finish_cnt))) {
LOG_WARN("failed to do backup ls task", K(ret), K_(task_attr));
}
if (OB_FAIL(ret)) {
} else if (ls_tasks.count() == finish_cnt) {
if (OB_FAIL(delete_backup_meta_info_files_())) {
LOG_WARN("failed to delete meta info files ", K(ret));
} else if (OB_FAIL(mark_backup_files_deleted_())) {
LOG_WARN("failed to mark backup files deleted", K(ret));
}
if (OB_FAIL(ret)) {
} else {
task_attr_.end_ts_ = ObTimeUtil::current_time();
task_attr_.status_.status_ = ObBackupCleanStatus::Status::COMPLETED;
if (OB_FAIL(advance_task_status_(*sql_proxy_, task_attr_.status_, OB_SUCCESS, task_attr_.end_ts_))) {
LOG_WARN("failed to advance task status", K(ret), K_(task_attr));
}
}
}
task_attr_.finish_ls_count_ = finish_cnt;
if (OB_FAIL(ret)) {
} else if (OB_FAIL(ObBackupCleanTaskOperator::update_ls_count(*sql_proxy_, task_attr_, false/*is_total*/))) {
LOG_WARN("failed to update finish ls count", K(ret), K_(task_attr));
}
return ret;
}
int ObBackupCleanTaskMgr::do_backup_clean_ls_tasks_(
ObArray<ObBackupCleanLSTaskAttr> &ls_tasks,
int64_t &finish_cnt)
{
int ret = OB_SUCCESS;
finish_cnt = 0;
int retry_count = 0;
if (ls_tasks.empty()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(ls_tasks));
} else {
for (int i = 0; OB_SUCC(ret) && i < ls_tasks.count(); ++i) {
ObBackupCleanLSTaskAttr &ls_attr = ls_tasks.at(i);
ObBackupCleanLSTaskMgr ls_task_mgr;
#ifdef ERRSIM
const int64_t OB_ERRSIM_BACKUP_MAX_RETRY_TIMES = 4;
if (ls_attr.retry_id_ >= OB_ERRSIM_BACKUP_MAX_RETRY_TIMES && OB_SUCCESS != ls_attr.result_) {
#else
if (ls_attr.retry_id_ >= OB_BACKUP_MAX_RETRY_TIMES && OB_SUCCESS != ls_attr.result_) {
#endif
job_attr_->can_retry_ = false;
ret = ls_attr.result_;
LOG_WARN("retry times exceeds the limit", K(ret), K(ls_attr.retry_id_), K(ls_attr.ls_id_), K(task_attr_), K(job_attr_->can_retry_));
} else if (OB_FAIL(ls_task_mgr.init(task_attr_, ls_attr, *task_scheduler_, *sql_proxy_, *backup_service_))) {
LOG_WARN("failed to init task advancer", K(ret), K(ls_attr));
} else if (OB_FAIL(ls_task_mgr.process(finish_cnt))) {
LOG_WARN("failed to process log stream task", K(ret));
}
}
}
return ret;
}
int ObBackupCleanTaskMgr::deal_failed_task(const int error)
{
int ret = OB_SUCCESS;
ObArray<ObBackupCleanLSTaskAttr> ls_tasks;
int result = OB_SUCCESS;
ObMySQLTransaction trans;
int64_t success_ls_count = 0;
const bool for_update = true;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObBackupCleanTaskMgr not init", K(ret));
} else if (OB_FAIL(trans.start(sql_proxy_, gen_meta_tenant_id(tenant_id_)))) {
LOG_WARN("failed to start trans", K(ret));
} else if (OB_FAIL(ObBackupCleanLSTaskOperator::get_ls_tasks_from_task_id(trans,
task_attr_.task_id_, task_attr_.tenant_id_, for_update, ls_tasks))) {
LOG_WARN("failed to get log stream tasks", K(ret));
} else {
if (OB_FAIL(do_failed_ls_task_(trans, ls_tasks, success_ls_count, result))) {
LOG_WARN("failed to do failed backup task", K(ret), K(ls_tasks));
}
result = (OB_SUCCESS == result) ? error : result;
if (OB_SUCC(ret)) {
ObBackupCleanStatus next_status;
if (OB_SUCCESS == result) {
next_status.status_ = ObBackupCleanStatus::Status::CANCELED;
result = OB_CANCELED;
} else {
next_status.status_ = ObBackupCleanStatus::Status::FAILED;
}
int64_t end_ts = ObTimeUtil::current_time();
task_attr_.finish_ls_count_ = success_ls_count;
if (OB_FAIL(advance_task_status_(trans, next_status, result, end_ts))) {
LOG_WARN("failed to advance task status to finish", K(ret), K_(task_attr));
} else if (OB_FAIL(ObBackupCleanTaskOperator::update_ls_count(trans, task_attr_, false/*is_total*/))) {
LOG_WARN("failed to update finish ls count", K(ret), K_(task_attr));
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(trans.end(true))) {
LOG_WARN("failed to commit trans", K(ret));
}
} else {
int tmp_ret = OB_SUCCESS;
if (OB_SUCCESS != (tmp_ret = trans.end(false))) {
LOG_WARN("failed to roll back status", K(ret), K(tmp_ret));
}
}
}
return ret;
}
int ObBackupCleanTaskMgr::do_failed_ls_task_(
common::ObISQLClient &sql_proxy,
ObIArray<ObBackupCleanLSTaskAttr> &ls_tasks,
int64_t &success_ls_count,
int &result)
{
int ret = OB_SUCCESS;
int64_t finish_cnt = 0;
ObBackupTaskStatus next_status;
next_status.status_ = ObBackupTaskStatus::Status::FINISH;
result = OB_SUCCESS;
if (ls_tasks.empty()) {
LOG_INFO("ls tasks is empty", K(ls_tasks));
// do nothing
} else {
for (int i = 0; OB_SUCC(ret) && i < ls_tasks.count(); ++i) {
ObBackupCleanLSTaskAttr &ls_attr = ls_tasks.at(i);
ObBackupCleanLSTaskMgr ls_task_mgr;
if (ObBackupTaskStatus::Status::FINISH == ls_attr.status_.status_) {
if (OB_SUCCESS == ls_attr.result_) {
success_ls_count++;
} else if (OB_SUCCESS == result) {
result = ls_attr.result_;
}
} else if (OB_FAIL(ls_task_mgr.init(task_attr_, ls_attr, *task_scheduler_, sql_proxy, *backup_service_))) {
LOG_WARN("failed to init task mgr", K(ret), K(ls_attr));
} else if (OB_FAIL(ls_task_mgr.cancel(finish_cnt))) {
LOG_WARN("failed to cancel task", K(ret));
}
}
}
return ret;
}
int ObBackupCleanTaskMgr::do_cleanup()
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObBackupCleanTaskMgr not init", K(ret));
} else if (OB_FAIL(move_task_to_history_())) {
LOG_WARN("failed to move task to his", K(ret), K_(task_attr));
}
return ret;
}
int ObBackupCleanTaskMgr::move_task_to_history_()
{
int ret = OB_SUCCESS;
if (OB_FAIL(backup_service_->check_leader())) {
LOG_WARN("failed to check leader", K(ret));
} else if (OB_FAIL(ObBackupCleanLSTaskOperator::move_ls_to_his(*sql_proxy_, task_attr_.tenant_id_, task_attr_.task_id_))) {
LOG_WARN("failed to move ls task to history", K(ret));
} else if (OB_FAIL(ObBackupCleanTaskOperator::move_task_to_history(*sql_proxy_, task_attr_.tenant_id_, task_attr_.task_id_))) {
LOG_WARN("failed to move task to history", K(ret));
}
return ret;
}
int ObBackupCleanTaskMgr::do_cancel_()
{
int ret = OB_SUCCESS;
int64_t finish_cnt = 0;
int64_t success_ls_count = 0;
ObArray<ObBackupCleanLSTaskAttr> ls_task;
ObMySQLTransaction trans;
const bool for_update = true;
if (OB_FAIL(trans.start(sql_proxy_, gen_meta_tenant_id(tenant_id_)))) {
LOG_WARN("failed to start trans", K(ret));
} else {
if (OB_FAIL(ObBackupCleanLSTaskOperator::get_ls_tasks_from_task_id(trans, task_attr_.task_id_, job_attr_->tenant_id_, for_update, ls_task))) {
LOG_WARN("failed to get log stream tasks", K(ret));
} else if (ls_task.empty()) {
// do nothing
} else {
for (int i = 0; OB_SUCC(ret) && i < ls_task.count(); ++i) {
ObBackupCleanLSTaskMgr task_mgr;
ObBackupCleanLSTaskAttr &ls_attr = ls_task.at(i);
if (ObBackupTaskStatus::Status::FINISH == ls_attr.status_.status_) {
++finish_cnt;
if (OB_SUCCESS == ls_attr.result_) {
success_ls_count++;
}
} else if (OB_FAIL(task_mgr.init(task_attr_, ls_attr, *task_scheduler_, trans, *backup_service_))) {
LOG_WARN("failed to init task mgr", K(ret), K(ls_attr));
} else if (OB_FAIL(task_mgr.cancel(finish_cnt))) {
LOG_WARN("failed to cancel task", K(ret));
}
}
}
if (OB_SUCC(ret) && ls_task.count() == finish_cnt) {
ObBackupCleanStatus next_status;
next_status.status_ = ObBackupCleanStatus::Status::CANCELED;
int64_t end_ts = ObTimeUtil::current_time();
task_attr_.finish_ls_count_ = success_ls_count;
if (OB_FAIL(advance_task_status_(trans, next_status, OB_CANCELED, end_ts))) {
LOG_WARN("failed to advance set task status to CANCELED", K(ret));
} else if (OB_FAIL(ObBackupCleanTaskOperator::update_ls_count(trans, task_attr_, false/*is_total*/))) {
LOG_WARN("failed to update finish ls count", K(ret), K_(task_attr));
} else {
FLOG_INFO("[BACKUP_CLEAN]advance set status CANCELED success", K(next_status));
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(trans.end(true))) {
LOG_WARN("failed to commit trans", K(ret));
}
} else {
int tmp_ret = OB_SUCCESS;
if (OB_SUCCESS != (tmp_ret = trans.end(false))) {
LOG_WARN("failed to roll back status", K(ret), K(tmp_ret));
}
}
}
return ret;
}
Reference in New Issue View Git Blame Copy Permalink