hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity
Files
c36014b57e8d6b04d458f690d46b75ffb5ffc890
oceanbase/src/storage/ob_partition_backup.cpp
oceanbase-admin cea7de1475 init push
2021-05-31 22:56:52 +08:00

1647 lines
60 KiB
C++
Raw Blame History

/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX STORAGE
#include "lib/utility/ob_tracepoint.h"
#include "lib/hash/ob_hashset.h"
#include "share/ob_force_print_log.h"
#include "storage/ob_partition_backup.h"
#include "storage/ob_partition_service.h"
#include "storage/ob_partition_log.h"
#include "storage/ob_partition_base_data_ob_reader.h"
#include "observer/ob_server_struct.h"
#include "observer/ob_server.h"
#include "clog/ob_clog_history_reporter.h"
#include "share/scheduler/ob_sys_task_stat.h"
#include "share/partition_table/ob_united_pt_operator.h"
#include "storage/ob_partition_base_data_restore_reader.h"
#include "storage/ob_partition_base_data_backup.h"
#include "share/ob_task_define.h"
#include "share/backup/ob_backup_struct.h"
#include "ob_table_mgr.h"
#include "ob_migrate_macro_block_writer.h"
#include "share/ob_debug_sync_point.h"
#include "ob_storage_struct.h"
#include "observer/omt/ob_tenant_node_balancer.h"
#include "storage/ob_partition_migration_status.h"
#include "storage/ob_pg_all_meta_checkpoint_reader.h"
#include "share/backup/ob_backup_path.h"
#include "storage/ob_partition_base_data_physical_restore.h"
#include "storage/ob_pg_storage.h"
namespace oceanbase {
using namespace common;
using namespace common::hash;
using namespace share;
using namespace obrpc;
using namespace blocksstable;
using namespace oceanbase::share::schema;
using namespace omt;
using namespace transaction;
namespace storage {
ObPartGroupBackupTask::ObPartGroupBackupTask() : ObPartGroupTask()
{}
ObPartGroupBackupTask::~ObPartGroupBackupTask()
{}
int ObPartGroupBackupTask::init(const ObIArray<ObReplicaOpArg>& task_list, const bool is_batch_mode,
storage::ObPartitionService* partition_service, const share::ObTaskId& task_id)
{
int ret = OB_SUCCESS;
const int64_t MAX_MAP_BUCKET_NUM = 1280;
SMART_VAR(ObPartMigrationTask, tmp_task)
{
ObChangeMemberOption change_member_option = NORMAL_CHANGE_MEMBER_LIST;
common::SpinWLockGuard guard(lock_);
if (is_inited_) {
ret = OB_INIT_TWICE;
STORAGE_LOG(WARN, "cannot init twice", K(ret));
} else if (NULL == partition_service || task_list.empty() || task_id.is_invalid()) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid args", K(ret), K(is_batch_mode), KP(partition_service), K(task_id), K(task_list));
} else if (OB_FAIL(cond_.init(ObWaitEventIds::GROUP_MIGRATE_TASK_IDLE_WAIT))) {
STORAGE_LOG(WARN, "failed to init cond", K(ret));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < task_list.count(); ++i) {
tmp_task.arg_ = task_list.at(i);
tmp_task.arg_.cluster_id_ =
(tmp_task.arg_.cluster_id_ == OB_INVALID_CLUSTER_ID) ? GCONF.cluster_id : tmp_task.arg_.cluster_id_;
tmp_task.status_ = ObPartMigrationTask::INIT;
tmp_task.result_ = OB_SUCCESS;
if (OB_FAIL(task_list_.push_back(tmp_task))) {
STORAGE_LOG(WARN, "failed to add task list", K(ret));
}
if (OB_SUCC(ret)) {
if (0 == i) {
type_ = task_list.at(i).type_;
tenant_id_ = task_list.at(i).key_.get_tenant_id();
change_member_option = task_list.at(i).change_member_option_;
if (!is_replica_op_valid(type_)) {
ret = OB_ERR_SYS;
STORAGE_LOG(ERROR, "invalid type", K(ret), K(i), K(type_), K(task_list));
}
} else if (type_ != task_list.at(i).type_ || tenant_id_ != task_list.at(i).key_.get_tenant_id() ||
change_member_option != task_list.at(i).change_member_option_) {
ret = OB_ERR_SYS;
STORAGE_LOG(ERROR,
"migration type or tenant_id or skip_change_member_list not same in task list",
K(ret),
K(i),
K(task_list));
}
}
}
if (OB_SUCC(ret)) {
is_inited_ = true;
is_batch_mode_ = is_batch_mode;
partition_service_ = partition_service;
first_error_code_ = OB_SUCCESS;
task_id_ = task_id;
is_finished_ = false;
need_idle_ = true;
change_member_option_ = change_member_option;
STORAGE_LOG(INFO, "succeed to init pg group backup task", K(*this));
}
}
}
return ret;
}
int ObPartGroupBackupTask::check_partition_validation()
{
int ret = OB_SUCCESS;
if (!is_inited_) {
ret = OB_NOT_INIT;
STORAGE_LOG(ERROR, "not inited", K(ret));
} else {
ObRecoveryPointSchemaFilter backup_filter;
const ObPhysicalBackupArg& backup_arg = task_list_[0].arg_.backup_arg_;
if (OB_FAIL(backup_filter.init(backup_arg.tenant_id_,
backup_arg.backup_schema_version_, /*backup_schema_version*/
backup_arg.backup_schema_version_ /*current_schema_version*/))) {
STORAGE_LOG(WARN, "backup schema filter init fail", K(ret));
}
for (int64_t i = 0; OB_SUCC(ret) && i < task_list_.count(); ++i) {
ObPartMigrationTask& sub_task = task_list_[i];
ObPartitionKey& pkey = sub_task.arg_.key_;
bool in_member_list = false;
bool is_working_partition = partition_service_->is_working_partition(pkey);
if (ObPartGroupMigrator::get_instance().is_stop()) {
ret = OB_SERVER_IS_STOPPING;
STORAGE_LOG(WARN, "server is stopping", K(ret));
} else if (BACKUP_REPLICA_OP != sub_task.arg_.type_) {
ret = OB_ERR_SYS;
LOG_ERROR("replica op type is not backup", K(ret), "arg", sub_task.arg_);
} else {
if (OB_UNLIKELY(!partition_service_->is_partition_exist(pkey))) {
ret = OB_ENTRY_NOT_EXIST;
ObTaskController::get().allow_next_syslog();
STORAGE_LOG(WARN, "can not backup replica which is not exsit", K(ret), K(pkey), K(sub_task));
} else {
// backup filter base on target version schema
bool is_exist = false;
if (OB_FAIL(backup_filter.check_partition_exist(pkey, is_exist))) {
STORAGE_LOG(WARN, "backup filter check partition failed", K(ret), K(pkey));
} else if (OB_UNLIKELY(!is_exist)) {
ret = OB_PARTITION_NOT_EXIST;
STORAGE_LOG(WARN, "backup partition is not exist after filter", K(ret), K(pkey), K(sub_task));
}
}
}
}
#ifdef ERRSIM
if (OB_SUCC(ret)) {
ret = E(EventTable::EN_CHECK_SUB_MIGRATION_TASK) OB_SUCCESS;
if (OB_FAIL(ret)) {
STORAGE_LOG(ERROR, "fake EN_CHECK_SUB_MIGRATION_TASK", K(ret));
}
}
#endif
}
return ret;
}
int ObPartGroupBackupTask::do_task()
{
int ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS;
const int64_t start_ts = ObTimeUtility::current_time();
ObTaskController::get().switch_task(share::ObTaskType::DATA_MAINTAIN);
ObCurTraceId::set(task_id_);
ObTaskController::get().allow_next_syslog();
LOG_INFO("start pg group backup task", K(*this));
if (!is_inited_) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "not inited", K(ret));
} else if (OB_FAIL(check_before_do_task())) {
LOG_WARN("Failed to check_before_do_task", K(ret));
} else if (OB_FAIL(check_all_pg_backup_point_created())) {
LOG_WARN("failed to check all pg backup point created", K(ret));
} else if (OB_FAIL(do_part_group_backup_minor_task())) {
LOG_WARN("failed to do part group backup minor task", K(ret));
} else if (OB_FAIL(do_backup_pg_metas())) {
LOG_WARN("failed to do backup pg metas", K(ret));
} else if (OB_FAIL(do_part_group_backup_major_task())) {
LOG_WARN("failed to do part group backup major task", K(ret));
}
if (OB_FAIL(ret)) {
common::SpinWLockGuard guard(lock_);
if (OB_SUCCESS == first_error_code_) {
first_error_code_ = ret;
LOG_INFO("set first error code", K(ret), K(first_error_code_));
}
}
if (OB_SUCCESS != (tmp_ret = finish_group_backup_task())) {
LOG_ERROR("failed to finish group backup task", K(ret));
}
ObTaskController::get().allow_next_syslog();
const int64_t cost_ts = ObTimeUtility::current_time() - start_ts;
LOG_INFO("group backup task finish", K(cost_ts), K(*this));
return ret;
}
int ObPartGroupBackupTask::check_before_do_task()
{
int ret = OB_SUCCESS;
if (!is_inited_) {
ret = OB_NOT_INIT;
STORAGE_LOG(ERROR, "not inited", K(ret));
} else if (OB_FAIL(check_partition_validation())) {
STORAGE_LOG(WARN, "failed to check_partition_validation", K(ret));
}
if (OB_FAIL(ret)) {
ObTaskController::get().allow_next_syslog();
first_error_code_ = ret;
STORAGE_LOG(WARN, "failed to check task, mark fail", K(ret), K(*this));
}
return ret;
}
int ObPartGroupBackupTask::do_backup_task(const ObBackupDataType& backup_data_type)
{
int ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS;
bool is_finished = false;
const int64_t start_ts = ObTimeUtility::current_time();
ObTaskController::get().switch_task(share::ObTaskType::DATA_MAINTAIN);
ObTaskController::get().allow_next_syslog();
LOG_INFO("start pg group backup task", K(*this));
if (!is_inited_) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "not inited", K(ret));
} else {
while (!is_finished) {
if (OB_SUCCESS != (tmp_ret = check_is_task_cancel())) {
STORAGE_LOG(WARN, "failed to check is task cancel", K(tmp_ret));
}
if (OB_SUCCESS != (tmp_ret = try_schedule_new_partition_backup(backup_data_type))) {
STORAGE_LOG(WARN, "failed to try_schedule_new_partition_migration", K(tmp_ret));
}
if (OB_SUCCESS != (tmp_ret = try_finish_group_backup(is_finished))) {
STORAGE_LOG(WARN, "failed to try_finish_group_backup", K(tmp_ret));
}
if (!is_finished) {
share::dag_yield();
ObThreadCondGuard guard(cond_);
if (need_idle_) {
if (OB_SUCCESS != (tmp_ret = cond_.wait(PART_GROUP_TASK_IDLE_TIME_MS))) {
if (OB_TIMEOUT != tmp_ret) {
STORAGE_LOG(WARN, "failed to idle", K(tmp_ret));
}
}
}
need_idle_ = true;
}
}
}
ObTaskController::get().allow_next_syslog();
const int64_t cost_ts = ObTimeUtility::current_time() - start_ts;
LOG_INFO("group backup task finish", K(cost_ts), K(task_list_), K(backup_data_type));
return ret;
}
int ObPartGroupBackupTask::do_part_group_backup_minor_task()
{
int ret = OB_SUCCESS;
ObBackupDataType backup_data_type;
int first_error_code = OB_SUCCESS;
if (!is_inited_) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "not inited", K(ret));
} else if (FALSE_IT(reset_tasks_status())) {
} else if (FALSE_IT(backup_data_type.set_minor_data_backup())) {
} else if (OB_FAIL(do_backup_task(backup_data_type))) {
STORAGE_LOG(WARN, "failed to do backup task", K(ret));
} else {
{
common::SpinRLockGuard guard(lock_);
first_error_code = first_error_code_;
}
if (OB_SUCCESS != first_error_code) {
ret = first_error_code;
STORAGE_LOG(WARN, "first error code has set, set failed", K(ret), K(first_error_code));
}
}
return ret;
}
int ObPartGroupBackupTask::do_part_group_backup_major_task()
{
int ret = OB_SUCCESS;
ObBackupDataType backup_data_type;
int first_error_code = OB_SUCCESS;
if (!is_inited_) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "not inited", K(ret));
} else if (FALSE_IT(reset_tasks_status())) {
} else if (FALSE_IT(backup_data_type.set_major_data_backup())) {
} else if (OB_FAIL(do_backup_task(backup_data_type))) {
STORAGE_LOG(WARN, "failed to do backup task", K(ret));
} else {
{
common::SpinRLockGuard guard(lock_);
first_error_code = first_error_code_;
}
if (OB_SUCCESS != first_error_code) {
ret = first_error_code;
STORAGE_LOG(WARN, "first error code has set, set failed", K(ret), K(first_error_code));
}
}
return ret;
}
int ObPartGroupBackupTask::set_task_list_result(
const int32_t first_error_code, const common::ObIArray<ObPartMigrationTask>& task_list)
{
int ret = OB_SUCCESS;
if (!is_inited_) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "not inited", K(ret));
} else if (task_list_.count() != task_list.count()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("backup task list num is not equal", K(ret), K(task_list_.count()), K(task_list.count()));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < task_list.count(); ++i) {
const ObPartMigrationTask& tmp_task = task_list.at(i);
ObPartMigrationTask& task = task_list_.at(i);
if (tmp_task.arg_.key_ != task.arg_.key_ || tmp_task.arg_.type_ != task.arg_.type_) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("backup task is not same", K(ret), K(task), K(tmp_task));
} else {
task.during_migrating_ = tmp_task.during_migrating_;
task.need_reset_migrate_status_ = tmp_task.need_reset_migrate_status_;
task.result_ = tmp_task.result_;
task.status_ = tmp_task.status_;
}
}
if (OB_SUCC(ret)) {
common::SpinWLockGuard guard(lock_);
if (OB_SUCCESS == first_error_code_) {
first_error_code_ = first_error_code;
STORAGE_LOG(WARN, "set first_error_code_", K(ret));
}
}
}
return ret;
}
int ObPartGroupBackupTask::finish_group_backup_task()
{
int ret = OB_SUCCESS;
int first_error_code = OB_SUCCESS;
if (!is_inited_) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "not inited", K(ret));
} else {
{
common::SpinRLockGuard guard(lock_);
first_error_code = first_error_code_;
}
SMART_VAR(ObReportPartMigrationTask, tmp_task)
{
if (OB_FAIL(report_list_.reserve(task_list_.count()))) {
STORAGE_LOG(WARN, "failed to reserve report list", K(ret));
}
for (int64_t i = 0; OB_SUCC(ret) && i < task_list_.count(); ++i) {
ObPartMigrationTask& sub_task = task_list_[i];
if (ObIPartMigrationTask::INIT == sub_task.status_) {
sub_task.result_ = first_error_code;
}
tmp_task.arg_ = sub_task.arg_;
tmp_task.status_ = ObIPartMigrationTask::FINISH;
tmp_task.result_ = sub_task.result_;
tmp_task.need_report_checksum_ = sub_task.ctx_.need_report_checksum_;
tmp_task.ctx_ = &sub_task.ctx_;
if (OB_FAIL(report_list_.push_back(tmp_task))) {
// report_list is reserved before, should not fail here
STORAGE_LOG(ERROR, "failed to add report list", K(ret));
}
}
if (OB_SUCC(ret)) {
is_finished_ = true;
}
}
}
return ret;
}
int ObPartGroupBackupTask::do_backup_pg_metas()
{
int ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS;
// backup meta
if (!is_inited_) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "not inited", K(ret));
} else if (OB_UNLIKELY(BACKUP_REPLICA_OP != type_)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "type must be backup", K(ret), K(type_));
} else if (FALSE_IT(reset_tasks_status())) {
} else {
common::SpinRLockGuard guard(lock_);
SMART_VAR(ObBackupMetaWriter, meta_writer)
{
common::ObInOutBandwidthThrottle* throttle = ObPartitionMigrator::get_instance().get_bandwidth_throttle();
if (OB_ISNULL(throttle)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "BandwidthThrottle should not be null here", K(ret));
} else if (OB_FAIL(meta_writer.open(*throttle, task_list_))) {
STORAGE_LOG(WARN, "init meta writer fail", K(ret));
} else {
if (OB_FAIL(meta_writer.process())) {
STORAGE_LOG(WARN, "write meta data fail", K(ret));
}
if (OB_SUCCESS != (tmp_ret = meta_writer.close())) {
LOG_WARN("failed to close meta writer", K(tmp_ret), K(ret));
if (OB_SUCC(ret)) {
ret = tmp_ret;
}
}
}
}
}
return ret;
}
int ObPartGroupBackupTask::try_schedule_new_partition_backup(const ObBackupDataType& backup_data_type)
{
int ret = OB_SUCCESS;
if (!is_inited_) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "part group backup task do not init", K(ret));
} else if (!backup_data_type.is_valid()) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "try schedule new partition backup get invalid argument", K(ret), K(backup_data_type));
} else {
{
common::SpinRLockGuard guard(lock_);
if (OB_SUCCESS != first_error_code_) {
ret = first_error_code_;
STORAGE_LOG(WARN, "first_error_code_ is set, skip schedule new partition backup", K(ret));
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(try_schedule_partition_backup(backup_data_type))) {
STORAGE_LOG(WARN, "failed to try schedule partition backup", K(ret));
}
}
}
return ret;
}
int ObPartGroupBackupTask::try_schedule_partition_backup(const ObBackupDataType& backup_data_type)
{
int ret = OB_SUCCESS;
bool need_schedule = true;
ObPartMigrationTask* task = NULL;
int64_t data_backup_concurrency = GCONF.backup_concurrency;
int32_t up_limit = 0;
if (OB_FAIL(ObDagScheduler::get_instance().get_up_limit(ObIDag::DAG_ULT_BACKUP, up_limit))) {
STORAGE_LOG(WARN, "failed to get up limit", K(ret));
} else {
data_backup_concurrency = data_backup_concurrency == 0 ? up_limit : data_backup_concurrency;
FLOG_INFO("backup concurrency", K(data_backup_concurrency), K(up_limit));
}
while (OB_SUCC(ret) && need_schedule &&
ObDagScheduler::get_instance().get_dag_count(ObIDag::DAG_TYPE_BACKUP) < data_backup_concurrency) {
if (OB_FAIL(inner_schedule_partition(task, need_schedule))) {
STORAGE_LOG(WARN, "failed to inner schedule partition", K(ret));
} else if (!need_schedule) {
// do nothing
} else if (OB_ISNULL(task)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "task need schedule but is NULL", K(ret), KP(task));
} else if (OB_FAIL(schedule_backup_dag(backup_data_type, task->ctx_))) {
STORAGE_LOG(WARN, "failed to schedule backup dag", K(ret), K(task->arg_));
}
}
return ret;
}
int ObPartGroupBackupTask::inner_schedule_partition(ObPartMigrationTask*& task, bool& need_schedule)
{
int ret = OB_SUCCESS;
need_schedule = false;
task = NULL;
common::SpinRLockGuard guard(lock_);
for (int64_t i = 0; OB_SUCC(ret) && i < task_list_.count(); ++i) {
task = &task_list_.at(i);
bool as_data_source = false;
if (type_ != task->arg_.type_ || !is_replica_op_valid(task->arg_.type_)) {
ret = OB_ERR_SYS;
STORAGE_LOG(ERROR, "invalid type, cannot handle this task", K(ret), K(i), K(*task));
} else if (REMOVE_REPLICA_OP == type_) {
// do nothing
// TODO() remove replica op backup need cancel all task and notice rs need retry
} else if (OB_FAIL(check_can_as_data_source(type_,
task->arg_.data_src_.get_replica_type(),
task->arg_.dst_.get_replica_type(),
as_data_source))) {
STORAGE_LOG(WARN, "fail to check can as data source", K(ret), K(task->arg_));
} else if (!as_data_source) {
ret = OB_NOT_SUPPORTED;
first_error_code_ = ret;
LOG_ERROR("invalid data src", K(ret), K(task->arg_));
} else if (ObPartMigrationTask::INIT == task->status_) {
if (OB_FAIL(build_migrate_ctx(task->arg_, task->ctx_))) {
STORAGE_LOG(WARN, "fail to build migrate ctx", K(ret));
} else {
need_schedule = true;
break;
}
}
}
return ret;
}
// caller must not hold wlock
int ObPartGroupBackupTask::schedule_backup_dag(const ObBackupDataType& backup_data_type, ObMigrateCtx& migrate_ctx)
{
int ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_ERROR("not inited", K(ret));
} else if (OB_FAIL(check_partition_checksum(migrate_ctx.replica_op_arg_.key_))) {
STORAGE_LOG(WARN,
"failed to check partition checksum, cannot schedule backup",
K(ret),
K(migrate_ctx.replica_op_arg_.key_));
} else if (OB_FAIL(migrate_ctx.generate_and_schedule_backup_dag(backup_data_type))) {
STORAGE_LOG(WARN, "failed to generate backup dag", K(ret));
}
if (OB_FAIL(ret)) {
share::ObTaskId fake_task_id;
const bool during_migrating = false;
const int64_t fake_backup_set_id = 0;
if (OB_SUCCESS !=
(tmp_ret = set_part_task_finish(
migrate_ctx.replica_op_arg_.key_, ret, fake_task_id, during_migrating, fake_backup_set_id))) {
LOG_WARN("failed to finish part task", K(ret));
}
}
return ret;
}
int ObPartGroupBackupTask::try_finish_group_backup(bool& is_finished)
{
int ret = OB_SUCCESS;
bool is_sub_task_finish = true;
report_list_.reset();
is_finished = false;
if (!is_inited_) {
ret = OB_NOT_INIT;
STORAGE_LOG(ERROR, "not inited", K(ret));
} else {
common::SpinWLockGuard guard(lock_);
for (int64_t i = 0; OB_SUCC(ret) && i < task_list_.count(); ++i) {
ObPartMigrationTask& task = task_list_.at(i);
if (ObPartMigrationTask::FINISH != task.status_) {
if (OB_SUCCESS != first_error_code_ && ObPartMigrationTask::INIT == task.status_) {
task.status_ = ObPartMigrationTask::FINISH;
task.result_ = first_error_code_;
} else {
is_sub_task_finish = false;
}
} else if (OB_SUCCESS != task.result_) {
if (OB_SUCCESS == first_error_code_) {
first_error_code_ = OB_ERR_SYS;
STORAGE_LOG(ERROR, "sub task has error , but first_error_code_ is succ, use OB_ERR_SYS", K(*this));
}
}
}
if (OB_SUCC(ret) && is_sub_task_finish) {
is_finished = true;
}
}
return ret;
}
void ObPartGroupBackupTask::reset_tasks_status()
{
common::SpinWLockGuard guard(lock_);
for (int64_t i = 0; i < task_list_.count(); ++i) {
ObPartMigrationTask& task = task_list_.at(i);
task.ctx_.rebuild_migrate_ctx();
task.status_ = ObPartMigrationTask::INIT;
task.result_ = OB_SUCCESS;
task.need_reset_migrate_status_ = false;
task.during_migrating_ = false;
}
}
// TODO() wait for backup point created
int ObPartGroupBackupTask::check_all_pg_backup_point_created()
{
int ret = OB_SUCCESS;
const int64_t start_ts = ObTimeUtility::current_time();
ObTaskController::get().allow_next_syslog();
LOG_INFO("start check all pg backup point created", K(*this));
if (!is_inited_) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "not inited", K(ret));
} else {
common::SpinRLockGuard guard(lock_);
for (int64_t i = 0; OB_SUCC(ret) && i < task_list_.count(); ++i) {
ObPartMigrationTask& task = task_list_.at(i);
const ObPartitionKey& pkey = task.arg_.key_;
const int64_t backup_snapshot_version = task.arg_.backup_arg_.backup_snapshot_version_;
if (OB_FAIL(check_pg_backup_point_created(pkey, backup_snapshot_version))) {
LOG_WARN("failed to check pg backup point created", K(ret), K(pkey), K(backup_snapshot_version));
}
}
}
ObTaskController::get().allow_next_syslog();
const int64_t cost_ts = ObTimeUtility::current_time() - start_ts;
LOG_INFO("end check all pg backup point created", K(cost_ts), K(task_list_.count()));
return ret;
}
int ObPartGroupBackupTask::check_pg_backup_point_created(
const ObPartitionKey& pg_key, const int64_t backup_snapshot_version)
{
int ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS;
bool is_exist = false;
// TODO() need wakeup minor merge
if (!is_inited_) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "not inited", K(ret));
} else if (!pg_key.is_valid() || backup_snapshot_version <= 0) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(
WARN, "check pg backup point created get invalid argument", K(ret), K(pg_key), K(backup_snapshot_version));
} else {
while (OB_SUCC(ret)) {
// check_backup_point exist
if (ObPartGroupMigrator::get_instance().is_stop()) {
ret = OB_SERVER_IS_STOPPING;
STORAGE_LOG(WARN, "server is stopping", K(ret));
} else {
ObIPartitionGroupGuard guard;
ObIPartitionGroup* partition_group = NULL;
if (OB_FAIL(ObPartitionService::get_instance().get_partition(pg_key, guard))) {
LOG_WARN("failed to get partition guard", K(ret), K(pg_key));
} else if (OB_ISNULL(partition_group = guard.get_partition_group())) {
ret = OB_ERR_SYS;
LOG_ERROR("partition must not null", K(ret), K(pg_key));
} else if (OB_FAIL(partition_group->get_pg_storage().get_recovery_data_mgr().check_backup_point_exist(
backup_snapshot_version, is_exist))) {
LOG_WARN("failed to check backup point exist", K(ret), K(pg_key));
}
}
if (OB_FAIL(ret)) {
} else if (is_exist) {
break;
} else {
if (REACH_TIME_INTERVAL(10 * 1000 * 1000)) { // 10s
LOG_INFO("backup point is not exist, need wait", K(backup_snapshot_version), K(pg_key));
}
share::dag_yield();
ObThreadCondGuard guard(cond_);
if (need_idle_) {
if (OB_SUCCESS != (tmp_ret = cond_.wait(PART_GROUP_BACKUP_POINT_CHECK_TIME_MS))) {
if (OB_TIMEOUT != tmp_ret) {
STORAGE_LOG(WARN, "failed to idle", K(tmp_ret));
}
}
}
need_idle_ = true;
}
}
}
return ret;
}
ObBackupPrepareTask::ObBackupPrepareTask()
: ObITask(TASK_TYPE_MIGRATE_PREPARE),
is_inited_(false),
ctx_(NULL),
cp_fty_(NULL),
bandwidth_throttle_(NULL),
partition_service_(NULL),
backup_meta_reader_(NULL)
{}
ObBackupPrepareTask::~ObBackupPrepareTask()
{
if (NULL != cp_fty_) {
if (NULL != backup_meta_reader_) {
cp_fty_->free(backup_meta_reader_);
}
}
}
int ObBackupPrepareTask::init(ObIPartitionComponentFactory& cp_fty,
common::ObInOutBandwidthThrottle& bandwidth_throttle, ObPartitionService& partition_service)
{
int ret = OB_SUCCESS;
if (is_inited_) {
ret = OB_INIT_TWICE;
LOG_ERROR("cannot init twice", K(ret));
} else if (ObIDag::DAG_TYPE_BACKUP != dag_->get_type()) {
ret = OB_ERR_SYS;
LOG_ERROR("dag type not match", K(ret), K(*dag_));
} else {
ctx_ = static_cast<ObMigrateDag*>(dag_)->get_ctx();
is_inited_ = true;
bandwidth_throttle_ = &bandwidth_throttle;
partition_service_ = &partition_service;
cp_fty_ = &cp_fty;
}
return ret;
}
int ObBackupPrepareTask::prepare_backup_reader()
{
int ret = OB_SUCCESS;
if (!is_inited_) {
ret = OB_NOT_INIT;
STORAGE_LOG(ERROR, "not inited", K(ret));
} else if (BACKUP_REPLICA_OP == ctx_->replica_op_arg_.type_) {
if (OB_ISNULL(ctx_->backup_meta_reader_ = cp_fty_->get_partition_group_meta_backup_reader())) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "pg meta reader should not be NULL", K(ret));
} else if (OB_FAIL(ctx_->backup_meta_reader_->init(ctx_->pg_meta_, ctx_->replica_op_arg_.backup_arg_))) {
STORAGE_LOG(WARN, "fail to init backup meta reader", K(ret));
}
}
return ret;
}
int ObBackupPrepareTask::process()
{
int ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS;
ObITask* last_task = nullptr;
if (NULL != ctx_) {
ctx_->action_ = ObMigrateCtx::PREPARE;
if (NULL != dag_) {
ctx_->task_id_ = dag_->get_dag_id();
}
if (OB_SUCCESS != (tmp_ret = (ctx_->trace_id_array_.push_back(*ObCurTraceId::get_trace_id())))) {
STORAGE_LOG(WARN, "failed to push back trace id to array", K(tmp_ret));
}
LOG_INFO("start ObBackupPrepareTask process", "arg", ctx_->replica_op_arg_);
}
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_ERROR("not inited", K(ret));
} else if (OB_FAIL(add_backup_status(ctx_))) {
LOG_WARN("failed to add backup status", K(ret));
} else if (OB_FAIL(prepare_backup())) {
LOG_WARN("failed to prepare backup", K(ret));
} else if (OB_FAIL(schedule_backup_tasks())) {
LOG_WARN("failed to schedule backup tasks", K(ret), K(*ctx_));
}
if (OB_SUCC(ret)) {
ObTaskController::get().allow_next_syslog();
LOG_INFO("finish prepare task", K(*ctx_));
}
if (OB_FAIL(ret) && NULL != ctx_) {
if (ctx_->is_need_retry(ret)) {
ctx_->need_rebuild_ = true;
LOG_INFO("migrate prepare task need retry", K(ret), K(*ctx_));
}
ctx_->set_result_code(ret);
}
LOG_INFO("end ObBackupPrepareTask process", K(ret));
return ret;
}
int ObBackupPrepareTask::prepare_backup()
{
int ret = OB_SUCCESS;
const bool is_write_lock = true;
if (!is_inited_) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "not inited", K(ret));
} else {
ObMigrateCtxGuard guard(is_write_lock, *ctx_);
ctx_->use_slave_safe_read_ts_ = false; // disable logical migrate
if (!ctx_->is_valid()) {
ret = OB_ERR_SYS;
STORAGE_LOG(ERROR, "invalid migrate ctx", K(ret), K(*this), K(*ctx_));
} else if (1 != ctx_->doing_task_cnt_) {
ret = OB_ERR_SYS;
STORAGE_LOG(ERROR, "doing task must be 1 during prepare action", K(ret), K(*this), K(*ctx_));
} else if (REMOVE_REPLICA_OP == ctx_->replica_op_arg_.type_) {
ret = OB_ERR_SYS;
STORAGE_LOG(ERROR, "should not prepare migrate for remove replica", K(ret), K(ctx_->replica_op_arg_));
} else if (OB_FAIL(build_backup_prepare_context())) {
LOG_WARN("failed to build backup prepare context", K(ret));
} else if (OB_FAIL(prepare_backup_reader())) {
LOG_WARN("failed to prepare backup reader", K(ret));
} else if (OB_FAIL(check_backup_data_continues())) {
LOG_WARN("failed to check backup data continues", K(ret));
} else if (OB_FAIL(build_backup_pg_partition_info())) {
LOG_WARN("failed to build backup pg partition info", K(ret), K(*ctx_));
} else {
ctx_->need_rebuild_ = false;
ObTaskController::get().allow_next_syslog();
STORAGE_LOG(INFO, "finish prepare backup", "pkey", ctx_->replica_op_arg_.key_, K(*ctx_));
}
if (OB_FAIL(ret)) {
if (NULL != ctx_) {
ctx_->result_ = ret;
}
}
}
return ret;
}
int ObBackupPrepareTask::add_backup_status(ObMigrateCtx* ctx)
{
int ret = OB_SUCCESS;
if (!is_inited_) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "not inited", K(ret));
} else if (NULL == ctx) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid arg", K(ret));
} else if (!ctx->is_valid()) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid args", K(ret), K(*ctx));
} else if (OB_FAIL(add_partition_backup_status(*ctx))) {
STORAGE_LOG(WARN, "failed to add partition_backup_status", K(ret), K(*ctx));
} else {
STORAGE_LOG(INFO, "succeed to add backup ctx", KP(ctx), K(*ctx));
}
return ret;
}
int ObBackupPrepareTask::add_partition_backup_status(const ObMigrateCtx& ctx)
{
int ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS;
if (!is_inited_) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "not inited", K(ret));
} else if (!ctx.is_valid()) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid args", K(ret), K(ctx));
} else {
ObPartitionMigrationStatus task;
task.task_id_ = ctx.task_id_;
task.migrate_type_ = ctx.replica_op_arg_.get_replica_op_type_str();
task.pkey_ = ctx.replica_op_arg_.key_;
task.clog_parent_.reset(); // we don't know it now
task.src_.reset(); // we don't know it now
task.dest_ = OBSERVER.get_self();
task.result_ = ctx.result_;
task.start_time_ = ctx.create_ts_;
task.action_ = ctx.action_;
task.replica_state_ = OB_UNKNOWN_REPLICA; // we don't know it now
task.doing_task_count_ = 0;
task.total_task_count_ = 0;
task.rebuild_count_ = 0;
task.continue_fail_count_ = 0;
task.data_statics_ = ctx.data_statics_;
// allow comment truncation, no need to set ret
(void)ctx.fill_comment(task.comment_, sizeof(task.comment_));
if (OB_SUCCESS != (tmp_ret = ObPartitionMigrationStatusMgr::get_instance().add_status(task))) {
STORAGE_LOG(WARN, "failed to add partition migration status", K(tmp_ret), K(task));
}
}
return ret;
}
int ObBackupPrepareTask::build_backup_prepare_context()
{
int ret = OB_SUCCESS;
ObIPartitionGroup* partition = NULL;
ObPGStorage* pg_storage = NULL;
int64_t trans_table_end_log_ts = 0;
int64_t trans_table_timestamp = 0;
ObPartitionGroupMeta local_pg_meta;
ObPartGroupMigrationTask* group_task = NULL;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not inited", K(ret));
} else if (OB_NOT_NULL(partition = ctx_->get_partition())) {
// do nothing
pg_storage = &(partition->get_pg_storage());
LOG_INFO("partition already hold, do not hold again");
} else if (OB_FAIL(ObPartitionService::get_instance().get_partition(
ctx_->replica_op_arg_.key_, ctx_->partition_guard_))) {
LOG_WARN("partition is not exist, cannot backup partition", K(ret), K(*ctx_));
} else if (OB_ISNULL(partition = ctx_->partition_guard_.get_partition_group()) ||
OB_ISNULL(pg_storage = &(partition->get_pg_storage()))) {
ret = OB_ERR_SYS;
STORAGE_LOG(ERROR, "partition must not null", K(ret));
} else if (OB_FAIL(pg_storage->get_backup_pg_meta_data(
ctx_->replica_op_arg_.backup_arg_.backup_snapshot_version_, ctx_->pg_meta_))) {
STORAGE_LOG(WARN, "failed to get backup pg meta data", K(ret), K(ctx_->replica_op_arg_));
} else if (!ctx_->pg_meta_.is_valid()) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "backup ctx pg meta is invalid", K(ret), K(*ctx_));
} else if (OB_FAIL(
pg_storage->get_trans_table_end_log_ts_and_timestamp(trans_table_end_log_ts, trans_table_timestamp))) {
LOG_WARN("failed to get trans table end_log_ts and timestamp", K(ret), K(ctx_->replica_op_arg_));
} else {
ctx_->local_publish_version_ = ctx_->pg_meta_.storage_info_.get_data_info().get_publish_version();
ctx_->local_last_replay_log_id_ = ctx_->pg_meta_.storage_info_.get_clog_info().get_last_replay_log_id();
ctx_->local_last_replay_log_ts_ =
std::max(trans_table_end_log_ts, ctx_->pg_meta_.storage_info_.get_data_info().get_last_replay_log_ts());
ctx_->migrate_src_info_.src_addr_ = ctx_->replica_op_arg_.data_src_.get_server();
ctx_->migrate_src_info_.cluster_id_ = ctx_->replica_op_arg_.cluster_id_;
}
if (OB_FAIL(ret) && NULL != ctx_) {
ctx_->can_rebuild_ = false;
LOG_WARN("failed during build backup prepare context, set cannot rebuild", K(ret), K(*ctx_));
}
return ret;
}
int ObBackupPrepareTask::build_backup_pg_partition_info()
{
int ret = OB_SUCCESS;
ObIPGPartitionBaseDataMetaObReader* reader = NULL;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not inited", K(ret));
} else if (OB_FAIL(get_partition_table_info_backup_reader(ctx_->migrate_src_info_, reader))) {
LOG_WARN("failed to get partition info reader", K(ret), K(ctx_));
} else if (OB_FAIL(build_table_partition_info(ctx_->replica_op_arg_, reader))) {
LOG_WARN("failed to build table partition info", K(ret), K(ctx_->replica_op_arg_));
}
if (NULL != reader) {
cp_fty_->free(reader);
}
return ret;
}
int ObBackupPrepareTask::build_backup_partition_info(const ObPGPartitionMetaInfo& partition_meta_info,
const common::ObIArray<obrpc::ObFetchTableInfoResult>& table_info_res,
const common::ObIArray<uint64_t>& table_id_list, ObPartitionMigrateCtx& part_migrate_ctx)
{
int ret = OB_SUCCESS;
ObMigrateTableInfo table_info;
int64_t cost_ts = ObTimeUtility::current_time();
const ObPartitionKey& pkey = partition_meta_info.meta_.pkey_;
ObMigratePartitionInfo& info = part_migrate_ctx.copy_info_;
part_migrate_ctx.ctx_ = ctx_;
DEBUG_SYNC(BEFORE_BUILD_MIGRATE_PARTITION_INFO);
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("not inited", K(ret));
} else if (!partition_meta_info.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("build backup partition info get invalid argument", K(ret), K(partition_meta_info));
} else if (OB_FAIL(info.meta_.deep_copy(partition_meta_info.meta_))) {
LOG_WARN("fail to deep copy partition store meta", K(ret), K(partition_meta_info.meta_));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < table_info_res.count(); ++i) {
table_info.reuse();
const obrpc::ObFetchTableInfoResult& table_res = table_info_res.at(i);
const uint64_t table_id = table_id_list.at(i);
LOG_INFO("build_backup_partition_info for table", "table_id", table_id);
if (OB_FAIL(build_backup_table_info(table_id, pkey, table_res, table_info))) {
LOG_WARN("failed to build backup table info", K(ret));
} else if (OB_FAIL(info.table_id_list_.push_back(table_id))) {
LOG_WARN("failed to push table id into array", K(ret));
} else if (OB_FAIL(info.table_infos_.push_back(table_info))) {
LOG_WARN("failed to add backup table info", K(ret));
} else {
LOG_INFO("add table info", K(table_info));
}
}
}
cost_ts = ObTimeUtility::current_time() - cost_ts;
LOG_INFO("build_backup_partition_info",
K(cost_ts),
"pkey",
info.meta_.pkey_,
"src",
info.src_,
"table_count",
info.table_id_list_.count());
return ret;
}
int ObBackupPrepareTask::build_backup_table_info(const uint64_t table_id, const ObPartitionKey& pkey,
const obrpc::ObFetchTableInfoResult& result, ObMigrateTableInfo& info)
{
int ret = OB_SUCCESS;
const ObIArray<ObITable::TableKey>& local_tables_info = result.table_keys_;
int64_t need_reserve_major_snapshot;
info.reuse();
info.table_id_ = table_id;
ObBackupDag* backup_dag = NULL;
ObBackupDataType backup_data_type;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_ERROR("not inited", K(ret));
} else if (OB_ISNULL(backup_dag = reinterpret_cast<ObBackupDag*>(get_dag()))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to get backup dag", K(ret));
} else if (FALSE_IT(backup_data_type = backup_dag->get_backup_data_type())) {
} else if (backup_data_type.is_major_backup()) {
if (OB_FAIL(build_backup_major_sstable(local_tables_info, info.major_sstables_))) {
LOG_WARN("failed to build backup major sstable", K(ret), K(table_id), K(pkey));
}
} else if (backup_data_type.is_minor_backup()) {
if (OB_FAIL(build_backup_minor_sstable(local_tables_info, info.minor_sstables_))) {
LOG_WARN("failed to build backup minor sstable", K(ret), K(table_id), K(pkey));
}
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("backup dag type is unexpected", K(ret), K(*ctx_), K(backup_data_type));
}
if (OB_FAIL(ret)) {
} else {
info.multi_version_start_ = result.multi_version_start_;
info.ready_for_read_ = result.is_ready_for_read_;
FLOG_INFO("succeed build_backup_table_info", K(result), K(info), K(pkey), K(table_id));
}
return ret;
}
int ObBackupPrepareTask::schedule_backup_tasks()
{
int ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS;
bool need_schedule = false;
ObArray<ObITask*> last_task_array;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_ERROR("not inited", K(ret));
} else if (OB_FAIL(generate_pg_backup_tasks(last_task_array))) {
LOG_WARN("failed to generate backup task", K(ret));
} else {
if (OB_SUCCESS != (tmp_ret = ctx_->update_partition_migration_status())) {
LOG_WARN("failed to update partition migration status", K(tmp_ret));
}
}
return ret;
}
int ObBackupPrepareTask::build_backup_major_sstable(
const ObIArray<ObITable::TableKey>& local_tables, ObIArray<ObMigrateTableInfo::SSTableInfo>& copy_sstables)
{
int ret = OB_SUCCESS;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("backup prepare task do not init", K(ret));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < local_tables.count(); ++i) {
const ObITable::TableKey& local_table = local_tables.at(i);
if (local_table.is_major_sstable()) {
ObMigrateTableInfo::SSTableInfo info;
info.src_table_key_ = local_table;
info.dest_base_version_ = local_table.get_base_version();
info.dest_log_ts_range_ = local_table.log_ts_range_;
if (OB_FAIL(copy_sstables.push_back(info))) {
LOG_WARN("failed to push backup table into array", K(ret), K(local_table), K(copy_sstables));
}
}
}
if (OB_SUCC(ret)) {
LOG_INFO("build_backup_major_sstable", K(copy_sstables), K(local_tables));
}
}
return ret;
}
int ObBackupPrepareTask::build_backup_minor_sstable(
const ObIArray<ObITable::TableKey>& local_tables, ObIArray<ObMigrateTableInfo::SSTableInfo>& copy_sstables)
{
int ret = OB_SUCCESS;
ObArray<ObMigrateTableInfo::SSTableInfo> tmp_copy_sstables;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("migrate prepare task do not init", K(ret));
} else {
ObMigrateTableInfo::SSTableInfo info;
for (int64_t i = 0; OB_SUCC(ret) && i < local_tables.count(); ++i) {
const ObITable::TableKey& local_table = local_tables.at(i);
if (local_table.is_minor_sstable()) {
info.reset();
info.src_table_key_ = local_table;
info.dest_base_version_ = local_table.trans_version_range_.base_version_;
info.dest_log_ts_range_ = local_table.log_ts_range_;
if (OB_FAIL(copy_sstables.push_back(info))) {
LOG_WARN("failed to push backup table into array", K(ret), K(local_table), K(copy_sstables));
}
}
}
if (OB_SUCC(ret)) {
LOG_INFO("build_backup_minor_sstable", K(copy_sstables), K(local_tables));
}
}
return ret;
}
int ObBackupPrepareTask::generate_pg_backup_tasks(ObIArray<ObITask*>& last_task_array)
{
int ret = OB_SUCCESS;
ObITask* last_task = NULL;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_ERROR("not inited", K(ret));
} else if (BACKUP_REPLICA_OP != ctx_->replica_op_arg_.type_) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("unexcepted op type, ", K(ret), K(ctx_->replica_op_arg_));
} else if (!ObReplicaTypeCheck::is_replica_with_ssstore(ctx_->replica_op_arg_.src_.get_replica_type())) {
ret = OB_REPLICA_CANNOT_BACKUP;
LOG_WARN("replica has no sstore.", K(ret), K(ctx_->replica_op_arg_));
} else if (OB_FAIL(generate_backup_tasks(last_task))) {
LOG_WARN("fail to generate backup tasks", K(ret));
} else if (OB_FAIL(last_task_array.push_back(last_task))) {
LOG_WARN("fail to push last task into last task array", K(ret));
}
return ret;
}
int ObBackupPrepareTask::generate_backup_tasks(ObITask*& last_task)
{
int ret = OB_SUCCESS;
ObFakeTask* wait_backup_finish_task = NULL;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_ERROR("not inited", K(ret));
} else if (OB_FAIL(generate_wait_backup_finish_task(wait_backup_finish_task))) {
LOG_WARN("failed to generate_wait_backup_finish_task", K(ret));
} else if (OB_ISNULL(wait_backup_finish_task)) {
ret = OB_ERR_SYS;
LOG_ERROR("wait_rebuild_finish_task must not null", K(ret));
} else if (OB_FAIL(generate_backup_tasks(*wait_backup_finish_task))) {
LOG_WARN("failed to generate_backup_tasks", K(ret));
}
if (OB_SUCC(ret)) {
last_task = wait_backup_finish_task;
}
return ret;
}
int ObBackupPrepareTask::generate_backup_tasks(ObFakeTask& wait_backup_finish_task)
{
int ret = OB_SUCCESS;
ObITask* parent_task = this;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_ERROR("not inited", K(ret));
} else if (OB_FAIL(generate_backup_sstable_tasks(parent_task))) {
LOG_WARN("failed to generate backup major tasks", K(ret));
} else if (OB_FAIL(parent_task->add_child(wait_backup_finish_task))) {
LOG_WARN("failed to add wait_migrate_finish_task", K(ret));
}
return ret;
}
int ObBackupPrepareTask::generate_backup_sstable_tasks(share::ObITask*& parent_task)
{
int ret = OB_SUCCESS;
ObFakeTask* wait_finish_task = NULL;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("migrate prepare task do not init", K(ret));
} else if (OB_ISNULL(parent_task)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), KP(parent_task));
} else if (OB_FAIL(dag_->alloc_task(wait_finish_task))) {
LOG_WARN("failed to alloc wait finish task", K(ret));
} else if (OB_FAIL(generate_backup_sstable_copy_task(parent_task, wait_finish_task))) {
LOG_WARN("failed to generate major sstable copy task", K(ret));
} else if (OB_FAIL(dag_->add_task(*wait_finish_task))) {
LOG_WARN("failed to add wait finish task", K(ret));
} else {
parent_task = wait_finish_task;
LOG_DEBUG("succeed to generate_backup_sstable_task");
}
return ret;
}
int ObBackupPrepareTask::generate_backup_sstable_copy_task(ObITask* parent_task, ObITask* child_task)
{
int ret = OB_SUCCESS;
ObBackupCopyPhysicalTask* copy_task = NULL;
ObBackupFinishTask* finish_task = NULL;
const int64_t task_idx = 0;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not inited", K(ret));
} else if (OB_ISNULL(parent_task) || OB_ISNULL(child_task)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", K(ret), KP(parent_task), KP(child_task));
} else if (OB_FAIL(dag_->alloc_task(finish_task))) {
LOG_WARN("failed to alloc finish task", K(ret));
} else if (OB_FAIL(build_backup_physical_ctx(ctx_->physical_backup_ctx_))) {
LOG_WARN("failed to build physical sstable ctx", K(ret));
} else if (OB_FAIL(finish_task->init(*ctx_))) {
LOG_WARN("failed to init finish task", K(ret));
} else if (OB_FAIL(finish_task->add_child(*child_task))) {
LOG_WARN("failed to add child", K(ret));
}
if (OB_SUCC(ret)) {
if (ctx_->physical_backup_ctx_.get_task_count() > 0) {
// parent->copy->finish->child
if (OB_FAIL(dag_->alloc_task(copy_task))) {
LOG_WARN("failed to alloc copy task", K(ret));
} else if (OB_FAIL(copy_task->init(task_idx, *ctx_))) {
LOG_WARN("failed to init copy task", K(ret));
} else if (OB_FAIL(parent_task->add_child(*copy_task))) {
LOG_WARN("failed to add child copy task", K(ret));
} else if (OB_FAIL(copy_task->add_child(*finish_task))) {
LOG_WARN("failed to add child finish task", K(ret));
} else if (OB_FAIL(dag_->add_task(*copy_task))) {
LOG_WARN("failed to add copy task to dag", K(ret));
}
} else {
// parent->finish->child
if (OB_FAIL(parent_task->add_child(*finish_task))) {
LOG_WARN("failed to add child finish_task for parent", K(ret));
}
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(dag_->add_task(*finish_task))) {
LOG_WARN("failed to add finish task to dag", K(ret));
}
}
return ret;
}
int ObBackupPrepareTask::build_backup_physical_ctx(ObBackupPhysicalPGCtx& ctx)
{
int ret = OB_SUCCESS;
ObBackupDataType backup_data_type;
ObBackupDag* backup_dag = NULL;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not inited", K(ret));
} else if (OB_ISNULL(backup_dag = reinterpret_cast<ObBackupDag*>(get_dag()))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to get backup dag", K(ret));
} else if (FALSE_IT(backup_data_type = backup_dag->get_backup_data_type())) {
} else if (OB_FAIL(ctx.open(
*bandwidth_throttle_, ctx_->replica_op_arg_.backup_arg_, ctx_->pg_meta_.pg_key_, backup_data_type))) {
LOG_WARN("ctx init failed", K(ret));
} else if (OB_FAIL(fetch_backup_sstables(backup_data_type, ctx.table_keys_))) {
LOG_WARN("failed to build sub task", K(ret));
} else if (ctx.table_keys_.empty()) {
LOG_INFO("pg don't have sstable, ", K(ret), K(ctx_->pg_meta_));
} else if (OB_FAIL(build_backup_sub_task(ctx))) {
LOG_WARN("failed to build sub task", K(ret));
} else {
ATOMIC_AAF(&ctx_->data_statics_.total_macro_block_, ctx.macro_block_count_);
LOG_INFO("succeed to build physical sstable ctx", K(ctx));
}
return ret;
}
int ObBackupPrepareTask::fetch_backup_sstables(
const ObBackupDataType& backup_data_type, ObIArray<ObITable::TableKey>& table_keys)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_ERROR("not inited", K(ret));
} else if (!backup_data_type.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("fetch backup sstables get invalid argument", K(ret), K(backup_data_type));
} else if (backup_data_type.is_major_backup()) {
if (OB_FAIL(fetch_backup_major_sstables(table_keys))) {
LOG_WARN("failed to fetch backup major sstables", K(ret));
}
} else {
// backup minor
if (OB_FAIL(fetch_backup_minor_sstables(table_keys))) {
LOG_WARN("failed to fetch backup major sstables", K(ret));
}
}
return ret;
}
int ObBackupPrepareTask::fetch_backup_major_sstables(ObIArray<ObITable::TableKey>& table_keys)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_ERROR("not inited", K(ret));
} else {
// partition
for (int64_t i = 0; OB_SUCC(ret) && i < ctx_->part_ctx_array_.count(); ++i) {
ObPartitionMigrateCtx& part_migrate_ctx = ctx_->part_ctx_array_.at(i);
if (OB_LIKELY(!part_migrate_ctx.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("part_migrate_ctx is invalid", K(ret), K(part_migrate_ctx));
} else {
// tables
const ObArray<ObMigrateTableInfo>& table_infos = part_migrate_ctx.copy_info_.table_infos_;
for (int64_t i = 0; OB_SUCC(ret) && i < table_infos.count(); ++i) {
const ObMigrateTableInfo& table_info = table_infos.at(i);
// major sstables
for (int64_t sstable_idx = 0; OB_SUCC(ret) && sstable_idx < table_info.major_sstables_.count();
++sstable_idx) {
const ObITable::TableKey& major_table_key = table_info.major_sstables_.at(sstable_idx).src_table_key_;
if (OB_UNLIKELY(!major_table_key.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid major table key", K(ret), K(major_table_key));
} else if (OB_UNLIKELY(!ObITable::is_major_sstable(major_table_key.table_type_))) {
ret = OB_ERR_SYS;
LOG_ERROR("table type is not major sstable", K(ret), K(major_table_key), K(table_info));
} else if (OB_FAIL(table_keys.push_back(major_table_key))) {
LOG_WARN("failed to push major table key into array", K(ret), K(major_table_key));
}
}
}
}
}
}
return ret;
}
int ObBackupPrepareTask::fetch_backup_minor_sstables(ObIArray<ObITable::TableKey>& table_keys)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_ERROR("not inited", K(ret));
} else {
// partition
for (int64_t i = 0; OB_SUCC(ret) && i < ctx_->part_ctx_array_.count(); ++i) {
ObPartitionMigrateCtx& part_migrate_ctx = ctx_->part_ctx_array_.at(i);
if (OB_LIKELY(!part_migrate_ctx.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("part_migrate_ctx is invalid", K(ret), K(part_migrate_ctx));
} else {
// tables
const ObArray<ObMigrateTableInfo>& table_infos = part_migrate_ctx.copy_info_.table_infos_;
for (int64_t i = 0; OB_SUCC(ret) && i < table_infos.count(); ++i) {
const ObMigrateTableInfo& table_info = table_infos.at(i);
// minor sstables
for (int64_t sstable_idx = 0; OB_SUCC(ret) && sstable_idx < table_info.minor_sstables_.count();
++sstable_idx) {
const ObITable::TableKey& minor_table_key = table_info.minor_sstables_.at(sstable_idx).src_table_key_;
if (OB_UNLIKELY(!minor_table_key.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid major table key", K(ret), K(minor_table_key));
} else if (OB_UNLIKELY(!ObITable::is_minor_sstable(minor_table_key.table_type_))) {
ret = OB_ERR_SYS;
LOG_ERROR("table type is not minor sstable", K(ret), K(minor_table_key), K(table_info));
} else if (OB_FAIL(table_keys.push_back(minor_table_key))) {
LOG_WARN("failed to push major table key into array", K(ret), K(minor_table_key));
}
}
}
}
}
}
return ret;
}
int ObBackupPrepareTask::build_backup_sub_task(ObBackupPhysicalPGCtx& ctx)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_ERROR("not inited", K(ret));
} else {
for (int i = 0; OB_SUCC(ret) && i < ctx.table_keys_.count(); ++i) {
ObTableHandle tmp_handle;
ObSSTable* sstable = NULL;
int64_t sstable_macro_count = 0;
ObITable::TableKey& table_key = ctx.table_keys_.at(i);
if (OB_FAIL(ObPartitionService::get_instance().acquire_sstable(table_key, tmp_handle))) {
STORAGE_LOG(WARN, "failed to get table", K(table_key), K(ret));
} else if (OB_FAIL(tmp_handle.get_sstable(sstable))) {
STORAGE_LOG(WARN, "failed to get table", K(table_key), K(ret));
} else if (OB_ISNULL(sstable)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "sstable should not be null", K(ret));
} else if (0 == (sstable_macro_count = sstable->get_meta().get_total_macro_block_count())) {
// empty sstable
} else {
ObBackupMacroBlockInfo block_info;
block_info.table_key_ = table_key;
block_info.start_index_ = 0;
block_info.cur_block_count_ = sstable_macro_count;
block_info.total_block_count_ = sstable_macro_count;
if (OB_FAIL(ctx.add_backup_macro_block_info(block_info))) {
STORAGE_LOG(WARN, "add backup block info failed", K(ret), K(block_info));
}
}
}
}
return ret;
}
int ObBackupPrepareTask::generate_wait_backup_finish_task(ObFakeTask*& wait_backup_finish_task)
{
int ret = OB_SUCCESS;
wait_backup_finish_task = NULL;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_ERROR("not inited", K(ret));
} else if (OB_FAIL(dag_->alloc_task(wait_backup_finish_task))) {
LOG_WARN("failed to alloc wait_rebuild_finish_task", K(ret));
} else if (OB_FAIL(add_child(*wait_backup_finish_task))) {
LOG_WARN("failed to add child wait_rebuild_finish_task", K(ret));
} else if (OB_FAIL(dag_->add_task(*wait_backup_finish_task))) {
LOG_WARN("failed to add wait_rebuild_finish_task", K(ret));
}
return ret;
}
int ObBackupPrepareTask::build_table_partition_info(
const ObReplicaOpArg& arg, ObIPGPartitionBaseDataMetaObReader* reader)
{
int ret = OB_SUCCESS;
ObPartitionMigrateCtx part_migrate_ctx;
ObArray<obrpc::ObFetchTableInfoResult> table_info_res;
ObArray<uint64_t> table_id_list;
ObPGPartitionMetaInfo partition_meta_info;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("migrate prepare task do not init", K(ret));
} else if (!arg.is_valid() || OB_ISNULL(reader)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("build table partition info get invalid argument", K(ret), K(arg), KP(reader));
} else {
while (OB_SUCC(ret)) {
partition_meta_info.reset();
table_info_res.reuse();
table_id_list.reuse();
part_migrate_ctx.reset();
if (OB_FAIL(reader->fetch_pg_partition_meta_info(partition_meta_info))) {
if (OB_ITER_END != ret) {
LOG_WARN("fail to fetch pg partition meta info", K(ret), K(ctx_->replica_op_arg_));
} else {
ret = OB_SUCCESS;
break;
}
} else if (OB_FAIL(table_id_list.assign(partition_meta_info.table_id_list_))) {
LOG_WARN("failed to assign table id list", K(ret));
} else if (OB_FAIL(table_info_res.assign(partition_meta_info.table_info_))) {
LOG_WARN("failed to copy needed table info", K(ret), K(partition_meta_info));
} else if (OB_FAIL(build_backup_partition_info(
partition_meta_info, table_info_res, table_id_list, part_migrate_ctx))) {
LOG_WARN("fail to build migrate partition info", K(ret), K(partition_meta_info));
} else if (OB_FAIL(ctx_->part_ctx_array_.push_back(part_migrate_ctx))) {
LOG_WARN("fail to push part migrate ctx into array", K(ret), K(part_migrate_ctx));
}
}
}
return ret;
}
int ObBackupPrepareTask::get_partition_table_info_backup_reader(
const ObMigrateSrcInfo& src_info, ObIPGPartitionBaseDataMetaObReader*& reader)
{
int ret = OB_SUCCESS;
UNUSED(src_info);
ObPGPartitionBaseDataMetaBackupReader* tmp_reader = NULL;
if (OB_ISNULL(cp_fty_)) {
ret = OB_ERR_SYS;
LOG_ERROR("cp fty should not be NULL", K(ret), KP(cp_fty_));
} else if (OB_ISNULL(tmp_reader = cp_fty_->get_pg_info_backup_reader())) {
ret = OB_ALLOCATE_MEMORY_FAILED;
STORAGE_LOG(WARN, "failed to get macro block ob reader", K(ret));
} else if (OB_ISNULL(ctx_->backup_meta_reader_)) {
ret = OB_ERR_SYS;
LOG_ERROR("ctx_->backup_meta_reader_ must not null", K(ret));
} else if (OB_FAIL(tmp_reader->init(ctx_->pg_meta_.partitions_, ctx_->backup_meta_reader_))) {
LOG_WARN("fail to init partition table info restore reader", K(ret));
} else {
reader = tmp_reader;
tmp_reader = NULL;
}
if (NULL != cp_fty_) {
if (NULL != tmp_reader) {
cp_fty_->free(tmp_reader);
tmp_reader = NULL;
}
}
return ret;
}
int ObBackupPrepareTask::check_backup_data_continues()
{
int ret = OB_SUCCESS;
ObIPartitionGroup* partition_group = NULL;
clog::ObPGLogArchiveStatus pg_log_archive_status;
ObLogArchiveBackupInfo archive_backup_info;
const int64_t start_ts = ObTimeUtility::current_time();
static const int64_t OB_MAX_RETRY_TIME = 60 * 1000 * 1000; // 60s
static const int64_t OB_SINGLE_SLEEP_US = 1 * 1000 * 1000; // 1s
const int64_t last_replay_log_id = ctx_->pg_meta_.storage_info_.get_clog_info().get_last_replay_log_id();
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("migrate prepare task do not init", K(ret));
} else if (BACKUP_REPLICA_OP != ctx_->replica_op_arg_.type_) {
// do nothing
} else if (OB_ISNULL(partition_group = ctx_->get_partition())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("backup local partition should not be NULL", K(ret), KP(partition_group), K(*ctx_));
} else {
while (OB_SUCC(ret)) {
if (OB_FAIL(partition_group->get_log_archive_status(pg_log_archive_status))) {
LOG_WARN("failed to get log archive status", K(ret), K(*ctx_));
} else if (OB_FAIL(ObBackupInfoMgr::get_instance().get_log_archive_backup_info(archive_backup_info))) {
LOG_WARN("failed to get log archive backup info", K(ret));
} else if (ObLogArchiveStatus::INVALID == pg_log_archive_status.status_) {
if (ObLogArchiveStatus::DOING == archive_backup_info.status_.status_) {
LOG_WARN("partition group log archive status is invalid, need retry",
K(pg_log_archive_status),
K(archive_backup_info));
ret = OB_EAGAIN;
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN(
"server do not archive log, can not backup", K(ret), K(pg_log_archive_status), K(archive_backup_info));
}
} else if (ObLogArchiveStatus::DOING == pg_log_archive_status.status_) {
if (pg_log_archive_status.archive_incarnation_ != archive_backup_info.status_.incarnation_ ||
pg_log_archive_status.log_archive_round_ != archive_backup_info.status_.round_) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("pg archive round or incarnation not equal to server archive info",
K(ret),
K(pg_log_archive_status),
K(archive_backup_info));
} else if (pg_log_archive_status.round_start_log_id_ < 1) {
ret = OB_LOG_ARCHIVE_STAT_NOT_MATCH;
LOG_WARN("log archive status is not match", K(ret), K(pg_log_archive_status));
} else if (last_replay_log_id + 1 < pg_log_archive_status.round_start_log_id_) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("base data and log archive data do not continues",
K(ret),
K(pg_log_archive_status),
K(ctx_->pg_meta_),
K(last_replay_log_id));
} else {
break;
}
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("log archive status is unexpected", K(ret), K(pg_log_archive_status));
}
if (OB_EAGAIN == ret) {
const int64_t cost_ts = ObTimeUtility::current_time() - start_ts;
if (cost_ts < OB_MAX_RETRY_TIME) {
usleep(OB_SINGLE_SLEEP_US);
ret = OB_SUCCESS;
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("server start do archive log, but partition retry many times status still wrong",
K(ret),
K(pg_log_archive_status),
K(archive_backup_info),
K(*ctx_));
}
}
}
}
return ret;
}
} // namespace storage
} // namespace oceanbase
Reference in New Issue View Git Blame Copy Permalink