From 36f69ff4abfa5d55a991f1d8455d53f09c3357cf Mon Sep 17 00:00:00 2001 From: hamstersox <673144759@qq.com> Date: Tue, 4 Jul 2023 07:17:58 +0000 Subject: [PATCH] Don't set data status when restore --- .../ob_all_virtual_tablet_info.cpp | 22 ++- .../backup/ob_backup_data_scheduler.cpp | 17 ++ .../backup/ob_backup_data_scheduler.h | 1 + .../high_availability/ob_ls_migration.cpp | 146 +++++++++++++++++- .../high_availability/ob_ls_migration.h | 27 +++- .../high_availability/ob_ls_restore.cpp | 1 - .../ob_physical_copy_task.cpp | 45 ++---- .../high_availability/ob_physical_copy_task.h | 15 +- .../ob_storage_ha_tablet_builder.cpp | 7 +- .../ob_tablet_group_restore.cpp | 104 ++++++------- .../ob_tablet_group_restore.h | 7 +- .../ob_transfer_backfill_tx.cpp | 7 +- .../ob_transfer_backfill_tx.h | 2 +- src/storage/ls/ob_ls_tablet_service.cpp | 7 +- src/storage/ob_storage_struct.cpp | 10 +- src/storage/ob_storage_struct.h | 4 +- src/storage/restore/ob_ls_restore_handler.cpp | 36 +++-- src/storage/restore/ob_ls_restore_handler.h | 1 + src/storage/tablet/ob_tablet.cpp | 3 +- 19 files changed, 334 insertions(+), 128 deletions(-) diff --git a/src/observer/virtual_table/ob_all_virtual_tablet_info.cpp b/src/observer/virtual_table/ob_all_virtual_tablet_info.cpp index 365718ca7..e1bac3bca 100644 --- a/src/observer/virtual_table/ob_all_virtual_tablet_info.cpp +++ b/src/observer/virtual_table/ob_all_virtual_tablet_info.cpp @@ -129,6 +129,7 @@ int ObAllVirtualTabletInfo::process_curr_tenant(ObNewRow *&row) ObTablet *tablet = nullptr; ObTabletCreateDeleteMdsUserData latest_user_data; bool is_committed = false; + bool is_empty_result = false; if (NULL == allocator_) { ret = OB_NOT_INIT; SERVER_LOG(WARN, "allocator_ shouldn't be NULL", K(allocator_), K(ret)); @@ -146,8 +147,16 @@ int ObAllVirtualTabletInfo::process_curr_tenant(ObNewRow *&row) ret = OB_ERR_UNEXPECTED; SERVER_LOG(WARN, "tablet should not null", K(ret), K(tablet_handle)); } else if (OB_FAIL(tablet->ObITabletMdsInterface::get_latest_tablet_status(latest_user_data, is_committed))) { - SERVER_LOG(WARN, "failed to get latest tablet status", K(ret), KPC(tablet)); - } else { + if (OB_EMPTY_RESULT == ret || OB_ERR_SHARED_LOCK_CONFLICT == ret) { + is_committed = false; + is_empty_result = true; + ret = OB_SUCCESS; + } else { + SERVER_LOG(WARN, "failed to get latest tablet status", K(ret), KPC(tablet)); + } + } + + if (OB_SUCC(ret)) { const ObTabletMeta &tablet_meta = tablet->get_tablet_meta(); const int64_t col_count = output_column_ids_.count(); for (int64_t i = 0; OB_SUCC(ret) && i < col_count; ++i) { @@ -222,10 +231,15 @@ int ObAllVirtualTabletInfo::process_curr_tenant(ObNewRow *&row) } } break; - case OB_APP_MIN_COLUMN_ID + 14: + case OB_APP_MIN_COLUMN_ID + 14: { // tablet_status - cur_row_.cells_[i].set_int(static_cast(latest_user_data.get_tablet_status())); + if (is_empty_result) { + cur_row_.cells_[i].set_int(static_cast(ObTabletStatus::MAX)); + } else { + cur_row_.cells_[i].set_int(static_cast(latest_user_data.get_tablet_status())); + } break; + } case OB_APP_MIN_COLUMN_ID + 15: // is_committed cur_row_.cells_[i].set_int(is_committed ? 1 : 0); diff --git a/src/rootserver/backup/ob_backup_data_scheduler.cpp b/src/rootserver/backup/ob_backup_data_scheduler.cpp index 2f9ad7318..f43b99258 100644 --- a/src/rootserver/backup/ob_backup_data_scheduler.cpp +++ b/src/rootserver/backup/ob_backup_data_scheduler.cpp @@ -1181,10 +1181,13 @@ int ObUserTenantBackupJobMgr::move_to_history_() LOG_INFO("start to move backup job to history", KPC(job_attr_)); ObMySQLTransaction trans; ObBackupSetTaskMgr set_task_mgr; + ObTimeoutCtx timeout_ctx; if (is_sys_tenant(job_attr_->initiator_tenant_id_) && OB_FAIL(report_failed_to_initiator_())) { LOG_WARN("fail to report job finish to initiator tenant id", K(ret), KPC(job_attr_)); } else if (OB_FAIL(trans.start(sql_proxy_, tenant_id_))) { LOG_WARN("[DATA_BACKUP]failed to start trans", K(ret)); + } else if (OB_FAIL(set_query_timeout_and_trx_timeout_(timeout_ctx))) { + LOG_WARN("failed to set query timeout and trx timeout", K(ret)); } else { if (OB_FAIL(set_task_mgr.init(tenant_id_, *job_attr_, *sql_proxy_, *rpc_proxy_, *task_scheduler_, *schema_service_, *backup_service_))) { @@ -1219,6 +1222,20 @@ int ObUserTenantBackupJobMgr::move_to_history_() return ret; } +int ObUserTenantBackupJobMgr::set_query_timeout_and_trx_timeout_(ObTimeoutCtx &timeout_ctx) +{ + int ret = OB_SUCCESS; + const int64_t ob_query_timeout = 600 * 1000 * 1000; // 600s + const int64_t ob_trx_timeout = 600 * 1000 * 1000; // 600s + const int64_t abs_timeout = ObTimeUtility::current_time() + ob_query_timeout; + if (OB_FAIL(timeout_ctx.set_trx_timeout_us(ob_trx_timeout))) { + LOG_WARN("failed to set trx timeout us", K(ret), K(ob_trx_timeout)); + } else if (OB_FAIL(timeout_ctx.set_abs_timeout(abs_timeout))) { + LOG_WARN("failed to set abs timeout", K(ret)); + } + return ret; +} + int ObUserTenantBackupJobMgr::report_failed_to_initiator_() { int ret = OB_SUCCESS; diff --git a/src/rootserver/backup/ob_backup_data_scheduler.h b/src/rootserver/backup/ob_backup_data_scheduler.h index 76b01edd9..e8a33ecad 100644 --- a/src/rootserver/backup/ob_backup_data_scheduler.h +++ b/src/rootserver/backup/ob_backup_data_scheduler.h @@ -162,6 +162,7 @@ private: int check_dest_validity_(); int cancel_(); int update_set_task_to_canceling_(); + int set_query_timeout_and_trx_timeout_(ObTimeoutCtx &timeout_ctx); private: DISALLOW_COPY_AND_ASSIGN(ObUserTenantBackupJobMgr); }; diff --git a/src/storage/high_availability/ob_ls_migration.cpp b/src/storage/high_availability/ob_ls_migration.cpp index 5761be39e..7f897af58 100644 --- a/src/storage/high_availability/ob_ls_migration.cpp +++ b/src/storage/high_availability/ob_ls_migration.cpp @@ -172,7 +172,7 @@ int ObCopyTabletCtx::set_copy_tablet_status(const ObCopyTabletStatus::STATUS &st return ret; } -int ObCopyTabletCtx::get_copy_tablet_status(ObCopyTabletStatus::STATUS &status) +int ObCopyTabletCtx::get_copy_tablet_status(ObCopyTabletStatus::STATUS &status) const { int ret = OB_SUCCESS; status = ObCopyTabletStatus::MAX_STATUS; @@ -2352,7 +2352,7 @@ int ObTabletMigrationTask::generate_migration_tasks_() int ret = OB_SUCCESS; ObITask *parent_task = this; ObTabletCopyFinishTask *tablet_copy_finish_task = nullptr; - + ObTabletFinishMigrationTask *tablet_finish_migration_task = nullptr; if (!is_inited_) { ret = OB_NOT_INIT; LOG_ERROR("not inited", K(ret)); @@ -2367,16 +2367,44 @@ int ObTabletMigrationTask::generate_migration_tasks_() LOG_WARN("failed to generate migrate major tasks", K(ret), K(*ctx_)); } else if (OB_FAIL(generate_ddl_copy_tasks_(tablet_copy_finish_task, parent_task))) { LOG_WARN("failed to generate ddl copy tasks", K(ret), K(*ctx_)); + } else if (OB_FAIL(generate_tablet_finish_migration_task_(tablet_finish_migration_task))) { + LOG_WARN("failed to generate tablet finish migration task", K(ret), K(ctx_->arg_)); + } else if (OB_FAIL(tablet_copy_finish_task->add_child(*tablet_finish_migration_task))) { + LOG_WARN("failed to add child finsih task for parent", K(ret), KPC(tablet_finish_migration_task)); } else if (OB_FAIL(parent_task->add_child(*tablet_copy_finish_task))) { LOG_WARN("failed to add tablet copy finish task as child", K(ret), KPC(ctx_)); } else if (OB_FAIL(dag_->add_task(*tablet_copy_finish_task))) { LOG_WARN("failed to add tablet copy finish task", K(ret), KPC(ctx_)); + } else if (OB_FAIL(dag_->add_task(*tablet_finish_migration_task))) { + LOG_WARN("failed to add tablet finish migration task", K(ret)); } else { LOG_INFO("generate sstable migration tasks", KPC(copy_tablet_ctx_), K(copy_table_key_array_)); } return ret; } +int ObTabletMigrationTask::generate_tablet_finish_migration_task_( + ObTabletFinishMigrationTask *&tablet_finish_migration_task) +{ + int ret = OB_SUCCESS; + ObTabletMigrationDag *tablet_migration_dag = nullptr; + ObLS *ls = nullptr; + if (OB_NOT_NULL(tablet_finish_migration_task)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tablet finish migration task must not be null", K(ret), KPC(tablet_finish_migration_task)); + } else if (FALSE_IT(tablet_migration_dag = static_cast(dag_))) { + } else if (OB_FAIL(dag_->alloc_task(tablet_finish_migration_task))) { + LOG_WARN("failed to alloc tablet finish task", K(ret), KPC(ctx_)); + } else if (OB_FAIL(tablet_migration_dag->get_ls(ls))) { + LOG_WARN("failed to get ls", K(ret), KPC(ctx_)); + } else if (OB_FAIL(tablet_finish_migration_task->init(*copy_tablet_ctx_, *ls))) { + LOG_WARN("failed to init tablet copy finish task", K(ret), KPC(ctx_), KPC(copy_tablet_ctx_)); + } else { + LOG_INFO("generate tablet migration finish task", "ls_id", ls->get_ls_id().id(), "tablet_id", copy_tablet_ctx_->tablet_id_); + } + return ret; +} + int ObTabletMigrationTask::generate_minor_copy_tasks_( ObTabletCopyFinishTask *tablet_copy_finish_task, share::ObITask *&parent_task) @@ -2629,7 +2657,8 @@ int ObTabletMigrationTask::generate_tablet_copy_finish_task_( LOG_WARN("failed to get ls", K(ret), KPC(ctx_)); } else if (OB_FAIL(ctx_->ha_table_info_mgr_.get_tablet_meta(copy_tablet_ctx_->tablet_id_, src_tablet_meta))) { LOG_WARN("failed to get src tablet meta", K(ret), KPC(copy_tablet_ctx_)); - } else if (OB_FAIL(tablet_copy_finish_task->init(copy_tablet_ctx_->tablet_id_, ls, reporter, restore_action, src_tablet_meta))) { + } else if (OB_FAIL(tablet_copy_finish_task->init( + copy_tablet_ctx_->tablet_id_, ls, reporter, restore_action, src_tablet_meta, copy_tablet_ctx_))) { LOG_WARN("failed to init tablet copy finish task", K(ret), KPC(ctx_), KPC(copy_tablet_ctx_)); } else { LOG_INFO("generate tablet copy finish task", "ls_id", ls->get_ls_id().id(), "tablet_id", copy_tablet_ctx_->tablet_id_); @@ -2808,6 +2837,117 @@ int ObTabletMigrationTask::check_tablet_replica_validity_(const common::ObTablet return ret; } +/******************ObTabletFinishMigrationTask*********************/ +ObTabletFinishMigrationTask::ObTabletFinishMigrationTask() + : ObITask(TASK_TYPE_MIGRATE_PREPARE), + is_inited_(false), + copy_tablet_ctx_(nullptr), + ls_(nullptr) +{ +} + +ObTabletFinishMigrationTask::~ObTabletFinishMigrationTask() +{ +} + +int ObTabletFinishMigrationTask::init(ObCopyTabletCtx &ctx, ObLS &ls) +{ + int ret = OB_SUCCESS; + if (is_inited_) { + ret = OB_INIT_TWICE; + LOG_WARN("tablet migration finish task init twice", K(ret)); + } else if (!ctx.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ctx)); + } else { + ha_dag_net_ctx_ = static_cast(this->get_dag())->get_ha_dag_net_ctx(); + copy_tablet_ctx_ = &ctx; + ls_ = &ls; + is_inited_ = true; + } + return ret; +} + +int ObTabletFinishMigrationTask::process() +{ + int ret = OB_SUCCESS; + LOG_INFO("start do tablet finish migration task", KPC(copy_tablet_ctx_)); + if (!is_inited_) { + ret = OB_NOT_INIT; + LOG_WARN("tablet finish migration task do not init", K(ret), KPC(copy_tablet_ctx_)); + } else { + bool is_failed = false; + if (ha_dag_net_ctx_->is_failed()) { + is_failed = true; + } else if (OB_FAIL(update_data_and_expected_status_())) { + LOG_WARN("failed to update data and expected status", K(ret)); + } + } + + if (OB_FAIL(ret)) { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = ObStorageHADagUtils::deal_with_fo(ret, this->get_dag()))) { + LOG_WARN("failed to deal with fo", K(ret), K(tmp_ret), KPC(copy_tablet_ctx_)); + } + } + return ret; +} + +int ObTabletFinishMigrationTask::update_data_and_expected_status_() +{ + int ret = OB_SUCCESS; + ObCopyTabletStatus::STATUS status; + if (!is_inited_) { + ret = OB_NOT_INIT; + LOG_WARN("tablet copy finish task do not init", K(ret)); + } else if (OB_FAIL(copy_tablet_ctx_->get_copy_tablet_status(status))) { + LOG_WARN("failed to get copy tablet status", KPC(copy_tablet_ctx_)); + } else if (ObCopyTabletStatus::TABLET_NOT_EXIST == status) { + const ObTabletExpectedStatus::STATUS expected_status = ObTabletExpectedStatus::DELETED; + if (OB_FAIL(ls_->get_tablet_svr()->update_tablet_ha_expected_status(copy_tablet_ctx_->tablet_id_, expected_status))) { + if (OB_TABLET_NOT_EXIST == ret) { + LOG_INFO("migration tablet maybe deleted, skip it", K(ret), KPC(copy_tablet_ctx_)); + ret = OB_SUCCESS; + } else { + LOG_WARN("failed to update tablet ha expected status", K(ret), K(expected_status), KPC(copy_tablet_ctx_)); + } + } else { + LOG_INFO("update tablet expected status", KPC(copy_tablet_ctx_), K(expected_status)); + SERVER_EVENT_ADD("storage_ha", "tablet_finish_migration_task", + "tenant_id", MTL_ID(), + "ls_id", ls_->get_ls_id().id(), + "tablet_id", copy_tablet_ctx_->tablet_id_, + "expected_status", expected_status); + } + } else { +#ifdef ERRSIM + if (OB_SUCC(ret)) { + ret = OB_E(EventTable::EN_UPDATE_TABLET_HA_STATUS_FAILED) OB_SUCCESS; + if (OB_FAIL(ret)) { + STORAGE_LOG(ERROR, "fake EN_UPDATE_TABLET_HA_STATUS_FAILED", K(ret)); + } + } +#endif + const ObTabletDataStatus::STATUS data_status = ObTabletDataStatus::COMPLETE; + if (OB_FAIL(ls_->update_tablet_ha_data_status(copy_tablet_ctx_->tablet_id_, data_status))) { + if (OB_TABLET_NOT_EXIST == ret) { + LOG_INFO("migration tablet maybe deleted, skip it", K(ret), KPC(copy_tablet_ctx_)); + ret = OB_SUCCESS; + } else { + LOG_WARN("[HA]failed to update tablet ha data status", K(ret), KPC(copy_tablet_ctx_), K(data_status)); + } + } else { + LOG_INFO("update tablet ha data status", KPC(copy_tablet_ctx_), K(data_status)); + SERVER_EVENT_ADD("storage_ha", "tablet_finish_migration_task", + "tenant_id", MTL_ID(), + "ls_id", ls_->get_ls_id().id(), + "tablet_id", copy_tablet_ctx_->tablet_id_, + "data_status", data_status); + } + } + return ret; +} + /******************ObDataTabletsMigrationDag*********************/ ObDataTabletsMigrationDag::ObDataTabletsMigrationDag() : ObMigrationDag(ObDagType::DAG_TYPE_DATA_TABLETS_MIGRATION), diff --git a/src/storage/high_availability/ob_ls_migration.h b/src/storage/high_availability/ob_ls_migration.h index 4d2b78c3c..c6ea016b5 100644 --- a/src/storage/high_availability/ob_ls_migration.h +++ b/src/storage/high_availability/ob_ls_migration.h @@ -80,15 +80,15 @@ public: DISALLOW_COPY_AND_ASSIGN(ObMigrationCtx); }; -struct ObCopyTabletCtx +struct ObCopyTabletCtx final: public ObICopyTabletCtx { public: ObCopyTabletCtx(); virtual ~ObCopyTabletCtx(); bool is_valid() const; void reset(); - int set_copy_tablet_status(const ObCopyTabletStatus::STATUS &status); - int get_copy_tablet_status(ObCopyTabletStatus::STATUS &status); + int set_copy_tablet_status(const ObCopyTabletStatus::STATUS &status) override; + int get_copy_tablet_status(ObCopyTabletStatus::STATUS &status) const override; VIRTUAL_TO_STRING_KV(K_(tablet_id), K_(status)); public: @@ -327,6 +327,7 @@ protected: DISALLOW_COPY_AND_ASSIGN(ObTabletMigrationDag); }; +class ObTabletFinishMigrationTask; class ObTabletMigrationTask : public share::ObITask { public: @@ -357,6 +358,8 @@ private: ObTabletCopyFinishTask *tablet_copy_finish_task, ObITask *parent_task, ObITask *child_task); + int generate_tablet_finish_migration_task_( + ObTabletFinishMigrationTask *&tablet_finish_migration_task); int build_copy_table_key_info_(); int build_copy_sstable_info_mgr_(); int generate_tablet_copy_finish_task_( @@ -383,6 +386,24 @@ private: DISALLOW_COPY_AND_ASSIGN(ObTabletMigrationTask); }; +class ObTabletFinishMigrationTask final : public share::ObITask +{ +public: + ObTabletFinishMigrationTask(); + virtual ~ObTabletFinishMigrationTask(); + int init(ObCopyTabletCtx &ctx, ObLS &ls); + virtual int process() override; + VIRTUAL_TO_STRING_KV(K("ObTabletFinishMigrationTask"), KP(this), KPC(ha_dag_net_ctx_), KPC(copy_tablet_ctx_), KPC(ls_)); +private: + int update_data_and_expected_status_(); +private: + bool is_inited_; + ObIHADagNetCtx *ha_dag_net_ctx_; + ObCopyTabletCtx *copy_tablet_ctx_; + ObLS *ls_; + DISALLOW_COPY_AND_ASSIGN(ObTabletFinishMigrationTask); +}; + class ObDataTabletsMigrationDag : public ObMigrationDag { public: diff --git a/src/storage/high_availability/ob_ls_restore.cpp b/src/storage/high_availability/ob_ls_restore.cpp index 8ffc39896..ebc2fe4a6 100644 --- a/src/storage/high_availability/ob_ls_restore.cpp +++ b/src/storage/high_availability/ob_ls_restore.cpp @@ -996,7 +996,6 @@ int ObStartLSRestoreTask::create_tablet_( void ObStartLSRestoreTask::set_tablet_to_restore(ObMigrationTabletParam &tablet_meta) { tablet_meta.ha_status_.set_restore_status(ObTabletRestoreStatus::PENDING); - tablet_meta.ha_status_.set_data_status(ObTabletDataStatus::INCOMPLETE); } int ObStartLSRestoreTask::update_ls_meta_and_create_all_tablets_() diff --git a/src/storage/high_availability/ob_physical_copy_task.cpp b/src/storage/high_availability/ob_physical_copy_task.cpp index 36db30162..bb465cd8e 100644 --- a/src/storage/high_availability/ob_physical_copy_task.cpp +++ b/src/storage/high_availability/ob_physical_copy_task.cpp @@ -1186,7 +1186,8 @@ ObTabletCopyFinishTask::ObTabletCopyFinishTask() major_tables_handle_(), restore_action_(ObTabletRestoreAction::MAX), src_tablet_meta_(nullptr), - status_(ObCopyTabletStatus::MAX_STATUS) + copy_tablet_ctx_(nullptr) + { } @@ -1200,17 +1201,20 @@ int ObTabletCopyFinishTask::init( ObLS *ls, observer::ObIMetaReport *reporter, const ObTabletRestoreAction::ACTION &restore_action, - const ObMigrationTabletParam *src_tablet_meta) + const ObMigrationTabletParam *src_tablet_meta, + ObICopyTabletCtx *copy_tablet_ctx) { int ret = OB_SUCCESS; if (is_inited_) { ret = OB_INIT_TWICE; LOG_WARN("tablet copy finish task init twice", K(ret)); } else if (!tablet_id.is_valid() || OB_ISNULL(ls) || OB_ISNULL(reporter) || OB_ISNULL(src_tablet_meta) - || !ObTabletRestoreAction::is_valid(restore_action)) { + || !ObTabletRestoreAction::is_valid(restore_action) || OB_ISNULL(copy_tablet_ctx)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("init tablet copy finish task get invalid argument", K(ret), K(tablet_id), KP(ls), KP(reporter), KP(src_tablet_meta), K(restore_action)); + } else if (OB_FAIL(copy_tablet_ctx->set_copy_tablet_status(ObCopyTabletStatus::TABLET_EXIST))) { + LOG_WARN("failed to set copy tablet status", K(ret)); } else { tablet_id_ = tablet_id; ls_ = ls; @@ -1218,7 +1222,7 @@ int ObTabletCopyFinishTask::init( ha_dag_ = static_cast(this->get_dag()); restore_action_ = restore_action; src_tablet_meta_ = src_tablet_meta; - status_ = ObCopyTabletStatus::TABLET_EXIST; + copy_tablet_ctx_ = copy_tablet_ctx; is_inited_ = true; } return ret; @@ -1249,7 +1253,7 @@ int ObTabletCopyFinishTask::process() LOG_WARN("failed to trim tablet", K(ret), K_(tablet_id)); } else if (OB_FAIL(check_tablet_valid_())) { LOG_WARN("failed to check tablet valid", K(ret), KPC(this)); - } else if (OB_FAIL(update_tablet_data_status_())) { + } else if (OB_FAIL(check_finish_copy_tablet_data_valid_())) { LOG_WARN("failed to update tablet data status", K(ret), K(tablet_id_)); } @@ -1383,7 +1387,7 @@ int ObTabletCopyFinishTask::create_new_table_store_with_major_() return ret; } -int ObTabletCopyFinishTask::update_tablet_data_status_() +int ObTabletCopyFinishTask::check_finish_copy_tablet_data_valid_() { int ret = OB_SUCCESS; ObTabletHandle tablet_handle; @@ -1403,8 +1407,6 @@ int ObTabletCopyFinishTask::update_tablet_data_status_() } else if (tablet->get_tablet_meta().has_next_tablet_) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet here should only has one", K(ret), KPC(tablet)); - } else if (tablet->get_tablet_meta().ha_status_.is_data_status_complete()) { - //do nothing } else if (OB_FAIL(ObStorageHATabletBuilderUtil::check_remote_logical_sstable_exist(tablet, is_logical_sstable_exist))) { LOG_WARN("failed to check remote logical sstable exist", K(ret), KPC(tablet)); } else if (is_logical_sstable_exist && tablet->get_tablet_meta().ha_status_.is_restore_status_full()) { @@ -1423,25 +1425,6 @@ int ObTabletCopyFinishTask::update_tablet_data_status_() LOG_WARN("tablet should has major sstable, unexpected", K(ret), KPC(tablet), K(major_sstables)); } -#ifdef ERRSIM - if (OB_SUCC(ret)) { - ret = OB_E(EventTable::EN_UPDATE_TABLET_HA_STATUS_FAILED) OB_SUCCESS; - if (OB_FAIL(ret)) { - STORAGE_LOG(ERROR, "fake EN_UPDATE_TABLET_HA_STATUS_FAILED", K(ret)); - } - } -#endif - - if (OB_SUCC(ret)) { - if (!tablet->get_tablet_meta().ha_status_.is_restore_status_full()) { - LOG_INFO("tablet is in restore status, do not update data stauts is full", K(ret), K(tablet_id_)); - } else if (OB_FAIL(ls_->update_tablet_ha_data_status(tablet_id_, data_status))) { - LOG_WARN("[HA]failed to update tablet ha data status", K(ret), K(tablet_id_), K(data_status)); - } else { - LOG_INFO("update tablet ha data status", K_(tablet_id), K(data_status)); - } - } - if (OB_SUCC(ret)) { int tmp_ret = OB_SUCCESS; if (OB_SUCCESS != (tmp_ret = reporter_->submit_tablet_update_task(ls_->get_tenant_id(), @@ -1512,7 +1495,9 @@ int ObTabletCopyFinishTask::set_tablet_status(const ObCopyTabletStatus::STATUS & LOG_WARN("set tablet status get invalid argument", K(ret), K(status)); } else { common::SpinWLockGuard guard(lock_); - status_ = status; + if (OB_FAIL(copy_tablet_ctx_->set_copy_tablet_status(status))) { + LOG_WARN("failed to set copy tablet status", K(ret)); + } } return ret; } @@ -1546,7 +1531,9 @@ int ObTabletCopyFinishTask::get_tablet_status(ObCopyTabletStatus::STATUS &status LOG_WARN("tablet copy finish task do not init", K(ret)); } else { common::SpinRLockGuard guard(lock_); - status = status_; + if (OB_FAIL(copy_tablet_ctx_->get_copy_tablet_status(status))) { + LOG_WARN("failed to get copy tablet status", K(ret)); + } } return ret; } diff --git a/src/storage/high_availability/ob_physical_copy_task.h b/src/storage/high_availability/ob_physical_copy_task.h index 806a5fa26..07c93a201 100644 --- a/src/storage/high_availability/ob_physical_copy_task.h +++ b/src/storage/high_availability/ob_physical_copy_task.h @@ -34,6 +34,13 @@ namespace oceanbase namespace storage { +struct ObICopyTabletCtx +{ +public: + virtual int set_copy_tablet_status(const ObCopyTabletStatus::STATUS &status) = 0; + virtual int get_copy_tablet_status(ObCopyTabletStatus::STATUS &status) const = 0; +}; + struct ObPhysicalCopyCtx { ObPhysicalCopyCtx(); @@ -218,7 +225,8 @@ public: ObLS *ls, observer::ObIMetaReport *reporter, const ObTabletRestoreAction::ACTION &restore_action, - const ObMigrationTabletParam *src_tablet_meta); + const ObMigrationTabletParam *src_tablet_meta, + ObICopyTabletCtx *copy_tablet_ctx); virtual int process() override; VIRTUAL_TO_STRING_KV(K("ObTabletCopyFinishTask"), KP(this)); int add_sstable(ObTableHandleV2 &table_handle); @@ -235,7 +243,7 @@ private: int create_new_table_store_with_minor_(); int trim_tablet_(); - int update_tablet_data_status_(); + int check_finish_copy_tablet_data_valid_(); int get_tables_handle_ptr_( const ObITable::TableKey &table_key, ObTablesHandleArray *&table_handle_ptr); @@ -254,11 +262,10 @@ private: ObTablesHandleArray major_tables_handle_; ObTabletRestoreAction::ACTION restore_action_; const ObMigrationTabletParam *src_tablet_meta_; - storage::ObCopyTabletStatus::STATUS status_; + ObICopyTabletCtx *copy_tablet_ctx_; DISALLOW_COPY_AND_ASSIGN(ObTabletCopyFinishTask); }; - } } #endif diff --git a/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp b/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp index 7b81042f1..d81d5c7e1 100755 --- a/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp +++ b/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp @@ -482,8 +482,8 @@ int ObStorageHATabletsBuilder::create_or_update_tablet_( ret = OB_INVALID_ARGUMENT; LOG_WARN("create or update tablet get invalid argument", K(ret), K(tablet_info), KP(ls)); } else if (ObCopyTabletStatus::TABLET_NOT_EXIST == tablet_info.status_ && tablet_info.tablet_id_.is_ls_inner_tablet()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sys tablet should exist", K(ret), K(tablet_info)); + ret = OB_TABLET_NOT_EXIST; + LOG_WARN("src ls inner tablet is not exist, src ls is maybe deleted", K(ret), K(tablet_info)); } else if (OB_FAIL(hold_local_reuse_sstable_(tablet_info.tablet_id_, local_tablet_hdl, major_tables, storage_schema, medium_info_list, allocator))) { LOG_WARN("failed to hold local reuse sstable", K(ret), K(tablet_info)); } else if (OB_FAIL(ls->rebuild_create_tablet(tablet_info.param_, keep_old))) { @@ -1360,7 +1360,8 @@ int ObStorageHATabletsBuilder::modified_tablet_info_( && !tablet_info.param_.ha_status_.is_data_status_complete()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet info ha status is unexpected", K(ret), K(tablet_info)); - } else if (OB_FAIL(tablet_info.param_.ha_status_.set_data_status(ObTabletDataStatus::INCOMPLETE))) { + } else if (ObTabletRestoreAction::is_restore_none(param_.restore_action_) // restore process doesn't consider data state + && OB_FAIL(tablet_info.param_.ha_status_.set_data_status(ObTabletDataStatus::INCOMPLETE))) { LOG_WARN("failed to set data status", K(ret), K(tablet_info)); } return ret; diff --git a/src/storage/high_availability/ob_tablet_group_restore.cpp b/src/storage/high_availability/ob_tablet_group_restore.cpp index 11c2bc185..82ecf11a0 100644 --- a/src/storage/high_availability/ob_tablet_group_restore.cpp +++ b/src/storage/high_availability/ob_tablet_group_restore.cpp @@ -28,6 +28,31 @@ using namespace share; namespace storage { +static int update_deleted_and_undefine_tablet(ObLS &ls, const ObTabletID &tablet_id) +{ + int ret = OB_SUCCESS; + const ObTabletExpectedStatus::STATUS expected_status = ObTabletExpectedStatus::DELETED; + const ObTabletRestoreStatus::STATUS restore_status = ObTabletRestoreStatus::UNDEFINED; + if (OB_FAIL(ls.get_tablet_svr()->update_tablet_ha_expected_status(tablet_id, expected_status))) { + if (OB_TABLET_NOT_EXIST == ret) { + LOG_INFO("restore tablet maybe deleted, skip update expected status to DELETED", K(ret), K(tablet_id)); + ret = OB_SUCCESS; + } else { + LOG_WARN("failed to update expected status to DELETED", K(ret), K(expected_status), K(tablet_id)); + } + } else if (OB_FAIL(ls.update_tablet_restore_status(tablet_id, restore_status))) { + if (OB_TABLET_NOT_EXIST == ret) { + LOG_INFO("restore tablet maybe deleted, skip update restore status to UNDEFINED", K(ret), K(tablet_id)); + ret = OB_SUCCESS; + } else { + LOG_WARN("failed to update restore status to UNDEFINED", K(ret), K(restore_status), K(tablet_id)); + } + } else { + LOG_INFO("remote tablet is not exist, update expected status to DELETED, and restore status to UNDEFINED", K(tablet_id)); + } + return ret; +} + /******************ObTabletGroupRestoreCtx*********************/ ObTabletGroupRestoreCtx::ObTabletGroupRestoreCtx() : ObIHADagNetCtx(), @@ -169,7 +194,7 @@ int ObTabletRestoreCtx::set_copy_tablet_status(const ObCopyTabletStatus::STATUS return ret; } -int ObTabletRestoreCtx::get_copy_tablet_status(ObCopyTabletStatus::STATUS &status) +int ObTabletRestoreCtx::get_copy_tablet_status(ObCopyTabletStatus::STATUS &status) const { int ret = OB_SUCCESS; status = ObCopyTabletStatus::MAX_STATUS; @@ -2468,7 +2493,7 @@ int ObTabletRestoreTask::generate_tablet_copy_finish_task_( tablet_restore_ctx_->tablet_id_, src_tablet_meta))) { LOG_WARN("failed to get src tablet meta", K(ret), KPC(tablet_restore_ctx_)); } else if (OB_FAIL(tablet_copy_finish_task->init(tablet_restore_ctx_->tablet_id_, ls_, reporter, - tablet_restore_ctx_->action_, src_tablet_meta))) { + tablet_restore_ctx_->action_, src_tablet_meta, tablet_restore_ctx_))) { LOG_WARN("failed to init tablet copy finish task", K(ret), KPC(ha_dag_net_ctx_), KPC(tablet_restore_ctx_)); } return ret; @@ -2552,26 +2577,8 @@ int ObTabletRestoreTask::update_ha_status_( } else if (status != ObCopyTabletStatus::TABLET_NOT_EXIST) { ret = OB_INVALID_ARGUMENT; LOG_WARN("update ha meta status get invalid argument", K(ret), K(status)); - } else { - const ObTabletExpectedStatus::STATUS expected_status = ObTabletExpectedStatus::DELETED; - const ObTabletRestoreStatus::STATUS restore_status = ObTabletRestoreStatus::UNDEFINED; - if (OB_FAIL(ls_->get_tablet_svr()->update_tablet_ha_expected_status(tablet_restore_ctx_->tablet_id_, expected_status))) { - if (OB_TABLET_NOT_EXIST == ret) { - LOG_INFO("restore tablet maybe deleted, skip update expected status to DELETED", K(ret), KPC(tablet_restore_ctx_)); - ret = OB_SUCCESS; - } else { - LOG_WARN("failed to update expected status to DELETED", K(ret), K(expected_status), KPC(tablet_restore_ctx_)); - } - } else if (OB_FAIL(ls_->update_tablet_restore_status(tablet_restore_ctx_->tablet_id_, restore_status))) { - if (OB_TABLET_NOT_EXIST == ret) { - LOG_INFO("restore tablet maybe deleted, skip update restore status to UNDEFINED", K(ret), KPC(tablet_restore_ctx_)); - ret = OB_SUCCESS; - } else { - LOG_WARN("failed to update restore status to UNDEFINED", K(ret), K(restore_status), KPC(tablet_restore_ctx_)); - } - } else { - LOG_INFO("remote tablet is not exist, update expected status to DELETED, and restore status to UNDEFINED", KPC(tablet_restore_ctx_)); - } + } else if (OB_FAIL(update_deleted_and_undefine_tablet(*ls_, tablet_restore_ctx_->tablet_id_))) { + LOG_WARN("failed to update deleted and undefine tablet", K(ret), KPC(tablet_restore_ctx_)); } return ret; } @@ -2707,14 +2714,18 @@ int ObTabletFinishRestoreTask::process() int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; LOG_INFO("start do tablet finish restore task", KPC(tablet_restore_ctx_)); - + ObCopyTabletStatus::STATUS status; if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("tablet finish restore task do not init", K(ret), KPC(tablet_restore_ctx_)); } else if (ha_dag_net_ctx_->is_failed()) { //do nothing - } else if (OB_FAIL(update_data_status_())) { - LOG_WARN("failed to update data status", K(ret), KPC(tablet_restore_ctx_)); + } else if (OB_FAIL(tablet_restore_ctx_->get_copy_tablet_status(status))) { + LOG_WARN("failed to get copy tablet status", K(ret)); + } else if (ObCopyTabletStatus::TABLET_NOT_EXIST == status) { + if (OB_FAIL(update_deleted_and_undefine_tablet(*ls_, tablet_restore_ctx_->tablet_id_))) { + LOG_WARN("failed to update deleted and undefine tablet", K(ret), KPC(tablet_restore_ctx_)); + } } else if (OB_FAIL(update_restore_status_())) { LOG_WARN("failed to update restore status", K(ret), KPC(tablet_restore_ctx_)); } else if (OB_FAIL(check_tablet_valid_())) { @@ -2733,7 +2744,7 @@ int ObTabletFinishRestoreTask::process() return ret; } -int ObTabletFinishRestoreTask::update_data_status_() +int ObTabletFinishRestoreTask::update_restore_status_() { int ret = OB_SUCCESS; ObTabletHandle tablet_handle; @@ -2770,38 +2781,27 @@ int ObTabletFinishRestoreTask::update_data_status_() ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet should has major sstable, unexpected", K(ret), K(tablet), K(major_sstables)); } - - if (OB_SUCC(ret)) { - const ObTabletDataStatus::STATUS data_status = ObTabletDataStatus::COMPLETE; - if (OB_FAIL(ls_->update_tablet_ha_data_status(tablet_restore_ctx_->tablet_id_, data_status))) { - LOG_WARN("[HA]failed to update tablet ha data status", K(ret), KPC(tablet_restore_ctx_), K(data_status)); + } + if (OB_SUCC(ret)) { + ObTabletRestoreStatus::STATUS tablet_restore_status = ObTabletRestoreStatus::RESTORE_STATUS_MAX; + if (OB_FAIL(ObTabletRestoreAction::trans_restore_action_to_restore_status( + tablet_restore_ctx_->action_, tablet_restore_status))) { + LOG_WARN("failed to trans restore action to restore status", K(ret), KPC(tablet_restore_ctx_)); + } else if (OB_FAIL(ls_->update_tablet_restore_status(tablet_restore_ctx_->tablet_id_, tablet_restore_status))) { + if (OB_TABLET_NOT_EXIST == ret) { + LOG_INFO("restore tablet maybe deleted, skip it", K(ret), KPC(tablet_restore_ctx_)); + ret = OB_SUCCESS; + } else { + LOG_WARN("failed to update tablet restore status", K(ret), KPC(tablet_restore_ctx_), K(tablet_restore_status)); } + } else { + FLOG_INFO("succeed to update restore", K(tablet_restore_ctx_->tablet_id_), + K(tablet_restore_ctx_->action_), K(tablet_restore_status)); } } return ret; } -int ObTabletFinishRestoreTask::update_restore_status_() -{ - int ret = OB_SUCCESS; - ObTabletRestoreStatus::STATUS tablet_restore_status = ObTabletRestoreStatus::RESTORE_STATUS_MAX; - - if (!is_inited_) { - ret = OB_NOT_INIT; - LOG_WARN("tablet finish restore task do not init", K(ret)); - } else if (OB_FAIL(ObTabletRestoreAction::trans_restore_action_to_restore_status( - tablet_restore_ctx_->action_, tablet_restore_status))) { - LOG_WARN("failed to trans restore action to restore status", K(ret), KPC(tablet_restore_ctx_)); - } else if (OB_FAIL(ls_->update_tablet_restore_status(tablet_restore_ctx_->tablet_id_, tablet_restore_status))) { - LOG_WARN("failed to update tablet restore status", K(ret), KPC(tablet_restore_ctx_), K(tablet_restore_status)); - } else { - FLOG_INFO("succeed to update restore", K(tablet_restore_ctx_->tablet_id_), - K(tablet_restore_ctx_->action_), K(tablet_restore_status)); - } - - return ret; -} - int ObTabletFinishRestoreTask::check_tablet_valid_() { int ret = OB_SUCCESS; diff --git a/src/storage/high_availability/ob_tablet_group_restore.h b/src/storage/high_availability/ob_tablet_group_restore.h index c629e579a..b340f8224 100644 --- a/src/storage/high_availability/ob_tablet_group_restore.h +++ b/src/storage/high_availability/ob_tablet_group_restore.h @@ -79,15 +79,15 @@ public: DISALLOW_COPY_AND_ASSIGN(ObTabletGroupRestoreCtx); }; -struct ObTabletRestoreCtx +struct ObTabletRestoreCtx final: public ObICopyTabletCtx { public: ObTabletRestoreCtx(); virtual ~ObTabletRestoreCtx(); bool is_valid() const; void reset(); - int set_copy_tablet_status(const ObCopyTabletStatus::STATUS &status); - int get_copy_tablet_status(ObCopyTabletStatus::STATUS &status); + int set_copy_tablet_status(const ObCopyTabletStatus::STATUS &status) override; + int get_copy_tablet_status(ObCopyTabletStatus::STATUS &status) const override; VIRTUAL_TO_STRING_KV(K_(tenant_id), K_(ls_id), K_(tablet_id), KPC_(restore_base_info), K_(is_leader), K_(action), KP_(meta_index_store), KP_(second_meta_index_store), K_(replica_type), KP_(ha_table_info_mgr), K_(status)); @@ -451,7 +451,6 @@ public: virtual int process() override; VIRTUAL_TO_STRING_KV(K("ObTabletRestoreTask"), KP(this), KPC(ha_dag_net_ctx_), KPC(tablet_restore_ctx_)); private: - int update_data_status_(); int update_restore_status_(); int check_tablet_valid_(); int record_server_event_(); diff --git a/src/storage/high_availability/ob_transfer_backfill_tx.cpp b/src/storage/high_availability/ob_transfer_backfill_tx.cpp index db60c82cd..31b14c011 100644 --- a/src/storage/high_availability/ob_transfer_backfill_tx.cpp +++ b/src/storage/high_availability/ob_transfer_backfill_tx.cpp @@ -1404,7 +1404,7 @@ int ObTransferReplaceTableTask::get_source_tablet_tables_( const common::ObTabletID &tablet_id, ObTableStoreIterator &sstable_iter, ObTabletHandle &tablet_handle, - ObTabletHAStatus &ha_status, + ObTabletRestoreStatus::STATUS &restore_status, common::ObArenaAllocator &allocator, ObTablesHandleArray &tables_handle) { @@ -1463,7 +1463,8 @@ int ObTransferReplaceTableTask::get_source_tablet_tables_( && ObTabletStatus::TRANSFER_OUT_DELETED != src_user_data.tablet_status_) { ret = OB_UNEXPECTED_TABLET_STATUS; LOG_WARN("tablet status should be TRANSFER_OUT or TRANSFER_OUT_DELETED", K(ret), KPC(tablet), K(src_user_data)); - } else if (FALSE_IT(ha_status = tablet->get_tablet_meta().ha_status_)) { + } else if (OB_FAIL(tablet->get_tablet_meta().ha_status_.get_restore_status(restore_status))) { + LOG_WARN("failed to get tablet restore status", K(ret)); } else if (OB_FAIL(tablet->fetch_table_store(wrapper))) { LOG_WARN("fetch table store fail", K(ret), KP(tablet)); } else if (OB_FAIL(check_src_memtable_is_empty_(tablet, transfer_scn))) { @@ -1588,7 +1589,7 @@ int ObTransferReplaceTableTask::transfer_replace_tables_( } else if (!dest_wrapper.get_member()->get_major_sstables().empty()) { ret = OB_INVALID_TABLE_STORE; LOG_WARN("tablet should not exist major sstable", K(ret), KPC(tablet)); - } else if (OB_FAIL(get_source_tablet_tables_(tablet, tablet_id, src_sstable_iter, src_tablet_handle, param.ha_status_, allocator, param.tables_handle_))) { + } else if (OB_FAIL(get_source_tablet_tables_(tablet, tablet_id, src_sstable_iter, src_tablet_handle, param.restore_status_, allocator, param.tables_handle_))) { LOG_WARN("failed to get source tablet tables", K(ret), K(tablet_id)); } else if (OB_FAIL(build_migration_param_(tablet, src_tablet_handle, mig_param))) { LOG_WARN("failed to build migration param", K(ret), KPC(tablet)); diff --git a/src/storage/high_availability/ob_transfer_backfill_tx.h b/src/storage/high_availability/ob_transfer_backfill_tx.h index 1f00f6f36..1836aab1d 100644 --- a/src/storage/high_availability/ob_transfer_backfill_tx.h +++ b/src/storage/high_availability/ob_transfer_backfill_tx.h @@ -208,7 +208,7 @@ private: const common::ObTabletID &tablet_id, ObTableStoreIterator &sstable_iter, ObTabletHandle &tablet_handle, - ObTabletHAStatus &ha_status, + ObTabletRestoreStatus::STATUS &restore_status, common::ObArenaAllocator &allocator, ObTablesHandleArray &tables_handle); int get_all_sstable_handles_( diff --git a/src/storage/ls/ob_ls_tablet_service.cpp b/src/storage/ls/ob_ls_tablet_service.cpp index 13d677b4c..5c4177524 100755 --- a/src/storage/ls/ob_ls_tablet_service.cpp +++ b/src/storage/ls/ob_ls_tablet_service.cpp @@ -6079,7 +6079,7 @@ int ObLSTabletService::set_frozen_for_all_memtables() int ObLSTabletService::ha_scan_all_tablets(const HandleTabletMetaFunc &handle_tablet_meta_f) { int ret = OB_SUCCESS; - + bool is_initial_state = false; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not inited", K(ret), K_(is_inited)); @@ -6104,6 +6104,11 @@ int ObLSTabletService::ha_scan_all_tablets(const HandleTabletMetaFunc &handle_ta } else if (OB_ISNULL(tablet = tablet_handle.get_obj())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet is nullptr", K(ret), K(tablet_handle)); + } else if (OB_FAIL(tablet->check_initial_state(is_initial_state))) { + LOG_WARN("failed to check initial state", K(ret)); + } else if (OB_UNLIKELY(is_initial_state)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tablet must not be initial state", K(ret), KPC(tablet), K(is_initial_state)); } else if (OB_FAIL(tablet->build_migration_tablet_param(tablet_info.param_))) { LOG_WARN("failed to build migration tablet param", K(ret)); } else if (OB_FAIL(tablet->get_ha_sstable_size(tablet_info.data_size_))) { diff --git a/src/storage/ob_storage_struct.cpp b/src/storage/ob_storage_struct.cpp index d8d17301e..db54cf653 100644 --- a/src/storage/ob_storage_struct.cpp +++ b/src/storage/ob_storage_struct.cpp @@ -322,9 +322,9 @@ ObBatchUpdateTableStoreParam::ObBatchUpdateTableStoreParam() is_transfer_replace_(false), start_scn_(SCN::min_scn()), tablet_meta_(nullptr), - update_ddl_sstable_(false) + update_ddl_sstable_(false), + restore_status_(ObTabletRestoreStatus::FULL) { - ha_status_.init_status(); } void ObBatchUpdateTableStoreParam::reset() @@ -336,7 +336,7 @@ void ObBatchUpdateTableStoreParam::reset() start_scn_.set_min(); tablet_meta_ = nullptr; update_ddl_sstable_ = false; - ha_status_.init_status(); + restore_status_ = ObTabletRestoreStatus::FULL; } bool ObBatchUpdateTableStoreParam::is_valid() const @@ -344,7 +344,7 @@ bool ObBatchUpdateTableStoreParam::is_valid() const return rebuild_seq_ > OB_INVALID_VERSION && (!update_logical_minor_sstable_ || (update_logical_minor_sstable_ && start_scn_ > SCN::min_scn() && OB_ISNULL(tablet_meta_))) - && ha_status_.is_valid(); + && ObTabletRestoreStatus::is_valid(restore_status_); } int ObBatchUpdateTableStoreParam::assign( @@ -363,7 +363,7 @@ int ObBatchUpdateTableStoreParam::assign( start_scn_ = param.start_scn_; tablet_meta_ = param.tablet_meta_; update_ddl_sstable_ = param.update_ddl_sstable_; - ha_status_ = param.ha_status_; + restore_status_ = param.restore_status_; } return ret; } diff --git a/src/storage/ob_storage_struct.h b/src/storage/ob_storage_struct.h index 1c51d9df4..f24414ca7 100755 --- a/src/storage/ob_storage_struct.h +++ b/src/storage/ob_storage_struct.h @@ -361,7 +361,7 @@ struct ObBatchUpdateTableStoreParam final int get_max_clog_checkpoint_scn(share::SCN &clog_checkpoint_scn) const; TO_STRING_KV(K_(tables_handle), K_(rebuild_seq), K_(update_logical_minor_sstable), K_(is_transfer_replace), - K_(start_scn), KP_(tablet_meta), K_(update_ddl_sstable), K_(ha_status)); + K_(start_scn), KP_(tablet_meta), K_(update_ddl_sstable), K_(restore_status)); ObTablesHandleArray tables_handle_; int64_t rebuild_seq_; @@ -370,7 +370,7 @@ struct ObBatchUpdateTableStoreParam final share::SCN start_scn_; const ObMigrationTabletParam *tablet_meta_; bool update_ddl_sstable_; - ObTabletHAStatus ha_status_; + ObTabletRestoreStatus::STATUS restore_status_; DISALLOW_COPY_AND_ASSIGN(ObBatchUpdateTableStoreParam); }; diff --git a/src/storage/restore/ob_ls_restore_handler.cpp b/src/storage/restore/ob_ls_restore_handler.cpp index 5c56a706e..de20b27c0 100644 --- a/src/storage/restore/ob_ls_restore_handler.cpp +++ b/src/storage/restore/ob_ls_restore_handler.cpp @@ -316,11 +316,8 @@ int ObLSRestoreHandler::check_before_do_restore_(bool &can_do_restore) can_do_restore = false; bool is_normal = false; bool is_exist = true; + bool is_in_member_or_learner_list = false; if (is_stop()) { - // when remove ls, set ls restore handler stop - if (REACH_TIME_INTERVAL(60 * 1000 * 1000)) { - LOG_WARN("ls is gc, no need to do restore", KPC(ls_)); - } } else if (OB_FAIL(check_meta_tenant_normal_(is_normal))) { LOG_WARN("fail to get meta tenant status", K(ret)); } else if (!is_normal) { @@ -334,9 +331,6 @@ int ObLSRestoreHandler::check_before_do_restore_(bool &can_do_restore) state_handler_ = nullptr; } } else if (restore_status.is_restore_failed()) { - if (REACH_TIME_INTERVAL(10 * 60 * 1000 * 1000)) { - LOG_WARN("ls restore failed, tenant restore can't continue", K(result_mgr_)); - } } else if (OB_FAIL(check_restore_job_exist_(is_exist))) { } else if (!is_exist) { if (OB_FAIL(ls_->set_restore_status(ObLSRestoreStatus(ObLSRestoreStatus::RESTORE_FAILED), get_rebuild_seq()))) { @@ -347,13 +341,31 @@ int ObLSRestoreHandler::check_before_do_restore_(bool &can_do_restore) } } else if (OB_FAIL(ls_->get_migration_status(migration_status))) { LOG_WARN("fail to get migration status", K(ret)); - } else if (ObMigrationStatusHelper::check_can_restore(migration_status)) { - // Migration and restore require serial execution - can_do_restore = true; - } else { + } else if (!ObMigrationStatusHelper::check_can_restore(migration_status)) { + } else if (OB_FAIL(check_in_member_or_learner_list_(is_in_member_or_learner_list))) { + LOG_WARN("failed to check in member or learner list", K(ret)); + } else if (!is_in_member_or_learner_list) { if (REACH_TIME_INTERVAL(60 * 1000 * 1000)) { - LOG_WARN("ls is in migration, restore wait later", KPC(ls_)); + LOG_INFO("ls is not in member or learner list", KPC(ls_)); } + } else { + can_do_restore = true; + } + return ret; +} + +int ObLSRestoreHandler::check_in_member_or_learner_list_(bool &is_in_member_or_learner_list) const +{ + int ret = OB_SUCCESS; + int64_t paxos_replica_num = 0; + common::ObMemberList member_list; + GlobalLearnerList learner_list; + ObAddr self_addr = GCONF.self_addr_; + is_in_member_or_learner_list = false; + if (OB_FAIL(ls_->get_log_handler()->get_paxos_member_list_and_learner_list(member_list, paxos_replica_num, learner_list))) { + LOG_WARN("failed to get paxos_member_list_and_learner_list", K(ret)); + } else { + is_in_member_or_learner_list = member_list.contains(self_addr) || learner_list.contains(self_addr); } return ret; } diff --git a/src/storage/restore/ob_ls_restore_handler.h b/src/storage/restore/ob_ls_restore_handler.h index 633093374..ce19b1531 100644 --- a/src/storage/restore/ob_ls_restore_handler.h +++ b/src/storage/restore/ob_ls_restore_handler.h @@ -96,6 +96,7 @@ public: private: int cancel_task_(); int check_before_do_restore_(bool &can_do_restore); + int check_in_member_or_learner_list_(bool &is_in_member_or_learner_list) const; int update_state_handle_(); int check_meta_tenant_normal_(bool &is_normal); int check_restore_job_exist_(bool &is_exist); diff --git a/src/storage/tablet/ob_tablet.cpp b/src/storage/tablet/ob_tablet.cpp index 3d64140f7..5c86250b4 100755 --- a/src/storage/tablet/ob_tablet.cpp +++ b/src/storage/tablet/ob_tablet.cpp @@ -648,7 +648,8 @@ int ObTablet::init( LOG_WARN("failed to check sstable column checksum", K(ret), KPC(this)); } else if (param.is_transfer_replace_ && OB_FAIL(tablet_meta_.reset_transfer_table())) { LOG_WARN("failed to set finish tansfer replace", K(ret), K(tablet_meta_), K(param)); - } else if (param.is_transfer_replace_ && OB_FALSE_IT(tablet_meta_.ha_status_ = param.ha_status_)) { + } else if (param.is_transfer_replace_ && OB_FAIL(tablet_meta_.ha_status_.set_restore_status(param.restore_status_))) { + LOG_WARN("failed to set tablet restore status", K(ret)); } else if (FALSE_IT(set_mem_addr())) { } else if (OB_FAIL(inner_inc_macro_ref_cnt())) { LOG_WARN("failed to increase macro ref cnt", K(ret));