From 07ecec736fa4de543a135d9f74b64569f91ff65e Mon Sep 17 00:00:00 2001 From: hamstersox <673144759@qq.com> Date: Fri, 22 Sep 2023 12:39:54 +0000 Subject: [PATCH] can't advance recover table status when failed to failover to primary --- .../ob_recover_table_job_scheduler.cpp | 46 +++++++++++++++++-- .../restore/ob_recover_table_job_scheduler.h | 3 +- src/share/restore/ob_import_table_struct.cpp | 6 +++ src/share/restore/ob_import_table_struct.h | 13 +++--- 4 files changed, 58 insertions(+), 10 deletions(-) diff --git a/src/rootserver/restore/ob_recover_table_job_scheduler.cpp b/src/rootserver/restore/ob_recover_table_job_scheduler.cpp index b5d80ba997..c6859c9ea7 100644 --- a/src/rootserver/restore/ob_recover_table_job_scheduler.cpp +++ b/src/rootserver/restore/ob_recover_table_job_scheduler.cpp @@ -436,6 +436,12 @@ void ObRecoverTableJobScheduler::user_process_(share::ObRecoverTableJob &job) } break; } + case ObRecoverTableStatus::Status::ACTIVE_AUX_TENANT: { + if (OB_FAIL(active_aux_tenant_(job))) { + LOG_WARN("failed to do user active aux tenant work", K(ret), K(job)); + } + break; + } case ObRecoverTableStatus::Status::GEN_IMPORT_JOB: { if (OB_FAIL(gen_import_job_(job))) { LOG_WARN("failed to do user import work", K(ret), K(job)); @@ -548,8 +554,6 @@ int ObRecoverTableJobScheduler::restore_aux_tenant_(share::ObRecoverTableJob &jo job.get_result().set_result(false, restore_history_info.comment_); } else if (OB_FAIL(check_aux_tenant_(job, aux_tenant_id))) { LOG_WARN("failed to check aux tenant", K(ret), K(aux_tenant_id)); - } else if (OB_FAIL(failover_to_leader_(job, aux_tenant_id))) { - LOG_WARN("failed to failover to leader", K(ret)); } int tmp_ret = OB_SUCCESS; @@ -560,7 +564,43 @@ int ObRecoverTableJobScheduler::restore_aux_tenant_(share::ObRecoverTableJob &jo return ret; } -int ObRecoverTableJobScheduler::failover_to_leader_( +int ObRecoverTableJobScheduler::active_aux_tenant_(share::ObRecoverTableJob &job) +{ + int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; + ObRestorePersistHelper restore_helper; + ObHisRestoreJobPersistInfo restore_history_info; + if (OB_FAIL(restore_helper.init(OB_SYS_TENANT_ID))) { + LOG_WARN("failed to init retore helper", K(ret)); + } else if (OB_FAIL(restore_helper.get_restore_job_history( + *sql_proxy_, job.get_initiator_job_id(), job.get_initiator_tenant_id(), restore_history_info))) { + LOG_WARN("failed to get restore job history", K(ret), + "initiator_job_id", job.get_job_id(), "initiator_tenant_id", job.get_tenant_id()); + } else if (OB_FAIL(failover_to_primary_(job, restore_history_info.restore_tenant_id_))) { + LOG_WARN("failed to failover to primary", K(ret), K(restore_history_info)); + } + if (OB_FAIL(ret)) { + int tmp_ret = OB_SUCCESS; + schema::ObSchemaGetterGuard guard; + if (OB_TMP_FAIL(ObImportTableUtil::get_tenant_schema_guard(*schema_service_, + restore_history_info.restore_tenant_id_, + guard))) { + if (OB_TENANT_NOT_EXIST == tmp_ret) { + ret = tmp_ret; + } + LOG_WARN("failed to get tenant schema guard", K(tmp_ret)); + } + } + + if (OB_SUCC(ret) || OB_TENANT_NOT_EXIST == ret) { + if (OB_SUCCESS != (tmp_ret = try_advance_status_(job, ret))) { + LOG_WARN("failed to advance status", K(tmp_ret), K(ret)); + } + } + return ret; +} + +int ObRecoverTableJobScheduler::failover_to_primary_( share::ObRecoverTableJob &job, const uint64_t aux_tenant_id) { int ret = OB_SUCCESS; diff --git a/src/rootserver/restore/ob_recover_table_job_scheduler.h b/src/rootserver/restore/ob_recover_table_job_scheduler.h index 0d0855ec8a..8489853e17 100644 --- a/src/rootserver/restore/ob_recover_table_job_scheduler.h +++ b/src/rootserver/restore/ob_recover_table_job_scheduler.h @@ -72,7 +72,8 @@ private: int user_prepare_(share::ObRecoverTableJob &job); int restore_aux_tenant_(share::ObRecoverTableJob &job); int check_aux_tenant_(share::ObRecoverTableJob &job, const uint64_t aux_tenant_id); - int failover_to_leader_(share::ObRecoverTableJob &job, const uint64_t aux_tenant_id); + int active_aux_tenant_(share::ObRecoverTableJob &job); + int failover_to_primary_(share::ObRecoverTableJob &job, const uint64_t aux_tenant_id); int check_tenant_compatibility( share::schema::ObSchemaGetterGuard &aux_tenant_guard, share::schema::ObSchemaGetterGuard &recover_tenant_guard, diff --git a/src/share/restore/ob_import_table_struct.cpp b/src/share/restore/ob_import_table_struct.cpp index c8fcb24c1f..cb7b8a442b 100644 --- a/src/share/restore/ob_import_table_struct.cpp +++ b/src/share/restore/ob_import_table_struct.cpp @@ -867,6 +867,7 @@ const char* ObRecoverTableStatus::get_str() const "PREPARE", "RECOVERING", "RESTORE_AUX_TENANT", + "ACTIVE_AUX_TENANT", "PRECHECK_IMPORT", "GEN_IMPORT_JOB", "IMPORTING", @@ -892,6 +893,7 @@ int ObRecoverTableStatus::set_status(const char *str) "PREPARE", "RECOVERING", "RESTORE_AUX_TENANT", + "ACTIVE_AUX_TENANT", "PRECHECK_IMPORT", "GEN_IMPORT_JOB", "IMPORTING", @@ -945,6 +947,10 @@ ObRecoverTableStatus ObRecoverTableStatus::get_user_next_status(const ObRecoverT break; } case ObRecoverTableStatus::Status::RESTORE_AUX_TENANT: { + ret = ObRecoverTableStatus::Status::ACTIVE_AUX_TENANT; + break; + } + case ObRecoverTableStatus::Status::ACTIVE_AUX_TENANT: { ret = ObRecoverTableStatus::Status::GEN_IMPORT_JOB; break; } diff --git a/src/share/restore/ob_import_table_struct.h b/src/share/restore/ob_import_table_struct.h index 0f6f413912..b3ace7b2b3 100644 --- a/src/share/restore/ob_import_table_struct.h +++ b/src/share/restore/ob_import_table_struct.h @@ -319,12 +319,13 @@ public: PREPARE = 0, RECOVERING = 1, RESTORE_AUX_TENANT = 2, - PRECHECK_IMPORT = 3, - GEN_IMPORT_JOB = 4, - IMPORTING = 5, - CANCELING = 6, - COMPLETED = 7, - FAILED = 8, + ACTIVE_AUX_TENANT = 3, + PRECHECK_IMPORT = 4, + GEN_IMPORT_JOB = 5, + IMPORTING = 6, + CANCELING = 7, + COMPLETED = 8, + FAILED = 9, MAX_STATUS }; public: