diff --git a/src/storage/high_availability/ob_ls_complete_migration.cpp b/src/storage/high_availability/ob_ls_complete_migration.cpp index 56e6ed1b87..4c22054903 100644 --- a/src/storage/high_availability/ob_ls_complete_migration.cpp +++ b/src/storage/high_availability/ob_ls_complete_migration.cpp @@ -329,6 +329,7 @@ int ObLSCompleteMigrationDagNet::trans_rebuild_fail_status_( bool is_valid_member = true; bool is_ls_deleted = false; new_migration_status = ObMigrationStatus::OB_MIGRATION_STATUS_MAX; + bool is_tenant_dropping_or_dropped = false; if (!ObMigrationStatusHelper::is_valid(current_migration_status)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("current migration status is invalid", K(ret), K(current_migration_status)); @@ -347,8 +348,12 @@ int ObLSCompleteMigrationDagNet::trans_rebuild_fail_status_( is_ls_deleted = false; // reset value if fail LOG_WARN("failed to get ls status from inner table", K(ret), K(tmp_ret), K(ls)); } + if (OB_TMP_FAIL(check_tenant_is_dropping_or_dropped_(is_tenant_dropping_or_dropped))) { + is_tenant_dropping_or_dropped = false; + LOG_WARN("failed to check tenant is droppping or dropped", K(ret), K(tmp_ret), K(ctx_)); + } if (FAILEDx(ObMigrationStatusHelper::trans_rebuild_fail_status( - current_migration_status, is_valid_member, is_ls_deleted, new_migration_status))) { + current_migration_status, is_valid_member, is_ls_deleted, is_tenant_dropping_or_dropped, new_migration_status))) { LOG_WARN("failed to trans rebuild fail status", K(ret), K(ctx_)); } } @@ -399,12 +404,6 @@ int ObLSCompleteMigrationDagNet::update_migration_status_(ObLS *ls) ret = OB_SERVER_IS_STOPPING; LOG_WARN("tenant dag scheduler has set stop, stop migration dag net", K(ret), K(ctx_)); break; - } else if (OB_FAIL(ObStorageHAUtils::check_tenant_will_be_deleted(is_tenant_deleted))) { - LOG_WARN("failed to check tenant deleted", K(ret), K(ctx_)); - } else if (is_tenant_deleted) { - ret = OB_TENANT_HAS_BEEN_DROPPED; - LOG_WARN("tenant has been stopped, stop migration dag net", K(ret), K(ctx_)); - break; } else { bool in_final_state = false; if (OB_FAIL(ls->get_migration_status(current_migration_status))) { @@ -523,6 +522,36 @@ int ObLSCompleteMigrationDagNet::report_ls_meta_table_(ObLS *ls) return ret; } +int ObLSCompleteMigrationDagNet::check_tenant_is_dropping_or_dropped_( + bool &is_tenant_dropping_or_dropped) +{ + int ret = OB_SUCCESS; + schema::ObMultiVersionSchemaService *schema_service = GCTX.schema_service_; + schema::ObSchemaGetterGuard guard; + is_tenant_dropping_or_dropped = false; + const ObTenantSchema *tenant_schema = nullptr; + + if (OB_ISNULL(schema_service)) { + ret = OB_ERR_UNEXPECTED; + CLOG_LOG(WARN, "schema_service is nullptr", KR(ret)); + } else if (OB_FAIL(schema_service->get_tenant_schema_guard(OB_SYS_TENANT_ID, guard))) { + LOG_WARN("fail to get schema guard", KR(ret), K(ctx_)); + } else if (OB_FAIL(guard.get_tenant_info(ctx_.tenant_id_, tenant_schema))) { + LOG_WARN("get tenant info failed", KR(ret), K(ctx_)); + } else if (OB_ISNULL(tenant_schema)) { + // Double check the tenant status to avoid any potential problems in the schema module. + if (OB_FAIL(guard.check_if_tenant_has_been_dropped(ctx_.tenant_id_, is_tenant_dropping_or_dropped))) { + LOG_WARN("fail to check if tenant has been dropped", KR(ret), K(ctx_)); + } else { + LOG_INFO("tenant info is nullptr, check the tenant status", + K(ctx_), K(is_tenant_dropping_or_dropped)); + } + } else { + is_tenant_dropping_or_dropped = tenant_schema->is_dropping(); + } + return ret; +} + /******************ObCompleteMigrationDag*********************/ ObCompleteMigrationDag::ObCompleteMigrationDag(const share::ObDagType::ObDagTypeEnum &dag_type) : ObStorageHADag(dag_type) diff --git a/src/storage/high_availability/ob_ls_complete_migration.h b/src/storage/high_availability/ob_ls_complete_migration.h index ffe0c8f2c2..2cb0595f1b 100644 --- a/src/storage/high_availability/ob_ls_complete_migration.h +++ b/src/storage/high_availability/ob_ls_complete_migration.h @@ -99,6 +99,8 @@ private: ObLS &ls, const ObMigrationStatus ¤t_migration_status, ObMigrationStatus &new_migration_status); + int check_tenant_is_dropping_or_dropped_( + bool &is_tenant_dropping_or_dropped); private: bool is_inited_; diff --git a/src/storage/high_availability/ob_storage_ha_struct.cpp b/src/storage/high_availability/ob_storage_ha_struct.cpp index 3cc86fc18b..0e9e7c7415 100644 --- a/src/storage/high_availability/ob_storage_ha_struct.cpp +++ b/src/storage/high_availability/ob_storage_ha_struct.cpp @@ -786,6 +786,7 @@ int ObMigrationStatusHelper::trans_rebuild_fail_status( const ObMigrationStatus &cur_status, const bool is_in_member_list, const bool is_ls_deleted, + const bool is_tenant_dropping_or_dropped, ObMigrationStatus &fail_status) { int ret = OB_SUCCESS; @@ -794,7 +795,7 @@ int ObMigrationStatusHelper::trans_rebuild_fail_status( if (OB_MIGRATION_STATUS_REBUILD != cur_status && OB_MIGRATION_STATUS_REBUILD_WAIT != cur_status) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", K(ret), K(cur_status)); - } else if (!is_in_member_list || is_ls_deleted) { + } else if (is_tenant_dropping_or_dropped || !is_in_member_list || is_ls_deleted) { fail_status = OB_MIGRATION_STATUS_REBUILD_FAIL; } else { fail_status = OB_MIGRATION_STATUS_REBUILD; diff --git a/src/storage/high_availability/ob_storage_ha_struct.h b/src/storage/high_availability/ob_storage_ha_struct.h index 91688b2d23..d2fcae9098 100644 --- a/src/storage/high_availability/ob_storage_ha_struct.h +++ b/src/storage/high_availability/ob_storage_ha_struct.h @@ -93,6 +93,7 @@ public: const ObMigrationStatus &cur_status, const bool is_in_member_list, const bool is_ls_deleted, + const bool is_tenant_dropping_or_dropped, ObMigrationStatus &fail_status); static int check_migration_in_final_state( const ObMigrationStatus &status, diff --git a/src/storage/ls/ob_ls.cpp b/src/storage/ls/ob_ls.cpp index 0eb5459742..480b51e232 100644 --- a/src/storage/ls/ob_ls.cpp +++ b/src/storage/ls/ob_ls.cpp @@ -1323,7 +1323,8 @@ int ObLS::get_replica_status(ObReplicaStatus &replica_status) ret = OB_ERR_UNEXPECTED; LOG_WARN("migration status is not valid", K(ret), K(migration_status)); } else if (OB_MIGRATION_STATUS_NONE == migration_status - || OB_MIGRATION_STATUS_REBUILD == migration_status) { + || OB_MIGRATION_STATUS_REBUILD == migration_status + || OB_MIGRATION_STATUS_REBUILD_WAIT == migration_status) { replica_status = REPLICA_STATUS_NORMAL; } else { replica_status = REPLICA_STATUS_OFFLINE;