From bf0a220832dadf2118cd175653ee17692ec3fcc1 Mon Sep 17 00:00:00 2001 From: maosy <630014370@qq.com> Date: Tue, 7 Mar 2023 13:43:44 +0000 Subject: [PATCH] Revert add iter scn to __all_tenant_info & primary cluster iter log --- .../data_dictionary/ob_data_dict_struct.cpp | 2 +- .../libobcdc/src/ob_log_ls_op_processor.cpp | 14 +- src/logservice/ob_garbage_collector.cpp | 4 +- src/logservice/ob_log_base_type.h | 8 - src/observer/omt/ob_multi_tenant.cpp | 9 +- src/rootserver/CMakeLists.txt | 6 +- src/rootserver/ob_common_ls_service.cpp | 597 ----- src/rootserver/ob_common_ls_service.h | 126 - src/rootserver/ob_ddl_service.cpp | 6 +- src/rootserver/ob_ls_service_helper.cpp | 1860 ------------- src/rootserver/ob_ls_service_helper.h | 279 -- src/rootserver/ob_primary_ls_service.cpp | 2350 ++++++++++++++++- src/rootserver/ob_primary_ls_service.h | 295 ++- src/rootserver/ob_recovery_ls_service.cpp | 657 ++++- src/rootserver/ob_recovery_ls_service.h | 63 +- src/rootserver/ob_root_utils.cpp | 1 + src/rootserver/ob_root_utils.h | 1 + src/rootserver/ob_tenant_info_loader.cpp | 15 - src/rootserver/ob_tenant_info_loader.h | 3 +- src/rootserver/ob_tenant_info_report.cpp | 110 - src/rootserver/ob_tenant_info_report.h | 84 - ...or.cpp => ob_tenant_recovery_reportor.cpp} | 94 +- ...portor.h => ob_tenant_recovery_reportor.h} | 17 +- .../ob_tenant_role_transition_service.cpp | 137 +- .../ob_tenant_role_transition_service.h | 11 +- src/rootserver/ob_tenant_thread_helper.cpp | 323 --- src/rootserver/ob_tenant_thread_helper.h | 120 - .../restore/ob_restore_scheduler.cpp | 41 +- src/rootserver/restore/ob_restore_scheduler.h | 2 +- .../ob_inner_table_schema.12251_12300.cpp | 19 - .../ob_inner_table_schema.15251_15300.cpp | 15 - .../ob_inner_table_schema.21151_21200.cpp | 2 +- .../ob_inner_table_schema.351_400.cpp | 19 - .../inner_table/ob_inner_table_schema_def.py | 7 - src/share/ls/ob_ls_i_life_manager.h | 37 - src/share/ls/ob_ls_life_manager.cpp | 49 +- src/share/ls/ob_ls_life_manager.h | 50 +- src/share/ls/ob_ls_operator.cpp | 66 +- src/share/ls/ob_ls_operator.h | 26 +- src/share/ls/ob_ls_recovery_stat_operator.cpp | 16 +- src/share/ob_primary_standby_service.cpp | 99 +- src/share/ob_primary_standby_service.h | 15 + src/share/ob_tenant_info_proxy.cpp | 203 +- src/share/ob_tenant_info_proxy.h | 42 +- src/share/ob_tenant_switchover_status.cpp | 1 + src/share/ob_tenant_switchover_status.h | 16 +- src/share/ob_upgrade_utils.cpp | 106 - src/share/ob_upgrade_utils.h | 8 +- src/share/rc/ob_tenant_base.h | 8 +- src/storage/ls/ob_ls.cpp | 37 +- src/storage/tx/ob_trans_service_v4.cpp | 1 + .../tx_storage/ob_checkpoint_service.cpp | 58 +- .../tx_storage/ob_checkpoint_service.h | 9 - .../ob_admin_dump_backup_data_executor.cpp | 2 +- .../rootserver/test_primary_ls_service.cpp | 30 +- 55 files changed, 3788 insertions(+), 4388 deletions(-) delete mode 100755 src/rootserver/ob_common_ls_service.cpp delete mode 100644 src/rootserver/ob_common_ls_service.h delete mode 100755 src/rootserver/ob_ls_service_helper.cpp delete mode 100644 src/rootserver/ob_ls_service_helper.h delete mode 100755 src/rootserver/ob_tenant_info_report.cpp delete mode 100644 src/rootserver/ob_tenant_info_report.h rename src/rootserver/{ob_ls_recovery_reportor.cpp => ob_tenant_recovery_reportor.cpp} (85%) rename src/rootserver/{ob_ls_recovery_reportor.h => ob_tenant_recovery_reportor.h} (82%) delete mode 100755 src/rootserver/ob_tenant_thread_helper.cpp delete mode 100644 src/rootserver/ob_tenant_thread_helper.h diff --git a/src/logservice/data_dictionary/ob_data_dict_struct.cpp b/src/logservice/data_dictionary/ob_data_dict_struct.cpp index 93cca14693..65c652764b 100644 --- a/src/logservice/data_dictionary/ob_data_dict_struct.cpp +++ b/src/logservice/data_dictionary/ob_data_dict_struct.cpp @@ -389,7 +389,7 @@ int ObDictTenantMeta::incremental_data_update(const share::ObLSAttr &ls_attr) if (OB_UNLIKELY(! ls_attr.is_valid())) { ret = OB_INVALID_ARGUMENT; DDLOG(WARN, "ls attr is invalid", KR(ret), K(ls_attr)); - } else if (share::is_ls_create_end_op(ls_attr.get_ls_operation_type())) { + } else if (share::is_ls_create_end_op(ls_attr.get_ls_operatin_type())) { if (OB_FAIL(ls_arr_.push_back(ls_attr.get_ls_id()))) { DDLOG(WARN, "ls_arr_ push_back failed", KR(ret), K(ls_attr), K(ls_arr_)); } diff --git a/src/logservice/libobcdc/src/ob_log_ls_op_processor.cpp b/src/logservice/libobcdc/src/ob_log_ls_op_processor.cpp index a8b16c07a4..6a24557c7d 100644 --- a/src/logservice/libobcdc/src/ob_log_ls_op_processor.cpp +++ b/src/logservice/libobcdc/src/ob_log_ls_op_processor.cpp @@ -38,20 +38,20 @@ int ObLogLSOpProcessor::process_ls_op( } else if (OB_ISNULL(tenant = guard.get_tenant())) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("get tenant fail, tenant is NULL", KR(ret), K(tenant_id)); - } else if (share::is_ls_create_pre_op(ls_attr.get_ls_operation_type())) { + } else if (share::is_ls_create_pre_op(ls_attr.get_ls_operatin_type())) { LOG_INFO("[LS_PROCESSOR] ls create pre operation", K(tenant_id), K(lsn), K(start_tstamp_ns), K(ls_attr)); - } else if (share::is_ls_create_end_op(ls_attr.get_ls_operation_type())) { + } else if (share::is_ls_create_end_op(ls_attr.get_ls_operatin_type())) { LOG_INFO("[LS_PROCESSOR] ls create end operation", K(tenant_id), K(lsn), K(start_tstamp_ns), K(ls_attr)); //create new ls; if (OB_FAIL(create_new_ls_(tenant, start_tstamp_ns, ls_attr))) { LOG_ERROR("failed to create new ls", KR(ret), K(tenant_id), K(lsn), K(start_tstamp_ns), K(ls_attr)); } - } else if (share::is_ls_create_abort_op(ls_attr.get_ls_operation_type())) { - } else if (share::is_ls_drop_pre_op(ls_attr.get_ls_operation_type())) { - } else if (share::is_ls_drop_end_op(ls_attr.get_ls_operation_type())) { - } else if (share::is_ls_tenant_drop_pre_op(ls_attr.get_ls_operation_type())) { + } else if (share::is_ls_create_abort_op(ls_attr.get_ls_operatin_type())) { + } else if (share::is_ls_drop_pre_op(ls_attr.get_ls_operatin_type())) { + } else if (share::is_ls_drop_end_op(ls_attr.get_ls_operatin_type())) { + } else if (share::is_ls_tenant_drop_pre_op(ls_attr.get_ls_operatin_type())) { //only sys ls - } else if (share::is_ls_tenant_drop_op(ls_attr.get_ls_operation_type())) { + } else if (share::is_ls_tenant_drop_op(ls_attr.get_ls_operatin_type())) { } else { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected operation type", KR(ret), K(tenant_id), K(lsn), K(start_tstamp_ns), K(ls_attr)); diff --git a/src/logservice/ob_garbage_collector.cpp b/src/logservice/ob_garbage_collector.cpp index 3cda548aee..4b0954a37a 100644 --- a/src/logservice/ob_garbage_collector.cpp +++ b/src/logservice/ob_garbage_collector.cpp @@ -29,7 +29,7 @@ #include "share/rc/ob_tenant_base.h" #include "share/ls/ob_ls_life_manager.h" #include "storage/tx_storage/ob_ls_handle.h" -#include "rootserver/ob_tenant_info_loader.h" // ObTenantInfoLoader +#include "rootserver/ob_tenant_recovery_reportor.h" // ObTenantRecoveryReportor #include "share/ob_occam_time_guard.h" namespace oceanbase @@ -651,7 +651,7 @@ void ObGCHandler::try_check_and_set_wait_gc_(ObGarbageCollector::LSStatus &ls_st int ObGCHandler::get_tenant_readable_scn_(SCN &readable_scn) { - return MTL(rootserver::ObTenantInfoLoader *)->get_tenant_readable_scn(readable_scn); + return MTL(rootserver::ObTenantRecoveryReportor *)->get_tenant_readable_scn(readable_scn); } // 由于日志流GC导致的归档日志不完整是无法被归档检查出来的异常, 因此需要保证GC与归档状态互斥; diff --git a/src/logservice/ob_log_base_type.h b/src/logservice/ob_log_base_type.h index 376dc83061..67983083e8 100644 --- a/src/logservice/ob_log_base_type.h +++ b/src/logservice/ob_log_base_type.h @@ -75,10 +75,6 @@ enum ObLogBaseType // for arbitration service ARBITRATION_SERVICE_LOG_BASE_TYPE = 21, - - TENANT_INFO_REPORTOR_LOG_BASE_TYPE = 22, - - COMMON_LS_SERVICE_LOG_BASE_TYPE = 23, // pay attention!!! // add log type in log_base_type_to_string // max value @@ -137,10 +133,6 @@ int log_base_type_to_string(const ObLogBaseType log_type, strncpy(str ,"DATA_DICTIONARY_SERVICE", str_len); } else if (log_type == ARBITRATION_SERVICE_LOG_BASE_TYPE) { strncpy(str ,"ARBITRATION_SERVICE", str_len); - } else if (log_type == TENANT_INFO_REPORTOR_LOG_BASE_TYPE) { - strncpy(str ,"TENANT_INFO_REPORTOR", str_len); - } else if (log_type == COMMON_LS_SERVICE_LOG_BASE_TYPE) { - strncpy(str ,"COMMON_LS_SERVICE", str_len); } else { ret = OB_INVALID_ARGUMENT; } diff --git a/src/observer/omt/ob_multi_tenant.cpp b/src/observer/omt/ob_multi_tenant.cpp index 55730cbf0c..a379408c04 100644 --- a/src/observer/omt/ob_multi_tenant.cpp +++ b/src/observer/omt/ob_multi_tenant.cpp @@ -85,14 +85,11 @@ #include "observer/omt/ob_tenant_config_mgr.h" #include "observer/report/ob_tenant_meta_checker.h" #include "storage/high_availability/ob_storage_ha_service.h" -#include "rootserver/ob_ls_recovery_reportor.h"//ObLSRecoveryReportor +#include "rootserver/ob_tenant_recovery_reportor.h"//ObTenantRecoveryReportor #include "rootserver/ob_tenant_info_loader.h"//ObTenantInfoLoader #include "rootserver/ob_primary_ls_service.h"//ObLSService #include "rootserver/ob_recovery_ls_service.h"//ObRecoveryLSService -#include "rootserver/ob_common_ls_service.h"//ObCommonLSService -#include "rootserver/ob_tenant_info_report.h"//ObTenantInfoReportor #include "rootserver/restore/ob_restore_scheduler.h" //ObRestoreService -#include "rootserver/ob_tenant_thread_helper.h"//ObTenantThreadHelper #include "logservice/leader_coordinator/ob_leader_coordinator.h" #include "storage/lob/ob_lob_manager.h" #include "share/deadlock/ob_deadlock_detector_mgr.h" @@ -404,11 +401,9 @@ int ObMultiTenant::init(ObAddr myaddr, MTL_BIND2(mtl_new_default, rootserver::ObPrimaryMajorFreezeService::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default); MTL_BIND2(mtl_new_default, rootserver::ObRestoreMajorFreezeService::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default); MTL_BIND2(mtl_new_default, ObTenantMetaChecker::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default); - MTL_BIND2(mtl_new_default, rootserver::ObLSRecoveryReportor::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default); + MTL_BIND2(mtl_new_default, rootserver::ObTenantRecoveryReportor::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default); MTL_BIND2(mtl_new_default, rootserver::ObTenantInfoLoader::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default); MTL_BIND2(mtl_new_default, rootserver::ObPrimaryLSService::mtl_init, nullptr, rootserver::ObPrimaryLSService::mtl_stop, rootserver::ObPrimaryLSService::mtl_wait, mtl_destroy_default); - MTL_BIND2(mtl_new_default, rootserver::ObCommonLSService::mtl_init, nullptr, rootserver::ObCommonLSService::mtl_stop, rootserver::ObCommonLSService::mtl_wait, mtl_destroy_default); - MTL_BIND2(mtl_new_default, rootserver::ObTenantInfoReportor::mtl_init, nullptr, rootserver::ObTenantInfoReportor::mtl_stop, rootserver::ObTenantInfoReportor::mtl_wait, mtl_destroy_default); MTL_BIND2(mtl_new_default, rootserver::ObRecoveryLSService::mtl_init, nullptr, rootserver::ObRecoveryLSService::mtl_stop, rootserver::ObRecoveryLSService::mtl_wait, mtl_destroy_default); MTL_BIND2(mtl_new_default, rootserver::ObRestoreService::mtl_init, nullptr, rootserver::ObRestoreService::mtl_stop, rootserver::ObRestoreService::mtl_wait, mtl_destroy_default); MTL_BIND2(mtl_new_default, coordinator::ObLeaderCoordinator::mtl_init, coordinator::ObLeaderCoordinator::mtl_start, coordinator::ObLeaderCoordinator::mtl_stop, coordinator::ObLeaderCoordinator::mtl_wait, mtl_destroy_default); diff --git a/src/rootserver/CMakeLists.txt b/src/rootserver/CMakeLists.txt index 0fffc9b49b..a1138982a9 100644 --- a/src/rootserver/CMakeLists.txt +++ b/src/rootserver/CMakeLists.txt @@ -66,14 +66,10 @@ ob_set_subtarget(ob_rootserver common ob_vtable_location_getter.cpp ob_zone_manager.cpp ob_zone_unit_provider.cpp - ob_tenant_thread_helper.cpp - ob_ls_recovery_reportor.cpp + ob_tenant_recovery_reportor.cpp ob_tenant_info_loader.cpp - ob_tenant_info_report.cpp ob_primary_ls_service.cpp ob_recovery_ls_service.cpp - ob_common_ls_service.cpp - ob_ls_service_helper.cpp ob_tenant_role_transition_service.cpp ob_server_balancer.cpp ob_system_admin_util.cpp diff --git a/src/rootserver/ob_common_ls_service.cpp b/src/rootserver/ob_common_ls_service.cpp deleted file mode 100755 index a9bb6a35c0..0000000000 --- a/src/rootserver/ob_common_ls_service.cpp +++ /dev/null @@ -1,597 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ - -#define USING_LOG_PREFIX RS -#include "ob_common_ls_service.h" -#include "lib/profile/ob_trace_id.h" -#include "share/ob_errno.h" -#include "share/ob_max_id_fetcher.h" -#include "share/schema/ob_schema_struct.h"//ObTenantInfo -#include "share/ls/ob_ls_creator.h" //ObLSCreator -#include "share/ls/ob_ls_life_manager.h"//ObLSLifeAgentManager -#include "share/ob_primary_zone_util.h"//ObPrimaryZoneUtil -#include "share/ob_share_util.h"//ObShareUtil -#include "share/ob_tenant_info_proxy.h"//ObAllTenantInfo -#include "share/ob_common_rpc_proxy.h"//common_rpc_proxy -#include "observer/ob_server_struct.h"//GCTX -#include "rootserver/ob_tenant_info_loader.h"//get_tenant_info -#include "logservice/palf/palf_base_info.h"//PalfBaseInfo - -namespace oceanbase -{ -using namespace common; -using namespace share; -using namespace transaction; -using namespace palf; -namespace rootserver -{ -//////////////ObCommonLSService -int ObCommonLSService::init() -{ - int ret = OB_SUCCESS; - tenant_id_ = MTL_ID(); - if (OB_UNLIKELY(inited_)) { - ret = OB_INIT_TWICE; - LOG_WARN("has inited", KR(ret)); - } else if (OB_FAIL(ObTenantThreadHelper::create("COMMONLSSe", - lib::TGDefIDs::SimpleLSService, *this))) { - LOG_WARN("failed to create thread", KR(ret)); - } else if (OB_FAIL(ObTenantThreadHelper::start())) { - LOG_WARN("fail to start", KR(ret)); - } else { - inited_ = true; - } - return ret; -} - -void ObCommonLSService::destroy() -{ - ObTenantThreadHelper::destroy(); - tenant_id_ = OB_INVALID_TENANT_ID; - inited_ = false; -} - -void ObCommonLSService::do_work() -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", K(ret)); - } else if (OB_FAIL(wait_tenant_schema_and_version_ready_(tenant_id_, DATA_VERSION_4_1_0_0))) { - LOG_WARN("failed to wait tenant schema version ready", KR(ret), K(tenant_id_), K(DATA_CURRENT_VERSION)); - } else { - int64_t idle_time_us = 100 * 1000L; - share::schema::ObTenantSchema user_tenant_schema; - while (!has_set_stop()) { - ret = OB_SUCCESS; - ObCurTraceId::init(GCONF.self_addr_); - if (is_meta_tenant(tenant_id_)) { - const uint64_t user_tenant_id = gen_user_tenant_id(tenant_id_); - if (OB_FAIL(get_tenant_schema(user_tenant_id, user_tenant_schema))) { - LOG_WARN("failed to get user tenant schema", KR(ret), K(user_tenant_id)); - } else if (user_tenant_schema.is_dropping()) { - if (OB_FAIL(try_force_drop_tenant_(user_tenant_schema))) { - LOG_WARN("failed to force drop tenant", KR(ret), K(user_tenant_id)); - } - } else if (OB_FAIL(try_create_ls_(user_tenant_schema))) { - LOG_WARN("failed to create ls", KR(ret), K(user_tenant_schema)); - } - } - - if (REACH_TENANT_TIME_INTERVAL(1 * 1000 * 1000)) { - ret = OB_SUCCESS;//ignore error - if (OB_FAIL(try_update_sys_ls_primary_zone_())) { - LOG_WARN("failed to update sys ls primary zone", KR(ret)); - } - if (is_meta_tenant(tenant_id_)) { - if (FAILEDx(try_adjust_user_ls_primary_zone_(user_tenant_schema))) { - LOG_WARN("failed to adjust user ls primary zone", KR(ret), K(user_tenant_schema)); - } - } - } - - user_tenant_schema.reset(); - idle(idle_time_us); - LOG_INFO("[COMMON_LS_SERVICE] finish one round", KR(ret), K(idle_time_us)); - } // end while - } -} - -int ObCommonLSService::try_update_sys_ls_primary_zone_() -{ - int ret = OB_SUCCESS; - share::schema::ObTenantSchema tenant_schema; - if (OB_ISNULL(GCTX.schema_service_) || OB_ISNULL(GCTX.sql_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error", KR(ret), KP(GCTX.sql_proxy_), KP(GCTX.schema_service_)); - } else if (OB_FAIL(get_tenant_schema(tenant_id_, tenant_schema))) { - LOG_WARN("failed to get tenant schema", KR(ret), K(tenant_id_)); - } else { - share::ObLSPrimaryZoneInfo primary_zone_info; - ObArray primary_zone; - share::ObLSStatusOperator status_op; - ObZone new_primary_zone; - ObSqlString new_zone_priority; - if (OB_FAIL(ObPrimaryZoneUtil::get_tenant_primary_zone_array( - tenant_schema, primary_zone))) { - LOG_WARN("failed to get tenant primary zone array", KR(ret), K(tenant_schema)); - } else if (OB_FAIL(status_op.get_ls_primary_zone_info(tenant_id_, - SYS_LS, primary_zone_info, *GCTX.sql_proxy_))) { - LOG_WARN("failed to get ls primary_zone info", KR(ret), K(tenant_id_)); - } else if (OB_UNLIKELY(0 == primary_zone.count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("primary zone is empty", KR(ret), K(tenant_schema)); - } else if (has_exist_in_array(primary_zone, primary_zone_info.get_primary_zone())) { - if (OB_FAIL(new_primary_zone.assign(primary_zone_info.get_primary_zone()))) { - LOG_WARN("failed to assign primary zone", KR(ret), K(primary_zone_info)); - } - } else if (OB_FAIL(new_primary_zone.assign(primary_zone.at(0)))) { - LOG_WARN("failed to assign primary zone", KR(ret), K(primary_zone)); - } - if (OB_FAIL(ret)) { - } else if (is_sys_tenant(tenant_id_)) { - //sys tenant use tenant normalize primary zone - if (OB_FAIL(ObPrimaryZoneUtil::get_tenant_zone_priority( - tenant_schema, new_zone_priority))) { - LOG_WARN("failed to get tenant primary zone array", KR(ret), K(tenant_schema)); - } - } else if (OB_FAIL(get_zone_priority(new_primary_zone, - tenant_schema, new_zone_priority))) { - LOG_WARN("failed to get normalize primary zone", KR(ret), K(new_primary_zone)); - } - if (FAILEDx(try_update_ls_primary_zone_( - primary_zone_info, new_primary_zone, new_zone_priority))) { - LOG_WARN("failed to update ls primary zone", KR(ret), K(primary_zone_info), - K(new_primary_zone), K(new_zone_priority)); - } else if (is_meta_tenant(tenant_id_)) { - //user sys ls has same primary zone with meta sys ls - share::ObLSPrimaryZoneInfo user_primary_zone_info; - const uint64_t user_tenant_id = gen_user_tenant_id(tenant_id_); - if (OB_FAIL(status_op.get_ls_primary_zone_info(user_tenant_id, SYS_LS, - user_primary_zone_info, *GCTX.sql_proxy_))) { - LOG_WARN("failed to get ls primary_zone info", KR(ret), K(tenant_id_), K(user_tenant_id)); - } else if (OB_FAIL(try_update_ls_primary_zone_(user_primary_zone_info, new_primary_zone, new_zone_priority))) { - LOG_WARN("failed to update ls primary zone", KR(ret), K(user_primary_zone_info), - K(new_primary_zone), K(new_zone_priority)); - } - } - } - return ret; -} - -int ObCommonLSService::try_adjust_user_ls_primary_zone_(const share::schema::ObTenantSchema &tenant_schema) -{ - int ret = OB_SUCCESS; - share::ObLSStatusOperator status_op; - share::ObLSPrimaryZoneInfoArray info_array; - ObArray primary_zone; - const uint64_t tenant_id = tenant_schema.get_tenant_id(); - if (OB_ISNULL(GCTX.schema_service_) || OB_ISNULL(GCTX.sql_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error", KR(ret), KP(GCTX.sql_proxy_), KP(GCTX.schema_service_)); - } else if (!is_user_tenant(tenant_id) || !tenant_schema.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("tenant iS invalid", KR(ret), K(tenant_id), K(tenant_schema)); - } else if (OB_FAIL(ObPrimaryZoneUtil::get_tenant_primary_zone_array( - tenant_schema, primary_zone))) { - LOG_WARN("failed to get tenant primary zone array", KR(ret), K(tenant_schema)); - } else { - if (OB_FAIL(status_op.get_tenant_primary_zone_info_array( - tenant_id, info_array, *GCTX.sql_proxy_))) { - LOG_WARN("failed to get tenant primary zone info array", KR(ret), K(tenant_id)); - } - } - - if (OB_SUCC(ret)) { - uint64_t last_ls_group_id = OB_INVALID_ID; - share::ObLSPrimaryZoneInfoArray tmp_info_array; - for (int64_t i = 0; OB_SUCC(ret) && i < info_array.count(); ++i) { - //TODO ls group id maybe zero - const ObLSPrimaryZoneInfo &info = info_array.at(i); - if (!info.get_ls_id().is_sys_ls()) { - if (OB_INVALID_ID == last_ls_group_id) { - last_ls_group_id = info.get_ls_group_id(); - } - if (last_ls_group_id != info.get_ls_group_id()) { - //process the ls group - if (OB_FAIL(adjust_primary_zone_by_ls_group_(primary_zone, tmp_info_array, tenant_schema))) { - LOG_WARN("failed to update primary zone of each ls group", KR(ret), - K(tmp_info_array), K(primary_zone), K(tenant_schema)); - } else { - tmp_info_array.reset(); - last_ls_group_id = info.get_ls_group_id(); - } - } - if (FAILEDx(tmp_info_array.push_back(info))) { - LOG_WARN("failed to push back primary info array", KR(ret), K(i)); - } - } - } - if (OB_SUCC(ret) && 0 < tmp_info_array.count()) { - if (OB_FAIL(adjust_primary_zone_by_ls_group_(primary_zone, tmp_info_array, tenant_schema))) { - LOG_WARN("failed to update primary zone of each ls group", KR(ret), - K(tmp_info_array), K(primary_zone), K(tenant_schema)); - } - } - } - - return ret; -} -//check every ls has right primary zone in ls group -//if primary_zone of ls not in primary zone, try to choose a right zone, -//if can not find a zone to modify, modify the ls to exist zone, -//load_balancer will choose the right ls to drop -//eg: -//if ls group has z1, z2 and primary zone is : z1. need modify z2 to z1 -//if ls group has z1, z2 and primary zone is : z1, z3. modify z2 to z3; -int ObCommonLSService::adjust_primary_zone_by_ls_group_( - const common::ObIArray &primary_zone_array, - const share::ObLSPrimaryZoneInfoArray &primary_zone_infos, - const share::schema::ObTenantSchema &tenant_schema) -{ - int ret = OB_SUCCESS; - const uint64_t tenant_id = tenant_schema.get_tenant_id(); - const int64_t ls_count = primary_zone_infos.count(); - const int64_t primary_zone_count = primary_zone_array.count(); - if (OB_UNLIKELY(0 == primary_zone_count - || 0 == ls_count || !tenant_schema.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(primary_zone_infos), - K(primary_zone_array), K(tenant_schema)); - } else if (OB_ISNULL(GCTX.sql_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy is null", KR(ret)); - } else if (1 == ls_count - && primary_zone_infos.at(0).get_ls_id().is_sys_ls()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("can not be sys ls", KR(ret), K(ls_count), K(primary_zone_infos)); - } else { - //Algorithm Description: - //Assumption: We have 5 ls, 3 primary_zones(z1, z2, z3). - //1. Set the primary zone of ls to tenant's primary zone, - //choose the least number of log streams on the zone is selected for all zones - //2. After all the primary_zone of the ls are in the primary_zonw of the tenant, - //choose the primary_zone with the most and least ls. Adjust a certain number of ls to the smallest zone without exceeding the balance - //while guaranteeing that the number of the zone with the most is no less than the average. - ObArray ls_primary_zone;//is match with primary_zone_infos - ObSEArray count_group_by_zone;//ls count of each primary zone - ObSqlString new_zone_priority; - if (OB_FAIL(set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, - count_group_by_zone))) { - LOG_WARN("failed to set ls to primary zone", KR(ret), K(primary_zone_array), K(primary_zone_infos)); - } else if (OB_FAIL(balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone))) { - LOG_WARN("failed to balance ls primary zone", KR(ret), K(ls_primary_zone), K(count_group_by_zone)); - } - - for (int64_t i = 0; OB_SUCC(ret) && i < ls_count; ++i) { - const ObZone &new_primary_zone = ls_primary_zone.at(i); - if (OB_FAIL(get_zone_priority(new_primary_zone, tenant_schema, new_zone_priority))) { - LOG_WARN("failed to get normalize primary zone", KR(ret), K(new_primary_zone)); - } - if (FAILEDx(try_update_ls_primary_zone_(primary_zone_infos.at(i), new_primary_zone, new_zone_priority))) { - LOG_WARN("failed to update ls primary zone", KR(ret), "primary_zone_info", primary_zone_infos.at(i), - K(new_primary_zone), K(new_zone_priority)); - } - } - } - return ret; -} - -int ObCommonLSService::set_ls_to_primary_zone( - const common::ObIArray &primary_zone_array, - const share::ObLSPrimaryZoneInfoArray &primary_zone_infos, - common::ObIArray &ls_primary_zone, - common::ObIArray &count_group_by_zone) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(0 == primary_zone_array.count() - || 0 == primary_zone_infos.count())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(primary_zone_array), K(primary_zone_infos)); - } else { - const int64_t primary_zone_count = primary_zone_array.count(); - const int64_t ls_count = primary_zone_infos.count(); - //ls may not in primary zone, record the index of primary_zone_infos not in primary zone - ObSEArray index_not_primary_zone; - int64_t index = 0; - ARRAY_FOREACH_X(primary_zone_array, idx, cnt, OB_SUCC(ret)) { - if (OB_FAIL(count_group_by_zone.push_back(0))) { - LOG_WARN("failed to push back", KR(ret), K(idx)); - } - } - for (int64_t i = 0; OB_SUCC(ret) && i < ls_count; ++i) { - const ObZone ¤t_zone = primary_zone_infos.at(i).get_primary_zone(); - if (has_exist_in_array(primary_zone_array, current_zone, &index)) { - count_group_by_zone.at(index)++; - } else if (OB_FAIL(index_not_primary_zone.push_back(i))) { - LOG_WARN("failed to push back", KR(ret), K(i), K(current_zone)); - } - if (FAILEDx(ls_primary_zone.push_back(current_zone))) { - LOG_WARN("failed to push back current zone", KR(ret), K(i), K(current_zone)); - } - } - //1. take all ls primary zone to tenant primary zone, choose the less primary zone count - int64_t min_count = INT64_MAX; - int64_t min_index = 0; - for (int64_t i = 0; OB_SUCC(ret) && i < index_not_primary_zone.count(); ++i) { - const int64_t ls_index = index_not_primary_zone.at(i); - min_count = INT64_MAX; - ARRAY_FOREACH_X(primary_zone_array, idx, cnt, OB_SUCC(ret)) { - if (min_count > count_group_by_zone.at(idx)) { - min_count = count_group_by_zone.at(idx); - min_index = idx; - } - }//end for search min count - if (OB_FAIL(ret)) { - } else if (min_index >= primary_zone_count) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to found min count", KR(ret), K(min_index), K(primary_zone_array)); - } else if (OB_FAIL(ls_primary_zone.at(ls_index).assign(primary_zone_array.at(min_index)))) { - LOG_WARN("failed to assign primary zone", KR(ret), K(min_index), K(min_count), K(ls_index)); - } else { - count_group_by_zone.at(min_index)++; - } - } - } - return ret; - -} - -int ObCommonLSService::balance_ls_primary_zone( - const common::ObIArray &primary_zone_array, - common::ObIArray &ls_primary_zone, - common::ObIArray &count_group_by_zone) -{ - int ret = OB_SUCCESS; - const int64_t ls_count = ls_primary_zone.count(); - const int64_t primary_zone_count = count_group_by_zone.count(); - if (OB_UNLIKELY(0 == primary_zone_count - || 0 == ls_count - || primary_zone_count != primary_zone_array.count())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(count_group_by_zone), K(ls_primary_zone)); - } else { - int64_t max_count = -1, max_index = 0; - int64_t min_count = INT64_MAX, min_index = 0; - do { - max_count = -1, max_index = 0; - min_count = INT64_MAX, min_index = 0; - for (int64_t i = 0; OB_SUCC(ret) && i < primary_zone_count; ++i) { - const int64_t ls_count = count_group_by_zone.at(i); - if (min_count > ls_count) { - min_count = ls_count; - min_index = i; - } - if (max_count < ls_count) { - max_count = ls_count; - max_index = i; - } - }//end for find min and max count - if (OB_UNLIKELY(max_index >= primary_zone_count || min_index >= primary_zone_count - || -1 == max_count || INT64_MAX == min_count)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to get primary zone", KR(ret), K(max_index), K(min_index), - K(min_count), K(max_count), - K(primary_zone_array), K(primary_zone_count), K(ls_primary_zone)); - } else if (max_count - min_count > 1) { - for (int64_t i = 0; OB_SUCC(ret) && i < ls_count; ++i) { - if (ls_primary_zone.at(i) == primary_zone_array.at(max_index)) { - if (OB_FAIL(ls_primary_zone.at(i).assign(primary_zone_array.at(min_index)))) { - LOG_WARN("failed to push back ls primary zone", KR(ret), K(min_index)); - } else { - count_group_by_zone.at(max_index)--; - count_group_by_zone.at(min_index)++; - } - break;//only change one by one - } - } - } - } while (max_count - min_count > 1); - } - return ret; -} - -int ObCommonLSService::try_update_ls_primary_zone_( - const share::ObLSPrimaryZoneInfo &primary_zone_info, - const common::ObZone &new_primary_zone, - const common::ObSqlString &zone_priority) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!primary_zone_info.is_valid() - || new_primary_zone.is_empty() || zone_priority.empty())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("primary zone info is invalid", KR(ret), K(primary_zone_info), - K(new_primary_zone), K(zone_priority)); - } else if (OB_ISNULL(GCTX.sql_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy is null", KR(ret)); - } else if (new_primary_zone != primary_zone_info.get_primary_zone() - || zone_priority.string() != primary_zone_info.get_zone_priority_str()) { - ObLSLifeAgentManager ls_life_agent(*GCTX.sql_proxy_); - if (OB_FAIL(ls_life_agent.update_ls_primary_zone(primary_zone_info.get_tenant_id(), - primary_zone_info.get_ls_id(), - new_primary_zone, zone_priority.string()))) { - LOG_WARN("failed to update ls primary zone", KR(ret), K(primary_zone_info), - K(new_primary_zone), K(zone_priority)); - } - LOG_INFO("[COMMON_LS_SERVICE] update ls primary zone", KR(ret), K(new_primary_zone), - K(zone_priority), K(primary_zone_info)); - } else { - //no need update - } - return ret; -} - -int ObCommonLSService::try_create_ls_(const share::schema::ObTenantSchema &tenant_schema) -{ - int ret = OB_SUCCESS; - const uint64_t tenant_id = tenant_schema.get_tenant_id(); - if (OB_ISNULL(GCTX.sql_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy is null", KR(ret), KP(GCTX.sql_proxy_)); - } else if (!is_user_tenant(tenant_id) || !tenant_schema.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("tenant is invalid", KR(ret), K(tenant_id), K(tenant_schema)); - } else { - share::ObLSStatusInfoArray status_info_array; - share::ObLSStatusOperator ls_op; - ObLSRecoveryStat recovery_stat; - ObLSRecoveryStatOperator ls_recovery_operator; - palf::PalfBaseInfo palf_base_info; - int tmp_ret = OB_SUCCESS; - if (OB_FAIL(ls_op.get_all_ls_status_by_order( - tenant_id, status_info_array, *GCTX.sql_proxy_))) { - LOG_WARN("failed to update ls status", KR(ret), K(tenant_id)); - } - for (int64_t i = 0; OB_SUCC(ret) && i < status_info_array.count(); ++i) { - const ObLSStatusInfo &status_info = status_info_array.at(i); - if (status_info.ls_is_creating()) { - recovery_stat.reset(); - if (OB_FAIL(check_can_create_ls_(tenant_schema))) { - LOG_WARN("failed to wait can do recovery", KR(ret), K(tenant_schema)); - } else if (OB_FAIL(ls_recovery_operator.get_ls_recovery_stat( - tenant_id, status_info.ls_id_, false /*for_update*/, - recovery_stat, *GCTX.sql_proxy_))) { - LOG_WARN("failed to get ls recovery stat", KR(ret), K(tenant_id), - K(status_info)); - } else if (OB_FAIL(do_create_user_ls(tenant_schema, status_info, - recovery_stat.get_create_scn(), - false, palf_base_info))) { - LOG_WARN("failed to create new ls", KR(ret), K(status_info), - K(recovery_stat)); - } - } - } // end for - } - return ret; -} - -int ObCommonLSService::check_can_create_ls_(const share::schema::ObTenantSchema &tenant_schema) -{ - int ret = OB_SUCCESS; - const uint64_t tenant_id = tenant_schema.get_tenant_id(); - if (OB_ISNULL(GCTX.sql_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy is null", KR(ret), KP(GCTX.sql_proxy_)); - } else if (!is_user_tenant(tenant_id) || !tenant_schema.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("tenant is invalid", KR(ret), K(tenant_id), K(tenant_schema)); - } else if (tenant_schema.is_restore()) { - //meta and user tenant unit must in the same observer - MTL_SWITCH(tenant_id) { - ObAllTenantInfo tenant_info; - ObTenantInfoLoader* tenant_loader = MTL(rootserver::ObTenantInfoLoader*); - if (OB_ISNULL(tenant_loader)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant report is null", KR(ret), K(tenant_id_)); - } else if (OB_FAIL(tenant_loader->get_tenant_info(tenant_info))) { - LOG_WARN("failed to get tenant info", KR(ret), K(tenant_id)); - } else if (OB_FAIL(check_can_do_recovery(tenant_info))) { - LOG_WARN("failed to wait can do recovery", KR(ret), K(tenant_info), K(tenant_id)); - } - } - } - return ret; -} -int ObCommonLSService::do_create_user_ls( - const share::schema::ObTenantSchema &tenant_schema, - const share::ObLSStatusInfo &info, const SCN &create_scn, - bool create_with_palf, const palf::PalfBaseInfo &palf_base_info) -{ - int ret = OB_SUCCESS; - LOG_INFO("[COMMON_LS_SERVICE] start to create ls", K(info), K(create_scn)); - const int64_t start_time = ObTimeUtility::fast_current_time(); - if (OB_UNLIKELY(!info.is_valid() || !info.ls_is_creating())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("info not valid", KR(ret), K(info)); - } else { - CK(OB_NOT_NULL(GCTX.sql_proxy_), OB_NOT_NULL(GCTX.srv_rpc_proxy_)) - common::ObArray locality_array; - int64_t paxos_replica_num = 0; - ObSchemaGetterGuard guard;//nothing - if (FAILEDx(tenant_schema.get_zone_replica_attr_array( - locality_array))) { - LOG_WARN("failed to get zone locality array", KR(ret)); - } else if (OB_FAIL(tenant_schema.get_paxos_replica_num( - guard, paxos_replica_num))) { - LOG_WARN("failed to get paxos replica num", KR(ret)); - } else { - ObLSCreator creator(*GCTX.srv_rpc_proxy_, info.tenant_id_, - info.ls_id_, GCTX.sql_proxy_); - if (OB_FAIL(creator.create_user_ls(info, paxos_replica_num, - locality_array, create_scn, - tenant_schema.get_compatibility_mode(), - create_with_palf, - palf_base_info))) { - LOG_WARN("failed to create user ls", KR(ret), K(info), K(locality_array), K(create_scn), - K(palf_base_info), K(create_with_palf)); - } - } - } - const int64_t cost = ObTimeUtility::fast_current_time() - start_time; - LOG_INFO("[COMMON_LS_SERVICE] end to create ls", KR(ret), K(info), K(cost)); - return ret; -} - -int ObCommonLSService::try_force_drop_tenant_( - const share::schema::ObTenantSchema &tenant_schema) -{ - int ret = OB_SUCCESS; - ObTimeoutCtx ctx; - const uint64_t tenant_id = tenant_schema.get_tenant_id(); - - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret)); - } else if (OB_ISNULL(GCTX.sql_proxy_) - || OB_ISNULL(GCTX.rs_rpc_proxy_) - || OB_ISNULL(GCTX.schema_service_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ptr is null", KR(ret), KP(GCTX.sql_proxy_), KP(GCTX.rs_rpc_proxy_), - KP(GCTX.schema_service_)); - } else if (!is_user_tenant(tenant_id) || !tenant_schema.is_valid() - || !tenant_schema.is_dropping()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("tenant is invalid", KR(ret), K(tenant_id), K(tenant_schema)); - } else { - LOG_INFO("try drop tenant", K(tenant_id)); - ObLSStatusOperator op; - share::ObLSStatusInfoArray ls_array; - if (OB_FAIL(ObShareUtil::set_default_timeout_ctx(ctx, GCONF.rpc_timeout))) { - LOG_WARN("fail to set timeout ctx", KR(ret), K(tenant_id)); - } else if (OB_FAIL(op.get_all_ls_status_by_order(tenant_id, ls_array, *GCTX.sql_proxy_))) { - LOG_WARN("fail to get all ls status", KR(ret), K(tenant_id)); - } else if (ls_array.count() <= 0) { - obrpc::ObDropTenantArg arg; - arg.exec_tenant_id_ = OB_SYS_TENANT_ID; - arg.tenant_name_ = tenant_schema.get_tenant_name(); - arg.tenant_id_ = tenant_schema.get_tenant_id(); - arg.if_exist_ = true; - arg.delay_to_drop_ = false; - ObSqlString sql; - const int64_t timeout_ts = ctx.get_timeout(); - if (OB_FAIL(sql.append_fmt("DROP TENANT IF EXISTS %s FORCE", arg.tenant_name_.ptr()))) { - LOG_WARN("fail to generate sql", KR(ret), K(arg)); - } else if (FALSE_IT(arg.ddl_stmt_str_ = sql.string())) { - } else if (OB_FAIL(GCTX.rs_rpc_proxy_->timeout(timeout_ts).drop_tenant(arg))) { - LOG_WARN("fail to drop tenant", KR(ret), K(arg), K(timeout_ts)); - } - } else { - // tenant's logstream is still dropping, check next round - } - } - return ret; -} - - -}//end of rootserver -} diff --git a/src/rootserver/ob_common_ls_service.h b/src/rootserver/ob_common_ls_service.h deleted file mode 100644 index eb3af16e35..0000000000 --- a/src/rootserver/ob_common_ls_service.h +++ /dev/null @@ -1,126 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ - -#ifndef OCEANBASE_ROOTSERVER_OB_COMMON_LS_SERVICE_H -#define OCEANBASE_ROOTSERVER_OB_COMMON_LS_SERVICE_H -#include "lib/thread/ob_reentrant_thread.h"//ObRsReentrantThread -#include "logservice/ob_log_base_type.h" -#include "share/scn.h"//SCN -#include "share/ls/ob_ls_status_operator.h"//ObLSStatusOperator -#include "share/ls/ob_ls_operator.h" //ObLSAttr -#include "share/ob_thread_mgr.h" //OBTGDefIDEnum -#include "logservice/palf/palf_iterator.h" //PalfBufferIterator -#include "share/unit/ob_unit_info.h"//ObUnit::Status -#include "lib/thread/thread_mgr_interface.h" // TGRunnable -#include "lib/lock/ob_thread_cond.h"//ObThreadCond -#include "rootserver/ob_tenant_thread_helper.h"//ObTenantThreadHelper - - -namespace oceanbase -{ -namespace obrpc -{ -class ObSrvRpcProxy; -} -namespace common -{ -class ObMySQLProxy; -class ObMySQLTransaction; -} -namespace share -{ -namespace schema -{ -class ObMultiVersionSchemaService; -class ObTenantSchema; -} - -} -namespace palf -{ -struct PalfBaseInfo; -} -namespace rootserver -{ -/*description: - *COMMON_LS_SERVICE thread: Started on the leader of the meta tenant sys ls - * 1. adjust sys ls primary zone of meta and user tenant - * 2. adjust user ls primary zone - * 3. make ls status from creating to created of __all_ls_status - * 4. check dropping tenant need drop tenant force; - * */ -class ObCommonLSService : public ObTenantThreadHelper, - public logservice::ObICheckpointSubHandler, - public logservice::ObIReplaySubHandler -{ -public: - ObCommonLSService():inited_(false), tenant_id_(OB_INVALID_TENANT_ID) {} - virtual ~ObCommonLSService() {} - int init(); - void destroy(); - virtual void do_work() override; - DEFINE_MTL_FUNC(ObCommonLSService) - -public: - virtual share::SCN get_rec_scn() override { return share::SCN::max_scn();} - virtual int flush(share::SCN &scn) override { return OB_SUCCESS; } - int replay(const void *buffer, const int64_t nbytes, const palf::LSN &lsn, const share::SCN &scn) override - { - UNUSED(buffer); - UNUSED(nbytes); - UNUSED(lsn); - UNUSED(scn); - return OB_SUCCESS; - } - -private: - int try_update_sys_ls_primary_zone_(); - int try_adjust_user_ls_primary_zone_(const share::schema::ObTenantSchema &tenant_schema); - int try_update_ls_primary_zone_( - const share::ObLSPrimaryZoneInfo &primary_zone_info, - const common::ObZone &new_primary_zone, - const common::ObSqlString &zone_priority); - static int set_ls_to_primary_zone(const common::ObIArray &primary_zone_array, - const share::ObLSPrimaryZoneInfoArray &primary_zone_infos, - common::ObIArray &ls_primary_zone, - common::ObIArray &count_group_by_zone); - static int balance_ls_primary_zone(const common::ObIArray &primary_zone_array, - common::ObIArray &ls_primary_zone, - common::ObIArray &count_group_by_zone); - int adjust_primary_zone_by_ls_group_(const common::ObIArray &primary_zone_array, - const share::ObLSPrimaryZoneInfoArray &primary_zone_infos, - const share::schema::ObTenantSchema &tenant_schema); - - - // force drop user tenant if tenant is in dropping status - int try_force_drop_tenant_( - const share::schema::ObTenantSchema &tenant_schema); - int try_create_ls_(const share::schema::ObTenantSchema &tenant_schema); - //restore tenant can not create ls before "CREATE_INIT_LS" status - int check_can_create_ls_(const share::schema::ObTenantSchema &tenant_schema); -public: - //restore_service need create init ls too - static int do_create_user_ls(const share::schema::ObTenantSchema &tenant_schema, - const share::ObLSStatusInfo &info, - const SCN &create_scn, - bool create_with_palf, - const palf::PalfBaseInfo &palf_base_info); -private: - bool inited_; - uint64_t tenant_id_; - -}; -} -} - - -#endif /* !OCEANBASE_ROOTSERVER_OB_COMMON_LS_SERVICE_H */ diff --git a/src/rootserver/ob_ddl_service.cpp b/src/rootserver/ob_ddl_service.cpp index c19c1261d3..8ba15b379a 100644 --- a/src/rootserver/ob_ddl_service.cpp +++ b/src/rootserver/ob_ddl_service.cpp @@ -72,7 +72,7 @@ #include "rootserver/ddl_task/ob_ddl_retry_task.h" #include "rootserver/freeze/ob_freeze_info_manager.h" #include "rootserver/freeze/ob_major_freeze_helper.h" -#include "rootserver/ob_tenant_thread_helper.h"//get_zone_priority +#include "rootserver/ob_primary_ls_service.h"//ObTenantLSInfo #include "lib/utility/ob_tracepoint.h" #include "observer/ob_server_struct.h" #include "storage/tx/ob_ts_mgr.h" @@ -20721,7 +20721,7 @@ int ObDDLService::create_tenant_sys_ls( } else if (OB_UNLIKELY(0 == primary_zone_list.count())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("primary zone is empty", KR(ret), K(tenant_schema)); - } else if (OB_FAIL(ObTenantThreadHelper::get_zone_priority( + } else if (OB_FAIL(ObTenantLSInfo::get_zone_priority( primary_zone_list.at(0), tenant_schema, zone_priority))) { LOG_WARN("failed to get zone priority", KR(ret), K(primary_zone_list), K(tenant_schema)); } else if (OB_FAIL(ls_creator.create_tenant_sys_ls( @@ -21203,7 +21203,7 @@ int ObDDLService::set_sys_ls_status(const uint64_t tenant_id) if (OB_FAIL(new_ls.init(SYS_LS, ls_group_id, flag, share::OB_LS_NORMAL, share::OB_LS_OP_CREATE_END, create_scn))) { LOG_WARN("failed to init new operation", KR(ret), K(flag), K(create_scn)); - } else if (OB_FAIL(ls_operator.insert_ls(new_ls, ls_group_id))) { + } else if (OB_FAIL(ls_operator.insert_ls(new_ls, ls_group_id, share::NORMAL_SWITCHOVER_STATUS))) { LOG_WARN("failed to insert new ls", KR(ret), K(new_ls), K(ls_group_id)); } } diff --git a/src/rootserver/ob_ls_service_helper.cpp b/src/rootserver/ob_ls_service_helper.cpp deleted file mode 100755 index 5bdfd20610..0000000000 --- a/src/rootserver/ob_ls_service_helper.cpp +++ /dev/null @@ -1,1860 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ - -#define USING_LOG_PREFIX RS -#include "ob_ls_service_helper.h" -#include "lib/profile/ob_trace_id.h" -#include "share/ob_errno.h" -#include "share/ob_max_id_fetcher.h" -#include "share/schema/ob_schema_struct.h"//ObTenantInfo -#include "share/schema/ob_schema_service.h"//ObMultiSchemaService -#include "share/ls/ob_ls_creator.h" //ObLSCreator -#include "share/ls/ob_ls_life_manager.h"//ObLSLifeAgentManager -#include "share/ls/ob_ls_table_operator.h"//ObLSTableOpertor -#include "share/ob_primary_zone_util.h"//ObPrimaryZoneUtil -#include "share/ob_unit_table_operator.h"//ObUnitTableOperator -#include "share/ob_zone_table_operation.h" //ObZoneTableOperation -#include "share/ob_share_util.h"//ObShareUtil -#include "share/restore/ob_physical_restore_table_operator.h"//ObTenantRestoreTableOperator -#include "share/ob_standby_upgrade.h"//ObStandbyUpgrade -#include "share/ob_upgrade_utils.h"//ObUpgradeChecker -#include "observer/ob_server_struct.h"//GCTX -#include "rootserver/ob_tenant_info_loader.h"//get_tenant_info -#include "rootserver/ob_tenant_thread_helper.h"//get_zone_priority -#include "storage/tx_storage/ob_ls_map.h" -#include "storage/tx_storage/ob_ls_service.h" -#include "storage/tx_storage/ob_ls_handle.h" //ObLSHandle -#include "logservice/palf/palf_base_info.h"//PalfBaseInfo -#include "logservice/ob_log_service.h"//ObLogService - -namespace oceanbase -{ -using namespace common; -using namespace share; -using namespace transaction; -using namespace palf; -namespace rootserver -{ -/////////ObUnitGroupInfo -bool ObUnitGroupInfo::is_valid() const -{ - return OB_INVALID_ID != unit_group_id_ - && share::ObUnit::UNIT_STATUS_MAX != unit_status_; -} - -int ObUnitGroupInfo::init(const uint64_t unit_group_id, - const share::ObUnit::Status &unit_status) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(OB_INVALID_ID == unit_group_id - || share::ObUnit::UNIT_STATUS_MAX == unit_status)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(unit_group_id), K(unit_status)); - } else { - unit_group_id_ = unit_group_id; - unit_status_ = unit_status; - } - return ret; -} - -void ObUnitGroupInfo::reset() -{ - unit_group_id_ = OB_INVALID_ID; - unit_status_ = share::ObUnit::UNIT_STATUS_MAX; - ls_group_ids_.reset(); -} - -int ObUnitGroupInfo::assign(const ObUnitGroupInfo &other) -{ - int ret = OB_SUCCESS; - if (this != &other) { - if (OB_FAIL(ls_group_ids_.assign(other.ls_group_ids_))) { - LOG_WARN("failed to assign", KR(ret), K(other)); - } else { - unit_group_id_ = other.unit_group_id_; - unit_status_ = other.unit_status_; - } - } - return ret; -} - -int ObUnitGroupInfo::remove_ls_group(const uint64_t ls_group_id) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(ObLSID::MIN_USER_LS_GROUP_ID >= ls_group_id)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("ls group id is invalid", KR(ret), K(ls_group_id)); - } else { - bool remove = false; - for (int64_t i = 0; OB_SUCC(ret) && i < ls_group_ids_.count(); ++i) { - if (ls_group_ids_.at(i) == ls_group_id) { - remove = true; - if (OB_FAIL(ls_group_ids_.remove(i))) { - LOG_WARN("failed to remove from array", KR(ret), K(i), - K(ls_group_id), "this", *this); - } - break; - } - } - if (OB_SUCC(ret) && !remove) { - ret = OB_ENTRY_NOT_EXIST; - LOG_WARN("failed to find ls group id", KR(ret), - K(ls_group_id), "this", *this); - } - } - return ret; -} - -bool ObUnitGroupInfo::operator==(const ObUnitGroupInfo &other) const -{ - return unit_group_id_ == other.unit_group_id_ - && unit_status_ == other.unit_status_; -} - -////////////ObLSGroupInfo -bool ObLSGroupInfo::is_valid() const -{ - return OB_INVALID_ID != unit_group_id_ - && OB_INVALID_ID != ls_group_id_; -} - -int ObLSGroupInfo::init(const uint64_t unit_group_id, - const uint64_t ls_group_id) -{ - int ret = OB_SUCCESS; - reset(); - if (OB_UNLIKELY(OB_INVALID_ID == unit_group_id - || OB_INVALID_ID == ls_group_id)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(unit_group_id), K(ls_group_id)); - } else { - unit_group_id_ = unit_group_id; - ls_group_id_ = ls_group_id; - } - return ret; -} - -int ObLSGroupInfo::assign(const ObLSGroupInfo &other) -{ - int ret = OB_SUCCESS; - if (this != &other) { - reset(); - if (OB_FAIL(ls_ids_.assign(other.ls_ids_))) { - LOG_WARN("failed to assign ls ids", KR(ret), K(other)); - } else { - unit_group_id_ = other.unit_group_id_; - ls_group_id_ = other.ls_group_id_; - } - } - return ret; -} - -void ObLSGroupInfo::reset() -{ - ls_group_id_ = OB_INVALID_ID; - unit_group_id_ = OB_INVALID_ID; - ls_ids_.reset(); -} - -int ObLSGroupInfo::remove_ls(const share::ObLSID &ls_id) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!ls_id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("ls id is invalid", KR(ret), K(ls_id)); - } else { - bool remove = false; - for (int64_t i = 0; OB_SUCC(ret) && i < ls_ids_.count(); ++i) { - if (ls_ids_.at(i) == ls_id) { - remove = true; - if (OB_FAIL(ls_ids_.remove(i))) { - LOG_WARN("failed to remove from array", KR(ret), K(i), - K(ls_id), "this", *this); - } - break; - } - } - if (OB_SUCC(ret) && !remove) { - ret = OB_ENTRY_NOT_EXIST; - LOG_WARN("failed to find ls id", KR(ret), - K(ls_id), "this", *this); - } - } - return ret; - -} - -///////////////ObLSStatusMachineParameter -//no need check status valid or ls_status is valid -int ObLSStatusMachineParameter::init(const share::ObLSID &id, - const share::ObLSStatusInfo &status_info, - const share::ObLSAttr &ls_info) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("id is invalid", KR(ret), K(id)); - } else if (OB_FAIL(status_info_.assign(status_info))) { - LOG_WARN("failed to assign status info", KR(ret), K(status_info)); - } else if (OB_FAIL(ls_info_.assign(ls_info))) { - LOG_WARN("failed to assign ls info", KR(ret), K(ls_info)); - } else { - ls_info_ = ls_info; - ls_id_ = id; - } - return ret; -} - -void ObLSStatusMachineParameter::reset() -{ - ls_id_.reset(); - ls_info_.reset(); - status_info_.reset(); -} - -/////////////ObTenantLSInfo - -void ObTenantLSInfo::reset() -{ - is_load_ = false; - status_array_.reset(); - unit_group_array_.reset(); - ls_group_array_.reset(); - primary_zone_.reset(); - max_ls_id_ = OB_INVALID_ID; - max_ls_group_id_ = OB_INVALID_ID; -} - - -bool ObTenantLSInfo::is_valid() const -{ - return ATOMIC_LOAD(&is_load_); -} - -int ObTenantLSInfo::revision_to_equal_status(common::ObMySQLTransaction &trans) -{ - int ret = OB_SUCCESS; - ObArray status_machine_array; - if (OB_FAIL(construct_ls_status_machine_(status_machine_array))) { - LOG_WARN("failed to construct ls status machine array", KR(ret)); - } else if (OB_ISNULL(sql_proxy_) || OB_ISNULL(tenant_schema_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy or tenant schema is null", KR(ret), KP(sql_proxy_), KP(tenant_schema_)); - } else { - ObLSLifeAgentManager ls_life_agent(*sql_proxy_); - const uint64_t tenant_id = tenant_schema_->get_tenant_id(); - ARRAY_FOREACH_NORET(status_machine_array, idx) { - const ObLSStatusMachineParameter &machine = status_machine_array.at(idx); - const share::ObLSStatusInfo &status_info = machine.status_info_; - const share::ObLSAttr &ls_info = machine.ls_info_; - if (ls_info.get_ls_status() == status_info.status_) { - //if ls and ls status is equal, need to process - } else if (!ls_info.is_valid()) { - if (status_info.ls_is_create_abort()) { - //ls may create_abort - if (OB_FAIL(ls_life_agent.drop_ls_in_trans(tenant_id, machine.ls_id_, - share::NORMAL_SWITCHOVER_STATUS, trans))) { - LOG_WARN("failed to delete ls", KR(ret), K(machine)); - } - } else { - //ls can not be waitoffline or other status - ret = OB_ERR_UNEXPECTED; - LOG_WARN("status machine not expected", KR(ret), K(machine)); - } - } else if (ls_info.ls_is_creating()) { - if (!status_info.is_valid()) { - if (OB_FAIL(create_new_ls_in_trans(ls_info.get_ls_id(), - ls_info.get_ls_group_id(), - ls_info.get_create_scn(), trans))) { - LOG_WARN("failed to create new ls", KR(ret), K(machine)); - } - } else if (status_info.ls_is_created() - || status_info.ls_is_create_abort()) { - //nothing todo - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("status machine not expected", KR(ret), K(machine)); - } - } else if (ls_info.ls_is_normal()) { - if (status_info.ls_is_created()) { - if (OB_FAIL(status_operator_.update_ls_status( - tenant_id, status_info.ls_id_, status_info.status_, - ls_info.get_ls_status(), share::NORMAL_SWITCHOVER_STATUS, trans))) { - LOG_WARN("failed to update ls status", KR(ret), K(status_info), K(tenant_id), K(ls_info)); - } - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("status machine not expected", KR(ret), K(machine)); - } - } else { - //in upgrade, tenant can not be dropping, so no need to take care of - //pre_tenant_dropping, tenant_dropping, waitoffline - ret = OB_ERR_UNEXPECTED; - LOG_WARN("the ls not expected in all_ls", KR(ret), K(machine)); - } - } - } - return ret; -} - - -//Regardless of the tenant being dropped, -//handle the asynchronous operation of status and history table -int ObTenantLSInfo::process_next_ls_status(int64_t &task_cnt) -{ - int ret = OB_SUCCESS; - ObArray status_machine_array; - - if (OB_FAIL(construct_ls_status_machine_(status_machine_array))) { - LOG_WARN("failed to construct ls status machine array", KR(ret)); - } else { - int tmp_ret = OB_SUCCESS; - bool is_steady = false; - ARRAY_FOREACH_NORET(status_machine_array, idx) { - const ObLSStatusMachineParameter &machine = status_machine_array.at(idx); - if (OB_TMP_FAIL(process_next_ls_status_(machine, is_steady))) { - LOG_WARN("failed to do next ls process", KR(ret), KR(tmp_ret), K(machine)); - } - if (OB_TMP_FAIL(tmp_ret) || !is_steady) { - task_cnt++; - } - } - } - - return ret; -} - -int ObTenantLSInfo::construct_ls_status_machine_( - common::ObIArray &status_machine_array) -{ - int ret = OB_SUCCESS; - status_machine_array.reset(); - share::ObLSStatusInfoArray status_info_array; - share::ObLSAttrArray ls_array; - if (OB_ISNULL(sql_proxy_) || OB_ISNULL(tenant_schema_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy or tenant schema is null", KR(ret), KP(tenant_schema_), KP(sql_proxy_)); - } else if (OB_FAIL(ls_operator_.get_all_ls_by_order(ls_array))) { - LOG_WARN("failed to get all ls by order", KR(ret)); - } else { - const uint64_t tenant_id = tenant_schema_->get_tenant_id(); - if (OB_FAIL(status_operator_.get_all_ls_status_by_order( - tenant_id, status_info_array, *sql_proxy_))) { - LOG_WARN("failed to get all ls status by order", KR(ret)); - } - } - if (OB_FAIL(ret)) { - } else if (OB_UNLIKELY(0 == status_info_array.count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls array can not be empty", KR(ret), K(ls_array), K(status_info_array)); - } else { - // merge by sort - int64_t status_index = 0; - int64_t ls_index = 0; - ObLSStatusMachineParameter status_machine; - const int64_t status_count = status_info_array.count(); - const int64_t ls_count = ls_array.count(); - share::ObLSStatusInfo status_info; - ObLSID ls_id; - ObLSAttr ls_info; - while ((status_index < status_count || ls_index < ls_count) && OB_SUCC(ret)) { - status_machine.reset(); - if (status_index == status_count) { - //status already end - if (OB_FAIL(ls_info.assign(ls_array.at(ls_index)))) { - LOG_WARN("failed to assign ls info", KR(ret), K(ls_index), K(ls_array)); - } else { - ls_id = ls_array.at(ls_index).get_ls_id(); - status_info.reset(); - ls_index++; - } - } else if (ls_index == ls_count) { - //ls already end - if (OB_FAIL(status_info.assign(status_info_array.at(status_index)))) { - LOG_WARN("failed to assign status info", KR(ret), K(status_index)); - } else { - status_index++; - ls_id = status_info.ls_id_; - ls_info.reset(); - } - } else { - const share::ObLSStatusInfo &tmp_status_info = status_info_array.at(status_index); - const share::ObLSAttr &tmp_ls_info = ls_array.at(ls_index); - if (tmp_status_info.ls_id_ == tmp_ls_info.get_ls_id()) { - status_index++; - ls_index++; - ls_id = tmp_status_info.ls_id_; - if (OB_FAIL(ls_info.assign(tmp_ls_info))) { - LOG_WARN("failed to assign tmp ls info", KR(ret), K(tmp_ls_info)); - } else if (OB_FAIL(status_info.assign(tmp_status_info))) { - LOG_WARN("failed to assign status info", KR(ret), K(tmp_status_info)); - } - } else if (tmp_status_info.ls_id_ > tmp_ls_info.get_ls_id()) { - ls_index++; - ls_id = tmp_ls_info.get_ls_id(); - status_info.reset(); - if (OB_FAIL(ls_info.assign(tmp_ls_info))) { - LOG_WARN("failed to assign tmp ls info", KR(ret), K(tmp_ls_info)); - } - } else { - status_index++; - ls_id = tmp_status_info.ls_id_; - ls_info.reset(); - if (OB_FAIL(status_info.assign(tmp_status_info))) { - LOG_WARN("failed to assign status info", KR(ret), K(tmp_status_info)); - } - } - } - if (FAILEDx(status_machine.init(ls_id, status_info, ls_info))) { - LOG_WARN("failed to init status machine", KR(ret), K(ls_id), K(status_info), K(ls_info)); - } else if (OB_FAIL(status_machine_array.push_back(status_machine))) { - LOG_WARN("failed to push back status machine", KR(ret), K(status_machine)); - } - } // end while - - } - return ret; -} - -int ObTenantLSInfo::drop_tenant() -{ - int ret = OB_SUCCESS; - share::ObLSAttrArray ls_array; - if (OB_FAIL(ls_operator_.get_all_ls_by_order(ls_array))) { - LOG_WARN("failed to get all ls", KR(ret)); - } else { - //1. set sys_ls to tenant_dropping in __all_ls - //2. set user_ls to tenant_dropping or abort in __all_ls - for (int64_t i = 0; OB_SUCC(ret) && i < ls_array.count(); ++i) { - share::ObLSAttr &attr = ls_array.at(i); - if (attr.get_ls_id().is_sys_ls()) { - if (attr.ls_is_normal()) { - if (OB_FAIL(ls_operator_.update_ls_status(attr.get_ls_id(), - attr.get_ls_status(), share::OB_LS_PRE_TENANT_DROPPING))) { - LOG_WARN("failed to update ls status", KR(ret), K(attr)); - } - } - } else if (attr.ls_is_creating()) { - //drop the status - if (OB_FAIL(ls_operator_.delete_ls(attr.get_ls_id(), attr.get_ls_status()))) { - LOG_WARN("failed to remove ls not normal", KR(ret), K(attr)); - } - } else { - //no matter the status is in normal or dropping - //may be the status in status info is created - if (!attr.ls_is_tenant_dropping()) { - if (OB_FAIL(ls_operator_.update_ls_status( - attr.get_ls_id(), attr.get_ls_status(), - share::OB_LS_TENANT_DROPPING))) { - LOG_WARN("failed to update ls status", KR(ret), K(attr)); - } else { - LOG_INFO("[LS_MGR] set ls to tenant dropping", KR(ret), K(attr)); - } - } - } - }//end for - } - return ret; -} -int ObTenantLSInfo::gather_stat(bool for_recovery) -{ - int ret = OB_SUCCESS; - reset(); - if (OB_ISNULL(tenant_schema_) || OB_ISNULL(sql_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected", KR(ret), KP(tenant_schema_), KP(sql_proxy_)); - } else { - ObUnitTableOperator unit_operator; - common::ObArray units; - const uint64_t tenant_id = tenant_schema_->get_tenant_id(); - if (OB_FAIL(unit_operator.init(*sql_proxy_))) { - LOG_WARN("failed to init unit operator", KR(ret)); - } else if (OB_FAIL(unit_operator.get_units_by_tenant( - tenant_id, units))) { - LOG_WARN("failed to get all tenant unit", KR(ret), K(tenant_id)); - } else { - ObUnitGroupInfo info; - for (int64_t j = 0; OB_SUCC(ret) && j < units.count(); ++j) { - info.reset(); - const ObUnit &unit = units.at(j); - if (OB_FAIL(info.init(unit.unit_group_id_, unit.status_))) { - LOG_WARN("failed to init unit info", KR(ret), K(unit)); - } else if (has_exist_in_array(unit_group_array_, info)) { - //nothing - } else if (OB_FAIL(unit_group_array_.push_back(info))) { - LOG_WARN("fail to push back", KR(ret), K(info)); - } - } - } - if (OB_SUCC(ret)) { - if (for_recovery) { - } else { - //get max ls id and max ls group id - share::ObMaxIdFetcher id_fetcher(*sql_proxy_); - if (OB_FAIL(id_fetcher.fetch_max_id(*sql_proxy_, tenant_id, - share::OB_MAX_USED_LS_GROUP_ID_TYPE, max_ls_group_id_))) { - LOG_WARN("failed to fetch max ls group id", KR(ret), K(tenant_id)); - } else if (OB_FAIL(id_fetcher.fetch_max_id(*sql_proxy_, tenant_id, - share::OB_MAX_USED_LS_ID_TYPE, max_ls_id_))) { - LOG_WARN("failed to fetch max ls id", KR(ret), K(tenant_id)); - } - } - } - if (FAILEDx(gather_all_ls_info_())) { - LOG_WARN("failed to get all ls info", KR(ret), K(tenant_id)); - } else if (OB_FAIL(ObPrimaryZoneUtil::get_tenant_primary_zone_array( - *tenant_schema_, primary_zone_))) { - LOG_WARN("failed to get tenant primary zone array", KR(ret), KPC(tenant_schema_)); - } else { - is_load_ = true; - } - } - LOG_INFO("[LS_MGR] gather stat", KR(ret), K(primary_zone_), K(unit_group_array_), - K(max_ls_id_), K(max_ls_group_id_)); - return ret; -} - -int ObTenantLSInfo::gather_all_ls_info_() -{ - int ret = OB_SUCCESS; - share::ObLSStatusInfoArray status_info_array; - if (OB_ISNULL(sql_proxy_) || OB_ISNULL(tenant_schema_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy or tenant schema is null", KR(ret), KP(tenant_schema_), KP(sql_proxy_)); - } else { - const uint64_t tenant_id = tenant_schema_->get_tenant_id(); - if (OB_FAIL(status_operator_.get_all_ls_status_by_order( - tenant_id, status_info_array, *sql_proxy_))) { - LOG_WARN("failed to get all ls status by order", KR(ret), K(tenant_id)); - } - } - if (OB_FAIL(ret)) { - } else { - const int64_t count = max(1, hash::cal_next_prime(status_info_array.count())); - if (OB_FAIL(status_map_.create(count, "LogStrInfo", "LogStrInfo"))) { - LOG_WARN("failed to create ls map", KR(ret), K(count)); - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < status_info_array.count(); ++i) { - const share::ObLSStatusInfo &info = status_info_array.at(i); - if (info.ls_is_wait_offline()) { - //ls is already offline, no need to process - } else if (FAILEDx(add_ls_status_info_(info))) { - LOG_WARN("failed to add ls status info", KR(ret), K(i), K(info)); - } - }// end for - } - } - return ret; -} - -int ObTenantLSInfo::add_ls_to_ls_group_(const share::ObLSStatusInfo &info) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!info.is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls status is invalid", KR(ret), K(info)); - } else { - int64_t group_index = OB_INVALID_INDEX_INT64; - for (int64_t j = 0; OB_SUCC(ret) && j < ls_group_array_.count(); - ++j) { - const ObLSGroupInfo &group = ls_group_array_.at(j); - if (info.ls_group_id_ == group.ls_group_id_) { - group_index = j; - break; - } - } - if (OB_SUCC(ret) && OB_INVALID_INDEX_INT64 == group_index) { - ObLSGroupInfo group; - group_index = ls_group_array_.count(); - if (OB_FAIL(group.init(info.unit_group_id_, info.ls_group_id_))) { - LOG_WARN("failed to init ls group", KR(ret), K(info)); - } else if (OB_FAIL(ls_group_array_.push_back(group))) { - LOG_WARN("failed to pushback group", KR(ret), K(group)); - } else if (OB_FAIL(add_ls_group_to_unit_group_(group))) { - LOG_WARN("failed to add ls group to unit group", KR(ret), K(group)); - } - } - if (OB_SUCC(ret)) { - if (OB_UNLIKELY(group_index >= ls_group_array_.count() - || OB_INVALID_INDEX_INT64 == group_index)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("group index is invalid", KR(ret), K(group_index), "count", - ls_group_array_.count()); - } else if (OB_FAIL(ls_group_array_.at(group_index) - .ls_ids_.push_back(info.ls_id_))) { - LOG_WARN("failed to push back ls id", KR(ret), K(group_index), - K(info)); - } - } - } - return ret; -} - -int ObTenantLSInfo::add_ls_group_to_unit_group_(const ObLSGroupInfo &group_info) -{ - int ret = OB_SUCCESS; - bool found = false; - if (OB_UNLIKELY(!group_info.is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("group_info is invalid", KR(ret), K(group_info)); - } else { - found = false; - for (int64_t j = 0; OB_SUCC(ret) && j < unit_group_array_.count(); ++j) { - ObUnitGroupInfo &unit_info = unit_group_array_.at(j); - if (group_info.unit_group_id_ == unit_info.unit_group_id_) { - found = true; - if (OB_FAIL(unit_info.ls_group_ids_.push_back( - group_info.ls_group_id_))) { - LOG_WARN("failed to push back ls group id", KR(ret), - K(group_info), K(unit_info)); - } - } - } // end j - if (OB_SUCC(ret) && !found) { - //can found unit_group for a ls group - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to find unit group for ls group", KR(ret), - K(group_info), K(unit_group_array_)); - } - } - - return ret; -} - -int ObTenantLSInfo::check_unit_group_valid_( - const uint64_t unit_group_id, bool &is_valid_info) -{ - int ret = OB_SUCCESS; - is_valid_info = true; - if (OB_UNLIKELY(OB_INVALID_VERSION == unit_group_id)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("ls id is invalid", KR(ret), K(unit_group_id)); - } else if (OB_UNLIKELY(!is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant stat not valid", KR(ret)); - } else { - //check unit group status is valid - ret = OB_ENTRY_NOT_EXIST; - for (int64_t i = 0; OB_FAIL(ret) && i < unit_group_array_.count(); ++i) { - const ObUnitGroupInfo &group_info = unit_group_array_.at(i); - if (unit_group_id == group_info.unit_group_id_) { - ret = OB_SUCCESS; - if (share::ObUnit::UNIT_STATUS_ACTIVE != group_info.unit_status_) { - is_valid_info = false; - } - break; - } - }//end for - if (OB_FAIL(ret)) { - LOG_WARN("failed to found unit group status", KR(ret), K(unit_group_id), - K(unit_group_array_)); - } - } - return ret; -} - -int ObTenantLSInfo::create_new_ls_in_trans( - const share::ObLSID &ls_id, - const uint64_t ls_group_id, - const SCN &create_scn, - ObMySQLTransaction &trans) -{ - int ret = OB_SUCCESS; - const bool is_recovery = true; - int64_t info_index = OB_INVALID_INDEX_INT64; - if (OB_UNLIKELY(!ls_id.is_valid() - || OB_INVALID_ID == ls_group_id - || !create_scn.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("ls id is invalid", KR(ret), K(ls_id), K(ls_group_id), K(create_scn)); - } else if (OB_UNLIKELY(!is_valid()) - || OB_ISNULL(tenant_schema_) - || OB_ISNULL(sql_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant stat not valid", KR(ret), KP(tenant_schema_), KP(sql_proxy_)); - } else { - const uint64_t tenant_id = tenant_schema_->get_tenant_id(); - share::ObLSStatusInfo new_info; - ObLSGroupInfo group_info; - ObZone primary_zone; - ObLSLifeAgentManager ls_life_agent(*sql_proxy_); - ObSqlString zone_priority; - if (OB_SUCC(get_ls_group_info( - ls_group_id, group_info))) { - // need a new primary zone - } else if (OB_ENTRY_NOT_EXIST == ret) { - // need a new unit group - ret = OB_SUCCESS; - int64_t unit_group_index = OB_INVALID_INDEX_INT64; - if (OB_FAIL(get_next_unit_group_(is_recovery, unit_group_index))) { - LOG_WARN("failed to get next unit group", KR(ret), K(is_recovery)); - } else if (OB_UNLIKELY(OB_INVALID_INDEX_INT64 == unit_group_index - || unit_group_index >= unit_group_array_.count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to next unit group", KR(ret), K(unit_group_index)); - } else if (OB_FAIL(group_info.init( - unit_group_array_.at(unit_group_index).unit_group_id_, - ls_group_id))) { - LOG_WARN("failed to init group info", KR(ret), K(unit_group_index), K(ls_group_id)); - } - } else { - LOG_WARN("failed to get ls group info", KR(ret), - K(ls_group_id)); - } - - if (FAILEDx(get_next_primary_zone_(is_recovery, group_info, primary_zone))) { - LOG_WARN("failed to get next primary zone", KR(ret), K(group_info)); - } else if (OB_FAIL(new_info.init(tenant_id, ls_id, - ls_group_id, - share::OB_LS_CREATING, - group_info.unit_group_id_, primary_zone))) { - LOG_WARN("failed to init new info", KR(ret), K(tenant_id), - K(ls_id), K(ls_group_id), K(group_info), K(primary_zone)); - } else if (OB_FAIL(ObTenantThreadHelper::get_zone_priority(primary_zone, *tenant_schema_, zone_priority))) { - LOG_WARN("failed to get normalize primary zone", KR(ret), K(primary_zone), K(zone_priority)); - } else if (OB_FAIL(ls_life_agent.create_new_ls_in_trans(new_info, create_scn, - zone_priority.string(), share::NORMAL_SWITCHOVER_STATUS, trans))) { - LOG_WARN("failed to insert ls info", KR(ret), K(new_info), K(create_scn), K(zone_priority)); - } else if (OB_FAIL(add_ls_status_info_(new_info))) { - LOG_WARN("failed to add new ls info to memory", KR(ret), K(new_info)); - } - } - return ret; -} - -int ObTenantLSInfo::add_ls_status_info_( - const share::ObLSStatusInfo &ls_info) -{ - int ret = OB_SUCCESS; - const int64_t index = status_array_.count(); - if (OB_UNLIKELY(!ls_info.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("ls info is invalid", KR(ret), K(ls_info)); - } else if (OB_FAIL(status_array_.push_back(ls_info))) { - LOG_WARN("failed to remove status", KR(ret), K(ls_info)); - } else if (OB_FAIL(status_map_.set_refactored(ls_info.ls_id_, index))) { - LOG_WARN("failed to remove ls from map", KR(ret), K(ls_info), K(index)); - } else if (ls_info.ls_id_.is_sys_ls()) { - //sys ls no ls group - } else if (OB_FAIL(add_ls_to_ls_group_(ls_info))) { - LOG_WARN("failed to add ls info", KR(ret), K(ls_info)); - } - return ret; -} - -int ObTenantLSInfo::get_ls_group_info( - const uint64_t ls_group_id, ObLSGroupInfo &info) const -{ - int ret = OB_SUCCESS; - info.reset(); - if (OB_UNLIKELY(OB_INVALID_ID == ls_group_id)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("ls group id is invalid", KR(ret), K(ls_group_id)); - } else if (OB_UNLIKELY(!is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant stat not valid", KR(ret)); - } else { - bool found = false; - for (int64_t i = 0; OB_SUCC(ret) && i < ls_group_array_.count(); ++i) { - const ObLSGroupInfo group_info = ls_group_array_.at(i); - if (ls_group_id == group_info.ls_group_id_) { - found = true; - if (OB_FAIL(info.assign(group_info))) { - LOG_WARN("failed to assign group info", KR(ret), K(group_info)); - } - break; - } - }//end for - if (OB_SUCC(ret) && !found) { - ret = OB_ENTRY_NOT_EXIST; - LOG_WARN("failed to find ls group info", KR(ret), - K(ls_group_id)); - } - } - return ret; -} - -int ObTenantLSInfo::get_ls_status_info( - const share::ObLSID &ls_id, - share::ObLSStatusInfo &info, - int64_t &info_index) const -{ - int ret = OB_SUCCESS; - info.reset(); - info_index = OB_INVALID_INDEX_INT64; - if (OB_UNLIKELY(!ls_id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("ls id is invalid", KR(ret), K(ls_id)); - } else if (OB_UNLIKELY(!is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant stat not valid", KR(ret)); - } else if (OB_FAIL(status_map_.get_refactored(ls_id, info_index))) { - if (OB_HASH_NOT_EXIST == ret) { - ret = OB_ENTRY_NOT_EXIST; - } - LOG_WARN("failed to find ls index", KR(ret), K(ls_id)); - } else if (OB_UNLIKELY(info_index < 0 || info_index >= status_array_.count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("info index not valid", KR(ret), K(info_index), K(ls_id), - "status_count", status_array_.count()); - } else if (OB_FAIL(info.assign(status_array_.at(info_index)))) { - LOG_WARN("failed to assign ls info", KR(ret), K(info_index), - "status", status_array_.at(info_index)); - } - return ret; -} - -int ObTenantLSInfo::get_next_unit_group_(const bool is_recovery, int64_t &group_index) -{ - int ret = OB_SUCCESS; - group_index = OB_INVALID_INDEX_INT64; - if (OB_UNLIKELY(!is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant stat not valid", KR(ret)); - } else { - int64_t ls_count = OB_INVALID_COUNT; - // Find the unit group with the least number of log streams - for (int64_t i = 0; OB_SUCC(ret) && i < unit_group_array_.count(); ++i) { - const ObUnitGroupInfo &info = unit_group_array_.at(i); - const int64_t count = info.ls_group_ids_.count(); - if (share::ObUnit::UNIT_STATUS_ACTIVE == info.unit_status_) { - if (OB_INVALID_COUNT == ls_count || ls_count > count) { - ls_count = count; - group_index = i; - if (0 == count) { - //the first has no ls group unit group - break; - } - } - } - } - if (OB_SUCC(ret)) { - if (OB_INVALID_INDEX_INT64 == group_index - || (!is_recovery && 0 != ls_count)) { - ret = OB_ENTRY_NOT_EXIST; - LOG_WARN("failed to find next unit group", KR(ret), - K(unit_group_array_), K(is_recovery), K(ls_count)); - } - } - } - LOG_INFO("get next primary zone", KR(ret), K(group_index), K(is_recovery)); - return ret; -} - -int ObTenantLSInfo::get_next_primary_zone_( - const bool is_recovery, - const ObLSGroupInfo &group_info, - ObZone &primary_zone) -{ - int ret = OB_SUCCESS; - primary_zone.reset(); - if (OB_UNLIKELY(!group_info.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("group info is invalid", KR(ret), K(group_info)); - } else if (OB_UNLIKELY(!is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant ls info not valid", KR(ret)); - } else { - share::ObLSStatusInfo info; - int64_t ls_count = OB_INVALID_COUNT; - int64_t info_index = OB_INVALID_INDEX_INT64; - for (int64_t i = 0; OB_SUCC(ret) && i < primary_zone_.count(); ++i) { - const ObZone &zone = primary_zone_.at(i); - int64_t primary_zone_count = 0; - for (int64_t j = 0; OB_SUCC(ret) && j < group_info.ls_ids_.count(); ++j) { - const share::ObLSID &id = group_info.ls_ids_.at(j); - if (OB_FAIL(get_ls_status_info(id, info, info_index))) { - LOG_WARN("failed to find ls info", KR(ret), K(id), K(j)); - } else if (zone == info.primary_zone_) { - primary_zone_count++; - } - }//end for j - if (OB_SUCC(ret)) { - if (OB_INVALID_COUNT == ls_count || ls_count > primary_zone_count) { - ls_count = primary_zone_count; - if (OB_FAIL(primary_zone.assign(zone))) { - LOG_WARN("failed to assign zone", KR(ret), K(zone)); - } else if (0 == primary_zone_count) { - //the first zone has no primary zone - break; - } - } - } - }//end for i - if (OB_SUCC(ret)) { - if (primary_zone.is_empty() || (!is_recovery && 0 != ls_count)) { - ret = OB_ENTRY_NOT_EXIST; - LOG_WARN("failed to find next primary zone", KR(ret), K(primary_zone_), - K(group_info), K(is_recovery), K(ls_count)); - } - } - LOG_INFO("get next primary zone", KR(ret), K(group_info), K(primary_zone)); - } - return ret; -} - -//for create new ls -//ls_info:creating, status_info:creating, status_info:created, ls_info:normal, status_info:normal - -//for drop ls, not supported - -//for drop tenant -//ls_info:tenant_dropping, status_info:tenant_dropping, ls_info:empty, status_info:wait_offline -//the sys ls can be tenant_dropping in status_info after all_users ls were deleted in ls_info -int ObTenantLSInfo::process_next_ls_status_(const ObLSStatusMachineParameter &machine, bool &is_steady) -{ - int ret = OB_SUCCESS; - const share::ObLSStatusInfo &status_info = machine.status_info_; - const share::ObLSAttr &ls_info = machine.ls_info_; - const uint64_t tenant_id = status_info.tenant_id_; - is_steady = false; - if (OB_UNLIKELY(!machine.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("machine is invalid", KR(ret), K(machine)); - } else if (!ls_info.is_valid()) { - if (status_info.ls_is_wait_offline()) { - is_steady = true; - } else if (status_info.ls_is_create_abort() - || status_info.ls_is_creating() - || status_info.ls_is_created()) { - //in switchover/failover, need create abort ls - //in drop tenant, __all_ls will be deleted while status is creating - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("status info is invalid", KR(ret), K(machine)); - } - } else if (ls_info.ls_is_creating()) { - if (status_info.ls_is_create_abort()) { - //delete ls, the ls must is creating - if (OB_FAIL(ls_operator_.delete_ls( - machine.ls_id_, share::OB_LS_CREATING))) { - LOG_WARN("failed to process creating info", KR(ret), K(machine)); - } - } else if (status_info.ls_is_created()) { - //set ls to normal - if (OB_FAIL(ls_operator_.update_ls_status( - machine.ls_id_, ls_info.get_ls_status(), share::OB_LS_NORMAL))) { - LOG_WARN("failed to update ls status", KR(ret), K(machine)); - } - } else if (status_info.ls_is_creating()) { - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("status info is invalid", KR(ret), K(machine)); - } - } else if (ls_info.ls_is_normal()) { - if (status_info.ls_is_normal()) { - is_steady = true; - } else if (status_info.ls_is_created()) { - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("status info is invalid", KR(ret), K(machine)); - } - } else if (ls_info.ls_is_dropping()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("drop ls not supported", KR(ret), K(machine)); - } else if (share::ls_is_pre_tenant_dropping_status(ls_info.get_ls_status())) { - if (!machine.ls_id_.is_sys_ls()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("normal ls can not in pre tenant dropping status", KR(ret), K(machine)); - } else if (!status_info.ls_is_pre_tenant_dropping()) { - } else if (OB_FAIL(sys_ls_tenant_drop_(status_info))) { - LOG_WARN("failed to process sys ls", KR(ret), K(status_info)); - } - } else if (ls_info.ls_is_tenant_dropping()) { - if (ls_info.get_ls_id().is_sys_ls()) { - //nothing - } else if (!status_info.ls_is_tenant_dropping()) { - } else if (OB_FAIL(do_tenant_drop_ls_(status_info))) { - LOG_WARN("failed to drop ls", KR(ret), K(machine)); - } - } else { - //other status can not be in __all_ls - //such as created, wait_offline - ret = OB_ERR_UNEXPECTED; - LOG_WARN("the ls not expected in all_ls", KR(ret), K(machine)); - } - return ret; -} - -//need create or delete new ls group -int ObTenantLSInfo::check_ls_group_match_unitnum() -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("tenant stat not valid", KR(ret)); - } - for (int64_t i = 0; OB_SUCC(ret) && i < unit_group_array_.count(); ++i) { - const ObUnitGroupInfo &info = unit_group_array_.at(i); - if (share::ObUnit::UNIT_STATUS_ACTIVE != info.unit_status_) { - // drop logstream, not support - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unit status not valid", KR(ret), K(info)); - } else if (0 == info.ls_group_ids_.count()) { - //create ls - if (OB_FAIL(create_new_ls_for_empty_unit_group_(info.unit_group_id_))) { - LOG_WARN("failed to create new ls for empty unit_group", KR(ret), - K(info)); - } - } - } - return ret; -} - -int ObTenantLSInfo::create_new_ls_for_empty_unit_group_(const uint64_t unit_group_id) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(OB_INVALID_INDEX_INT64 == unit_group_id)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("group id is invalid", KR(ret), K(unit_group_id)); - } else if (OB_UNLIKELY(!is_valid()) - || OB_ISNULL(tenant_schema_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant ls info not valid", KR(ret), KP(tenant_schema_)); - } else if (OB_UNLIKELY(0 == primary_zone_.count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("primary zone is invalid", KR(ret)); - } else { - const uint64_t tenant_id = tenant_schema_->get_tenant_id(); - share::ObLSID new_id; - share::ObLSStatusInfo new_info; - uint64_t new_ls_group_id = OB_INVALID_INDEX_INT64; - if (OB_FAIL(fetch_new_ls_group_id(tenant_id, new_ls_group_id))) { - LOG_WARN("failed to fetch new id", KR(ret), K(tenant_id)); - } - for (int64_t i = 0; OB_SUCC(ret) && i < primary_zone_.count(); ++i) { - const ObZone &zone = primary_zone_.at(i); - new_info.reset(); - if (OB_FAIL(fetch_new_ls_id(tenant_id, new_id))) { - LOG_WARN("failed to get new id", KR(ret), K(tenant_id)); - } else if (OB_FAIL(new_info.init(tenant_id, new_id, new_ls_group_id, - share::OB_LS_CREATING, unit_group_id, - zone))) { - LOG_WARN("failed to init new info", KR(ret), K(new_id), - K(new_ls_group_id), K(unit_group_id), K(zone), K(tenant_id)); - } else if (OB_FAIL(create_new_ls_(new_info))) { - LOG_WARN("failed to add ls info", KR(ret), K(new_info)); - } - LOG_INFO("[LS_MGR] create new ls for empty unit group", KR(ret), K(new_info)); - }//end for - } - return ret; -} - -//need create or delete new ls -//TODO for primary cluster, The primary_zone of multiple LSs may be the same, -//the redundant ones need to be deleted, and only one is kept. -int ObTenantLSInfo::check_ls_match_primary_zone() -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!is_valid()) || OB_ISNULL(tenant_schema_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant ls info not valid", KR(ret), KP(tenant_schema_)); - } else { - const uint64_t tenant_id = tenant_schema_->get_tenant_id(); - ObZone zone; - share::ObLSID new_id; - share::ObLSStatusInfo new_info; - for (int64_t i = 0; OB_SUCC(ret) && i < ls_group_array_.count(); ++i) { - ObLSGroupInfo &group_info = ls_group_array_.at(i); - //check the unit group is active - //if unit group is in deleting, no need check primary zone - bool is_valid = false; - int64_t group_id_count = group_info.ls_ids_.count(); - if (OB_FAIL(check_unit_group_valid_(group_info.unit_group_id_, is_valid))) { - LOG_WARN("failed to check unit group valid", KR(ret), K(group_info)); - } - while (OB_SUCC(ret) && is_valid - && primary_zone_.count() > group_id_count) { - group_id_count++; - if (OB_FAIL(get_next_primary_zone_(false/*is_recovery*/, group_info, zone))) { - LOG_WARN("failed to get next primary zone", K(group_info)); - } else if (OB_FAIL(fetch_new_ls_id(tenant_id, new_id))) { - LOG_WARN("failed to get new id", KR(ret), K(tenant_id)); - } else if (OB_FAIL(new_info.init(tenant_id, new_id, group_info.ls_group_id_, - share::OB_LS_CREATING, - group_info.unit_group_id_, zone))) { - LOG_WARN("failed to init new info", KR(ret), K(new_id), - K(group_info), K(zone), K(tenant_id)); - } else if (OB_FAIL(create_new_ls_(new_info))) { - LOG_WARN("failed to create new ls", KR(ret), K(new_info)); - } - LOG_INFO("[LS_MGR] create new ls of primary zone", KR(ret), K(new_info), - K(group_id_count)); - } - } - } - return ret; -} - -int ObTenantLSInfo::create_new_ls_(const share::ObLSStatusInfo &status_info) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("tenant stat not valid", KR(ret)); - } else if (OB_UNLIKELY(!status_info.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("ls info is invalid", KR(ret), K(status_info)); - } else if (OB_ISNULL(sql_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy is null", KR(ret)); - } else { - share::ObLSAttr ls_info; - share::ObLSFlag flag = share::OB_LS_FLAG_NORMAL;//TODO - SCN create_scn; - if (OB_FAIL(ObLSAttrOperator::get_tenant_gts(status_info.tenant_id_, create_scn))) { - LOG_WARN("failed to get tenant gts", KR(ret), K(status_info)); - } else if (OB_FAIL(ls_info.init(status_info.ls_id_, status_info.ls_group_id_, flag, - share::OB_LS_CREATING, share::OB_LS_OP_CREATE_PRE, create_scn))) { - LOG_WARN("failed to init new operation", KR(ret), K(status_info), K(create_scn)); - } else if (OB_FAIL(ls_operator_.insert_ls(ls_info, max_ls_group_id_))) { - LOG_WARN("failed to insert new operation", KR(ret), K(ls_info), K(max_ls_group_id_)); - } - LOG_INFO("[LS_MGR] create new ls", KR(ret), K(status_info)); - } - return ret; -} - -int ObTenantLSInfo::fetch_new_ls_group_id( - const uint64_t tenant_id, - uint64_t &ls_group_id) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("tenant id is invalid", KR(ret), K(tenant_id)); - } else if (OB_ISNULL(sql_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy is null", KR(ret)); - } else { - ls_group_id = OB_INVALID_ID; - share::ObMaxIdFetcher id_fetcher(*sql_proxy_); - if (OB_FAIL(id_fetcher.fetch_new_max_id( - tenant_id, share::OB_MAX_USED_LS_GROUP_ID_TYPE, ls_group_id))) { - LOG_WARN("fetch_max_id failed", KR(ret), K(tenant_id), "id_type", - share::OB_MAX_USED_LS_GROUP_ID_TYPE); - } else { - if (ls_group_id != ++max_ls_group_id_) { - ret = OB_EAGAIN; - LOG_WARN("ls may create concurrency, need retry", KR(ret), K(ls_group_id), - K(max_ls_group_id_)); - } - } - } - return ret; -} - -int ObTenantLSInfo::fetch_new_ls_id(const uint64_t tenant_id, share::ObLSID &id) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("tenant id is invalid", KR(ret), K(tenant_id)); - } else if (OB_ISNULL(sql_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy is null", KR(ret)); - } else { - share::ObMaxIdFetcher id_fetcher(*sql_proxy_); - uint64_t id_value = OB_INVALID_ID; - if (OB_FAIL(id_fetcher.fetch_new_max_id( - tenant_id, share::OB_MAX_USED_LS_ID_TYPE, id_value))) { - LOG_WARN("fetch_max_id failed", KR(ret), K(tenant_id), "id_type", - share::OB_MAX_USED_LS_ID_TYPE); - } else { - if (id_value != ++max_ls_id_) { - ret = OB_EAGAIN; - LOG_WARN("ls may create concurrency, need retry", KR(ret), K(id_value), - K(max_ls_id_)); - } else { - share::ObLSID new_id(id_value); - id = new_id; - } - } - } - return ret; -} - -int ObTenantLSInfo::do_tenant_drop_ls_(const share::ObLSStatusInfo &status_info) -{ - int ret = OB_SUCCESS; - LOG_INFO("[LS_MGR] start to tenant drop ls", K(status_info)); - const int64_t start_time = ObTimeUtility::fast_current_time(); - bool can_offline = false; - if (OB_UNLIKELY(!status_info.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("info not valid", KR(ret), K(status_info)); - } else if (OB_ISNULL(sql_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy is null", KR(ret)); - } else if (!status_info.ls_is_tenant_dropping()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("status info not expected", KR(ret), K(status_info)); - } - if (FAILEDx(check_ls_can_offline_by_rpc(status_info, can_offline))) { - // send rpc to observer - LOG_WARN("failed to check ls can offline", KR(ret), K(status_info)); - } else if (can_offline) { - //sys ls already block, can not delete ls - const uint64_t tenant_id = status_info.tenant_id_; - if (OB_FAIL(ls_operator_.delete_ls(status_info.ls_id_, share::OB_LS_TENANT_DROPPING))) { - LOG_WARN("failed to delete ls", KR(ret), K(status_info)); - } - } - const int64_t cost = ObTimeUtility::fast_current_time() - start_time; - LOG_INFO("[LS_MGR] end to tenant drop ls", KR(ret), K(status_info), K(can_offline), K(cost)); - return ret; -} - -int ObTenantLSInfo::sys_ls_tenant_drop_(const share::ObLSStatusInfo &info) -{ - int ret = OB_SUCCESS; - const ObLSStatus target_status = share::OB_LS_TENANT_DROPPING; - const ObLSStatus pre_status = share::OB_LS_PRE_TENANT_DROPPING; - if (OB_UNLIKELY(!info.is_valid() - || !info.ls_id_.is_sys_ls())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(info)); - } else if (OB_ISNULL(sql_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy is null", KR(ret), KP(sql_proxy_)); - } else if (pre_status != info.status_) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sys ls can not in other status", KR(ret), K(info)); - } - bool can_offline = false; - if (FAILEDx(check_sys_ls_can_offline_(can_offline))) { - LOG_WARN("failed to check sys ls can offline", KR(ret)); - } else if (can_offline) { - if (OB_FAIL(ls_operator_.update_ls_status(info.ls_id_, pre_status, target_status))) { - LOG_WARN("failed to update ls status", KR(ret), K(info), K(pre_status), K(target_status)); - } - } - return ret; -} - -int ObTenantLSInfo::check_sys_ls_can_offline_(bool &can_offline) -{ - int ret = OB_SUCCESS; - share::ObLSStatusInfoArray status_info_array; - can_offline = true; - if (OB_ISNULL(sql_proxy_) || OB_ISNULL(tenant_schema_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy or tenant schema is null", KR(ret), KP(tenant_schema_), KP(sql_proxy_)); - } else { - const uint64_t tenant_id = tenant_schema_->get_tenant_id(); - if (OB_FAIL(status_operator_.get_all_ls_status_by_order( - tenant_id, status_info_array, *sql_proxy_))) { - LOG_WARN("failed to get all ls status", KR(ret), K(tenant_id)); - } else if (0 == status_info_array.count()) { - //sys ls not exist - can_offline = true; - } - for (int64_t i = 0; OB_SUCC(ret) && i < status_info_array.count() && can_offline; ++i) { - const share::ObLSStatusInfo &status_info = status_info_array.at(i); - if (status_info.ls_id_.is_sys_ls()) { - } else { - can_offline = false; - LOG_INFO("[LS_MGR] sys ls can not offline", K(status_info)); - } - } - if (OB_SUCC(ret) && can_offline) { - LOG_INFO("[LS_MGR] sys ls can offline", K(status_info_array)); - } - } - return ret; -} - -int ObTenantLSInfo::check_ls_can_offline_by_rpc(const share::ObLSStatusInfo &info, bool &can_offline) -{ - int ret = OB_SUCCESS; - share::ObLSInfo ls_info; - const share::ObLSReplica *replica = NULL; - if (OB_UNLIKELY(!info.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("info not valid", KR(ret), K(info)); - } else if (OB_ISNULL(lst_operator_) || OB_ISNULL(rpc_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("lst operator or proxy is null", KR(ret), KP(lst_operator_), - KP(rpc_proxy_)); - } else if (OB_FAIL(lst_operator_->get(GCONF.cluster_id, info.tenant_id_, - info.ls_id_, share::ObLSTable::DEFAULT_MODE, ls_info))) { - LOG_WARN("failed to get ls info", KR(ret), K(info)); - } else if (OB_FAIL(ls_info.find_leader(replica))) { - LOG_WARN("failed to find leader", KR(ret), K(ls_info)); - } else if (OB_ISNULL(replica)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("replica is null", KR(ret), K(ls_info)); - } else { - const int64_t timeout = GCONF.rpc_timeout; - obrpc::ObCheckLSCanOfflineArg arg; - if (OB_FAIL(arg.init(info.tenant_id_, info.ls_id_, info.status_))) { - LOG_WARN("failed to init arg", KR(ret), K(arg)); - } else if (OB_FAIL(rpc_proxy_->to(replica->get_server()) - .by(info.tenant_id_) - .timeout(timeout) - .check_ls_can_offline(arg))) { - can_offline = false; - LOG_WARN("failed to check ls can offline", KR(ret), K(arg), K(info), - K(timeout), K(replica)); - } else { - can_offline = true; - } - } - return ret; -} - -////////////ObRecoveryLSHelper -int ObRecoveryLSHelper::do_work(palf::PalfBufferIterator &iterator, - share::SCN &start_scn) -{ - int ret = OB_SUCCESS; - ObTenantInfoLoader *tenant_loader = MTL(ObTenantInfoLoader*); - ObAllTenantInfo tenant_info; - // two thread for seed log and recovery_ls_manager - if (!is_user_tenant(tenant_id_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls recovery thread must run on user tenant", KR(ret), - K(tenant_id_)); - } else if (OB_ISNULL(tenant_loader)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant report is null", KR(ret), K(tenant_id_)); - } else if (OB_FAIL(tenant_loader->get_tenant_info(tenant_info))) { - LOG_WARN("failed to get tenant info", KR(ret)); - } else if (OB_FAIL(ObTenantThreadHelper::check_can_do_recovery(tenant_info))) { - LOG_WARN("failed to check do recovery", KR(ret), K(tenant_info)); - } else if (!start_scn.is_valid()) { - ObAllTenantInfo new_tenant_info; - if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(tenant_id_, proxy_, - false, new_tenant_info))) { - LOG_WARN("failed to load tenant info", KR(ret), K(tenant_id_)); - } else if (new_tenant_info.get_recovery_until_scn() == new_tenant_info.get_sys_recovery_scn()) { - ret = OB_EAGAIN; - if (REACH_TIME_INTERVAL(60 * 1000 * 1000)) { // every minute - LOG_INFO("has recovered to recovery_until_scn", KR(ret), - K(new_tenant_info)); - } - } else if (OB_FAIL( - seek_log_iterator_(new_tenant_info.get_sys_recovery_scn(), iterator))) { - LOG_WARN("failed to seek log iterator", KR(ret), K(new_tenant_info)); - } else { - start_scn = new_tenant_info.get_sys_recovery_scn(); - LOG_INFO("start to seek at", K(start_scn)); - } - } - if (OB_FAIL(ret)) { - } else if (start_scn.is_valid() && - OB_FAIL(process_ls_log_(tenant_info, start_scn, iterator))) { - if (OB_ITER_STOP != ret) { - LOG_WARN("failed to process ls log", KR(ret), K(start_scn), - K(tenant_info)); - } - } - if (OB_FAIL(ret)) { - start_scn.reset(); - } - return ret; -} - -int ObRecoveryLSHelper::seek_log_iterator_(const SCN &sys_recovery_scn, PalfBufferIterator &iterator) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!sys_recovery_scn.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("sync scn is invalid", KR(ret), K(sys_recovery_scn)); - } else { - ObLSService *ls_svr = MTL(ObLSService *); - ObLSHandle ls_handle; - if (OB_FAIL(ls_svr->get_ls(SYS_LS, ls_handle, storage::ObLSGetMod::RS_MOD))) { - LOG_WARN("failed to get ls", KR(ret)); - } else { - logservice::ObLogHandler *log_handler = NULL; - ObLS *ls = NULL; - palf::LSN start_lsn; - if (OB_ISNULL(ls = ls_handle.get_ls()) - || OB_ISNULL(log_handler = ls->get_log_handler())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls or log handle is null", KR(ret), KP(ls), KP(log_handler)); - } else if (sys_recovery_scn.is_base_scn()) { - //The recovery_scn on the primary tenant is initialized to base_scn, - //but too small scn may cause locate to fail, - //so when base_scn is encountered, iterates the log directly from the initialized LSN. - start_lsn = palf::LSN(palf::PALF_INITIAL_LSN_VAL); - } else if (OB_FAIL(log_handler->locate_by_scn_coarsely(sys_recovery_scn, start_lsn))) { - LOG_WARN("failed to locate lsn", KR(ret), K(sys_recovery_scn)); - } - - if (FAILEDx(log_handler->seek(start_lsn, iterator))) { - LOG_WARN("failed to seek iterator", KR(ret), K(sys_recovery_scn)); - } - } - } - return ret; -} - -int ObRecoveryLSHelper::process_ls_log_( - const ObAllTenantInfo &tenant_info, - SCN &start_scn, - PalfBufferIterator &iterator) -{ - int ret = OB_SUCCESS; - palf::LogEntry log_entry; - palf::LSN target_lsn; - SCN sys_recovery_scn; - SCN last_sys_recovery_scn; - int tmp_ret = OB_SUCCESS; - if (OB_UNLIKELY(!tenant_info.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("tenant_info is invalid", KR(ret), K(tenant_info)); - } - while (OB_SUCC(ret) && OB_SUCC(iterator.next())) { - if (OB_FAIL(iterator.get_entry(log_entry, target_lsn))) { - LOG_WARN("failed to get log", KR(ret), K(log_entry)); - } else if (OB_ISNULL(log_entry.get_data_buf())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("log entry is null", KR(ret)); - } else { - sys_recovery_scn = log_entry.get_scn(); - LOG_TRACE("get log", K(log_entry), K(target_lsn), K(start_scn), K(sys_recovery_scn)); - const char *log_buf = log_entry.get_data_buf(); - const int64_t log_length = log_entry.get_data_len(); - logservice::ObLogBaseHeader header; - const int64_t HEADER_SIZE = header.get_serialize_size(); - int64_t log_pos = 0; - if (OB_UNLIKELY(sys_recovery_scn <= start_scn)) { - //通过scn定位的LSN是不准确的，可能会获取多余的数据，所以需要把小于等于sys_recovery_scn的过滤掉 - continue; - } else if (tenant_info.get_recovery_until_scn() < sys_recovery_scn) { - if (OB_FAIL(report_tenant_sys_recovery_scn_(tenant_info.get_recovery_until_scn(), false, proxy_))) { - LOG_WARN("failed to report_tenant_sys_recovery_scn_", KR(ret), K(sys_recovery_scn), K(tenant_info), - K(log_entry), K(target_lsn), K(start_scn)); - // SYS LS has recovered to the recovery_until_scn, need stop iterate SYS LS log and reset start_scn - } else if (OB_FAIL(seek_log_iterator_(tenant_info.get_recovery_until_scn(), iterator))) { - LOG_WARN("failed to seek log iterator", KR(ret), K(sys_recovery_scn), K(tenant_info), - K(log_entry), K(target_lsn), K(start_scn)); - } else { - ret = OB_ITER_STOP; - LOG_WARN("SYS LS has recovered to the recovery_until_scn, need stop iterate SYS LS log", - KR(ret), K(sys_recovery_scn), K(tenant_info), K(log_entry), K(target_lsn), K(start_scn)); - start_scn.reset(); - } - } else if (OB_FAIL(header.deserialize(log_buf, HEADER_SIZE, log_pos))) { - LOG_WARN("failed to deserialize", KR(ret), K(HEADER_SIZE)); - } else if (OB_UNLIKELY(log_pos >= log_length)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("log pos is not expected", KR(ret), K(log_pos), K(log_length)); - // nothing - } else if (logservice::TRANS_SERVICE_LOG_BASE_TYPE == header.get_log_type()) { - ObTxLogBlock tx_log_block; - ObTxLogBlockHeader tx_block_header; - if (OB_FAIL(tx_log_block.init(log_buf, log_length, log_pos, tx_block_header))) { - LOG_WARN("failed to init tx log block", KR(ret), K(log_length)); - } else if (OB_FAIL(process_ls_tx_log_(tx_log_block, sys_recovery_scn))) { - LOG_WARN("failed to process ls tx log", KR(ret), K(tx_log_block), K(sys_recovery_scn)); - } - } else {} - if (OB_SUCC(ret)) { - last_sys_recovery_scn = sys_recovery_scn; - } - if ((OB_FAIL(ret) && last_sys_recovery_scn.is_valid()) - || (OB_SUCC(ret) && REACH_TIME_INTERVAL(100 * 1000))) { - //if ls_operator can not process, need to report last sync scn - //Forced reporting every second - if (OB_TMP_FAIL(report_tenant_sys_recovery_scn_(last_sys_recovery_scn, false, proxy_))) { - ret = OB_SUCC(ret) ? tmp_ret : ret; - LOG_WARN("failed to report ls recovery stat", KR(ret), KR(tmp_ret), K(last_sys_recovery_scn)); - } - } - } - if (REACH_TIME_INTERVAL(10 * 1000 * 1000)) { - if (OB_TMP_FAIL(try_to_process_sys_ls_offline_())) { - LOG_WARN("failed to process sys ls offline", KR(ret), KR(tmp_ret)); - } - } - }//end for each log - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - if (!sys_recovery_scn.is_valid()) { - } else if (OB_FAIL(report_tenant_sys_recovery_scn_(sys_recovery_scn, false, proxy_))) { - LOG_WARN("failed to report ls recovery stat", KR(ret), K(sys_recovery_scn)); - } - } else if (OB_SUCC(ret)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("iterator must be end", KR(ret)); - } else { - LOG_WARN("failed to get next log", KR(ret)); - } - - return ret; -} - -int ObRecoveryLSHelper::process_ls_tx_log_(ObTxLogBlock &tx_log_block, const SCN &sys_recovery_scn) -{ - int ret = OB_SUCCESS; - bool has_operation = false; - while (OB_SUCC(ret)) { - transaction::ObTxLogHeader tx_header; - if (OB_FAIL(tx_log_block.get_next_log(tx_header))) { - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - break; - } else { - LOG_WARN("failed to get next log", KR(ret)); - } - } else if (transaction::ObTxLogType::TX_COMMIT_LOG != - tx_header.get_tx_log_type()) { - // nothing - } else { - ObTxCommitLogTempRef temp_ref; - ObTxCommitLog commit_log(temp_ref); - const int64_t COMMIT_SIZE = commit_log.get_serialize_size(); - //TODO commit log may too large - if (OB_FAIL(tx_log_block.deserialize_log_body(commit_log))) { - LOG_WARN("failed to deserialize", KR(ret)); - } else { - const ObTxBufferNodeArray &source_data = - commit_log.get_multi_source_data(); - for (int64_t i = 0; OB_SUCC(ret) && i < source_data.count(); ++i) { - const ObTxBufferNode &node = source_data.at(i); - if (ObTxDataSourceType::STANDBY_UPGRADE == node.get_data_source_type()) { - if (OB_FAIL(process_upgrade_log_(node))) { - LOG_WARN("failed to process_upgrade_log_", KR(ret), K(node)); - } - } else if (ObTxDataSourceType::LS_TABLE != node.get_data_source_type()) { - // nothing - } else if (has_operation) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("one clog has more than one operation", KR(ret), K(commit_log)); - } else { - has_operation = true; - ObLSAttr ls_attr; - const int64_t LS_SIZE = ls_attr.get_serialize_size(); - int64_t pos = 0; - if (OB_FAIL(ls_attr.deserialize(node.get_data_buf().ptr(), LS_SIZE, - pos))) { - LOG_WARN("failed to deserialize", KR(ret), K(node), K(LS_SIZE)); - } else if (OB_UNLIKELY(pos > LS_SIZE)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to get ls attr", KR(ret), K(pos), K(LS_SIZE)); - } else { - LOG_INFO("get ls operation", K(ls_attr), K(sys_recovery_scn)); - //TODO ls recovery is too fast for ls manager, so it maybe failed, while change ls status - //consider how to retry - if (OB_FAIL(process_ls_operator_(ls_attr, - sys_recovery_scn))) { - LOG_WARN("failed to process ls operator", KR(ret), K(ls_attr), - K(sys_recovery_scn)); - } - } - } - }// end for - } - } - } // end while for each tx_log - - return ret; -} - -int ObRecoveryLSHelper::process_upgrade_log_(const ObTxBufferNode &node) -{ - int ret = OB_SUCCESS; - uint64_t standby_data_version = 0; - - if (!node.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("node is invalid", KR(ret), K(node)); - } else { - ObStandbyUpgrade primary_data_version; - int64_t pos = 0; - if (OB_FAIL(primary_data_version.deserialize(node.get_data_buf().ptr(), node.get_data_buf().length(), pos))) { - LOG_WARN("failed to deserialize", KR(ret), K(node), KPHEX(node.get_data_buf().ptr(), node.get_data_buf().length())); - } else if (OB_UNLIKELY(pos > node.get_data_buf().length())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to get primary_data_version", KR(ret), K(pos), K(node.get_data_buf().length())); - } else { - LOG_INFO("get primary_data_version", K(primary_data_version)); - if (!primary_data_version.is_valid()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("primary_data_version not valid", KR(ret), K(primary_data_version)); - } else if (OB_FAIL(ObUpgradeChecker::get_data_version_by_cluster_version( - GET_MIN_CLUSTER_VERSION(), - standby_data_version))) { - LOG_WARN("failed to get_data_version_by_cluster_version", KR(ret), K(GET_MIN_CLUSTER_VERSION())); - } else if (primary_data_version.get_data_version() > standby_data_version) { - ret = OB_EAGAIN; - if (REACH_TIME_INTERVAL(30 * 60 * 1000 * 1000)) { // 30min - LOG_ERROR("standby version is not new enough to recover primary clog", KR(ret), - K(primary_data_version), K(standby_data_version)); - } - } - } - } - return ret; -} - -int ObRecoveryLSHelper::process_ls_operator_(const share::ObLSAttr &ls_attr, const SCN &sys_recovery_scn) -{ - int ret = OB_SUCCESS; - common::ObMySQLTransaction trans; - const uint64_t meta_tenant_id = gen_meta_tenant_id(tenant_id_); - if (OB_UNLIKELY(!ls_attr.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("ls attr is invalid", KR(ret), K(ls_attr)); - } else if (OB_ISNULL(proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("proxy is null", KR(ret)); - } else if (OB_FAIL(check_valid_to_operator_ls_(ls_attr, sys_recovery_scn))) { - LOG_WARN("failed to check valid to operator ls", KR(ret), K(sys_recovery_scn), K(ls_attr)); - } else if (OB_FAIL(trans.start(proxy_, meta_tenant_id))) { - LOG_WARN("failed to start trans", KR(ret), K(meta_tenant_id)); - } else if (OB_FAIL(process_ls_operator_in_trans_(ls_attr, sys_recovery_scn, trans))) { - LOG_WARN("failed to process ls operator in trans", KR(ret), K(ls_attr), K(sys_recovery_scn)); - } else if (OB_FAIL(report_tenant_sys_recovery_scn_trans_(sys_recovery_scn, true, trans))) { - LOG_WARN("failed to report tenant sys recovery scn", KR(ret), K(sys_recovery_scn)); - } - - if (trans.is_started()) { - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) { - ret = OB_SUCC(ret) ? tmp_ret : ret; - LOG_WARN("failed to end trans", KR(ret), K(tmp_ret)); - } - } - return ret; -} - -int ObRecoveryLSHelper::check_valid_to_operator_ls_( - const share::ObLSAttr &ls_attr, const SCN &sys_recovery_scn) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("proxy is null", KR(ret)); - } else if (OB_UNLIKELY(!sys_recovery_scn.is_valid() || !ls_attr.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("syns scn is invalid", KR(ret), K(sys_recovery_scn), K(ls_attr)); - } else if (share::is_ls_tenant_drop_pre_op(ls_attr.get_ls_operation_type())) { - ObTenantSchema tenant_schema; - if (OB_FAIL(ObTenantThreadHelper::get_tenant_schema(tenant_id_, tenant_schema))) { - LOG_WARN("failed to get tenant schema", KR(ret), K(tenant_id_)); - } else if (!tenant_schema.is_dropping()) { - ret = OB_ITER_STOP; - LOG_WARN("can not process ls operator before tenant dropping", K(ls_attr), K(tenant_schema)); - } - } - if (OB_FAIL(ret)) { - } else { - SCN min_sys_recovery_scn; - SCN min_wrs; - ObLSRecoveryStatOperator ls_recovery; - if (OB_FAIL( - ls_recovery.get_tenant_recovery_stat(tenant_id_, *proxy_, min_sys_recovery_scn, min_wrs))) { - LOG_WARN("failed to get user scn", KR(ret), K(tenant_id_)); - } else if (min_sys_recovery_scn >= sys_recovery_scn) { - // other ls has larger sync scn, ls can operator - } else { - ret = OB_NEED_WAIT; - LOG_WARN("can not process ls operator, need wait other ls sync", KR(ret), - K(min_sys_recovery_scn), K(sys_recovery_scn)); - } - } - return ret; -} - -int ObRecoveryLSHelper::process_ls_operator_in_trans_( - const share::ObLSAttr &ls_attr, const SCN &sys_recovery_scn, ObMySQLTransaction &trans) -{ - int ret = OB_SUCCESS; - const uint64_t meta_tenant_id = gen_meta_tenant_id(tenant_id_); - if (OB_UNLIKELY(!ls_attr.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("ls attr is invalid", KR(ret), K(ls_attr)); - } else if (OB_ISNULL(proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("proxy is null", KR(ret)); - } else { - ObLSStatusOperator ls_operator; - share::ObLSStatusInfo ls_status; - ObLSLifeAgentManager ls_life_agent(*proxy_); - if (OB_UNLIKELY(!ls_attr.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("ls attr is invalid", KR(ret), K(ls_attr)); - } else if (share::is_ls_create_pre_op(ls_attr.get_ls_operation_type())) { - //create new ls; - if (OB_FAIL(create_new_ls_(ls_attr, sys_recovery_scn, trans))) { - LOG_WARN("failed to create new ls", KR(ret), K(sys_recovery_scn), K(ls_attr)); - } - } else if (share::is_ls_create_abort_op(ls_attr.get_ls_operation_type())) { - if (OB_FAIL(ls_life_agent.drop_ls_in_trans(tenant_id_, ls_attr.get_ls_id(), share::NORMAL_SWITCHOVER_STATUS, trans))) { - LOG_WARN("failed to drop ls", KR(ret), K(tenant_id_), K(ls_attr)); - } - } else if (share::is_ls_drop_end_op(ls_attr.get_ls_operation_type())) { - if (OB_FAIL(ls_life_agent.set_ls_offline_in_trans(tenant_id_, ls_attr.get_ls_id(), - ls_attr.get_ls_status(), sys_recovery_scn, share::NORMAL_SWITCHOVER_STATUS, trans))) { - LOG_WARN("failed to set offline", KR(ret), K(tenant_id_), K(ls_attr), K(sys_recovery_scn)); - } - } else { - ObLSStatus target_status = share::OB_LS_EMPTY; - if (OB_FAIL(ls_operator.get_ls_status_info(tenant_id_, ls_attr.get_ls_id(), - ls_status, trans))) { - LOG_WARN("failed to get ls status", KR(ret), K(tenant_id_), K(ls_attr)); - } else if (ls_status.ls_is_creating()) { - ret = OB_EAGAIN; - LOG_WARN("ls not created, need wait", KR(ret), K(ls_status)); - } else if (share::is_ls_create_end_op(ls_attr.get_ls_operation_type())) { - // set ls to normal - target_status = share::OB_LS_NORMAL; - } else if (share::is_ls_tenant_drop_op(ls_attr.get_ls_operation_type())) { - target_status = share::OB_LS_TENANT_DROPPING; - } else if (share::is_ls_drop_pre_op(ls_attr.get_ls_operation_type())) { - target_status = share::OB_LS_DROPPING; - } else if (share::is_ls_tenant_drop_pre_op(ls_attr.get_ls_operation_type())) { - target_status = share::OB_LS_PRE_TENANT_DROPPING; - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected operation type", KR(ret), K(ls_attr)); - } - if (FAILEDx(ls_operator.update_ls_status(tenant_id_, ls_attr.get_ls_id(), - ls_status.status_, target_status, - share::NORMAL_SWITCHOVER_STATUS, trans))) { - LOG_WARN("failed to update ls status", KR(ret), K(tenant_id_), K(ls_attr), - K(ls_status), K(target_status)); - } - LOG_INFO("[LS_RECOVERY] update ls status", KR(ret), K(ls_attr), K(target_status)); - } - } - - return ret; -} -int ObRecoveryLSHelper::create_new_ls_(const share::ObLSAttr &ls_attr, - const SCN &sys_recovery_scn, - common::ObMySQLTransaction &trans) -{ - int ret = OB_SUCCESS; - if (!share::is_ls_create_pre_op(ls_attr.get_ls_operation_type())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("ls not create pre operation", KR(ret), K(ls_attr)); - } else { - //create new ls; - share::schema::ObTenantSchema tenant_schema; - if (OB_FAIL(ObTenantThreadHelper::get_tenant_schema(tenant_id_, tenant_schema))) { - LOG_WARN("failed to get tenant schema", KR(ret), K(tenant_id_)); - } else { - ObTenantLSInfo tenant_stat(GCTX.sql_proxy_, &tenant_schema, tenant_id_, - GCTX.srv_rpc_proxy_, GCTX.lst_operator_); - if (OB_FAIL(tenant_stat.gather_stat(true))) { - LOG_WARN("failed to gather stat", KR(ret)); - } else if (OB_FAIL(tenant_stat.create_new_ls_in_trans(ls_attr.get_ls_id(), - ls_attr.get_ls_group_id(), ls_attr.get_create_scn(), trans))) { - LOG_WARN("failed to add new ls status info", KR(ret), K(ls_attr), K(sys_recovery_scn)); - } - } - LOG_INFO("[LS_RECOVERY] create new ls", KR(ret), K(ls_attr)); - } - return ret; -} - -int ObRecoveryLSHelper::report_tenant_sys_recovery_scn_(const SCN &sys_recovery_scn, - const bool update_sys_recovery_scn, ObISQLClient *proxy) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(proxy)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql can't null", K(ret), KP(proxy)); - } else if (SCN::base_scn() != sys_recovery_scn) { - const uint64_t exec_tenant_id = gen_meta_tenant_id(tenant_id_); - START_TRANSACTION(proxy, exec_tenant_id) - if (FAILEDx(report_tenant_sys_recovery_scn_trans_(sys_recovery_scn, update_sys_recovery_scn, trans))) { - LOG_WARN("failed to report tenant sys recovery scn in trans", KR(ret), K(sys_recovery_scn)); - } - END_TRANSACTION(trans) - } - return ret; -} - -int ObRecoveryLSHelper::report_tenant_sys_recovery_scn_trans_(const share::SCN &sys_recovery_scn, - const bool update_sys_recovery_scn, common::ObMySQLTransaction &trans) -{ - int ret = OB_SUCCESS; - ObTenantInfoLoader *tenant_report = MTL(ObTenantInfoLoader*); - ObAllTenantInfo tenant_info; - if (OB_ISNULL(tenant_report)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant report is null", KR(ret), K(tenant_id_)); - } else if (OB_FAIL(tenant_report->get_tenant_info(tenant_info))) { - LOG_WARN("failed to get tenant info", KR(ret)); - } else if (OB_FAIL(ObAllTenantInfoProxy::update_tenant_sys_recovery_scn_in_trans( - tenant_id_, sys_recovery_scn, update_sys_recovery_scn, trans))) { - LOG_WARN("failed to update tenant sys recovery scn", KR(ret), K(tenant_id_), - K(sys_recovery_scn), K(tenant_info), K(update_sys_recovery_scn)); - } else { - //double check sync can not fallback - //log will be truncated during the flashback. - //If a certain iteration report crosses the flashback, - //recovery scn too large may be reported. Therefore, - //it is verified that the reported bit is not rolled back within the transaction. - palf::PalfHandleGuard palf_handle_guard; - logservice::ObLogService *ls_svr = MTL(logservice::ObLogService*); - SCN new_scn; - if (OB_FAIL(ls_svr->open_palf(SYS_LS, palf_handle_guard))) { - LOG_WARN("failed to open palf", KR(ret), K(ls_id)); - } else if (OB_FAIL(palf_handle_guard.get_end_scn(new_scn))) { - LOG_WARN("failed to get end ts", KR(ret), K(ls_id)); - } else if (new_scn < sys_recovery_scn) { - ret = OB_OP_NOT_ALLOW; - LOG_WARN("maybe flashback, can not report sync scn", KR(ret), K(sys_recovery_scn), K(new_scn)); - } - } - return ret; -} - -int ObRecoveryLSHelper::try_to_process_sys_ls_offline_() -{ - int ret = OB_SUCCESS; - ObTenantSchema tenant_schema; - if (OB_FAIL(ObTenantThreadHelper::get_tenant_schema(tenant_id_, tenant_schema))) { - LOG_WARN("failed to get tenant schema", KR(ret), K(tenant_id_)); - } else if (!tenant_schema.is_dropping()) { - //not dropping, nothing - } else if (OB_ISNULL(proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("proxy is null", KR(ret)); - } else { - //check sys ls is tenant dropping, and check can offline - share::ObLSStatusInfo ls_status; - share::ObLSStatusOperator ls_op; - if (OB_FAIL(ls_op.get_ls_status_info(tenant_id_, SYS_LS, ls_status, *proxy_))) { - LOG_WARN("failed to get ls status", KR(ret), K(tenant_id_)); - } else if (!ls_status.ls_is_tenant_dropping()) { - //nothing todo - } else { - bool can_offline = false; - ObTenantLSInfo tenant_stat(GCTX.sql_proxy_, &tenant_schema, tenant_id_, - GCTX.srv_rpc_proxy_, GCTX.lst_operator_); - if (OB_FAIL(tenant_stat.check_ls_can_offline_by_rpc(ls_status, can_offline))) { - LOG_WARN("failed to check ls can offline", KR(ret), K(ls_status)); - } else if (!can_offline) { - //nothing - } else { - ObLSLifeAgentManager ls_life_agent(*proxy_); - if (OB_FAIL(ls_life_agent.set_ls_offline(tenant_id_, SYS_LS, ls_status.status_, - SCN::base_scn(), share::NORMAL_SWITCHOVER_STATUS))) { - LOG_WARN("failed to set ls offline", KR(ret), K(ls_status)); - } - } - } - } - return ret; -} - -} -} diff --git a/src/rootserver/ob_ls_service_helper.h b/src/rootserver/ob_ls_service_helper.h deleted file mode 100644 index b8e2b32017..0000000000 --- a/src/rootserver/ob_ls_service_helper.h +++ /dev/null @@ -1,279 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ - -#ifndef OCEANBASE_ROOTSERVER_OB_LS_SERVICE_HELPER_H -#define OCEANBASE_ROOTSERVER_OB_LS_SERVICE_HELPER_H -#include "lib/thread/ob_reentrant_thread.h"//ObRsReentrantThread -#include "logservice/ob_log_base_type.h" -#include "share/scn.h"//SCN -#include "share/ls/ob_ls_status_operator.h"//ObLSStatusOperator -#include "share/ls/ob_ls_operator.h" //ObLSAttr -#include "share/ob_thread_mgr.h" //OBTGDefIDEnum -#include "logservice/palf/palf_iterator.h" //PalfBufferIterator -#include "share/unit/ob_unit_info.h"//ObUnit::Status -#include "lib/thread/thread_mgr_interface.h" // TGRunnable -#include "lib/lock/ob_thread_cond.h"//ObThreadCond - - -namespace oceanbase -{ -namespace obrpc -{ -class ObSrvRpcProxy; -} -namespace common -{ -class ObMySQLProxy; -class ObISQLClient; -class ObMySQLTransaction; -class ObClusterVersion; -} -namespace share -{ -class ObLSTableOperator; -class SCN; -namespace schema -{ -class ObMultiVersionSchemaService; -class ObTenantSchema; -} -} -namespace logservice -{ -class ObLogHandler; -} -namespace transaction -{ -class ObTxLogBlock; -class ObTxBufferNode; -} -namespace palf -{ -struct PalfBaseInfo; -} -namespace rootserver -{ -struct ObUnitGroupInfo -{ - ObUnitGroupInfo() : unit_group_id_(OB_INVALID_ID), unit_status_(share::ObUnit::UNIT_STATUS_MAX), - ls_group_ids_() {} - virtual ~ObUnitGroupInfo() {} - bool is_valid() const; - int init(const uint64_t unit_group_id, - const share::ObUnit::Status &unit_status); - void reset(); - int assign(const ObUnitGroupInfo &other); - int remove_ls_group(const uint64_t ls_group_id); - bool operator==(const ObUnitGroupInfo &other) const; - - uint64_t unit_group_id_; - share::ObUnit::Status unit_status_; - ObArray ls_group_ids_; - TO_STRING_KV(K_(unit_group_id), K_(unit_status), K_(ls_group_ids)); -}; -typedef ObArray ObUnitGroupInfoArray; -typedef ObIArray ObUnitGroupInfoIArray; - -struct ObLSGroupInfo -{ - ObLSGroupInfo() : ls_group_id_(OB_INVALID_ID), unit_group_id_(OB_INVALID_ID), - ls_ids_() {} - virtual ~ObLSGroupInfo() {} - bool is_valid() const; - int init(const uint64_t unit_group_id, const uint64_t ls_group_id); - int assign(const ObLSGroupInfo &other); - void reset(); - int remove_ls(const share::ObLSID &ls_id); - uint64_t ls_group_id_; - uint64_t unit_group_id_; - ObArray ls_ids_; - TO_STRING_KV(K_(ls_group_id), K_(unit_group_id), K_(ls_ids)); -}; - -typedef ObArray ObLSGroupInfoArray; -typedef ObIArray ObLSGroupInfoIArray; - -struct ObLSStatusMachineParameter -{ - ObLSStatusMachineParameter() : ls_id_(), status_info_(), ls_info_() {} - virtual ~ObLSStatusMachineParameter() {} - bool is_valid() const - { - return ls_id_.is_valid() - && (share::OB_LS_EMPTY == status_info_.status_ - || status_info_.ls_id_ == ls_id_); - } - int init(const share::ObLSID &id, const share::ObLSStatusInfo &status_info, - const share::ObLSAttr &ls_info); - void reset(); - share::ObLSID ls_id_; - share::ObLSStatusInfo status_info_;//for create ls and status of __all_ls_status - share::ObLSAttr ls_info_; - TO_STRING_KV(K_(ls_id), K_(status_info), K_(ls_info)); -}; - -/*descripthin: Tenant log stream status information: Statistical log stream - * status on __all_ls_status and __all_ls, tenant primary_zone and unit_num - * information. Provides location information for the newly created log stream. - * Whether the build needs to create or delete log streams.*/ -class ObTenantLSInfo -{ -public: - ObTenantLSInfo(ObMySQLProxy *sql_proxy, - const share::schema::ObTenantSchema *tenant_schema, - const uint64_t tenant_id, obrpc::ObSrvRpcProxy *rpc_proxy, - share::ObLSTableOperator *lst_operator) - : sql_proxy_(sql_proxy), - tenant_schema_(tenant_schema), - status_operator_(), - ls_operator_(tenant_id, sql_proxy), - status_array_(), - unit_group_array_(), - ls_group_array_(), - primary_zone_(), - rpc_proxy_(rpc_proxy), - lst_operator_(lst_operator), - max_ls_id_(OB_INVALID_ID), - max_ls_group_id_(OB_INVALID_ID) {} - - virtual ~ObTenantLSInfo(){}; - void reset(); - bool is_valid() const; - - int gather_stat(bool for_recovery); - int process_next_ls_status(int64_t &task_cnt); - - // need create or delete new ls group - int check_ls_group_match_unitnum(); - - //need create or delete new ls - int check_ls_match_primary_zone(); - - int fetch_new_ls_group_id(const uint64_t tenant_id, uint64_t &ls_group_id); - int fetch_new_ls_id(const uint64_t tenant_id, share::ObLSID &ls_id); - - //get ls group info from ls_group_array_ by ls_group_id - //the interface must used after gather_stat(); - int get_ls_group_info(const uint64_t ls_group_id, ObLSGroupInfo &info) const; - - //get ls status info from status_array_ by ls_id - //the interface must used after gather_stat(); - int get_ls_status_info(const share::ObLSID &id, share::ObLSStatusInfo &info, - int64_t &info_index) const; - //process dropping tenant, set status to tenant_dropping in __all_ls - int drop_tenant(); - //for recovery tenant, create new ls according to ls_id and ls_group_id - int create_new_ls_in_trans(const share::ObLSID &ls_id, - const uint64_t ls_group_id, - const share::SCN &create_scn, - common::ObMySQLTransaction &trans); - int construct_ls_status_machine_( - common::ObIArray &status_machine_array); - //only for upgrade, before 4.1, make __all_ls_status equal to __all_ls - int revision_to_equal_status(common::ObMySQLTransaction &trans); - int check_ls_can_offline_by_rpc(const share::ObLSStatusInfo &info, - bool &can_offline); -private: - // get from __all_ls_status and __all_ls - int gather_all_ls_info_(); - - // base on status_array construct ls_array - int add_ls_to_ls_group_(const share::ObLSStatusInfo &info); - - // base on ls_array construct unit_group_array and __all_unit - int add_ls_group_to_unit_group_(const ObLSGroupInfo &group_info); - - int check_unit_group_valid_(const uint64_t unit_group_id, bool &is_valid); - int add_ls_status_info_(const share::ObLSStatusInfo &ls_info); - int create_new_ls_for_empty_unit_group_(const uint64_t unit_group_id); - - int get_next_unit_group_(const bool is_recovery, int64_t &group_index); - // get the primary zone not in ls group - int get_next_primary_zone_(const bool is_recovery, const ObLSGroupInfo &group_info, - ObZone &primary_zone); - int create_new_ls_(const share::ObLSStatusInfo &ls_info); - - // drop ls group info - int do_tenant_drop_ls_(const share::ObLSStatusInfo &ls_info); - int sys_ls_tenant_drop_(const share::ObLSStatusInfo &info); - int check_sys_ls_can_offline_(bool &can_offline); - - -private: - //modify __all_ls - int process_next_ls_status_(const ObLSStatusMachineParameter &status_machine, bool &is_steady); - -private: - ObMySQLProxy *sql_proxy_; - const share::schema::ObTenantSchema *tenant_schema_; - share::ObLSStatusOperator status_operator_; - share::ObLSAttrOperator ls_operator_; - bool is_load_; - share::ObLSStatusInfoArray status_array_; - common::hash::ObHashMap status_map_; - ObUnitGroupInfoArray unit_group_array_; - ObLSGroupInfoArray ls_group_array_; - ObArray primary_zone_; - obrpc::ObSrvRpcProxy *rpc_proxy_; - share::ObLSTableOperator *lst_operator_; - //ensure ls can not be concurrent created - uint64_t max_ls_id_; - uint64_t max_ls_group_id_; -}; - -/*description: - *only active on the leader of the system log stream under the user tenant*/ -class ObRecoveryLSHelper -{ -public: - ObRecoveryLSHelper(const uint64_t tenant_id, - common::ObMySQLProxy *sql_proxy) : - tenant_id_(tenant_id), proxy_(sql_proxy) {} - virtual ~ObRecoveryLSHelper() {} - int do_work(palf::PalfBufferIterator &iterator, - share::SCN &start_scn); -private: - //get log iterator by start_scn - int seek_log_iterator_(const share::SCN &syn_scn, - palf::PalfBufferIterator &iterator); - int process_ls_log_(const ObAllTenantInfo &tenant_info, - share::SCN &start_scn, - palf::PalfBufferIterator &iterator); - int process_upgrade_log_(const transaction::ObTxBufferNode &node); - int process_ls_tx_log_(transaction::ObTxLogBlock &tx_log, - const share::SCN &syn_scn); - int process_ls_operator_(const share::ObLSAttr &ls_attr, - const share::SCN &syn_scn); - int create_new_ls_(const share::ObLSAttr &ls_attr, - const share::SCN &syn_scn, - common::ObMySQLTransaction &trans); - //wait other ls is larger than sycn ts - int check_valid_to_operator_ls_(const share::ObLSAttr &ls_attr, - const share::SCN &syn_scn); - //readable scn need report - int report_tenant_sys_recovery_scn_(const share::SCN &sys_recovery_scn, const bool update_sys_recovery_scn, - ObISQLClient *proxy); - int report_tenant_sys_recovery_scn_trans_(const share::SCN &sys_recovery_scn, const bool update_sys_recovery_scn, - common::ObMySQLTransaction &trans); - int process_ls_operator_in_trans_(const share::ObLSAttr &ls_attr, - const share::SCN &sys_recovery_scn, common::ObMySQLTransaction &trans); - int try_to_process_sys_ls_offline_(); -private: - uint64_t tenant_id_; - common::ObMySQLProxy *proxy_; -}; - -} -} - - -#endif /* !OCEANBASE_ROOTSERVER_OB_LS_SERVICE_HELPER_H */ diff --git a/src/rootserver/ob_primary_ls_service.cpp b/src/rootserver/ob_primary_ls_service.cpp index 90c6956094..a77f8fdc39 100755 --- a/src/rootserver/ob_primary_ls_service.cpp +++ b/src/rootserver/ob_primary_ls_service.cpp @@ -14,14 +14,23 @@ #include "ob_primary_ls_service.h" #include "lib/profile/ob_trace_id.h" #include "share/ob_errno.h" +#include "share/ob_max_id_fetcher.h" +#include "share/schema/ob_schema_struct.h"//ObTenantInfo +#include "share/schema/ob_schema_service.h"//ObMultiSchemaService #include "share/ls/ob_ls_creator.h" //ObLSCreator #include "share/ls/ob_ls_life_manager.h"//ObLSLifeAgentManager +#include "share/ls/ob_ls_table_operator.h"//ObLSTableOpertor +#include "share/ob_primary_zone_util.h"//ObPrimaryZoneUtil +#include "share/ob_unit_table_operator.h"//ObUnitTableOperator +#include "share/ob_zone_table_operation.h" //ObZoneTableOperation #include "share/ob_share_util.h"//ObShareUtil #include "observer/ob_server_struct.h"//GCTX +#include "rootserver/ob_tenant_recovery_reportor.h"//update_ls_recovery +#include "rootserver/ob_tenant_role_transition_service.h" +#include "storage/tx_storage/ob_ls_map.h" #include "storage/tx_storage/ob_ls_service.h" #include "storage/tx_storage/ob_ls_handle.h" //ObLSHandle #include "logservice/palf/palf_base_info.h"//PalfBaseInfo -#include "rootserver/ob_ls_service_helper.h"//ObTenantLSInfo namespace oceanbase { @@ -31,6 +40,2022 @@ using namespace transaction; using namespace palf; namespace rootserver { +/////////ObUnitGroupInfo +bool ObUnitGroupInfo::is_valid() const +{ + return OB_INVALID_ID != unit_group_id_ + && share::ObUnit::UNIT_STATUS_MAX != unit_status_; +} + +int ObUnitGroupInfo::init(const uint64_t unit_group_id, + const share::ObUnit::Status &unit_status) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(OB_INVALID_ID == unit_group_id + || share::ObUnit::UNIT_STATUS_MAX == unit_status)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(unit_group_id), K(unit_status)); + } else { + unit_group_id_ = unit_group_id; + unit_status_ = unit_status; + } + return ret; +} + +void ObUnitGroupInfo::reset() +{ + unit_group_id_ = OB_INVALID_ID; + unit_status_ = share::ObUnit::UNIT_STATUS_MAX; + ls_group_ids_.reset(); +} + +int ObUnitGroupInfo::assign(const ObUnitGroupInfo &other) +{ + int ret = OB_SUCCESS; + if (this != &other) { + if (OB_FAIL(ls_group_ids_.assign(other.ls_group_ids_))) { + LOG_WARN("failed to assign", KR(ret), K(other)); + } else { + unit_group_id_ = other.unit_group_id_; + unit_status_ = other.unit_status_; + } + } + return ret; +} + +int ObUnitGroupInfo::remove_ls_group(const uint64_t ls_group_id) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(ObLSID::MIN_USER_LS_GROUP_ID >= ls_group_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("ls group id is invalid", KR(ret), K(ls_group_id)); + } else { + bool remove = false; + for (int64_t i = 0; OB_SUCC(ret) && i < ls_group_ids_.count(); ++i) { + if (ls_group_ids_.at(i) == ls_group_id) { + remove = true; + if (OB_FAIL(ls_group_ids_.remove(i))) { + LOG_WARN("failed to remove from array", KR(ret), K(i), + K(ls_group_id), "this", *this); + } + break; + } + } + if (OB_SUCC(ret) && !remove) { + ret = OB_ENTRY_NOT_EXIST; + LOG_WARN("failed to find ls group id", KR(ret), + K(ls_group_id), "this", *this); + } + } + return ret; +} + +bool ObUnitGroupInfo::operator==(const ObUnitGroupInfo &other) const +{ + return unit_group_id_ == other.unit_group_id_ + && unit_status_ == other.unit_status_; +} + +////////////ObLSGroupInfo +bool ObLSGroupInfo::is_valid() const +{ + return OB_INVALID_ID != unit_group_id_ + && OB_INVALID_ID != ls_group_id_; +} + +int ObLSGroupInfo::init(const uint64_t unit_group_id, + const uint64_t ls_group_id) +{ + int ret = OB_SUCCESS; + reset(); + if (OB_UNLIKELY(OB_INVALID_ID == unit_group_id + || OB_INVALID_ID == ls_group_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(unit_group_id), K(ls_group_id)); + } else { + unit_group_id_ = unit_group_id; + ls_group_id_ = ls_group_id; + } + return ret; +} + +int ObLSGroupInfo::assign(const ObLSGroupInfo &other) +{ + int ret = OB_SUCCESS; + if (this != &other) { + reset(); + if (OB_FAIL(ls_ids_.assign(other.ls_ids_))) { + LOG_WARN("failed to assign ls ids", KR(ret), K(other)); + } else { + unit_group_id_ = other.unit_group_id_; + ls_group_id_ = other.ls_group_id_; + } + } + return ret; +} + +void ObLSGroupInfo::reset() +{ + ls_group_id_ = OB_INVALID_ID; + unit_group_id_ = OB_INVALID_ID; + ls_ids_.reset(); +} + +int ObLSGroupInfo::remove_ls(const share::ObLSID &ls_id) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!ls_id.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("ls id is invalid", KR(ret), K(ls_id)); + } else { + bool remove = false; + for (int64_t i = 0; OB_SUCC(ret) && i < ls_ids_.count(); ++i) { + if (ls_ids_.at(i) == ls_id) { + remove = true; + if (OB_FAIL(ls_ids_.remove(i))) { + LOG_WARN("failed to remove from array", KR(ret), K(i), + K(ls_id), "this", *this); + } + break; + } + } + if (OB_SUCC(ret) && !remove) { + ret = OB_ENTRY_NOT_EXIST; + LOG_WARN("failed to find ls id", KR(ret), + K(ls_id), "this", *this); + } + } + return ret; + +} + +///////////////ObLSStatusMachineParameter +//no need check status valid or ls_status is valid +int ObLSStatusMachineParameter::init(const share::ObLSID &id, + const share::ObLSStatusInfo &status_info, + const share::ObLSStatus &ls_info) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!id.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("id is invalid", KR(ret), K(id)); + } else if (OB_FAIL(status_info_.assign(status_info))) { + LOG_WARN("failed to assign status info", KR(ret), K(status_info)); + } else { + ls_info_ = ls_info; + ls_id_ = id; + } + return ret; +} + +void ObLSStatusMachineParameter::reset() +{ + ls_id_.reset(); + ls_info_ = share::OB_LS_EMPTY; + status_info_.reset(); +} + +/////////////ObTenantLSInfo + +void ObTenantLSInfo::reset() +{ + is_load_ = false; + status_array_.reset(); + unit_group_array_.reset(); + ls_group_array_.reset(); + primary_zone_.reset(); + max_ls_id_ = OB_INVALID_ID; + max_ls_group_id_ = OB_INVALID_ID; +} + + +bool ObTenantLSInfo::is_valid() const +{ + return ATOMIC_LOAD(&is_load_); +} + +//Regardless of the tenant being dropped, +//handle the asynchronous operation of status and history table +int ObTenantLSInfo::process_ls_status_missmatch( + const bool lock_sys_ls, + const share::ObTenantSwitchoverStatus &working_sw_status) +{ + int ret = OB_SUCCESS; + share::ObLSStatusInfoArray status_info_array; + share::ObLSAttrArray ls_array; + ObArray status_machine_array; + if (OB_ISNULL(sql_proxy_) || OB_ISNULL(tenant_schema_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy or tenant schema is null", KR(ret), KP(tenant_schema_), KP(sql_proxy_)); + } else if (OB_FAIL(ls_operator_.get_all_ls_by_order(lock_sys_ls, ls_array))) { + LOG_WARN("failed to insert ls operation", KR(ret)); + } else { + const uint64_t tenant_id = tenant_schema_->get_tenant_id(); + if (OB_FAIL(status_operator_.get_all_ls_status_by_order( + tenant_id, status_info_array, *sql_proxy_))) { + LOG_WARN("failed to update ls status", KR(ret)); + } + } + if (FAILEDx(construct_ls_status_machine_(status_info_array, ls_array, + status_machine_array))) { + LOG_WARN("failed to construct ls status machine array", KR(ret), + K(status_info_array), K(ls_array)); + } else { + int tmp_ret = OB_SUCCESS; + ARRAY_FOREACH_NORET(status_machine_array, idx) { + const ObLSStatusMachineParameter &machine = status_machine_array.at(idx); + //ignore error of each ls + //may ls can not create success + if (OB_SUCCESS != (tmp_ret = fix_ls_status_(machine, working_sw_status))) { + LOG_WARN("failed to fix ls status", KR(ret), KR(tmp_ret), K(machine)); + ret = OB_SUCC(ret) ? tmp_ret : ret; + } + } + } + + return ret; +} + +int ObTenantLSInfo::construct_ls_status_machine_( + const share::ObLSStatusInfoIArray &status_info_array, + const share::ObLSAttrIArray &ls_array, + common::ObIArray &status_machine_array) +{ + int ret = OB_SUCCESS; + status_machine_array.reset(); + if (OB_UNLIKELY(0 == status_info_array.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls array can not be empty", KR(ret), K(ls_array), K(status_info_array)); + } else { + // merge by sort + int64_t status_index = 0; + int64_t ls_index = 0; + ObLSStatusMachineParameter status_machine; + const int64_t status_count = status_info_array.count(); + const int64_t ls_count = ls_array.count(); + share::ObLSStatusInfo status_info; + ObLSID ls_id; + ObLSStatus ls_info_status = share::OB_LS_EMPTY; + while ((status_index < status_count || ls_index < ls_count) && OB_SUCC(ret)) { + status_machine.reset(); + if (status_index == status_count) { + //status already end + ls_id = ls_array.at(ls_index).get_ls_id(); + ls_info_status = ls_array.at(ls_index).get_ls_status(); + status_info.reset(); + ls_index++; + } else if (ls_index == ls_count) { + //ls already end + if (OB_FAIL(status_info.assign(status_info_array.at(status_index)))) { + LOG_WARN("failed to assign status info", KR(ret), K(status_index)); + } else { + status_index++; + ls_id = status_info.ls_id_; + ls_info_status = share::OB_LS_EMPTY; + } + } else { + const share::ObLSStatusInfo &tmp_status_info = status_info_array.at(status_index); + const share::ObLSAttr &tmp_ls_info = ls_array.at(ls_index); + if (tmp_status_info.ls_id_ == tmp_ls_info.get_ls_id()) { + status_index++; + ls_index++; + ls_info_status = tmp_ls_info.get_ls_status(); + ls_id = tmp_status_info.ls_id_; + if (OB_FAIL(status_info.assign(tmp_status_info))) { + LOG_WARN("failed to assign status info", KR(ret), K(tmp_status_info)); + } + } else if (tmp_status_info.ls_id_ > tmp_ls_info.get_ls_id()) { + ls_index++; + ls_id = tmp_ls_info.get_ls_id(); + ls_info_status = tmp_ls_info.get_ls_status(); + status_info.reset(); + } else { + status_index++; + ls_id = tmp_status_info.ls_id_; + ls_info_status = share::OB_LS_EMPTY; + if (OB_FAIL(status_info.assign(tmp_status_info))) { + LOG_WARN("failed to assign status info", KR(ret), K(tmp_status_info)); + } + } + } + if (FAILEDx(status_machine.init(ls_id, status_info, ls_info_status))) { + LOG_WARN("failed to push back status", KR(ret), K(ls_id), K(status_info), K(ls_info_status)); + } else if (OB_FAIL(status_machine_array.push_back(status_machine))) { + LOG_WARN("failed to push back status machine", KR(ret), K(status_machine)); + } + } // end while + + } + return ret; +} + +//for create new ls +//ls_info:creating, status_info:creating, status_info:created, ls_info:normal, status_info:normal + +//for drop ls +//ls_info:dropping, status_info:dropping, ls_info:empty, status_info:wait_offline + +//for drop tenant +//ls_info:tenant_dropping, status_info:tenant_dropping, ls_info:empty, status_info:wait_offline +//the sys ls can be tenant_dropping in status_info after all_users ls were deleted in ls_info +int ObTenantLSInfo::fix_ls_status_(const ObLSStatusMachineParameter &machine, + const ObTenantSwitchoverStatus &working_sw_status) +{ + int ret = OB_SUCCESS; + const share::ObLSStatusInfo &status_info = machine.status_info_; + const uint64_t tenant_id = status_info.tenant_id_; + ObLSLifeAgentManager ls_life_agent(*sql_proxy_); + if (OB_UNLIKELY(!machine.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("status machine is valid", KR(ret), K(machine)); + } else if (OB_ISNULL(sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy is null", KR(ret), KP(sql_proxy_)); + } else if (ls_is_empty_status(machine.ls_info_)) { + //the ls not in ls and in status + if (status_info.ls_is_wait_offline()) { + //already offline, status will be deleted by GC + } else if (status_info.ls_is_dropping() + || status_info.ls_is_tenant_dropping()) { + SCN drop_scn; + if (OB_FAIL(ObLSAttrOperator::get_tenant_gts(tenant_id, drop_scn))) { + LOG_WARN("failed to get gts", KR(ret), K(tenant_id)); + } else if (OB_FAIL(ls_life_agent.set_ls_offline(tenant_id, + status_info.ls_id_, status_info.status_, + drop_scn, working_sw_status))) { + LOG_WARN("failed to update ls status", KR(ret), K(status_info), K(tenant_id), K(drop_scn)); + } + } else if (status_info.ls_is_creating() + || status_info.ls_is_created() + || status_info.ls_is_create_abort()) { + //ls may create_abort + if (OB_FAIL(ls_life_agent.drop_ls(tenant_id, machine.ls_id_, working_sw_status))) { + LOG_WARN("failed to delete ls", KR(ret), K(machine)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("status machine not expected", KR(ret), K(machine)); + } + } else if (ls_is_creating_status(machine.ls_info_)) { + //the ls may is in creating + if (!status_info.is_valid()) { + // TODO fix later + //in primary cluster, can create continue, but no need + if (OB_FAIL(ls_operator_.delete_ls(machine.ls_id_, machine.ls_info_, working_sw_status))) { + LOG_WARN("failed to delete ls", KR(ret), K(machine)); + } + } else if (status_info.ls_is_creating() && working_sw_status.is_normal_status()) { + //TODO check the primary zone or unit group id is valid + ObLSRecoveryStat recovery_stat; + ObLSRecoveryStatOperator ls_recovery_operator; + if (OB_FAIL(ls_recovery_operator.get_ls_recovery_stat(tenant_id, status_info.ls_id_, + false/*for_update*/, recovery_stat, *sql_proxy_))) { + LOG_WARN("failed to get ls recovery stat", KR(ret), K(tenant_id), K(status_info)); + } else if (OB_FAIL(do_create_ls_(status_info, recovery_stat.get_create_scn()))) { + LOG_WARN("failed to create new ls", KR(ret), K(status_info), + K(recovery_stat)); + } + } else if (status_info.ls_is_created()) { + if (OB_FAIL(process_ls_status_after_created_(status_info, working_sw_status))) { + LOG_WARN("failed to create ls", KR(ret), K(status_info)); + } + } else if (status_info.ls_is_create_abort()) { + //drop ls + if (OB_FAIL(ls_operator_.delete_ls(status_info.ls_id_, machine.ls_info_, working_sw_status))) { + LOG_WARN("failed to remove ls not normal", KR(ret), K(machine)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("status machine not expected", KR(ret), K(machine)); + } + } else if (ls_is_normal_status(machine.ls_info_)) { + //the ls is normal + if (status_info.ls_is_normal()) { + //nothing + } else if (status_info.ls_is_created()) { + if (OB_FAIL(status_operator_.update_ls_status( + tenant_id, status_info.ls_id_, status_info.status_, + share::OB_LS_NORMAL, working_sw_status, *sql_proxy_))) { + LOG_WARN("failed to update ls status", KR(ret), K(status_info), K(tenant_id)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("status machine not expected", KR(ret), K(machine)); + } + } else if (ls_is_dropping_status(machine.ls_info_)) { + if (FAILEDx(do_drop_ls_(status_info, working_sw_status))) { + LOG_WARN("failed to drop ls", KR(ret), K(status_info)); + } + } else if (ls_is_pre_tenant_dropping_status(machine.ls_info_)) { + if (!machine.ls_id_.is_sys_ls()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("normal ls can not in pre tenant dropping status", KR(ret), K(machine)); + } else if (OB_FAIL(sys_ls_tenant_drop_(status_info, working_sw_status))) { + LOG_WARN("failed to process sys ls", KR(ret), K(status_info)); + } + } else if (ls_is_tenant_dropping_status(machine.ls_info_)) { + if (status_info.ls_is_wait_offline()) { + //sys ls will tenant_dropping and wait offline + } else if (OB_FAIL(do_tenant_drop_ls_(status_info, working_sw_status))) { + LOG_WARN("failed to drop ls", KR(ret), K(machine)); + } + } else { + //other status can not be in __all_ls + //such as created, wait_offline + ret = OB_ERR_UNEXPECTED; + LOG_WARN("the ls not expected in all_ls", KR(ret), K(machine)); + } + return ret; +} + +int ObTenantLSInfo::drop_tenant(const ObTenantSwitchoverStatus &working_sw_status) +{ + int ret = OB_SUCCESS; + share::ObLSAttrArray ls_array; + if (OB_FAIL(ls_operator_.get_all_ls_by_order(ls_array))) { + LOG_WARN("failed to get all ls", KR(ret)); + } else { + //1. set sys_ls to tenant_dropping in __all_ls + //2. set user_ls to tenant_dropping or abort in __all_ls + for (int64_t i = 0; OB_SUCC(ret) && i < ls_array.count(); ++i) { + share::ObLSAttr &attr = ls_array.at(i); + if (attr.get_ls_id().is_sys_ls()) { + if (attr.ls_is_normal()) { + if (OB_FAIL(ls_operator_.update_ls_status(attr.get_ls_id(), + attr.get_ls_status(), share::OB_LS_PRE_TENANT_DROPPING, working_sw_status))) { + LOG_WARN("failed to update ls status", KR(ret), K(attr)); + } + } + } else if (attr.ls_is_creating()) { + //drop the status + if (OB_FAIL(ls_operator_.delete_ls(attr.get_ls_id(), attr.get_ls_status(), working_sw_status))) { + LOG_WARN("failed to remove ls not normal", KR(ret), K(attr)); + } + } else { + //no matter the status is in normal or dropping + //may be the status in status info is created + if (!attr.ls_is_tenant_dropping()) { + if (OB_FAIL(ls_operator_.update_ls_status( + attr.get_ls_id(), attr.get_ls_status(), + share::OB_LS_TENANT_DROPPING, working_sw_status))) { + LOG_WARN("failed to update ls status", KR(ret), K(attr)); + } else { + LOG_INFO("[LS_MGR] set ls to tenant dropping", KR(ret), K(attr)); + } + } + } + }//end for + } + return ret; +} +int ObTenantLSInfo::gather_stat(bool for_recovery) +{ + int ret = OB_SUCCESS; + reset(); + if (OB_ISNULL(tenant_schema_) || OB_ISNULL(sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected", KR(ret), KP(tenant_schema_), KP(sql_proxy_)); + } else { + ObUnitTableOperator unit_operator; + common::ObArray units; + const uint64_t tenant_id = tenant_schema_->get_tenant_id(); + if (OB_FAIL(unit_operator.init(*sql_proxy_))) { + LOG_WARN("failed to init unit operator", KR(ret)); + } else if (OB_FAIL(unit_operator.get_units_by_tenant( + tenant_id, units))) { + LOG_WARN("failed to get all tenant unit", KR(ret), K(tenant_id)); + } else { + ObUnitGroupInfo info; + for (int64_t j = 0; OB_SUCC(ret) && j < units.count(); ++j) { + info.reset(); + const ObUnit &unit = units.at(j); + if (OB_FAIL(info.init(unit.unit_group_id_, unit.status_))) { + LOG_WARN("failed to init unit info", KR(ret), K(unit)); + } else if (has_exist_in_array(unit_group_array_, info)) { + //nothing + } else if (OB_FAIL(unit_group_array_.push_back(info))) { + LOG_WARN("fail to push back", KR(ret), K(info)); + } + } + } + if (OB_SUCC(ret)) { + if (for_recovery) { + } else { + //get max ls id and max ls group id + share::ObMaxIdFetcher id_fetcher(*sql_proxy_); + if (OB_FAIL(id_fetcher.fetch_max_id(*sql_proxy_, tenant_id, + share::OB_MAX_USED_LS_GROUP_ID_TYPE, max_ls_group_id_))) { + LOG_WARN("failed to fetch max ls group id", KR(ret), K(tenant_id)); + } else if (OB_FAIL(id_fetcher.fetch_max_id(*sql_proxy_, tenant_id, + share::OB_MAX_USED_LS_ID_TYPE, max_ls_id_))) { + LOG_WARN("failed to fetch max ls id", KR(ret), K(tenant_id)); + } + } + } + if (FAILEDx(gather_all_ls_info_())) { + LOG_WARN("failed to get all ls info", KR(ret), K(tenant_id)); + } else if (OB_FAIL(ObPrimaryZoneUtil::get_tenant_primary_zone_array( + *tenant_schema_, primary_zone_))) { + LOG_WARN("failed to get tenant primary zone array", KR(ret), KPC(tenant_schema_)); + } else { + is_load_ = true; + } + } + LOG_INFO("[LS_MGR] gather stat", KR(ret), K(primary_zone_), K(unit_group_array_), + K(max_ls_id_), K(max_ls_group_id_)); + return ret; +} + +int ObTenantLSInfo::gather_all_ls_info_() +{ + int ret = OB_SUCCESS; + share::ObLSStatusInfoArray status_info_array; + if (OB_ISNULL(sql_proxy_) || OB_ISNULL(tenant_schema_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy or tenant schema is null", KR(ret), KP(tenant_schema_), KP(sql_proxy_)); + } else { + const uint64_t tenant_id = tenant_schema_->get_tenant_id(); + if (OB_FAIL(status_operator_.get_all_ls_status_by_order( + tenant_id, status_info_array, *sql_proxy_))) { + LOG_WARN("failed to update ls status", KR(ret), K(tenant_id)); + } + } + if (OB_FAIL(ret)) { + } else { + const int64_t count = max(1, hash::cal_next_prime(status_info_array.count())); + if (OB_FAIL(status_map_.create(count, "LogStrInfo", "LogStrInfo"))) { + LOG_WARN("failed to create ls map", KR(ret), K(count)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < status_info_array.count(); ++i) { + const share::ObLSStatusInfo &info = status_info_array.at(i); + if (info.ls_is_wait_offline()) { + //ls is already offline, no need to process + } else if (FAILEDx(add_ls_status_info_(info))) { + LOG_WARN("failed to add ls status info", KR(ret), K(i), K(info)); + } + }// end for + } + } + return ret; +} + +int ObTenantLSInfo::add_ls_to_ls_group_(const share::ObLSStatusInfo &info) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!info.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls status is invalid", KR(ret), K(info)); + } else { + int64_t group_index = OB_INVALID_INDEX_INT64; + for (int64_t j = 0; OB_SUCC(ret) && j < ls_group_array_.count(); + ++j) { + const ObLSGroupInfo &group = ls_group_array_.at(j); + if (info.ls_group_id_ == group.ls_group_id_) { + group_index = j; + break; + } + } + if (OB_SUCC(ret) && OB_INVALID_INDEX_INT64 == group_index) { + ObLSGroupInfo group; + group_index = ls_group_array_.count(); + if (OB_FAIL(group.init(info.unit_group_id_, info.ls_group_id_))) { + LOG_WARN("failed to init ls group", KR(ret), K(info)); + } else if (OB_FAIL(ls_group_array_.push_back(group))) { + LOG_WARN("failed to pushback group", KR(ret), K(group)); + } else if (OB_FAIL(add_ls_group_to_unit_group_(group))) { + LOG_WARN("failed to add ls group to unit group", KR(ret), K(group)); + } + } + if (OB_SUCC(ret)) { + if (OB_UNLIKELY(group_index >= ls_group_array_.count() + || OB_INVALID_INDEX_INT64 == group_index)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("group index is invalid", KR(ret), K(group_index), "count", + ls_group_array_.count()); + } else if (OB_FAIL(ls_group_array_.at(group_index) + .ls_ids_.push_back(info.ls_id_))) { + LOG_WARN("failed to push back ls id", KR(ret), K(group_index), + K(info)); + } + } + } + return ret; +} + +int ObTenantLSInfo::add_ls_group_to_unit_group_(const ObLSGroupInfo &group_info) +{ + int ret = OB_SUCCESS; + bool found = false; + if (OB_UNLIKELY(!group_info.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("group_info is invalid", KR(ret), K(group_info)); + } else { + found = false; + for (int64_t j = 0; OB_SUCC(ret) && j < unit_group_array_.count(); ++j) { + ObUnitGroupInfo &unit_info = unit_group_array_.at(j); + if (group_info.unit_group_id_ == unit_info.unit_group_id_) { + found = true; + if (OB_FAIL(unit_info.ls_group_ids_.push_back( + group_info.ls_group_id_))) { + LOG_WARN("failed to push back ls group id", KR(ret), + K(group_info), K(unit_info)); + } + } + } // end j + if (OB_SUCC(ret) && !found) { + //can found unit_group for a ls group + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to find unit group for ls group", KR(ret), + K(group_info), K(unit_group_array_)); + } + } + + return ret; +} + +int ObTenantLSInfo::check_unit_group_valid_( + const uint64_t unit_group_id, bool &is_valid_info) +{ + int ret = OB_SUCCESS; + is_valid_info = true; + if (OB_UNLIKELY(OB_INVALID_VERSION == unit_group_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("ls id is invalid", KR(ret), K(unit_group_id)); + } else if (OB_UNLIKELY(!is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant stat not valid", KR(ret)); + } else { + //check unit group status is valid + ret = OB_ENTRY_NOT_EXIST; + for (int64_t i = 0; OB_FAIL(ret) && i < unit_group_array_.count(); ++i) { + const ObUnitGroupInfo &group_info = unit_group_array_.at(i); + if (unit_group_id == group_info.unit_group_id_) { + ret = OB_SUCCESS; + if (share::ObUnit::UNIT_STATUS_ACTIVE != group_info.unit_status_) { + is_valid_info = false; + } + break; + } + }//end for + if (OB_FAIL(ret)) { + LOG_WARN("failed to found unit group status", KR(ret), K(unit_group_id), + K(unit_group_array_)); + } + } + return ret; +} + +//for standby +int ObTenantLSInfo::create_new_ls_for_recovery( + const share::ObLSID &ls_id, + const uint64_t ls_group_id, + const SCN &create_scn, + ObMySQLTransaction &trans) +{ + int ret = OB_SUCCESS; + const bool is_recovery = true; + int64_t info_index = OB_INVALID_INDEX_INT64; + if (OB_UNLIKELY(!ls_id.is_valid() + || OB_INVALID_ID == ls_group_id + || !create_scn.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("ls id is invalid", KR(ret), K(ls_id), K(ls_group_id), K(create_scn)); + } else if (OB_UNLIKELY(!is_valid()) + || OB_ISNULL(tenant_schema_) + || OB_ISNULL(sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant stat not valid", KR(ret), KP(tenant_schema_), KP(sql_proxy_)); + } else { + const uint64_t tenant_id = tenant_schema_->get_tenant_id(); + share::ObLSStatusInfo new_info; + ObLSGroupInfo group_info; + ObZone primary_zone; + ObLSLifeAgentManager ls_life_agent(*sql_proxy_); + ObSqlString zone_priority; + if (OB_SUCC(get_ls_group_info( + ls_group_id, group_info))) { + // need a new primary zone + } else if (OB_ENTRY_NOT_EXIST == ret) { + // need a new unit group + ret = OB_SUCCESS; + int64_t unit_group_index = OB_INVALID_INDEX_INT64; + if (OB_FAIL(get_next_unit_group_(is_recovery, unit_group_index))) { + LOG_WARN("failed to get next unit group", KR(ret), K(is_recovery)); + } else if (OB_UNLIKELY(OB_INVALID_INDEX_INT64 == unit_group_index + || unit_group_index >= unit_group_array_.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to next unit group", KR(ret), K(unit_group_index)); + } else if (OB_FAIL(group_info.init( + unit_group_array_.at(unit_group_index).unit_group_id_, + ls_group_id))) { + LOG_WARN("failed to init group info", KR(ret), K(unit_group_index), K(ls_group_id)); + } + } else { + LOG_WARN("failed to get ls group info", KR(ret), + K(ls_group_id)); + } + + if (FAILEDx(get_next_primary_zone_(is_recovery, group_info, primary_zone))) { + LOG_WARN("failed to get next primary zone", KR(ret), K(group_info)); + } else if (OB_FAIL(new_info.init(tenant_id, ls_id, + ls_group_id, + share::OB_LS_CREATING, + group_info.unit_group_id_, primary_zone))) { + LOG_WARN("failed to init new info", KR(ret), K(tenant_id), + K(ls_id), K(ls_group_id), K(group_info), K(primary_zone)); + } else if (OB_FAIL(get_zone_priority(primary_zone, *tenant_schema_, zone_priority))) { + LOG_WARN("failed to get normalize primary zone", KR(ret), K(primary_zone), K(zone_priority)); + } else if (OB_FAIL(ls_life_agent.create_new_ls_in_trans(new_info, create_scn, + zone_priority.string(), share::NORMAL_SWITCHOVER_STATUS, trans))) { + LOG_WARN("failed to insert ls info", KR(ret), K(new_info), K(create_scn), K(zone_priority)); + } else if (OB_FAIL(add_ls_status_info_(new_info))) { + LOG_WARN("failed to add new ls info to memory", KR(ret), K(new_info)); + } + } + return ret; +} + +int ObTenantLSInfo::process_ls_stats_for_recovery() +{ + int ret = OB_SUCCESS; + share::ObLSStatusInfoArray status_info_array; + ObLSRecoveryStat recovery_stat; + ObLSRecoveryStatOperator ls_recovery_operator; + if (OB_ISNULL(sql_proxy_) || OB_ISNULL(tenant_schema_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy or tenant schema is null", KR(ret), KP(tenant_schema_), KP(sql_proxy_)); + } else { + const uint64_t tenant_id = tenant_schema_->get_tenant_id(); + int tmp_ret = OB_SUCCESS; + if (OB_FAIL(status_operator_.get_all_ls_status_by_order( + tenant_id, status_info_array, *sql_proxy_))) { + LOG_WARN("failed to update ls status", KR(ret), K(tenant_id)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < status_info_array.count(); ++i) { + const ObLSStatusInfo &status_info = status_info_array.at(i); + recovery_stat.reset(); + if (status_info.ls_is_creating()) { + if (OB_FAIL(ls_recovery_operator.get_ls_recovery_stat(tenant_id, status_info.ls_id_, + false/*for_update*/, recovery_stat, *sql_proxy_))) { + LOG_WARN("failed to get ls recovery stat", KR(ret), K(tenant_id), K(status_info)); + } else if (OB_FAIL(do_create_ls_(status_info, recovery_stat.get_create_scn()))) { + LOG_WARN("failed to create new ls", KR(ret), K(status_info), + K(recovery_stat)); + } + } + }//end for + //adjust primary zone + if (OB_SUCCESS != (tmp_ret = adjust_user_tenant_primary_zone())) { + ret = OB_SUCC(ret) ? tmp_ret : ret; + LOG_WARN("failed to adjust user tenant primary zone", KR(ret), + KR(tmp_ret), K(tenant_schema_)); + } + } + return ret; +} + +int ObTenantLSInfo::add_ls_status_info_( + const share::ObLSStatusInfo &ls_info) +{ + int ret = OB_SUCCESS; + const int64_t index = status_array_.count(); + if (OB_UNLIKELY(!ls_info.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("ls info is invalid", KR(ret), K(ls_info)); + } else if (OB_FAIL(status_array_.push_back(ls_info))) { + LOG_WARN("failed to remove status", KR(ret), K(ls_info)); + } else if (OB_FAIL(status_map_.set_refactored(ls_info.ls_id_, index))) { + LOG_WARN("failed to remove ls from map", KR(ret), K(ls_info), K(index)); + } else if (ls_info.ls_id_.is_sys_ls()) { + //sys ls no ls group + } else if (OB_FAIL(add_ls_to_ls_group_(ls_info))) { + LOG_WARN("failed to add ls info", KR(ret), K(ls_info)); + } + return ret; +} + +int ObTenantLSInfo::get_ls_group_info( + const uint64_t ls_group_id, ObLSGroupInfo &info) const +{ + int ret = OB_SUCCESS; + info.reset(); + if (OB_UNLIKELY(OB_INVALID_ID == ls_group_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("ls group id is invalid", KR(ret), K(ls_group_id)); + } else if (OB_UNLIKELY(!is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant stat not valid", KR(ret)); + } else { + bool found = false; + for (int64_t i = 0; OB_SUCC(ret) && i < ls_group_array_.count(); ++i) { + const ObLSGroupInfo group_info = ls_group_array_.at(i); + if (ls_group_id == group_info.ls_group_id_) { + found = true; + if (OB_FAIL(info.assign(group_info))) { + LOG_WARN("failed to assign group info", KR(ret), K(group_info)); + } + break; + } + }//end for + if (OB_SUCC(ret) && !found) { + ret = OB_ENTRY_NOT_EXIST; + LOG_WARN("failed to find ls group info", KR(ret), + K(ls_group_id)); + } + } + return ret; +} + +int ObTenantLSInfo::get_ls_status_info( + const share::ObLSID &ls_id, + share::ObLSStatusInfo &info, + int64_t &info_index) const +{ + int ret = OB_SUCCESS; + info.reset(); + info_index = OB_INVALID_INDEX_INT64; + if (OB_UNLIKELY(!ls_id.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("ls id is invalid", KR(ret), K(ls_id)); + } else if (OB_UNLIKELY(!is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant stat not valid", KR(ret)); + } else if (OB_FAIL(status_map_.get_refactored(ls_id, info_index))) { + if (OB_HASH_NOT_EXIST == ret) { + ret = OB_ENTRY_NOT_EXIST; + } + LOG_WARN("failed to find ls index", KR(ret), K(ls_id)); + } else if (OB_UNLIKELY(info_index < 0 || info_index >= status_array_.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("info index not valid", KR(ret), K(info_index), K(ls_id), + "status_count", status_array_.count()); + } else if (OB_FAIL(info.assign(status_array_.at(info_index)))) { + LOG_WARN("failed to assign ls info", KR(ret), K(info_index), + "status", status_array_.at(info_index)); + } + return ret; +} + +int ObTenantLSInfo::get_next_unit_group_(const bool is_recovery, int64_t &group_index) +{ + int ret = OB_SUCCESS; + group_index = OB_INVALID_INDEX_INT64; + if (OB_UNLIKELY(!is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant stat not valid", KR(ret)); + } else { + int64_t ls_count = OB_INVALID_COUNT; + // Find the unit group with the least number of log streams + for (int64_t i = 0; OB_SUCC(ret) && i < unit_group_array_.count(); ++i) { + const ObUnitGroupInfo &info = unit_group_array_.at(i); + const int64_t count = info.ls_group_ids_.count(); + if (share::ObUnit::UNIT_STATUS_ACTIVE == info.unit_status_) { + if (OB_INVALID_COUNT == ls_count || ls_count > count) { + ls_count = count; + group_index = i; + if (0 == count) { + //the first has no ls group unit group + break; + } + } + } + } + if (OB_SUCC(ret)) { + if (OB_INVALID_INDEX_INT64 == group_index + || (!is_recovery && 0 != ls_count)) { + ret = OB_ENTRY_NOT_EXIST; + LOG_WARN("failed to find next unit group", KR(ret), + K(unit_group_array_), K(is_recovery), K(ls_count)); + } + } + } + LOG_INFO("get next primary zone", KR(ret), K(group_index), K(is_recovery)); + return ret; +} + +int ObTenantLSInfo::get_next_primary_zone_( + const bool is_recovery, + const ObLSGroupInfo &group_info, + ObZone &primary_zone) +{ + int ret = OB_SUCCESS; + primary_zone.reset(); + if (OB_UNLIKELY(!group_info.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("group info is invalid", KR(ret), K(group_info)); + } else if (OB_UNLIKELY(!is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant ls info not valid", KR(ret)); + } else { + share::ObLSStatusInfo info; + int64_t ls_count = OB_INVALID_COUNT; + int64_t info_index = OB_INVALID_INDEX_INT64; + for (int64_t i = 0; OB_SUCC(ret) && i < primary_zone_.count(); ++i) { + const ObZone &zone = primary_zone_.at(i); + int64_t primary_zone_count = 0; + for (int64_t j = 0; OB_SUCC(ret) && j < group_info.ls_ids_.count(); ++j) { + const share::ObLSID &id = group_info.ls_ids_.at(j); + if (OB_FAIL(get_ls_status_info(id, info, info_index))) { + LOG_WARN("failed to find ls info", KR(ret), K(id), K(j)); + } else if (zone == info.primary_zone_) { + primary_zone_count++; + } + }//end for j + if (OB_SUCC(ret)) { + if (OB_INVALID_COUNT == ls_count || ls_count > primary_zone_count) { + ls_count = primary_zone_count; + if (OB_FAIL(primary_zone.assign(zone))) { + LOG_WARN("failed to assign zone", KR(ret), K(zone)); + } else if (0 == primary_zone_count) { + //the first zone has no primary zone + break; + } + } + } + }//end for i + if (OB_SUCC(ret)) { + if (primary_zone.is_empty() || (!is_recovery && 0 != ls_count)) { + ret = OB_ENTRY_NOT_EXIST; + LOG_WARN("failed to find next primary zone", KR(ret), K(primary_zone_), + K(group_info), K(is_recovery), K(ls_count)); + } + } + LOG_INFO("get next primary zone", KR(ret), K(group_info), K(primary_zone)); + } + return ret; +} + +//need create or delete new ls group +int ObTenantLSInfo::check_ls_group_match_unitnum() +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tenant stat not valid", KR(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < unit_group_array_.count(); ++i) { + const ObUnitGroupInfo &info = unit_group_array_.at(i); + if (share::ObUnit::UNIT_STATUS_MAX == info.unit_status_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unit status not valid", KR(ret), K(info)); + } else if (share::ObUnit::UNIT_STATUS_DELETING == info.unit_status_) { + // drop logstream, + if (0 == info.ls_group_ids_.count()) { + //nothing + } else if (OB_FAIL(try_drop_ls_of_deleting_unit_group_(info))) { + LOG_WARN("failed to drop ls of the unit group", KR(ret), K(info)); + } + } else if (0 == info.ls_group_ids_.count()) { + //create ls + if (OB_FAIL(create_new_ls_for_empty_unit_group_(info.unit_group_id_))) { + LOG_WARN("failed to create new ls for empty unit_group", KR(ret), + K(info)); + } + } + } + return ret; +} + +int ObTenantLSInfo::create_new_ls_for_empty_unit_group_(const uint64_t unit_group_id) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(OB_INVALID_INDEX_INT64 == unit_group_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("group id is invalid", KR(ret), K(unit_group_id)); + } else if (OB_UNLIKELY(!is_valid()) + || OB_ISNULL(tenant_schema_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant ls info not valid", KR(ret), KP(tenant_schema_)); + } else if (OB_UNLIKELY(0 == primary_zone_.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("primary zone is invalid", KR(ret)); + } else { + const uint64_t tenant_id = tenant_schema_->get_tenant_id(); + share::ObLSID new_id; + share::ObLSStatusInfo new_info; + uint64_t new_ls_group_id = OB_INVALID_INDEX_INT64; + if (OB_FAIL(fetch_new_ls_group_id(tenant_id, new_ls_group_id))) { + LOG_WARN("failed to fetch new id", KR(ret), K(tenant_id)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < primary_zone_.count(); ++i) { + const ObZone &zone = primary_zone_.at(i); + new_info.reset(); + if (OB_FAIL(fetch_new_ls_id(tenant_id, new_id))) { + LOG_WARN("failed to get new id", KR(ret), K(tenant_id)); + } else if (OB_FAIL(new_info.init(tenant_id, new_id, new_ls_group_id, + share::OB_LS_CREATING, unit_group_id, + zone))) { + LOG_WARN("failed to init new info", KR(ret), K(new_id), + K(new_ls_group_id), K(unit_group_id), K(zone), K(tenant_id)); + } else if (OB_FAIL(create_new_ls_(new_info, share::NORMAL_SWITCHOVER_STATUS))) { + LOG_WARN("failed to add ls info", KR(ret), K(new_info)); + } + LOG_INFO("[LS_MGR] create new ls for empty unit group", KR(ret), K(new_info)); + }//end for + } + return ret; +} + +int ObTenantLSInfo::try_drop_ls_of_deleting_unit_group_(const ObUnitGroupInfo &info) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(share::ObUnit::UNIT_STATUS_DELETING != info.unit_status_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("unit group status is invalid", KR(ret), K(info)); + } else if (OB_UNLIKELY(!is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant ls info not valid", KR(ret)); + } else { + ARRAY_FOREACH_X(info.ls_group_ids_, idx, cnt, OB_SUCC(ret)) { + ObLSGroupInfo ls_group_info; + const int64_t ls_group_id = info.ls_group_ids_.at(idx); + if (OB_FAIL(get_ls_group_info(ls_group_id, ls_group_info))) { + LOG_WARN("failed to get ls group info", KR(ret), K(idx), + K(ls_group_id)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < ls_group_info.ls_ids_.count(); ++i) { + const share::ObLSID &ls_id = ls_group_info.ls_ids_.at(i); + share::ObLSStatusInfo info; + int64_t info_index = 0; + if (OB_FAIL(get_ls_status_info(ls_id, info, info_index))) { + LOG_WARN("failed to get ls status info", KR(ret), K(ls_id)); + } else if (OB_FAIL(drop_ls_(info, share::NORMAL_SWITCHOVER_STATUS))) { + LOG_WARN("failed to drop ls", KR(ret), K(info)); + } + LOG_INFO("[LS_MGR] drop ls for deleting unit group", KR(ret), + K(ls_id), K(info)); + }//end for each ls in ls group info + } + }//end for each ls group in unit group info + } + return ret; +} + +int ObTenantLSInfo::drop_ls_(const share::ObLSStatusInfo &info, + const ObTenantSwitchoverStatus &working_sw_status) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tenant stat not valid", KR(ret)); + } else if (OB_UNLIKELY(!info.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("ls info is invalid", KR(ret), K(info)); + } else if (info.ls_is_normal()) { + //try to set ls to dropping + if (OB_FAIL(ls_operator_.update_ls_status(info.ls_id_, + share::OB_LS_NORMAL, share::OB_LS_DROPPING, working_sw_status))) { + LOG_WARN("failed to update ls status", KR(ret), K(info)); + } + } else if (info.ls_is_created()) { + //if ls_status is created, the ls_info may creating or normal + //creating can not to dropping, so need retry after fix_status_machine. + ret = OB_NEED_RETRY; + LOG_WARN("not certain status in ls info, need retry", KR(ret), K(info)); + } else if (info.ls_is_dropping() + || info.ls_is_wait_offline()) { + //nothing + } else if (info.ls_is_creating()) { + //if status is in creating, the ls must in creating too + if (OB_FAIL(ls_operator_.delete_ls(info.ls_id_, share::OB_LS_CREATING, working_sw_status))) { + LOG_WARN("failed to process creating info", KR(ret), K(info)); + } + } else { + //other ls status not expected + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls status not expected for dropping ls", KR(ret), K(info)); + } + return ret; +} + +//need create or delete new ls +//TODO for primary cluster, The primary_zone of multiple LSs may be the same, +//the redundant ones need to be deleted, and only one is kept. +int ObTenantLSInfo::check_ls_match_primary_zone() +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_valid()) || OB_ISNULL(tenant_schema_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant ls info not valid", KR(ret), KP(tenant_schema_)); + } else { + const uint64_t tenant_id = tenant_schema_->get_tenant_id(); + ObZone zone; + share::ObLSID new_id; + share::ObLSStatusInfo new_info; + for (int64_t i = 0; OB_SUCC(ret) && i < ls_group_array_.count(); ++i) { + ObLSGroupInfo &group_info = ls_group_array_.at(i); + //check the unit group is active + //if unit group is in deleting, no need check primary zone + bool is_valid = false; + int64_t group_id_count = group_info.ls_ids_.count(); + if (OB_FAIL(check_unit_group_valid_(group_info.unit_group_id_, is_valid))) { + LOG_WARN("failed to check unit group valid", KR(ret), K(group_info)); + } + while (OB_SUCC(ret) && is_valid + && primary_zone_.count() > group_id_count) { + group_id_count++; + if (OB_FAIL(get_next_primary_zone_(false/*is_recovery*/, group_info, zone))) { + LOG_WARN("failed to get next primary zone", K(group_info)); + } else if (OB_FAIL(fetch_new_ls_id(tenant_id, new_id))) { + LOG_WARN("failed to get new id", KR(ret), K(tenant_id)); + } else if (OB_FAIL(new_info.init(tenant_id, new_id, group_info.ls_group_id_, + share::OB_LS_CREATING, + group_info.unit_group_id_, zone))) { + LOG_WARN("failed to init new info", KR(ret), K(new_id), + K(group_info), K(zone), K(tenant_id)); + } else if (OB_FAIL(create_new_ls_(new_info, share::NORMAL_SWITCHOVER_STATUS))) { + LOG_WARN("failed to create new ls", KR(ret), K(new_info)); + } + LOG_INFO("[LS_MGR] create new ls of primary zone", KR(ret), K(new_info), + K(group_id_count)); + } + } + } + return ret; +} + +int ObTenantLSInfo::get_zone_priority(const ObZone &primary_zone, + const share::schema::ObTenantSchema &tenant_schema, + ObSqlString &primary_zone_str) +{ + int ret = OB_SUCCESS; + primary_zone_str.reset(); + if (OB_UNLIKELY(!tenant_schema.is_valid() || primary_zone.is_empty())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(primary_zone), K(tenant_schema)); + } else if (OB_FAIL(ObPrimaryZoneUtil::get_ls_primary_zone_priority(primary_zone, + tenant_schema, primary_zone_str))) { + LOG_WARN("failed to get ls primary zone priority", KR(ret), K(primary_zone), K(tenant_schema)); + } + LOG_DEBUG("get zone priority", KR(ret), K(primary_zone_str), K(tenant_schema)); + return ret; +} + + +int ObTenantLSInfo::create_new_ls_(const share::ObLSStatusInfo &status_info, + const share::ObTenantSwitchoverStatus &working_sw_status) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tenant stat not valid", KR(ret)); + } else if (OB_UNLIKELY(!status_info.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("ls info is invalid", KR(ret), K(status_info)); + } else if (OB_ISNULL(sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy is null", KR(ret)); + } else { + share::ObLSLifeAgentManager ls_life_agent(*sql_proxy_); + share::ObLSAttr ls_info; + share::ObLSFlag flag = share::OB_LS_FLAG_NORMAL;//TODO + SCN create_scn; + ObSqlString zone_priority; + if (OB_FAIL(ObLSAttrOperator::get_tenant_gts(status_info.tenant_id_, create_scn))) { + LOG_WARN("failed to get tenant gts", KR(ret), K(status_info)); + } else if (OB_FAIL(ls_info.init(status_info.ls_id_, status_info.ls_group_id_, flag, + share::OB_LS_CREATING, share::OB_LS_OP_CREATE_PRE, create_scn))) { + LOG_WARN("failed to init new operation", KR(ret), K(status_info), K(create_scn)); + } else if (OB_FAIL(ls_operator_.insert_ls(ls_info, max_ls_group_id_, working_sw_status))) { + LOG_WARN("failed to insert new operation", KR(ret), K(ls_info), K(max_ls_group_id_)); + } else if (OB_FAIL(get_zone_priority(status_info.primary_zone_, + *tenant_schema_, zone_priority))) { + LOG_WARN("failed to get normalize primary zone", KR(ret), K(status_info), + K(zone_priority), K(tenant_schema_)); + } else if (OB_FAIL(ls_life_agent.create_new_ls(status_info, create_scn, + zone_priority.string(), working_sw_status))) { + LOG_WARN("failed to create new ls", KR(ret), K(status_info), K(create_scn), + K(zone_priority)); + } else if (OB_FAIL(do_create_ls_(status_info, create_scn))) { + LOG_WARN("failed to create ls", KR(ret), K(status_info), K(create_scn)); + } else if (OB_FAIL(process_ls_status_after_created_(status_info, working_sw_status))) { + LOG_WARN("failed to update ls status", KR(ret), K(status_info)); + } + LOG_INFO("[LS_MGR] create new ls", KR(ret), K(status_info)); + } + return ret; +} + +int ObTenantLSInfo::fetch_new_ls_group_id( + const uint64_t tenant_id, + uint64_t &ls_group_id) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tenant id is invalid", KR(ret), K(tenant_id)); + } else if (OB_ISNULL(sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy is null", KR(ret)); + } else { + ls_group_id = OB_INVALID_ID; + share::ObMaxIdFetcher id_fetcher(*sql_proxy_); + if (OB_FAIL(id_fetcher.fetch_new_max_id( + tenant_id, share::OB_MAX_USED_LS_GROUP_ID_TYPE, ls_group_id))) { + LOG_WARN("fetch_max_id failed", KR(ret), K(tenant_id), "id_type", + share::OB_MAX_USED_LS_GROUP_ID_TYPE); + } else { + if (ls_group_id != ++max_ls_group_id_) { + ret = OB_EAGAIN; + LOG_WARN("ls may create concurrency, need retry", KR(ret), K(ls_group_id), + K(max_ls_group_id_)); + } + } + } + return ret; +} + +int ObTenantLSInfo::fetch_new_ls_id(const uint64_t tenant_id, share::ObLSID &id) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tenant id is invalid", KR(ret), K(tenant_id)); + } else if (OB_ISNULL(sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy is null", KR(ret)); + } else { + share::ObMaxIdFetcher id_fetcher(*sql_proxy_); + uint64_t id_value = OB_INVALID_ID; + if (OB_FAIL(id_fetcher.fetch_new_max_id( + tenant_id, share::OB_MAX_USED_LS_ID_TYPE, id_value))) { + LOG_WARN("fetch_max_id failed", KR(ret), K(tenant_id), "id_type", + share::OB_MAX_USED_LS_ID_TYPE); + } else { + if (id_value != ++max_ls_id_) { + ret = OB_EAGAIN; + LOG_WARN("ls may create concurrency, need retry", KR(ret), K(id_value), + K(max_ls_id_)); + } else { + share::ObLSID new_id(id_value); + id = new_id; + } + } + } + return ret; +} + +int ObTenantLSInfo::create_ls_with_palf( + const share::ObLSStatusInfo &info, + const SCN &create_scn, + const bool create_ls_with_palf, + const palf::PalfBaseInfo &palf_base_info) +{ + int ret = OB_SUCCESS; + LOG_INFO("[LS_EXEC] start to create ls", K(info)); + const int64_t start_time = ObTimeUtility::fast_current_time(); + if (OB_UNLIKELY(!info.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("info not valid", KR(ret), K(info)); + } else if (OB_ISNULL(rpc_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("rpc proxy is null", KR(ret)); + } else if (OB_UNLIKELY(!info.ls_is_created() && !info.ls_is_creating())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("info status not expected", KR(ret), K(info)); + } else if (info.ls_is_creating()) { + common::ObArray locality_array; + int64_t paxos_replica_num = 0; + ObSchemaGetterGuard guard;//nothing + if (OB_ISNULL(tenant_schema_)) { + ret = OB_TENANT_NOT_EXIST; + LOG_WARN("tenant not exist", KR(ret), K(info)); + } else if (tenant_schema_->is_dropping()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant is in dropping, no need create ls", KR(ret), K(tenant_schema_)); + } else if (OB_FAIL(tenant_schema_->get_zone_replica_attr_array( + locality_array))) { + LOG_WARN("failed to get zone locality array", KR(ret)); + } else if (OB_FAIL(tenant_schema_->get_paxos_replica_num( + guard, paxos_replica_num))) { + LOG_WARN("failed to get paxos replica num", KR(ret)); + } else { + ObLSCreator creator(*rpc_proxy_, info.tenant_id_, + info.ls_id_, sql_proxy_); + if (OB_FAIL(creator.create_user_ls(info, paxos_replica_num, + locality_array, create_scn, + tenant_schema_->get_compatibility_mode(), + create_ls_with_palf, + palf_base_info))) { + LOG_WARN("failed to create user ls", KR(ret), K(info), K(locality_array), K(create_scn), + K(palf_base_info), K(create_ls_with_palf)); + } + } + } + const int64_t cost = ObTimeUtility::fast_current_time() - start_time; + LOG_INFO("[LS_EXEC] end to create ls", KR(ret), K(info), K(cost)); + return ret; +} + +int ObTenantLSInfo::do_create_ls_(const share::ObLSStatusInfo &info, + const SCN &create_scn) +{ + int ret = OB_SUCCESS; + const int64_t start_time = ObTimeUtility::fast_current_time(); + if (OB_UNLIKELY(!info.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("info not valid", KR(ret), K(info)); + } else { + palf::PalfBaseInfo palf_base_info; + if (OB_FAIL(create_ls_with_palf(info, create_scn, false, palf_base_info))) { + LOG_WARN("failed to create ls with palf", KR(ret), K(info), K(create_scn), K(palf_base_info)); + } + } + return ret; +} + +int ObTenantLSInfo::process_ls_status_after_created_(const share::ObLSStatusInfo &status_info, + const ObTenantSwitchoverStatus &working_sw_status) +{ + //do not check ls status + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!status_info.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("info not valid", KR(ret), K(status_info)); + } else if (OB_ISNULL(sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy is null", KR(ret)); + } else if (OB_FAIL(ls_operator_.update_ls_status( + status_info.ls_id_, share::OB_LS_CREATING, share::OB_LS_NORMAL, working_sw_status))) { + LOG_WARN("failed to update ls status", KR(ret), K(status_info)); + } else if (OB_FAIL(status_operator_.update_ls_status( + status_info.tenant_id_, status_info.ls_id_, share::OB_LS_CREATED, + share::OB_LS_NORMAL, working_sw_status, *sql_proxy_))) { + LOG_WARN("failed to update ls status", KR(ret), K(status_info)); + } + return ret; +} + +int ObTenantLSInfo::do_tenant_drop_ls_(const share::ObLSStatusInfo &status_info, + const ObTenantSwitchoverStatus &working_sw_status) +{ + int ret = OB_SUCCESS; + LOG_INFO("[LS_EXEC] start to tenant drop ls", K(status_info)); + const int64_t start_time = ObTimeUtility::fast_current_time(); + bool can_offline = false; + if (OB_UNLIKELY(!status_info.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("info not valid", KR(ret), K(status_info)); + } else if (OB_ISNULL(sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy is null", KR(ret)); + } else if (OB_UNLIKELY(status_info.ls_is_creating() || status_info.ls_is_wait_offline())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("status info not expected", KR(ret), K(status_info)); + } else if (!status_info.ls_is_tenant_dropping()) { + //status_info may in created, normal, dropping, tenant_dropping + if (OB_FAIL(status_operator_.update_ls_status( + status_info.tenant_id_, status_info.ls_id_, + status_info.status_, share::OB_LS_TENANT_DROPPING, working_sw_status, + *sql_proxy_))) { + LOG_WARN("failed to update ls status", KR(ret), K(status_info)); + } + } + if (FAILEDx(check_ls_can_offline_by_rpc_(status_info, share::OB_LS_TENANT_DROPPING, can_offline))) { + // send rpc to observer + LOG_WARN("failed to check ls can offline", KR(ret), K(status_info)); + } else if (can_offline) { + ObLSLifeAgentManager ls_life_agent(*sql_proxy_); + SCN drop_scn; + if (!status_info.ls_id_.is_sys_ls()) { + //sys ls cannot delete ls, after ls is in tenant dropping + if (OB_FAIL(ls_operator_.delete_ls(status_info.ls_id_, share::OB_LS_TENANT_DROPPING, + working_sw_status))) { + LOG_WARN("failed to delete ls", KR(ret), K(status_info)); + } else if (OB_FAIL(ObLSAttrOperator::get_tenant_gts(status_info.tenant_id_, drop_scn))) { + LOG_WARN("failed to get gts", KR(ret), K(status_info)); + } + } else { + //TODO sys ls can not get GTS after tenant_dropping + drop_scn.set_base(); + } + if (FAILEDx(ls_life_agent.set_ls_offline(status_info.tenant_id_, + status_info.ls_id_, status_info.status_, drop_scn, working_sw_status))) { + LOG_WARN("failed to update ls info", KR(ret), K(status_info), K(drop_scn)); + } + } + const int64_t cost = ObTimeUtility::fast_current_time() - start_time; + LOG_INFO("[LS_EXEC] end to tenant drop ls", KR(ret), K(status_info), K(cost)); + return ret; +} + +int ObTenantLSInfo::do_drop_ls_(const share::ObLSStatusInfo &status_info, + const ObTenantSwitchoverStatus &working_sw_status) +{ + int ret = OB_SUCCESS; + LOG_INFO("[LS_EXEC] start to drop ls", K(status_info)); + const int64_t start_time = ObTimeUtility::fast_current_time(); + bool tablet_empty = false; + if (OB_UNLIKELY(!status_info.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("info not valid", KR(ret), K(status_info)); + } else if (OB_ISNULL(sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy is null", KR(ret)); + } else if (status_info.ls_is_normal()) { + if (OB_FAIL(status_operator_.update_ls_status( + status_info.tenant_id_, status_info.ls_id_, + status_info.status_, share::OB_LS_DROPPING, + working_sw_status, + *sql_proxy_))) { + LOG_WARN("failed to update ls status", KR(ret), K(status_info)); + } + } else if (status_info.ls_is_dropping()) { + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("status not expected", KR(ret), K(status_info)); + } + if (FAILEDx(check_ls_empty_(status_info, tablet_empty))) { + LOG_WARN("failed to check ls empty", KR(ret), K(status_info)); + } + if (OB_SUCC(ret) && tablet_empty) { + // send rpc to observer + bool can_offline = false; + if (OB_FAIL(check_ls_can_offline_by_rpc_(status_info, share::OB_LS_DROPPING, can_offline))) { + LOG_WARN("failed to check ls can offline", KR(ret), K(status_info)); + } else if (can_offline) { + ObLSLifeAgentManager ls_life_agent(*sql_proxy_); + SCN drop_scn; + if (OB_FAIL(ls_operator_.delete_ls(status_info.ls_id_, share::OB_LS_DROPPING, + working_sw_status))) { + LOG_WARN("failed to delete ls", KR(ret), K(status_info)); + } else if (OB_FAIL(ObLSAttrOperator::get_tenant_gts(status_info.tenant_id_, drop_scn))) { + LOG_WARN("failed to get gts", KR(ret), K(status_info)); + } else if (OB_FAIL(ls_life_agent.set_ls_offline(status_info.tenant_id_, + status_info.ls_id_, status_info.status_, drop_scn, working_sw_status))) { + LOG_WARN("failed to update ls info", KR(ret), K(status_info), K(drop_scn)); + } + } + } + const int64_t cost = ObTimeUtility::fast_current_time() - start_time; + LOG_INFO("[LS_EXEC] end to drop ls", KR(ret), K(status_info), K(cost), K(tablet_empty)); + return ret; +} + + +int ObTenantLSInfo::sys_ls_tenant_drop_(const share::ObLSStatusInfo &info, + const share::ObTenantSwitchoverStatus &working_sw_status) +{ + int ret = OB_SUCCESS; + const ObLSStatus target_status = share::OB_LS_TENANT_DROPPING; + const ObLSStatus pre_status = share::OB_LS_PRE_TENANT_DROPPING; + if (OB_UNLIKELY(!info.is_valid() + || !info.ls_id_.is_sys_ls())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(info)); + } else if (OB_ISNULL(sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy is null", KR(ret), KP(sql_proxy_)); + } else if (info.ls_is_normal()) { + if (OB_FAIL(status_operator_.update_ls_status(info.tenant_id_, + info.ls_id_, info.status_, pre_status, working_sw_status, + *sql_proxy_))) { + LOG_WARN("failed to update ls status", KR(ret), K(info), K(pre_status)); + } + } else if (pre_status == info.status_) { + //nothing + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sys ls can not in other status", KR(ret), K(info)); + } + bool can_offline = false; + if (FAILEDx(check_sys_ls_can_offline_(can_offline))) { + LOG_WARN("failed to check sys ls can offline", KR(ret)); + } else if (can_offline) { + if (OB_FAIL(ls_operator_.update_ls_status(info.ls_id_, pre_status, target_status, + working_sw_status))) { + LOG_WARN("failed to update ls status", KR(ret), K(info), K(pre_status), K(target_status)); + } else if (OB_FAIL(status_operator_.update_ls_status(info.tenant_id_, + info.ls_id_, pre_status, target_status, working_sw_status, *sql_proxy_))) { + LOG_WARN("failed to update ls status", KR(ret), K(info), K(pre_status), K(target_status)); + } + } + return ret; +} + +int ObTenantLSInfo::check_sys_ls_can_offline_(bool &can_offline) +{ + int ret = OB_SUCCESS; + share::ObLSStatusInfoArray status_info_array; + can_offline = true; + if (OB_ISNULL(sql_proxy_) || OB_ISNULL(tenant_schema_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy or tenant schema is null", KR(ret), KP(tenant_schema_), KP(sql_proxy_)); + } else { + const uint64_t tenant_id = tenant_schema_->get_tenant_id(); + if (OB_FAIL(status_operator_.get_all_ls_status_by_order( + tenant_id, status_info_array, *sql_proxy_))) { + LOG_WARN("failed to update ls status", KR(ret), K(tenant_id)); + } else if (0 == status_info_array.count()) { + //if has multi ls_mgr + can_offline = true; + } + for (int64_t i = 0; OB_SUCC(ret) && i < status_info_array.count() && can_offline; ++i) { + const share::ObLSStatusInfo &status_info = status_info_array.at(i); + if (status_info.ls_id_.is_sys_ls()) { + } else if (status_info.ls_is_wait_offline()) { + //nothing + } else { + can_offline = false; + LOG_INFO("[LS_MGR] sys ls can not offline", K(status_info)); + break; + } + } + if (OB_SUCC(ret) && can_offline) { + LOG_INFO("[LS_MGR] sys ls can offline", K(status_info_array)); + } + } + return ret; +} + +int ObTenantLSInfo::check_ls_empty_(const share::ObLSStatusInfo &info, bool &empty) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!info.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("info not valid", KR(ret), K(info)); + } else { + ObSqlString sql; + if (OB_FAIL(sql.assign_fmt( + "SELECT * FROM %s where ls_id = %ld", + OB_ALL_TABLET_TO_LS_TNAME, info.ls_id_.id()))) { + LOG_WARN("failed to assign sql", KR(ret), K(sql)); + } else if (OB_ISNULL(sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy is null", KR(ret)); + } else { + HEAP_VAR(ObMySQLProxy::MySQLResult, res) { + common::sqlclient::ObMySQLResult *result = NULL; + if (OB_FAIL(sql_proxy_->read(res, info.tenant_id_, sql.ptr()))) { + LOG_WARN("failed to read", KR(ret), K(info), K(sql)); + } else if (OB_ISNULL(result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get sql result", KR(ret)); + } else if (OB_SUCC(result->next())) { + empty = false; + } else if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + empty = true; + } else { + LOG_WARN("failed to get next", KR(ret), K(sql)); + } + } + } + } + return ret; +} + +int ObTenantLSInfo::check_ls_can_offline_by_rpc_(const share::ObLSStatusInfo &info, + const share::ObLSStatus ¤t_ls_status, bool &can_offline) +{ + int ret = OB_SUCCESS; + share::ObLSInfo ls_info; + const share::ObLSReplica *replica = NULL; + if (OB_UNLIKELY(!info.is_valid() + || !ls_is_tenant_dropping_status(current_ls_status) && ! ls_is_dropping_status(current_ls_status))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("info not valid", KR(ret), K(info), K(current_ls_status)); + } else if (OB_ISNULL(lst_operator_) || OB_ISNULL(rpc_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("lst operator or proxy is null", KR(ret), KP(lst_operator_), + KP(rpc_proxy_)); + } else if (OB_FAIL(lst_operator_->get(GCONF.cluster_id, info.tenant_id_, + info.ls_id_, share::ObLSTable::DEFAULT_MODE, ls_info))) { + LOG_WARN("failed to get ls info", KR(ret), K(info)); + } else if (OB_FAIL(ls_info.find_leader(replica))) { + LOG_WARN("failed to find leader", KR(ret), K(ls_info)); + } else if (OB_ISNULL(replica)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("replica is null", KR(ret), K(ls_info)); + } else { + const int64_t timeout = GCONF.rpc_timeout; + obrpc::ObCheckLSCanOfflineArg arg; + if (OB_FAIL(arg.init(info.tenant_id_, info.ls_id_, current_ls_status))) { + LOG_WARN("failed to init arg", KR(ret), K(info), K(current_ls_status)); + } else if (OB_FAIL(rpc_proxy_->to(replica->get_server()) + .by(info.tenant_id_) + .timeout(timeout) + .check_ls_can_offline(arg))) { + can_offline = false; + LOG_WARN("failed to check ls can offline", KR(ret), K(arg), K(info), + K(timeout), K(replica)); + } else { + can_offline = true; + } + } + return ret; +} +int ObTenantLSInfo::adjust_user_tenant_primary_zone() +{ + int ret = OB_SUCCESS; + share::ObLSStatusOperator status_op; + share::ObLSPrimaryZoneInfoArray info_array; + ObArray primary_zone; + if (OB_ISNULL(sql_proxy_) || OB_ISNULL(tenant_schema_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error", KR(ret), KP(sql_proxy_), KP(tenant_schema_)); + } else if (OB_FAIL(ObPrimaryZoneUtil::get_tenant_primary_zone_array( + *tenant_schema_, primary_zone))) { + LOG_WARN("failed to get tenant primary zone array", KR(ret), KPC(tenant_schema_)); + } else { + const uint64_t tenant_id = tenant_schema_->get_tenant_id(); + if (OB_FAIL(status_op.get_tenant_primary_zone_info_array( + tenant_id, info_array, *sql_proxy_))) { + LOG_WARN("failed to get tenant primary zone info array", KR(ret), K(tenant_id)); + } + } + + if (OB_SUCC(ret)) { + uint64_t last_ls_group_id = OB_INVALID_ID; + share::ObLSPrimaryZoneInfoArray tmp_info_array; + for (int64_t i = 0; OB_SUCC(ret) && i < info_array.count(); ++i) { + const ObLSPrimaryZoneInfo &info = info_array.at(i); + if (OB_INVALID_ID == last_ls_group_id) { + last_ls_group_id = info.get_ls_group_id(); + } + if (last_ls_group_id != info.get_ls_group_id()) { + //process the ls group + if (OB_FAIL(adjust_primary_zone_by_ls_group_(primary_zone, tmp_info_array, *tenant_schema_))) { + LOG_WARN("failed to update primary zone of each ls group", KR(ret), + K(tmp_info_array), K(primary_zone), KPC(tenant_schema_)); + } else { + tmp_info_array.reset(); + last_ls_group_id = info.get_ls_group_id(); + } + } + if (FAILEDx(tmp_info_array.push_back(info))) { + LOG_WARN("failed to push back primary info array", KR(ret), K(i)); + } + } + if (OB_SUCC(ret) && 0 < tmp_info_array.count()) { + if (OB_FAIL(adjust_primary_zone_by_ls_group_(primary_zone, tmp_info_array, *tenant_schema_))) { + LOG_WARN("failed to update primary zone of each ls group", KR(ret), + K(tmp_info_array), K(primary_zone), KPC(tenant_schema_)); + } + } + } + + return ret; +} + + +int ObTenantLSInfo::try_update_ls_primary_zone_( + const share::ObLSPrimaryZoneInfo &primary_zone_info, + const common::ObZone &new_primary_zone, + const common::ObSqlString &zone_priority) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!primary_zone_info.is_valid() + || new_primary_zone.is_empty() || zone_priority.empty())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("primary zone info is invalid", KR(ret), K(primary_zone_info), + K(new_primary_zone), K(zone_priority)); + } else if (OB_ISNULL(GCTX.sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy is null", KR(ret)); + } else if (new_primary_zone != primary_zone_info.get_primary_zone() + || zone_priority.string() != primary_zone_info.get_zone_priority_str()) { + ObLSLifeAgentManager ls_life_agent(*GCTX.sql_proxy_); + if (OB_FAIL(ls_life_agent.update_ls_primary_zone(primary_zone_info.get_tenant_id(), + primary_zone_info.get_ls_id(), + new_primary_zone, zone_priority.string()))) { + LOG_WARN("failed to update ls primary zone", KR(ret), K(primary_zone_info), + K(new_primary_zone), K(zone_priority)); + } + LOG_INFO("[LS_MGR] update ls primary zone", KR(ret), K(new_primary_zone), + K(zone_priority), K(primary_zone_info)); + } else { + //no need update + } + return ret; +} + +//check every ls has right primary zone in ls group +//if primary_zone of ls not in primary zone, try to choose a right zone, +//if can not find a zone to modify, modify the ls to exist zone, +//load_balancer will choose the right ls to drop +//eg: +//if ls group has z1, z2 and primary zone is : z1. need modify z2 to z1 +//if ls group has z1, z2 and primary zone is : z1, z3. modify z2 to z3; +int ObTenantLSInfo::adjust_primary_zone_by_ls_group_( + const common::ObIArray &primary_zone_array, + const share::ObLSPrimaryZoneInfoArray &primary_zone_infos, + const share::schema::ObTenantSchema &tenant_schema) +{ + int ret = OB_SUCCESS; + const uint64_t tenant_id = tenant_schema.get_tenant_id(); + const int64_t ls_count = primary_zone_infos.count(); + const int64_t primary_zone_count = primary_zone_array.count(); + if (OB_UNLIKELY(0 == primary_zone_count + || 0 == ls_count || !tenant_schema.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(primary_zone_infos), + K(primary_zone_array), K(tenant_schema)); + } else if (OB_ISNULL(GCTX.sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy is null", KR(ret)); + } else if (1 == ls_count + && primary_zone_infos.at(0).get_ls_id().is_sys_ls()) { + //user sys ls is equal to meta sys ls + share::ObLSStatusOperator status_op; + share::ObLSPrimaryZoneInfo meta_primary_zone; + const uint64_t meta_tenant_id = gen_meta_tenant_id(tenant_id); + if (OB_FAIL(status_op.get_ls_primary_zone_info(meta_tenant_id, + SYS_LS, meta_primary_zone, *GCTX.sql_proxy_))) { + LOG_WARN("failed to get ls primary_zone info", KR(ret), K(meta_tenant_id)); + } else if (OB_FAIL(try_update_ls_primary_zone_(primary_zone_infos.at(0), + meta_primary_zone.get_primary_zone(), meta_primary_zone.get_zone_priority()))) { + LOG_WARN("failed to update primary zone", KR(ret), K(primary_zone_infos), K(meta_primary_zone)); + } + } else { + //Algorithm Description: + //Assumption: We have 5 ls, 3 primary_zones(z1, z2, z3). + //1. Set the primary zone of ls to tenant's primary zone, + //choose the least number of log streams on the zone is selected for all zones + //2. After all the primary_zone of the ls are in the primary_zonw of the tenant, + //choose the primary_zone with the most and least ls. Adjust a certain number of ls to the smallest zone without exceeding the balance + //while guaranteeing that the number of the zone with the most is no less than the average. + ObArray ls_primary_zone;//is match with primary_zone_infos + ObSEArray count_group_by_zone;//ls count of each primary zone + ObSqlString new_zone_priority; + if (OB_FAIL(set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, + count_group_by_zone))) { + LOG_WARN("failed to set ls to primary zone", KR(ret), K(primary_zone_array), K(primary_zone_infos)); + } else if (OB_FAIL(balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone))) { + LOG_WARN("failed to balance ls primary zone", KR(ret), K(ls_primary_zone), K(count_group_by_zone)); + } + + for (int64_t i = 0; OB_SUCC(ret) && i < ls_count; ++i) { + const ObZone &new_primary_zone = ls_primary_zone.at(i); + if (OB_FAIL(ObTenantLSInfo::get_zone_priority(new_primary_zone, tenant_schema, new_zone_priority))) { + LOG_WARN("failed to get normalize primary zone", KR(ret), K(new_primary_zone)); + } + if (FAILEDx(try_update_ls_primary_zone_(primary_zone_infos.at(i), new_primary_zone, new_zone_priority))) { + LOG_WARN("failed to update ls primary zone", KR(ret), "primary_zone_info", primary_zone_infos.at(i), + K(new_primary_zone), K(new_zone_priority)); + } + } + } + return ret; +} + +int ObTenantLSInfo::set_ls_to_primary_zone( + const common::ObIArray &primary_zone_array, + const share::ObLSPrimaryZoneInfoArray &primary_zone_infos, + common::ObIArray &ls_primary_zone, + common::ObIArray &count_group_by_zone) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(0 == primary_zone_array.count() + || 0 == primary_zone_infos.count())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(primary_zone_array), K(primary_zone_infos)); + } else { + const int64_t primary_zone_count = primary_zone_array.count(); + const int64_t ls_count = primary_zone_infos.count(); + //ls may not in primary zone, record the index of primary_zone_infos not in primary zone + ObSEArray index_not_primary_zone; + int64_t index = 0; + ARRAY_FOREACH_X(primary_zone_array, idx, cnt, OB_SUCC(ret)) { + if (OB_FAIL(count_group_by_zone.push_back(0))) { + LOG_WARN("failed to push back", KR(ret), K(idx)); + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < ls_count; ++i) { + const ObZone ¤t_zone = primary_zone_infos.at(i).get_primary_zone(); + if (has_exist_in_array(primary_zone_array, current_zone, &index)) { + count_group_by_zone.at(index)++; + } else if (OB_FAIL(index_not_primary_zone.push_back(i))) { + LOG_WARN("failed to push back", KR(ret), K(i), K(current_zone)); + } + if (FAILEDx(ls_primary_zone.push_back(current_zone))) { + LOG_WARN("failed to push back current zone", KR(ret), K(i), K(current_zone)); + } + } + //1. take all ls primary zone to tenant primary zone, choose the less primary zone count + int64_t min_count = INT64_MAX; + int64_t min_index = 0; + for (int64_t i = 0; OB_SUCC(ret) && i < index_not_primary_zone.count(); ++i) { + const int64_t ls_index = index_not_primary_zone.at(i); + min_count = INT64_MAX; + ARRAY_FOREACH_X(primary_zone_array, idx, cnt, OB_SUCC(ret)) { + if (min_count > count_group_by_zone.at(idx)) { + min_count = count_group_by_zone.at(idx); + min_index = idx; + } + }//end for search min count + if (OB_FAIL(ret)) { + } else if (min_index >= primary_zone_count) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to found min count", KR(ret), K(min_index), K(primary_zone_array)); + } else if (OB_FAIL(ls_primary_zone.at(ls_index).assign(primary_zone_array.at(min_index)))) { + LOG_WARN("failed to assign primary zone", KR(ret), K(min_index), K(min_count), K(ls_index)); + } else { + count_group_by_zone.at(min_index)++; + } + } + } + return ret; + +} + +int ObTenantLSInfo::balance_ls_primary_zone( + const common::ObIArray &primary_zone_array, + common::ObIArray &ls_primary_zone, + common::ObIArray &count_group_by_zone) +{ + int ret = OB_SUCCESS; + const int64_t ls_count = ls_primary_zone.count(); + const int64_t primary_zone_count = count_group_by_zone.count(); + if (OB_UNLIKELY(0 == primary_zone_count + || 0 == ls_count + || primary_zone_count != primary_zone_array.count())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(count_group_by_zone), K(ls_primary_zone)); + } else { + int64_t max_count = -1, max_index = 0; + int64_t min_count = INT64_MAX, min_index = 0; + const int64_t each_ls_max_count = ceil((double)(ls_count)/primary_zone_count); + do { + max_count = -1, max_index = 0; + min_count = INT64_MAX, min_index = 0; + for (int64_t i = 0; OB_SUCC(ret) && i < primary_zone_count; ++i) { + const int64_t ls_count = count_group_by_zone.at(i); + if (min_count > ls_count) { + min_count = ls_count; + min_index = i; + } + if (max_count < ls_count) { + max_count = ls_count; + max_index = i; + } + }//end for find min and max count + if (OB_UNLIKELY(max_index >= primary_zone_count || min_index >= primary_zone_count + || -1 == max_count || INT64_MAX == min_count)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get primary zone", KR(ret), K(max_index), K(min_index), + K(min_count), K(max_count), + K(primary_zone_array), K(primary_zone_count), K(ls_primary_zone)); + } else if (max_count - min_count > 1) { + //choose the max count to min count + int64_t need_change = min(each_ls_max_count - min_count, max_count - each_ls_max_count); + for (int64_t i = 0; OB_SUCC(ret) && i < ls_count && need_change > 0; ++i) { + if (ls_primary_zone.at(i) == primary_zone_array.at(max_index)) { + if (OB_FAIL(ls_primary_zone.at(i).assign(primary_zone_array.at(min_index)))) { + LOG_WARN("failed to push back ls primary zone", KR(ret), K(min_index)); + } else { + need_change--; + count_group_by_zone.at(max_index)--; + count_group_by_zone.at(min_index)++; + } + } + } + } + } while (max_count - min_count > 1); + } + return ret; +} + + +//////////////ObTenantThreadHelper +int ObTenantThreadHelper::create( + const char* thread_name, int tg_def_id, ObTenantThreadHelper &tenant_thread) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(is_created_)) { + ret = OB_INIT_TWICE; + LOG_WARN("has inited", KR(ret)); + } else if (OB_ISNULL(thread_name)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("thread name is null", KR(ret)); + } else if (OB_FAIL(TG_CREATE_TENANT(tg_def_id, tg_id_))) { + LOG_ERROR("create tg failed", KR(ret)); + } else if (OB_FAIL(TG_SET_RUNNABLE(tg_id_, *this))) { + LOG_ERROR("set thread runable fail", KR(ret)); + } else if (OB_FAIL(thread_cond_.init(ObWaitEventIds::REENTRANT_THREAD_COND_WAIT))) { + LOG_WARN("fail to init cond, ", KR(ret)); + } else { + thread_name_ = thread_name; + is_created_ = true; + is_first_time_to_start_ = true; + } + return ret; +} + +int ObTenantThreadHelper::start() +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_created_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (is_first_time_to_start_) { + if (OB_FAIL(TG_START(tg_id_))) { + LOG_WARN("fail ed to start at first time", KR(ret), K(tg_id_), K(thread_name_)); + } else { + is_first_time_to_start_ = false; + } + } else if (OB_FAIL(TG_REENTRANT_LOGICAL_START(tg_id_))) { + LOG_WARN("failed to start", KR(ret)); + } + LOG_INFO("[TENANT THREAD] thread start", KR(ret), K(tg_id_), K(thread_name_)); + return ret; +} + +void ObTenantThreadHelper::stop() +{ + LOG_INFO("[TENANT THREAD] thread stop start", K(tg_id_), K(thread_name_)); + if (-1 != tg_id_) { + TG_REENTRANT_LOGICAL_STOP(tg_id_); + } + LOG_INFO("[TENANT THREAD] thread stop finish", K(tg_id_), K(thread_name_)); +} + +void ObTenantThreadHelper::wait() +{ + LOG_INFO("[TENANT THREAD] thread wait start", K(tg_id_), K(thread_name_)); + if (-1 != tg_id_) { + TG_REENTRANT_LOGICAL_WAIT(tg_id_); + } + LOG_INFO("[TENANT THREAD] thread wait finish", K(tg_id_), K(thread_name_)); +} + +void ObTenantThreadHelper::mtl_thread_stop() +{ + LOG_INFO("[TENANT THREAD] thread stop start", K(tg_id_), K(thread_name_)); + if (-1 != tg_id_) { + TG_STOP(tg_id_); + } + LOG_INFO("[TENANT THREAD] thread stop finish", K(tg_id_), K(thread_name_)); +} + +void ObTenantThreadHelper::mtl_thread_wait() +{ + LOG_INFO("[TENANT THREAD] thread wait start", K(tg_id_), K(thread_name_)); + if (-1 != tg_id_) { + { + ObThreadCondGuard guard(thread_cond_); + thread_cond_.broadcast(); + } + TG_WAIT(tg_id_); + is_first_time_to_start_ = true; + } + LOG_INFO("[TENANT THREAD] thread wait finish", K(tg_id_), K(thread_name_)); +} + +void ObTenantThreadHelper::destroy() +{ + LOG_INFO("[TENANT THREAD] thread destory start", K(tg_id_), K(thread_name_)); + if (-1 != tg_id_) { + TG_STOP(tg_id_); + { + ObThreadCondGuard guard(thread_cond_); + thread_cond_.broadcast(); + } + TG_WAIT(tg_id_); + TG_DESTROY(tg_id_); + tg_id_ = -1; + } + is_created_ = false; + is_first_time_to_start_ = true; + LOG_INFO("[TENANT THREAD] thread destory finish", K(tg_id_), K(thread_name_)); +} + +void ObTenantThreadHelper::switch_to_follower_forcedly() +{ + stop(); +} +int ObTenantThreadHelper::switch_to_leader() +{ + int ret = OB_SUCCESS; + LOG_INFO("[TENANT THREAD] thread start", K(tg_id_), K(thread_name_)); + if (OB_FAIL(start())) { + LOG_WARN("failed to start thread", KR(ret)); + } else { + ObThreadCondGuard guard(thread_cond_); + if (OB_FAIL(thread_cond_.broadcast())) { + LOG_WARN("failed to weakup thread cond", KR(ret)); + } + } + LOG_INFO("[TENANT THREAD] thread start finish", K(tg_id_), K(thread_name_)); + return ret; +} +void ObTenantThreadHelper::run1() +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_created_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else { + lib::set_thread_name(thread_name_); + LOG_INFO("thread run", K(thread_name_)); + do_work(); + } +} +void ObTenantThreadHelper::idle(const int64_t idle_time_us) +{ + ObThreadCondGuard guard(thread_cond_); + thread_cond_.wait_us(idle_time_us); +} //////////////ObPrimaryLSService int ObPrimaryLSService::init() @@ -41,7 +2066,7 @@ int ObPrimaryLSService::init() ret = OB_INIT_TWICE; LOG_WARN("has inited", KR(ret)); } else if (OB_FAIL(ObTenantThreadHelper::create("PLSSer", - lib::TGDefIDs::LSService, *this))) { + lib::TGDefIDs::SimpleLSService, *this))) { LOG_WARN("failed to create thread", KR(ret)); } else if (OB_FAIL(ObTenantThreadHelper::start())) { LOG_WARN("fail to start", KR(ret)); @@ -64,49 +2089,134 @@ void ObPrimaryLSService::do_work() if (OB_UNLIKELY(!inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); - } else if (OB_FAIL(wait_tenant_schema_and_version_ready_(tenant_id_, DATA_VERSION_4_1_0_0))) { - LOG_WARN("failed to wait tenant schema version ready", KR(ret), K(tenant_id_), K(DATA_CURRENT_VERSION)); } else { - int64_t idle_time_us = 500 * 1000L; + int64_t idle_time_us = 100 * 1000L; int tmp_ret = OB_SUCCESS; - const uint64_t thread_idx = get_thread_idx(); - share::schema::ObTenantSchema tenant_schema; - palf::PalfBufferIterator iterator; - SCN start_scn; while (!has_set_stop()) { - tenant_schema.reset(); - ObCurTraceId::init(GCONF.self_addr_); - if (0 == thread_idx) { - if (OB_FAIL(get_tenant_schema(tenant_id_, tenant_schema))) { - LOG_WARN("failed to get tenant schema", KR(ret), K(tenant_id_)); - } else if (OB_TMP_FAIL(process_user_tenant_thread0_(tenant_schema))) { - ret = OB_SUCC(ret) ? tmp_ret : ret; - LOG_WARN("failed to process user tenant thread0", KR(ret), - KR(tmp_ret), K(tenant_id_)); - } - } else if (OB_TMP_FAIL( - process_user_tenant_thread1_(iterator, start_scn))) { - ret = OB_SUCC(ret) ? tmp_ret : ret; - LOG_WARN("failed to process user tenant thread1", KR(ret), KR(tmp_ret), - K(tenant_id_), K(start_scn)); - } + idle_time_us = 1000 * 1000L; + { + ObCurTraceId::init(GCONF.self_addr_); + share::schema::ObSchemaGetterGuard schema_guard; + const share::schema::ObTenantSchema *tenant_schema = NULL; + if (OB_ISNULL(GCTX.schema_service_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error", KR(ret)); + } else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard( + OB_SYS_TENANT_ID, schema_guard))) { + LOG_WARN("fail to get schema guard", KR(ret)); + } else if (OB_FAIL(schema_guard.get_tenant_info(tenant_id_, tenant_schema))) { + LOG_WARN("failed to get tenant ids", KR(ret), K(tenant_id_)); + } else if (OB_ISNULL(tenant_schema)) { + ret = OB_TENANT_NOT_EXIST; + LOG_WARN("tenant not exist", KR(ret), K(tenant_id_)); + } else if (!is_user_tenant(tenant_id_)) { + if (OB_SUCCESS != (tmp_ret = process_meta_tenant_(*tenant_schema))) { + ret = OB_SUCC(ret) ? tmp_ret : ret; + LOG_WARN("failed to process tenant", KR(ret), KR(tmp_ret), K(tenant_id_)); + } + //drop tenant, no need process sys tenant, ignore failuer + if (!is_sys_tenant(tenant_id_)) { + const uint64_t user_tenant_id = gen_user_tenant_id(tenant_id_); + if (OB_SUCCESS != (tmp_ret = try_force_drop_tenant_(user_tenant_id))) { + ret = OB_SUCC(ret) ? tmp_ret : ret; + LOG_WARN("failed to drop tenant", KR(ret), KR(tmp_ret), K(user_tenant_id)); + } + } + } else { + if (OB_SUCCESS !=( tmp_ret = process_user_tenant_(*tenant_schema))) { + ret = OB_SUCC(ret) ? tmp_ret : ret; + LOG_WARN("failed to process tenant", KR(ret), KR(tmp_ret), K(tenant_id_)); + } - tenant_schema.reset(); + if (OB_SUCCESS != (tmp_ret = report_sys_ls_recovery_stat_())) { + //ignore error of report, no need wakeup + LOG_WARN("failed to report sys ls recovery stat", KR(ret), KR(tmp_ret), K(tenant_id_)); + } + } + }//for schema guard, must be free + if (OB_FAIL(ret)) { + idle_time_us = 100 * 1000; + } idle(idle_time_us); - LOG_INFO("[PRIMARY_LS_SERVICE] finish one round", KR(ret), K(idle_time_us), K(thread_idx)); - } // end while + LOG_INFO("[LS_SER] finish one round", KR(ret), K(idle_time_us)); + }// end while } } -int ObPrimaryLSService::process_user_tenant_thread0_(const share::schema::ObTenantSchema &tenant_schema) +//meta tenant no need process create_new_ls or drop ls +//only need to agjust primary zone of sys_ls +int ObPrimaryLSService::process_meta_tenant_(const share::schema::ObTenantSchema &tenant_schema) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (OB_UNLIKELY(!tenant_schema.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tenant schema is invalid", KR(ret), K(tenant_schema)); + } else if (OB_ISNULL(GCTX.sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy is null", KR(ret)); + } else if (!share::schema::ObSchemaService::is_formal_version(tenant_schema.get_schema_version()) + || !tenant_schema.is_normal()) { + } else if (tenant_schema.is_dropping()) { + } else { + share::ObLSPrimaryZoneInfo primary_zone_info; + ObArray primary_zone; + share::ObLSStatusOperator status_op; + ObZone new_primary_zone; + ObSqlString new_zone_priority; + if (OB_FAIL(ObPrimaryZoneUtil::get_tenant_primary_zone_array( + tenant_schema, primary_zone))) { + LOG_WARN("failed to get tenant primary zone array", KR(ret), K(tenant_schema)); + } else if (OB_FAIL(status_op.get_ls_primary_zone_info(tenant_id_, + SYS_LS, primary_zone_info, *GCTX.sql_proxy_))) { + LOG_WARN("failed to get ls primary_zone info", KR(ret), K(tenant_id_)); + } else if (OB_UNLIKELY(0 == primary_zone.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("primary zone is empty", KR(ret), K(tenant_schema)); + } else if (has_exist_in_array(primary_zone, primary_zone_info.get_primary_zone())) { + if (OB_FAIL(new_primary_zone.assign(primary_zone_info.get_primary_zone()))) { + LOG_WARN("failed to assign primary zone", KR(ret), K(primary_zone_info)); + } + } else if (OB_FAIL(new_primary_zone.assign(primary_zone.at(0)))) { + LOG_WARN("failed to assign primary zone", KR(ret), K(primary_zone)); + } + if (OB_FAIL(ret)) { + } else if (is_sys_tenant(tenant_id_)) { + //sys tenant use tenant normalize primary zone + if (OB_FAIL(ObPrimaryZoneUtil::get_tenant_zone_priority( + tenant_schema, new_zone_priority))) { + LOG_WARN("failed to get tenant primary zone array", KR(ret), K(tenant_schema)); + } + } else if (OB_FAIL(ObTenantLSInfo::get_zone_priority(new_primary_zone, tenant_schema, new_zone_priority))) { + LOG_WARN("failed to get normalize primary zone", KR(ret), K(new_primary_zone)); + } + if (FAILEDx(ObTenantLSInfo::try_update_ls_primary_zone_( + primary_zone_info, new_primary_zone, new_zone_priority))) { + LOG_WARN("failed to update ls primary zone", KR(ret), K(primary_zone_info), + K(new_primary_zone), K(new_zone_priority)); + } + + if (OB_SUCC(ret) && !is_sys_tenant(tenant_id_)) { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = gather_tenant_recovery_stat_())) { + ret = OB_SUCC(ret) ? tmp_ret : ret; + LOG_WARN("failed to gather tenant recovery stat", KR(ret), KR(tmp_ret)); + } + } + } + return ret; +} + +int ObPrimaryLSService::process_user_tenant_(const share::schema::ObTenantSchema &tenant_schema) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; const uint64_t tenant_id = tenant_schema.get_tenant_id(); ObTenantLSInfo tenant_stat(GCTX.sql_proxy_, &tenant_schema, tenant_id, GCTX.srv_rpc_proxy_, GCTX.lst_operator_); - int64_t task_cnt = 0; if (OB_UNLIKELY(!inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", KR(ret)); @@ -120,61 +2230,159 @@ int ObPrimaryLSService::process_user_tenant_thread0_(const share::schema::ObTena //if tenant schema is in dropping //set the creating ls to create_abort, //set the normal or dropping tenant to drop_tennat_pre - if (OB_FAIL(tenant_stat.drop_tenant())) { + if (OB_FAIL(tenant_stat.drop_tenant(share::NORMAL_SWITCHOVER_STATUS))) { LOG_WARN("failed to drop tenant", KR(ret), K(tenant_id)); - } else if (OB_FAIL(tenant_stat.process_next_ls_status(task_cnt))) { - LOG_WARN("failed to process next ls status", KR(ret), KR(tmp_ret)); + } else if (OB_FAIL(tenant_stat.process_ls_status_missmatch(false/* lock_sys_ls */, + share::NORMAL_SWITCHOVER_STATUS))) { + LOG_WARN("failed to process ls status missmatch", KR(ret), KR(tmp_ret)); } } else { //normal tenant //some ls may failed to create ls, but can continue - if (OB_SUCCESS != (tmp_ret = tenant_stat.process_next_ls_status(task_cnt))) { + if (OB_SUCCESS != (tmp_ret = tenant_stat.process_ls_status_missmatch(false/* lock_sys_ls */, + share::NORMAL_SWITCHOVER_STATUS))) { ret = OB_SUCC(ret) ? tmp_ret : ret; - LOG_WARN("failed to process next ls status", KR(ret), KR(tmp_ret)); - } else if (0 == task_cnt) { - //process each ls group and primary zone is matching - //process each unit group has the right ls group - //overwrite ret - if (OB_SUCCESS != (tmp_ret = tenant_stat.gather_stat(false))) { + LOG_WARN("failed to process ls status missmatch", KR(ret), KR(tmp_ret)); + } + + //process each ls group and primary zone is matching + //process each unit group has the right ls group + //overwrite ret + if (OB_SUCCESS != (tmp_ret = tenant_stat.gather_stat(false))) { + ret = OB_SUCC(ret) ? tmp_ret : ret; + LOG_WARN("failed to gather stat", KR(ret), KR(tmp_ret), K(tenant_id)); + } else { + if (OB_SUCCESS != (tmp_ret = tenant_stat.check_ls_match_primary_zone())) { ret = OB_SUCC(ret) ? tmp_ret : ret; - LOG_WARN("failed to gather stat", KR(ret), KR(tmp_ret), K(tenant_id)); - } else { - if (OB_SUCCESS != (tmp_ret = tenant_stat.check_ls_match_primary_zone())) { - ret = OB_SUCC(ret) ? tmp_ret : ret; - LOG_WARN("failed to check ls match with primary zone", KR(ret), KR(tmp_ret)); - } - if (OB_SUCCESS != (tmp_ret = tenant_stat.check_ls_group_match_unitnum())) { - ret = OB_SUCC(ret) ? tmp_ret : ret; - LOG_WARN("failed to check ls group match unitnum", KR(ret), KR(tmp_ret)); - } + LOG_WARN("failed to check ls match with primary zone", KR(ret), KR(tmp_ret)); + } + if (OB_SUCCESS != (tmp_ret = tenant_stat.check_ls_group_match_unitnum())) { + ret = OB_SUCC(ret) ? tmp_ret : ret; + LOG_WARN("failed to check ls group match unitnum", KR(ret), KR(tmp_ret)); } } - } - - LOG_INFO("[PRIMARY_LS_SERVICE_THREAD0] finish process tenant", - KR(ret), KR(tmp_ret), K(tenant_id), K(task_cnt), K(tenant_schema)); - return ret; -} - -int ObPrimaryLSService::process_user_tenant_thread1_(palf::PalfBufferIterator &iterator, - share::SCN &start_scn) -{ - int ret = OB_SUCCESS; - if (!is_user_tenant(tenant_id_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls recovery thread must run on user tenant", KR(ret), - K(tenant_id_)); - } else { - ObRecoveryLSHelper recovery_ls(tenant_id_, GCTX.sql_proxy_); - if (OB_FAIL(recovery_ls.do_work(iterator, start_scn))) { - LOG_WARN("failed to recovery ls", KR(ret), K(start_scn)); + if (OB_SUCCESS != (tmp_ret = tenant_stat.adjust_user_tenant_primary_zone())) { + //ignore error + LOG_WARN("failed to adjust user tenant primary zone", KR(ret), KR(tmp_ret)); } } - LOG_INFO("[PRIMARY_LS_SERVICE_THREAD1] finish process tenant", KR(ret), K(start_scn)); + + LOG_INFO("finish process tenant", KR(ret), KR(tmp_ret), K(tenant_id), K(tenant_schema)); return ret; } -}//end of rootserver + +int ObPrimaryLSService::report_sys_ls_recovery_stat_() +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret), K(inited_)); + } else { + ObLSService *ls_svr = MTL(ObLSService *); + ObLSHandle ls_handle; + if (OB_FAIL(ls_svr->get_ls(SYS_LS, ls_handle, storage::ObLSGetMod::RS_MOD))) { + LOG_WARN("failed to get ls", KR(ret)); + } else if (OB_FAIL(ObTenantRecoveryReportor::update_ls_recovery(ls_handle.get_ls(), GCTX.sql_proxy_))) { + LOG_WARN("failed to update ls recovery", KR(ret)); + } + } + return ret; +} + +int ObPrimaryLSService::try_force_drop_tenant_( + const uint64_t user_tenant_id) +{ + int ret = OB_SUCCESS; + ObTimeoutCtx ctx; + share::schema::ObSchemaGetterGuard schema_guard; + const share::schema::ObTenantSchema *tenant_schema = NULL; + + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (OB_ISNULL(GCTX.sql_proxy_) + || OB_ISNULL(GCTX.rs_rpc_proxy_) + || OB_ISNULL(GCTX.schema_service_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ptr is null", KR(ret), KP(GCTX.sql_proxy_), KP(GCTX.rs_rpc_proxy_), + KP(GCTX.schema_service_)); + } else if (OB_UNLIKELY(!is_user_tenant(user_tenant_id))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("not user tenant", KR(ret), K(user_tenant_id)); + } else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard( + OB_SYS_TENANT_ID, schema_guard))) { + LOG_WARN("fail to get schema guard", KR(ret)); + } else if (OB_FAIL(schema_guard.get_tenant_info(user_tenant_id, tenant_schema))) { + LOG_WARN("failed to get tenant ids", KR(ret), K(user_tenant_id)); + } else if (OB_ISNULL(tenant_schema)) { + ret = OB_TENANT_NOT_EXIST; + LOG_WARN("tenant not exist", KR(ret), K(user_tenant_id)); + } else if (!tenant_schema->is_dropping()) { + } else { + LOG_INFO("try drop tenant", K(user_tenant_id)); + ObLSStatusOperator op; + share::ObLSStatusInfoArray ls_array; + if (OB_FAIL(ObShareUtil::set_default_timeout_ctx(ctx, GCONF.rpc_timeout))) { + LOG_WARN("fail to set timeout ctx", KR(ret), K(user_tenant_id)); + } else if (OB_FAIL(op.get_all_ls_status_by_order(user_tenant_id, ls_array, *GCTX.sql_proxy_))) { + LOG_WARN("fail to get all ls status", KR(ret), K(user_tenant_id)); + } else if (ls_array.count() <= 0) { + obrpc::ObDropTenantArg arg; + arg.exec_tenant_id_ = OB_SYS_TENANT_ID; + arg.tenant_name_ = tenant_schema->get_tenant_name(); + arg.tenant_id_ = tenant_schema->get_tenant_id(); + arg.if_exist_ = true; + arg.delay_to_drop_ = false; + ObSqlString sql; + const int64_t timeout_ts = ctx.get_timeout(); + if (OB_FAIL(sql.append_fmt("DROP TENANT IF EXISTS %s FORCE", arg.tenant_name_.ptr()))) { + LOG_WARN("fail to generate sql", KR(ret), K(arg)); + } else if (FALSE_IT(arg.ddl_stmt_str_ = sql.string())) { + } else if (OB_FAIL(GCTX.rs_rpc_proxy_->timeout(timeout_ts).drop_tenant(arg))) { + LOG_WARN("fail to drop tenant", KR(ret), K(arg), K(timeout_ts)); + } + } else { + // tenant's logstream is still dropping, check next round + } + } + return ret; +} + +int ObPrimaryLSService::gather_tenant_recovery_stat_() +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (is_sys_tenant(tenant_id_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sys tenant no need gather tenant recovery stat", KR(ret), K(tenant_id_)); + } else if (OB_ISNULL(GCTX.sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy is null", KR(ret)); + } else { + ObLSRecoveryStatOperator ls_recovery_op; + ObAllTenantInfoProxy info_proxy; + ObAllTenantInfo tenant_info; + const uint64_t user_tenant_id = gen_user_tenant_id(tenant_id_); + SCN sync_scn; + SCN min_wrs_scn; + DEBUG_SYNC(BLOCK_TENANT_SYNC_SNAPSHOT_INC); + if (OB_FAIL(ls_recovery_op.get_tenant_recovery_stat( + user_tenant_id, *GCTX.sql_proxy_, sync_scn, min_wrs_scn))) { + LOG_WARN("failed to get tenant recovery stat", KR(ret), K(user_tenant_id)); + //TODO replayable_scn is equal to sync_scn + } else if (OB_FAIL(info_proxy.update_tenant_recovery_status( + user_tenant_id, GCTX.sql_proxy_, share::NORMAL_SWITCHOVER_STATUS, sync_scn, + sync_scn, min_wrs_scn))) { + LOG_WARN("failed to update tenant recovery stat", KR(ret), + K(user_tenant_id), K(sync_scn), K(min_wrs_scn)); + } + } + return ret; +} +} } diff --git a/src/rootserver/ob_primary_ls_service.h b/src/rootserver/ob_primary_ls_service.h index f370d968f5..f6af3c1642 100644 --- a/src/rootserver/ob_primary_ls_service.h +++ b/src/rootserver/ob_primary_ls_service.h @@ -18,11 +18,9 @@ #include "share/ls/ob_ls_status_operator.h"//ObLSStatusOperator #include "share/ls/ob_ls_operator.h" //ObLSAttr #include "share/ob_thread_mgr.h" //OBTGDefIDEnum -#include "logservice/palf/palf_iterator.h" //PalfBufferIterator #include "share/unit/ob_unit_info.h"//ObUnit::Status #include "lib/thread/thread_mgr_interface.h" // TGRunnable #include "lib/lock/ob_thread_cond.h"//ObThreadCond -#include "rootserver/ob_tenant_thread_helper.h"//ObTenantThreadHelper namespace oceanbase @@ -35,12 +33,10 @@ namespace common { class ObMySQLProxy; class ObMySQLTransaction; -class ObClusterVersion; } namespace share { class ObLSTableOperator; -class SCN; namespace schema { class ObMultiVersionSchemaService; @@ -54,6 +50,284 @@ struct PalfBaseInfo; } namespace rootserver { +struct ObUnitGroupInfo +{ + ObUnitGroupInfo() : unit_group_id_(OB_INVALID_ID), unit_status_(share::ObUnit::UNIT_STATUS_MAX), + ls_group_ids_() {} + virtual ~ObUnitGroupInfo() {} + bool is_valid() const; + int init(const uint64_t unit_group_id, + const share::ObUnit::Status &unit_status); + void reset(); + int assign(const ObUnitGroupInfo &other); + int remove_ls_group(const uint64_t ls_group_id); + bool operator==(const ObUnitGroupInfo &other) const; + + uint64_t unit_group_id_; + share::ObUnit::Status unit_status_; + ObArray ls_group_ids_; + TO_STRING_KV(K_(unit_group_id), K_(unit_status), K_(ls_group_ids)); +}; +typedef ObArray ObUnitGroupInfoArray; +typedef ObIArray ObUnitGroupInfoIArray; + +struct ObLSGroupInfo +{ + ObLSGroupInfo() : ls_group_id_(OB_INVALID_ID), unit_group_id_(OB_INVALID_ID), + ls_ids_() {} + virtual ~ObLSGroupInfo() {} + bool is_valid() const; + int init(const uint64_t unit_group_id, const uint64_t ls_group_id); + int assign(const ObLSGroupInfo &other); + void reset(); + int remove_ls(const share::ObLSID &ls_id); + uint64_t ls_group_id_; + uint64_t unit_group_id_; + ObArray ls_ids_; + TO_STRING_KV(K_(ls_group_id), K_(unit_group_id), K_(ls_ids)); +}; + +typedef ObArray ObLSGroupInfoArray; +typedef ObIArray ObLSGroupInfoIArray; + +struct ObLSStatusMachineParameter +{ + ObLSStatusMachineParameter() : ls_id_(), status_info_(), ls_info_(share::OB_LS_EMPTY) {} + virtual ~ObLSStatusMachineParameter() {} + bool is_valid() const + { + return ls_id_.is_valid() + && (share::OB_LS_EMPTY == status_info_.status_ + || status_info_.ls_id_ == ls_id_); + } + int init(const share::ObLSID &id, const share::ObLSStatusInfo &status_info, + const share::ObLSStatus &ls_info); + void reset(); + share::ObLSID ls_id_; + share::ObLSStatusInfo status_info_;//for create ls and status of __all_ls_status + share::ObLSStatus ls_info_;//status in __all_ls + TO_STRING_KV(K_(ls_id), K_(status_info), K_(ls_info)); +}; + +/*descripthin: Tenant log stream status information: Statistical log stream + * status on __all_ls_status and __all_ls, tenant primary_zone and unit_num + * information. Provides location information for the newly created log stream. + * Whether the build needs to create or delete log streams.*/ +class ObTenantLSInfo +{ +public: + ObTenantLSInfo(ObMySQLProxy *sql_proxy, + const share::schema::ObTenantSchema *tenant_schema, + const uint64_t tenant_id, obrpc::ObSrvRpcProxy *rpc_proxy, + share::ObLSTableOperator *lst_operator) + : sql_proxy_(sql_proxy), + tenant_schema_(tenant_schema), + status_operator_(), + ls_operator_(tenant_id, sql_proxy), + status_array_(), + unit_group_array_(), + ls_group_array_(), + primary_zone_(), + rpc_proxy_(rpc_proxy), + lst_operator_(lst_operator), + max_ls_id_(OB_INVALID_ID), + max_ls_group_id_(OB_INVALID_ID) {} + + virtual ~ObTenantLSInfo(){}; + void reset(); + bool is_valid() const; + + int gather_stat(bool for_recovery); + //some ls not in __all_ls, but in __all_ls_status + //it need delete by gc, no need process it anymore. + //check the ls status, and delete no need ls + int process_ls_status_missmatch( + const bool lock_sys_ls, + const share::ObTenantSwitchoverStatus &working_sw_status); + + // need create or delete new ls group + int check_ls_group_match_unitnum(); + + //need create or delete new ls + int check_ls_match_primary_zone(); + + int fetch_new_ls_group_id(const uint64_t tenant_id, uint64_t &ls_group_id); + int fetch_new_ls_id(const uint64_t tenant_id, share::ObLSID &ls_id); + + //get ls group info from ls_group_array_ by ls_group_id + //the interface must used after gather_stat(); + int get_ls_group_info(const uint64_t ls_group_id, ObLSGroupInfo &info) const; + + //get ls status info from status_array_ by ls_id + //the interface must used after gather_stat(); + int get_ls_status_info(const share::ObLSID &id, share::ObLSStatusInfo &info, + int64_t &info_index) const; + //process dropping tenant, set status to tenant_dropping in __all_ls + int drop_tenant(const share::ObTenantSwitchoverStatus &working_sw_status); + //for recovery tenant, create new ls according to ls_id and ls_group_id + int create_new_ls_for_recovery(const share::ObLSID &ls_id, + const uint64_t ls_group_id, + const share::SCN &create_scn, + common::ObMySQLTransaction &trans); + //for recovery tenant, if ls is in creating in __all_ls_status, create the ls + int process_ls_stats_for_recovery(); + + int adjust_user_tenant_primary_zone(); + + /* + * description: create ls with palf base info for recovery + @param[in] info: status info in __all_ls_status_info + @param[in] create_scn : create scn of ls + @param[in] create_ls_with_palf: restore create init ls + @param[in] palf_base_info : palf base info + */ + int create_ls_with_palf(const share::ObLSStatusInfo &info, const share::SCN &create_scn, + const bool create_ls_with_palf, + const palf::PalfBaseInfo &palf_base_info); + + static int get_zone_priority(const ObZone &primary_zone, + const share::schema::ObTenantSchema &tenant_schema, + ObSqlString &primary_zone_str); + static int try_update_ls_primary_zone_(const share::ObLSPrimaryZoneInfo &primary_zone_info, + const common::ObZone &new_primary_zone, + const common::ObSqlString &zone_priority); + static int construct_ls_status_machine_( + const share::ObLSStatusInfoIArray &statua_info_array, + const share::ObLSAttrIArray &ls_array, + common::ObIArray &status_machine_array); + +private: + int fix_ls_status_(const ObLSStatusMachineParameter &status_machine, + const share::ObTenantSwitchoverStatus &working_sw_status); + // get from __all_ls_status and __all_ls + int gather_all_ls_info_(); + + // base on status_array construct ls_array + int add_ls_to_ls_group_(const share::ObLSStatusInfo &info); + + // base on ls_array construct unit_group_array and __all_unit + int add_ls_group_to_unit_group_(const ObLSGroupInfo &group_info); + + int process_creating_ls_(const share::ObLSAttr &ls_operation); + + int check_unit_group_valid_(const uint64_t unit_group_id, bool &is_valid); + int add_ls_status_info_(const share::ObLSStatusInfo &ls_info); + int create_new_ls_for_empty_unit_group_(const uint64_t unit_group_id); + + int get_next_unit_group_(const bool is_recovery, int64_t &group_index); + // get the primary zone not in ls group + int get_next_primary_zone_(const bool is_recovery, const ObLSGroupInfo &group_info, + ObZone &primary_zone); + int create_new_ls_(const share::ObLSStatusInfo &ls_info, + const share::ObTenantSwitchoverStatus &working_sw_status); + + // drop ls group info + int try_drop_ls_of_deleting_unit_group_(const ObUnitGroupInfo &info); + int drop_ls_(const share::ObLSStatusInfo &ls_info, + const share::ObTenantSwitchoverStatus &working_sw_status); + int do_drop_ls_(const share::ObLSStatusInfo &ls_info, + const share::ObTenantSwitchoverStatus &working_sw_status); + int do_tenant_drop_ls_(const share::ObLSStatusInfo &ls_info, + const share::ObTenantSwitchoverStatus &working_sw_status); + int do_create_ls_(const share::ObLSStatusInfo &info, const share::SCN &create_scn); + int sys_ls_tenant_drop_(const share::ObLSStatusInfo &info, + const share::ObTenantSwitchoverStatus &working_sw_status); + int check_sys_ls_can_offline_(bool &can_offline); + int check_ls_empty_(const share::ObLSStatusInfo &info, bool &empty); + int check_ls_can_offline_by_rpc_(const share::ObLSStatusInfo &info, + const share::ObLSStatus ¤t_ls_status, + bool &can_offline); + int process_ls_status_after_created_(const share::ObLSStatusInfo &info, + const share::ObTenantSwitchoverStatus &working_sw_status); + +private: + static int set_ls_to_primary_zone(const common::ObIArray &primary_zone_array, + const share::ObLSPrimaryZoneInfoArray &primary_zone_infos, + common::ObIArray &ls_primary_zone, + common::ObIArray &count_group_by_zone); + static int balance_ls_primary_zone(const common::ObIArray &primary_zone_array, + common::ObIArray &ls_primary_zone, + common::ObIArray &count_group_by_zone); + int adjust_primary_zone_by_ls_group_(const common::ObIArray &primary_zone_array, + const share::ObLSPrimaryZoneInfoArray &primary_zone_infos, + const share::schema::ObTenantSchema &tenant_schema); + +private: + ObMySQLProxy *sql_proxy_; + const share::schema::ObTenantSchema *tenant_schema_; + share::ObLSStatusOperator status_operator_; + share::ObLSAttrOperator ls_operator_; + bool is_load_; + share::ObLSStatusInfoArray status_array_; + common::hash::ObHashMap status_map_; + ObUnitGroupInfoArray unit_group_array_; + ObLSGroupInfoArray ls_group_array_; + ObArray primary_zone_; + obrpc::ObSrvRpcProxy *rpc_proxy_; + share::ObLSTableOperator *lst_operator_; + //ensure ls can not be concurrent created + uint64_t max_ls_id_; + uint64_t max_ls_group_id_; +}; + +class ObTenantThreadHelper : public lib::TGRunnable, + public logservice::ObIRoleChangeSubHandler +{ +public: + ObTenantThreadHelper() : tg_id_(-1), thread_cond_(), is_created_(false), is_first_time_to_start_(true), thread_name_("") {} + virtual ~ObTenantThreadHelper() {} + virtual void do_work() = 0; + virtual void run1() override; + virtual void destroy(); + int start(); + void stop(); + void wait(); + void mtl_thread_stop(); + void mtl_thread_wait(); + int create(const char* thread_name, int tg_def_id, ObTenantThreadHelper &tenant_thread); + void idle(const int64_t idle_time_us); +public: + virtual void switch_to_follower_forcedly() override; + + virtual int switch_to_leader() override; + virtual int switch_to_follower_gracefully() override + { + stop(); + return OB_SUCCESS; + } + virtual int resume_leader() override + { + return OB_SUCCESS; + } + +#define DEFINE_MTL_FUNC(TYPE)\ + static int mtl_init(TYPE *&ka) {\ + int ret = OB_SUCCESS;\ + if (OB_ISNULL(ka)) {\ + ret = OB_ERR_UNEXPECTED;\ + } else if (OB_FAIL(ka->init())) {\ + }\ + return ret;\ + }\ + static void mtl_stop(TYPE *&ka) {\ + if (OB_NOT_NULL(ka)) {\ + ka->mtl_thread_stop();\ + }\ + }\ + static void mtl_wait(TYPE *&ka) {\ + if (OB_NOT_NULL(ka)) {\ + ka->mtl_thread_wait();\ + }\ + } + +protected: + int tg_id_; +private: + common::ObThreadCond thread_cond_; + bool is_created_; + bool is_first_time_to_start_; + const char* thread_name_; +}; /*description: *Log stream management thread: Started on the leader of the system log stream @@ -90,9 +364,16 @@ public: private: - int process_user_tenant_thread0_(const share::schema::ObTenantSchema &tenant_schema); - int process_user_tenant_thread1_(palf::PalfBufferIterator &iterator, - share::SCN &start_scn); + int process_user_tenant_(const share::schema::ObTenantSchema &tenant_schema); + //no need create ls or drop ls, but need adjust primary_zone + int process_meta_tenant_(const share::schema::ObTenantSchema &tenant_schema); + //for primary cluster, sys ls recovery stat need report, + //standby cluster will be reported in RecoveryLSService + int report_sys_ls_recovery_stat_(); + // force drop user tenant if tenant is in dropping status + int try_force_drop_tenant_( + const uint64_t user_tenant_id); + int gather_tenant_recovery_stat_(); private: bool inited_; uint64_t tenant_id_; diff --git a/src/rootserver/ob_recovery_ls_service.cpp b/src/rootserver/ob_recovery_ls_service.cpp index cdbbedeb47..aa24c294ce 100644 --- a/src/rootserver/ob_recovery_ls_service.cpp +++ b/src/rootserver/ob_recovery_ls_service.cpp @@ -20,16 +20,28 @@ #include "logservice/ob_log_handler.h" //ObLogHandler #include "logservice/palf/log_entry.h" //LogEntry #include "logservice/palf/log_define.h" -#include "logservice/ob_log_service.h" #include "share/scn.h"//SCN +#include "logservice/ob_garbage_collector.h"//ObGCLSLog +#include "logservice/restoreservice/ob_log_restore_handler.h"//ObLogRestoreHandler #include "observer/ob_server_struct.h" //GCTX +#include "rootserver/ob_primary_ls_service.h" //ObTenantLSInfo +#include "rootserver/ob_tenant_recovery_reportor.h" //ObTenantRecoveryReportor +#include "rootserver/ob_tenant_info_loader.h" // ObTenantInfoLoader +#include "share/ls/ob_ls_life_manager.h" //ObLSLifeManger +#include "share/ls/ob_ls_operator.h" //ObLSAttr +#include "share/ls/ob_ls_recovery_stat_operator.h" //ObLSRecoveryLSStatOperator +#include "share/ob_errno.h" #include "share/ob_share_util.h" //ObShareUtil #include "share/schema/ob_multi_version_schema_service.h" //ObMultiSchemaService +#include "share/restore/ob_physical_restore_info.h" //restore_status +#include "share/restore/ob_physical_restore_table_operator.h"//ObPhysicalRestoreTableOperator +#include "share/ob_primary_standby_service.h" // ObPrimaryStandbyService #include "share/ob_standby_upgrade.h" // ObStandbyUpgrade #include "share/ob_upgrade_utils.h" // ObUpgradeChecker #include "share/ob_global_stat_proxy.h" // ObGlobalStatProxy -#include "share/ob_cluster_version.h"//GET_TENANT_DATA_VERSION -#include "rootserver/ob_ls_service_helper.h"//ObRecoveryLSHelper +#include "storage/tx/ob_tx_log.h" //ObTxLogHeader +#include "storage/tx_storage/ob_ls_service.h" //ObLSService +#include "storage/tx_storage/ob_ls_handle.h" //ObLSHandle namespace oceanbase { @@ -48,7 +60,7 @@ int ObRecoveryLSService::init() ret = OB_INIT_TWICE; LOG_WARN("has inited", KR(ret)); } else if (OB_FAIL(ObTenantThreadHelper::create("RecLSSer", - lib::TGDefIDs::SimpleLSService, *this))) { + lib::TGDefIDs::LSService, *this))) { LOG_WARN("failed to create thread", KR(ret)); } else if (OB_FAIL(ObTenantThreadHelper::start())) { LOG_WARN("failed to start thread", KR(ret)); @@ -76,40 +88,318 @@ void ObRecoveryLSService::do_work() if (OB_UNLIKELY(!inited_) || OB_ISNULL(proxy_)) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret), K(inited_), KP(proxy_)); - } else if (OB_FAIL(wait_tenant_data_version_ready_(tenant_id_, DATA_VERSION_4_1_0_0))) { - LOG_WARN("failed to wait tenant and version ready", KR(ret), K(tenant_id_)); } else { + ObLSRecoveryStatOperator ls_recovery; palf::PalfBufferIterator iterator; - int64_t idle_time_us = 100 * 1000L; + int tmp_ret = OB_SUCCESS; + const int64_t idle_time_us = 100 * 1000L; SCN start_scn; - while (!has_set_stop()) { ObCurTraceId::init(GCONF.self_addr_); - // two thread for seed log and recovery_ls_manager + uint64_t thread_idx = get_thread_idx(); + ObTenantInfoLoader *tenant_info_loader = MTL(ObTenantInfoLoader*); + ObAllTenantInfo tenant_info; + //two thread for seed log and recovery_ls_manager if (!is_user_tenant(tenant_id_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("ls recovery thread must run on user tenant", KR(ret), K(tenant_id_)); + } else if (OB_ISNULL(tenant_info_loader)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant report is null", KR(ret), K(tenant_id_)); + } else if (OB_FAIL(tenant_info_loader->get_tenant_info(tenant_info))) { + LOG_WARN("failed to get tenant info", KR(ret)); + } else if (OB_FAIL(check_can_do_recovery_(tenant_info))) { + LOG_WARN("failed to check do recovery", KR(ret), K(tenant_info)); + } else if (0 == thread_idx) { + if (OB_SUCCESS != (tmp_ret = process_recovery_ls_manager())) { + ret = OB_SUCC(ret) ? tmp_ret : ret; + LOG_WARN("failed to process recovery ls manager", KR(ret), KR(tmp_ret)); + } } else { - ObRecoveryLSHelper recovery_ls(tenant_id_, proxy_); - if (OB_FAIL(recovery_ls.do_work(iterator, start_scn))) { - LOG_WARN("failed to recovery ls", KR(ret), K(start_scn)); + if (!start_scn.is_valid()) { + ObLSRecoveryStat ls_recovery_stat; + if (OB_FAIL(ls_recovery.get_ls_recovery_stat(tenant_id_, + SYS_LS, false, ls_recovery_stat, *proxy_))) { + LOG_WARN("failed to load sys recovery stat", KR(ret), K(tenant_id_)); + } else if (SCN::base_scn() == ls_recovery_stat.get_sync_scn()) { + //等待物理恢复设置完系统日志流初始同步点之后开始迭代 + ret = OB_EAGAIN; + LOG_WARN("restore init sync scn has not setted, need wait", KR(ret), K(ls_recovery_stat)); + } else if (OB_FAIL(report_sys_ls_recovery_stat_(ls_recovery_stat.get_sync_scn()))) { + //may recovery end, but readable scn need report + LOG_WARN("failed to report ls recovery stat", KR(ret), K(ls_recovery_stat)); + } else if (tenant_info.get_recovery_until_scn() == ls_recovery_stat.get_sync_scn()) { + ret = OB_EAGAIN; + if (REACH_TIME_INTERVAL(60 * 1000 * 1000)) { // every minute + LOG_INFO("has recovered to recovery_until_scn", KR(ret), K(ls_recovery_stat), K(tenant_info)); + } + } else if (OB_FAIL(seek_log_iterator_(ls_recovery_stat.get_sync_scn(), iterator))) { + LOG_WARN("failed to seek log iterator", KR(ret), K(ls_recovery_stat)); + } else { + start_scn = ls_recovery_stat.get_sync_scn(); + LOG_INFO("start to seek at", K(start_scn)); + } } - if (OB_ITER_STOP == ret) { - //standby or recovery may restore finish, - //so when the error code is OB_ITER_STOP, adjust idle_time to 10s - idle_time_us = 10 * 1000 * 1000;//10s + if (OB_FAIL(ret)) { + } else if (start_scn.is_valid() && OB_FAIL(process_ls_log_(tenant_info, start_scn, iterator))) { + if (OB_ITER_STOP != ret) { + LOG_WARN("failed to process ls log", KR(ret), K(start_scn), K(tenant_info)); + } + } + if (OB_FAIL(ret)) { + start_scn.reset(); + } + + if (REACH_TIME_INTERVAL(10 * 1000 * 1000)) { // every 10 second + (void)try_tenant_upgrade_end_(); } } - if (REACH_TENANT_TIME_INTERVAL(10 * 1000 * 1000)) { // every 10 second - (void)try_tenant_upgrade_end_(); - } - LOG_INFO("[RECOVERY_LS_SERVICE] finish one round", KR(ret), - K(start_scn)); + LOG_INFO("[LS_RECOVERY] finish one round", KR(ret), KR(tmp_ret), K(start_scn), K(thread_idx), K(tenant_info)); idle(idle_time_us); } } } +int ObRecoveryLSService::seek_log_iterator_(const SCN &sync_scn, PalfBufferIterator &iterator) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret), K(inited_)); + } else if (OB_UNLIKELY(!sync_scn.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("sync scn is invalid", KR(ret), K(sync_scn)); + } else { + ObLSService *ls_svr = MTL(ObLSService *); + ObLSHandle ls_handle; + if (OB_FAIL(ls_svr->get_ls(SYS_LS, ls_handle, storage::ObLSGetMod::RS_MOD))) { + LOG_WARN("failed to get ls", KR(ret)); + } else { + ObLogHandler *log_handler = NULL; + ObLS *ls = NULL; + palf::LSN start_lsn; + if (OB_ISNULL(ls = ls_handle.get_ls()) + || OB_ISNULL(log_handler = ls->get_log_handler())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls or log handle is null", KR(ret), KP(ls), + KP(log_handler)); + } else if (OB_FAIL(log_handler->locate_by_scn_coarsely(sync_scn, start_lsn))) { + LOG_WARN("failed to locate lsn", KR(ret), K(sync_scn)); + } else if (OB_FAIL(log_handler->seek(start_lsn, iterator))) { + LOG_WARN("failed to seek iterator", KR(ret), K(sync_scn)); + } + } + } + return ret; +} + +int ObRecoveryLSService::process_ls_log_( + const ObAllTenantInfo &tenant_info, + SCN &start_scn, + PalfBufferIterator &iterator) +{ + int ret = OB_SUCCESS; + palf::LogEntry log_entry; + palf::LSN target_lsn; + SCN sync_scn; + SCN last_sync_scn; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret), K(inited_)); + } else if (OB_UNLIKELY(!tenant_info.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tenant_info is invalid", KR(ret), K(tenant_info)); + } + while (OB_SUCC(ret) && OB_SUCC(iterator.next())) { + if (OB_FAIL(iterator.get_entry(log_entry, target_lsn))) { + LOG_WARN("failed to get log", KR(ret), K(log_entry)); + } else if (OB_ISNULL(log_entry.get_data_buf())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("log entry is null", KR(ret)); + } else { + LOG_DEBUG("get log", K(log_entry), K(target_lsn), K(start_scn)); + sync_scn = log_entry.get_scn(); + const char *log_buf = log_entry.get_data_buf(); + const int64_t log_length = log_entry.get_data_len(); + logservice::ObLogBaseHeader header; + const int64_t HEADER_SIZE = header.get_serialize_size(); + int64_t log_pos = 0; + if (OB_UNLIKELY(sync_scn <= start_scn)) { + //通过scn定位的LSN是不准确的，可能会获取多余的数据，所以需要把小于等于sync_scn的过滤掉 + continue; + } else if (tenant_info.get_recovery_until_scn() < sync_scn) { + if (OB_FAIL(report_sys_ls_recovery_stat_(tenant_info.get_recovery_until_scn()))) { + LOG_WARN("failed to report_sys_ls_recovery_stat_", KR(ret), K(sync_scn), K(tenant_info), + K(log_entry), K(target_lsn), K(start_scn)); + // SYS LS has recovered to the recovery_until_scn, need stop iterate SYS LS log and reset start_scn + } else if (OB_FAIL(seek_log_iterator_(tenant_info.get_recovery_until_scn(), iterator))) { + LOG_WARN("failed to seek log iterator", KR(ret), K(sync_scn), K(tenant_info), + K(log_entry), K(target_lsn), K(start_scn)); + } else { + ret = OB_ITER_STOP; + LOG_WARN("SYS LS has recovered to the recovery_until_scn, need stop iterate SYS LS log", + KR(ret), K(sync_scn), K(tenant_info), K(log_entry), K(target_lsn), K(start_scn)); + start_scn.reset(); + } + } else if (OB_FAIL(header.deserialize(log_buf, HEADER_SIZE, log_pos))) { + LOG_WARN("failed to deserialize", KR(ret), K(HEADER_SIZE)); + } else if (OB_UNLIKELY(log_pos >= log_length)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("log pos is not expected", KR(ret), K(log_pos), K(log_length)); + } else if (logservice::GC_LS_LOG_BASE_TYPE == header.get_log_type()) { + ObGCLSLog gc_log; + if (OB_FAIL(gc_log.deserialize(log_buf, log_length, log_pos))) { + LOG_WARN("failed to deserialize gc log", KR(ret), K(log_length)); + } else if (ObGCLSLOGType::OFFLINE_LS == gc_log.get_log_type()) { + //set sys ls offline + if (OB_FAIL(process_gc_log_(gc_log, sync_scn))) { + LOG_WARN("failed to process gc log", KR(ret), K(sync_scn)); + } + } + // nothing + } else if (logservice::TRANS_SERVICE_LOG_BASE_TYPE == header.get_log_type()) { + ObTxLogBlock tx_log_block; + ObTxLogBlockHeader tx_block_header; + if (OB_FAIL(tx_log_block.init(log_buf, log_length, log_pos, tx_block_header))) { + LOG_WARN("failed to init tx log block", KR(ret), K(log_length)); + } else if (OB_FAIL(process_ls_tx_log_(tx_log_block, sync_scn))) { + LOG_WARN("failed to process ls tx log", KR(ret), K(tx_log_block), K(sync_scn)); + } + } else {} + if (OB_SUCC(ret)) { + last_sync_scn = sync_scn; + } else if (last_sync_scn.is_valid()) { + //if ls_operator can not process, need to report last sync scn + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(report_sys_ls_recovery_stat_(last_sync_scn))) { + LOG_WARN("failed to report ls recovery stat", KR(ret), KR(tmp_ret), K(last_sync_scn)); + } + } + } + }//end for each log + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + if (OB_FAIL(report_sys_ls_recovery_stat_(sync_scn))) { + LOG_WARN("failed to report ls recovery stat", KR(ret), K(sync_scn)); + } + } else if (OB_SUCC(ret)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("iterator must be end", KR(ret)); + } else { + LOG_WARN("failed to get next log", KR(ret)); + } + + return ret; +} + +int ObRecoveryLSService::process_ls_tx_log_(ObTxLogBlock &tx_log_block, const SCN &sync_scn) +{ + int ret = OB_SUCCESS; + bool has_operation = false; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret), K(inited_)); + } + while (OB_SUCC(ret)) { + transaction::ObTxLogHeader tx_header; + if (OB_FAIL(tx_log_block.get_next_log(tx_header))) { + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + break; + } else { + LOG_WARN("failed to get next log", KR(ret)); + } + } else if (transaction::ObTxLogType::TX_COMMIT_LOG != + tx_header.get_tx_log_type()) { + // nothing + } else { + ObTxCommitLogTempRef temp_ref; + ObTxCommitLog commit_log(temp_ref); + const int64_t COMMIT_SIZE = commit_log.get_serialize_size(); + //TODO commit log may too large + if (OB_FAIL(tx_log_block.deserialize_log_body(commit_log))) { + LOG_WARN("failed to deserialize", KR(ret)); + } else { + const ObTxBufferNodeArray &source_data = + commit_log.get_multi_source_data(); + for (int64_t i = 0; OB_SUCC(ret) && i < source_data.count(); ++i) { + const ObTxBufferNode &node = source_data.at(i); + if (ObTxDataSourceType::STANDBY_UPGRADE == node.get_data_source_type()) { + if (OB_FAIL(process_upgrade_log_(node))) { + LOG_WARN("failed to process_upgrade_log_", KR(ret), K(node)); + } + } else if (ObTxDataSourceType::LS_TABLE != node.get_data_source_type()) { + // nothing + } else if (has_operation) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("one clog has more than one operation", KR(ret), K(commit_log)); + } else { + has_operation = true; + ObLSAttr ls_attr; + const int64_t LS_SIZE = ls_attr.get_serialize_size(); + int64_t pos = 0; + if (OB_FAIL(ls_attr.deserialize(node.get_data_buf().ptr(), LS_SIZE, + pos))) { + LOG_WARN("failed to deserialize", KR(ret), K(node), K(LS_SIZE)); + } else if (OB_UNLIKELY(pos > LS_SIZE)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get ls attr", KR(ret), K(pos), K(LS_SIZE)); + } else { + LOG_INFO("get ls operation", K(ls_attr), K(sync_scn)); + //TODO ls recovery is too fast for ls manager, so it maybe failed, while change ls status + //consider how to retry + if (OB_FAIL(process_ls_operator_(ls_attr, + sync_scn))) { + LOG_WARN("failed to process ls operator", KR(ret), K(ls_attr), + K(sync_scn)); + } + } + } + }// end for + } + } + } // end while for each tx_log + + return ret; +} + +int ObRecoveryLSService::process_upgrade_log_(const ObTxBufferNode &node) +{ + int ret = OB_SUCCESS; + uint64_t standby_data_version = 0; + + if (!node.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("node is invalid", KR(ret), K(node)); + } else { + ObStandbyUpgrade primary_data_version; + int64_t pos = 0; + if (OB_FAIL(primary_data_version.deserialize(node.get_data_buf().ptr(), node.get_data_buf().length(), pos))) { + LOG_WARN("failed to deserialize", KR(ret), K(node), KPHEX(node.get_data_buf().ptr(), node.get_data_buf().length())); + } else if (OB_UNLIKELY(pos > node.get_data_buf().length())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get primary_data_version", KR(ret), K(pos), K(node.get_data_buf().length())); + } else { + LOG_INFO("get primary_data_version", K(primary_data_version)); + if (!primary_data_version.is_valid()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("primary_data_version not valid", KR(ret), K(primary_data_version)); + } else if (OB_FAIL(ObUpgradeChecker::get_data_version_by_cluster_version( + GET_MIN_CLUSTER_VERSION(), + standby_data_version))) { + LOG_WARN("failed to get_data_version_by_cluster_version", KR(ret), K(GET_MIN_CLUSTER_VERSION())); + } else if (primary_data_version.get_data_version() > standby_data_version) { + ret = OB_EAGAIN; + if (REACH_TIME_INTERVAL(30 * 60 * 1000 * 1000)) { // 30min + LOG_ERROR("standby version is not new enough to recover primary clog", KR(ret), + K(primary_data_version), K(standby_data_version)); + } + } + } + } + return ret; +} + int ObRecoveryLSService::get_min_data_version_(uint64_t &compatible) { int ret = OB_SUCCESS; @@ -212,6 +502,331 @@ void ObRecoveryLSService::try_tenant_upgrade_end_() } } +int ObRecoveryLSService::process_gc_log_(logservice::ObGCLSLog &gc_log, const SCN &sync_scn) +{ + int ret = OB_SUCCESS; + common::ObMySQLTransaction trans; + const uint64_t meta_tenant_id = gen_meta_tenant_id(tenant_id_); + ObLSLifeAgentManager ls_life_agent(*proxy_); + if (OB_ISNULL(proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("proxy is null", KR(ret)); + } else if (OB_FAIL(trans.start(proxy_, meta_tenant_id))) { + LOG_WARN("failed to start trans", KR(ret), K(meta_tenant_id)); + } else if (OB_FAIL(ls_life_agent.set_ls_offline_in_trans( + tenant_id_, SYS_LS, share::OB_LS_TENANT_DROPPING, sync_scn, share::NORMAL_SWITCHOVER_STATUS, + trans))) { + LOG_WARN("failed to set offline", KR(ret), K(tenant_id_), K(sync_scn)); + } else { + ObLSRecoveryStatOperator ls_recovery; + ObLSRecoveryStat ls_recovery_stat; + if (OB_FAIL(construct_ls_recovery_stat(sync_scn, ls_recovery_stat))) { + LOG_WARN("failed to construct ls recovery stat", KR(ret), K(sync_scn)); + } else if (OB_FAIL(ls_recovery.update_ls_recovery_stat_in_trans( + ls_recovery_stat, trans))) { + LOG_WARN("failed to update ls recovery stat", KR(ret), K(ls_recovery_stat)); + } + } + if (trans.is_started()) { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) { + ret = OB_SUCC(ret) ? tmp_ret : ret; + LOG_WARN("failed to end trans", KR(ret), K(tmp_ret)); + } + } + return ret; +} + +int ObRecoveryLSService::construct_ls_recovery_stat(const SCN &sync_scn, + ObLSRecoveryStat &ls_stat) +{ + int ret = OB_SUCCESS; + ls_stat.reset(); + SCN readable_scn; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret), K(inited_)); + } else if (OB_FAIL(ObTenantRecoveryReportor::get_readable_scn(SYS_LS, readable_scn))) { + LOG_WARN("failed to get readable scn", KR(ret)); + } else if (OB_FAIL(ls_stat.init_only_recovery_stat( + tenant_id_, SYS_LS, sync_scn, readable_scn))) { + LOG_WARN("failed to init ls recovery stat", KR(ret), K(tenant_id_), + K(sync_scn), K(readable_scn)); + } + return ret; +} + +int ObRecoveryLSService::process_ls_operator_(const share::ObLSAttr &ls_attr, const SCN &sync_scn) +{ + int ret = OB_SUCCESS; + common::ObMySQLTransaction trans; + const uint64_t meta_tenant_id = gen_meta_tenant_id(tenant_id_); + if (OB_UNLIKELY(!ls_attr.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("ls attr is invalid", KR(ret), K(ls_attr)); + } else if (OB_ISNULL(proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("proxy is null", KR(ret)); + } else if (OB_FAIL(check_valid_to_operator_ls_(ls_attr, sync_scn))) { + LOG_WARN("failed to check valid to operator ls", KR(ret), K(sync_scn), K(ls_attr)); + } else if (OB_FAIL(trans.start(proxy_, meta_tenant_id))) { + LOG_WARN("failed to start trans", KR(ret), K(meta_tenant_id)); + } else if (OB_FAIL(process_ls_operator_in_trans_(ls_attr, sync_scn, trans))) { + LOG_WARN("failed to process ls operator in trans", KR(ret), K(ls_attr), K(sync_scn)); + } + + if (OB_SUCC(ret)) { + ObLSRecoveryStat ls_recovery_stat; + ObLSRecoveryStatOperator ls_recovery; + if (OB_FAIL(construct_ls_recovery_stat(sync_scn, ls_recovery_stat))) { + LOG_WARN("failed to construct ls recovery stat", KR(ret), K(sync_scn)); + } else if (OB_FAIL(ls_recovery.update_ls_recovery_stat_in_trans( + ls_recovery_stat, trans))) { + LOG_WARN("failed to update ls recovery stat", KR(ret), K(ls_recovery_stat)); + } + } + if (trans.is_started()) { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) { + ret = OB_SUCC(ret) ? tmp_ret : ret; + LOG_WARN("failed to end trans", KR(ret), K(tmp_ret)); + } + } + return ret; +} + +int ObRecoveryLSService::check_valid_to_operator_ls_( + const share::ObLSAttr &ls_attr, const SCN &sync_scn) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret), K(inited_)); + } else if (OB_ISNULL(proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("proxy is null", KR(ret)); + } else if (OB_UNLIKELY(!sync_scn.is_valid() || !ls_attr.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("syns scn is invalid", KR(ret), K(sync_scn), K(ls_attr)); + } else if (share::is_ls_tenant_drop_pre_op(ls_attr.get_ls_operatin_type())) { + ret = OB_ITER_STOP; + LOG_WARN("can not process ls operator after tenant dropping", K(ls_attr)); + } else if (share::is_ls_tenant_drop_op(ls_attr.get_ls_operatin_type())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls recovery must stop while pre tenant dropping", KR(ret), K(ls_attr)); + } else { + SCN user_scn; + ObLSRecoveryStatOperator ls_recovery; + if (OB_FAIL(ls_recovery.get_user_ls_sync_scn(tenant_id_, *proxy_, user_scn))) { + LOG_WARN("failed to get user scn", KR(ret), K(tenant_id_)); + } else if (user_scn >= sync_scn) { + //other ls has larger sync scn, ls can operator + } else { + ret = OB_NEED_WAIT; + LOG_WARN("can not process ls operator, need wait other ls sync", KR(ret), + K(user_scn), K(sync_scn)); + } + } + + return ret; +} + +int ObRecoveryLSService::process_ls_operator_in_trans_( + const share::ObLSAttr &ls_attr, const SCN &sync_scn, ObMySQLTransaction &trans) +{ + int ret = OB_SUCCESS; + const uint64_t meta_tenant_id = gen_meta_tenant_id(tenant_id_); + if (OB_UNLIKELY(!ls_attr.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("ls attr is invalid", KR(ret), K(ls_attr)); + } else if (OB_ISNULL(proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("proxy is null", KR(ret)); + } else { + ObLSStatusOperator ls_operator; + share::ObLSStatusInfo ls_status; + ObLSLifeAgentManager ls_life_agent(*proxy_); + if (OB_UNLIKELY(!ls_attr.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("ls attr is invalid", KR(ret), K(ls_attr)); + } else if (share::is_ls_create_pre_op(ls_attr.get_ls_operatin_type())) { + //create new ls; + if (OB_FAIL(create_new_ls_(ls_attr, sync_scn, trans))) { + LOG_WARN("failed to create new ls", KR(ret), K(sync_scn), K(ls_attr)); + } + } else if (share::is_ls_create_abort_op(ls_attr.get_ls_operatin_type())) { + if (OB_FAIL(ls_life_agent.drop_ls_in_trans(tenant_id_, ls_attr.get_ls_id(), share::NORMAL_SWITCHOVER_STATUS, trans))) { + LOG_WARN("failed to drop ls", KR(ret), K(tenant_id_), K(ls_attr)); + } + } else if (share::is_ls_drop_end_op(ls_attr.get_ls_operatin_type())) { + if (OB_FAIL(ls_life_agent.set_ls_offline_in_trans(tenant_id_, ls_attr.get_ls_id(), + ls_attr.get_ls_status(), sync_scn, share::NORMAL_SWITCHOVER_STATUS, trans))) { + LOG_WARN("failed to set offline", KR(ret), K(tenant_id_), K(ls_attr), K(sync_scn)); + } + } else { + ObLSStatus target_status = share::OB_LS_EMPTY; + if (OB_FAIL(ls_operator.get_ls_status_info(tenant_id_, ls_attr.get_ls_id(), + ls_status, trans))) { + LOG_WARN("failed to get ls status", KR(ret), K(tenant_id_), K(ls_attr)); + } else if (ls_status.ls_is_creating()) { + ret = OB_EAGAIN; + LOG_WARN("ls not created, need wait", KR(ret), K(ls_status)); + } else if (share::is_ls_create_end_op(ls_attr.get_ls_operatin_type())) { + // set ls to normal + target_status = share::OB_LS_NORMAL; + } else if (share::is_ls_tenant_drop_op(ls_attr.get_ls_operatin_type())) { + target_status = share::OB_LS_TENANT_DROPPING; + } else if (share::is_ls_drop_pre_op(ls_attr.get_ls_operatin_type())) { + target_status = share::OB_LS_DROPPING; + } else if (share::is_ls_tenant_drop_pre_op(ls_attr.get_ls_operatin_type())) { + target_status = share::OB_LS_PRE_TENANT_DROPPING; + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected operation type", KR(ret), K(ls_attr)); + } + if (FAILEDx(ls_operator.update_ls_status(tenant_id_, ls_attr.get_ls_id(), + ls_status.status_, target_status, + share::NORMAL_SWITCHOVER_STATUS, trans))) { + LOG_WARN("failed to update ls status", KR(ret), K(tenant_id_), K(ls_attr), + K(ls_status), K(target_status)); + } + LOG_INFO("[LS_RECOVERY] update ls status", KR(ret), K(ls_attr), K(target_status)); + } + } + + return ret; +} +int ObRecoveryLSService::create_new_ls_(const share::ObLSAttr &ls_attr, + const SCN &sync_scn, + common::ObMySQLTransaction &trans) +{ + int ret = OB_SUCCESS; + if (!share::is_ls_create_pre_op(ls_attr.get_ls_operatin_type())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("ls not create pre operation", KR(ret), K(ls_attr)); + } else { + //create new ls; + share::schema::ObSchemaGetterGuard schema_guard; + const share::schema::ObTenantSchema *tenant_schema = NULL; + if (OB_ISNULL(GCTX.schema_service_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error", KR(ret)); + } else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard( + OB_SYS_TENANT_ID, schema_guard))) { + LOG_WARN("fail to get schema guard", KR(ret)); + } else if (OB_FAIL(schema_guard.get_tenant_info(tenant_id_, tenant_schema))) { + LOG_WARN("failed to get tenant ids", KR(ret), K(tenant_id_)); + } else if (OB_ISNULL(tenant_schema)) { + ret = OB_TENANT_NOT_EXIST; + LOG_WARN("tenant not exist", KR(ret), K(tenant_id_)); + } else { + ObTenantLSInfo tenant_stat(GCTX.sql_proxy_, tenant_schema, tenant_id_, + GCTX.srv_rpc_proxy_, GCTX.lst_operator_); + if (OB_FAIL(tenant_stat.gather_stat(true))) { + LOG_WARN("failed to gather stat", KR(ret)); + } else if (OB_FAIL(tenant_stat.create_new_ls_for_recovery(ls_attr.get_ls_id(), + ls_attr.get_ls_group_id(), ls_attr.get_create_scn(), trans))) { + LOG_WARN("failed to add new ls status info", KR(ret), K(ls_attr), K(sync_scn)); + } + } + LOG_INFO("[LS_RECOVERY] create new ls", KR(ret), K(ls_attr)); + } + return ret; +} + + +int ObRecoveryLSService::process_recovery_ls_manager() +{ + int ret = OB_SUCCESS; + share::schema::ObSchemaGetterGuard schema_guard; + const share::schema::ObTenantSchema *tenant_schema = NULL; + + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret), K(inited_)); + } else if (OB_ISNULL(GCTX.schema_service_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error", KR(ret)); + } else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard( + OB_SYS_TENANT_ID, schema_guard))) { + LOG_WARN("fail to get schema guard", KR(ret)); + } else if (OB_FAIL(schema_guard.get_tenant_info(tenant_id_, tenant_schema))) { + LOG_WARN("failed to get tenant ids", KR(ret), K(tenant_id_)); + } else if (OB_ISNULL(tenant_schema)) { + ret = OB_TENANT_NOT_EXIST; + LOG_WARN("tenant not exist", KR(ret), K(tenant_id_)); + } else { + ObTenantLSInfo tenant_stat(GCTX.sql_proxy_, tenant_schema, tenant_id_, + GCTX.srv_rpc_proxy_, GCTX.lst_operator_); + if (OB_FAIL(tenant_stat.process_ls_stats_for_recovery())) { + LOG_WARN("failed to do process in standby", KR(ret), K(tenant_id_)); + } + } + return ret; +} + +int ObRecoveryLSService::check_can_do_recovery_(const ObAllTenantInfo &tenant_info) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret), K(inited_)); + } else if (OB_UNLIKELY(!tenant_info.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tenant_info is invalid", KR(ret), K(tenant_info)); + } else if (tenant_info.is_primary()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("primary tenant can not do recovery", KR(ret), K(tenant_info)); + } else if (tenant_info.is_standby()) { + //standby can do recovery + } else if (tenant_info.is_restore()) { + //need to check success to create init ls + share::ObPhysicalRestoreTableOperator restore_table_operator; + share::ObPhysicalRestoreJob job_info; + if (OB_ISNULL(proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql can't null", K(ret), K(proxy_)); + } else if (OB_FAIL(restore_table_operator.init(proxy_, tenant_id_))) { + LOG_WARN("fail to init restore table operator", KR(ret), K(tenant_id_)); + } else if (OB_FAIL(restore_table_operator.get_job_by_tenant_id(tenant_id_, + job_info))) { + LOG_WARN("fail to get restore job", K(ret), K(tenant_id_)); + } else if (job_info.is_valid_status_to_recovery()) { + //can do recovery + } else { + ret = OB_NEED_WAIT; + LOG_WARN("restore tenant not valid to recovery", KR(ret), K(job_info)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected tenant role", KR(ret), K(tenant_info)); + } + return ret; +} + +int ObRecoveryLSService::report_sys_ls_recovery_stat_(const SCN &sync_scn) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret), K(inited_)); + } else if (OB_ISNULL(proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql can't null", K(ret), K(proxy_)); + } else if (SCN::base_scn() != sync_scn) { + ObLSRecoveryStatOperator ls_recovery; + ObLSRecoveryStat ls_recovery_stat; + if (OB_FAIL(construct_ls_recovery_stat(sync_scn, ls_recovery_stat))) { + LOG_WARN("failed to construct ls recovery stat", KR(ret), K(sync_scn)); + } else if (OB_FAIL(ls_recovery.update_ls_recovery_stat(ls_recovery_stat, + *proxy_))) { + LOG_WARN("failed to update ls recovery stat", KR(ret), + K(ls_recovery_stat)); + } + } + return ret; +} + }//end of namespace rootserver }//end of namespace oceanbase diff --git a/src/rootserver/ob_recovery_ls_service.h b/src/rootserver/ob_recovery_ls_service.h index 53d85b3f34..2913e3d47c 100644 --- a/src/rootserver/ob_recovery_ls_service.h +++ b/src/rootserver/ob_recovery_ls_service.h @@ -14,7 +14,11 @@ #define OCEANBASE_ROOTSERVER_OB_RECCOVERY_LS_SERVICE_H #include "lib/thread/ob_reentrant_thread.h"//ObRsReentrantThread #include "logservice/ob_log_base_type.h"//ObIRoleChangeSubHandler ObICheckpointSubHandler ObIReplaySubHandler -#include "ob_tenant_thread_helper.h" //ObTenantThreadHelper +#include "logservice/palf/lsn.h"//palf::LSN +#include "logservice/palf/palf_iterator.h" //PalfBufferIterator +#include "ob_primary_ls_service.h" //ObTenantThreadHelper +#include "lib/lock/ob_spin_lock.h" //ObSpinLock +#include "storage/tx/ob_multi_data_source.h" //ObTxBufferNode namespace oceanbase { @@ -28,11 +32,35 @@ class ObMySQLProxy; class ObISQLClient; class ObMySQLTransaction; } - - - +namespace share +{ +class ObLSTableOperator; +struct ObLSAttr; +struct ObLSRecoveryStat; +class SCN; +namespace schema +{ +class ObMultiVersionSchemaService; +class ObTenantSchema; +} +} +namespace logservice +{ +class ObLogHandler; +class ObGCLSLog; +} +namespace transaction +{ +class ObTxLogBlock; +} namespace rootserver { +/*description: + *Restores the status of each log stream according to the logs to which the + *system log stream is synchronized, and updates the recovery progress of the + *system log stream. This thread should only exist in the standby database or + *the recovery process, and needs to be registered in the RestoreHandler. + *This thread is only active on the leader of the system log stream under the user tenant*/ class ObRecoveryLSService : public ObTenantThreadHelper { public: @@ -44,8 +72,35 @@ public: virtual void do_work() override; DEFINE_MTL_FUNC(ObRecoveryLSService) private: + //get log iterator by start_scn + int seek_log_iterator_(const share::SCN &syn_scn, + palf::PalfBufferIterator &iterator); + int process_ls_log_(const ObAllTenantInfo &tenant_info, + share::SCN &start_scn, + palf::PalfBufferIterator &iterator); + int process_upgrade_log_(const transaction::ObTxBufferNode &node); + int process_gc_log_(logservice::ObGCLSLog &gc_log, + const share::SCN &syn_scn); + int process_ls_tx_log_(transaction::ObTxLogBlock &tx_log, + const share::SCN &syn_scn); + int process_ls_operator_(const share::ObLSAttr &ls_attr, + const share::SCN &syn_scn); + int create_new_ls_(const share::ObLSAttr &ls_attr, + const share::SCN &syn_scn, + common::ObMySQLTransaction &trans); + int process_recovery_ls_manager(); + int construct_ls_recovery_stat(const share::SCN &syn_scn, + share::ObLSRecoveryStat &ls_stat); + //wait other ls is larger than sycn ts + int check_valid_to_operator_ls_(const share::ObLSAttr &ls_attr, + const share::SCN &syn_scn); + int check_can_do_recovery_(const ObAllTenantInfo &tenant_info); + //readable scn need report + int report_sys_ls_recovery_stat_(const share::SCN &sync_scn); void try_tenant_upgrade_end_(); int get_min_data_version_(uint64_t &compatible); + int process_ls_operator_in_trans_(const share::ObLSAttr &ls_attr, + const share::SCN &sync_scn, common::ObMySQLTransaction &trans); private: bool inited_; uint64_t tenant_id_; diff --git a/src/rootserver/ob_root_utils.cpp b/src/rootserver/ob_root_utils.cpp index c9dd756605..386d5b4be0 100644 --- a/src/rootserver/ob_root_utils.cpp +++ b/src/rootserver/ob_root_utils.cpp @@ -1773,6 +1773,7 @@ int ObRootUtils::get_tenant_intersection(ObUnitManager &unit_mgr, return ret; } + template bool ObRootUtils::has_intersection(const common::ObIArray &this_array, const common::ObIArray &other_array) diff --git a/src/rootserver/ob_root_utils.h b/src/rootserver/ob_root_utils.h index 8069cfcbba..11fdf9d3e0 100644 --- a/src/rootserver/ob_root_utils.h +++ b/src/rootserver/ob_root_utils.h @@ -642,6 +642,7 @@ public: common::ObIArray &this_server_list, common::ObIArray &other_server_list, common::ObIArray &tenant_ids); + }; class ObClusterInfoGetter diff --git a/src/rootserver/ob_tenant_info_loader.cpp b/src/rootserver/ob_tenant_info_loader.cpp index 11d2300476..8f9b858c3a 100644 --- a/src/rootserver/ob_tenant_info_loader.cpp +++ b/src/rootserver/ob_tenant_info_loader.cpp @@ -267,20 +267,5 @@ int ObAllTenantInfoCache::get_tenant_info(share::ObAllTenantInfo &tenant_info) return ret; } -int ObTenantInfoLoader::get_tenant_readable_scn(SCN &readable_scn) -{ - int ret = OB_SUCCESS; - share::ObAllTenantInfo tenant_info; - if (OB_FAIL(get_tenant_info(tenant_info))) { - LOG_WARN("get_tenant_info failed", K(ret)); - } else if (OB_UNLIKELY(! tenant_info.is_valid())) { - ret = OB_EAGAIN; - LOG_WARN("tenant info not valid", K(ret), K(tenant_info)); - } else { - readable_scn = tenant_info.get_standby_scn(); - } - return ret; -} - } } diff --git a/src/rootserver/ob_tenant_info_loader.h b/src/rootserver/ob_tenant_info_loader.h index 7e17cbe7d2..1d1149703e 100644 --- a/src/rootserver/ob_tenant_info_loader.h +++ b/src/rootserver/ob_tenant_info_loader.h @@ -78,7 +78,6 @@ public: } virtual void run2() override; int get_tenant_info(share::ObAllTenantInfo &tenant_info); - int get_tenant_readable_scn(share::SCN &readable_scn); /** * @description: * get valid sts, only return sts refreshed after specified_time @@ -108,4 +107,4 @@ private: } // namespace rootserver } // namespace oceanbase -#endif /* !OCEANBASE_ROOTSERVER_OB_TENANT_INFO_LOADER_H */ +#endif /* !OCEANBASE_ROOTSERVER_OB_TENANT_INFO_LOADER_H */ \ No newline at end of file diff --git a/src/rootserver/ob_tenant_info_report.cpp b/src/rootserver/ob_tenant_info_report.cpp deleted file mode 100755 index 2d27d523b5..0000000000 --- a/src/rootserver/ob_tenant_info_report.cpp +++ /dev/null @@ -1,110 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ - -#define USING_LOG_PREFIX RS -#include "ob_tenant_info_report.h" -#include "lib/profile/ob_trace_id.h" -#include "share/ob_errno.h" -#include "share/ob_share_util.h"//ObShareUtil -#include "observer/ob_server_struct.h"//GCTX -#include "share/ls/ob_ls_recovery_stat_operator.h"//ObLSRecoveryStatOperator - -namespace oceanbase -{ -using namespace common; -using namespace share; -using namespace transaction; -using namespace palf; -namespace rootserver -{ -//////////////ObTenantInfoReportor - -int ObTenantInfoReportor::init() -{ - int ret = OB_SUCCESS; - tenant_id_ = MTL_ID(); - if (OB_UNLIKELY(inited_)) { - ret = OB_INIT_TWICE; - LOG_WARN("has inited", KR(ret)); - } else if (OB_FAIL(ObTenantThreadHelper::create("TenInfoRep", - lib::TGDefIDs::SimpleLSService, *this))) { - LOG_WARN("failed to create thread", KR(ret)); - } else if (OB_FAIL(ObTenantThreadHelper::start())) { - LOG_WARN("fail to start", KR(ret)); - } else { - inited_ = true; - } - return ret; -} - -void ObTenantInfoReportor::destroy() -{ - ObTenantThreadHelper::destroy(); - tenant_id_ = OB_INVALID_TENANT_ID; - inited_ = false; -} - -void ObTenantInfoReportor::do_work() -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", K(ret)); - } else { - int64_t idle_time_us = 500 * 1000L; - while (!has_set_stop()) { - ObCurTraceId::init(GCONF.self_addr_); - if (is_sys_tenant(tenant_id_)) { - idle_time_us = 3600 * 1000 * 1000L; - } else if (OB_FAIL(gather_tenant_recovery_stat_())) { - LOG_WARN("failed to gather tenant recovery stat", KR(ret), K(tenant_id_)); - } - idle(idle_time_us); - }// end while - } -} - -int ObTenantInfoReportor::gather_tenant_recovery_stat_() -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret)); - } else if (!is_meta_tenant(tenant_id_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sys or user tenant no need gather tenant recovery stat", KR(ret), K(tenant_id_)); - } else if (OB_ISNULL(GCTX.sql_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy is null", KR(ret)); - } else { - ObLSRecoveryStatOperator ls_recovery_op; - ObAllTenantInfoProxy info_proxy; - ObAllTenantInfo tenant_info; - const uint64_t user_tenant_id = gen_user_tenant_id(tenant_id_); - SCN sync_scn; - SCN readable_scn; - DEBUG_SYNC(BLOCK_TENANT_SYNC_SNAPSHOT_INC); - if (OB_FAIL(ls_recovery_op.get_tenant_recovery_stat( - user_tenant_id, *GCTX.sql_proxy_, sync_scn, readable_scn))) { - LOG_WARN("failed to get tenant recovery stat", KR(ret), K(user_tenant_id)); - //TODO replayable_scn is equal to sync_scn - } else if (OB_FAIL(info_proxy.update_tenant_recovery_status( - user_tenant_id, GCTX.sql_proxy_, sync_scn, - sync_scn, readable_scn))) { - LOG_WARN("failed to update tenant recovery stat", KR(ret), - K(user_tenant_id), K(sync_scn), K(readable_scn)); - } - } - return ret; -} -} -} diff --git a/src/rootserver/ob_tenant_info_report.h b/src/rootserver/ob_tenant_info_report.h deleted file mode 100644 index a387b386c0..0000000000 --- a/src/rootserver/ob_tenant_info_report.h +++ /dev/null @@ -1,84 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ - -#ifndef OCEANBASE_ROOTSERVER_OB_TENANT_INFO_REPORT_H -#define OCEANBASE_ROOTSERVER_OB_TENANT_INFO_REPORT_H -#include "lib/thread/ob_reentrant_thread.h"//ObRsReentrantThread -#include "logservice/ob_log_base_type.h" -#include "share/scn.h"//SCN -#include "share/ob_thread_mgr.h" //OBTGDefIDEnum -#include "lib/thread/thread_mgr_interface.h" // TGRunnable -#include "lib/lock/ob_thread_cond.h"//ObThreadCond -#include "rootserver/ob_tenant_thread_helper.h"//ObTenantThreadHelper - - -namespace oceanbase -{ -namespace obrpc -{ -class ObSrvRpcProxy; -} -namespace common -{ -class ObMySQLProxy; -class ObMySQLTransaction; -} -namespace share -{ -class ObLSTableOperator; -class SCN; - -} -namespace palf -{ -struct PalfBaseInfo; -} -namespace rootserver -{ -/*description: - * report __all_tenant_info - */ -class ObTenantInfoReportor : public ObTenantThreadHelper, - public logservice::ObICheckpointSubHandler, - public logservice::ObIReplaySubHandler -{ -public: - ObTenantInfoReportor():inited_(false), tenant_id_(OB_INVALID_TENANT_ID) {} - virtual ~ObTenantInfoReportor() {} - int init(); - void destroy(); - virtual void do_work() override; - DEFINE_MTL_FUNC(ObTenantInfoReportor) - -public: - virtual share::SCN get_rec_scn() override { return share::SCN::max_scn();} - virtual int flush(share::SCN &scn) override { return OB_SUCCESS; } - int replay(const void *buffer, const int64_t nbytes, const palf::LSN &lsn, const share::SCN &scn) override - { - UNUSED(buffer); - UNUSED(nbytes); - UNUSED(lsn); - UNUSED(scn); - return OB_SUCCESS; - } -private: - int gather_tenant_recovery_stat_(); -private: - bool inited_; - uint64_t tenant_id_; - -}; -} -} - - -#endif /* !OCEANBASE_ROOTSERVER_OB_TENANT_INFO_REPORT_H */ diff --git a/src/rootserver/ob_ls_recovery_reportor.cpp b/src/rootserver/ob_tenant_recovery_reportor.cpp similarity index 85% rename from src/rootserver/ob_ls_recovery_reportor.cpp rename to src/rootserver/ob_tenant_recovery_reportor.cpp index c810486a3c..c9933fe32f 100644 --- a/src/rootserver/ob_ls_recovery_reportor.cpp +++ b/src/rootserver/ob_tenant_recovery_reportor.cpp @@ -12,7 +12,7 @@ #define USING_LOG_PREFIX RS -#include "rootserver/ob_ls_recovery_reportor.h" +#include "rootserver/ob_tenant_recovery_reportor.h" #include "rootserver/ob_tenant_info_loader.h" #include "rootserver/ob_tenant_role_transition_service.h"//ObTenantRoleTransitionConstants #include "storage/tx_storage/ob_ls_service.h" //ObLSService @@ -36,11 +36,11 @@ using namespace storage; using namespace palf; namespace rootserver { -int ObLSRecoveryReportor::mtl_init(ObLSRecoveryReportor *&ka) +int ObTenantRecoveryReportor::mtl_init(ObTenantRecoveryReportor *&ka) { return ka->init(); } -int ObLSRecoveryReportor::init() +int ObTenantRecoveryReportor::init() { int ret = OB_SUCCESS; lib::ThreadPool::set_run_wrapper(MTL_CTX()); @@ -56,7 +56,7 @@ int ObLSRecoveryReportor::init() } else if (OB_ISNULL(GCTX.sql_proxy_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("sql proxy is null", KR(ret)); - } else if (OB_FAIL(create(thread_cnt, "LSRecRep"))) { + } else if (OB_FAIL(create(thread_cnt, "TeRec"))) { LOG_WARN("failed to create tenant recovery stat thread", KR(ret), K(thread_cnt)); } @@ -66,7 +66,7 @@ int ObLSRecoveryReportor::init() } return ret; } -void ObLSRecoveryReportor::destroy() +void ObTenantRecoveryReportor::destroy() { LOG_INFO("tenant recovery service destory", KPC(this)); stop(); @@ -76,7 +76,7 @@ void ObLSRecoveryReportor::destroy() sql_proxy_ = NULL; } -int ObLSRecoveryReportor::start() +int ObTenantRecoveryReportor::start() { int ret = OB_SUCCESS; if (IS_NOT_INIT) { @@ -94,16 +94,16 @@ int ObLSRecoveryReportor::start() return ret; } -void ObLSRecoveryReportor::stop() +void ObTenantRecoveryReportor::stop() { logical_stop(); } -void ObLSRecoveryReportor::wait() +void ObTenantRecoveryReportor::wait() { logical_wait(); } -void ObLSRecoveryReportor::wakeup() +void ObTenantRecoveryReportor::wakeup() { if (OB_NOT_INIT) { } else { @@ -112,7 +112,7 @@ void ObLSRecoveryReportor::wakeup() } } -void ObLSRecoveryReportor::run2() +void ObTenantRecoveryReportor::run2() { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; @@ -122,8 +122,7 @@ void ObLSRecoveryReportor::run2() LOG_WARN("not init", KR(ret)); } else { ObThreadCondGuard guard(get_cond()); - //const int64_t idle_time = ObTenantRoleTransitionConstants::TENANT_INFO_REFRESH_TIME_US; - const int64_t idle_time = 500 * 1000; + const int64_t idle_time = ObTenantRoleTransitionConstants::TENANT_INFO_REFRESH_TIME_US; const uint64_t meta_tenant_id = gen_meta_tenant_id(tenant_id_); while (!stop_) { if (OB_ISNULL(GCTX.schema_service_)) { @@ -157,7 +156,7 @@ void ObLSRecoveryReportor::run2() } } -int ObLSRecoveryReportor::submit_tenant_refresh_schema_task_() +int ObTenantRecoveryReportor::submit_tenant_refresh_schema_task_() { int ret = OB_SUCCESS; ObAllTenantInfo tenant_info; @@ -201,7 +200,7 @@ int ObLSRecoveryReportor::submit_tenant_refresh_schema_task_() } return ret; } -int ObLSRecoveryReportor::update_ls_recovery_stat_() +int ObTenantRecoveryReportor::update_ls_recovery_stat_() { int ret = OB_SUCCESS; if (IS_NOT_INIT) { @@ -236,7 +235,10 @@ int ObLSRecoveryReportor::update_ls_recovery_stat_() LOG_WARN("failed to update_ls_replayable_point", KR(tmp_ret), KPC(ls), K(tenant_info)); } - if (OB_FAIL(update_ls_recovery(ls, sql_proxy_))) { + if (ls->is_sys_ls()) { + // nothing todo + // sys ls of user tenant is in ls_recovery + } else if (OB_FAIL(update_ls_recovery(ls, sql_proxy_))) { LOG_WARN("failed to update ls recovery", KR(ret), KPC(ls)); } } @@ -254,7 +256,7 @@ int ObLSRecoveryReportor::update_ls_recovery_stat_() return ret; } -int ObLSRecoveryReportor::update_ls_recovery(ObLS *ls, common::ObMySQLProxy *sql_proxy) +int ObTenantRecoveryReportor::update_ls_recovery(ObLS *ls, common::ObMySQLProxy *sql_proxy) { int ret = OB_SUCCESS; logservice::ObLogService *ls_svr = MTL(logservice::ObLogService*); @@ -270,9 +272,8 @@ int ObLSRecoveryReportor::update_ls_recovery(ObLS *ls, common::ObMySQLProxy *sql ObLSRecoveryStat ls_recovery_stat; const ObLSID ls_id = ls->get_ls_id(); const uint64_t tenant_id = MTL_ID(); - ObMySQLTransaction trans; + ObLSRecoveryStatOperator ls_recovery; - const uint64_t exec_tenant_id = ls_recovery.get_exec_tenant_id(tenant_id); if (OB_FAIL(ls_svr->get_palf_role(ls_id, role, first_proposal_id))) { LOG_WARN("failed to get first role", KR(ret), K(ls_id), KPC(ls)); } else if (!is_strong_leader(role)) { @@ -293,31 +294,10 @@ int ObLSRecoveryReportor::update_ls_recovery(ObLS *ls, common::ObMySQLProxy *sql ret = OB_EAGAIN; LOG_INFO("role change, try again", KR(ret), K(role), K(first_proposal_id), K(second_proposal_id), KPC(ls)); - } else if (OB_FAIL(trans.start(sql_proxy, exec_tenant_id))) { - LOG_WARN("failed to start trans", KR(ret), K(exec_tenant_id), K(ls_recovery_stat)); - } else if (OB_FAIL(ls_recovery.update_ls_recovery_stat_in_trans(ls_recovery_stat, - trans))) { + } else if (OB_FAIL(ls_recovery.update_ls_recovery_stat(ls_recovery_stat, + *sql_proxy))) { LOG_WARN("failed to update ls recovery stat", KR(ret), K(ls_recovery_stat)); - } else { - //double check sync can not fallback - palf::PalfHandleGuard palf_handle_guard; - SCN new_scn; - if (OB_FAIL(ls_svr->open_palf(ls_id, palf_handle_guard))) { - LOG_WARN("failed to open palf", KR(ret), K(ls_id)); - } else if (OB_FAIL(palf_handle_guard.get_end_scn(new_scn))) { - LOG_WARN("failed to get end ts", KR(ret), K(ls_id)); - } else if (new_scn < sync_scn) { - ret = OB_OP_NOT_ALLOW; - LOG_WARN("maybe flashback, can not report sync scn", KR(ret), K(ls_recovery_stat), K(new_scn)); - } - } - if (trans.is_started()) { - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) { - LOG_WARN("failed to commit trans", KR(ret), KR(tmp_ret)); - ret = OB_SUCC(ret) ? tmp_ret : ret; - } } } const int64_t PRINT_INTERVAL = 10 * 1000 * 1000L; @@ -335,7 +315,27 @@ int ObLSRecoveryReportor::update_ls_recovery(ObLS *ls, common::ObMySQLProxy *sql } -int ObLSRecoveryReportor::update_replayable_point_() +int ObTenantRecoveryReportor::get_tenant_readable_scn(SCN &readable_scn) +{ + int ret = OB_SUCCESS; + share::ObAllTenantInfo tenant_info; + rootserver::ObTenantInfoLoader *tenant_info_loader = MTL(rootserver::ObTenantInfoLoader*); + + if (OB_ISNULL(tenant_info_loader)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("mtl pointer is null", KR(ret), KP(tenant_info_loader)); + } else if (OB_FAIL(tenant_info_loader->get_tenant_info(tenant_info))) { + LOG_WARN("get_tenant_info failed", K(ret)); + } else if (OB_UNLIKELY(! tenant_info.is_valid())) { + ret = OB_EAGAIN; + LOG_WARN("tenant info not valid", K(ret), K(tenant_info)); + } else { + readable_scn = tenant_info.get_standby_scn(); + } + return ret; +} + +int ObTenantRecoveryReportor::update_replayable_point_() { int ret = OB_SUCCESS; if (IS_NOT_INIT) { @@ -352,7 +352,7 @@ int ObLSRecoveryReportor::update_replayable_point_() return ret; } -int ObLSRecoveryReportor::update_replayable_point_from_tenant_info_() +int ObTenantRecoveryReportor::update_replayable_point_from_tenant_info_() { int ret = OB_SUCCESS; logservice::ObLogService *log_service = MTL(logservice::ObLogService*); @@ -373,7 +373,7 @@ int ObLSRecoveryReportor::update_replayable_point_from_tenant_info_() return ret; } -int ObLSRecoveryReportor::update_replayable_point_from_meta_() +int ObTenantRecoveryReportor::update_replayable_point_from_meta_() { int ret = OB_SUCCESS; SCN replayable_point; @@ -413,10 +413,12 @@ int ObLSRecoveryReportor::update_replayable_point_from_meta_() return ret; } -int ObLSRecoveryReportor::get_sync_point_(const share::ObLSID &id, +int ObTenantRecoveryReportor::get_sync_point_(const share::ObLSID &id, SCN &sync_scn, SCN &read_scn) { int ret = OB_SUCCESS; + palf::AccessMode access_mode; + int64_t unused_mode_version; palf::PalfHandleGuard palf_handle_guard; if (OB_FAIL(MTL(logservice::ObLogService*)->open_palf(id, palf_handle_guard))) { LOG_WARN("failed to open palf", KR(ret), K(id)); @@ -430,7 +432,7 @@ int ObLSRecoveryReportor::get_sync_point_(const share::ObLSID &id, } -int ObLSRecoveryReportor::get_readable_scn(const share::ObLSID &id, SCN &readable_scn) +int ObTenantRecoveryReportor::get_readable_scn(const share::ObLSID &id, SCN &readable_scn) { int ret = OB_SUCCESS; storage::ObLSHandle ls_handle; diff --git a/src/rootserver/ob_ls_recovery_reportor.h b/src/rootserver/ob_tenant_recovery_reportor.h similarity index 82% rename from src/rootserver/ob_ls_recovery_reportor.h rename to src/rootserver/ob_tenant_recovery_reportor.h index 428c81132e..31aceb18ca 100644 --- a/src/rootserver/ob_ls_recovery_reportor.h +++ b/src/rootserver/ob_tenant_recovery_reportor.h @@ -10,8 +10,8 @@ * See the Mulan PubL v2 for more details. */ -#ifndef OCEANBASE_ROOTSERVER_OB_LS_RECOVERY_SERVICE_H -#define OCEANBASE_ROOTSERVER_OB_LS_RECOVERY_SERVICE_H +#ifndef OCEANBASE_ROOTSERVER_OB_TENANT_RECOVERY_SERVICE_H +#define OCEANBASE_ROOTSERVER_OB_TENANT_RECOVERY_SERVICE_H #include "lib/thread/ob_reentrant_thread.h"//ObRsReentrantThread #include "lib/utility/ob_print_utils.h" //TO_STRING_KV @@ -40,15 +40,15 @@ namespace rootserver * standby machine-readable timestamp of the majority, the minimum standby * machine-readable timestamp of all replicas, the synchronization point, etc. * Statistics for the syslog stream are not in this thread.*/ -class ObLSRecoveryReportor : public share::ObReentrantThread +class ObTenantRecoveryReportor : public share::ObReentrantThread { public: - ObLSRecoveryReportor() + ObTenantRecoveryReportor() : is_inited_(false), tenant_id_(common::OB_INVALID_TENANT_ID), sql_proxy_(nullptr) {} - ~ObLSRecoveryReportor() {} - static int mtl_init(ObLSRecoveryReportor *&ka); + ~ObTenantRecoveryReportor() {} + static int mtl_init(ObTenantRecoveryReportor *&ka); int init(); void destroy(); int start(); @@ -62,6 +62,7 @@ public: //description: update ls recovery static int update_ls_recovery(storage::ObLS *ls, common::ObMySQLProxy *sql_proxy); + int get_tenant_readable_scn(share::SCN &readable_scn); static int get_readable_scn(const share::ObLSID &id, share::SCN &read_scn); private: static int get_sync_point_(const share::ObLSID &id, share::SCN &scn, share::SCN &read_scn); @@ -79,11 +80,11 @@ private: int update_replayable_point_from_tenant_info_(); int update_replayable_point_from_meta_(); int submit_tenant_refresh_schema_task_(); - DISALLOW_COPY_AND_ASSIGN(ObLSRecoveryReportor); + DISALLOW_COPY_AND_ASSIGN(ObTenantRecoveryReportor); }; } // namespace rootserver } // namespace oceanbase -#endif /* !OCEANBASE_ROOTSERVER_OB_LS_RECOVERY_SERVICE_H */ +#endif /* !OCEANBASE_ROOTSERVER_OB_TENANT_RECOVERY_SERVICE_H */ diff --git a/src/rootserver/ob_tenant_role_transition_service.cpp b/src/rootserver/ob_tenant_role_transition_service.cpp index 46185b672e..9416f31703 100644 --- a/src/rootserver/ob_tenant_role_transition_service.cpp +++ b/src/rootserver/ob_tenant_role_transition_service.cpp @@ -309,6 +309,8 @@ int ObTenantRoleTransitionService::do_prepare_flashback_for_failover_to_primary_ } else if (OB_UNLIKELY(switchover_epoch_ != tenant_info.get_switchover_epoch())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("tenant switchover status not valid", KR(ret), K(tenant_info), K_(switchover_epoch)); + } else if (OB_FAIL(update_tenant_stat_info_())) { + LOG_WARN("failed to update tenant stat info", KR(ret), K(tenant_info), K_(switchover_epoch)); } else if (OB_FAIL(OB_PRIMARY_STANDBY_SERVICE.do_recover_tenant(tenant_id_, share::PREPARE_FLASHBACK_FOR_FAILOVER_TO_PRIMARY_SWITCHOVER_STATUS, obrpc::ObRecoverTenantArg::RecoverType::CANCEL, @@ -336,20 +338,15 @@ int ObTenantRoleTransitionService::do_switch_access_mode_to_flashback( const share::ObAllTenantInfo &tenant_info) { int ret = OB_SUCCESS; - ObLSStatusOperator ls_status_op; - share::ObLSStatusInfoArray all_ls_status_array; + if (OB_FAIL(check_inner_stat())) { LOG_WARN("error unexpected", KR(ret), K(tenant_id_), KP(sql_proxy_), KP(rpc_proxy_)); } else if (OB_UNLIKELY(!tenant_info.is_flashback_status() || switchover_epoch_ != tenant_info.get_switchover_epoch())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("tenant switchover status not valid", KR(ret), K(tenant_info), K(switchover_epoch_)); - } else if (OB_FAIL(ls_status_op.get_all_ls_status_by_order_for_switch_tenant(tenant_id_, - false/* ignore_need_create_abort */, all_ls_status_array, *sql_proxy_))) { - LOG_WARN("failed to get_all_ls_status_by_order", KR(ret), K(tenant_id_)); - } else if (OB_FAIL(change_ls_access_mode_(all_ls_status_array, - palf::AccessMode::FLASHBACK, SCN::base_scn()))) { - LOG_WARN("failed to get access mode", KR(ret), K(tenant_info), K(all_ls_status_array)); + } else if (OB_FAIL(change_ls_access_mode_(palf::AccessMode::FLASHBACK, SCN::base_scn()))) { + LOG_WARN("failed to get access mode", KR(ret), K(tenant_info)); } return ret; } @@ -403,10 +400,7 @@ int ObTenantRoleTransitionService::do_switch_access_mode_to_append( { int ret = OB_SUCCESS; palf::AccessMode access_mode = logservice::ObLogService::get_palf_access_mode(target_tenant_role); - SCN ref_scn; - ObLSRecoveryStatOperator ls_recovery_op; - ObLSStatusOperator status_op; - ObLSStatusInfoArray status_info_array; + const SCN &ref_scn = tenant_info.get_sync_scn(); if (OB_FAIL(check_inner_stat())) { LOG_WARN("error unexpected", KR(ret), K(tenant_id_), KP(sql_proxy_), KP(rpc_proxy_)); } else if (OB_UNLIKELY(!tenant_info.is_switching_to_primary_status() @@ -416,25 +410,8 @@ int ObTenantRoleTransitionService::do_switch_access_mode_to_append( LOG_WARN("tenant switchover status not valid", KR(ret), K(tenant_info), K(target_tenant_role), K(switchover_epoch_)); //TODO(yaoying):xianming - } else if (OB_ISNULL(sql_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy is null", KR(ret)); - } else if (OB_FAIL(ls_recovery_op.get_tenant_max_sync_scn(tenant_id_, *sql_proxy_, ref_scn))) { - LOG_WARN("failed to get tenant max sync scn", KR(ret), K(tenant_id_)); - } else { - // After flashback, it is necessary to ensure that the newly written log - // must be greater than all the sync_scn that have been reported, so ref_ts - // will be set as the largest sync_scn of all ls. Since sys_recovery_scn is - // reported independently, the maximum value of these two values should be taken . - ref_scn = SCN::max(tenant_info.get_sys_recovery_scn(), ref_scn); - } - - if (FAILEDx(status_op.get_all_ls_status_by_order_for_switch_tenant( - tenant_id_, false/*ignore need create abort*/, status_info_array, *sql_proxy_))) { - LOG_WARN("failed to get all ls status info", KR(ret), K(tenant_id_)); - } else if (OB_FAIL(change_ls_access_mode_(status_info_array, access_mode, ref_scn))) { - LOG_WARN("failed to get access mode", KR(ret), K(access_mode), K(ref_scn), - K(tenant_info), K(status_info_array)); + } else if (OB_FAIL(change_ls_access_mode_(access_mode, ref_scn))) { + LOG_WARN("failed to get access mode", KR(ret), K(access_mode), K(ref_scn), K(tenant_info)); } else { common::ObMySQLTransaction trans; share::ObAllTenantInfo cur_tenant_info; @@ -471,14 +448,12 @@ int ObTenantRoleTransitionService::do_switch_access_mode_to_append( return ret; } - -int ObTenantRoleTransitionService::switchover_to_standby(const share::ObAllTenantInfo &tenant_info) +int ObTenantRoleTransitionService::do_switch_access_mode_to_raw_rw( + const share::ObAllTenantInfo &tenant_info) { int ret = OB_SUCCESS; palf::AccessMode access_mode = logservice::ObLogService::get_palf_access_mode(STANDBY_TENANT_ROLE); ObLSStatusOperator status_op; - ObLSStatusInfoArray status_info_array; - share::ObLSStatusInfo sys_ls; if (OB_FAIL(check_inner_stat())) { LOG_WARN("error unexpected", KR(ret), K(tenant_id_), KP(sql_proxy_), KP(rpc_proxy_)); @@ -488,69 +463,30 @@ int ObTenantRoleTransitionService::switchover_to_standby(const share::ObAllTenan ret = OB_INVALID_ARGUMENT; LOG_WARN("tenant switchover status not valid", KR(ret), K(tenant_info), K(switchover_epoch_)); - } else if (OB_FAIL(status_op.get_ls_status_info(tenant_id_, SYS_LS, sys_ls, *sql_proxy_))) { - LOG_WARN("failed to get sys ls status info", KR(ret), K(tenant_id_)); - } else if (OB_FAIL(status_info_array.push_back(sys_ls))) { - LOG_WARN("failed to push back", KR(ret), K(sys_ls)); - } else if (OB_FAIL(change_ls_access_mode_(status_info_array, access_mode, SCN::base_scn()))) { - LOG_WARN("failed to switch sys ls access mode", KR(ret), K(status_info_array), K(access_mode)); - } else if (OB_FAIL(wait_sys_recovery_finish_(status_info_array))) { - LOG_WARN("faild to wait sys recovery finish", KR(ret), K(status_info_array)); } else if (OB_FAIL(status_op.create_abort_ls_in_switch_tenant( tenant_id_, tenant_info.get_switchover_status(), tenant_info.get_switchover_epoch(), *sql_proxy_))) { LOG_WARN("failed to create abort ls", KR(ret), K(tenant_info)); - } else if (OB_FAIL(status_op.get_all_ls_status_by_order_for_switch_tenant( - tenant_id_, false/*ignore need create abort*/, status_info_array, *sql_proxy_))) { - LOG_WARN("failed to get all ls status info", KR(ret), K(tenant_id_)); - } else if (OB_FAIL(change_ls_access_mode_(status_info_array, access_mode, SCN::base_scn()))) { + } else if (OB_FAIL(change_ls_access_mode_(access_mode, SCN::base_scn()))) { LOG_WARN("failed to get access mode", KR(ret), K(access_mode), K(tenant_info)); } return ret; } -int ObTenantRoleTransitionService::wait_sys_recovery_finish_(const ObLSStatusInfoIArray &ls_status_array) +int ObTenantRoleTransitionService::update_tenant_stat_info_() { int ret = OB_SUCCESS; - ObArray checkpoints; - if (OB_FAIL(check_inner_stat())) { LOG_WARN("error unexpected", KR(ret), K(tenant_id_), KP(sql_proxy_), KP(rpc_proxy_)); - } else if (OB_FAIL(get_checkpoints_by_rpc_(tenant_id_, ls_status_array, false, checkpoints))) { - LOG_WARN("failed to get checkpoints by rpc", KR(ret), K(tenant_id_), K(ls_status_array)); - } else if (OB_UNLIKELY(1 != checkpoints.count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("expectect only one checkpoint", KR(ret), K(ls_status_array)); } else { - //wait sys recovery scn to latest - ObAllTenantInfo curr_tenant_info; - bool sync_to_latest = false; - const SCN sys_scn = checkpoints.at(0).cur_sync_scn_; - while (!THIS_WORKER.is_timeout() && !sync_to_latest && OB_SUCC(ret)) { - //ignore error - if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(tenant_id_, sql_proxy_, false, curr_tenant_info))) { - LOG_WARN("failed to load tenant info", KR(ret), K(tenant_id_)); - } else if (curr_tenant_info.get_sys_recovery_scn() >= sys_scn) { - sync_to_latest = true; - } else { - LOG_WARN("sys ls not recovery finish, need wait", K(tenant_id_), K(sys_scn), K(curr_tenant_info)); - usleep(10L * 1000L); - } - } - if (THIS_WORKER.is_timeout() || !sync_to_latest) { - // return NOT_ALLOW instead of timeout - ret = OB_OP_NOT_ALLOW; - LOG_WARN("wait sys ls recovery finish timeout, can not swithover to standby", KR(ret)); - LOG_USER_ERROR(OB_OP_NOT_ALLOW, "wait sys ls recovery timeout, switchover to standby"); - } else if (OB_SUCC(ret)) { - LOG_INFO("finish check sync to latest"); - } + //TODO, get all ls sync_scn, update __all_tenant_info, + //the new sync_scn cannot larger than recovery_scn and sync_scn of sys_ls } + LOG_INFO("[ROLE_TRANSITION] finish update tenant stat info", KR(ret), K(tenant_id_)); return ret; } -int ObTenantRoleTransitionService::change_ls_access_mode_(const share::ObLSStatusInfoIArray &status_info_array, - palf::AccessMode target_access_mode, +int ObTenantRoleTransitionService::change_ls_access_mode_(palf::AccessMode target_access_mode, const SCN &ref_scn) { int ret = OB_SUCCESS; @@ -577,11 +513,8 @@ int ObTenantRoleTransitionService::change_ls_access_mode_(const share::ObLSStatu need_retry = false; ret = OB_TIMEOUT; LOG_WARN("already timeout", KR(ret)); - } else if (OB_FAIL(get_ls_access_mode_(status_info_array, ls_mode_info))) { + } else if (OB_FAIL(get_ls_access_mode_(ls_mode_info))) { LOG_WARN("failed to get ls access mode", KR(ret)); - } else if (OB_UNLIKELY(status_info_array.count() != ls_mode_info.count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls count not equal to ls mode info count", KR(ret), K(status_info_array), K(ls_mode_info)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < ls_mode_info.count(); ++i) { const LSAccessModeInfo &info = ls_mode_info.at(i); @@ -626,8 +559,7 @@ int ObTenantRoleTransitionService::change_ls_access_mode_(const share::ObLSStatu } -int ObTenantRoleTransitionService::get_ls_access_mode_(const share::ObLSStatusInfoIArray &status_info_array, - ObIArray &ls_access_info) +int ObTenantRoleTransitionService::get_ls_access_mode_(ObIArray &ls_access_info) { int ret = OB_SUCCESS; ls_access_info.reset(); @@ -637,10 +569,14 @@ int ObTenantRoleTransitionService::get_ls_access_mode_(const share::ObLSStatusIn ret = OB_ERR_UNEXPECTED; LOG_WARN("location service is null", KR(ret)); } else { + share::ObLSStatusInfoArray status_info_array; ObLSStatusOperator status_op; ObTimeoutCtx ctx; if (OB_FAIL(ObShareUtil::set_default_timeout_ctx(ctx, GCONF.rpc_timeout))) { LOG_WARN("fail to set timeout ctx", KR(ret)); + } else if (OB_FAIL(status_op.get_all_ls_status_by_order_for_switch_tenant(tenant_id_, + false/* ignore_need_create_abort */, status_info_array, *sql_proxy_))) { + LOG_WARN("fail to get_all_ls_status_by_order", KR(ret), K_(tenant_id), KP(sql_proxy_)); } else { ObAddr leader; ObGetLSAccessModeProxy proxy( @@ -864,7 +800,6 @@ int ObTenantRoleTransitionService::switchover_update_tenant_status( const SCN final_sync_scn = MAX(max_checkpoint_scn, gts_upper_limit); const SCN final_replayable_scn = final_sync_scn; - const SCN final_sys_recovery_scn = SCN::max(final_sync_scn, max_sys_ls_sync_scn); SCN final_readable_scn = SCN::min_scn(); SCN final_recovery_until_scn = SCN::min_scn(); if (OB_FAIL(trans.start(sql_proxy_, exec_tenant_id))) { @@ -882,8 +817,8 @@ int ObTenantRoleTransitionService::switchover_update_tenant_status( // switch_to_primary // Does not change STS final_readable_scn = tmp_tenant_info.get_standby_scn(); - // To prevent unexpected sync log or recovery sys ls log, set recovery_until_scn = max(sync_scn, sys_recovery_scn) - final_recovery_until_scn = MAX(final_sync_scn, final_sys_recovery_scn); + // To prevent unexpected sync log, set recovery_until_scn = sync_scn + final_recovery_until_scn = final_sync_scn; } else { // switch_to_standby // STS >= GTS @@ -910,11 +845,10 @@ int ObTenantRoleTransitionService::switchover_update_tenant_status( final_replayable_scn, final_readable_scn, final_recovery_until_scn, - final_sys_recovery_scn, old_switchover_epoch))) { LOG_WARN("failed to update_tenant_status", KR(ret), K(tenant_id), K(new_role), K(old_status), K(new_status), K(final_sync_scn), K(final_replayable_scn), - K(final_readable_scn), K(final_recovery_until_scn), K(old_switchover_epoch), K(final_sys_recovery_scn)); + K(final_readable_scn), K(final_recovery_until_scn), K(old_switchover_epoch)); } else if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info( tenant_id, &trans, false /* for update */, new_tenant_info))) { LOG_WARN("failed to load tenant info", KR(ret), K(tenant_id)); @@ -964,17 +898,14 @@ int ObTenantRoleTransitionService::wait_tenant_sync_to_latest_until_timeout_( if (OB_FAIL(ret) || !has_restore_source) { } else { bool has_sync_to_latest = false; - ObAllTenantInfo new_tenant_info; - while (!THIS_WORKER.is_timeout() && !has_sync_to_latest) { - //ignore error - if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(tenant_id_, sql_proxy_, - false, new_tenant_info))) { - LOG_WARN("failed to load tenant info", KR(ret), K(tenant_id_)); - } else if (OB_FAIL(check_sync_to_latest_(tenant_id, new_tenant_info, has_sync_to_latest))) { + while (!THIS_WORKER.is_timeout()) { + has_sync_to_latest = false; + if (OB_FAIL(check_sync_to_latest_(tenant_id, tenant_info, has_sync_to_latest))) { LOG_WARN("fail to check_sync_to_latest_", KR(ret), K(tenant_id), - K(new_tenant_info), K(has_sync_to_latest)); + K(tenant_info), K(has_sync_to_latest)); } else if (has_sync_to_latest) { LOG_INFO("sync to latest", K(has_sync_to_latest), K(tenant_id)); + break; } else { LOG_WARN("not sync to latest, wait a while", K(tenant_id)); } @@ -1013,6 +944,7 @@ int ObTenantRoleTransitionService::check_sync_to_latest_(const uint64_t tenant_i share::ObLSStatusInfoArray sys_ls_status_array; common::ObArray switchover_checkpoints; ObLSRecoveryStatOperator ls_recovery_operator; + ObLSRecoveryStat sys_ls_recovery_stat; SCN sys_ls_sync_scn = SCN::min_scn(); bool sys_ls_sync_to_latest = false; share::ObLSStatusInfo ls_status; @@ -1035,12 +967,15 @@ int ObTenantRoleTransitionService::check_sync_to_latest_(const uint64_t tenant_i ret = OB_ERR_UNEXPECTED; LOG_WARN("checkpoints count is not 1", KR(ret), K(switchover_checkpoints), K(tenant_id), K(tenant_info), K(sys_ls_status_array)); + } else if (OB_FAIL(ls_recovery_operator.get_ls_recovery_stat(tenant_id, SYS_LS, + false/*for_update*/, sys_ls_recovery_stat, *sql_proxy_))) { + LOG_WARN("failed to get ls recovery stat", KR(ret), K(tenant_id)); } else if (OB_FAIL(get_sys_ls_sync_scn_(switchover_checkpoints, sys_ls_sync_scn, sys_ls_sync_to_latest))) { LOG_WARN("failed to get_sys_ls_sync_scn_", KR(ret), K(switchover_checkpoints)); } else if (!(sys_ls_sync_scn.is_valid_and_not_min() && sys_ls_sync_to_latest - && tenant_info.get_sys_recovery_scn() == sys_ls_sync_scn)) { + && sys_ls_recovery_stat.get_sync_scn() == sys_ls_sync_scn)) { LOG_WARN("sys ls not sync, keep waiting", KR(ret), K(sys_ls_sync_scn), K(sys_ls_sync_to_latest), - K(tenant_info), K(switchover_checkpoints)); + K(sys_ls_recovery_stat), K(switchover_checkpoints)); // SYS LS is sync, check other LS } else if (OB_FAIL(ls_status_op.get_all_ls_status_by_order_for_switch_tenant(tenant_id, true/* ignore_need_create_abort */, all_ls_status_array, *sql_proxy_))) { diff --git a/src/rootserver/ob_tenant_role_transition_service.h b/src/rootserver/ob_tenant_role_transition_service.h index 277788670e..8529c7d688 100644 --- a/src/rootserver/ob_tenant_role_transition_service.h +++ b/src/rootserver/ob_tenant_role_transition_service.h @@ -97,10 +97,10 @@ public: switch_optype_(switch_optype) {} virtual ~ObTenantRoleTransitionService() {} int failover_to_primary(); - int switchover_to_standby(const share::ObAllTenantInfo &tenant_info); int check_inner_stat(); int do_switch_access_mode_to_append(const share::ObAllTenantInfo &tenant_info, const share::ObTenantRole &target_tenant_role); + int do_switch_access_mode_to_raw_rw(const share::ObAllTenantInfo &tenant_info); void set_switchover_epoch(const int64_t switchover_epoch) { switchover_epoch_ = switchover_epoch; @@ -156,11 +156,10 @@ private: int do_failover_to_primary_(const share::ObAllTenantInfo &tenant_info); int do_prepare_flashback_(share::ObAllTenantInfo &tenant_info); int do_flashback_(const share::ObAllTenantInfo &tenant_info); - int change_ls_access_mode_(const share::ObLSStatusInfoIArray &status_info_array, - palf::AccessMode target_access_mode, + int change_ls_access_mode_(palf::AccessMode target_access_mode, const share::SCN &ref_scn); - int get_ls_access_mode_(const share::ObLSStatusInfoIArray &status_info_array, - ObIArray &ls_access_info); + int update_tenant_stat_info_(); + int get_ls_access_mode_(ObIArray &ls_access_info); int do_change_ls_access_mode_(const ObIArray &ls_access_info, palf::AccessMode target_access_mode, const share::SCN &ref_scn); @@ -220,8 +219,6 @@ private: int do_prepare_flashback_for_switch_to_primary_(share::ObAllTenantInfo &tenant_info); int do_prepare_flashback_for_failover_to_primary_(share::ObAllTenantInfo &tenant_info); - //switchover - int wait_sys_recovery_finish_(const ObLSStatusInfoIArray &ls_status); private: const static int64_t SEC_UNIT = 1000L * 1000L; const static int64_t PRINT_INTERVAL = 10 * 1000 * 1000L; diff --git a/src/rootserver/ob_tenant_thread_helper.cpp b/src/rootserver/ob_tenant_thread_helper.cpp deleted file mode 100755 index 0b7d78610f..0000000000 --- a/src/rootserver/ob_tenant_thread_helper.cpp +++ /dev/null @@ -1,323 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ - -#define USING_LOG_PREFIX RS -#include "ob_tenant_thread_helper.h" -#include "lib/profile/ob_trace_id.h" -#include "lib/thread/thread_mgr.h"//TG -#include "share/ob_errno.h" -#include "share/schema/ob_schema_struct.h"//ObTenantInfo -#include "share/schema/ob_schema_service.h"//ObMultiSchemaService -#include "share/schema/ob_schema_getter_guard.h"//ObSchemaGetterGuard -#include "share/ob_share_util.h"//ObShareUtil -#include "share/ob_tenant_info_proxy.h"//ObAllTenantInfo -#include "share/restore/ob_physical_restore_table_operator.h"//restore_job -#include "share/restore/ob_physical_restore_info.h"//restore_info -#include "share/ob_primary_zone_util.h"//get_ls_primary_zone_priority -#include "observer/ob_server_struct.h"//GCTX - -namespace oceanbase -{ -using namespace common; -using namespace share; -using namespace transaction; -using namespace palf; -using namespace lib; -namespace rootserver -{ -//////////////ObTenantThreadHelper -int ObTenantThreadHelper::create( - const char* thread_name, int tg_def_id, ObTenantThreadHelper &tenant_thread) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(is_created_)) { - ret = OB_INIT_TWICE; - LOG_WARN("has inited", KR(ret)); - } else if (OB_ISNULL(thread_name)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("thread name is null", KR(ret)); - } else if (OB_FAIL(TG_CREATE_TENANT(tg_def_id, tg_id_))) { - LOG_ERROR("create tg failed", KR(ret)); - } else if (OB_FAIL(TG_SET_RUNNABLE(tg_id_, *this))) { - LOG_ERROR("set thread runable fail", KR(ret)); - } else if (OB_FAIL(thread_cond_.init(ObWaitEventIds::REENTRANT_THREAD_COND_WAIT))) { - LOG_WARN("fail to init cond, ", KR(ret)); - } else { - thread_name_ = thread_name; - is_created_ = true; - is_first_time_to_start_ = true; - } - return ret; -} - -int ObTenantThreadHelper::start() -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!is_created_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret)); - } else if (is_first_time_to_start_) { - if (OB_FAIL(TG_START(tg_id_))) { - LOG_WARN("fail ed to start at first time", KR(ret), K(tg_id_), K(thread_name_)); - } else { - is_first_time_to_start_ = false; - } - } else if (OB_FAIL(TG_REENTRANT_LOGICAL_START(tg_id_))) { - LOG_WARN("failed to start", KR(ret)); - } - LOG_INFO("[TENANT THREAD] thread start", KR(ret), K(tg_id_), K(thread_name_)); - return ret; -} - -void ObTenantThreadHelper::stop() -{ - LOG_INFO("[TENANT THREAD] thread logical stop start", K(tg_id_), K(thread_name_)); - if (-1 != tg_id_) { - TG_REENTRANT_LOGICAL_STOP(tg_id_); - } - LOG_INFO("[TENANT THREAD] thread logical stop finish", K(tg_id_), K(thread_name_)); -} - -void ObTenantThreadHelper::wait() -{ - LOG_INFO("[TENANT THREAD] thread logical wait start", K(tg_id_), K(thread_name_)); - if (-1 != tg_id_) { - TG_REENTRANT_LOGICAL_WAIT(tg_id_); - } - LOG_INFO("[TENANT THREAD] thread logical wait finish", K(tg_id_), K(thread_name_)); -} -void ObTenantThreadHelper::mtl_thread_stop() -{ - LOG_INFO("[TENANT THREAD] thread stop start", K(tg_id_), K(thread_name_)); - if (-1 != tg_id_) { - TG_STOP(tg_id_); - } - LOG_INFO("[TENANT THREAD] thread stop finish", K(tg_id_), K(thread_name_)); -} - -void ObTenantThreadHelper::mtl_thread_wait() -{ - LOG_INFO("[TENANT THREAD] thread wait start", K(tg_id_), K(thread_name_)); - if (-1 != tg_id_) { - { - ObThreadCondGuard guard(thread_cond_); - thread_cond_.broadcast(); - } - TG_WAIT(tg_id_); - is_first_time_to_start_ = true; - } - LOG_INFO("[TENANT THREAD] thread wait finish", K(tg_id_), K(thread_name_)); -} - -void ObTenantThreadHelper::destroy() -{ - LOG_INFO("[TENANT THREAD] thread destory start", K(tg_id_), K(thread_name_)); - if (-1 != tg_id_) { - mtl_thread_stop(); - mtl_thread_wait(); - TG_DESTROY(tg_id_); - tg_id_ = -1; - } - is_created_ = false; - is_first_time_to_start_ = true; - LOG_INFO("[TENANT THREAD] thread destory finish", K(tg_id_), K(thread_name_)); -} - -void ObTenantThreadHelper::switch_to_follower_forcedly() -{ - stop(); -} -int ObTenantThreadHelper::switch_to_leader() -{ - int ret = OB_SUCCESS; - LOG_INFO("[TENANT THREAD] thread start", K(tg_id_), K(thread_name_)); - if (OB_FAIL(start())) { - LOG_WARN("failed to start thread", KR(ret)); - } else { - ObThreadCondGuard guard(thread_cond_); - if (OB_FAIL(thread_cond_.broadcast())) { - LOG_WARN("failed to weakup thread cond", KR(ret)); - } - } - LOG_INFO("[TENANT THREAD] thread start finish", K(tg_id_), K(thread_name_)); - return ret; -} -int ObTenantThreadHelper::wait_tenant_data_version_ready_( - const uint64_t tenant_id, const uint64_t &data_version) -{ - int ret = OB_SUCCESS; - bool is_ready = false; - uint64_t tenant_data_version = 0; - while (!is_ready && !has_set_stop()) { - ret = OB_SUCCESS; - - if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, tenant_data_version))) { - LOG_WARN("failed to get min data version", KR(ret), K(tenant_id)); - } else if (tenant_data_version < data_version) { - ret = OB_NEED_WAIT; - LOG_WARN("tenant version not target, need wait", KR(ret), - K(tenant_data_version), K(data_version)); - } else { - is_ready = true; - } - - if (!is_ready) { - idle(10 * 1000 * 1000); - } - } - - if (has_set_stop()) { - LOG_WARN("thread has been stopped", K(is_ready), K(tenant_id)); - ret = OB_IN_STOP_STATE; - } - return ret; -} - -int ObTenantThreadHelper::wait_tenant_schema_and_version_ready_( - const uint64_t tenant_id, const uint64_t &data_version) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(GCTX.schema_service_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("schema ptr is null", KR(ret), KP(GCTX.schema_service_)); - } else if (OB_FAIL(wait_tenant_data_version_ready_(tenant_id, data_version))) { - LOG_WARN("failed to wait tenant data version", KR(ret), K(tenant_id), K(data_version)); - } else { - bool is_ready = false; - share::schema::ObTenantSchema tenant_schema; - while (!is_ready && !has_set_stop()) { - ret = OB_SUCCESS; - if (OB_FAIL(get_tenant_schema(tenant_id, tenant_schema))) { - LOG_WARN("failed to get tenant schema", KR(ret), K(tenant_id)); - } else if (!tenant_schema.is_normal()) { - ret = OB_NEED_WAIT; - LOG_WARN("tenant schema not ready, no need tenant balance", KR(ret), K(tenant_schema)); - } else { - is_ready = true; - } - - if (!is_ready) { - idle(10 * 1000 *1000); - } - } - - if (has_set_stop()) { - LOG_WARN("thread has been stopped", K(is_ready), K(tenant_id)); - ret = OB_IN_STOP_STATE; - } - } - return ret; -} - -void ObTenantThreadHelper::run1() { - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!is_created_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", K(ret)); - } else { - lib::set_thread_name(thread_name_); - LOG_INFO("thread run", K(thread_name_)); - do_work(); - } -} -void ObTenantThreadHelper::idle(const int64_t idle_time_us) -{ - ObThreadCondGuard guard(thread_cond_); - thread_cond_.wait_us(idle_time_us); -} - -int ObTenantThreadHelper::get_tenant_schema(const uint64_t tenant_id, - share::schema::ObTenantSchema &tenant_schema) -{ - int ret = OB_SUCCESS; - share::schema::ObSchemaGetterGuard schema_guard; - const share::schema::ObTenantSchema *cur_tenant_schema = NULL; - if (!is_valid_tenant_id(tenant_id)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("tenant id is invalid", KR(ret), K(tenant_id)); - } else if (OB_ISNULL(GCTX.schema_service_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error", KR(ret), KP(GCTX.schema_service_)); - } else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard( - OB_SYS_TENANT_ID, schema_guard))) { - LOG_WARN("fail to get schema guard", KR(ret)); - } else if (OB_FAIL(schema_guard.get_tenant_info(tenant_id, - cur_tenant_schema))) { - LOG_WARN("failed to get tenant ids", KR(ret), K(tenant_id)); - } else if (OB_ISNULL(cur_tenant_schema)) { - ret = OB_TENANT_NOT_EXIST; - LOG_WARN("tenant not exist", KR(ret), K(tenant_id)); - } else if (OB_FAIL(tenant_schema.assign(*cur_tenant_schema))) { - LOG_WARN("failed to get cur tenant schema", KR(ret), KP(cur_tenant_schema)); - } - return ret; -} - -int ObTenantThreadHelper::check_can_do_recovery(const ObAllTenantInfo &tenant_info) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!tenant_info.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("tenant_info is invalid", KR(ret), K(tenant_info)); - } else if (tenant_info.get_sys_recovery_scn().is_min()) { - //during upgrade, not expected - ret = OB_NEED_RETRY; - LOG_WARN("sys recovery scn can not be min now", KR(ret), K(tenant_info)); - } else if (OB_ISNULL(GCTX.sql_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy is null", KR(ret)); - } else if (tenant_info.is_primary()) { - } else if (tenant_info.is_standby()) { - //standby can do recovery - } else if (tenant_info.is_restore()) { - //need to check success to create init ls - const uint64_t tenant_id = tenant_info.get_tenant_id(); - share::ObPhysicalRestoreTableOperator restore_table_operator; - share::ObPhysicalRestoreJob job_info; - if (FAILEDx(restore_table_operator.init(GCTX.sql_proxy_, tenant_id))) { - LOG_WARN("fail to init restore table operator", KR(ret), K(tenant_id)); - } else if (OB_FAIL(restore_table_operator.get_job_by_tenant_id(tenant_id, - job_info))) { - LOG_WARN("fail to get restore job", K(ret), K(tenant_id)); - } else if (job_info.is_valid_status_to_recovery()) { - //can do recovery - } else { - ret = OB_NEED_WAIT; - LOG_WARN("restore tenant not valid to recovery", KR(ret), K(job_info)); - } - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected tenant role", KR(ret), K(tenant_info)); - } - return ret; -} - -int ObTenantThreadHelper::get_zone_priority(const ObZone &primary_zone, - const share::schema::ObTenantSchema &tenant_schema, - ObSqlString &primary_zone_str) -{ - int ret = OB_SUCCESS; - primary_zone_str.reset(); - if (OB_UNLIKELY(!tenant_schema.is_valid() || primary_zone.is_empty())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(primary_zone), K(tenant_schema)); - } else if (OB_FAIL(ObPrimaryZoneUtil::get_ls_primary_zone_priority(primary_zone, - tenant_schema, primary_zone_str))) { - LOG_WARN("failed to get ls primary zone priority", KR(ret), K(primary_zone), K(tenant_schema)); - } - LOG_DEBUG("get zone priority", KR(ret), K(primary_zone_str), K(tenant_schema)); - return ret; -} - - -}//end of rootserver -} diff --git a/src/rootserver/ob_tenant_thread_helper.h b/src/rootserver/ob_tenant_thread_helper.h deleted file mode 100644 index a88c32d083..0000000000 --- a/src/rootserver/ob_tenant_thread_helper.h +++ /dev/null @@ -1,120 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ - -#ifndef OCEANBASE_ROOTSERVER_OB_TENANT_THREAD_HELPER_H -#define OCEANBASE_ROOTSERVER_OB_TENANT_THREAD_HELPER_H -#include "lib/thread/ob_reentrant_thread.h"//ObRsReentrantThread -#include "logservice/ob_log_base_type.h" -#include "share/ob_thread_mgr.h" //OBTGDefIDEnum -#include "lib/thread/thread_mgr_interface.h" // TGRunnable -#include "lib/lock/ob_thread_cond.h"//ObThreadCond -#include "common/ob_zone.h"//ObZone - - -namespace oceanbase -{ -namespace obrpc -{ -class ObSrvRpcProxy; -} -namespace common -{ -class ObSqlString; -} -namespace share -{ -class ObAllTenantInfo; -namespace schema -{ -class ObTenantSchema; -} -} -namespace logservice -{ -class ObLogHandler; -} - -namespace rootserver -{ -class ObTenantThreadHelper : public lib::TGRunnable, - public logservice::ObIRoleChangeSubHandler -{ -public: - ObTenantThreadHelper() : tg_id_(-1), thread_cond_(), is_created_(false), is_first_time_to_start_(true), thread_name_("") {} - virtual ~ObTenantThreadHelper() {} - virtual void do_work() = 0; - virtual void run1() override; - virtual void destroy(); - int start(); - void stop(); - void wait(); - void mtl_thread_stop(); - void mtl_thread_wait(); - int create(const char* thread_name, int tg_def_id, ObTenantThreadHelper &tenant_thread); - void idle(const int64_t idle_time_us); -public: - virtual void switch_to_follower_forcedly() override; - - virtual int switch_to_leader() override; - virtual int switch_to_follower_gracefully() override - { - stop(); - return OB_SUCCESS; - } - virtual int resume_leader() override - { - return OB_SUCCESS; - } -public: - static int get_tenant_schema(const uint64_t tenant_id, - share::schema::ObTenantSchema &tenant_schema); - static int check_can_do_recovery(const share::ObAllTenantInfo &tenant_info); - static int get_zone_priority(const ObZone &primary_zone, - const share::schema::ObTenantSchema &tenant_schema, - common::ObSqlString &primary_zone_str); -protected: - int wait_tenant_schema_and_version_ready_( - const uint64_t tenant_id, const uint64_t &data_version); - int wait_tenant_data_version_ready_( - const uint64_t tenant_id, const uint64_t &data_version); - int tg_id_; -private: - common::ObThreadCond thread_cond_; - bool is_created_; - bool is_first_time_to_start_; - const char* thread_name_; -}; - -#define DEFINE_MTL_FUNC(TYPE)\ - static int mtl_init(TYPE *&ka) {\ - int ret = OB_SUCCESS;\ - if (OB_ISNULL(ka)) {\ - ret = OB_ERR_UNEXPECTED;\ - } else if (OB_FAIL(ka->init())) {\ - }\ - return ret;\ - }\ - static void mtl_stop(TYPE *&ka) {\ - if (OB_NOT_NULL(ka)) {\ - ka->mtl_thread_stop();\ - }\ - }\ - static void mtl_wait(TYPE *&ka) {\ - if (OB_NOT_NULL(ka)) {\ - ka->mtl_thread_wait();\ - }\ - } -} -} - - -#endif /* !OCEANBASE_ROOTSERVER_OB_TENANT_THREAD_HELPER_H */ diff --git a/src/rootserver/restore/ob_restore_scheduler.cpp b/src/rootserver/restore/ob_restore_scheduler.cpp index 8d1f99ba56..951cac4ae6 100644 --- a/src/rootserver/restore/ob_restore_scheduler.cpp +++ b/src/rootserver/restore/ob_restore_scheduler.cpp @@ -15,8 +15,7 @@ #include "ob_restore_scheduler.h" #include "rootserver/ob_rs_event_history_table_operator.h" #include "rootserver/ob_unit_manager.h"//convert_pool_name_lis -#include "rootserver/ob_ls_service_helper.h"//create_new_ls_in_trans -#include "rootserver/ob_common_ls_service.h"//do_create_user_ls +#include "rootserver/ob_tenant_role_transition_service.h"//failover_to_primary #include "share/ob_schema_status_proxy.h" #include "share/schema/ob_schema_utils.h" #include "share/schema/ob_schema_mgr.h" @@ -955,16 +954,19 @@ int ObRestoreService::restore_init_ls(const share::ObPhysicalRestoreJob &job_inf } else if (OB_FAIL(store.read_ls_attr_info(backup_ls_attr))) { LOG_WARN("failed to read ls info", KR(ret)); } else { - const SCN &sys_recovery_scn = backup_ls_attr.backup_scn_; - ObAllTenantInfo tenant_info; - LOG_INFO("start to create ls and set sys_recovery_scn", K(sys_recovery_scn), K(backup_ls_attr)); - if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(tenant_id_, sql_proxy_, false, tenant_info))) { - LOG_WARN("failed to update tenant info", KR(ret), K(tenant_id_)); - } else if (tenant_info.get_sys_recovery_scn() >= sys_recovery_scn) { - //no need update sys recovery scn - } else if (OB_FAIL(ObAllTenantInfoProxy::update_tenant_sys_recovery_scn(tenant_id_, sys_recovery_scn, - false, sql_proxy_))) { - LOG_WARN("failed to update tenant sys recovery scn", KR(ret), K(tenant_id_), K(sys_recovery_scn), K(tenant_info)); + const SCN &sync_scn = backup_ls_attr.backup_scn_; + const SCN readable_scn = SCN::base_scn(); + ObLSRecoveryStatOperator ls_recovery; + ObLSRecoveryStat ls_recovery_stat; + LOG_INFO("start to create ls and set sync scn", K(sync_scn), K(backup_ls_attr)); + if (OB_FAIL(ls_recovery_stat.init_only_recovery_stat( + tenant_id_, SYS_LS, sync_scn, readable_scn))) { + LOG_WARN("failed to init ls recovery stat", KR(ret), K(backup_ls_attr.backup_scn_), + K(sync_scn), K(readable_scn)); + } else if (OB_FAIL(ls_recovery.update_ls_recovery_stat(ls_recovery_stat, + *sql_proxy_))) { + LOG_WARN("failed to update ls recovery stat", KR(ret), + K(ls_recovery_stat)); } } if (FAILEDx(create_all_ls_(*tenant_schema, backup_ls_attr.ls_attr_array_))) { @@ -1032,7 +1034,7 @@ int ObRestoreService::create_all_ls_( LOG_INFO("[RESTORE] ls already exist", K(ls_info), K(tenant_id_)); } else if (OB_ENTRY_NOT_EXIST != ret) { LOG_WARN("failed to get ls status info", KR(ret), K(tenant_id_), K(ls_info)); - } else if (OB_FAIL(tenant_stat.create_new_ls_in_trans( + } else if (OB_FAIL(tenant_stat.create_new_ls_for_recovery( ls_info.get_ls_id(), ls_info.get_ls_group_id(), ls_info.get_create_scn(), trans))) { LOG_WARN("failed to add new ls status info", KR(ret), K(ls_info)); @@ -1072,10 +1074,12 @@ int ObRestoreService::wait_all_ls_created_(const share::schema::ObTenantSchema & ObLSRecoveryStat recovery_stat; ObLSRecoveryStatOperator ls_recovery_operator; + ObTenantLSInfo tenant_stat(sql_proxy_, &tenant_schema, tenant_id_, + srv_rpc_proxy_, GCTX.lst_operator_); if (OB_FAIL(status_op.get_all_ls_status_by_order(tenant_id, ls_array, *sql_proxy_))) { LOG_WARN("failed to get all ls status", KR(ret), K(tenant_id)); - } + } for (int64_t i = 0; OB_SUCC(ret) && i < ls_array.count(); ++i) { const ObLSStatusInfo &info = ls_array.at(i); if (info.ls_is_creating()) { @@ -1087,11 +1091,10 @@ int ObRestoreService::wait_all_ls_created_(const share::schema::ObTenantSchema & job_info, info.ls_id_, palf_base_info))) { LOG_WARN("failed to get restore ls palf info", KR(ret), K(info), K(job_info)); - } else if (OB_FAIL(ObCommonLSService::do_create_user_ls( - tenant_schema, info, recovery_stat.get_create_scn(), - true, /*create with palf*/ - palf_base_info))) { - LOG_WARN("failed to create ls with palf", KR(ret), K(info), K(tenant_schema), + } else if (OB_FAIL(tenant_stat.create_ls_with_palf(info, recovery_stat.get_create_scn(), + true,/*create with palf*/ + palf_base_info))) { + LOG_WARN("failed to create ls with palf", KR(ret), K(info), K(palf_base_info)); } } diff --git a/src/rootserver/restore/ob_restore_scheduler.h b/src/rootserver/restore/ob_restore_scheduler.h index 3ff790b870..46040a8892 100644 --- a/src/rootserver/restore/ob_restore_scheduler.h +++ b/src/rootserver/restore/ob_restore_scheduler.h @@ -15,7 +15,7 @@ #include "share/backup/ob_backup_info_mgr.h" #include "rootserver/restore/ob_restore_util.h" -#include "rootserver/ob_tenant_thread_helper.h"//ObTenantThreadHelper +#include "rootserver/ob_primary_ls_service.h"//ObTenantThreadHelper #include "share/backup/ob_backup_struct.h" #include "share/ob_rpc_struct.h" #include "share/ob_common_rpc_proxy.h" diff --git a/src/share/inner_table/ob_inner_table_schema.12251_12300.cpp b/src/share/inner_table/ob_inner_table_schema.12251_12300.cpp index e2685d4eee..ed52ddc96a 100644 --- a/src/share/inner_table/ob_inner_table_schema.12251_12300.cpp +++ b/src/share/inner_table/ob_inner_table_schema.12251_12300.cpp @@ -1377,25 +1377,6 @@ int ObInnerTableSchema::all_virtual_tenant_info_schema(ObTableSchema &table_sche log_mode_default, log_mode_default); //default_value } - - if (OB_SUCC(ret)) { - ObObj sys_recovery_scn_default; - sys_recovery_scn_default.set_uint64(OB_MIN_SCN_TS_NS); - ADD_COLUMN_SCHEMA_T("sys_recovery_scn", //column_name - ++column_id, //column_id - 0, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObUInt64Type, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(uint64_t), //column_length - -1, //column_precision - -1, //column_scale - false, //is_nullable - false, //is_autoincrement - sys_recovery_scn_default, - sys_recovery_scn_default); //default_value - } table_schema.set_index_using_type(USING_BTREE); table_schema.set_row_store_type(ENCODING_ROW_STORE); table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); diff --git a/src/share/inner_table/ob_inner_table_schema.15251_15300.cpp b/src/share/inner_table/ob_inner_table_schema.15251_15300.cpp index 179ae233f3..38cc7fce79 100644 --- a/src/share/inner_table/ob_inner_table_schema.15251_15300.cpp +++ b/src/share/inner_table/ob_inner_table_schema.15251_15300.cpp @@ -4777,21 +4777,6 @@ int ObInnerTableSchema::all_virtual_tenant_info_ora_schema(ObTableSchema &table_ false, //is_nullable false); //is_autoincrement } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("SYS_RECOVERY_SCN", //column_name - ++column_id, //column_id - 0, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false, //is_nullable - false); //is_autoincrement - } table_schema.set_index_using_type(USING_BTREE); table_schema.set_row_store_type(ENCODING_ROW_STORE); table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); diff --git a/src/share/inner_table/ob_inner_table_schema.21151_21200.cpp b/src/share/inner_table/ob_inner_table_schema.21151_21200.cpp index e40c0ebc31..160a455772 100644 --- a/src/share/inner_table/ob_inner_table_schema.21151_21200.cpp +++ b/src/share/inner_table/ob_inner_table_schema.21151_21200.cpp @@ -410,7 +410,7 @@ int ObInnerTableSchema::dba_ob_tenants_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT A.TENANT_ID, TENANT_NAME, (CASE WHEN A.TENANT_ID = 1 THEN 'SYS' WHEN (A.TENANT_ID & 0x1) = 1 THEN 'META' ELSE 'USER' END) AS TENANT_TYPE, A.gmt_create AS CREATE_TIME, A.gmt_modified AS MODIFY_TIME, PRIMARY_ZONE, LOCALITY, CASE previous_locality WHEN "" THEN NULL ELSE previous_locality END AS PREVIOUS_LOCALITY, CASE compatibility_mode WHEN 0 THEN 'MYSQL' WHEN 1 THEN 'ORACLE' ELSE NULL END AS COMPATIBILITY_MODE, STATUS, CASE in_recyclebin WHEN 0 THEN 'NO' ELSE 'YES' END AS IN_RECYCLEBIN, CASE locked WHEN 0 THEN 'NO' ELSE 'YES' END AS LOCKED, (CASE WHEN A.TENANT_ID = 1 THEN 'PRIMARY' WHEN (A.TENANT_ID & 0x1) = 1 THEN 'PRIMARY' ELSE TENANT_ROLE END) AS TENANT_ROLE, (CASE WHEN A.TENANT_ID = 1 THEN 'NORMAL' WHEN (A.TENANT_ID & 0x1) = 1 THEN 'NORMAL' ELSE SWITCHOVER_STATUS END) AS SWITCHOVER_STATUS, (CASE WHEN A.TENANT_ID = 1 THEN 0 WHEN (A.TENANT_ID & 0x1) = 1 THEN 0 ELSE SWITCHOVER_EPOCH END) AS SWITCHOVER_EPOCH, (CASE WHEN A.TENANT_ID = 1 THEN NULL WHEN (A.TENANT_ID & 0x1) = 1 THEN NULL ELSE SYNC_SCN END) AS SYNC_SCN, (CASE WHEN A.TENANT_ID = 1 THEN NULL WHEN (A.TENANT_ID & 0x1) = 1 THEN NULL ELSE REPLAYABLE_SCN END) AS REPLAYABLE_SCN, (CASE WHEN A.TENANT_ID = 1 THEN NULL WHEN (A.TENANT_ID & 0x1) = 1 THEN NULL ELSE READABLE_SCN END) AS READABLE_SCN, (CASE WHEN A.TENANT_ID = 1 THEN NULL WHEN (A.TENANT_ID & 0x1) = 1 THEN NULL ELSE RECOVERY_UNTIL_SCN END) AS RECOVERY_UNTIL_SCN, (CASE WHEN A.TENANT_ID = 1 THEN NULL WHEN (A.TENANT_ID & 0x1) = 1 THEN NULL ELSE SYS_RECOVERY_SCN END) AS SYS_RECOVERY_SCN, (CASE WHEN A.TENANT_ID = 1 THEN 'NOARCHIVELOG' WHEN (A.TENANT_ID & 0x1) = 1 THEN 'NOARCHIVELOG' ELSE LOG_MODE END) AS LOG_MODE, ARBITRATION_SERVICE_STATUS FROM OCEANBASE.__ALL_VIRTUAL_TENANT_MYSQL_SYS_AGENT AS A LEFT JOIN OCEANBASE.__ALL_VIRTUAL_TENANT_INFO AS B ON A.TENANT_ID = B.TENANT_ID )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT A.TENANT_ID, TENANT_NAME, (CASE WHEN A.TENANT_ID = 1 THEN 'SYS' WHEN (A.TENANT_ID & 0x1) = 1 THEN 'META' ELSE 'USER' END) AS TENANT_TYPE, A.gmt_create AS CREATE_TIME, A.gmt_modified AS MODIFY_TIME, PRIMARY_ZONE, LOCALITY, CASE previous_locality WHEN "" THEN NULL ELSE previous_locality END AS PREVIOUS_LOCALITY, CASE compatibility_mode WHEN 0 THEN 'MYSQL' WHEN 1 THEN 'ORACLE' ELSE NULL END AS COMPATIBILITY_MODE, STATUS, CASE in_recyclebin WHEN 0 THEN 'NO' ELSE 'YES' END AS IN_RECYCLEBIN, CASE locked WHEN 0 THEN 'NO' ELSE 'YES' END AS LOCKED, (CASE WHEN A.TENANT_ID = 1 THEN 'PRIMARY' WHEN (A.TENANT_ID & 0x1) = 1 THEN 'PRIMARY' ELSE TENANT_ROLE END) AS TENANT_ROLE, (CASE WHEN A.TENANT_ID = 1 THEN 'NORMAL' WHEN (A.TENANT_ID & 0x1) = 1 THEN 'NORMAL' ELSE SWITCHOVER_STATUS END) AS SWITCHOVER_STATUS, (CASE WHEN A.TENANT_ID = 1 THEN 0 WHEN (A.TENANT_ID & 0x1) = 1 THEN 0 ELSE SWITCHOVER_EPOCH END) AS SWITCHOVER_EPOCH, (CASE WHEN A.TENANT_ID = 1 THEN NULL WHEN (A.TENANT_ID & 0x1) = 1 THEN NULL ELSE SYNC_SCN END) AS SYNC_SCN, (CASE WHEN A.TENANT_ID = 1 THEN NULL WHEN (A.TENANT_ID & 0x1) = 1 THEN NULL ELSE REPLAYABLE_SCN END) AS REPLAYABLE_SCN, (CASE WHEN A.TENANT_ID = 1 THEN NULL WHEN (A.TENANT_ID & 0x1) = 1 THEN NULL ELSE READABLE_SCN END) AS READABLE_SCN, (CASE WHEN A.TENANT_ID = 1 THEN NULL WHEN (A.TENANT_ID & 0x1) = 1 THEN NULL ELSE RECOVERY_UNTIL_SCN END) AS RECOVERY_UNTIL_SCN, (CASE WHEN A.TENANT_ID = 1 THEN 'NOARCHIVELOG' WHEN (A.TENANT_ID & 0x1) = 1 THEN 'NOARCHIVELOG' ELSE LOG_MODE END) AS LOG_MODE, ARBITRATION_SERVICE_STATUS FROM OCEANBASE.__ALL_VIRTUAL_TENANT_MYSQL_SYS_AGENT AS A LEFT JOIN OCEANBASE.__ALL_VIRTUAL_TENANT_INFO AS B ON A.TENANT_ID = B.TENANT_ID )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema.351_400.cpp b/src/share/inner_table/ob_inner_table_schema.351_400.cpp index adfbbd1253..b4698cd2e0 100644 --- a/src/share/inner_table/ob_inner_table_schema.351_400.cpp +++ b/src/share/inner_table/ob_inner_table_schema.351_400.cpp @@ -5775,25 +5775,6 @@ int ObInnerTableSchema::all_tenant_info_schema(ObTableSchema &table_schema) log_mode_default, log_mode_default); //default_value } - - if (OB_SUCC(ret)) { - ObObj sys_recovery_scn_default; - sys_recovery_scn_default.set_uint64(OB_MIN_SCN_TS_NS); - ADD_COLUMN_SCHEMA_T("sys_recovery_scn", //column_name - ++column_id, //column_id - 0, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObUInt64Type, //column_type - CS_TYPE_INVALID, //column_collation_type - sizeof(uint64_t), //column_length - -1, //column_precision - -1, //column_scale - false, //is_nullable - false, //is_autoincrement - sys_recovery_scn_default, - sys_recovery_scn_default); //default_value - } table_schema.set_index_using_type(USING_BTREE); table_schema.set_row_store_type(ENCODING_ROW_STORE); table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index 232b2f3c5f..bf50ba88db 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -4050,7 +4050,6 @@ def_table_schema( ('readable_scn', 'uint'), ('recovery_until_scn', 'uint', 'false', 'OB_MAX_SCN_TS_NS'), ('log_mode', 'varchar:100', 'false', 'NOARCHIVELOG'), - ("sys_recovery_scn", 'uint', 'false', 'OB_MIN_SCN_TS_NS'), ], ) @@ -15457,12 +15456,6 @@ SELECT A.TENANT_ID, ELSE RECOVERY_UNTIL_SCN END) AS RECOVERY_UNTIL_SCN, - (CASE - WHEN A.TENANT_ID = 1 THEN NULL - WHEN (A.TENANT_ID & 0x1) = 1 THEN NULL - ELSE SYS_RECOVERY_SCN - END) AS SYS_RECOVERY_SCN, - (CASE WHEN A.TENANT_ID = 1 THEN 'NOARCHIVELOG' WHEN (A.TENANT_ID & 0x1) = 1 THEN 'NOARCHIVELOG' diff --git a/src/share/ls/ob_ls_i_life_manager.h b/src/share/ls/ob_ls_i_life_manager.h index 99212bc113..47b307d705 100644 --- a/src/share/ls/ob_ls_i_life_manager.h +++ b/src/share/ls/ob_ls_i_life_manager.h @@ -241,44 +241,7 @@ int ObLSTemplateOperator::exec_write(const uint64_t &tenant_id, return ret; } -#define DEFINE_IN_TRANS_FUC(func_name, ...)\ -int func_name ##_in_trans(__VA_ARGS__, ObMySQLTransaction &trans);\ -int func_name(__VA_ARGS__); -#define DEFINE_IN_TRANS_FUC1(func_name, ...)\ -static int func_name ##_in_trans(__VA_ARGS__, ObMySQLTransaction &trans);\ -static int func_name(__VA_ARGS__, ObISQLClient *proxy); - -#define TAKE_IN_TRANS(func_name, proxy, exec_tenant_id, ...)\ -do {\ - ObMySQLTransaction trans; \ - if (FAILEDx(trans.start(proxy, exec_tenant_id))) {\ - SHARE_LOG(WARN, "failed to start trans", KR(ret), K(exec_tenant_id));\ - } else if (OB_FAIL(func_name##_in_trans(__VA_ARGS__, trans))) {\ - SHARE_LOG(WARN, "failed to do it in trans", KR(ret));\ - }\ - if (trans.is_started()) {\ - int tmp_ret = OB_SUCCESS;\ - if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) {\ - SHARE_LOG(WARN, "failed to commit trans", KR(ret), KR(tmp_ret));\ - ret = OB_SUCC(ret) ? tmp_ret : ret;\ - }\ - }\ -} while (0) - -#define START_TRANSACTION(proxy, exec_tenant_id) \ - ObMySQLTransaction trans; \ - if (FAILEDx(trans.start(proxy, exec_tenant_id))) { \ - SHARE_LOG(WARN, "failed to start trans", KR(ret), K(exec_tenant_id)); \ - } -#define END_TRANSACTION(trans)\ - if (trans.is_started()) {\ - int tmp_ret = OB_SUCCESS;\ - if (OB_TMP_FAIL(trans.end(OB_SUCC(ret)))) {\ - ret = OB_SUCC(ret) ? tmp_ret : ret;\ - SHARE_LOG(WARN, "failed to end trans", KR(ret), K(tmp_ret));\ - }\ - } } } diff --git a/src/share/ls/ob_ls_life_manager.cpp b/src/share/ls/ob_ls_life_manager.cpp index 938d215642..bb2c1f0e9d 100644 --- a/src/share/ls/ob_ls_life_manager.cpp +++ b/src/share/ls/ob_ls_life_manager.cpp @@ -34,13 +34,21 @@ int ObLSLifeAgentManager::create_new_ls( if (OB_UNLIKELY(!ls_info.is_valid() || !create_ls_scn.is_valid() || zone_priority.empty())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(ls_info), K(create_ls_scn), K(zone_priority)); - } else { - TAKE_IN_TRANS(create_new_ls, proxy_, - exec_tenant_id, ls_info, create_ls_scn, zone_priority, working_sw_status); + } else if (OB_FAIL(trans.start(proxy_, exec_tenant_id))) { + LOG_WARN("failed to start trans", KR(ret), K(exec_tenant_id), K(ls_info)); + } else if (OB_FAIL(create_new_ls_in_trans(ls_info, create_ls_scn, zone_priority, working_sw_status, trans))) { + LOG_WARN("failed to create new ls", KR(ret), K(ls_info), K(create_ls_scn), K(zone_priority)); } + if (trans.is_started()) { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) { + LOG_WARN("failed to commit trans", KR(ret), KR(tmp_ret)); + ret = OB_SUCC(ret) ? tmp_ret : ret; + } + } + return ret; } - int ObLSLifeAgentManager::drop_ls(const uint64_t &tenant_id, const share::ObLSID &ls_id, const ObTenantSwitchoverStatus &working_sw_status) @@ -51,10 +59,19 @@ int ObLSLifeAgentManager::drop_ls(const uint64_t &tenant_id, if (OB_UNLIKELY(!ls_id.is_valid() || OB_INVALID_TENANT_ID == tenant_id)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(ls_id), K(tenant_id)); - } else { - TAKE_IN_TRANS(drop_ls, proxy_, exec_tenant_id, tenant_id, - ls_id, working_sw_status); + } else if (OB_FAIL(trans.start(proxy_, exec_tenant_id))) { + LOG_WARN("failed to start trans", KR(ret), K(exec_tenant_id), K(tenant_id)); + } else if (OB_FAIL(drop_ls_in_trans(tenant_id, ls_id, working_sw_status, trans))) { + LOG_WARN("failed to create new ls", KR(ret), K(tenant_id), K(ls_id)); } + if (trans.is_started()) { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) { + LOG_WARN("failed to commit trans", KR(ret), KR(tmp_ret)); + ret = OB_SUCC(ret) ? tmp_ret : ret; + } + } + return ret; } @@ -73,10 +90,22 @@ int ObLSLifeAgentManager::set_ls_offline(const uint64_t &tenant_id, || (!ls_is_dropping_status(ls_status) && !ls_is_tenant_dropping_status(ls_status)))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(ls_id), K(tenant_id), K(drop_scn), K(ls_status)); - } else { - TAKE_IN_TRANS(set_ls_offline, proxy_, exec_tenant_id, tenant_id, - ls_id, ls_status, drop_scn, working_sw_status); + } else if (OB_FAIL(trans.start(proxy_, exec_tenant_id))) { + LOG_WARN("failed to start trans", KR(ret), K(exec_tenant_id), K(tenant_id)); + } else if (OB_FAIL(set_ls_offline_in_trans(tenant_id, ls_id, ls_status, drop_scn, + working_sw_status, trans))) { + LOG_WARN("failed to create new ls", KR(ret), K(tenant_id), K(ls_id), + K(ls_status), K(drop_scn)); } + + if (trans.is_started()) { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) { + LOG_WARN("failed to commit trans", KR(ret), KR(tmp_ret)); + ret = OB_SUCC(ret) ? tmp_ret : ret; + } + } + return ret; } diff --git a/src/share/ls/ob_ls_life_manager.h b/src/share/ls/ob_ls_life_manager.h index 937e9eb0b8..9097f48949 100644 --- a/src/share/ls/ob_ls_life_manager.h +++ b/src/share/ls/ob_ls_life_manager.h @@ -71,19 +71,19 @@ public: * @param[in]zone_priority: the primary_zone of OB_ALL_LS_ELECTION_REFERENCE_INFO * @param[in] working_sw_status only support working on specified switchover status * */ - DEFINE_IN_TRANS_FUC(create_new_ls, const ObLSStatusInfo &ls_info,\ - const SCN &create_ls_scn,\ - const common::ObString &zone_priority,\ - const share::ObTenantSwitchoverStatus &working_sw_status) + int create_new_ls(const ObLSStatusInfo &ls_info, + const SCN &create_ls_scn, + const common::ObString &zone_priority, + const share::ObTenantSwitchoverStatus &working_sw_status); /* * description: for primary cluster and GC of standby, delete ls from each inner_table * @param[in] tenant_id: tenant_id * @param[in] ls_id: need delete ls * @param[in] working_sw_status only support working on specified switchover status * */ - DEFINE_IN_TRANS_FUC(drop_ls, const uint64_t &tenant_id,\ - const share::ObLSID &ls_id,\ - const ObTenantSwitchoverStatus &working_sw_status) + int drop_ls(const uint64_t &tenant_id, + const share::ObLSID &ls_id, + const ObTenantSwitchoverStatus &working_sw_status); /* * description: for primary cluster set ls to wait offline from tenant_dropping or dropping status * @param[in] tenant_id: tenant_id @@ -92,11 +92,11 @@ public: * @param[in] drop_scn: there is no user data after drop_scn except offline * @param[in] working_sw_status only support working on specified switchover status * */ - DEFINE_IN_TRANS_FUC(set_ls_offline, const uint64_t &tenant_id,\ - const share::ObLSID &ls_id,\ - const ObLSStatus &ls_status,\ - const SCN &drop_scn,\ - const ObTenantSwitchoverStatus &working_sw_status) + int set_ls_offline(const uint64_t &tenant_id, + const share::ObLSID &ls_id, + const ObLSStatus &ls_status, + const SCN &drop_scn, + const ObTenantSwitchoverStatus &working_sw_status); /* * description: update ls primary zone, need update __all_ls_status and __all_ls_election_reference * @param[in] tenant_id: tenant_id @@ -110,6 +110,32 @@ public: const common::ObZone &primary_zone, const common::ObString &zone_priority); +public: + /* + * description: for standby cluster, create new ls + */ + int create_new_ls_in_trans(const ObLSStatusInfo &ls_info, + const SCN &create_ls_scn, + const common::ObString &zone_priority, + const share::ObTenantSwitchoverStatus &working_sw_status, + ObMySQLTransaction &trans); + /* + * description: for standby cluster, create new ls + */ + int drop_ls_in_trans(const uint64_t &tenant_id, + const share::ObLSID &ls_id, + const ObTenantSwitchoverStatus &working_sw_status, + ObMySQLTransaction &trans); + /* + * description: for standby cluster set ls to offline + * */ + int set_ls_offline_in_trans(const uint64_t &tenant_id, + const share::ObLSID &ls_id, + const ObLSStatus &ls_status, + const SCN &drop_scn, + const ObTenantSwitchoverStatus &working_sw_status, + ObMySQLTransaction &trans); + private: bool inited_; //table_operator diff --git a/src/share/ls/ob_ls_operator.cpp b/src/share/ls/ob_ls_operator.cpp index 9ca3d8602f..4c461361b9 100644 --- a/src/share/ls/ob_ls_operator.cpp +++ b/src/share/ls/ob_ls_operator.cpp @@ -116,15 +116,17 @@ bool ObLSAttrOperator::is_valid() const int ObLSAttrOperator::operator_ls_( const ObLSAttr &ls_attr, const common::ObSqlString &sql, - const uint64_t target_max_ls_group_id) + const uint64_t target_max_ls_group_id, + const ObTenantSwitchoverStatus &working_sw_status) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!ls_attr.is_valid() || sql.empty() - || OB_INVALID_ID == target_max_ls_group_id)) { + || OB_INVALID_ID == target_max_ls_group_id + || !working_sw_status.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("operation is invalid", KR(ret), K(ls_attr), K(sql), - K(target_max_ls_group_id)); + K(target_max_ls_group_id), K(working_sw_status)); } else { ObMySQLTransaction trans; const bool for_update = true; @@ -144,6 +146,10 @@ int ObLSAttrOperator::operator_ls_( LOG_WARN("failed to load sys ls status", KR(ret)); } else if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(tenant_id_, proxy_, false /* for_update */, tenant_info))) { LOG_WARN("failed to load tenant info", KR(ret), K_(tenant_id)); + } else if (working_sw_status != tenant_info.get_switchover_status()) { + ret = OB_NEED_RETRY; + LOG_WARN("tenant not in specified switchover status", K_(tenant_id), K(working_sw_status), + K(tenant_info)); } else if (OB_LS_OP_CREATE_PRE == ls_attr.get_ls_operation_type()) { if (OB_LS_NORMAL != sys_ls_attr.get_ls_status()) { //for sys ls, need insert_ls, but ls_status is normal @@ -190,7 +196,8 @@ int ObLSAttrOperator::operator_ls_( int ObLSAttrOperator::insert_ls( const ObLSAttr &ls_attr, - const uint64_t max_ls_group_id) + const uint64_t max_ls_group_id, + const ObTenantSwitchoverStatus &working_sw_status) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!ls_attr.is_valid() || OB_INVALID_ID == max_ls_group_id)) { @@ -208,7 +215,7 @@ int ObLSAttrOperator::insert_ls( ObLSStatusOperator::ls_status_to_str(ls_attr.get_ls_status()), "", ls_attr.get_create_scn().get_val_for_inner_table_field()))) { LOG_WARN("failed to assign sql", KR(ret), K(ls_attr), K(sql)); - } else if (OB_FAIL(operator_ls_(ls_attr, sql, max_ls_group_id))) { + } else if (OB_FAIL(operator_ls_(ls_attr, sql, max_ls_group_id, working_sw_status))) { LOG_WARN("failed to operator ls", KR(ret), K(ls_attr), K(sql)); } } @@ -217,7 +224,8 @@ int ObLSAttrOperator::insert_ls( } int ObLSAttrOperator::delete_ls( - const ObLSID &ls_id, const share::ObLSStatus &old_status) + const ObLSID &ls_id, const share::ObLSStatus &old_status, + const ObTenantSwitchoverStatus &working_sw_status) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!ls_id.is_valid() @@ -250,7 +258,8 @@ int ObLSAttrOperator::delete_ls( ls_attr.get_ls_flag(), ls_attr.get_ls_status(), operation_type, SCN::base_scn()))) { LOG_WARN("failed to init new ls attr", KR(ret), K(ls_id), K(ls_attr), K(operation_type)); - } else if (OB_FAIL(operator_ls_(new_ls_attr, sql, new_ls_attr.get_ls_group_id()))) { + } else if (OB_FAIL(operator_ls_(new_ls_attr, sql, new_ls_attr.get_ls_group_id(), + working_sw_status))) { LOG_WARN("failed to operator ls", KR(ret), K(new_ls_attr), K(sql)); } LOG_INFO("[LS_OPERATOR] delete ls", KR(ret), K(ls_id), K(old_status)); @@ -314,7 +323,8 @@ int ObLSAttrOperator::process_sub_trans_(const ObLSAttr &ls_attr, ObMySQLTransac int ObLSAttrOperator::update_ls_status(const ObLSID &id, const share::ObLSStatus &old_status, - const share::ObLSStatus &new_status) + const share::ObLSStatus &new_status, + const ObTenantSwitchoverStatus &working_sw_status) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!id.is_valid() @@ -343,7 +353,8 @@ int ObLSAttrOperator::update_ls_status(const ObLSID &id, operation_type, SCN::base_scn()))) { LOG_WARN("failed to init new ls attr", KR(ret), K(id), K(ls_attr), K(operation_type)); - } else if (OB_FAIL(operator_ls_(new_ls_attr, sql, new_ls_attr.get_ls_group_id()))) { + } else if (OB_FAIL(operator_ls_(new_ls_attr, sql, new_ls_attr.get_ls_group_id(), + working_sw_status))) { LOG_WARN("failed to operator ls", KR(ret), K(new_ls_attr), K(sql)); } LOG_INFO("[LS_OPERATOR] update ls status", KR(ret), K(ls_attr), K(new_ls_attr)); @@ -538,6 +549,43 @@ int ObLSAttrOperator::get_tenant_gts(const uint64_t &tenant_id, SCN >s_scn) return ret; } +int ObLSAttrOperator::get_all_ls_by_order(const bool lock_sys_ls, ObLSAttrIArray &ls_operation_array) +{ + int ret = OB_SUCCESS; + ls_operation_array.reset(); + ObMySQLTransaction trans; + ObLSAttr sys_ls_attr; + + if (OB_UNLIKELY(!is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("operation is not valid", KR(ret), "operation", *this); + } else if (OB_FAIL(trans.start(proxy_, tenant_id_))) { + LOG_WARN("failed to start transaction", KR(ret), K_(tenant_id)); + /* to get accurate LS list need lock SYS_LS */ + } else if (lock_sys_ls && OB_FAIL(get_ls_attr(SYS_LS, true /* for_update */, trans, sys_ls_attr))) { + LOG_WARN("failed to load sys ls status", KR(ret)); + } else { + ObSqlString sql; + if (OB_FAIL(sql.assign_fmt( + "select * from %s order by ls_id", + OB_ALL_LS_TNAME))) { + LOG_WARN("failed to assign sql", KR(ret), K(sql)); + } else if (OB_FAIL(exec_read(tenant_id_, sql, trans, this, ls_operation_array))) { + LOG_WARN("failed to construct ls attr", KR(ret), K(sql), K_(tenant_id)); + } + } + + if (trans.is_started()) { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) { + LOG_WARN("failed to end trans", KR(ret), KR(tmp_ret)); + ret = OB_SUCC(ret) ? tmp_ret : ret; + } + } + + return ret; +} + } } diff --git a/src/share/ls/ob_ls_operator.h b/src/share/ls/ob_ls_operator.h index dddc42639d..b10b81d6b4 100644 --- a/src/share/ls/ob_ls_operator.h +++ b/src/share/ls/ob_ls_operator.h @@ -105,6 +105,10 @@ struct ObLSAttr { return ls_is_normal_status(status_); } + ObLSOperationType get_ls_operatin_type() const + { + return operation_type_; + } ObLSID get_ls_id() const { return id_; @@ -168,11 +172,24 @@ public: bool is_valid() const; int get_all_ls_by_order( ObLSAttrIArray &ls_array); - int insert_ls(const ObLSAttr &ls_attr, const uint64_t max_ls_group_id); + /** + * @description: + * get ls list from all_ls table, + * if want to get accurate LS list, set lock_sys_ls to true to lock SYS LS in __all_ls table + * to make sure mutual exclusion with load balancing thread + * @param[in] lock_sys_ls whether lock SYS LS in __all_ls table + * @param[out] ls_operation_array ls list + * @return return code + */ + int get_all_ls_by_order(const bool lock_sys_ls, ObLSAttrIArray &ls_operation_array); + int insert_ls(const ObLSAttr &ls_attr, const uint64_t max_ls_group_id, + const ObTenantSwitchoverStatus &working_sw_status); //prevent the concurrency of create and drop ls int delete_ls(const ObLSID &id, - const share::ObLSStatus &old_status); - int update_ls_status(const ObLSID &id, const share::ObLSStatus &old_status, const share::ObLSStatus &new_status); + const share::ObLSStatus &old_status, + const ObTenantSwitchoverStatus &working_sw_status); + int update_ls_status(const ObLSID &id, const share::ObLSStatus &old_status, const share::ObLSStatus &new_status, + const ObTenantSwitchoverStatus &working_sw_status); static ObLSOperationType get_ls_operation_by_status(const ObLSStatus &ls_status); int get_ls_attr(const ObLSID &id, const bool for_update, common::ObISQLClient &client, ObLSAttr &ls_attr); /* @@ -185,7 +202,8 @@ public: private: int process_sub_trans_(const ObLSAttr &ls_attr, ObMySQLTransaction &trans); - int operator_ls_(const ObLSAttr &ls_attr, const common::ObSqlString &sql, const uint64_t max_ls_group_id); + int operator_ls_(const ObLSAttr &ls_attr, const common::ObSqlString &sql, const uint64_t max_ls_group_id, + const ObTenantSwitchoverStatus &working_sw_status); private: uint64_t tenant_id_; common::ObMySQLProxy *proxy_; diff --git a/src/share/ls/ob_ls_recovery_stat_operator.cpp b/src/share/ls/ob_ls_recovery_stat_operator.cpp index 7d81b2c20a..53a70cad63 100644 --- a/src/share/ls/ob_ls_recovery_stat_operator.cpp +++ b/src/share/ls/ob_ls_recovery_stat_operator.cpp @@ -17,7 +17,6 @@ #include "share/ob_errno.h" #include "share/config/ob_server_config.h" #include "share/inner_table/ob_inner_table_schema.h" -#include "share/ob_tenant_info_proxy.h"//ObAllTenantInfo #include "lib/string/ob_sql_string.h"//ObSqlString #include "lib/mysqlclient/ob_mysql_transaction.h"//ObMySQLTransaction #include "common/ob_timeout_ctx.h" @@ -220,18 +219,22 @@ int ObLSRecoveryStatOperator::update_ls_recovery_stat_in_trans( true, old_ls_recovery, trans))) { LOG_WARN("failed to get ls current recovery stat", KR(ret), K(ls_recovery)); } else if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(ls_recovery.get_tenant_id(), &trans, - true /* for update */, tenant_info))) { + true /* for update */, tenant_info))) { LOG_WARN("failed to load tenant info", KR(ret), K(ls_recovery)); - } else if (!tenant_info.get_switchover_status().can_report_recovery_status()) { - ret = OB_OP_NOT_ALLOW; - LOG_WARN("status can not report recovery status", KR(ret), K(tenant_info)); + } else if (OB_UNLIKELY(!tenant_info.is_normal_status())) { + ret = OB_NEED_RETRY; + LOG_WARN("switchover status is not normal, do not update ls recovery", KR(ret), + K(ls_recovery), K(tenant_info)); } else { const SCN sync_scn = ls_recovery.get_sync_scn() > old_ls_recovery.get_sync_scn() ? ls_recovery.get_sync_scn() : old_ls_recovery.get_sync_scn(); const SCN &readable_scn = ls_recovery.get_readable_scn() > old_ls_recovery.get_readable_scn() ? ls_recovery.get_readable_scn() : old_ls_recovery.get_readable_scn(); common::ObSqlString sql; - if (OB_FAIL(sql.assign_fmt("UPDATE %s SET sync_scn = %lu, readable_scn = " + if (ls_recovery.get_ls_id() == share::SYS_LS && tenant_info.get_recovery_until_scn() < sync_scn) { + ret = OB_NEED_RETRY; + LOG_WARN("can not recovery bigger than recovery_until_scn", KR(ret), K(sync_scn), K(tenant_info)); + } else if (OB_FAIL(sql.assign_fmt("UPDATE %s SET sync_scn = %lu, readable_scn = " "%lu where ls_id = %ld and tenant_id = %lu", OB_ALL_LS_RECOVERY_STAT_TNAME, sync_scn.get_val_for_inner_table_field(), readable_scn.get_val_for_inner_table_field(), @@ -243,7 +246,6 @@ int ObLSRecoveryStatOperator::update_ls_recovery_stat_in_trans( } return ret; } - int ObLSRecoveryStatOperator::set_ls_offline(const uint64_t &tenant_id, const share::ObLSID &ls_id, const ObLSStatus &ls_status, diff --git a/src/share/ob_primary_standby_service.cpp b/src/share/ob_primary_standby_service.cpp index b62a2d1300..a386fa03fe 100644 --- a/src/share/ob_primary_standby_service.cpp +++ b/src/share/ob_primary_standby_service.cpp @@ -19,7 +19,7 @@ #include "rootserver/ob_rs_event_history_table_operator.h" // ROOTSERVICE_EVENT_ADD #include "rootserver/ob_tenant_role_transition_service.h" // ObTenantRoleTransitionService #include "rootserver/ob_primary_ls_service.h"//ObTenantLSInfo -#include "share/restore/ob_log_restore_source_mgr.h" // ObLogRestoreSourceMgr +#include "share/ls/ob_ls_recovery_stat_operator.h"// ObLSRecoveryStatOperator #include "share/ls/ob_ls_life_manager.h" //ObLSLifeAgentManager #include "share/ls/ob_ls_operator.h" //ObLSAttr #include "storage/tx/ob_timestamp_service.h" // ObTimestampService @@ -281,6 +281,8 @@ int ObPrimaryStandbyService::do_recover_tenant( const uint64_t exec_tenant_id = gen_meta_tenant_id(tenant_id); common::ObMySQLTransaction trans; const ObSimpleTenantSchema *tenant_schema = nullptr; + ObLSRecoveryStatOperator ls_recovery_operator; + ObLSRecoveryStat sys_ls_recovery; if (OB_FAIL(check_inner_stat_())) { LOG_WARN("inner stat error", KR(ret), K_(inited)); } else if (!obrpc::ObRecoverTenantArg::is_valid(recover_type, recovery_until_scn) @@ -311,16 +313,19 @@ int ObPrimaryStandbyService::do_recover_tenant( } else if (tenant_info.get_switchover_status() != working_sw_status) { ret = OB_OP_NOT_ALLOW; LOG_WARN("unexpected tenant switchover status", KR(ret), K(working_sw_status), K(tenant_info)); + } else if (OB_FAIL(ls_recovery_operator.get_ls_recovery_stat(tenant_id, share::SYS_LS, + true /*for_update*/, sys_ls_recovery, trans))) { + LOG_WARN("failed to get ls recovery stat", KR(ret), K(tenant_id)); } else if (obrpc::ObRecoverTenantArg::RecoverType::UNTIL == recover_type && (recovery_until_scn < tenant_info.get_sync_scn() - || recovery_until_scn < tenant_info.get_sys_recovery_scn())) { + || recovery_until_scn < sys_ls_recovery.get_sync_scn())) { ret = OB_OP_NOT_ALLOW; - LOG_WARN("recover before tenant sync_scn or sys recovery scn is not allow", KR(ret), K(tenant_info), - K(tenant_id), K(recover_type), K(recovery_until_scn)); - LOG_USER_ERROR(OB_OP_NOT_ALLOW, "recover before tenant sync_scn or sys recovery scn is"); + LOG_WARN("recover before tenant sync_scn or SYS LS sync_scn is not allow", KR(ret), K(tenant_info), + K(tenant_id), K(recover_type), K(recovery_until_scn), K(sys_ls_recovery)); + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "recover before tenant sync_scn or SYS LS sync_scn is"); } else if (tenant_schema->is_normal()) { const SCN &recovery_until_scn_to_set = obrpc::ObRecoverTenantArg::RecoverType::UNTIL == recover_type ? - recovery_until_scn : SCN::max(tenant_info.get_sync_scn(), tenant_info.get_sys_recovery_scn()); + recovery_until_scn : SCN::max(tenant_info.get_sync_scn(), sys_ls_recovery.get_sync_scn()); if (tenant_info.get_recovery_until_scn() == recovery_until_scn_to_set) { LOG_WARN("recovery_until_scn is same with original", KR(ret), K(tenant_info), K(tenant_id), K(recover_type), K(recovery_until_scn)); @@ -418,15 +423,23 @@ int ObPrimaryStandbyService::switch_to_standby( K(tenant_id)); } } + case share::ObTenantSwitchoverStatus::PREPARE_SWITCHING_TO_STANDBY_STATUS: { + if (OB_FAIL(ret)) { + } else if (OB_FAIL(switch_to_standby_prepare_ls_status_(tenant_id, + tenant_info.get_switchover_status(), + tenant_info.get_switchover_epoch(), + tenant_info))) { + LOG_WARN("failed to switch_to_standby_prepare_ls_status_", KR(ret), K(tenant_id), K(tenant_info)); + } + } case share::ObTenantSwitchoverStatus::SWITCHING_TO_STANDBY_STATUS: { if (OB_FAIL(ret)) { } else { - //change sys ls asscess mode and wait sys_recovery_scn to latest ObTenantRoleTransitionService role_transition_service(tenant_id, sql_proxy_, GCTX.srv_rpc_proxy_, switch_optype); (void)role_transition_service.set_switchover_epoch(tenant_info.get_switchover_epoch()); - if (OB_FAIL(role_transition_service.switchover_to_standby(tenant_info))) { - LOG_WARN("failed to switchover to standby", KR(ret), K(tenant_info)); + if (OB_FAIL(role_transition_service.do_switch_access_mode_to_raw_rw(tenant_info))) { + LOG_WARN("failed to do_switch_access_mode", KR(ret), K(tenant_id), K(tenant_info)); } else if (OB_FAIL(role_transition_service.switchover_update_tenant_status(tenant_id, false /* switch_to_standby */, share::STANDBY_TENANT_ROLE, @@ -495,8 +508,8 @@ int ObPrimaryStandbyService::update_tenant_status_before_sw_to_standby_( LOG_WARN("tenant not expect switchover epoch", KR(ret), K(tenant_info), K(cur_switchover_epoch)); } else if (OB_FAIL(ObAllTenantInfoProxy::update_tenant_role( tenant_id, &trans, cur_switchover_epoch, - STANDBY_TENANT_ROLE, cur_switchover_status, - share::SWITCHING_TO_STANDBY_SWITCHOVER_STATUS, new_switchover_ts))) { + PRIMARY_TENANT_ROLE, cur_switchover_status, + share::PREP_SWITCHING_TO_STANDBY_SWITCHOVER_STATUS, new_switchover_ts))) { LOG_WARN("failed to update tenant role", KR(ret), K(tenant_id), K(cur_switchover_epoch), K(tenant_info)); } else if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info( tenant_id, &trans, true, new_tenant_info))) { @@ -520,6 +533,68 @@ int ObPrimaryStandbyService::update_tenant_status_before_sw_to_standby_( return ret; } +int ObPrimaryStandbyService::switch_to_standby_prepare_ls_status_( + const uint64_t tenant_id, + const ObTenantSwitchoverStatus &status, + const int64_t switchover_epoch, + ObAllTenantInfo &new_tenant_info) +{ + int ret = OB_SUCCESS; + ObMySQLTransaction trans; + ObLSAttr sys_ls_attr; + share::ObLSAttrOperator ls_operator(tenant_id, sql_proxy_); + share::schema::ObSchemaGetterGuard schema_guard; + const share::schema::ObTenantSchema *tenant_schema = NULL; + int64_t new_switchover_epoch = OB_INVALID_VERSION; + + if (!is_user_tenant(tenant_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id)); + } else if (OB_ISNULL(GCTX.schema_service_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("schema_service_ is NULL", KR(ret)); + } else if (OB_FAIL(check_inner_stat_())) { + LOG_WARN("inner stat error", KR(ret), K_(inited)); + } else if (OB_UNLIKELY(!status.is_prepare_switching_to_standby_status())) { + ret = OB_OP_NOT_ALLOW; + LOG_WARN("switchover status not match, switchover to standby not allow", KR(ret), K(status)); + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "switchover status not match, switchover to standby"); + } else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID, schema_guard))) { + LOG_WARN("fail to get schema guard", KR(ret)); + } else if (OB_FAIL(schema_guard.get_tenant_info(tenant_id, tenant_schema))) { + LOG_WARN("failed to get tenant ids", KR(ret), K(tenant_id)); + } else if (OB_ISNULL(tenant_schema)) { + ret = OB_TENANT_NOT_EXIST; + LOG_WARN("tenant not exist", KR(ret), K(tenant_id)); + } else { + ObTenantLSInfo tenant_stat(GCTX.sql_proxy_, tenant_schema, tenant_id, + GCTX.srv_rpc_proxy_, GCTX.lst_operator_); + /* lock SYS_LS to get accurate LS list, then fix ls status to make ls status consistency + between __all_ls&__all_ls_status. + Refer to ls operator, insert/update/delete of ls table are executed in the SYS_LS lock + and normal switchover status */ + if (OB_FAIL(tenant_stat.process_ls_status_missmatch(true/* lock_sys_ls */, + share::PREP_SWITCHING_TO_STANDBY_SWITCHOVER_STATUS))) { + LOG_WARN("failed to process_ls_status_missmatch", KR(ret)); + } else if (OB_FAIL(ObAllTenantInfoProxy::update_tenant_role( + tenant_id, sql_proxy_, switchover_epoch, + share::STANDBY_TENANT_ROLE, status, + share::SWITCHING_TO_STANDBY_SWITCHOVER_STATUS, new_switchover_epoch))) { + LOG_WARN("failed to update tenant role", KR(ret), K(tenant_id), K(switchover_epoch)); + } else if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info( + tenant_id, sql_proxy_, false, new_tenant_info))) { + LOG_WARN("failed to load tenant info", KR(ret), K(tenant_id)); + } else if (OB_UNLIKELY(new_tenant_info.get_switchover_epoch() != new_switchover_epoch)) { + ret = OB_NEED_RETRY; + LOG_WARN("switchover is concurrency", KR(ret), K(switchover_epoch), K(new_tenant_info)); + } + + DEBUG_SYNC(SWITCHING_TO_STANDBY); + } + + return ret; +} + int ObPrimaryStandbyService::write_upgrade_barrier_log( ObMySQLTransaction &trans, const uint64_t tenant_id, @@ -562,5 +637,5 @@ int ObPrimaryStandbyService::write_upgrade_barrier_log( return ret; } -}//end of share +} } diff --git a/src/share/ob_primary_standby_service.h b/src/share/ob_primary_standby_service.h index 1409baf384..9d64401735 100644 --- a/src/share/ob_primary_standby_service.h +++ b/src/share/ob_primary_standby_service.h @@ -159,6 +159,21 @@ private: const uint64_t tenant_id, ObAllTenantInfo &new_tenant_info); + /** + * @description: + * when switch to standby, prepare ls_status in all_ls and all_ls_status to proper status + * @param[in] tenant_id the tenant id to check + * @param[in] status only prepare in specified switchover status + * @param[in] switchover_epoch only prepare in specified switchover epoch + * @param[out] new_tenant_info return the updated tenant_info + * @return return code + */ + int switch_to_standby_prepare_ls_status_( + const uint64_t tenant_id, + const ObTenantSwitchoverStatus &status, + const int64_t switchover_epoch, + ObAllTenantInfo &new_tenant_info); + private: const static int64_t SEC_UNIT = 1000L * 1000L; const static int64_t PRINT_INTERVAL = 10 * 1000 * 1000L; diff --git a/src/share/ob_tenant_info_proxy.cpp b/src/share/ob_tenant_info_proxy.cpp index 367d16eebd..db822c2ae3 100644 --- a/src/share/ob_tenant_info_proxy.cpp +++ b/src/share/ob_tenant_info_proxy.cpp @@ -18,6 +18,7 @@ #include "share/config/ob_server_config.h"//GCONF #include "share/inner_table/ob_inner_table_schema.h"//ALL_TENANT_INFO_TNAME #include "share/ls/ob_ls_i_life_manager.h"//TODO SCN VALUE +#include "share/ls/ob_ls_recovery_stat_operator.h"//ObLSRecoveryStatOperator #include "lib/string/ob_sql_string.h"//ObSqlString #include "lib/mysqlclient/ob_mysql_transaction.h"//ObMySQLTrans #include "common/ob_timeout_ctx.h"//ObTimeoutCtx @@ -36,22 +37,14 @@ bool is_valid_tenant_scn( const SCN &sync_scn, const SCN &replayable_scn, const SCN &standby_scn, - const SCN &recovery_until_scn, - const SCN &sys_recovery_scn) + const SCN &recovery_until_scn) { - return standby_scn <= replayable_scn && replayable_scn <= sync_scn && sync_scn <= recovery_until_scn - && (sys_recovery_scn.is_min() || sync_scn <= sys_recovery_scn) - && (sys_recovery_scn.is_min() || sys_recovery_scn <= recovery_until_scn); + return standby_scn <= replayable_scn && replayable_scn <= sync_scn && sync_scn <= recovery_until_scn; } -SCN gen_new_sync_scn(const SCN &cur_sync_scn, const SCN &desired_sync_scn, const SCN &cur_recovery_until_scn, - const SCN &sys_recovery_scn) +SCN gen_new_sync_scn(const SCN &cur_sync_scn, const SCN &desired_sync_scn, const SCN &cur_recovery_until_scn) { - SCN new_sync_scn = MIN(MAX(cur_sync_scn, desired_sync_scn), cur_recovery_until_scn); - if (!sys_recovery_scn.is_min()) { - new_sync_scn = MIN(new_sync_scn, sys_recovery_scn); - } - return new_sync_scn; + return MIN(MAX(cur_sync_scn, desired_sync_scn), cur_recovery_until_scn); } SCN gen_new_replayable_scn(const SCN &cur_replayable_scn, const SCN &desired_replayable_scn, const SCN &new_sync_scn) @@ -76,8 +69,7 @@ bool ObAllTenantInfo::is_valid() const && tenant_role_.is_valid() && switchover_status_.is_valid() && log_mode_.is_valid() - && sys_recovery_scn_.is_valid() - && is_valid_tenant_scn(sync_scn_, replayable_scn_, standby_scn_, recovery_until_scn_, sys_recovery_scn_); + && is_valid_tenant_scn(sync_scn_, replayable_scn_, standby_scn_, recovery_until_scn_); } int ObAllTenantInfo::init( @@ -89,8 +81,7 @@ int ObAllTenantInfo::init( const SCN &replayable_scn, const SCN &standby_scn, const SCN &recovery_until_scn, - const ObArchiveMode &log_mode, - const SCN &sys_recovery_scn) + const ObArchiveMode &log_mode) { int ret = OB_SUCCESS; if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id @@ -102,13 +93,11 @@ int ObAllTenantInfo::init( || !standby_scn.is_valid_and_not_min() || !recovery_until_scn.is_valid_and_not_min() || !log_mode.is_valid() - //sys recovery scn is valid and maybe min during upgrade - || !sys_recovery_scn.is_valid() - || !is_valid_tenant_scn(sync_scn, replayable_scn, standby_scn, recovery_until_scn, sys_recovery_scn))) { + || !is_valid_tenant_scn(sync_scn, replayable_scn, standby_scn, recovery_until_scn))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(tenant_role), K(switchover_status), K(switchover_epoch), K(sync_scn), K(replayable_scn), K(standby_scn), K(recovery_until_scn), - K(log_mode), K(sys_recovery_scn)); + K(log_mode)); } else { tenant_id_ = tenant_id; tenant_role_ = tenant_role; @@ -119,7 +108,6 @@ int ObAllTenantInfo::init( standby_scn_ = standby_scn; recovery_until_scn_ = recovery_until_scn; log_mode_ = log_mode; - sys_recovery_scn_ = sys_recovery_scn; } return ret; } @@ -137,7 +125,6 @@ void ObAllTenantInfo::assign(const ObAllTenantInfo &other) standby_scn_ = other.standby_scn_; recovery_until_scn_ = other.recovery_until_scn_; log_mode_ = other.log_mode_; - sys_recovery_scn_ = other.sys_recovery_scn_; } return ; } @@ -153,11 +140,10 @@ void ObAllTenantInfo::reset() standby_scn_.set_min() ; recovery_until_scn_.set_min(); log_mode_.reset(); - sys_recovery_scn_.set_min(); } OB_SERIALIZE_MEMBER(ObAllTenantInfo, tenant_id_, tenant_role_, switchover_status_, switchover_epoch_, sync_scn_, - replayable_scn_, standby_scn_, recovery_until_scn_, log_mode_, sys_recovery_scn_); + replayable_scn_, standby_scn_, recovery_until_scn_, log_mode_); ObAllTenantInfo& ObAllTenantInfo::operator= (const ObAllTenantInfo &other) { @@ -192,8 +178,8 @@ int ObAllTenantInfoProxy::init_tenant_info( } else if (OB_FAIL(sql.assign_fmt( "insert into %s (tenant_id, tenant_role, " "switchover_status, switchover_epoch, " - "sync_scn, replayable_scn, readable_scn, recovery_until_scn, log_mode, sys_recovery_scn) " - "values(%lu, '%s', '%s', %ld, %lu, %lu, %lu, %lu, '%s', %lu)", + "sync_scn, replayable_scn, readable_scn, recovery_until_scn, log_mode) " + "values(%lu, '%s', '%s', %ld, %lu, %lu, %lu, %lu, '%s')", OB_ALL_TENANT_INFO_TNAME, tenant_info.get_tenant_id(), tenant_info.get_tenant_role().to_str(), tenant_info.get_switchover_status().to_str(), @@ -202,8 +188,7 @@ int ObAllTenantInfoProxy::init_tenant_info( tenant_info.get_replayable_scn().get_val_for_inner_table_field(), tenant_info.get_standby_scn().get_val_for_inner_table_field(), tenant_info.get_recovery_until_scn().get_val_for_inner_table_field(), - tenant_info.get_log_mode().to_str(), - tenant_info.get_sys_recovery_scn().get_val_for_inner_table_field()))) { + tenant_info.get_log_mode().to_str()))) { LOG_WARN("failed to assign sql", KR(ret), K(tenant_info), K(sql)); } else if (OB_FAIL(proxy->write(exec_tenant_id, sql.ptr(), affected_rows))) { LOG_WARN("failed to execute sql", KR(ret), K(exec_tenant_id), K(sql)); @@ -366,7 +351,8 @@ int ObAllTenantInfoProxy::load_tenant_info(const uint64_t tenant_id, } int ObAllTenantInfoProxy::update_tenant_recovery_status( - const uint64_t tenant_id, ObMySQLProxy *proxy, const SCN &sync_scn, + const uint64_t tenant_id, ObMySQLProxy *proxy, + ObTenantSwitchoverStatus status, const SCN &sync_scn, const SCN &replay_scn, const SCN &readable_scn) { int ret = OB_SUCCESS; @@ -376,9 +362,10 @@ int ObAllTenantInfoProxy::update_tenant_recovery_status( common::ObMySQLTransaction trans; ObAllTenantInfo old_tenant_info; ObTimeoutCtx ctx; - if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id)) { + if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id || + !status.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("tenant_id is invalid", KR(ret), K(tenant_id)); + LOG_WARN("tenant_info is invalid", KR(ret), K(tenant_id), K(status)); } else if (OB_ISNULL(proxy)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("proxy is null", KR(ret), KP(proxy)); @@ -390,8 +377,7 @@ int ObAllTenantInfoProxy::update_tenant_recovery_status( } else if (OB_FAIL(load_tenant_info(tenant_id, &trans, true, old_tenant_info))) { LOG_WARN("failed to load all tenant info", KR(ret), K(tenant_id)); } else { - SCN new_sync_scn = gen_new_sync_scn(old_tenant_info.get_sync_scn(), sync_scn, - old_tenant_info.get_recovery_until_scn(), old_tenant_info.get_sys_recovery_scn()); + SCN new_sync_scn = gen_new_sync_scn(old_tenant_info.get_sync_scn(), sync_scn, old_tenant_info.get_recovery_until_scn()); SCN new_replay_scn = gen_new_replayable_scn(old_tenant_info.get_replayable_scn(), replay_scn, new_sync_scn); SCN new_scn = gen_new_standby_scn(old_tenant_info.get_standby_scn(), readable_scn, new_replay_scn); @@ -404,14 +390,14 @@ int ObAllTenantInfoProxy::update_tenant_recovery_status( } else if (OB_FAIL(sql.assign_fmt( "update %s set sync_scn = %ld, replayable_scn = %ld, " "readable_scn = %ld where tenant_id = %lu " - "and readable_scn <= replayable_scn and " - "replayable_scn <= sync_scn and sync_scn <= recovery_until_scn and sync_scn <= %lu", - OB_ALL_TENANT_INFO_TNAME, + "and switchover_status = '%s' and readable_scn <= replayable_scn and " + "replayable_scn <= sync_scn and sync_scn <= recovery_until_scn", OB_ALL_TENANT_INFO_TNAME, new_sync_scn.get_val_for_inner_table_field(), new_replay_scn.get_val_for_inner_table_field(), new_scn.get_val_for_inner_table_field(), - tenant_id, new_sync_scn.get_val_for_inner_table_field()))) { - LOG_WARN("failed to assign sql", KR(ret), K(tenant_id), K(sql)); + tenant_id, status.to_str()))) { + LOG_WARN("failed to assign sql", KR(ret), K(tenant_id), K(status), + K(sql)); } else if (OB_FAIL(trans.write(exec_tenant_id, sql.ptr(), affected_rows))) { LOG_WARN("failed to execute sql", KR(ret), K(exec_tenant_id), K(sql)); } else if (!is_single_row(affected_rows)) { @@ -445,14 +431,12 @@ int ObAllTenantInfoProxy::fill_cell(common::sqlclient::ObMySQLResult *result, Ob uint64_t replay_scn_val = OB_INVALID_SCN_VAL; uint64_t sts_scn_val = OB_INVALID_SCN_VAL; uint64_t recovery_until_scn_val = OB_INVALID_SCN_VAL; - uint64_t sys_recovery_scn_val = OB_INVALID_SCN_VAL; ObString log_mode_str; ObString log_mode_default_value("NOARCHIVELOG"); SCN sync_scn; SCN replay_scn; SCN sts_scn; SCN recovery_until_scn; - SCN sys_recovery_scn; EXTRACT_VARCHAR_FIELD_MYSQL(*result, "tenant_role", tenant_role_str); EXTRACT_VARCHAR_FIELD_MYSQL(*result, "switchover_status", status_str); EXTRACT_INT_FIELD_MYSQL(*result, "tenant_id", tenant_id, uint64_t); @@ -461,7 +445,6 @@ int ObAllTenantInfoProxy::fill_cell(common::sqlclient::ObMySQLResult *result, Ob EXTRACT_UINT_FIELD_MYSQL(*result, "replayable_scn", replay_scn_val, uint64_t); EXTRACT_UINT_FIELD_MYSQL(*result, "readable_scn", sts_scn_val, uint64_t); EXTRACT_UINT_FIELD_MYSQL_WITH_DEFAULT_VALUE(*result, "recovery_until_scn", recovery_until_scn_val, uint64_t, false /* skip_null_error */, true /* skip_column_error */, OB_MAX_SCN_TS_NS); - EXTRACT_UINT_FIELD_MYSQL_WITH_DEFAULT_VALUE(*result, "sys_recovery_scn", sys_recovery_scn_val, uint64_t, false /* skip_null_error */, true /* skip_column_error */, OB_MIN_SCN_TS_NS); EXTRACT_VARCHAR_FIELD_MYSQL_WITH_DEFAULT_VALUE(*result, "log_mode", log_mode_str, false /* skip_null_error */, true /* skip_column_error */, log_mode_default_value); ObTenantRole tmp_tenant_role(tenant_role_str); @@ -478,15 +461,13 @@ int ObAllTenantInfoProxy::fill_cell(common::sqlclient::ObMySQLResult *result, Ob LOG_WARN("failed to convert_for_inner_table_field", KR(ret), K(sts_scn_val)); } else if (OB_FAIL(recovery_until_scn.convert_for_inner_table_field(recovery_until_scn_val))) { LOG_WARN("failed to convert_for_inner_table_field", KR(ret), K(recovery_until_scn_val)); - } else if (OB_FAIL(sys_recovery_scn.convert_for_inner_table_field(sys_recovery_scn_val))) { - LOG_WARN("failed to conver for inner table field", KR(ret), K(sys_recovery_scn_val)); } else if (OB_FAIL(tenant_info.init( tenant_id, tmp_tenant_role, tmp_tenant_sw_status, switchover_epoch, - sync_scn, replay_scn, sts_scn, recovery_until_scn, tmp_log_mode, sys_recovery_scn))) { + sync_scn, replay_scn, sts_scn, recovery_until_scn, tmp_log_mode))) { LOG_WARN("failed to init tenant info", KR(ret), K(tenant_id), K(tmp_tenant_role), K(tenant_role_str), K(tmp_tenant_sw_status), K(status_str), K(switchover_epoch), K(sync_scn), K(recovery_until_scn), - K(log_mode_str), K(tmp_log_mode), K(sys_recovery_scn)); + K(log_mode_str), K(tmp_log_mode)); } } return ret; @@ -617,6 +598,8 @@ int ObAllTenantInfoProxy::update_tenant_recovery_until_scn( ObSqlString sql; int64_t affected_rows = 0; ObTimeoutCtx ctx; + ObLSRecoveryStatOperator ls_recovery_operator; + ObLSRecoveryStat sys_ls_recovery; ObLogRestoreSourceMgr restore_source_mgr; uint64_t compat_version = 0; @@ -632,26 +615,27 @@ int ObAllTenantInfoProxy::update_tenant_recovery_until_scn( } else if (OB_UNLIKELY(!recovery_until_scn.is_valid_and_not_min())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("recovery_until_scn invalid", KR(ret), K(recovery_until_scn)); + } else if (OB_FAIL(ls_recovery_operator.get_ls_recovery_stat(tenant_id, share::SYS_LS, + true /*for_update*/, sys_ls_recovery, trans))) { + LOG_WARN("failed to get ls recovery stat", KR(ret), K(tenant_id)); + } else if (recovery_until_scn < sys_ls_recovery.get_sync_scn()) { + ret = OB_OP_NOT_ALLOW; + LOG_WARN("recover before SYS LS sync_scn is not allowed", KR(ret), K(tenant_id), K(recovery_until_scn), K(sys_ls_recovery)); + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "recover before SYS LS sync_scn is"); } else if (OB_FAIL(rootserver::ObRootUtils::get_rs_default_timeout_ctx(ctx))) { LOG_WARN("fail to get timeout ctx", KR(ret), K(ctx)); } else if (OB_FAIL(sql.assign_fmt( - "update %s set recovery_until_scn = %lu where tenant_id = %lu " - "and sync_scn <= %lu and switchover_epoch = %ld and sys_recovery_scn <= %lu", - OB_ALL_TENANT_INFO_TNAME, - recovery_until_scn.get_val_for_inner_table_field(), tenant_id, - recovery_until_scn.get_val_for_inner_table_field(), - switchover_epoch, - recovery_until_scn.get_val_for_inner_table_field()))) { + "update %s set recovery_until_scn = %lu where tenant_id = %lu and sync_scn <= %lu and switchover_epoch = %ld", + OB_ALL_TENANT_INFO_TNAME, recovery_until_scn.get_val_for_inner_table_field(), tenant_id, + recovery_until_scn.get_val_for_inner_table_field(), switchover_epoch))) { LOG_WARN("failed to assign sql", KR(ret), K(tenant_id), K(recovery_until_scn), K(sql)); } else if (OB_FAIL(trans.write(exec_tenant_id, sql.ptr(), affected_rows))) { LOG_WARN("failed to execute sql", KR(ret), K(exec_tenant_id), K(sql)); } else if (0 == affected_rows) { ret = OB_OP_NOT_ALLOW; - LOG_WARN("state changed, check sync_scn, switchover status and sys_recovery_scn", KR(ret), K(tenant_id), + LOG_WARN("state changed, check sync_scn and switchover status", KR(ret), K(tenant_id), K(switchover_epoch), K(recovery_until_scn), K(sql)); - LOG_USER_ERROR(OB_OP_NOT_ALLOW, - "state changed, check sync_scn, sys_recovery_scn and " - "switchover status, recover is"); + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "state changed, check sync_scn and switchover status, recover is"); } else if (!is_single_row(affected_rows)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("expect updating one row", KR(ret), K(affected_rows), @@ -664,10 +648,10 @@ int ObAllTenantInfoProxy::update_tenant_recovery_until_scn( } int64_t cost = ObTimeUtility::current_time() - begin_time; - LOG_INFO("update_recovery_until_scn finish", KR(ret), K(tenant_id), + LOG_INFO("update_recovery_until_scn finish", KR(ret), K(tenant_id), K(sys_ls_recovery), K(recovery_until_scn), K(affected_rows), K(switchover_epoch), K(sql), K(cost)); ROOTSERVICE_EVENT_ADD("tenant_info", "update_recovery_until_scn", K(ret), K(tenant_id), - K(recovery_until_scn), K(affected_rows), K(switchover_epoch)); + K(recovery_until_scn), K(affected_rows), K(switchover_epoch), K(sys_ls_recovery)); return ret; } @@ -681,7 +665,6 @@ int ObAllTenantInfoProxy::update_tenant_status( const share::SCN &replayable_scn, const share::SCN &readable_scn, const share::SCN &recovery_until_scn, - const share::SCN &sys_recovery_scn, const int64_t old_switchover_epoch) { int64_t begin_time = ObTimeUtility::current_time(); @@ -692,7 +675,6 @@ int ObAllTenantInfoProxy::update_tenant_status( ObTimeoutCtx ctx; int64_t new_switchover_epoch = OB_INVALID_VERSION; ObLogRestoreSourceMgr restore_source_mgr; - ObAllTenantInfo tenant_info; if (OB_UNLIKELY(!is_user_tenant(tenant_id) || !new_role.is_valid() @@ -703,24 +685,15 @@ int ObAllTenantInfoProxy::update_tenant_status( || !replayable_scn.is_valid_and_not_min() || !readable_scn.is_valid_and_not_min() || !recovery_until_scn.is_valid_and_not_min() - || !sys_recovery_scn.is_valid_and_not_min() || OB_INVALID_VERSION == old_switchover_epoch)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("tenant_info is invalid", KR(ret), K(tenant_id), K(new_role), K(old_status), K(new_status), K(sync_scn), K(replayable_scn), K(readable_scn), K(recovery_until_scn), K(old_switchover_epoch)); - //use tenant_info check parameters valid } else if (OB_FAIL(get_new_switchover_epoch_(old_switchover_epoch, old_status, new_status, new_switchover_epoch))) { LOG_WARN("fail to get_new_switchover_epoch_", KR(ret), K(old_switchover_epoch), K(old_status), K(new_status)); - } else if (OB_FAIL(tenant_info.init(tenant_id, new_role, new_status, new_switchover_epoch, - sync_scn, replayable_scn, readable_scn, - recovery_until_scn, NOARCHIVE_MODE, sys_recovery_scn))) { - LOG_WARN("failed to init tenant_info", KR(ret), K(tenant_id), K(new_role), - K(new_status), K(new_switchover_epoch), K(sync_scn), - K(replayable_scn), K(readable_scn), K(recovery_until_scn), - K(sys_recovery_scn)); } else if (OB_UNLIKELY(OB_INVALID_VERSION == new_switchover_epoch)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("new_switchover_ts is invalid", KR(ret), K(new_switchover_epoch), @@ -730,22 +703,20 @@ int ObAllTenantInfoProxy::update_tenant_status( } else if (OB_FAIL(sql.assign_fmt( "update %s set tenant_role = '%s', switchover_status = '%s', " "switchover_epoch = %ld, sync_scn = %lu, replayable_scn = %lu, " - "readable_scn = %lu, recovery_until_scn = %lu ,sys_recovery_scn = %lu where tenant_id = %lu " + "readable_scn = %lu, recovery_until_scn = %lu where tenant_id = %lu " "and switchover_status = '%s' and switchover_epoch = %ld " "and readable_scn <= replayable_scn and replayable_scn <= sync_scn and sync_scn <= recovery_until_scn " - "and sync_scn <= %lu and replayable_scn <= %lu and readable_scn <= %lu and sys_recovery_scn <= %lu", + "and sync_scn <= %lu and replayable_scn <= %lu and readable_scn <= %lu ", OB_ALL_TENANT_INFO_TNAME, new_role.to_str(), new_status.to_str(), new_switchover_epoch, sync_scn.get_val_for_inner_table_field(), replayable_scn.get_val_for_inner_table_field(), readable_scn.get_val_for_inner_table_field(), recovery_until_scn.get_val_for_inner_table_field(), - sys_recovery_scn.get_val_for_inner_table_field(), tenant_id, old_status.to_str(), old_switchover_epoch, sync_scn.get_val_for_inner_table_field(), replayable_scn.get_val_for_inner_table_field(), - readable_scn.get_val_for_inner_table_field(), - sys_recovery_scn.get_val_for_inner_table_field()))) { + readable_scn.get_val_for_inner_table_field()))) { LOG_WARN("failed to assign sql", KR(ret), K(tenant_id), K(sql)); } else if (OB_FAIL(trans.write(exec_tenant_id, sql.ptr(), affected_rows))) { LOG_WARN("failed to execute sql", KR(ret), K(exec_tenant_id), K(sql)); @@ -762,6 +733,22 @@ int ObAllTenantInfoProxy::update_tenant_status( LOG_WARN("failed to update_recovery_until_scn", KR(ret), K(tenant_id), K(recovery_until_scn)); } + ObAllTenantInfo tenant_info; + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = tenant_info.init(tenant_id, + new_role, + new_status, + new_switchover_epoch, + sync_scn, + replayable_scn, + readable_scn, + recovery_until_scn))) { + LOG_WARN("failed to init tenant_info", KR(ret), KR(tmp_ret), K(tenant_id), K(new_role), + K(new_status), K(new_switchover_epoch), K(sync_scn), + K(replayable_scn), K(readable_scn), K(recovery_until_scn)); + ret = OB_SUCC(ret) ? tmp_ret : ret; + } + int64_t cost = ObTimeUtility::current_time() - begin_time; ROOTSERVICE_EVENT_ADD("tenant_info", "update_tenant_role", K(ret), K(tenant_id), K(tenant_info), K(old_status), K(old_switchover_epoch), K(cost)); @@ -830,72 +817,6 @@ int ObAllTenantInfoProxy::update_tenant_log_mode( return ret; } -int ObAllTenantInfoProxy::update_tenant_sys_recovery_scn( - const uint64_t tenant_id, - const share::SCN &sys_recovery_scn, - bool update_sync_scn, ObISQLClient *proxy) -{ - int ret = OB_SUCCESS; - const uint64_t exec_tenant_id = gen_meta_tenant_id(tenant_id); - - if (OB_UNLIKELY(!is_user_tenant(tenant_id) - || !sys_recovery_scn.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(sys_recovery_scn)); - } else { - TAKE_IN_TRANS(update_tenant_sys_recovery_scn, proxy, exec_tenant_id, - tenant_id, sys_recovery_scn, update_sync_scn); - } - return ret; -} - -int ObAllTenantInfoProxy::update_tenant_sys_recovery_scn_in_trans( - const uint64_t tenant_id, - const share::SCN &sys_recovery_scn, - bool update_sync_scn, common::ObMySQLTransaction &trans) -{ - int ret = OB_SUCCESS; - const uint64_t exec_tenant_id = gen_meta_tenant_id(tenant_id); - ObSqlString sql; - int64_t affected_rows = 0; - ObTimeoutCtx ctx; - ObAllTenantInfo tenant_info; - - if (OB_UNLIKELY(!is_user_tenant(tenant_id) - || !sys_recovery_scn.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(sys_recovery_scn)); - } else if (OB_FAIL(rootserver::ObRootUtils::get_rs_default_timeout_ctx(ctx))) { - LOG_WARN("fail to get timeout ctx", KR(ret), K(ctx)); - } else if (OB_FAIL(load_tenant_info(tenant_id, &trans, true, tenant_info))) { - LOG_WARN("failed to load tenant info", KR(ret), K(tenant_id)); - } else if (!tenant_info.get_switchover_status().can_report_recovery_status() - || sys_recovery_scn <= tenant_info.get_sys_recovery_scn() - || sys_recovery_scn > tenant_info.get_recovery_until_scn() - || sys_recovery_scn < tenant_info.get_sync_scn()) { - ret = OB_NEED_RETRY; - LOG_WARN("can not report recovery status", KR(ret), - K(tenant_info), K(sys_recovery_scn)); - } else if (OB_FAIL(sql.assign_fmt( - "update %s set sys_recovery_scn = %lu ", - OB_ALL_TENANT_INFO_TNAME, sys_recovery_scn.get_val_for_inner_table_field()))) { - LOG_WARN("failed to assign sql", KR(ret), K(tenant_id), K(sys_recovery_scn), K(sql)); - } else if (update_sync_scn && OB_FAIL(sql.append_fmt(", sync_scn = %lu, replayable_scn = %lu ", - sys_recovery_scn.get_val_for_inner_table_field(), - sys_recovery_scn.get_val_for_inner_table_field()))) { - LOG_WARN("failed to append sql", KR(ret), K(sys_recovery_scn), K(sql)); - } else if (OB_FAIL(trans.write(exec_tenant_id, sql.ptr(), affected_rows))) { - LOG_WARN("failed to execute sql", KR(ret), K(exec_tenant_id), K(sql)); - } else if (!is_single_row(affected_rows)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("expect updating one row", KR(ret), K(affected_rows), K(sql)); - } - return ret; - - LOG_TRACE("update sys_recovery_scn", KR(ret), K(tenant_id), K(sys_recovery_scn), - K(update_sync_scn)); -} - -} // namespace share +} } diff --git a/src/share/ob_tenant_info_proxy.h b/src/share/ob_tenant_info_proxy.h index dded8fa67d..802e2296ff 100644 --- a/src/share/ob_tenant_info_proxy.h +++ b/src/share/ob_tenant_info_proxy.h @@ -43,12 +43,9 @@ bool is_valid_tenant_scn( const share::SCN &sync_scn, const share::SCN &replayable_scn, const share::SCN &standby_scn, - const share::SCN &recovery_until_scn, - const share::SCN &sys_recovery_scn); + const share::SCN &recovery_until_scn); -SCN gen_new_sync_scn(const share::SCN &cur_sync_scn, const share::SCN &desired_sync_scn, - const share::SCN &cur_recovery_until_scn, - const share::SCN &sys_recovery_scn); +SCN gen_new_sync_scn(const share::SCN &cur_sync_scn, const share::SCN &desired_sync_scn, const share::SCN &cur_recovery_until_scn); struct ObAllTenantInfo { @@ -67,7 +64,6 @@ public: * @param[in] standby_scn * @param[in] recovery_until_scn * @param[in] log_mode - * @param[in] sys_recovery_scn */ int init(const uint64_t tenant_id, const ObTenantRole &tenant_role, @@ -77,8 +73,7 @@ public: const SCN &replayable_scn = SCN::base_scn(), const SCN &standby_scn = SCN::base_scn(), const SCN &recovery_until_scn = SCN::base_scn(), - const ObArchiveMode &log_mode = NOARCHIVE_MODE, - const SCN &sys_recovery_scn = SCN::base_scn()); + const ObArchiveMode &log_mode = NOARCHIVE_MODE); ObAllTenantInfo &operator=(const ObAllTenantInfo &other); void assign(const ObAllTenantInfo &other); void reset(); @@ -95,7 +90,12 @@ public: * Because STS will be changed when switchover to standby. */ bool is_sts_ready() const { return !(tenant_role_.is_primary() - || is_switching_to_standby_status());} + || tenant_is_switchover_to_standby()); } + + // ************* Functions that describe what tenant is doing ********************* + // tenant is in switchover from primary to standby + bool tenant_is_switchover_to_standby() const { return is_prepare_switching_to_standby_status() + || is_switching_to_standby_status(); } // ObTenantSwitchoverStatus related function #define IS_TENANT_STATUS(STATUS) \ @@ -106,12 +106,13 @@ IS_TENANT_STATUS(switching_to_primary) IS_TENANT_STATUS(prepare_flashback_for_failover_to_primary) IS_TENANT_STATUS(flashback) IS_TENANT_STATUS(switching_to_standby) +IS_TENANT_STATUS(prepare_switching_to_standby) IS_TENANT_STATUS(prepare_flashback_for_switch_to_primary) #undef IS_TENANT_STATUS TO_STRING_KV(K_(tenant_id), K_(tenant_role), K_(switchover_status), K_(switchover_epoch), K_(sync_scn), K_(replayable_scn), - K_(standby_scn), K_(recovery_until_scn), K_(log_mode), K_(sys_recovery_scn)); + K_(standby_scn), K_(recovery_until_scn), K_(log_mode)); // Getter&Setter const ObTenantRole &get_tenant_role() const { return tenant_role_; } @@ -131,7 +132,6 @@ public:\ Property_declare_var(share::SCN, standby_scn) Property_declare_var(share::SCN, recovery_until_scn) Property_declare_var(ObArchiveMode, log_mode) - Property_declare_var(share::SCN, sys_recovery_scn) #undef Property_declare_var private: ObTenantRole tenant_role_; @@ -173,12 +173,14 @@ public: * @description: update tenant recovery status * @param[in] tenant_id * @param[in] proxy + * @param[in] status: the target status while update recovery status * @param[in] sync_scn : sync point * @param[in] replay_scn : max replay point * @param[in] reabable_scn : standby readable scn */ static int update_tenant_recovery_status(const uint64_t tenant_id, ObMySQLProxy *proxy, + ObTenantSwitchoverStatus status, const SCN &sync_scn, const SCN &replay_scn, const SCN &reabable_scn); @@ -221,7 +223,6 @@ public: const share::SCN &replayable_scn, const share::SCN &readable_scn, const share::SCN &recovery_until_scn, - const share::SCN &sys_recovery_scn, const int64_t old_switchover_epoch); /** @@ -285,23 +286,6 @@ public: const ObArchiveMode &old_log_mode, const ObArchiveMode &new_log_mode); - /** - * @description: update tenant sys ls log sys recovery scn - * @param[in] tenant_id - * @param[in] trans - * @param[in] sys_recovery_scn new sys recovery scn - * @param[in] update_sync_scn : When iterating to the LS operations, these operations can be executed provided that - * the sync_scn of other ls have reached the LS operation's point, - * so when updating the sys_recovery_scn, can the sync_scn be updated at the same time. - * return : - * OB_SUCCESS update tenant sys recovery scn successfully - * OB_NEED_RETRY sys recovery scn backoff, need retry - */ - DEFINE_IN_TRANS_FUC1(update_tenant_sys_recovery_scn, - const uint64_t tenant_id, - const share::SCN &sys_recovery_scn, - bool update_sync_scn); - }; } } diff --git a/src/share/ob_tenant_switchover_status.cpp b/src/share/ob_tenant_switchover_status.cpp index 8513110376..ccbe394695 100644 --- a/src/share/ob_tenant_switchover_status.cpp +++ b/src/share/ob_tenant_switchover_status.cpp @@ -27,6 +27,7 @@ static const char* TENANT_SWITCHOVER_ARRAY[] = "SWITCHING TO PRIMARY", "PREPARE FLASHBACK", "FLASHBACK", + "PREPARE SWITCHING TO STANDBY", "SWITCHING TO STANDBY", "PREPARE SWITCHING TO PRIMARY", }; diff --git a/src/share/ob_tenant_switchover_status.h b/src/share/ob_tenant_switchover_status.h index 1c712c0dbc..ba3f08a5bc 100644 --- a/src/share/ob_tenant_switchover_status.h +++ b/src/share/ob_tenant_switchover_status.h @@ -33,9 +33,10 @@ public: SWITCHING_TO_PRIMARY_STATUS = 2, PREPARE_FLASHBACK_FOR_FAILOVER_TO_PRIMARY_STATUS = 3, FLASHBACK_STATUS = 4, - SWITCHING_TO_STANDBY_STATUS = 5, - PREPARE_FLASHBACK_FOR_SWITCH_TO_PRIMARY_STATUS = 6, - MAX_STATUS = 7 + PREPARE_SWITCHING_TO_STANDBY_STATUS = 5, + SWITCHING_TO_STANDBY_STATUS = 6, + PREPARE_FLASHBACK_FOR_SWITCH_TO_PRIMARY_STATUS = 7, + MAX_STATUS = 8 }; public: ObTenantSwitchoverStatus() : value_(INVALID_STATUS) {} @@ -48,13 +49,6 @@ public: bool is_valid() const { return INVALID_STATUS != value_; } ObTenantSwitchoverStatus::Status value() const { return value_; } const char* to_str() const; - //In the flashback status, the log needs to be truncated, - //and sync_scn will fall back, so at this stage, - //__all_ls_recovery_stat/__all_tenant_info will not be reported - bool can_report_recovery_status() const - { - return FLASHBACK_STATUS != value_; - } // compare operator bool operator == (const ObTenantSwitchoverStatus &other) const { return value_ == other.value_; } @@ -75,6 +69,7 @@ IS_TENANT_STATUS(NORMAL_STATUS, normal) IS_TENANT_STATUS(SWITCHING_TO_PRIMARY_STATUS, switching_to_primary) IS_TENANT_STATUS(PREPARE_FLASHBACK_FOR_FAILOVER_TO_PRIMARY_STATUS, prepare_flashback_for_failover_to_primary) IS_TENANT_STATUS(FLASHBACK_STATUS, flashback) +IS_TENANT_STATUS(PREPARE_SWITCHING_TO_STANDBY_STATUS, prepare_switching_to_standby) IS_TENANT_STATUS(SWITCHING_TO_STANDBY_STATUS, switching_to_standby) IS_TENANT_STATUS(PREPARE_FLASHBACK_FOR_SWITCH_TO_PRIMARY_STATUS, prepare_flashback_for_switch_to_primary) #undef IS_TENANT_STATUS @@ -89,6 +84,7 @@ static const ObTenantSwitchoverStatus NORMAL_SWITCHOVER_STATUS(ObTenantSwitchove static const ObTenantSwitchoverStatus PREPARE_FLASHBACK_FOR_FAILOVER_TO_PRIMARY_SWITCHOVER_STATUS(ObTenantSwitchoverStatus::PREPARE_FLASHBACK_FOR_FAILOVER_TO_PRIMARY_STATUS); static const ObTenantSwitchoverStatus FLASHBACK_SWITCHOVER_STATUS(ObTenantSwitchoverStatus::FLASHBACK_STATUS); static const ObTenantSwitchoverStatus SWITCHING_TO_PRIMARY_SWITCHOVER_STATUS(ObTenantSwitchoverStatus::SWITCHING_TO_PRIMARY_STATUS); +static const ObTenantSwitchoverStatus PREP_SWITCHING_TO_STANDBY_SWITCHOVER_STATUS(ObTenantSwitchoverStatus::PREPARE_SWITCHING_TO_STANDBY_STATUS); static const ObTenantSwitchoverStatus SWITCHING_TO_STANDBY_SWITCHOVER_STATUS(ObTenantSwitchoverStatus::SWITCHING_TO_STANDBY_STATUS); static const ObTenantSwitchoverStatus PREPARE_FLASHBACK_FOR_SWITCH_TO_PRIMARY_SWITCHOVER_STATUS(ObTenantSwitchoverStatus::PREPARE_FLASHBACK_FOR_SWITCH_TO_PRIMARY_STATUS); diff --git a/src/share/ob_upgrade_utils.cpp b/src/share/ob_upgrade_utils.cpp index a14a2c3130..09f08f3a1a 100644 --- a/src/share/ob_upgrade_utils.cpp +++ b/src/share/ob_upgrade_utils.cpp @@ -21,10 +21,6 @@ #include "rootserver/ob_root_service.h" #include "sql/resolver/expr/ob_raw_expr_util.h" #include "share/ob_rpc_struct.h" -#include "share/ob_tenant_info_proxy.h"//AllTenantInfo -#include "share/ls/ob_ls_i_life_manager.h"//start_transaction -#include "rootserver/ob_ls_service_helper.h" -#include "ob_upgrade_utils.h" namespace oceanbase { @@ -795,8 +791,6 @@ int ObUpgradeFor4100Processor::post_upgrade() LOG_WARN("fail to check inner stat", KR(ret)); } else if (OB_FAIL(recompile_all_views_and_synonyms(tenant_id))) { LOG_WARN("fail to init rewrite rule version", K(ret), K(tenant_id)); - } else if (OB_FAIL(init_tenant_sys_recovery_scn(tenant_id))) { - LOG_WARN("failed to init tenant sys recovery scn", KR(ret), K(tenant_id)); } return ret; } @@ -930,106 +924,6 @@ int ObUpgradeFor4100Processor::recompile_all_views_and_synonyms(const uint64_t t return ret; } -int ObUpgradeFor4100Processor::init_tenant_sys_recovery_scn(const uint64_t tenant_id) -{ - int ret = OB_SUCCESS; - int64_t start = ObTimeUtility::current_time(); - if (!is_user_tenant(tenant_id)) { - //no need to update meta tenant or sys tenant sys recovery scn - } else { - //check sys recovery scn, make status to equal and set sys_recovery_scn - const uint64_t exec_tenant_id = gen_meta_tenant_id(tenant_id); - ObAllTenantInfo tenant_info; - SCN sync_scn; - ObTenantSchema tenant_schema; - if (OB_FAIL(rootserver::ObTenantThreadHelper::get_tenant_schema(tenant_id, tenant_schema))) { - LOG_WARN("failed to get tenant schema", KR(ret), K(tenant_id)); - } - START_TRANSACTION(sql_proxy_, exec_tenant_id); - rootserver::ObTenantLSInfo tenant_stat(sql_proxy_, &tenant_schema, - tenant_id, GCTX.srv_rpc_proxy_, - GCTX.lst_operator_); - if (FAILEDx(ObAllTenantInfoProxy::load_tenant_info(tenant_id, &trans, - true, tenant_info))) { - LOG_WARN("failed to load tenant info", KR(ret), K(tenant_id)); - } else if (!tenant_info.get_sys_recovery_scn().is_min()) { - LOG_INFO("tenant sys recovery scn already update, no need to init", K(tenant_info)); - } else if (OB_FAIL(tenant_stat.revision_to_equal_status(trans))) { - LOG_WARN("failed to make revision status", KR(ret), K(tenant_id)); - } else if (OB_FAIL(get_sys_ls_max_sync_scn_(tenant_id, sync_scn))) { - LOG_WARN("failed to get sys ls max sync scn", KR(ret), K(tenant_id)); - } else if (OB_FAIL(ObAllTenantInfoProxy::update_tenant_sys_recovery_scn_in_trans( - tenant_id, sync_scn, false, trans))) { - LOG_WARN("failed to update tenant sys recovery scn", KR(ret), K(tenant_id), - K(sync_scn), K(tenant_info)); - } else { - LOG_INFO("init tenant sys recovery scn success", K(tenant_id), K(sync_scn)); - } - END_TRANSACTION(trans); - } - - LOG_INFO("add tenant sys recovery scn finish", K(ret), K(tenant_id_), "cost", ObTimeUtility::current_time() - start); - return ret; -} - -int ObUpgradeFor4100Processor::get_sys_ls_max_sync_scn_(const uint64_t tenant_id, SCN &sync_scn) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(GCTX.sql_proxy_) || OB_ISNULL(GCTX.srv_rpc_proxy_) || - OB_ISNULL(GCTX.location_service_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("error unexpcted", KR(ret), KP(GCTX.sql_proxy_), KP(GCTX.srv_rpc_proxy_), - KP(GCTX.location_service_)); - } else { - ObTimeoutCtx ctx; - ObAddr leader; - obrpc::ObGetLSSyncScnArg arg; - ObGetLSSyncScnRes res; - int tmp_ret = OB_SUCCESS; - rootserver::ObGetLSSyncScnProxy proxy( - *GCTX.srv_rpc_proxy_, &obrpc::ObSrvRpcProxy::get_ls_sync_scn); - const int64_t timeout = GCONF.internal_sql_execute_timeout; - if (FAILEDx(arg.init(tenant_id, SYS_LS, false))) { - LOG_WARN("failed to init arg", KR(ret), K(tenant_id)); - } else if (OB_FAIL(ObShareUtil::set_default_timeout_ctx(ctx, timeout))) { - LOG_WARN("fail to set timeout ctx", KR(ret), K(timeout)); - } - while (OB_SUCC(ret)) { - proxy.reuse(); - if (ctx.is_timeouted()) { - ret = OB_TIMEOUT; - LOG_WARN("already timeout, failed to get sys ls sync scn", KR(ret), K(ctx), K(tenant_id)); - } else { - if (OB_FAIL(GCTX.location_service_->get_leader( - GCONF.cluster_id, tenant_id, SYS_LS, true, leader))) { - LOG_WARN("failed to get leader", KR(ret), K(tenant_id)); - } else if (OB_FAIL(proxy.call(leader, ctx.get_timeout(), tenant_id, arg))) { - LOG_WARN("failed to get ls sync scn", KR(ret), K(arg), K(ctx), - K(leader)); - } - if (OB_TMP_FAIL(proxy.wait())) { - ret = OB_SUCC(ret) ? tmp_ret : ret; - LOG_WARN("failed to wait all", KR(ret), KR(tmp_ret)); - } else if (OB_FAIL(ret)) { - } else if (1 != proxy.get_results().count() || - OB_ISNULL(proxy.get_results().at(0))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to get rpc result", KR(ret), "proxy result", proxy.get_results()); - } else { - sync_scn = proxy.get_results().at(0)->get_cur_sync_scn(); - break; - } - if (OB_FAIL(ret)) { - //ignore error of each rpc until timeout - LOG_WARN("failed to get sys ls sync scn, sleep and try again", KR(ret)); - ret = OB_SUCCESS; - usleep(100 * 1000L); - } - } - } - } - return ret; -} /* =========== 4100 upgrade processor end ============= */ /* =========== special upgrade processor end ============= */ } // end share diff --git a/src/share/ob_upgrade_utils.h b/src/share/ob_upgrade_utils.h index 90c69cc162..bd4f329887 100644 --- a/src/share/ob_upgrade_utils.h +++ b/src/share/ob_upgrade_utils.h @@ -22,13 +22,9 @@ namespace oceanbase { -namespace common -{ -class ObTimeoutCtx; -} namespace share { -class SCN; + static const int64_t UPGRADE_JOB_TYPE_COUNT = 1; static const rootserver::ObRsJobType upgrade_job_type_array[UPGRADE_JOB_TYPE_COUNT] = { rootserver::JOB_TYPE_INVALID, @@ -190,8 +186,6 @@ private: int post_upgrade_for_srs(); int init_rewrite_rule_version(const uint64_t tenant_id); static int recompile_all_views_and_synonyms(const uint64_t tenant_id); - int init_tenant_sys_recovery_scn(const uint64_t tenant_id); - int get_sys_ls_max_sync_scn_(const uint64_t tenant_id, share::SCN &sys_scn); }; /* =========== special upgrade processor end ============= */ diff --git a/src/share/rc/ob_tenant_base.h b/src/share/rc/ob_tenant_base.h index 76216b464e..77866cc0cd 100644 --- a/src/share/rc/ob_tenant_base.h +++ b/src/share/rc/ob_tenant_base.h @@ -124,11 +124,9 @@ namespace rootserver { class ObPrimaryMajorFreezeService; class ObRestoreMajorFreezeService; - class ObLSRecoveryReportor; + class ObTenantRecoveryReportor; class ObTenantInfoLoader; - class ObTenantInfoReportor; class ObPrimaryLSService; - class ObCommonLSService; class ObRestoreService; class ObRecoveryLSService; class ObArbitrationService; @@ -200,10 +198,8 @@ using ObPartTransCtxObjPool = common::ObServerObjectPoolset_cache_ls(this); } - if (OB_SUCC(ret) && ls_id.is_sys_ls() && is_user_tenant(tenant_id)) { + if (OB_SUCC(ret) && ls_id.is_sys_ls()) { + //meta tenant need create thread for ls manager REGISTER_TO_LOGSERVICE(logservice::PRIMARY_LS_SERVICE_LOG_BASE_TYPE, MTL(rootserver::ObPrimaryLSService *)); - LOG_INFO("primary ls manager register to logservice success"); - } - if (OB_SUCC(ret) && ls_id.is_sys_ls() && !is_user_tenant(tenant_id)) { - REGISTER_TO_LOGSERVICE(logservice::COMMON_LS_SERVICE_LOG_BASE_TYPE, MTL(rootserver::ObCommonLSService *)); - LOG_INFO("common ls manager register to logservice success"); - + LOG_INFO("primary ls manager registre to logservice success"); } - if (OB_SUCC(ret) && ls_id.is_sys_ls() && is_meta_tenant(tenant_id)) { - //sys no need to report - REGISTER_TO_LOGSERVICE(logservice::TENANT_INFO_REPORTOR_LOG_BASE_TYPE, MTL(rootserver::ObTenantInfoReportor *)); - LOG_INFO("tenant info report register to logservice success"); - } if (OB_SUCC(ret) && is_user_tenant(tenant_id) && ls_id.is_sys_ls()) { // only user tenant need dump datadict REGISTER_TO_LOGSERVICE(logservice::DATA_DICT_LOG_BASE_TYPE, MTL(datadict::ObDataDictService *)); //only user table need recovery REGISTER_TO_RESTORESERVICE(logservice::RECOVERY_LS_SERVICE_LOG_BASE_TYPE, MTL(rootserver::ObRecoveryLSService *)); - LOG_INFO("recovery ls manager register to restoreservice success"); + LOG_INFO("recovery ls manager registre to restoreservice success"); } if (OB_SUCC(ret) && !is_user_tenant(tenant_id) && ls_id.is_sys_ls()) { @@ -643,24 +632,20 @@ void ObLS::destroy() UNREGISTER_FROM_LOGSERVICE(logservice::GAIS_LOG_BASE_TYPE, MTL(share::ObGlobalAutoIncService *)); MTL(share::ObGlobalAutoIncService *)->set_cache_ls(nullptr); } - if (ls_meta_.ls_id_.is_sys_ls() && is_user_tenant(MTL_ID())) { + if (OB_SUCC(ret) && ls_meta_.ls_id_.is_sys_ls()) { rootserver::ObPrimaryLSService* ls_service = MTL(rootserver::ObPrimaryLSService*); UNREGISTER_FROM_LOGSERVICE(logservice::PRIMARY_LS_SERVICE_LOG_BASE_TYPE, ls_service); } - if (ls_meta_.ls_id_.is_sys_ls() && !is_user_tenant(MTL_ID())) { - rootserver::ObCommonLSService *ls_service = MTL(rootserver::ObCommonLSService*); - UNREGISTER_FROM_LOGSERVICE(logservice::COMMON_LS_SERVICE_LOG_BASE_TYPE, ls_service); - rootserver::ObRestoreService * restore_service = MTL(rootserver::ObRestoreService*); - UNREGISTER_FROM_LOGSERVICE(logservice::RESTORE_SERVICE_LOG_BASE_TYPE, restore_service); - } - if (ls_meta_.ls_id_.is_sys_ls() && is_meta_tenant(MTL_ID())) { - UNREGISTER_FROM_LOGSERVICE(logservice::TENANT_INFO_REPORTOR_LOG_BASE_TYPE, MTL(rootserver::ObTenantInfoReportor*)); - } - if (is_user_tenant(MTL_ID()) && ls_meta_.ls_id_.is_sys_ls()) { + + if (OB_SUCC(ret) && is_user_tenant(MTL_ID()) && ls_meta_.ls_id_.is_sys_ls()) { UNREGISTER_FROM_LOGSERVICE(logservice::DATA_DICT_LOG_BASE_TYPE, MTL(datadict::ObDataDictService *)); rootserver::ObRecoveryLSService* ls_service = MTL(rootserver::ObRecoveryLSService*); UNREGISTER_FROM_RESTORESERVICE(logservice::RECOVERY_LS_SERVICE_LOG_BASE_TYPE, ls_service); } + if (OB_SUCC(ret) && !is_user_tenant(MTL_ID()) && ls_meta_.ls_id_.is_sys_ls()) { + rootserver::ObRestoreService * restore_service = MTL(rootserver::ObRestoreService*); + UNREGISTER_FROM_LOGSERVICE(logservice::RESTORE_SERVICE_LOG_BASE_TYPE, restore_service); + } tx_table_.destroy(); lock_table_.destroy(); ls_tablet_svr_.destroy(); diff --git a/src/storage/tx/ob_trans_service_v4.cpp b/src/storage/tx/ob_trans_service_v4.cpp index 0d2599abab..cbadac298c 100644 --- a/src/storage/tx/ob_trans_service_v4.cpp +++ b/src/storage/tx/ob_trans_service_v4.cpp @@ -39,6 +39,7 @@ #include "storage/tx_storage/ob_ls_handle.h" #include "storage/ls/ob_ls.h" #include "ob_xa_service.h" +#include "rootserver/ob_tenant_recovery_reportor.h" /* interface(s) */ namespace oceanbase { diff --git a/src/storage/tx_storage/ob_checkpoint_service.cpp b/src/storage/tx_storage/ob_checkpoint_service.cpp index db9d35a80d..8438d4893a 100644 --- a/src/storage/tx_storage/ob_checkpoint_service.cpp +++ b/src/storage/tx_storage/ob_checkpoint_service.cpp @@ -25,11 +25,7 @@ #include "logservice/palf/palf_env.h" #include "logservice/palf/lsn.h" #include "logservice/archiveservice/ob_archive_service.h" -#include "logservice/ob_log_handler.h" //ObLogHandler #include "storage/tx_storage/ob_ls_handle.h" -#include "rootserver/ob_tenant_info_loader.h"//get_tenant_info -#include "share/ob_cluster_version.h"//get_tenant_data_version -#include "share/ob_tenant_info_proxy.h"//ObAllTenantInfo namespace oceanbase { @@ -181,29 +177,13 @@ void ObCheckPointService::ObCheckpointTask::runTimerTask() K(archive_lsn), K(checkpoint_lsn), KPC(ls)); checkpoint_lsn = archive_lsn; } + if (OB_FAIL(ls->get_log_handler()->advance_base_lsn(checkpoint_lsn))) { + ARCHIVE_LOG(WARN, "advance base lsn failed", K(ret), K(checkpoint_lsn)); + } else { + FLOG_INFO("[CHECKPOINT] advance palf base lsn successfully", + K(checkpoint_lsn), K(ls->get_ls_id())); + } } - if (OB_SUCC(ret) && ls->get_ls_id().is_sys_ls()) { - //The check_point point of the sys ls refers to the sys_recovery_scn of __all_tenant_info - const uint64_t tenant_id = ls->get_tenant_id(); - palf::LSN sys_recovery_lsn; - if (!is_user_tenant(tenant_id)) { - //nothing todo - } else if (OB_FAIL(get_sys_ls_recovery_lsn_(ls->get_log_handler(), sys_recovery_lsn))) { - LOG_WARN("failed to get sys ls recovery lsn", KR(ret)); - } else if (sys_recovery_lsn < checkpoint_lsn) { - LOG_TRACE( - "sys ls recovery lsn is small than checkpoint_lsn, set base_lsn with sys recovery lsn", - K(sys_recovery_lsn), K(checkpoint_lsn)); - checkpoint_lsn = sys_recovery_lsn; - } // end for user tenant - } - if (FAILEDx(ls->get_log_handler()->advance_base_lsn(checkpoint_lsn))) { - ARCHIVE_LOG(WARN, "advance base lsn failed", K(ret), K(checkpoint_lsn)); - } else { - FLOG_INFO("[CHECKPOINT] advance palf base lsn successfully", - K(checkpoint_lsn), K(ls->get_ls_id())); - } - } if (ret == OB_ITER_END) { ret = OB_SUCCESS; @@ -216,32 +196,6 @@ void ObCheckPointService::ObCheckpointTask::runTimerTask() } } -int ObCheckPointService::ObCheckpointTask::get_sys_ls_recovery_lsn_( - logservice::ObLogHandler *log_handler, palf::LSN &recovery_lsn) -{ - int ret = OB_SUCCESS; - share::ObAllTenantInfo tenant_info; - SCN recovery_scn; - recovery_lsn.reset(); - if (OB_FAIL(MTL(rootserver::ObTenantInfoLoader *)->get_tenant_info(tenant_info))) { - LOG_WARN("failed to get tenant info", KR(ret)); - } else if (tenant_info.get_sys_recovery_scn().is_min()) { - // min is during upgrade, use sync scn instead - recovery_scn = tenant_info.get_sync_scn(); - } else { - recovery_scn = tenant_info.get_sys_recovery_scn(); - } - if (OB_FAIL(ret)) { - } else if (OB_ISNULL(log_handler)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("log handle is null", KR(ret), K(tenant_info)); - } else if (OB_FAIL(log_handler->locate_by_scn_coarsely(recovery_scn, - recovery_lsn))) { - LOG_WARN("failed to locate scn", KR(ret), K(tenant_info)); - } - return ret; -} - bool ObCheckPointService::get_disk_usage_threshold_(int64_t &threshold) { int ret = OB_SUCCESS; diff --git a/src/storage/tx_storage/ob_checkpoint_service.h b/src/storage/tx_storage/ob_checkpoint_service.h index 5a6a4b059c..720c1d8091 100644 --- a/src/storage/tx_storage/ob_checkpoint_service.h +++ b/src/storage/tx_storage/ob_checkpoint_service.h @@ -19,13 +19,6 @@ namespace oceanbase { -namespace logservice -{ -class ObLogHandler; -} -namespace palf { -struct LSN; -} namespace storage { namespace checkpoint @@ -82,8 +75,6 @@ private: virtual ~ObCheckpointTask() {} virtual void runTimerTask(); - private: - int get_sys_ls_recovery_lsn_(logservice::ObLogHandler *log_handler, palf::LSN &recovery_lsn); }; class ObTraversalFlushTask : public common::ObTimerTask diff --git a/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp b/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp index bed96c43ec..ec55101014 100644 --- a/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp +++ b/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp @@ -2152,7 +2152,7 @@ int ObAdminDumpBackupDataExecutor::dump_ls_attr_info_(const share::ObLSAttr &ls_ PrintHelper::print_dump_line("ls_group_id", ls_attr.get_ls_group_id()); PrintHelper::print_dump_line("flag", ls_attr.get_ls_flag()); PrintHelper::print_dump_line("status", ls_attr.get_ls_status()); - PrintHelper::print_dump_line("operation_type", ls_attr.get_ls_operation_type()); + PrintHelper::print_dump_line("operation_type", ls_attr.get_ls_operatin_type()); return ret; } diff --git a/unittest/rootserver/test_primary_ls_service.cpp b/unittest/rootserver/test_primary_ls_service.cpp index c3e560b397..576797bb9d 100644 --- a/unittest/rootserver/test_primary_ls_service.cpp +++ b/unittest/rootserver/test_primary_ls_service.cpp @@ -14,7 +14,7 @@ #include #include #define private public -#include "rootserver/ob_common_ls_service.h" +#include "rootserver/ob_primary_ls_service.h" namespace oceanbase { using namespace common; using namespace share; @@ -57,12 +57,12 @@ TEST_F(TestPrimaryLSService, zone_balance) ASSERT_EQ(ret, OB_SUCCESS); ret = primary_zone_array.push_back(z1); ASSERT_EQ(ret, OB_SUCCESS); - ret = ObCommonLSService::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone); + ret = ObTenantLSInfo::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone); ASSERT_EQ(ret, OB_SUCCESS); LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone)); ASSERT_EQ(z1, ls_primary_zone.at(0)); ASSERT_EQ(1, count_group_by_zone.at(0)); - ret = ObCommonLSService::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone); + ret = ObTenantLSInfo::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone); ASSERT_EQ(ret, OB_SUCCESS); ASSERT_EQ(z1, ls_primary_zone.at(0)); ASSERT_EQ(1, count_group_by_zone.at(0)); @@ -74,14 +74,14 @@ TEST_F(TestPrimaryLSService, zone_balance) ASSERT_EQ(ret, OB_SUCCESS); ret = primary_zone_array.push_back(z2); ASSERT_EQ(ret, OB_SUCCESS); - ret = ObCommonLSService::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone); + ret = ObTenantLSInfo::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone); LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone)); ASSERT_EQ(ret, OB_SUCCESS); ASSERT_EQ(z2, ls_primary_zone.at(0)); ASSERT_EQ(z2, ls_primary_zone.at(1)); ASSERT_EQ(0, count_group_by_zone.at(0)); ASSERT_EQ(2, count_group_by_zone.at(1)); - ret = ObCommonLSService::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone); + ret = ObTenantLSInfo::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone); LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone)); ASSERT_EQ(ret, OB_SUCCESS); ASSERT_EQ(z1, ls_primary_zone.at(0)); @@ -97,14 +97,14 @@ TEST_F(TestPrimaryLSService, zone_balance) ASSERT_EQ(ret, OB_SUCCESS); ret = primary_zone_array.push_back(z3); ASSERT_EQ(ret, OB_SUCCESS); - ret = ObCommonLSService::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone); + ret = ObTenantLSInfo::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone); LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone)); ASSERT_EQ(ret, OB_SUCCESS); ASSERT_EQ(z1, ls_primary_zone.at(0)); ASSERT_EQ(z3, ls_primary_zone.at(1)); ASSERT_EQ(1, count_group_by_zone.at(0)); ASSERT_EQ(1, count_group_by_zone.at(1)); - ret = ObCommonLSService::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone); + ret = ObTenantLSInfo::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone); LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone)); ASSERT_EQ(ret, OB_SUCCESS); ASSERT_EQ(z1, ls_primary_zone.at(0)); @@ -117,7 +117,7 @@ TEST_F(TestPrimaryLSService, zone_balance) count_group_by_zone.reset(); ret = primary_zone_array.push_back(z2); ASSERT_EQ(ret, OB_SUCCESS); - ret = ObCommonLSService::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone); + ret = ObTenantLSInfo::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone); LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone)); ASSERT_EQ(ret, OB_SUCCESS); ASSERT_EQ(z2, ls_primary_zone.at(0)); @@ -125,7 +125,7 @@ TEST_F(TestPrimaryLSService, zone_balance) ASSERT_EQ(0, count_group_by_zone.at(0)); ASSERT_EQ(0, count_group_by_zone.at(1)); ASSERT_EQ(2, count_group_by_zone.at(2)); - ret = ObCommonLSService::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone); + ret = ObTenantLSInfo::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone); LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone)); ASSERT_EQ(ret, OB_SUCCESS); ASSERT_EQ(z1, ls_primary_zone.at(0)); @@ -145,7 +145,7 @@ TEST_F(TestPrimaryLSService, zone_balance) ASSERT_EQ(ret, OB_SUCCESS); ret = primary_zone_infos.push_back(info); ASSERT_EQ(ret, OB_SUCCESS); - ret = ObCommonLSService::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone); + ret = ObTenantLSInfo::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone); LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone)); ASSERT_EQ(ret, OB_SUCCESS); ASSERT_EQ(z2, ls_primary_zone.at(0)); @@ -155,7 +155,7 @@ TEST_F(TestPrimaryLSService, zone_balance) ASSERT_EQ(1, count_group_by_zone.at(0)); ASSERT_EQ(0, count_group_by_zone.at(1)); ASSERT_EQ(3, count_group_by_zone.at(2)); - ret = ObCommonLSService::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone); + ret = ObTenantLSInfo::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone); LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone)); ASSERT_EQ(ret, OB_SUCCESS); ASSERT_EQ(z3, ls_primary_zone.at(0)); @@ -184,7 +184,7 @@ TEST_F(TestPrimaryLSService, zone_balance) ASSERT_EQ(ret, OB_SUCCESS); ret = primary_zone_infos.push_back(info); ASSERT_EQ(ret, OB_SUCCESS); - ret = ObCommonLSService::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone); + ret = ObTenantLSInfo::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone); LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone)); ASSERT_EQ(ret, OB_SUCCESS); ASSERT_EQ(z1, ls_primary_zone.at(0)); @@ -195,7 +195,7 @@ TEST_F(TestPrimaryLSService, zone_balance) ASSERT_EQ(3, count_group_by_zone.at(0)); ASSERT_EQ(1, count_group_by_zone.at(1)); ASSERT_EQ(1, count_group_by_zone.at(2)); - ret = ObCommonLSService::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone); + ret = ObTenantLSInfo::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone); LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone)); ASSERT_EQ(z3, ls_primary_zone.at(0)); ASSERT_EQ(z1, ls_primary_zone.at(1)); @@ -232,7 +232,7 @@ TEST_F(TestPrimaryLSService, zone_balance) ASSERT_EQ(ret, OB_SUCCESS); ret = primary_zone_infos.push_back(info); ASSERT_EQ(ret, OB_SUCCESS); - ret = ObCommonLSService::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone); + ret = ObTenantLSInfo::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone); LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone)); ASSERT_EQ(ret, OB_SUCCESS); ASSERT_EQ(z1, ls_primary_zone.at(0)); @@ -246,7 +246,7 @@ TEST_F(TestPrimaryLSService, zone_balance) ASSERT_EQ(4, count_group_by_zone.at(0)); ASSERT_EQ(0, count_group_by_zone.at(1)); ASSERT_EQ(4, count_group_by_zone.at(2)); - ret = ObCommonLSService::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone); + ret = ObTenantLSInfo::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone); LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone)); ASSERT_EQ(z3, ls_primary_zone.at(0)); ASSERT_EQ(z1, ls_primary_zone.at(1));