From 09f099c4e2f7d1e2253bda48f15ace1f8985656f Mon Sep 17 00:00:00 2001 From: maosy <630014370@qq.com> Date: Mon, 27 Feb 2023 10:41:19 +0000 Subject: [PATCH] add sys_recovery_scn to __all_tenant_info && primary tenant iter log --- .../data_dictionary/ob_data_dict_struct.cpp | 2 +- .../libobcdc/src/ob_log_ls_op_processor.cpp | 14 +- src/logservice/ob_garbage_collector.cpp | 4 +- src/logservice/ob_log_base_type.h | 8 + src/observer/omt/ob_multi_tenant.cpp | 8 +- src/rootserver/CMakeLists.txt | 6 +- src/rootserver/ob_common_ls_service.cpp | 602 +++++ src/rootserver/ob_common_ls_service.h | 126 + src/rootserver/ob_ddl_service.cpp | 6 +- ...portor.cpp => ob_ls_recovery_reportor.cpp} | 89 +- ...y_reportor.h => ob_ls_recovery_reportor.h} | 17 +- src/rootserver/ob_ls_service_helper.cpp | 1860 +++++++++++++ src/rootserver/ob_ls_service_helper.h | 279 ++ src/rootserver/ob_primary_ls_service.cpp | 2311 +---------------- src/rootserver/ob_primary_ls_service.h | 273 +- src/rootserver/ob_recovery_ls_service.cpp | 657 +---- src/rootserver/ob_recovery_ls_service.h | 63 +- src/rootserver/ob_root_utils.cpp | 1 - src/rootserver/ob_root_utils.h | 1 - src/rootserver/ob_tenant_info_loader.cpp | 15 + src/rootserver/ob_tenant_info_loader.h | 3 +- src/rootserver/ob_tenant_info_report.cpp | 114 + src/rootserver/ob_tenant_info_report.h | 84 + .../ob_tenant_role_transition_service.cpp | 137 +- .../ob_tenant_role_transition_service.h | 11 +- src/rootserver/ob_tenant_thread_helper.cpp | 305 +++ src/rootserver/ob_tenant_thread_helper.h | 101 + .../restore/ob_restore_scheduler.cpp | 41 +- src/rootserver/restore/ob_restore_scheduler.h | 2 +- .../ob_inner_table_schema.12251_12300.cpp | 19 + .../ob_inner_table_schema.15251_15300.cpp | 15 + .../ob_inner_table_schema.21151_21200.cpp | 2 +- .../ob_inner_table_schema.351_400.cpp | 19 + .../inner_table/ob_inner_table_schema_def.py | 7 + src/share/ls/ob_ls_i_life_manager.h | 37 + src/share/ls/ob_ls_life_manager.cpp | 49 +- src/share/ls/ob_ls_life_manager.h | 50 +- src/share/ls/ob_ls_operator.cpp | 66 +- src/share/ls/ob_ls_operator.h | 26 +- src/share/ls/ob_ls_recovery_stat_operator.cpp | 16 +- src/share/ob_primary_standby_service.cpp | 99 +- src/share/ob_primary_standby_service.h | 15 - src/share/ob_tenant_info_proxy.cpp | 202 +- src/share/ob_tenant_info_proxy.h | 42 +- src/share/ob_tenant_switchover_status.cpp | 1 - src/share/ob_tenant_switchover_status.h | 16 +- src/share/ob_upgrade_utils.cpp | 106 + src/share/ob_upgrade_utils.h | 8 +- src/share/rc/ob_tenant_base.h | 8 +- src/storage/ls/ob_ls.cpp | 37 +- src/storage/tx/ob_trans_service_v4.cpp | 1 - .../tx_storage/ob_checkpoint_service.cpp | 58 +- .../tx_storage/ob_checkpoint_service.h | 9 + .../ob_admin_dump_backup_data_executor.cpp | 2 +- .../rootserver/test_primary_ls_service.cpp | 30 +- 55 files changed, 4347 insertions(+), 3733 deletions(-) create mode 100755 src/rootserver/ob_common_ls_service.cpp create mode 100644 src/rootserver/ob_common_ls_service.h rename src/rootserver/{ob_tenant_recovery_reportor.cpp => ob_ls_recovery_reportor.cpp} (86%) rename src/rootserver/{ob_tenant_recovery_reportor.h => ob_ls_recovery_reportor.h} (82%) create mode 100755 src/rootserver/ob_ls_service_helper.cpp create mode 100644 src/rootserver/ob_ls_service_helper.h create mode 100755 src/rootserver/ob_tenant_info_report.cpp create mode 100644 src/rootserver/ob_tenant_info_report.h create mode 100755 src/rootserver/ob_tenant_thread_helper.cpp create mode 100644 src/rootserver/ob_tenant_thread_helper.h diff --git a/src/logservice/data_dictionary/ob_data_dict_struct.cpp b/src/logservice/data_dictionary/ob_data_dict_struct.cpp index 65c652764..93cca1469 100644 --- a/src/logservice/data_dictionary/ob_data_dict_struct.cpp +++ b/src/logservice/data_dictionary/ob_data_dict_struct.cpp @@ -389,7 +389,7 @@ int ObDictTenantMeta::incremental_data_update(const share::ObLSAttr &ls_attr) if (OB_UNLIKELY(! ls_attr.is_valid())) { ret = OB_INVALID_ARGUMENT; DDLOG(WARN, "ls attr is invalid", KR(ret), K(ls_attr)); - } else if (share::is_ls_create_end_op(ls_attr.get_ls_operatin_type())) { + } else if (share::is_ls_create_end_op(ls_attr.get_ls_operation_type())) { if (OB_FAIL(ls_arr_.push_back(ls_attr.get_ls_id()))) { DDLOG(WARN, "ls_arr_ push_back failed", KR(ret), K(ls_attr), K(ls_arr_)); } diff --git a/src/logservice/libobcdc/src/ob_log_ls_op_processor.cpp b/src/logservice/libobcdc/src/ob_log_ls_op_processor.cpp index 6a24557c7..a8b16c07a 100644 --- a/src/logservice/libobcdc/src/ob_log_ls_op_processor.cpp +++ b/src/logservice/libobcdc/src/ob_log_ls_op_processor.cpp @@ -38,20 +38,20 @@ int ObLogLSOpProcessor::process_ls_op( } else if (OB_ISNULL(tenant = guard.get_tenant())) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("get tenant fail, tenant is NULL", KR(ret), K(tenant_id)); - } else if (share::is_ls_create_pre_op(ls_attr.get_ls_operatin_type())) { + } else if (share::is_ls_create_pre_op(ls_attr.get_ls_operation_type())) { LOG_INFO("[LS_PROCESSOR] ls create pre operation", K(tenant_id), K(lsn), K(start_tstamp_ns), K(ls_attr)); - } else if (share::is_ls_create_end_op(ls_attr.get_ls_operatin_type())) { + } else if (share::is_ls_create_end_op(ls_attr.get_ls_operation_type())) { LOG_INFO("[LS_PROCESSOR] ls create end operation", K(tenant_id), K(lsn), K(start_tstamp_ns), K(ls_attr)); //create new ls; if (OB_FAIL(create_new_ls_(tenant, start_tstamp_ns, ls_attr))) { LOG_ERROR("failed to create new ls", KR(ret), K(tenant_id), K(lsn), K(start_tstamp_ns), K(ls_attr)); } - } else if (share::is_ls_create_abort_op(ls_attr.get_ls_operatin_type())) { - } else if (share::is_ls_drop_pre_op(ls_attr.get_ls_operatin_type())) { - } else if (share::is_ls_drop_end_op(ls_attr.get_ls_operatin_type())) { - } else if (share::is_ls_tenant_drop_pre_op(ls_attr.get_ls_operatin_type())) { + } else if (share::is_ls_create_abort_op(ls_attr.get_ls_operation_type())) { + } else if (share::is_ls_drop_pre_op(ls_attr.get_ls_operation_type())) { + } else if (share::is_ls_drop_end_op(ls_attr.get_ls_operation_type())) { + } else if (share::is_ls_tenant_drop_pre_op(ls_attr.get_ls_operation_type())) { //only sys ls - } else if (share::is_ls_tenant_drop_op(ls_attr.get_ls_operatin_type())) { + } else if (share::is_ls_tenant_drop_op(ls_attr.get_ls_operation_type())) { } else { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected operation type", KR(ret), K(tenant_id), K(lsn), K(start_tstamp_ns), K(ls_attr)); diff --git a/src/logservice/ob_garbage_collector.cpp b/src/logservice/ob_garbage_collector.cpp index 4b0954a37..3cda548ae 100644 --- a/src/logservice/ob_garbage_collector.cpp +++ b/src/logservice/ob_garbage_collector.cpp @@ -29,7 +29,7 @@ #include "share/rc/ob_tenant_base.h" #include "share/ls/ob_ls_life_manager.h" #include "storage/tx_storage/ob_ls_handle.h" -#include "rootserver/ob_tenant_recovery_reportor.h" // ObTenantRecoveryReportor +#include "rootserver/ob_tenant_info_loader.h" // ObTenantInfoLoader #include "share/ob_occam_time_guard.h" namespace oceanbase @@ -651,7 +651,7 @@ void ObGCHandler::try_check_and_set_wait_gc_(ObGarbageCollector::LSStatus &ls_st int ObGCHandler::get_tenant_readable_scn_(SCN &readable_scn) { - return MTL(rootserver::ObTenantRecoveryReportor *)->get_tenant_readable_scn(readable_scn); + return MTL(rootserver::ObTenantInfoLoader *)->get_tenant_readable_scn(readable_scn); } // 由于日志流GC导致的归档日志不完整是无法被归档检查出来的异常, 因此需要保证GC与归档状态互斥; diff --git a/src/logservice/ob_log_base_type.h b/src/logservice/ob_log_base_type.h index 916c9c5a3..7dfdea656 100644 --- a/src/logservice/ob_log_base_type.h +++ b/src/logservice/ob_log_base_type.h @@ -75,6 +75,10 @@ enum ObLogBaseType // for arbitration service ARBITRATION_SERVICE_LOG_BASE_TYPE = 21, + + TENANT_INFO_REPORTOR_LOG_BASE_TYPE = 22, + + COMMON_LS_SERVICE_LOG_BASE_TYPE = 23, // pay attention!!! // add log type in log_base_type_to_string // max value @@ -131,6 +135,10 @@ int log_base_type_to_string(const ObLogBaseType log_type, strncpy(str ,"DATA_DICTIONARY_SERVICE", str_len); } else if (log_type == ARBITRATION_SERVICE_LOG_BASE_TYPE) { strncpy(str ,"ARBITRATION_SERVICE", str_len); + } else if (log_type == TENANT_INFO_REPORTOR_LOG_BASE_TYPE) { + strncpy(str ,"TENANT_INFO_REPORTOR", str_len); + } else if (log_type == COMMON_LS_SERVICE_LOG_BASE_TYPE) { + strncpy(str ,"COMMON_LS_SERVICE", str_len); } else { ret = OB_INVALID_ARGUMENT; } diff --git a/src/observer/omt/ob_multi_tenant.cpp b/src/observer/omt/ob_multi_tenant.cpp index 5f6bfd821..741dccaf1 100644 --- a/src/observer/omt/ob_multi_tenant.cpp +++ b/src/observer/omt/ob_multi_tenant.cpp @@ -85,10 +85,12 @@ #include "observer/omt/ob_tenant_config_mgr.h" #include "observer/report/ob_tenant_meta_checker.h" #include "storage/high_availability/ob_storage_ha_service.h" -#include "rootserver/ob_tenant_recovery_reportor.h"//ObTenantRecoveryReportor +#include "rootserver/ob_ls_recovery_reportor.h"//ObLSRecoveryReportor #include "rootserver/ob_tenant_info_loader.h"//ObTenantInfoLoader #include "rootserver/ob_primary_ls_service.h"//ObLSService #include "rootserver/ob_recovery_ls_service.h"//ObRecoveryLSService +#include "rootserver/ob_common_ls_service.h"//ObCommonLSService +#include "rootserver/ob_tenant_info_report.h"//ObTenantInfoReportor #include "rootserver/restore/ob_restore_scheduler.h" //ObRestoreService #include "logservice/leader_coordinator/ob_leader_coordinator.h" #include "storage/lob/ob_lob_manager.h" @@ -401,9 +403,11 @@ int ObMultiTenant::init(ObAddr myaddr, MTL_BIND2(mtl_new_default, rootserver::ObPrimaryMajorFreezeService::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default); MTL_BIND2(mtl_new_default, rootserver::ObRestoreMajorFreezeService::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default); MTL_BIND2(mtl_new_default, ObTenantMetaChecker::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default); - MTL_BIND2(mtl_new_default, rootserver::ObTenantRecoveryReportor::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default); + MTL_BIND2(mtl_new_default, rootserver::ObLSRecoveryReportor::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default); MTL_BIND2(mtl_new_default, rootserver::ObTenantInfoLoader::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default); MTL_BIND2(mtl_new_default, rootserver::ObPrimaryLSService::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default); + MTL_BIND2(mtl_new_default, rootserver::ObCommonLSService::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default); + MTL_BIND2(mtl_new_default, rootserver::ObTenantInfoReportor::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default); MTL_BIND2(mtl_new_default, rootserver::ObRecoveryLSService::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default); MTL_BIND2(mtl_new_default, rootserver::ObRestoreService::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default); MTL_BIND2(mtl_new_default, coordinator::ObLeaderCoordinator::mtl_init, coordinator::ObLeaderCoordinator::mtl_start, coordinator::ObLeaderCoordinator::mtl_stop, coordinator::ObLeaderCoordinator::mtl_wait, mtl_destroy_default); diff --git a/src/rootserver/CMakeLists.txt b/src/rootserver/CMakeLists.txt index a1138982a..0fffc9b49 100644 --- a/src/rootserver/CMakeLists.txt +++ b/src/rootserver/CMakeLists.txt @@ -66,10 +66,14 @@ ob_set_subtarget(ob_rootserver common ob_vtable_location_getter.cpp ob_zone_manager.cpp ob_zone_unit_provider.cpp - ob_tenant_recovery_reportor.cpp + ob_tenant_thread_helper.cpp + ob_ls_recovery_reportor.cpp ob_tenant_info_loader.cpp + ob_tenant_info_report.cpp ob_primary_ls_service.cpp ob_recovery_ls_service.cpp + ob_common_ls_service.cpp + ob_ls_service_helper.cpp ob_tenant_role_transition_service.cpp ob_server_balancer.cpp ob_system_admin_util.cpp diff --git a/src/rootserver/ob_common_ls_service.cpp b/src/rootserver/ob_common_ls_service.cpp new file mode 100755 index 000000000..6f8023330 --- /dev/null +++ b/src/rootserver/ob_common_ls_service.cpp @@ -0,0 +1,602 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX RS +#include "ob_common_ls_service.h" +#include "lib/profile/ob_trace_id.h" +#include "share/ob_errno.h" +#include "share/ob_max_id_fetcher.h" +#include "share/schema/ob_schema_struct.h"//ObTenantInfo +#include "share/ls/ob_ls_creator.h" //ObLSCreator +#include "share/ls/ob_ls_life_manager.h"//ObLSLifeAgentManager +#include "share/ob_primary_zone_util.h"//ObPrimaryZoneUtil +#include "share/ob_share_util.h"//ObShareUtil +#include "share/ob_tenant_info_proxy.h"//ObAllTenantInfo +#include "share/ob_common_rpc_proxy.h"//common_rpc_proxy +#include "observer/ob_server_struct.h"//GCTX +#include "rootserver/ob_tenant_info_loader.h"//get_tenant_info +#include "logservice/palf/palf_base_info.h"//PalfBaseInfo + +namespace oceanbase +{ +using namespace common; +using namespace share; +using namespace transaction; +using namespace palf; +namespace rootserver +{ +//////////////ObCommonLSService +int ObCommonLSService::mtl_init(ObCommonLSService *&ka) +{ + return ka->init(); +} + +int ObCommonLSService::init() +{ + int ret = OB_SUCCESS; + tenant_id_ = MTL_ID(); + if (OB_UNLIKELY(inited_)) { + ret = OB_INIT_TWICE; + LOG_WARN("has inited", KR(ret)); + } else if (OB_FAIL(ObTenantThreadHelper::create("COMMONLSSe", + lib::TGDefIDs::SimpleLSService, *this))) { + LOG_WARN("failed to create thread", KR(ret)); + } else if (OB_FAIL(ObTenantThreadHelper::start())) { + LOG_WARN("fail to start", KR(ret)); + } else { + inited_ = true; + } + return ret; +} + +void ObCommonLSService::destroy() +{ + ObTenantThreadHelper::destroy(); + tenant_id_ = OB_INVALID_TENANT_ID; + inited_ = false; +} + +void ObCommonLSService::do_work() +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (OB_FAIL(wait_tenant_schema_and_version_ready_(tenant_id_, DATA_VERSION_4_1_0_0))) { + LOG_WARN("failed to wait tenant schema version ready", KR(ret), K(tenant_id_), K(DATA_CURRENT_VERSION)); + } else { + int64_t idle_time_us = 100 * 1000L; + share::schema::ObTenantSchema user_tenant_schema; + while (!has_set_stop()) { + ret = OB_SUCCESS; + ObCurTraceId::init(GCONF.self_addr_); + if (is_meta_tenant(tenant_id_)) { + const uint64_t user_tenant_id = gen_user_tenant_id(tenant_id_); + if (OB_FAIL(get_tenant_schema(user_tenant_id, user_tenant_schema))) { + LOG_WARN("failed to get user tenant schema", KR(ret), K(user_tenant_id)); + } else if (user_tenant_schema.is_dropping()) { + if (OB_FAIL(try_force_drop_tenant_(user_tenant_schema))) { + LOG_WARN("failed to force drop tenant", KR(ret), K(user_tenant_id)); + } + } else if (OB_FAIL(try_create_ls_(user_tenant_schema))) { + LOG_WARN("failed to create ls", KR(ret), K(user_tenant_schema)); + } + } + + if (REACH_TIME_INTERVAL(1 * 1000 * 1000)) { + ret = OB_SUCCESS;//ignore error + if (OB_FAIL(try_update_sys_ls_primary_zone_())) { + LOG_WARN("failed to update sys ls primary zone", KR(ret)); + } + if (is_meta_tenant(tenant_id_)) { + if (FAILEDx(try_adjust_user_ls_primary_zone_(user_tenant_schema))) { + LOG_WARN("failed to adjust user ls primary zone", KR(ret), K(user_tenant_schema)); + } + } + } + + user_tenant_schema.reset(); + idle(idle_time_us); + LOG_INFO("[COMMON_LS_SERVICE] finish one round", KR(ret), K(idle_time_us)); + } // end while + } +} + +int ObCommonLSService::try_update_sys_ls_primary_zone_() +{ + int ret = OB_SUCCESS; + share::schema::ObTenantSchema tenant_schema; + if (OB_ISNULL(GCTX.schema_service_) || OB_ISNULL(GCTX.sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error", KR(ret), KP(GCTX.sql_proxy_), KP(GCTX.schema_service_)); + } else if (OB_FAIL(get_tenant_schema(tenant_id_, tenant_schema))) { + LOG_WARN("failed to get tenant schema", KR(ret), K(tenant_id_)); + } else { + share::ObLSPrimaryZoneInfo primary_zone_info; + ObArray primary_zone; + share::ObLSStatusOperator status_op; + ObZone new_primary_zone; + ObSqlString new_zone_priority; + if (OB_FAIL(ObPrimaryZoneUtil::get_tenant_primary_zone_array( + tenant_schema, primary_zone))) { + LOG_WARN("failed to get tenant primary zone array", KR(ret), K(tenant_schema)); + } else if (OB_FAIL(status_op.get_ls_primary_zone_info(tenant_id_, + SYS_LS, primary_zone_info, *GCTX.sql_proxy_))) { + LOG_WARN("failed to get ls primary_zone info", KR(ret), K(tenant_id_)); + } else if (OB_UNLIKELY(0 == primary_zone.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("primary zone is empty", KR(ret), K(tenant_schema)); + } else if (has_exist_in_array(primary_zone, primary_zone_info.get_primary_zone())) { + if (OB_FAIL(new_primary_zone.assign(primary_zone_info.get_primary_zone()))) { + LOG_WARN("failed to assign primary zone", KR(ret), K(primary_zone_info)); + } + } else if (OB_FAIL(new_primary_zone.assign(primary_zone.at(0)))) { + LOG_WARN("failed to assign primary zone", KR(ret), K(primary_zone)); + } + if (OB_FAIL(ret)) { + } else if (is_sys_tenant(tenant_id_)) { + //sys tenant use tenant normalize primary zone + if (OB_FAIL(ObPrimaryZoneUtil::get_tenant_zone_priority( + tenant_schema, new_zone_priority))) { + LOG_WARN("failed to get tenant primary zone array", KR(ret), K(tenant_schema)); + } + } else if (OB_FAIL(get_zone_priority(new_primary_zone, + tenant_schema, new_zone_priority))) { + LOG_WARN("failed to get normalize primary zone", KR(ret), K(new_primary_zone)); + } + if (FAILEDx(try_update_ls_primary_zone_( + primary_zone_info, new_primary_zone, new_zone_priority))) { + LOG_WARN("failed to update ls primary zone", KR(ret), K(primary_zone_info), + K(new_primary_zone), K(new_zone_priority)); + } else if (is_meta_tenant(tenant_id_)) { + //user sys ls has same primary zone with meta sys ls + share::ObLSPrimaryZoneInfo user_primary_zone_info; + const uint64_t user_tenant_id = gen_user_tenant_id(tenant_id_); + if (OB_FAIL(status_op.get_ls_primary_zone_info(user_tenant_id, SYS_LS, + user_primary_zone_info, *GCTX.sql_proxy_))) { + LOG_WARN("failed to get ls primary_zone info", KR(ret), K(tenant_id_), K(user_tenant_id)); + } else if (OB_FAIL(try_update_ls_primary_zone_(user_primary_zone_info, new_primary_zone, new_zone_priority))) { + LOG_WARN("failed to update ls primary zone", KR(ret), K(user_primary_zone_info), + K(new_primary_zone), K(new_zone_priority)); + } + } + } + return ret; +} + +int ObCommonLSService::try_adjust_user_ls_primary_zone_(const share::schema::ObTenantSchema &tenant_schema) +{ + int ret = OB_SUCCESS; + share::ObLSStatusOperator status_op; + share::ObLSPrimaryZoneInfoArray info_array; + ObArray primary_zone; + const uint64_t tenant_id = tenant_schema.get_tenant_id(); + if (OB_ISNULL(GCTX.schema_service_) || OB_ISNULL(GCTX.sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error", KR(ret), KP(GCTX.sql_proxy_), KP(GCTX.schema_service_)); + } else if (!is_user_tenant(tenant_id) || !tenant_schema.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tenant iS invalid", KR(ret), K(tenant_id), K(tenant_schema)); + } else if (OB_FAIL(ObPrimaryZoneUtil::get_tenant_primary_zone_array( + tenant_schema, primary_zone))) { + LOG_WARN("failed to get tenant primary zone array", KR(ret), K(tenant_schema)); + } else { + if (OB_FAIL(status_op.get_tenant_primary_zone_info_array( + tenant_id, info_array, *GCTX.sql_proxy_))) { + LOG_WARN("failed to get tenant primary zone info array", KR(ret), K(tenant_id)); + } + } + + if (OB_SUCC(ret)) { + uint64_t last_ls_group_id = OB_INVALID_ID; + share::ObLSPrimaryZoneInfoArray tmp_info_array; + for (int64_t i = 0; OB_SUCC(ret) && i < info_array.count(); ++i) { + //TODO ls group id maybe zero + const ObLSPrimaryZoneInfo &info = info_array.at(i); + if (!info.get_ls_id().is_sys_ls()) { + if (OB_INVALID_ID == last_ls_group_id) { + last_ls_group_id = info.get_ls_group_id(); + } + if (last_ls_group_id != info.get_ls_group_id()) { + //process the ls group + if (OB_FAIL(adjust_primary_zone_by_ls_group_(primary_zone, tmp_info_array, tenant_schema))) { + LOG_WARN("failed to update primary zone of each ls group", KR(ret), + K(tmp_info_array), K(primary_zone), K(tenant_schema)); + } else { + tmp_info_array.reset(); + last_ls_group_id = info.get_ls_group_id(); + } + } + if (FAILEDx(tmp_info_array.push_back(info))) { + LOG_WARN("failed to push back primary info array", KR(ret), K(i)); + } + } + } + if (OB_SUCC(ret) && 0 < tmp_info_array.count()) { + if (OB_FAIL(adjust_primary_zone_by_ls_group_(primary_zone, tmp_info_array, tenant_schema))) { + LOG_WARN("failed to update primary zone of each ls group", KR(ret), + K(tmp_info_array), K(primary_zone), K(tenant_schema)); + } + } + } + + return ret; +} +//check every ls has right primary zone in ls group +//if primary_zone of ls not in primary zone, try to choose a right zone, +//if can not find a zone to modify, modify the ls to exist zone, +//load_balancer will choose the right ls to drop +//eg: +//if ls group has z1, z2 and primary zone is : z1. need modify z2 to z1 +//if ls group has z1, z2 and primary zone is : z1, z3. modify z2 to z3; +int ObCommonLSService::adjust_primary_zone_by_ls_group_( + const common::ObIArray &primary_zone_array, + const share::ObLSPrimaryZoneInfoArray &primary_zone_infos, + const share::schema::ObTenantSchema &tenant_schema) +{ + int ret = OB_SUCCESS; + const uint64_t tenant_id = tenant_schema.get_tenant_id(); + const int64_t ls_count = primary_zone_infos.count(); + const int64_t primary_zone_count = primary_zone_array.count(); + if (OB_UNLIKELY(0 == primary_zone_count + || 0 == ls_count || !tenant_schema.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(primary_zone_infos), + K(primary_zone_array), K(tenant_schema)); + } else if (OB_ISNULL(GCTX.sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy is null", KR(ret)); + } else if (1 == ls_count + && primary_zone_infos.at(0).get_ls_id().is_sys_ls()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("can not be sys ls", KR(ret), K(ls_count), K(primary_zone_infos)); + } else { + //Algorithm Description: + //Assumption: We have 5 ls, 3 primary_zones(z1, z2, z3). + //1. Set the primary zone of ls to tenant's primary zone, + //choose the least number of log streams on the zone is selected for all zones + //2. After all the primary_zone of the ls are in the primary_zonw of the tenant, + //choose the primary_zone with the most and least ls. Adjust a certain number of ls to the smallest zone without exceeding the balance + //while guaranteeing that the number of the zone with the most is no less than the average. + ObArray ls_primary_zone;//is match with primary_zone_infos + ObSEArray count_group_by_zone;//ls count of each primary zone + ObSqlString new_zone_priority; + if (OB_FAIL(set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, + count_group_by_zone))) { + LOG_WARN("failed to set ls to primary zone", KR(ret), K(primary_zone_array), K(primary_zone_infos)); + } else if (OB_FAIL(balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone))) { + LOG_WARN("failed to balance ls primary zone", KR(ret), K(ls_primary_zone), K(count_group_by_zone)); + } + + for (int64_t i = 0; OB_SUCC(ret) && i < ls_count; ++i) { + const ObZone &new_primary_zone = ls_primary_zone.at(i); + if (OB_FAIL(get_zone_priority(new_primary_zone, tenant_schema, new_zone_priority))) { + LOG_WARN("failed to get normalize primary zone", KR(ret), K(new_primary_zone)); + } + if (FAILEDx(try_update_ls_primary_zone_(primary_zone_infos.at(i), new_primary_zone, new_zone_priority))) { + LOG_WARN("failed to update ls primary zone", KR(ret), "primary_zone_info", primary_zone_infos.at(i), + K(new_primary_zone), K(new_zone_priority)); + } + } + } + return ret; +} + +int ObCommonLSService::set_ls_to_primary_zone( + const common::ObIArray &primary_zone_array, + const share::ObLSPrimaryZoneInfoArray &primary_zone_infos, + common::ObIArray &ls_primary_zone, + common::ObIArray &count_group_by_zone) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(0 == primary_zone_array.count() + || 0 == primary_zone_infos.count())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(primary_zone_array), K(primary_zone_infos)); + } else { + const int64_t primary_zone_count = primary_zone_array.count(); + const int64_t ls_count = primary_zone_infos.count(); + //ls may not in primary zone, record the index of primary_zone_infos not in primary zone + ObSEArray index_not_primary_zone; + int64_t index = 0; + ARRAY_FOREACH_X(primary_zone_array, idx, cnt, OB_SUCC(ret)) { + if (OB_FAIL(count_group_by_zone.push_back(0))) { + LOG_WARN("failed to push back", KR(ret), K(idx)); + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < ls_count; ++i) { + const ObZone ¤t_zone = primary_zone_infos.at(i).get_primary_zone(); + if (has_exist_in_array(primary_zone_array, current_zone, &index)) { + count_group_by_zone.at(index)++; + } else if (OB_FAIL(index_not_primary_zone.push_back(i))) { + LOG_WARN("failed to push back", KR(ret), K(i), K(current_zone)); + } + if (FAILEDx(ls_primary_zone.push_back(current_zone))) { + LOG_WARN("failed to push back current zone", KR(ret), K(i), K(current_zone)); + } + } + //1. take all ls primary zone to tenant primary zone, choose the less primary zone count + int64_t min_count = INT64_MAX; + int64_t min_index = 0; + for (int64_t i = 0; OB_SUCC(ret) && i < index_not_primary_zone.count(); ++i) { + const int64_t ls_index = index_not_primary_zone.at(i); + min_count = INT64_MAX; + ARRAY_FOREACH_X(primary_zone_array, idx, cnt, OB_SUCC(ret)) { + if (min_count > count_group_by_zone.at(idx)) { + min_count = count_group_by_zone.at(idx); + min_index = idx; + } + }//end for search min count + if (OB_FAIL(ret)) { + } else if (min_index >= primary_zone_count) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to found min count", KR(ret), K(min_index), K(primary_zone_array)); + } else if (OB_FAIL(ls_primary_zone.at(ls_index).assign(primary_zone_array.at(min_index)))) { + LOG_WARN("failed to assign primary zone", KR(ret), K(min_index), K(min_count), K(ls_index)); + } else { + count_group_by_zone.at(min_index)++; + } + } + } + return ret; + +} + +int ObCommonLSService::balance_ls_primary_zone( + const common::ObIArray &primary_zone_array, + common::ObIArray &ls_primary_zone, + common::ObIArray &count_group_by_zone) +{ + int ret = OB_SUCCESS; + const int64_t ls_count = ls_primary_zone.count(); + const int64_t primary_zone_count = count_group_by_zone.count(); + if (OB_UNLIKELY(0 == primary_zone_count + || 0 == ls_count + || primary_zone_count != primary_zone_array.count())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(count_group_by_zone), K(ls_primary_zone)); + } else { + int64_t max_count = -1, max_index = 0; + int64_t min_count = INT64_MAX, min_index = 0; + do { + max_count = -1, max_index = 0; + min_count = INT64_MAX, min_index = 0; + for (int64_t i = 0; OB_SUCC(ret) && i < primary_zone_count; ++i) { + const int64_t ls_count = count_group_by_zone.at(i); + if (min_count > ls_count) { + min_count = ls_count; + min_index = i; + } + if (max_count < ls_count) { + max_count = ls_count; + max_index = i; + } + }//end for find min and max count + if (OB_UNLIKELY(max_index >= primary_zone_count || min_index >= primary_zone_count + || -1 == max_count || INT64_MAX == min_count)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get primary zone", KR(ret), K(max_index), K(min_index), + K(min_count), K(max_count), + K(primary_zone_array), K(primary_zone_count), K(ls_primary_zone)); + } else if (max_count - min_count > 1) { + for (int64_t i = 0; OB_SUCC(ret) && i < ls_count; ++i) { + if (ls_primary_zone.at(i) == primary_zone_array.at(max_index)) { + if (OB_FAIL(ls_primary_zone.at(i).assign(primary_zone_array.at(min_index)))) { + LOG_WARN("failed to push back ls primary zone", KR(ret), K(min_index)); + } else { + count_group_by_zone.at(max_index)--; + count_group_by_zone.at(min_index)++; + } + break;//only change one by one + } + } + } + } while (max_count - min_count > 1); + } + return ret; +} + +int ObCommonLSService::try_update_ls_primary_zone_( + const share::ObLSPrimaryZoneInfo &primary_zone_info, + const common::ObZone &new_primary_zone, + const common::ObSqlString &zone_priority) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!primary_zone_info.is_valid() + || new_primary_zone.is_empty() || zone_priority.empty())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("primary zone info is invalid", KR(ret), K(primary_zone_info), + K(new_primary_zone), K(zone_priority)); + } else if (OB_ISNULL(GCTX.sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy is null", KR(ret)); + } else if (new_primary_zone != primary_zone_info.get_primary_zone() + || zone_priority.string() != primary_zone_info.get_zone_priority_str()) { + ObLSLifeAgentManager ls_life_agent(*GCTX.sql_proxy_); + if (OB_FAIL(ls_life_agent.update_ls_primary_zone(primary_zone_info.get_tenant_id(), + primary_zone_info.get_ls_id(), + new_primary_zone, zone_priority.string()))) { + LOG_WARN("failed to update ls primary zone", KR(ret), K(primary_zone_info), + K(new_primary_zone), K(zone_priority)); + } + LOG_INFO("[COMMON_LS_SERVICE] update ls primary zone", KR(ret), K(new_primary_zone), + K(zone_priority), K(primary_zone_info)); + } else { + //no need update + } + return ret; +} + +int ObCommonLSService::try_create_ls_(const share::schema::ObTenantSchema &tenant_schema) +{ + int ret = OB_SUCCESS; + const uint64_t tenant_id = tenant_schema.get_tenant_id(); + if (OB_ISNULL(GCTX.sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy is null", KR(ret), KP(GCTX.sql_proxy_)); + } else if (!is_user_tenant(tenant_id) || !tenant_schema.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tenant is invalid", KR(ret), K(tenant_id), K(tenant_schema)); + } else { + share::ObLSStatusInfoArray status_info_array; + share::ObLSStatusOperator ls_op; + ObLSRecoveryStat recovery_stat; + ObLSRecoveryStatOperator ls_recovery_operator; + palf::PalfBaseInfo palf_base_info; + int tmp_ret = OB_SUCCESS; + if (OB_FAIL(ls_op.get_all_ls_status_by_order( + tenant_id, status_info_array, *GCTX.sql_proxy_))) { + LOG_WARN("failed to update ls status", KR(ret), K(tenant_id)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < status_info_array.count(); ++i) { + const ObLSStatusInfo &status_info = status_info_array.at(i); + if (status_info.ls_is_creating()) { + recovery_stat.reset(); + if (OB_FAIL(check_can_create_ls_(tenant_schema))) { + LOG_WARN("failed to wait can do recovery", KR(ret), K(tenant_schema)); + } else if (OB_FAIL(ls_recovery_operator.get_ls_recovery_stat( + tenant_id, status_info.ls_id_, false /*for_update*/, + recovery_stat, *GCTX.sql_proxy_))) { + LOG_WARN("failed to get ls recovery stat", KR(ret), K(tenant_id), + K(status_info)); + } else if (OB_FAIL(do_create_user_ls(tenant_schema, status_info, + recovery_stat.get_create_scn(), + false, palf_base_info))) { + LOG_WARN("failed to create new ls", KR(ret), K(status_info), + K(recovery_stat)); + } + } + } // end for + } + return ret; +} + +int ObCommonLSService::check_can_create_ls_(const share::schema::ObTenantSchema &tenant_schema) +{ + int ret = OB_SUCCESS; + const uint64_t tenant_id = tenant_schema.get_tenant_id(); + if (OB_ISNULL(GCTX.sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy is null", KR(ret), KP(GCTX.sql_proxy_)); + } else if (!is_user_tenant(tenant_id) || !tenant_schema.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tenant is invalid", KR(ret), K(tenant_id), K(tenant_schema)); + } else if (tenant_schema.is_restore()) { + //meta and user tenant unit must in the same observer + MTL_SWITCH(tenant_id) { + ObAllTenantInfo tenant_info; + ObTenantInfoLoader* tenant_loader = MTL(rootserver::ObTenantInfoLoader*); + if (OB_ISNULL(tenant_loader)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant report is null", KR(ret), K(tenant_id_)); + } else if (OB_FAIL(tenant_loader->get_tenant_info(tenant_info))) { + LOG_WARN("failed to get tenant info", KR(ret), K(tenant_id)); + } else if (OB_FAIL(check_can_do_recovery(tenant_info))) { + LOG_WARN("failed to wait can do recovery", KR(ret), K(tenant_info), K(tenant_id)); + } + } + } + return ret; +} +int ObCommonLSService::do_create_user_ls( + const share::schema::ObTenantSchema &tenant_schema, + const share::ObLSStatusInfo &info, const SCN &create_scn, + bool create_with_palf, const palf::PalfBaseInfo &palf_base_info) +{ + int ret = OB_SUCCESS; + LOG_INFO("[COMMON_LS_SERVICE] start to create ls", K(info), K(create_scn)); + const int64_t start_time = ObTimeUtility::fast_current_time(); + if (OB_UNLIKELY(!info.is_valid() || !info.ls_is_creating())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("info not valid", KR(ret), K(info)); + } else { + CK(OB_NOT_NULL(GCTX.sql_proxy_), OB_NOT_NULL(GCTX.srv_rpc_proxy_)) + common::ObArray locality_array; + int64_t paxos_replica_num = 0; + ObSchemaGetterGuard guard;//nothing + if (FAILEDx(tenant_schema.get_zone_replica_attr_array( + locality_array))) { + LOG_WARN("failed to get zone locality array", KR(ret)); + } else if (OB_FAIL(tenant_schema.get_paxos_replica_num( + guard, paxos_replica_num))) { + LOG_WARN("failed to get paxos replica num", KR(ret)); + } else { + ObLSCreator creator(*GCTX.srv_rpc_proxy_, info.tenant_id_, + info.ls_id_, GCTX.sql_proxy_); + if (OB_FAIL(creator.create_user_ls(info, paxos_replica_num, + locality_array, create_scn, + tenant_schema.get_compatibility_mode(), + create_with_palf, + palf_base_info))) { + LOG_WARN("failed to create user ls", KR(ret), K(info), K(locality_array), K(create_scn), + K(palf_base_info), K(create_with_palf)); + } + } + } + const int64_t cost = ObTimeUtility::fast_current_time() - start_time; + LOG_INFO("[COMMON_LS_SERVICE] end to create ls", KR(ret), K(info), K(cost)); + return ret; +} + +int ObCommonLSService::try_force_drop_tenant_( + const share::schema::ObTenantSchema &tenant_schema) +{ + int ret = OB_SUCCESS; + ObTimeoutCtx ctx; + const uint64_t tenant_id = tenant_schema.get_tenant_id(); + + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (OB_ISNULL(GCTX.sql_proxy_) + || OB_ISNULL(GCTX.rs_rpc_proxy_) + || OB_ISNULL(GCTX.schema_service_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ptr is null", KR(ret), KP(GCTX.sql_proxy_), KP(GCTX.rs_rpc_proxy_), + KP(GCTX.schema_service_)); + } else if (!is_user_tenant(tenant_id) || !tenant_schema.is_valid() + || !tenant_schema.is_dropping()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tenant is invalid", KR(ret), K(tenant_id), K(tenant_schema)); + } else { + LOG_INFO("try drop tenant", K(tenant_id)); + ObLSStatusOperator op; + share::ObLSStatusInfoArray ls_array; + if (OB_FAIL(ObShareUtil::set_default_timeout_ctx(ctx, GCONF.rpc_timeout))) { + LOG_WARN("fail to set timeout ctx", KR(ret), K(tenant_id)); + } else if (OB_FAIL(op.get_all_ls_status_by_order(tenant_id, ls_array, *GCTX.sql_proxy_))) { + LOG_WARN("fail to get all ls status", KR(ret), K(tenant_id)); + } else if (ls_array.count() <= 0) { + obrpc::ObDropTenantArg arg; + arg.exec_tenant_id_ = OB_SYS_TENANT_ID; + arg.tenant_name_ = tenant_schema.get_tenant_name(); + arg.tenant_id_ = tenant_schema.get_tenant_id(); + arg.if_exist_ = true; + arg.delay_to_drop_ = false; + ObSqlString sql; + const int64_t timeout_ts = ctx.get_timeout(); + if (OB_FAIL(sql.append_fmt("DROP TENANT IF EXISTS %s FORCE", arg.tenant_name_.ptr()))) { + LOG_WARN("fail to generate sql", KR(ret), K(arg)); + } else if (FALSE_IT(arg.ddl_stmt_str_ = sql.string())) { + } else if (OB_FAIL(GCTX.rs_rpc_proxy_->timeout(timeout_ts).drop_tenant(arg))) { + LOG_WARN("fail to drop tenant", KR(ret), K(arg), K(timeout_ts)); + } + } else { + // tenant's logstream is still dropping, check next round + } + } + return ret; +} + + +}//end of rootserver +} diff --git a/src/rootserver/ob_common_ls_service.h b/src/rootserver/ob_common_ls_service.h new file mode 100644 index 000000000..d0131a856 --- /dev/null +++ b/src/rootserver/ob_common_ls_service.h @@ -0,0 +1,126 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_ROOTSERVER_OB_COMMON_LS_SERVICE_H +#define OCEANBASE_ROOTSERVER_OB_COMMON_LS_SERVICE_H +#include "lib/thread/ob_reentrant_thread.h"//ObRsReentrantThread +#include "logservice/ob_log_base_type.h" +#include "share/scn.h"//SCN +#include "share/ls/ob_ls_status_operator.h"//ObLSStatusOperator +#include "share/ls/ob_ls_operator.h" //ObLSAttr +#include "share/ob_thread_mgr.h" //OBTGDefIDEnum +#include "logservice/palf/palf_iterator.h" //PalfBufferIterator +#include "share/unit/ob_unit_info.h"//ObUnit::Status +#include "lib/thread/thread_mgr_interface.h" // TGRunnable +#include "lib/lock/ob_thread_cond.h"//ObThreadCond +#include "rootserver/ob_tenant_thread_helper.h"//ObTenantThreadHelper + + +namespace oceanbase +{ +namespace obrpc +{ +class ObSrvRpcProxy; +} +namespace common +{ +class ObMySQLProxy; +class ObMySQLTransaction; +} +namespace share +{ +namespace schema +{ +class ObMultiVersionSchemaService; +class ObTenantSchema; +} + +} +namespace palf +{ +struct PalfBaseInfo; +} +namespace rootserver +{ +/*description: + *COMMON_LS_SERVICE thread: Started on the leader of the meta tenant sys ls + * 1. adjust sys ls primary zone of meta and user tenant + * 2. adjust user ls primary zone + * 3. make ls status from creating to created of __all_ls_status + * 4. check dropping tenant need drop tenant force; + * */ +class ObCommonLSService : public ObTenantThreadHelper, + public logservice::ObICheckpointSubHandler, + public logservice::ObIReplaySubHandler +{ +public: + ObCommonLSService():inited_(false), tenant_id_(OB_INVALID_TENANT_ID) {} + virtual ~ObCommonLSService() {} + static int mtl_init(ObCommonLSService *&ka); + int init(); + void destroy(); + virtual void do_work() override; + +public: + virtual share::SCN get_rec_scn() override { return share::SCN::max_scn();} + virtual int flush(share::SCN &scn) override { return OB_SUCCESS; } + int replay(const void *buffer, const int64_t nbytes, const palf::LSN &lsn, const share::SCN &scn) override + { + UNUSED(buffer); + UNUSED(nbytes); + UNUSED(lsn); + UNUSED(scn); + return OB_SUCCESS; + } + +private: + int try_update_sys_ls_primary_zone_(); + int try_adjust_user_ls_primary_zone_(const share::schema::ObTenantSchema &tenant_schema); + int try_update_ls_primary_zone_( + const share::ObLSPrimaryZoneInfo &primary_zone_info, + const common::ObZone &new_primary_zone, + const common::ObSqlString &zone_priority); + static int set_ls_to_primary_zone(const common::ObIArray &primary_zone_array, + const share::ObLSPrimaryZoneInfoArray &primary_zone_infos, + common::ObIArray &ls_primary_zone, + common::ObIArray &count_group_by_zone); + static int balance_ls_primary_zone(const common::ObIArray &primary_zone_array, + common::ObIArray &ls_primary_zone, + common::ObIArray &count_group_by_zone); + int adjust_primary_zone_by_ls_group_(const common::ObIArray &primary_zone_array, + const share::ObLSPrimaryZoneInfoArray &primary_zone_infos, + const share::schema::ObTenantSchema &tenant_schema); + + + // force drop user tenant if tenant is in dropping status + int try_force_drop_tenant_( + const share::schema::ObTenantSchema &tenant_schema); + int try_create_ls_(const share::schema::ObTenantSchema &tenant_schema); + //restore tenant can not create ls before "CREATE_INIT_LS" status + int check_can_create_ls_(const share::schema::ObTenantSchema &tenant_schema); +public: + //restore_service need create init ls too + static int do_create_user_ls(const share::schema::ObTenantSchema &tenant_schema, + const share::ObLSStatusInfo &info, + const SCN &create_scn, + bool create_with_palf, + const palf::PalfBaseInfo &palf_base_info); +private: + bool inited_; + uint64_t tenant_id_; + +}; +} +} + + +#endif /* !OCEANBASE_ROOTSERVER_OB_COMMON_LS_SERVICE_H */ diff --git a/src/rootserver/ob_ddl_service.cpp b/src/rootserver/ob_ddl_service.cpp index 7ed002196..ddd6f901c 100644 --- a/src/rootserver/ob_ddl_service.cpp +++ b/src/rootserver/ob_ddl_service.cpp @@ -72,7 +72,7 @@ #include "rootserver/ddl_task/ob_ddl_retry_task.h" #include "rootserver/freeze/ob_freeze_info_manager.h" #include "rootserver/freeze/ob_major_freeze_helper.h" -#include "rootserver/ob_primary_ls_service.h"//ObTenantLSInfo +#include "rootserver/ob_tenant_thread_helper.h"//get_zone_priority #include "lib/utility/ob_tracepoint.h" #include "observer/ob_server_struct.h" #include "storage/tx/ob_ts_mgr.h" @@ -20702,7 +20702,7 @@ int ObDDLService::create_tenant_sys_ls( } else if (OB_UNLIKELY(0 == primary_zone_list.count())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("primary zone is empty", KR(ret), K(tenant_schema)); - } else if (OB_FAIL(ObTenantLSInfo::get_zone_priority( + } else if (OB_FAIL(ObTenantThreadHelper::get_zone_priority( primary_zone_list.at(0), tenant_schema, zone_priority))) { LOG_WARN("failed to get zone priority", KR(ret), K(primary_zone_list), K(tenant_schema)); } else if (OB_FAIL(ls_creator.create_tenant_sys_ls( @@ -21169,7 +21169,7 @@ int ObDDLService::set_sys_ls_status(const uint64_t tenant_id) if (OB_FAIL(new_ls.init(SYS_LS, ls_group_id, flag, share::OB_LS_NORMAL, share::OB_LS_OP_CREATE_END, create_scn))) { LOG_WARN("failed to init new operation", KR(ret), K(flag), K(create_scn)); - } else if (OB_FAIL(ls_operator.insert_ls(new_ls, ls_group_id, share::NORMAL_SWITCHOVER_STATUS))) { + } else if (OB_FAIL(ls_operator.insert_ls(new_ls, ls_group_id))) { LOG_WARN("failed to insert new ls", KR(ret), K(new_ls), K(ls_group_id)); } } diff --git a/src/rootserver/ob_tenant_recovery_reportor.cpp b/src/rootserver/ob_ls_recovery_reportor.cpp similarity index 86% rename from src/rootserver/ob_tenant_recovery_reportor.cpp rename to src/rootserver/ob_ls_recovery_reportor.cpp index c9933fe32..b9a964e01 100644 --- a/src/rootserver/ob_tenant_recovery_reportor.cpp +++ b/src/rootserver/ob_ls_recovery_reportor.cpp @@ -12,7 +12,7 @@ #define USING_LOG_PREFIX RS -#include "rootserver/ob_tenant_recovery_reportor.h" +#include "rootserver/ob_ls_recovery_reportor.h" #include "rootserver/ob_tenant_info_loader.h" #include "rootserver/ob_tenant_role_transition_service.h"//ObTenantRoleTransitionConstants #include "storage/tx_storage/ob_ls_service.h" //ObLSService @@ -36,11 +36,11 @@ using namespace storage; using namespace palf; namespace rootserver { -int ObTenantRecoveryReportor::mtl_init(ObTenantRecoveryReportor *&ka) +int ObLSRecoveryReportor::mtl_init(ObLSRecoveryReportor *&ka) { return ka->init(); } -int ObTenantRecoveryReportor::init() +int ObLSRecoveryReportor::init() { int ret = OB_SUCCESS; lib::ThreadPool::set_run_wrapper(MTL_CTX()); @@ -66,7 +66,7 @@ int ObTenantRecoveryReportor::init() } return ret; } -void ObTenantRecoveryReportor::destroy() +void ObLSRecoveryReportor::destroy() { LOG_INFO("tenant recovery service destory", KPC(this)); stop(); @@ -76,7 +76,7 @@ void ObTenantRecoveryReportor::destroy() sql_proxy_ = NULL; } -int ObTenantRecoveryReportor::start() +int ObLSRecoveryReportor::start() { int ret = OB_SUCCESS; if (IS_NOT_INIT) { @@ -94,16 +94,16 @@ int ObTenantRecoveryReportor::start() return ret; } -void ObTenantRecoveryReportor::stop() +void ObLSRecoveryReportor::stop() { logical_stop(); } -void ObTenantRecoveryReportor::wait() +void ObLSRecoveryReportor::wait() { logical_wait(); } -void ObTenantRecoveryReportor::wakeup() +void ObLSRecoveryReportor::wakeup() { if (OB_NOT_INIT) { } else { @@ -112,7 +112,7 @@ void ObTenantRecoveryReportor::wakeup() } } -void ObTenantRecoveryReportor::run2() +void ObLSRecoveryReportor::run2() { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; @@ -156,7 +156,7 @@ void ObTenantRecoveryReportor::run2() } } -int ObTenantRecoveryReportor::submit_tenant_refresh_schema_task_() +int ObLSRecoveryReportor::submit_tenant_refresh_schema_task_() { int ret = OB_SUCCESS; ObAllTenantInfo tenant_info; @@ -200,7 +200,7 @@ int ObTenantRecoveryReportor::submit_tenant_refresh_schema_task_() } return ret; } -int ObTenantRecoveryReportor::update_ls_recovery_stat_() +int ObLSRecoveryReportor::update_ls_recovery_stat_() { int ret = OB_SUCCESS; if (IS_NOT_INIT) { @@ -235,10 +235,7 @@ int ObTenantRecoveryReportor::update_ls_recovery_stat_() LOG_WARN("failed to update_ls_replayable_point", KR(tmp_ret), KPC(ls), K(tenant_info)); } - if (ls->is_sys_ls()) { - // nothing todo - // sys ls of user tenant is in ls_recovery - } else if (OB_FAIL(update_ls_recovery(ls, sql_proxy_))) { + if (OB_FAIL(update_ls_recovery(ls, sql_proxy_))) { LOG_WARN("failed to update ls recovery", KR(ret), KPC(ls)); } } @@ -256,7 +253,7 @@ int ObTenantRecoveryReportor::update_ls_recovery_stat_() return ret; } -int ObTenantRecoveryReportor::update_ls_recovery(ObLS *ls, common::ObMySQLProxy *sql_proxy) +int ObLSRecoveryReportor::update_ls_recovery(ObLS *ls, common::ObMySQLProxy *sql_proxy) { int ret = OB_SUCCESS; logservice::ObLogService *ls_svr = MTL(logservice::ObLogService*); @@ -272,8 +269,9 @@ int ObTenantRecoveryReportor::update_ls_recovery(ObLS *ls, common::ObMySQLProxy ObLSRecoveryStat ls_recovery_stat; const ObLSID ls_id = ls->get_ls_id(); const uint64_t tenant_id = MTL_ID(); - + ObMySQLTransaction trans; ObLSRecoveryStatOperator ls_recovery; + const uint64_t exec_tenant_id = ls_recovery.get_exec_tenant_id(tenant_id); if (OB_FAIL(ls_svr->get_palf_role(ls_id, role, first_proposal_id))) { LOG_WARN("failed to get first role", KR(ret), K(ls_id), KPC(ls)); } else if (!is_strong_leader(role)) { @@ -294,10 +292,31 @@ int ObTenantRecoveryReportor::update_ls_recovery(ObLS *ls, common::ObMySQLProxy ret = OB_EAGAIN; LOG_INFO("role change, try again", KR(ret), K(role), K(first_proposal_id), K(second_proposal_id), KPC(ls)); - } else if (OB_FAIL(ls_recovery.update_ls_recovery_stat(ls_recovery_stat, - *sql_proxy))) { + } else if (OB_FAIL(trans.start(sql_proxy, exec_tenant_id))) { + LOG_WARN("failed to start trans", KR(ret), K(exec_tenant_id), K(ls_recovery_stat)); + } else if (OB_FAIL(ls_recovery.update_ls_recovery_stat_in_trans(ls_recovery_stat, + trans))) { LOG_WARN("failed to update ls recovery stat", KR(ret), K(ls_recovery_stat)); + } else { + //double check sync can not fallback + palf::PalfHandleGuard palf_handle_guard; + SCN new_scn; + if (OB_FAIL(ls_svr->open_palf(ls_id, palf_handle_guard))) { + LOG_WARN("failed to open palf", KR(ret), K(ls_id)); + } else if (OB_FAIL(palf_handle_guard.get_end_scn(new_scn))) { + LOG_WARN("failed to get end ts", KR(ret), K(ls_id)); + } else if (new_scn < sync_scn) { + ret = OB_OP_NOT_ALLOW; + LOG_WARN("maybe flashback, can not report sync scn", KR(ret), K(ls_recovery_stat), K(new_scn)); + } + } + if (trans.is_started()) { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) { + LOG_WARN("failed to commit trans", KR(ret), KR(tmp_ret)); + ret = OB_SUCC(ret) ? tmp_ret : ret; + } } } const int64_t PRINT_INTERVAL = 10 * 1000 * 1000L; @@ -315,27 +334,7 @@ int ObTenantRecoveryReportor::update_ls_recovery(ObLS *ls, common::ObMySQLProxy } -int ObTenantRecoveryReportor::get_tenant_readable_scn(SCN &readable_scn) -{ - int ret = OB_SUCCESS; - share::ObAllTenantInfo tenant_info; - rootserver::ObTenantInfoLoader *tenant_info_loader = MTL(rootserver::ObTenantInfoLoader*); - - if (OB_ISNULL(tenant_info_loader)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("mtl pointer is null", KR(ret), KP(tenant_info_loader)); - } else if (OB_FAIL(tenant_info_loader->get_tenant_info(tenant_info))) { - LOG_WARN("get_tenant_info failed", K(ret)); - } else if (OB_UNLIKELY(! tenant_info.is_valid())) { - ret = OB_EAGAIN; - LOG_WARN("tenant info not valid", K(ret), K(tenant_info)); - } else { - readable_scn = tenant_info.get_standby_scn(); - } - return ret; -} - -int ObTenantRecoveryReportor::update_replayable_point_() +int ObLSRecoveryReportor::update_replayable_point_() { int ret = OB_SUCCESS; if (IS_NOT_INIT) { @@ -352,7 +351,7 @@ int ObTenantRecoveryReportor::update_replayable_point_() return ret; } -int ObTenantRecoveryReportor::update_replayable_point_from_tenant_info_() +int ObLSRecoveryReportor::update_replayable_point_from_tenant_info_() { int ret = OB_SUCCESS; logservice::ObLogService *log_service = MTL(logservice::ObLogService*); @@ -373,7 +372,7 @@ int ObTenantRecoveryReportor::update_replayable_point_from_tenant_info_() return ret; } -int ObTenantRecoveryReportor::update_replayable_point_from_meta_() +int ObLSRecoveryReportor::update_replayable_point_from_meta_() { int ret = OB_SUCCESS; SCN replayable_point; @@ -413,12 +412,10 @@ int ObTenantRecoveryReportor::update_replayable_point_from_meta_() return ret; } -int ObTenantRecoveryReportor::get_sync_point_(const share::ObLSID &id, +int ObLSRecoveryReportor::get_sync_point_(const share::ObLSID &id, SCN &sync_scn, SCN &read_scn) { int ret = OB_SUCCESS; - palf::AccessMode access_mode; - int64_t unused_mode_version; palf::PalfHandleGuard palf_handle_guard; if (OB_FAIL(MTL(logservice::ObLogService*)->open_palf(id, palf_handle_guard))) { LOG_WARN("failed to open palf", KR(ret), K(id)); @@ -432,7 +429,7 @@ int ObTenantRecoveryReportor::get_sync_point_(const share::ObLSID &id, } -int ObTenantRecoveryReportor::get_readable_scn(const share::ObLSID &id, SCN &readable_scn) +int ObLSRecoveryReportor::get_readable_scn(const share::ObLSID &id, SCN &readable_scn) { int ret = OB_SUCCESS; storage::ObLSHandle ls_handle; diff --git a/src/rootserver/ob_tenant_recovery_reportor.h b/src/rootserver/ob_ls_recovery_reportor.h similarity index 82% rename from src/rootserver/ob_tenant_recovery_reportor.h rename to src/rootserver/ob_ls_recovery_reportor.h index 31aceb18c..428c81132 100644 --- a/src/rootserver/ob_tenant_recovery_reportor.h +++ b/src/rootserver/ob_ls_recovery_reportor.h @@ -10,8 +10,8 @@ * See the Mulan PubL v2 for more details. */ -#ifndef OCEANBASE_ROOTSERVER_OB_TENANT_RECOVERY_SERVICE_H -#define OCEANBASE_ROOTSERVER_OB_TENANT_RECOVERY_SERVICE_H +#ifndef OCEANBASE_ROOTSERVER_OB_LS_RECOVERY_SERVICE_H +#define OCEANBASE_ROOTSERVER_OB_LS_RECOVERY_SERVICE_H #include "lib/thread/ob_reentrant_thread.h"//ObRsReentrantThread #include "lib/utility/ob_print_utils.h" //TO_STRING_KV @@ -40,15 +40,15 @@ namespace rootserver * standby machine-readable timestamp of the majority, the minimum standby * machine-readable timestamp of all replicas, the synchronization point, etc. * Statistics for the syslog stream are not in this thread.*/ -class ObTenantRecoveryReportor : public share::ObReentrantThread +class ObLSRecoveryReportor : public share::ObReentrantThread { public: - ObTenantRecoveryReportor() + ObLSRecoveryReportor() : is_inited_(false), tenant_id_(common::OB_INVALID_TENANT_ID), sql_proxy_(nullptr) {} - ~ObTenantRecoveryReportor() {} - static int mtl_init(ObTenantRecoveryReportor *&ka); + ~ObLSRecoveryReportor() {} + static int mtl_init(ObLSRecoveryReportor *&ka); int init(); void destroy(); int start(); @@ -62,7 +62,6 @@ public: //description: update ls recovery static int update_ls_recovery(storage::ObLS *ls, common::ObMySQLProxy *sql_proxy); - int get_tenant_readable_scn(share::SCN &readable_scn); static int get_readable_scn(const share::ObLSID &id, share::SCN &read_scn); private: static int get_sync_point_(const share::ObLSID &id, share::SCN &scn, share::SCN &read_scn); @@ -80,11 +79,11 @@ private: int update_replayable_point_from_tenant_info_(); int update_replayable_point_from_meta_(); int submit_tenant_refresh_schema_task_(); - DISALLOW_COPY_AND_ASSIGN(ObTenantRecoveryReportor); + DISALLOW_COPY_AND_ASSIGN(ObLSRecoveryReportor); }; } // namespace rootserver } // namespace oceanbase -#endif /* !OCEANBASE_ROOTSERVER_OB_TENANT_RECOVERY_SERVICE_H */ +#endif /* !OCEANBASE_ROOTSERVER_OB_LS_RECOVERY_SERVICE_H */ diff --git a/src/rootserver/ob_ls_service_helper.cpp b/src/rootserver/ob_ls_service_helper.cpp new file mode 100755 index 000000000..561ff0b8c --- /dev/null +++ b/src/rootserver/ob_ls_service_helper.cpp @@ -0,0 +1,1860 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX RS +#include "ob_ls_service_helper.h" +#include "lib/profile/ob_trace_id.h" +#include "share/ob_errno.h" +#include "share/ob_max_id_fetcher.h" +#include "share/schema/ob_schema_struct.h"//ObTenantInfo +#include "share/schema/ob_schema_service.h"//ObMultiSchemaService +#include "share/ls/ob_ls_creator.h" //ObLSCreator +#include "share/ls/ob_ls_life_manager.h"//ObLSLifeAgentManager +#include "share/ls/ob_ls_table_operator.h"//ObLSTableOpertor +#include "share/ob_primary_zone_util.h"//ObPrimaryZoneUtil +#include "share/ob_unit_table_operator.h"//ObUnitTableOperator +#include "share/ob_zone_table_operation.h" //ObZoneTableOperation +#include "share/ob_share_util.h"//ObShareUtil +#include "share/restore/ob_physical_restore_table_operator.h"//ObTenantRestoreTableOperator +#include "share/ob_standby_upgrade.h"//ObStandbyUpgrade +#include "share/ob_upgrade_utils.h"//ObUpgradeChecker +#include "observer/ob_server_struct.h"//GCTX +#include "rootserver/ob_tenant_info_loader.h"//get_tenant_info +#include "rootserver/ob_tenant_thread_helper.h"//get_zone_priority +#include "storage/tx_storage/ob_ls_map.h" +#include "storage/tx_storage/ob_ls_service.h" +#include "storage/tx_storage/ob_ls_handle.h" //ObLSHandle +#include "logservice/palf/palf_base_info.h"//PalfBaseInfo +#include "logservice/ob_log_service.h"//ObLogService + +namespace oceanbase +{ +using namespace common; +using namespace share; +using namespace transaction; +using namespace palf; +namespace rootserver +{ +/////////ObUnitGroupInfo +bool ObUnitGroupInfo::is_valid() const +{ + return OB_INVALID_ID != unit_group_id_ + && share::ObUnit::UNIT_STATUS_MAX != unit_status_; +} + +int ObUnitGroupInfo::init(const uint64_t unit_group_id, + const share::ObUnit::Status &unit_status) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(OB_INVALID_ID == unit_group_id + || share::ObUnit::UNIT_STATUS_MAX == unit_status)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(unit_group_id), K(unit_status)); + } else { + unit_group_id_ = unit_group_id; + unit_status_ = unit_status; + } + return ret; +} + +void ObUnitGroupInfo::reset() +{ + unit_group_id_ = OB_INVALID_ID; + unit_status_ = share::ObUnit::UNIT_STATUS_MAX; + ls_group_ids_.reset(); +} + +int ObUnitGroupInfo::assign(const ObUnitGroupInfo &other) +{ + int ret = OB_SUCCESS; + if (this != &other) { + if (OB_FAIL(ls_group_ids_.assign(other.ls_group_ids_))) { + LOG_WARN("failed to assign", KR(ret), K(other)); + } else { + unit_group_id_ = other.unit_group_id_; + unit_status_ = other.unit_status_; + } + } + return ret; +} + +int ObUnitGroupInfo::remove_ls_group(const uint64_t ls_group_id) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(ObLSID::MIN_USER_LS_GROUP_ID >= ls_group_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("ls group id is invalid", KR(ret), K(ls_group_id)); + } else { + bool remove = false; + for (int64_t i = 0; OB_SUCC(ret) && i < ls_group_ids_.count(); ++i) { + if (ls_group_ids_.at(i) == ls_group_id) { + remove = true; + if (OB_FAIL(ls_group_ids_.remove(i))) { + LOG_WARN("failed to remove from array", KR(ret), K(i), + K(ls_group_id), "this", *this); + } + break; + } + } + if (OB_SUCC(ret) && !remove) { + ret = OB_ENTRY_NOT_EXIST; + LOG_WARN("failed to find ls group id", KR(ret), + K(ls_group_id), "this", *this); + } + } + return ret; +} + +bool ObUnitGroupInfo::operator==(const ObUnitGroupInfo &other) const +{ + return unit_group_id_ == other.unit_group_id_ + && unit_status_ == other.unit_status_; +} + +////////////ObLSGroupInfo +bool ObLSGroupInfo::is_valid() const +{ + return OB_INVALID_ID != unit_group_id_ + && OB_INVALID_ID != ls_group_id_; +} + +int ObLSGroupInfo::init(const uint64_t unit_group_id, + const uint64_t ls_group_id) +{ + int ret = OB_SUCCESS; + reset(); + if (OB_UNLIKELY(OB_INVALID_ID == unit_group_id + || OB_INVALID_ID == ls_group_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(unit_group_id), K(ls_group_id)); + } else { + unit_group_id_ = unit_group_id; + ls_group_id_ = ls_group_id; + } + return ret; +} + +int ObLSGroupInfo::assign(const ObLSGroupInfo &other) +{ + int ret = OB_SUCCESS; + if (this != &other) { + reset(); + if (OB_FAIL(ls_ids_.assign(other.ls_ids_))) { + LOG_WARN("failed to assign ls ids", KR(ret), K(other)); + } else { + unit_group_id_ = other.unit_group_id_; + ls_group_id_ = other.ls_group_id_; + } + } + return ret; +} + +void ObLSGroupInfo::reset() +{ + ls_group_id_ = OB_INVALID_ID; + unit_group_id_ = OB_INVALID_ID; + ls_ids_.reset(); +} + +int ObLSGroupInfo::remove_ls(const share::ObLSID &ls_id) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!ls_id.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("ls id is invalid", KR(ret), K(ls_id)); + } else { + bool remove = false; + for (int64_t i = 0; OB_SUCC(ret) && i < ls_ids_.count(); ++i) { + if (ls_ids_.at(i) == ls_id) { + remove = true; + if (OB_FAIL(ls_ids_.remove(i))) { + LOG_WARN("failed to remove from array", KR(ret), K(i), + K(ls_id), "this", *this); + } + break; + } + } + if (OB_SUCC(ret) && !remove) { + ret = OB_ENTRY_NOT_EXIST; + LOG_WARN("failed to find ls id", KR(ret), + K(ls_id), "this", *this); + } + } + return ret; + +} + +///////////////ObLSStatusMachineParameter +//no need check status valid or ls_status is valid +int ObLSStatusMachineParameter::init(const share::ObLSID &id, + const share::ObLSStatusInfo &status_info, + const share::ObLSAttr &ls_info) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!id.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("id is invalid", KR(ret), K(id)); + } else if (OB_FAIL(status_info_.assign(status_info))) { + LOG_WARN("failed to assign status info", KR(ret), K(status_info)); + } else if (OB_FAIL(ls_info_.assign(ls_info))) { + LOG_WARN("failed to assign ls info", KR(ret), K(ls_info)); + } else { + ls_info_ = ls_info; + ls_id_ = id; + } + return ret; +} + +void ObLSStatusMachineParameter::reset() +{ + ls_id_.reset(); + ls_info_.reset(); + status_info_.reset(); +} + +/////////////ObTenantLSInfo + +void ObTenantLSInfo::reset() +{ + is_load_ = false; + status_array_.reset(); + unit_group_array_.reset(); + ls_group_array_.reset(); + primary_zone_.reset(); + max_ls_id_ = OB_INVALID_ID; + max_ls_group_id_ = OB_INVALID_ID; +} + + +bool ObTenantLSInfo::is_valid() const +{ + return ATOMIC_LOAD(&is_load_); +} + +int ObTenantLSInfo::revision_to_equal_status(common::ObMySQLTransaction &trans) +{ + int ret = OB_SUCCESS; + ObArray status_machine_array; + if (OB_FAIL(construct_ls_status_machine_(status_machine_array))) { + LOG_WARN("failed to construct ls status machine array", KR(ret)); + } else if (OB_ISNULL(sql_proxy_) || OB_ISNULL(tenant_schema_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy or tenant schema is null", KR(ret), KP(sql_proxy_), KP(tenant_schema_)); + } else { + ObLSLifeAgentManager ls_life_agent(*sql_proxy_); + const uint64_t tenant_id = tenant_schema_->get_tenant_id(); + ARRAY_FOREACH_NORET(status_machine_array, idx) { + const ObLSStatusMachineParameter &machine = status_machine_array.at(idx); + const share::ObLSStatusInfo &status_info = machine.status_info_; + const share::ObLSAttr &ls_info = machine.ls_info_; + if (ls_info.get_ls_status() == status_info.status_) { + //if ls and ls status is equal, need to process + } else if (!ls_info.is_valid()) { + if (status_info.ls_is_create_abort()) { + //ls may create_abort + if (OB_FAIL(ls_life_agent.drop_ls_in_trans(tenant_id, machine.ls_id_, + share::NORMAL_SWITCHOVER_STATUS, trans))) { + LOG_WARN("failed to delete ls", KR(ret), K(machine)); + } + } else { + //ls can not be waitoffline or other status + ret = OB_ERR_UNEXPECTED; + LOG_WARN("status machine not expected", KR(ret), K(machine)); + } + } else if (ls_info.ls_is_creating()) { + if (!status_info.is_valid()) { + if (OB_FAIL(create_new_ls_in_trans(ls_info.get_ls_id(), + ls_info.get_ls_group_id(), + ls_info.get_create_scn(), trans))) { + LOG_WARN("failed to create new ls", KR(ret), K(machine)); + } + } else if (status_info.ls_is_created() + || status_info.ls_is_create_abort()) { + //nothing todo + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("status machine not expected", KR(ret), K(machine)); + } + } else if (ls_info.ls_is_normal()) { + if (status_info.ls_is_created()) { + if (OB_FAIL(status_operator_.update_ls_status( + tenant_id, status_info.ls_id_, status_info.status_, + ls_info.get_ls_status(), share::NORMAL_SWITCHOVER_STATUS, trans))) { + LOG_WARN("failed to update ls status", KR(ret), K(status_info), K(tenant_id), K(ls_info)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("status machine not expected", KR(ret), K(machine)); + } + } else { + //in upgrade, tenant can not be dropping, so no need to take care of + //pre_tenant_dropping, tenant_dropping, waitoffline + ret = OB_ERR_UNEXPECTED; + LOG_WARN("the ls not expected in all_ls", KR(ret), K(machine)); + } + } + } + return ret; +} + + +//Regardless of the tenant being dropped, +//handle the asynchronous operation of status and history table +int ObTenantLSInfo::process_next_ls_status(int64_t &task_cnt) +{ + int ret = OB_SUCCESS; + ObArray status_machine_array; + + if (OB_FAIL(construct_ls_status_machine_(status_machine_array))) { + LOG_WARN("failed to construct ls status machine array", KR(ret)); + } else { + int tmp_ret = OB_SUCCESS; + bool is_steady = false; + ARRAY_FOREACH_NORET(status_machine_array, idx) { + const ObLSStatusMachineParameter &machine = status_machine_array.at(idx); + if (OB_TMP_FAIL(process_next_ls_status_(machine, is_steady))) { + LOG_WARN("failed to do next ls process", KR(ret), KR(tmp_ret), K(machine)); + } + if (OB_TMP_FAIL(tmp_ret) || !is_steady) { + task_cnt++; + } + } + } + + return ret; +} + +int ObTenantLSInfo::construct_ls_status_machine_( + common::ObIArray &status_machine_array) +{ + int ret = OB_SUCCESS; + status_machine_array.reset(); + share::ObLSStatusInfoArray status_info_array; + share::ObLSAttrArray ls_array; + if (OB_ISNULL(sql_proxy_) || OB_ISNULL(tenant_schema_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy or tenant schema is null", KR(ret), KP(tenant_schema_), KP(sql_proxy_)); + } else if (OB_FAIL(ls_operator_.get_all_ls_by_order(ls_array))) { + LOG_WARN("failed to get all ls by order", KR(ret)); + } else { + const uint64_t tenant_id = tenant_schema_->get_tenant_id(); + if (OB_FAIL(status_operator_.get_all_ls_status_by_order( + tenant_id, status_info_array, *sql_proxy_))) { + LOG_WARN("failed to get all ls status by order", KR(ret)); + } + } + if (OB_FAIL(ret)) { + } else if (OB_UNLIKELY(0 == status_info_array.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls array can not be empty", KR(ret), K(ls_array), K(status_info_array)); + } else { + // merge by sort + int64_t status_index = 0; + int64_t ls_index = 0; + ObLSStatusMachineParameter status_machine; + const int64_t status_count = status_info_array.count(); + const int64_t ls_count = ls_array.count(); + share::ObLSStatusInfo status_info; + ObLSID ls_id; + ObLSAttr ls_info; + while ((status_index < status_count || ls_index < ls_count) && OB_SUCC(ret)) { + status_machine.reset(); + if (status_index == status_count) { + //status already end + if (OB_FAIL(ls_info.assign(ls_array.at(ls_index)))) { + LOG_WARN("failed to assign ls info", KR(ret), K(ls_index), K(ls_array)); + } else { + ls_id = ls_array.at(ls_index).get_ls_id(); + status_info.reset(); + ls_index++; + } + } else if (ls_index == ls_count) { + //ls already end + if (OB_FAIL(status_info.assign(status_info_array.at(status_index)))) { + LOG_WARN("failed to assign status info", KR(ret), K(status_index)); + } else { + status_index++; + ls_id = status_info.ls_id_; + ls_info.reset(); + } + } else { + const share::ObLSStatusInfo &tmp_status_info = status_info_array.at(status_index); + const share::ObLSAttr &tmp_ls_info = ls_array.at(ls_index); + if (tmp_status_info.ls_id_ == tmp_ls_info.get_ls_id()) { + status_index++; + ls_index++; + ls_id = tmp_status_info.ls_id_; + if (OB_FAIL(ls_info.assign(tmp_ls_info))) { + LOG_WARN("failed to assign tmp ls info", KR(ret), K(tmp_ls_info)); + } else if (OB_FAIL(status_info.assign(tmp_status_info))) { + LOG_WARN("failed to assign status info", KR(ret), K(tmp_status_info)); + } + } else if (tmp_status_info.ls_id_ > tmp_ls_info.get_ls_id()) { + ls_index++; + ls_id = tmp_ls_info.get_ls_id(); + status_info.reset(); + if (OB_FAIL(ls_info.assign(tmp_ls_info))) { + LOG_WARN("failed to assign tmp ls info", KR(ret), K(tmp_ls_info)); + } + } else { + status_index++; + ls_id = tmp_status_info.ls_id_; + ls_info.reset(); + if (OB_FAIL(status_info.assign(tmp_status_info))) { + LOG_WARN("failed to assign status info", KR(ret), K(tmp_status_info)); + } + } + } + if (FAILEDx(status_machine.init(ls_id, status_info, ls_info))) { + LOG_WARN("failed to init status machine", KR(ret), K(ls_id), K(status_info), K(ls_info)); + } else if (OB_FAIL(status_machine_array.push_back(status_machine))) { + LOG_WARN("failed to push back status machine", KR(ret), K(status_machine)); + } + } // end while + + } + return ret; +} + +int ObTenantLSInfo::drop_tenant() +{ + int ret = OB_SUCCESS; + share::ObLSAttrArray ls_array; + if (OB_FAIL(ls_operator_.get_all_ls_by_order(ls_array))) { + LOG_WARN("failed to get all ls", KR(ret)); + } else { + //1. set sys_ls to tenant_dropping in __all_ls + //2. set user_ls to tenant_dropping or abort in __all_ls + for (int64_t i = 0; OB_SUCC(ret) && i < ls_array.count(); ++i) { + share::ObLSAttr &attr = ls_array.at(i); + if (attr.get_ls_id().is_sys_ls()) { + if (attr.ls_is_normal()) { + if (OB_FAIL(ls_operator_.update_ls_status(attr.get_ls_id(), + attr.get_ls_status(), share::OB_LS_PRE_TENANT_DROPPING))) { + LOG_WARN("failed to update ls status", KR(ret), K(attr)); + } + } + } else if (attr.ls_is_creating()) { + //drop the status + if (OB_FAIL(ls_operator_.delete_ls(attr.get_ls_id(), attr.get_ls_status()))) { + LOG_WARN("failed to remove ls not normal", KR(ret), K(attr)); + } + } else { + //no matter the status is in normal or dropping + //may be the status in status info is created + if (!attr.ls_is_tenant_dropping()) { + if (OB_FAIL(ls_operator_.update_ls_status( + attr.get_ls_id(), attr.get_ls_status(), + share::OB_LS_TENANT_DROPPING))) { + LOG_WARN("failed to update ls status", KR(ret), K(attr)); + } else { + LOG_INFO("[LS_MGR] set ls to tenant dropping", KR(ret), K(attr)); + } + } + } + }//end for + } + return ret; +} +int ObTenantLSInfo::gather_stat(bool for_recovery) +{ + int ret = OB_SUCCESS; + reset(); + if (OB_ISNULL(tenant_schema_) || OB_ISNULL(sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected", KR(ret), KP(tenant_schema_), KP(sql_proxy_)); + } else { + ObUnitTableOperator unit_operator; + common::ObArray units; + const uint64_t tenant_id = tenant_schema_->get_tenant_id(); + if (OB_FAIL(unit_operator.init(*sql_proxy_))) { + LOG_WARN("failed to init unit operator", KR(ret)); + } else if (OB_FAIL(unit_operator.get_units_by_tenant( + tenant_id, units))) { + LOG_WARN("failed to get all tenant unit", KR(ret), K(tenant_id)); + } else { + ObUnitGroupInfo info; + for (int64_t j = 0; OB_SUCC(ret) && j < units.count(); ++j) { + info.reset(); + const ObUnit &unit = units.at(j); + if (OB_FAIL(info.init(unit.unit_group_id_, unit.status_))) { + LOG_WARN("failed to init unit info", KR(ret), K(unit)); + } else if (has_exist_in_array(unit_group_array_, info)) { + //nothing + } else if (OB_FAIL(unit_group_array_.push_back(info))) { + LOG_WARN("fail to push back", KR(ret), K(info)); + } + } + } + if (OB_SUCC(ret)) { + if (for_recovery) { + } else { + //get max ls id and max ls group id + share::ObMaxIdFetcher id_fetcher(*sql_proxy_); + if (OB_FAIL(id_fetcher.fetch_max_id(*sql_proxy_, tenant_id, + share::OB_MAX_USED_LS_GROUP_ID_TYPE, max_ls_group_id_))) { + LOG_WARN("failed to fetch max ls group id", KR(ret), K(tenant_id)); + } else if (OB_FAIL(id_fetcher.fetch_max_id(*sql_proxy_, tenant_id, + share::OB_MAX_USED_LS_ID_TYPE, max_ls_id_))) { + LOG_WARN("failed to fetch max ls id", KR(ret), K(tenant_id)); + } + } + } + if (FAILEDx(gather_all_ls_info_())) { + LOG_WARN("failed to get all ls info", KR(ret), K(tenant_id)); + } else if (OB_FAIL(ObPrimaryZoneUtil::get_tenant_primary_zone_array( + *tenant_schema_, primary_zone_))) { + LOG_WARN("failed to get tenant primary zone array", KR(ret), KPC(tenant_schema_)); + } else { + is_load_ = true; + } + } + LOG_INFO("[LS_MGR] gather stat", KR(ret), K(primary_zone_), K(unit_group_array_), + K(max_ls_id_), K(max_ls_group_id_)); + return ret; +} + +int ObTenantLSInfo::gather_all_ls_info_() +{ + int ret = OB_SUCCESS; + share::ObLSStatusInfoArray status_info_array; + if (OB_ISNULL(sql_proxy_) || OB_ISNULL(tenant_schema_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy or tenant schema is null", KR(ret), KP(tenant_schema_), KP(sql_proxy_)); + } else { + const uint64_t tenant_id = tenant_schema_->get_tenant_id(); + if (OB_FAIL(status_operator_.get_all_ls_status_by_order( + tenant_id, status_info_array, *sql_proxy_))) { + LOG_WARN("failed to get all ls status by order", KR(ret), K(tenant_id)); + } + } + if (OB_FAIL(ret)) { + } else { + const int64_t count = max(1, hash::cal_next_prime(status_info_array.count())); + if (OB_FAIL(status_map_.create(count, "LogStrInfo", "LogStrInfo"))) { + LOG_WARN("failed to create ls map", KR(ret), K(count)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < status_info_array.count(); ++i) { + const share::ObLSStatusInfo &info = status_info_array.at(i); + if (info.ls_is_wait_offline()) { + //ls is already offline, no need to process + } else if (FAILEDx(add_ls_status_info_(info))) { + LOG_WARN("failed to add ls status info", KR(ret), K(i), K(info)); + } + }// end for + } + } + return ret; +} + +int ObTenantLSInfo::add_ls_to_ls_group_(const share::ObLSStatusInfo &info) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!info.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls status is invalid", KR(ret), K(info)); + } else { + int64_t group_index = OB_INVALID_INDEX_INT64; + for (int64_t j = 0; OB_SUCC(ret) && j < ls_group_array_.count(); + ++j) { + const ObLSGroupInfo &group = ls_group_array_.at(j); + if (info.ls_group_id_ == group.ls_group_id_) { + group_index = j; + break; + } + } + if (OB_SUCC(ret) && OB_INVALID_INDEX_INT64 == group_index) { + ObLSGroupInfo group; + group_index = ls_group_array_.count(); + if (OB_FAIL(group.init(info.unit_group_id_, info.ls_group_id_))) { + LOG_WARN("failed to init ls group", KR(ret), K(info)); + } else if (OB_FAIL(ls_group_array_.push_back(group))) { + LOG_WARN("failed to pushback group", KR(ret), K(group)); + } else if (OB_FAIL(add_ls_group_to_unit_group_(group))) { + LOG_WARN("failed to add ls group to unit group", KR(ret), K(group)); + } + } + if (OB_SUCC(ret)) { + if (OB_UNLIKELY(group_index >= ls_group_array_.count() + || OB_INVALID_INDEX_INT64 == group_index)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("group index is invalid", KR(ret), K(group_index), "count", + ls_group_array_.count()); + } else if (OB_FAIL(ls_group_array_.at(group_index) + .ls_ids_.push_back(info.ls_id_))) { + LOG_WARN("failed to push back ls id", KR(ret), K(group_index), + K(info)); + } + } + } + return ret; +} + +int ObTenantLSInfo::add_ls_group_to_unit_group_(const ObLSGroupInfo &group_info) +{ + int ret = OB_SUCCESS; + bool found = false; + if (OB_UNLIKELY(!group_info.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("group_info is invalid", KR(ret), K(group_info)); + } else { + found = false; + for (int64_t j = 0; OB_SUCC(ret) && j < unit_group_array_.count(); ++j) { + ObUnitGroupInfo &unit_info = unit_group_array_.at(j); + if (group_info.unit_group_id_ == unit_info.unit_group_id_) { + found = true; + if (OB_FAIL(unit_info.ls_group_ids_.push_back( + group_info.ls_group_id_))) { + LOG_WARN("failed to push back ls group id", KR(ret), + K(group_info), K(unit_info)); + } + } + } // end j + if (OB_SUCC(ret) && !found) { + //can found unit_group for a ls group + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to find unit group for ls group", KR(ret), + K(group_info), K(unit_group_array_)); + } + } + + return ret; +} + +int ObTenantLSInfo::check_unit_group_valid_( + const uint64_t unit_group_id, bool &is_valid_info) +{ + int ret = OB_SUCCESS; + is_valid_info = true; + if (OB_UNLIKELY(OB_INVALID_VERSION == unit_group_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("ls id is invalid", KR(ret), K(unit_group_id)); + } else if (OB_UNLIKELY(!is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant stat not valid", KR(ret)); + } else { + //check unit group status is valid + ret = OB_ENTRY_NOT_EXIST; + for (int64_t i = 0; OB_FAIL(ret) && i < unit_group_array_.count(); ++i) { + const ObUnitGroupInfo &group_info = unit_group_array_.at(i); + if (unit_group_id == group_info.unit_group_id_) { + ret = OB_SUCCESS; + if (share::ObUnit::UNIT_STATUS_ACTIVE != group_info.unit_status_) { + is_valid_info = false; + } + break; + } + }//end for + if (OB_FAIL(ret)) { + LOG_WARN("failed to found unit group status", KR(ret), K(unit_group_id), + K(unit_group_array_)); + } + } + return ret; +} + +int ObTenantLSInfo::create_new_ls_in_trans( + const share::ObLSID &ls_id, + const uint64_t ls_group_id, + const SCN &create_scn, + ObMySQLTransaction &trans) +{ + int ret = OB_SUCCESS; + const bool is_recovery = true; + int64_t info_index = OB_INVALID_INDEX_INT64; + if (OB_UNLIKELY(!ls_id.is_valid() + || OB_INVALID_ID == ls_group_id + || !create_scn.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("ls id is invalid", KR(ret), K(ls_id), K(ls_group_id), K(create_scn)); + } else if (OB_UNLIKELY(!is_valid()) + || OB_ISNULL(tenant_schema_) + || OB_ISNULL(sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant stat not valid", KR(ret), KP(tenant_schema_), KP(sql_proxy_)); + } else { + const uint64_t tenant_id = tenant_schema_->get_tenant_id(); + share::ObLSStatusInfo new_info; + ObLSGroupInfo group_info; + ObZone primary_zone; + ObLSLifeAgentManager ls_life_agent(*sql_proxy_); + ObSqlString zone_priority; + if (OB_SUCC(get_ls_group_info( + ls_group_id, group_info))) { + // need a new primary zone + } else if (OB_ENTRY_NOT_EXIST == ret) { + // need a new unit group + ret = OB_SUCCESS; + int64_t unit_group_index = OB_INVALID_INDEX_INT64; + if (OB_FAIL(get_next_unit_group_(is_recovery, unit_group_index))) { + LOG_WARN("failed to get next unit group", KR(ret), K(is_recovery)); + } else if (OB_UNLIKELY(OB_INVALID_INDEX_INT64 == unit_group_index + || unit_group_index >= unit_group_array_.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to next unit group", KR(ret), K(unit_group_index)); + } else if (OB_FAIL(group_info.init( + unit_group_array_.at(unit_group_index).unit_group_id_, + ls_group_id))) { + LOG_WARN("failed to init group info", KR(ret), K(unit_group_index), K(ls_group_id)); + } + } else { + LOG_WARN("failed to get ls group info", KR(ret), + K(ls_group_id)); + } + + if (FAILEDx(get_next_primary_zone_(is_recovery, group_info, primary_zone))) { + LOG_WARN("failed to get next primary zone", KR(ret), K(group_info)); + } else if (OB_FAIL(new_info.init(tenant_id, ls_id, + ls_group_id, + share::OB_LS_CREATING, + group_info.unit_group_id_, primary_zone))) { + LOG_WARN("failed to init new info", KR(ret), K(tenant_id), + K(ls_id), K(ls_group_id), K(group_info), K(primary_zone)); + } else if (OB_FAIL(ObTenantThreadHelper::get_zone_priority(primary_zone, *tenant_schema_, zone_priority))) { + LOG_WARN("failed to get normalize primary zone", KR(ret), K(primary_zone), K(zone_priority)); + } else if (OB_FAIL(ls_life_agent.create_new_ls_in_trans(new_info, create_scn, + zone_priority.string(), share::NORMAL_SWITCHOVER_STATUS, trans))) { + LOG_WARN("failed to insert ls info", KR(ret), K(new_info), K(create_scn), K(zone_priority)); + } else if (OB_FAIL(add_ls_status_info_(new_info))) { + LOG_WARN("failed to add new ls info to memory", KR(ret), K(new_info)); + } + } + return ret; +} + +int ObTenantLSInfo::add_ls_status_info_( + const share::ObLSStatusInfo &ls_info) +{ + int ret = OB_SUCCESS; + const int64_t index = status_array_.count(); + if (OB_UNLIKELY(!ls_info.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("ls info is invalid", KR(ret), K(ls_info)); + } else if (OB_FAIL(status_array_.push_back(ls_info))) { + LOG_WARN("failed to remove status", KR(ret), K(ls_info)); + } else if (OB_FAIL(status_map_.set_refactored(ls_info.ls_id_, index))) { + LOG_WARN("failed to remove ls from map", KR(ret), K(ls_info), K(index)); + } else if (ls_info.ls_id_.is_sys_ls()) { + //sys ls no ls group + } else if (OB_FAIL(add_ls_to_ls_group_(ls_info))) { + LOG_WARN("failed to add ls info", KR(ret), K(ls_info)); + } + return ret; +} + +int ObTenantLSInfo::get_ls_group_info( + const uint64_t ls_group_id, ObLSGroupInfo &info) const +{ + int ret = OB_SUCCESS; + info.reset(); + if (OB_UNLIKELY(OB_INVALID_ID == ls_group_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("ls group id is invalid", KR(ret), K(ls_group_id)); + } else if (OB_UNLIKELY(!is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant stat not valid", KR(ret)); + } else { + bool found = false; + for (int64_t i = 0; OB_SUCC(ret) && i < ls_group_array_.count(); ++i) { + const ObLSGroupInfo group_info = ls_group_array_.at(i); + if (ls_group_id == group_info.ls_group_id_) { + found = true; + if (OB_FAIL(info.assign(group_info))) { + LOG_WARN("failed to assign group info", KR(ret), K(group_info)); + } + break; + } + }//end for + if (OB_SUCC(ret) && !found) { + ret = OB_ENTRY_NOT_EXIST; + LOG_WARN("failed to find ls group info", KR(ret), + K(ls_group_id)); + } + } + return ret; +} + +int ObTenantLSInfo::get_ls_status_info( + const share::ObLSID &ls_id, + share::ObLSStatusInfo &info, + int64_t &info_index) const +{ + int ret = OB_SUCCESS; + info.reset(); + info_index = OB_INVALID_INDEX_INT64; + if (OB_UNLIKELY(!ls_id.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("ls id is invalid", KR(ret), K(ls_id)); + } else if (OB_UNLIKELY(!is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant stat not valid", KR(ret)); + } else if (OB_FAIL(status_map_.get_refactored(ls_id, info_index))) { + if (OB_HASH_NOT_EXIST == ret) { + ret = OB_ENTRY_NOT_EXIST; + } + LOG_WARN("failed to find ls index", KR(ret), K(ls_id)); + } else if (OB_UNLIKELY(info_index < 0 || info_index >= status_array_.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("info index not valid", KR(ret), K(info_index), K(ls_id), + "status_count", status_array_.count()); + } else if (OB_FAIL(info.assign(status_array_.at(info_index)))) { + LOG_WARN("failed to assign ls info", KR(ret), K(info_index), + "status", status_array_.at(info_index)); + } + return ret; +} + +int ObTenantLSInfo::get_next_unit_group_(const bool is_recovery, int64_t &group_index) +{ + int ret = OB_SUCCESS; + group_index = OB_INVALID_INDEX_INT64; + if (OB_UNLIKELY(!is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant stat not valid", KR(ret)); + } else { + int64_t ls_count = OB_INVALID_COUNT; + // Find the unit group with the least number of log streams + for (int64_t i = 0; OB_SUCC(ret) && i < unit_group_array_.count(); ++i) { + const ObUnitGroupInfo &info = unit_group_array_.at(i); + const int64_t count = info.ls_group_ids_.count(); + if (share::ObUnit::UNIT_STATUS_ACTIVE == info.unit_status_) { + if (OB_INVALID_COUNT == ls_count || ls_count > count) { + ls_count = count; + group_index = i; + if (0 == count) { + //the first has no ls group unit group + break; + } + } + } + } + if (OB_SUCC(ret)) { + if (OB_INVALID_INDEX_INT64 == group_index + || (!is_recovery && 0 != ls_count)) { + ret = OB_ENTRY_NOT_EXIST; + LOG_WARN("failed to find next unit group", KR(ret), + K(unit_group_array_), K(is_recovery), K(ls_count)); + } + } + } + LOG_INFO("get next primary zone", KR(ret), K(group_index), K(is_recovery)); + return ret; +} + +int ObTenantLSInfo::get_next_primary_zone_( + const bool is_recovery, + const ObLSGroupInfo &group_info, + ObZone &primary_zone) +{ + int ret = OB_SUCCESS; + primary_zone.reset(); + if (OB_UNLIKELY(!group_info.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("group info is invalid", KR(ret), K(group_info)); + } else if (OB_UNLIKELY(!is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant ls info not valid", KR(ret)); + } else { + share::ObLSStatusInfo info; + int64_t ls_count = OB_INVALID_COUNT; + int64_t info_index = OB_INVALID_INDEX_INT64; + for (int64_t i = 0; OB_SUCC(ret) && i < primary_zone_.count(); ++i) { + const ObZone &zone = primary_zone_.at(i); + int64_t primary_zone_count = 0; + for (int64_t j = 0; OB_SUCC(ret) && j < group_info.ls_ids_.count(); ++j) { + const share::ObLSID &id = group_info.ls_ids_.at(j); + if (OB_FAIL(get_ls_status_info(id, info, info_index))) { + LOG_WARN("failed to find ls info", KR(ret), K(id), K(j)); + } else if (zone == info.primary_zone_) { + primary_zone_count++; + } + }//end for j + if (OB_SUCC(ret)) { + if (OB_INVALID_COUNT == ls_count || ls_count > primary_zone_count) { + ls_count = primary_zone_count; + if (OB_FAIL(primary_zone.assign(zone))) { + LOG_WARN("failed to assign zone", KR(ret), K(zone)); + } else if (0 == primary_zone_count) { + //the first zone has no primary zone + break; + } + } + } + }//end for i + if (OB_SUCC(ret)) { + if (primary_zone.is_empty() || (!is_recovery && 0 != ls_count)) { + ret = OB_ENTRY_NOT_EXIST; + LOG_WARN("failed to find next primary zone", KR(ret), K(primary_zone_), + K(group_info), K(is_recovery), K(ls_count)); + } + } + LOG_INFO("get next primary zone", KR(ret), K(group_info), K(primary_zone)); + } + return ret; +} + +//for create new ls +//ls_info:creating, status_info:creating, status_info:created, ls_info:normal, status_info:normal + +//for drop ls, not supported + +//for drop tenant +//ls_info:tenant_dropping, status_info:tenant_dropping, ls_info:empty, status_info:wait_offline +//the sys ls can be tenant_dropping in status_info after all_users ls were deleted in ls_info +int ObTenantLSInfo::process_next_ls_status_(const ObLSStatusMachineParameter &machine, bool &is_steady) +{ + int ret = OB_SUCCESS; + const share::ObLSStatusInfo &status_info = machine.status_info_; + const share::ObLSAttr &ls_info = machine.ls_info_; + const uint64_t tenant_id = status_info.tenant_id_; + is_steady = false; + if (OB_UNLIKELY(!machine.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("machine is invalid", KR(ret), K(machine)); + } else if (!ls_info.is_valid()) { + if (status_info.ls_is_wait_offline()) { + is_steady = true; + } else if (status_info.ls_is_create_abort() + || status_info.ls_is_creating() + || status_info.ls_is_created()) { + //in switchover/failover, need create abort ls + //in drop tenant, __all_ls will be deleted while status is creating + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("status info is invalid", KR(ret), K(machine)); + } + } else if (ls_info.ls_is_creating()) { + if (status_info.ls_is_create_abort()) { + //delete ls, the ls must is creating + if (OB_FAIL(ls_operator_.delete_ls( + machine.ls_id_, share::OB_LS_CREATING))) { + LOG_WARN("failed to process creating info", KR(ret), K(machine)); + } + } else if (status_info.ls_is_created()) { + //set ls to normal + if (OB_FAIL(ls_operator_.update_ls_status( + machine.ls_id_, ls_info.get_ls_status(), share::OB_LS_NORMAL))) { + LOG_WARN("failed to update ls status", KR(ret), K(machine)); + } + } else if (status_info.ls_is_creating()) { + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("status info is invalid", KR(ret), K(machine)); + } + } else if (ls_info.ls_is_normal()) { + if (status_info.ls_is_normal()) { + is_steady = true; + } else if (status_info.ls_is_created()) { + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("status info is invalid", KR(ret), K(machine)); + } + } else if (ls_info.ls_is_dropping()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("drop ls not supported", KR(ret), K(machine)); + } else if (share::ls_is_pre_tenant_dropping_status(ls_info.get_ls_status())) { + if (!machine.ls_id_.is_sys_ls()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("normal ls can not in pre tenant dropping status", KR(ret), K(machine)); + } else if (!status_info.ls_is_pre_tenant_dropping()) { + } else if (OB_FAIL(sys_ls_tenant_drop_(status_info))) { + LOG_WARN("failed to process sys ls", KR(ret), K(status_info)); + } + } else if (ls_info.ls_is_tenant_dropping()) { + if (ls_info.get_ls_id().is_sys_ls()) { + //nothing + } else if (!status_info.ls_is_tenant_dropping()) { + } else if (OB_FAIL(do_tenant_drop_ls_(status_info))) { + LOG_WARN("failed to drop ls", KR(ret), K(machine)); + } + } else { + //other status can not be in __all_ls + //such as created, wait_offline + ret = OB_ERR_UNEXPECTED; + LOG_WARN("the ls not expected in all_ls", KR(ret), K(machine)); + } + return ret; +} + +//need create or delete new ls group +int ObTenantLSInfo::check_ls_group_match_unitnum() +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tenant stat not valid", KR(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < unit_group_array_.count(); ++i) { + const ObUnitGroupInfo &info = unit_group_array_.at(i); + if (share::ObUnit::UNIT_STATUS_ACTIVE != info.unit_status_) { + // drop logstream, not support + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unit status not valid", KR(ret), K(info)); + } else if (0 == info.ls_group_ids_.count()) { + //create ls + if (OB_FAIL(create_new_ls_for_empty_unit_group_(info.unit_group_id_))) { + LOG_WARN("failed to create new ls for empty unit_group", KR(ret), + K(info)); + } + } + } + return ret; +} + +int ObTenantLSInfo::create_new_ls_for_empty_unit_group_(const uint64_t unit_group_id) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(OB_INVALID_INDEX_INT64 == unit_group_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("group id is invalid", KR(ret), K(unit_group_id)); + } else if (OB_UNLIKELY(!is_valid()) + || OB_ISNULL(tenant_schema_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant ls info not valid", KR(ret), KP(tenant_schema_)); + } else if (OB_UNLIKELY(0 == primary_zone_.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("primary zone is invalid", KR(ret)); + } else { + const uint64_t tenant_id = tenant_schema_->get_tenant_id(); + share::ObLSID new_id; + share::ObLSStatusInfo new_info; + uint64_t new_ls_group_id = OB_INVALID_INDEX_INT64; + if (OB_FAIL(fetch_new_ls_group_id(tenant_id, new_ls_group_id))) { + LOG_WARN("failed to fetch new id", KR(ret), K(tenant_id)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < primary_zone_.count(); ++i) { + const ObZone &zone = primary_zone_.at(i); + new_info.reset(); + if (OB_FAIL(fetch_new_ls_id(tenant_id, new_id))) { + LOG_WARN("failed to get new id", KR(ret), K(tenant_id)); + } else if (OB_FAIL(new_info.init(tenant_id, new_id, new_ls_group_id, + share::OB_LS_CREATING, unit_group_id, + zone))) { + LOG_WARN("failed to init new info", KR(ret), K(new_id), + K(new_ls_group_id), K(unit_group_id), K(zone), K(tenant_id)); + } else if (OB_FAIL(create_new_ls_(new_info))) { + LOG_WARN("failed to add ls info", KR(ret), K(new_info)); + } + LOG_INFO("[LS_MGR] create new ls for empty unit group", KR(ret), K(new_info)); + }//end for + } + return ret; +} + +//need create or delete new ls +//TODO for primary cluster, The primary_zone of multiple LSs may be the same, +//the redundant ones need to be deleted, and only one is kept. +int ObTenantLSInfo::check_ls_match_primary_zone() +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_valid()) || OB_ISNULL(tenant_schema_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant ls info not valid", KR(ret), KP(tenant_schema_)); + } else { + const uint64_t tenant_id = tenant_schema_->get_tenant_id(); + ObZone zone; + share::ObLSID new_id; + share::ObLSStatusInfo new_info; + for (int64_t i = 0; OB_SUCC(ret) && i < ls_group_array_.count(); ++i) { + ObLSGroupInfo &group_info = ls_group_array_.at(i); + //check the unit group is active + //if unit group is in deleting, no need check primary zone + bool is_valid = false; + int64_t group_id_count = group_info.ls_ids_.count(); + if (OB_FAIL(check_unit_group_valid_(group_info.unit_group_id_, is_valid))) { + LOG_WARN("failed to check unit group valid", KR(ret), K(group_info)); + } + while (OB_SUCC(ret) && is_valid + && primary_zone_.count() > group_id_count) { + group_id_count++; + if (OB_FAIL(get_next_primary_zone_(false/*is_recovery*/, group_info, zone))) { + LOG_WARN("failed to get next primary zone", K(group_info)); + } else if (OB_FAIL(fetch_new_ls_id(tenant_id, new_id))) { + LOG_WARN("failed to get new id", KR(ret), K(tenant_id)); + } else if (OB_FAIL(new_info.init(tenant_id, new_id, group_info.ls_group_id_, + share::OB_LS_CREATING, + group_info.unit_group_id_, zone))) { + LOG_WARN("failed to init new info", KR(ret), K(new_id), + K(group_info), K(zone), K(tenant_id)); + } else if (OB_FAIL(create_new_ls_(new_info))) { + LOG_WARN("failed to create new ls", KR(ret), K(new_info)); + } + LOG_INFO("[LS_MGR] create new ls of primary zone", KR(ret), K(new_info), + K(group_id_count)); + } + } + } + return ret; +} + +int ObTenantLSInfo::create_new_ls_(const share::ObLSStatusInfo &status_info) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tenant stat not valid", KR(ret)); + } else if (OB_UNLIKELY(!status_info.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("ls info is invalid", KR(ret), K(status_info)); + } else if (OB_ISNULL(sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy is null", KR(ret)); + } else { + share::ObLSAttr ls_info; + share::ObLSFlag flag = share::OB_LS_FLAG_NORMAL;//TODO + SCN create_scn; + if (OB_FAIL(ObLSAttrOperator::get_tenant_gts(status_info.tenant_id_, create_scn))) { + LOG_WARN("failed to get tenant gts", KR(ret), K(status_info)); + } else if (OB_FAIL(ls_info.init(status_info.ls_id_, status_info.ls_group_id_, flag, + share::OB_LS_CREATING, share::OB_LS_OP_CREATE_PRE, create_scn))) { + LOG_WARN("failed to init new operation", KR(ret), K(status_info), K(create_scn)); + } else if (OB_FAIL(ls_operator_.insert_ls(ls_info, max_ls_group_id_))) { + LOG_WARN("failed to insert new operation", KR(ret), K(ls_info), K(max_ls_group_id_)); + } + LOG_INFO("[LS_MGR] create new ls", KR(ret), K(status_info)); + } + return ret; +} + +int ObTenantLSInfo::fetch_new_ls_group_id( + const uint64_t tenant_id, + uint64_t &ls_group_id) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tenant id is invalid", KR(ret), K(tenant_id)); + } else if (OB_ISNULL(sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy is null", KR(ret)); + } else { + ls_group_id = OB_INVALID_ID; + share::ObMaxIdFetcher id_fetcher(*sql_proxy_); + if (OB_FAIL(id_fetcher.fetch_new_max_id( + tenant_id, share::OB_MAX_USED_LS_GROUP_ID_TYPE, ls_group_id))) { + LOG_WARN("fetch_max_id failed", KR(ret), K(tenant_id), "id_type", + share::OB_MAX_USED_LS_GROUP_ID_TYPE); + } else { + if (ls_group_id != ++max_ls_group_id_) { + ret = OB_EAGAIN; + LOG_WARN("ls may create concurrency, need retry", KR(ret), K(ls_group_id), + K(max_ls_group_id_)); + } + } + } + return ret; +} + +int ObTenantLSInfo::fetch_new_ls_id(const uint64_t tenant_id, share::ObLSID &id) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tenant id is invalid", KR(ret), K(tenant_id)); + } else if (OB_ISNULL(sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy is null", KR(ret)); + } else { + share::ObMaxIdFetcher id_fetcher(*sql_proxy_); + uint64_t id_value = OB_INVALID_ID; + if (OB_FAIL(id_fetcher.fetch_new_max_id( + tenant_id, share::OB_MAX_USED_LS_ID_TYPE, id_value))) { + LOG_WARN("fetch_max_id failed", KR(ret), K(tenant_id), "id_type", + share::OB_MAX_USED_LS_ID_TYPE); + } else { + if (id_value != ++max_ls_id_) { + ret = OB_EAGAIN; + LOG_WARN("ls may create concurrency, need retry", KR(ret), K(id_value), + K(max_ls_id_)); + } else { + share::ObLSID new_id(id_value); + id = new_id; + } + } + } + return ret; +} + +int ObTenantLSInfo::do_tenant_drop_ls_(const share::ObLSStatusInfo &status_info) +{ + int ret = OB_SUCCESS; + LOG_INFO("[LS_MGR] start to tenant drop ls", K(status_info)); + const int64_t start_time = ObTimeUtility::fast_current_time(); + bool can_offline = false; + if (OB_UNLIKELY(!status_info.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("info not valid", KR(ret), K(status_info)); + } else if (OB_ISNULL(sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy is null", KR(ret)); + } else if (!status_info.ls_is_tenant_dropping()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("status info not expected", KR(ret), K(status_info)); + } + if (FAILEDx(check_ls_can_offline_by_rpc(status_info, can_offline))) { + // send rpc to observer + LOG_WARN("failed to check ls can offline", KR(ret), K(status_info)); + } else if (can_offline) { + //sys ls already block, can not delete ls + const uint64_t tenant_id = status_info.tenant_id_; + if (OB_FAIL(ls_operator_.delete_ls(status_info.ls_id_, share::OB_LS_TENANT_DROPPING))) { + LOG_WARN("failed to delete ls", KR(ret), K(status_info)); + } + } + const int64_t cost = ObTimeUtility::fast_current_time() - start_time; + LOG_INFO("[LS_MGR] end to tenant drop ls", KR(ret), K(status_info), K(can_offline), K(cost)); + return ret; +} + +int ObTenantLSInfo::sys_ls_tenant_drop_(const share::ObLSStatusInfo &info) +{ + int ret = OB_SUCCESS; + const ObLSStatus target_status = share::OB_LS_TENANT_DROPPING; + const ObLSStatus pre_status = share::OB_LS_PRE_TENANT_DROPPING; + if (OB_UNLIKELY(!info.is_valid() + || !info.ls_id_.is_sys_ls())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(info)); + } else if (OB_ISNULL(sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy is null", KR(ret), KP(sql_proxy_)); + } else if (pre_status != info.status_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sys ls can not in other status", KR(ret), K(info)); + } + bool can_offline = false; + if (FAILEDx(check_sys_ls_can_offline_(can_offline))) { + LOG_WARN("failed to check sys ls can offline", KR(ret)); + } else if (can_offline) { + if (OB_FAIL(ls_operator_.update_ls_status(info.ls_id_, pre_status, target_status))) { + LOG_WARN("failed to update ls status", KR(ret), K(info), K(pre_status), K(target_status)); + } + } + return ret; +} + +int ObTenantLSInfo::check_sys_ls_can_offline_(bool &can_offline) +{ + int ret = OB_SUCCESS; + share::ObLSStatusInfoArray status_info_array; + can_offline = true; + if (OB_ISNULL(sql_proxy_) || OB_ISNULL(tenant_schema_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy or tenant schema is null", KR(ret), KP(tenant_schema_), KP(sql_proxy_)); + } else { + const uint64_t tenant_id = tenant_schema_->get_tenant_id(); + if (OB_FAIL(status_operator_.get_all_ls_status_by_order( + tenant_id, status_info_array, *sql_proxy_))) { + LOG_WARN("failed to get all ls status", KR(ret), K(tenant_id)); + } else if (0 == status_info_array.count()) { + //sys ls not exist + can_offline = true; + } + for (int64_t i = 0; OB_SUCC(ret) && i < status_info_array.count() && can_offline; ++i) { + const share::ObLSStatusInfo &status_info = status_info_array.at(i); + if (status_info.ls_id_.is_sys_ls()) { + } else { + can_offline = false; + LOG_INFO("[LS_MGR] sys ls can not offline", K(status_info)); + } + } + if (OB_SUCC(ret) && can_offline) { + LOG_INFO("[LS_MGR] sys ls can offline", K(status_info_array)); + } + } + return ret; +} + +int ObTenantLSInfo::check_ls_can_offline_by_rpc(const share::ObLSStatusInfo &info, bool &can_offline) +{ + int ret = OB_SUCCESS; + share::ObLSInfo ls_info; + const share::ObLSReplica *replica = NULL; + if (OB_UNLIKELY(!info.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("info not valid", KR(ret), K(info)); + } else if (OB_ISNULL(lst_operator_) || OB_ISNULL(rpc_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("lst operator or proxy is null", KR(ret), KP(lst_operator_), + KP(rpc_proxy_)); + } else if (OB_FAIL(lst_operator_->get(GCONF.cluster_id, info.tenant_id_, + info.ls_id_, share::ObLSTable::DEFAULT_MODE, ls_info))) { + LOG_WARN("failed to get ls info", KR(ret), K(info)); + } else if (OB_FAIL(ls_info.find_leader(replica))) { + LOG_WARN("failed to find leader", KR(ret), K(ls_info)); + } else if (OB_ISNULL(replica)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("replica is null", KR(ret), K(ls_info)); + } else { + const int64_t timeout = GCONF.rpc_timeout; + obrpc::ObCheckLSCanOfflineArg arg; + if (OB_FAIL(arg.init(info.tenant_id_, info.ls_id_, info.status_))) { + LOG_WARN("failed to init arg", KR(ret), K(arg)); + } else if (OB_FAIL(rpc_proxy_->to(replica->get_server()) + .by(info.tenant_id_) + .timeout(timeout) + .check_ls_can_offline(arg))) { + can_offline = false; + LOG_WARN("failed to check ls can offline", KR(ret), K(arg), K(info), + K(timeout), K(replica)); + } else { + can_offline = true; + } + } + return ret; +} + +////////////ObRecoveryLSHelper +int ObRecoveryLSHelper::do_work(palf::PalfBufferIterator &iterator, + share::SCN &start_scn) +{ + int ret = OB_SUCCESS; + ObTenantInfoLoader *tenant_loader = MTL(ObTenantInfoLoader*); + ObAllTenantInfo tenant_info; + // two thread for seed log and recovery_ls_manager + if (!is_user_tenant(tenant_id_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls recovery thread must run on user tenant", KR(ret), + K(tenant_id_)); + } else if (OB_ISNULL(tenant_loader)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant report is null", KR(ret), K(tenant_id_)); + } else if (OB_FAIL(tenant_loader->get_tenant_info(tenant_info))) { + LOG_WARN("failed to get tenant info", KR(ret)); + } else if (OB_FAIL(ObTenantThreadHelper::check_can_do_recovery(tenant_info))) { + LOG_WARN("failed to check do recovery", KR(ret), K(tenant_info)); + } else if (!start_scn.is_valid()) { + ObAllTenantInfo new_tenant_info; + if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(tenant_id_, proxy_, + false, new_tenant_info))) { + LOG_WARN("failed to load tenant info", KR(ret), K(tenant_id_)); + } else if (new_tenant_info.get_recovery_until_scn() == new_tenant_info.get_sys_recovery_scn()) { + ret = OB_EAGAIN; + if (REACH_TIME_INTERVAL(60 * 1000 * 1000)) { // every minute + LOG_INFO("has recovered to recovery_until_scn", KR(ret), + K(new_tenant_info)); + } + } else if (OB_FAIL( + seek_log_iterator_(new_tenant_info.get_sys_recovery_scn(), iterator))) { + LOG_WARN("failed to seek log iterator", KR(ret), K(new_tenant_info)); + } else { + start_scn = new_tenant_info.get_sys_recovery_scn(); + LOG_INFO("start to seek at", K(start_scn)); + } + } + if (OB_FAIL(ret)) { + } else if (start_scn.is_valid() && + OB_FAIL(process_ls_log_(tenant_info, start_scn, iterator))) { + if (OB_ITER_STOP != ret) { + LOG_WARN("failed to process ls log", KR(ret), K(start_scn), + K(tenant_info)); + } + } + if (OB_FAIL(ret)) { + start_scn.reset(); + } + return ret; +} + +int ObRecoveryLSHelper::seek_log_iterator_(const SCN &sys_recovery_scn, PalfBufferIterator &iterator) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!sys_recovery_scn.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("sync scn is invalid", KR(ret), K(sys_recovery_scn)); + } else { + ObLSService *ls_svr = MTL(ObLSService *); + ObLSHandle ls_handle; + if (OB_FAIL(ls_svr->get_ls(SYS_LS, ls_handle, storage::ObLSGetMod::RS_MOD))) { + LOG_WARN("failed to get ls", KR(ret)); + } else { + logservice::ObLogHandler *log_handler = NULL; + ObLS *ls = NULL; + palf::LSN start_lsn; + if (OB_ISNULL(ls = ls_handle.get_ls()) + || OB_ISNULL(log_handler = ls->get_log_handler())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls or log handle is null", KR(ret), KP(ls), KP(log_handler)); + } else if (sys_recovery_scn.is_base_scn()) { + //The recovery_scn on the primary tenant is initialized to base_scn, + //but too small scn may cause locate to fail, + //so when base_scn is encountered, iterates the log directly from the initialized LSN. + start_lsn = palf::LSN(palf::PALF_INITIAL_LSN_VAL); + } else if (OB_FAIL(log_handler->locate_by_scn_coarsely(sys_recovery_scn, start_lsn))) { + LOG_WARN("failed to locate lsn", KR(ret), K(sys_recovery_scn)); + } + + if (FAILEDx(log_handler->seek(start_lsn, iterator))) { + LOG_WARN("failed to seek iterator", KR(ret), K(sys_recovery_scn)); + } + } + } + return ret; +} + +int ObRecoveryLSHelper::process_ls_log_( + const ObAllTenantInfo &tenant_info, + SCN &start_scn, + PalfBufferIterator &iterator) +{ + int ret = OB_SUCCESS; + palf::LogEntry log_entry; + palf::LSN target_lsn; + SCN sys_recovery_scn; + SCN last_sys_recovery_scn; + int tmp_ret = OB_SUCCESS; + if (OB_UNLIKELY(!tenant_info.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tenant_info is invalid", KR(ret), K(tenant_info)); + } + while (OB_SUCC(ret) && OB_SUCC(iterator.next())) { + if (OB_FAIL(iterator.get_entry(log_entry, target_lsn))) { + LOG_WARN("failed to get log", KR(ret), K(log_entry)); + } else if (OB_ISNULL(log_entry.get_data_buf())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("log entry is null", KR(ret)); + } else { + sys_recovery_scn = log_entry.get_scn(); + LOG_TRACE("get log", K(log_entry), K(target_lsn), K(start_scn), K(sys_recovery_scn)); + const char *log_buf = log_entry.get_data_buf(); + const int64_t log_length = log_entry.get_data_len(); + logservice::ObLogBaseHeader header; + const int64_t HEADER_SIZE = header.get_serialize_size(); + int64_t log_pos = 0; + if (OB_UNLIKELY(sys_recovery_scn <= start_scn)) { + //通过scn定位的LSN是不准确的，可能会获取多余的数据，所以需要把小于等于sys_recovery_scn的过滤掉 + continue; + } else if (tenant_info.get_recovery_until_scn() < sys_recovery_scn) { + if (OB_FAIL(report_tenant_sys_recovery_scn_(tenant_info.get_recovery_until_scn(), false, proxy_))) { + LOG_WARN("failed to report_tenant_sys_recovery_scn_", KR(ret), K(sys_recovery_scn), K(tenant_info), + K(log_entry), K(target_lsn), K(start_scn)); + // SYS LS has recovered to the recovery_until_scn, need stop iterate SYS LS log and reset start_scn + } else if (OB_FAIL(seek_log_iterator_(tenant_info.get_recovery_until_scn(), iterator))) { + LOG_WARN("failed to seek log iterator", KR(ret), K(sys_recovery_scn), K(tenant_info), + K(log_entry), K(target_lsn), K(start_scn)); + } else { + ret = OB_ITER_STOP; + LOG_WARN("SYS LS has recovered to the recovery_until_scn, need stop iterate SYS LS log", + KR(ret), K(sys_recovery_scn), K(tenant_info), K(log_entry), K(target_lsn), K(start_scn)); + start_scn.reset(); + } + } else if (OB_FAIL(header.deserialize(log_buf, HEADER_SIZE, log_pos))) { + LOG_WARN("failed to deserialize", KR(ret), K(HEADER_SIZE)); + } else if (OB_UNLIKELY(log_pos >= log_length)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("log pos is not expected", KR(ret), K(log_pos), K(log_length)); + // nothing + } else if (logservice::TRANS_SERVICE_LOG_BASE_TYPE == header.get_log_type()) { + ObTxLogBlock tx_log_block; + ObTxLogBlockHeader tx_block_header; + if (OB_FAIL(tx_log_block.init(log_buf, log_length, log_pos, tx_block_header))) { + LOG_WARN("failed to init tx log block", KR(ret), K(log_length)); + } else if (OB_FAIL(process_ls_tx_log_(tx_log_block, sys_recovery_scn))) { + LOG_WARN("failed to process ls tx log", KR(ret), K(tx_log_block), K(sys_recovery_scn)); + } + } else {} + if (OB_SUCC(ret)) { + last_sys_recovery_scn = sys_recovery_scn; + } + if ((OB_FAIL(ret) && last_sys_recovery_scn.is_valid()) + || (OB_SUCC(ret) && REACH_TIME_INTERVAL(100 * 1000))) { + //if ls_operator can not process, need to report last sync scn + //Forced reporting every second + if (OB_TMP_FAIL(report_tenant_sys_recovery_scn_(last_sys_recovery_scn, false, proxy_))) { + ret = OB_SUCC(ret) ? tmp_ret : ret; + LOG_WARN("failed to report ls recovery stat", KR(ret), KR(tmp_ret), K(last_sys_recovery_scn)); + } + } + } + if (REACH_TIME_INTERVAL(10 * 1000 * 1000)) { + if (OB_TMP_FAIL(try_to_process_sys_ls_offline_())) { + LOG_WARN("failed to process sys ls offline", KR(ret), KR(tmp_ret)); + } + } + }//end for each log + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + if (!sys_recovery_scn.is_valid()) { + } else if (OB_FAIL(report_tenant_sys_recovery_scn_(sys_recovery_scn, false, proxy_))) { + LOG_WARN("failed to report ls recovery stat", KR(ret), K(sys_recovery_scn)); + } + } else if (OB_SUCC(ret)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("iterator must be end", KR(ret)); + } else { + LOG_WARN("failed to get next log", KR(ret)); + } + + return ret; +} + +int ObRecoveryLSHelper::process_ls_tx_log_(ObTxLogBlock &tx_log_block, const SCN &sys_recovery_scn) +{ + int ret = OB_SUCCESS; + bool has_operation = false; + while (OB_SUCC(ret)) { + transaction::ObTxLogHeader tx_header; + if (OB_FAIL(tx_log_block.get_next_log(tx_header))) { + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + break; + } else { + LOG_WARN("failed to get next log", KR(ret)); + } + } else if (transaction::ObTxLogType::TX_COMMIT_LOG != + tx_header.get_tx_log_type()) { + // nothing + } else { + ObTxCommitLogTempRef temp_ref; + ObTxCommitLog commit_log(temp_ref); + const int64_t COMMIT_SIZE = commit_log.get_serialize_size(); + //TODO commit log may too large + if (OB_FAIL(tx_log_block.deserialize_log_body(commit_log))) { + LOG_WARN("failed to deserialize", KR(ret)); + } else { + const ObTxBufferNodeArray &source_data = + commit_log.get_multi_source_data(); + for (int64_t i = 0; OB_SUCC(ret) && i < source_data.count(); ++i) { + const ObTxBufferNode &node = source_data.at(i); + if (ObTxDataSourceType::STANDBY_UPGRADE == node.get_data_source_type()) { + if (OB_FAIL(process_upgrade_log_(node))) { + LOG_WARN("failed to process_upgrade_log_", KR(ret), K(node)); + } + } else if (ObTxDataSourceType::LS_TABLE != node.get_data_source_type()) { + // nothing + } else if (has_operation) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("one clog has more than one operation", KR(ret), K(commit_log)); + } else { + has_operation = true; + ObLSAttr ls_attr; + const int64_t LS_SIZE = ls_attr.get_serialize_size(); + int64_t pos = 0; + if (OB_FAIL(ls_attr.deserialize(node.get_data_buf().ptr(), LS_SIZE, + pos))) { + LOG_WARN("failed to deserialize", KR(ret), K(node), K(LS_SIZE)); + } else if (OB_UNLIKELY(pos > LS_SIZE)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get ls attr", KR(ret), K(pos), K(LS_SIZE)); + } else { + LOG_INFO("get ls operation", K(ls_attr), K(sys_recovery_scn)); + //TODO ls recovery is too fast for ls manager, so it maybe failed, while change ls status + //consider how to retry + if (OB_FAIL(process_ls_operator_(ls_attr, + sys_recovery_scn))) { + LOG_WARN("failed to process ls operator", KR(ret), K(ls_attr), + K(sys_recovery_scn)); + } + } + } + }// end for + } + } + } // end while for each tx_log + + return ret; +} + +int ObRecoveryLSHelper::process_upgrade_log_(const ObTxBufferNode &node) +{ + int ret = OB_SUCCESS; + uint64_t standby_data_version = 0; + + if (!node.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("node is invalid", KR(ret), K(node)); + } else { + ObStandbyUpgrade primary_data_version; + int64_t pos = 0; + if (OB_FAIL(primary_data_version.deserialize(node.get_data_buf().ptr(), node.get_data_buf().length(), pos))) { + LOG_WARN("failed to deserialize", KR(ret), K(node), KPHEX(node.get_data_buf().ptr(), node.get_data_buf().length())); + } else if (OB_UNLIKELY(pos > node.get_data_buf().length())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get primary_data_version", KR(ret), K(pos), K(node.get_data_buf().length())); + } else { + LOG_INFO("get primary_data_version", K(primary_data_version)); + if (!primary_data_version.is_valid()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("primary_data_version not valid", KR(ret), K(primary_data_version)); + } else if (OB_FAIL(ObUpgradeChecker::get_data_version_by_cluster_version( + GET_MIN_CLUSTER_VERSION(), + standby_data_version))) { + LOG_WARN("failed to get_data_version_by_cluster_version", KR(ret), K(GET_MIN_CLUSTER_VERSION())); + } else if (primary_data_version.get_data_version() > standby_data_version) { + ret = OB_EAGAIN; + if (REACH_TIME_INTERVAL(30 * 60 * 1000 * 1000)) { // 30min + LOG_ERROR("standby version is not new enough to recover primary clog", KR(ret), + K(primary_data_version), K(standby_data_version)); + } + } + } + } + return ret; +} + +int ObRecoveryLSHelper::process_ls_operator_(const share::ObLSAttr &ls_attr, const SCN &sys_recovery_scn) +{ + int ret = OB_SUCCESS; + common::ObMySQLTransaction trans; + const uint64_t meta_tenant_id = gen_meta_tenant_id(tenant_id_); + if (OB_UNLIKELY(!ls_attr.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("ls attr is invalid", KR(ret), K(ls_attr)); + } else if (OB_ISNULL(proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("proxy is null", KR(ret)); + } else if (OB_FAIL(check_valid_to_operator_ls_(ls_attr, sys_recovery_scn))) { + LOG_WARN("failed to check valid to operator ls", KR(ret), K(sys_recovery_scn), K(ls_attr)); + } else if (OB_FAIL(trans.start(proxy_, meta_tenant_id))) { + LOG_WARN("failed to start trans", KR(ret), K(meta_tenant_id)); + } else if (OB_FAIL(process_ls_operator_in_trans_(ls_attr, sys_recovery_scn, trans))) { + LOG_WARN("failed to process ls operator in trans", KR(ret), K(ls_attr), K(sys_recovery_scn)); + } else if (OB_FAIL(report_tenant_sys_recovery_scn_trans_(sys_recovery_scn, true, trans))) { + LOG_WARN("failed to report tenant sys recovery scn", KR(ret), K(sys_recovery_scn)); + } + + if (trans.is_started()) { + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) { + ret = OB_SUCC(ret) ? tmp_ret : ret; + LOG_WARN("failed to end trans", KR(ret), K(tmp_ret)); + } + } + return ret; +} + +int ObRecoveryLSHelper::check_valid_to_operator_ls_( + const share::ObLSAttr &ls_attr, const SCN &sys_recovery_scn) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("proxy is null", KR(ret)); + } else if (OB_UNLIKELY(!sys_recovery_scn.is_valid() || !ls_attr.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("syns scn is invalid", KR(ret), K(sys_recovery_scn), K(ls_attr)); + } else if (share::is_ls_tenant_drop_pre_op(ls_attr.get_ls_operation_type())) { + ObTenantSchema tenant_schema; + if (OB_FAIL(ObTenantThreadHelper::get_tenant_schema(tenant_id_, tenant_schema))) { + LOG_WARN("failed to get tenant schema", KR(ret), K(tenant_id_)); + } else if (!tenant_schema.is_dropping()) { + ret = OB_ITER_STOP; + LOG_WARN("can not process ls operator before tenant dropping", K(ls_attr), K(tenant_schema)); + } + } + if (OB_FAIL(ret)) { + } else { + SCN min_sys_recovery_scn; + SCN min_wrs; + ObLSRecoveryStatOperator ls_recovery; + if (OB_FAIL( + ls_recovery.get_tenant_recovery_stat(tenant_id_, *proxy_, min_sys_recovery_scn, min_wrs))) { + LOG_WARN("failed to get user scn", KR(ret), K(tenant_id_)); + } else if (min_sys_recovery_scn >= sys_recovery_scn) { + // other ls has larger sync scn, ls can operator + } else { + ret = OB_NEED_WAIT; + LOG_WARN("can not process ls operator, need wait other ls sync", KR(ret), + K(min_sys_recovery_scn), K(sys_recovery_scn)); + } + } + return ret; +} + +int ObRecoveryLSHelper::process_ls_operator_in_trans_( + const share::ObLSAttr &ls_attr, const SCN &sys_recovery_scn, ObMySQLTransaction &trans) +{ + int ret = OB_SUCCESS; + const uint64_t meta_tenant_id = gen_meta_tenant_id(tenant_id_); + if (OB_UNLIKELY(!ls_attr.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("ls attr is invalid", KR(ret), K(ls_attr)); + } else if (OB_ISNULL(proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("proxy is null", KR(ret)); + } else { + ObLSStatusOperator ls_operator; + share::ObLSStatusInfo ls_status; + ObLSLifeAgentManager ls_life_agent(*proxy_); + if (OB_UNLIKELY(!ls_attr.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("ls attr is invalid", KR(ret), K(ls_attr)); + } else if (share::is_ls_create_pre_op(ls_attr.get_ls_operation_type())) { + //create new ls; + if (OB_FAIL(create_new_ls_(ls_attr, sys_recovery_scn, trans))) { + LOG_WARN("failed to create new ls", KR(ret), K(sys_recovery_scn), K(ls_attr)); + } + } else if (share::is_ls_create_abort_op(ls_attr.get_ls_operation_type())) { + if (OB_FAIL(ls_life_agent.drop_ls_in_trans(tenant_id_, ls_attr.get_ls_id(), share::NORMAL_SWITCHOVER_STATUS, trans))) { + LOG_WARN("failed to drop ls", KR(ret), K(tenant_id_), K(ls_attr)); + } + } else if (share::is_ls_drop_end_op(ls_attr.get_ls_operation_type())) { + if (OB_FAIL(ls_life_agent.set_ls_offline_in_trans(tenant_id_, ls_attr.get_ls_id(), + ls_attr.get_ls_status(), sys_recovery_scn, share::NORMAL_SWITCHOVER_STATUS, trans))) { + LOG_WARN("failed to set offline", KR(ret), K(tenant_id_), K(ls_attr), K(sys_recovery_scn)); + } + } else { + ObLSStatus target_status = share::OB_LS_EMPTY; + if (OB_FAIL(ls_operator.get_ls_status_info(tenant_id_, ls_attr.get_ls_id(), + ls_status, trans))) { + LOG_WARN("failed to get ls status", KR(ret), K(tenant_id_), K(ls_attr)); + } else if (ls_status.ls_is_creating()) { + ret = OB_EAGAIN; + LOG_WARN("ls not created, need wait", KR(ret), K(ls_status)); + } else if (share::is_ls_create_end_op(ls_attr.get_ls_operation_type())) { + // set ls to normal + target_status = share::OB_LS_NORMAL; + } else if (share::is_ls_tenant_drop_op(ls_attr.get_ls_operation_type())) { + target_status = share::OB_LS_TENANT_DROPPING; + } else if (share::is_ls_drop_pre_op(ls_attr.get_ls_operation_type())) { + target_status = share::OB_LS_DROPPING; + } else if (share::is_ls_tenant_drop_pre_op(ls_attr.get_ls_operation_type())) { + target_status = share::OB_LS_PRE_TENANT_DROPPING; + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected operation type", KR(ret), K(ls_attr)); + } + if (FAILEDx(ls_operator.update_ls_status(tenant_id_, ls_attr.get_ls_id(), + ls_status.status_, target_status, + share::NORMAL_SWITCHOVER_STATUS, trans))) { + LOG_WARN("failed to update ls status", KR(ret), K(tenant_id_), K(ls_attr), + K(ls_status), K(target_status)); + } + LOG_INFO("[LS_RECOVERY] update ls status", KR(ret), K(ls_attr), K(target_status)); + } + } + + return ret; +} +int ObRecoveryLSHelper::create_new_ls_(const share::ObLSAttr &ls_attr, + const SCN &sys_recovery_scn, + common::ObMySQLTransaction &trans) +{ + int ret = OB_SUCCESS; + if (!share::is_ls_create_pre_op(ls_attr.get_ls_operation_type())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("ls not create pre operation", KR(ret), K(ls_attr)); + } else { + //create new ls; + share::schema::ObTenantSchema tenant_schema; + if (OB_FAIL(ObTenantThreadHelper::get_tenant_schema(tenant_id_, tenant_schema))) { + LOG_WARN("failed to get tenant schema", KR(ret), K(tenant_id_)); + } else { + ObTenantLSInfo tenant_stat(GCTX.sql_proxy_, &tenant_schema, tenant_id_, + GCTX.srv_rpc_proxy_, GCTX.lst_operator_); + if (OB_FAIL(tenant_stat.gather_stat(true))) { + LOG_WARN("failed to gather stat", KR(ret)); + } else if (OB_FAIL(tenant_stat.create_new_ls_in_trans(ls_attr.get_ls_id(), + ls_attr.get_ls_group_id(), ls_attr.get_create_scn(), trans))) { + LOG_WARN("failed to add new ls status info", KR(ret), K(ls_attr), K(sys_recovery_scn)); + } + } + LOG_INFO("[LS_RECOVERY] create new ls", KR(ret), K(ls_attr)); + } + return ret; +} + +int ObRecoveryLSHelper::report_tenant_sys_recovery_scn_(const SCN &sys_recovery_scn, + const bool update_sys_recovery_scn, ObISQLClient *proxy) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(proxy)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql can't null", K(ret), KP(proxy)); + } else if (SCN::base_scn() != sys_recovery_scn) { + const uint64_t exec_tenant_id = gen_meta_tenant_id(tenant_id_); + START_TRANSACTION(proxy, exec_tenant_id) + if (FAILEDx(report_tenant_sys_recovery_scn_trans_(sys_recovery_scn, update_sys_recovery_scn, trans))) { + LOG_WARN("failed to report tenant sys recovery scn in trans", KR(ret), K(sys_recovery_scn)); + } + END_TRANSACTION(trans) + } + return ret; +} + +int ObRecoveryLSHelper::report_tenant_sys_recovery_scn_trans_(const share::SCN &sys_recovery_scn, + const bool update_sys_recovery_scn, common::ObMySQLTransaction &trans) +{ + int ret = OB_SUCCESS; + ObTenantInfoLoader *tenant_report = MTL(ObTenantInfoLoader*); + ObAllTenantInfo tenant_info; + if (OB_ISNULL(tenant_report)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tenant report is null", KR(ret), K(tenant_id_)); + } else if (OB_FAIL(tenant_report->get_tenant_info(tenant_info))) { + LOG_WARN("failed to get tenant info", KR(ret)); + } else if (OB_FAIL(ObAllTenantInfoProxy::update_tenant_sys_recovery_scn_in_trans( + tenant_id_, sys_recovery_scn, true, trans))) { + LOG_WARN("failed to update tenant sys recovery scn", KR(ret), K(tenant_id_), + K(sys_recovery_scn), K(tenant_info)); + } else { + //double check sync can not fallback + //log will be truncated during the flashback. + //If a certain iteration report crosses the flashback, + //recovery scn too large may be reported. Therefore, + //it is verified that the reported bit is not rolled back within the transaction. + palf::PalfHandleGuard palf_handle_guard; + logservice::ObLogService *ls_svr = MTL(logservice::ObLogService*); + SCN new_scn; + if (OB_FAIL(ls_svr->open_palf(SYS_LS, palf_handle_guard))) { + LOG_WARN("failed to open palf", KR(ret), K(ls_id)); + } else if (OB_FAIL(palf_handle_guard.get_end_scn(new_scn))) { + LOG_WARN("failed to get end ts", KR(ret), K(ls_id)); + } else if (new_scn < sys_recovery_scn) { + ret = OB_OP_NOT_ALLOW; + LOG_WARN("maybe flashback, can not report sync scn", KR(ret), K(sys_recovery_scn), K(new_scn)); + } + } + return ret; +} + +int ObRecoveryLSHelper::try_to_process_sys_ls_offline_() +{ + int ret = OB_SUCCESS; + ObTenantSchema tenant_schema; + if (OB_FAIL(ObTenantThreadHelper::get_tenant_schema(tenant_id_, tenant_schema))) { + LOG_WARN("failed to get tenant schema", KR(ret), K(tenant_id_)); + } else if (!tenant_schema.is_dropping()) { + //not dropping, nothing + } else if (OB_ISNULL(proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("proxy is null", KR(ret)); + } else { + //check sys ls is tenant dropping, and check can offline + share::ObLSStatusInfo ls_status; + share::ObLSStatusOperator ls_op; + if (OB_FAIL(ls_op.get_ls_status_info(tenant_id_, SYS_LS, ls_status, *proxy_))) { + LOG_WARN("failed to get ls status", KR(ret), K(tenant_id_)); + } else if (!ls_status.ls_is_tenant_dropping()) { + //nothing todo + } else { + bool can_offline = false; + ObTenantLSInfo tenant_stat(GCTX.sql_proxy_, &tenant_schema, tenant_id_, + GCTX.srv_rpc_proxy_, GCTX.lst_operator_); + if (OB_FAIL(tenant_stat.check_ls_can_offline_by_rpc(ls_status, can_offline))) { + LOG_WARN("failed to check ls can offline", KR(ret), K(ls_status)); + } else if (!can_offline) { + //nothing + } else { + ObLSLifeAgentManager ls_life_agent(*proxy_); + if (OB_FAIL(ls_life_agent.set_ls_offline(tenant_id_, SYS_LS, ls_status.status_, + SCN::base_scn(), share::NORMAL_SWITCHOVER_STATUS))) { + LOG_WARN("failed to set ls offline", KR(ret), K(ls_status)); + } + } + } + } + return ret; +} + +} +} diff --git a/src/rootserver/ob_ls_service_helper.h b/src/rootserver/ob_ls_service_helper.h new file mode 100644 index 000000000..b8e2b3201 --- /dev/null +++ b/src/rootserver/ob_ls_service_helper.h @@ -0,0 +1,279 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_ROOTSERVER_OB_LS_SERVICE_HELPER_H +#define OCEANBASE_ROOTSERVER_OB_LS_SERVICE_HELPER_H +#include "lib/thread/ob_reentrant_thread.h"//ObRsReentrantThread +#include "logservice/ob_log_base_type.h" +#include "share/scn.h"//SCN +#include "share/ls/ob_ls_status_operator.h"//ObLSStatusOperator +#include "share/ls/ob_ls_operator.h" //ObLSAttr +#include "share/ob_thread_mgr.h" //OBTGDefIDEnum +#include "logservice/palf/palf_iterator.h" //PalfBufferIterator +#include "share/unit/ob_unit_info.h"//ObUnit::Status +#include "lib/thread/thread_mgr_interface.h" // TGRunnable +#include "lib/lock/ob_thread_cond.h"//ObThreadCond + + +namespace oceanbase +{ +namespace obrpc +{ +class ObSrvRpcProxy; +} +namespace common +{ +class ObMySQLProxy; +class ObISQLClient; +class ObMySQLTransaction; +class ObClusterVersion; +} +namespace share +{ +class ObLSTableOperator; +class SCN; +namespace schema +{ +class ObMultiVersionSchemaService; +class ObTenantSchema; +} +} +namespace logservice +{ +class ObLogHandler; +} +namespace transaction +{ +class ObTxLogBlock; +class ObTxBufferNode; +} +namespace palf +{ +struct PalfBaseInfo; +} +namespace rootserver +{ +struct ObUnitGroupInfo +{ + ObUnitGroupInfo() : unit_group_id_(OB_INVALID_ID), unit_status_(share::ObUnit::UNIT_STATUS_MAX), + ls_group_ids_() {} + virtual ~ObUnitGroupInfo() {} + bool is_valid() const; + int init(const uint64_t unit_group_id, + const share::ObUnit::Status &unit_status); + void reset(); + int assign(const ObUnitGroupInfo &other); + int remove_ls_group(const uint64_t ls_group_id); + bool operator==(const ObUnitGroupInfo &other) const; + + uint64_t unit_group_id_; + share::ObUnit::Status unit_status_; + ObArray ls_group_ids_; + TO_STRING_KV(K_(unit_group_id), K_(unit_status), K_(ls_group_ids)); +}; +typedef ObArray ObUnitGroupInfoArray; +typedef ObIArray ObUnitGroupInfoIArray; + +struct ObLSGroupInfo +{ + ObLSGroupInfo() : ls_group_id_(OB_INVALID_ID), unit_group_id_(OB_INVALID_ID), + ls_ids_() {} + virtual ~ObLSGroupInfo() {} + bool is_valid() const; + int init(const uint64_t unit_group_id, const uint64_t ls_group_id); + int assign(const ObLSGroupInfo &other); + void reset(); + int remove_ls(const share::ObLSID &ls_id); + uint64_t ls_group_id_; + uint64_t unit_group_id_; + ObArray ls_ids_; + TO_STRING_KV(K_(ls_group_id), K_(unit_group_id), K_(ls_ids)); +}; + +typedef ObArray ObLSGroupInfoArray; +typedef ObIArray ObLSGroupInfoIArray; + +struct ObLSStatusMachineParameter +{ + ObLSStatusMachineParameter() : ls_id_(), status_info_(), ls_info_() {} + virtual ~ObLSStatusMachineParameter() {} + bool is_valid() const + { + return ls_id_.is_valid() + && (share::OB_LS_EMPTY == status_info_.status_ + || status_info_.ls_id_ == ls_id_); + } + int init(const share::ObLSID &id, const share::ObLSStatusInfo &status_info, + const share::ObLSAttr &ls_info); + void reset(); + share::ObLSID ls_id_; + share::ObLSStatusInfo status_info_;//for create ls and status of __all_ls_status + share::ObLSAttr ls_info_; + TO_STRING_KV(K_(ls_id), K_(status_info), K_(ls_info)); +}; + +/*descripthin: Tenant log stream status information: Statistical log stream + * status on __all_ls_status and __all_ls, tenant primary_zone and unit_num + * information. Provides location information for the newly created log stream. + * Whether the build needs to create or delete log streams.*/ +class ObTenantLSInfo +{ +public: + ObTenantLSInfo(ObMySQLProxy *sql_proxy, + const share::schema::ObTenantSchema *tenant_schema, + const uint64_t tenant_id, obrpc::ObSrvRpcProxy *rpc_proxy, + share::ObLSTableOperator *lst_operator) + : sql_proxy_(sql_proxy), + tenant_schema_(tenant_schema), + status_operator_(), + ls_operator_(tenant_id, sql_proxy), + status_array_(), + unit_group_array_(), + ls_group_array_(), + primary_zone_(), + rpc_proxy_(rpc_proxy), + lst_operator_(lst_operator), + max_ls_id_(OB_INVALID_ID), + max_ls_group_id_(OB_INVALID_ID) {} + + virtual ~ObTenantLSInfo(){}; + void reset(); + bool is_valid() const; + + int gather_stat(bool for_recovery); + int process_next_ls_status(int64_t &task_cnt); + + // need create or delete new ls group + int check_ls_group_match_unitnum(); + + //need create or delete new ls + int check_ls_match_primary_zone(); + + int fetch_new_ls_group_id(const uint64_t tenant_id, uint64_t &ls_group_id); + int fetch_new_ls_id(const uint64_t tenant_id, share::ObLSID &ls_id); + + //get ls group info from ls_group_array_ by ls_group_id + //the interface must used after gather_stat(); + int get_ls_group_info(const uint64_t ls_group_id, ObLSGroupInfo &info) const; + + //get ls status info from status_array_ by ls_id + //the interface must used after gather_stat(); + int get_ls_status_info(const share::ObLSID &id, share::ObLSStatusInfo &info, + int64_t &info_index) const; + //process dropping tenant, set status to tenant_dropping in __all_ls + int drop_tenant(); + //for recovery tenant, create new ls according to ls_id and ls_group_id + int create_new_ls_in_trans(const share::ObLSID &ls_id, + const uint64_t ls_group_id, + const share::SCN &create_scn, + common::ObMySQLTransaction &trans); + int construct_ls_status_machine_( + common::ObIArray &status_machine_array); + //only for upgrade, before 4.1, make __all_ls_status equal to __all_ls + int revision_to_equal_status(common::ObMySQLTransaction &trans); + int check_ls_can_offline_by_rpc(const share::ObLSStatusInfo &info, + bool &can_offline); +private: + // get from __all_ls_status and __all_ls + int gather_all_ls_info_(); + + // base on status_array construct ls_array + int add_ls_to_ls_group_(const share::ObLSStatusInfo &info); + + // base on ls_array construct unit_group_array and __all_unit + int add_ls_group_to_unit_group_(const ObLSGroupInfo &group_info); + + int check_unit_group_valid_(const uint64_t unit_group_id, bool &is_valid); + int add_ls_status_info_(const share::ObLSStatusInfo &ls_info); + int create_new_ls_for_empty_unit_group_(const uint64_t unit_group_id); + + int get_next_unit_group_(const bool is_recovery, int64_t &group_index); + // get the primary zone not in ls group + int get_next_primary_zone_(const bool is_recovery, const ObLSGroupInfo &group_info, + ObZone &primary_zone); + int create_new_ls_(const share::ObLSStatusInfo &ls_info); + + // drop ls group info + int do_tenant_drop_ls_(const share::ObLSStatusInfo &ls_info); + int sys_ls_tenant_drop_(const share::ObLSStatusInfo &info); + int check_sys_ls_can_offline_(bool &can_offline); + + +private: + //modify __all_ls + int process_next_ls_status_(const ObLSStatusMachineParameter &status_machine, bool &is_steady); + +private: + ObMySQLProxy *sql_proxy_; + const share::schema::ObTenantSchema *tenant_schema_; + share::ObLSStatusOperator status_operator_; + share::ObLSAttrOperator ls_operator_; + bool is_load_; + share::ObLSStatusInfoArray status_array_; + common::hash::ObHashMap status_map_; + ObUnitGroupInfoArray unit_group_array_; + ObLSGroupInfoArray ls_group_array_; + ObArray primary_zone_; + obrpc::ObSrvRpcProxy *rpc_proxy_; + share::ObLSTableOperator *lst_operator_; + //ensure ls can not be concurrent created + uint64_t max_ls_id_; + uint64_t max_ls_group_id_; +}; + +/*description: + *only active on the leader of the system log stream under the user tenant*/ +class ObRecoveryLSHelper +{ +public: + ObRecoveryLSHelper(const uint64_t tenant_id, + common::ObMySQLProxy *sql_proxy) : + tenant_id_(tenant_id), proxy_(sql_proxy) {} + virtual ~ObRecoveryLSHelper() {} + int do_work(palf::PalfBufferIterator &iterator, + share::SCN &start_scn); +private: + //get log iterator by start_scn + int seek_log_iterator_(const share::SCN &syn_scn, + palf::PalfBufferIterator &iterator); + int process_ls_log_(const ObAllTenantInfo &tenant_info, + share::SCN &start_scn, + palf::PalfBufferIterator &iterator); + int process_upgrade_log_(const transaction::ObTxBufferNode &node); + int process_ls_tx_log_(transaction::ObTxLogBlock &tx_log, + const share::SCN &syn_scn); + int process_ls_operator_(const share::ObLSAttr &ls_attr, + const share::SCN &syn_scn); + int create_new_ls_(const share::ObLSAttr &ls_attr, + const share::SCN &syn_scn, + common::ObMySQLTransaction &trans); + //wait other ls is larger than sycn ts + int check_valid_to_operator_ls_(const share::ObLSAttr &ls_attr, + const share::SCN &syn_scn); + //readable scn need report + int report_tenant_sys_recovery_scn_(const share::SCN &sys_recovery_scn, const bool update_sys_recovery_scn, + ObISQLClient *proxy); + int report_tenant_sys_recovery_scn_trans_(const share::SCN &sys_recovery_scn, const bool update_sys_recovery_scn, + common::ObMySQLTransaction &trans); + int process_ls_operator_in_trans_(const share::ObLSAttr &ls_attr, + const share::SCN &sys_recovery_scn, common::ObMySQLTransaction &trans); + int try_to_process_sys_ls_offline_(); +private: + uint64_t tenant_id_; + common::ObMySQLProxy *proxy_; +}; + +} +} + + +#endif /* !OCEANBASE_ROOTSERVER_OB_LS_SERVICE_HELPER_H */ diff --git a/src/rootserver/ob_primary_ls_service.cpp b/src/rootserver/ob_primary_ls_service.cpp index 4f4629494..237f48af8 100755 --- a/src/rootserver/ob_primary_ls_service.cpp +++ b/src/rootserver/ob_primary_ls_service.cpp @@ -14,23 +14,14 @@ #include "ob_primary_ls_service.h" #include "lib/profile/ob_trace_id.h" #include "share/ob_errno.h" -#include "share/ob_max_id_fetcher.h" -#include "share/schema/ob_schema_struct.h"//ObTenantInfo -#include "share/schema/ob_schema_service.h"//ObMultiSchemaService #include "share/ls/ob_ls_creator.h" //ObLSCreator #include "share/ls/ob_ls_life_manager.h"//ObLSLifeAgentManager -#include "share/ls/ob_ls_table_operator.h"//ObLSTableOpertor -#include "share/ob_primary_zone_util.h"//ObPrimaryZoneUtil -#include "share/ob_unit_table_operator.h"//ObUnitTableOperator -#include "share/ob_zone_table_operation.h" //ObZoneTableOperation #include "share/ob_share_util.h"//ObShareUtil #include "observer/ob_server_struct.h"//GCTX -#include "rootserver/ob_tenant_recovery_reportor.h"//update_ls_recovery -#include "rootserver/ob_tenant_role_transition_service.h" -#include "storage/tx_storage/ob_ls_map.h" #include "storage/tx_storage/ob_ls_service.h" #include "storage/tx_storage/ob_ls_handle.h" //ObLSHandle #include "logservice/palf/palf_base_info.h"//PalfBaseInfo +#include "rootserver/ob_ls_service_helper.h"//ObTenantLSInfo namespace oceanbase { @@ -40,1999 +31,6 @@ using namespace transaction; using namespace palf; namespace rootserver { -/////////ObUnitGroupInfo -bool ObUnitGroupInfo::is_valid() const -{ - return OB_INVALID_ID != unit_group_id_ - && share::ObUnit::UNIT_STATUS_MAX != unit_status_; -} - -int ObUnitGroupInfo::init(const uint64_t unit_group_id, - const share::ObUnit::Status &unit_status) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(OB_INVALID_ID == unit_group_id - || share::ObUnit::UNIT_STATUS_MAX == unit_status)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(unit_group_id), K(unit_status)); - } else { - unit_group_id_ = unit_group_id; - unit_status_ = unit_status; - } - return ret; -} - -void ObUnitGroupInfo::reset() -{ - unit_group_id_ = OB_INVALID_ID; - unit_status_ = share::ObUnit::UNIT_STATUS_MAX; - ls_group_ids_.reset(); -} - -int ObUnitGroupInfo::assign(const ObUnitGroupInfo &other) -{ - int ret = OB_SUCCESS; - if (this != &other) { - if (OB_FAIL(ls_group_ids_.assign(other.ls_group_ids_))) { - LOG_WARN("failed to assign", KR(ret), K(other)); - } else { - unit_group_id_ = other.unit_group_id_; - unit_status_ = other.unit_status_; - } - } - return ret; -} - -int ObUnitGroupInfo::remove_ls_group(const uint64_t ls_group_id) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(ObLSID::MIN_USER_LS_GROUP_ID >= ls_group_id)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("ls group id is invalid", KR(ret), K(ls_group_id)); - } else { - bool remove = false; - for (int64_t i = 0; OB_SUCC(ret) && i < ls_group_ids_.count(); ++i) { - if (ls_group_ids_.at(i) == ls_group_id) { - remove = true; - if (OB_FAIL(ls_group_ids_.remove(i))) { - LOG_WARN("failed to remove from array", KR(ret), K(i), - K(ls_group_id), "this", *this); - } - break; - } - } - if (OB_SUCC(ret) && !remove) { - ret = OB_ENTRY_NOT_EXIST; - LOG_WARN("failed to find ls group id", KR(ret), - K(ls_group_id), "this", *this); - } - } - return ret; -} - -bool ObUnitGroupInfo::operator==(const ObUnitGroupInfo &other) const -{ - return unit_group_id_ == other.unit_group_id_ - && unit_status_ == other.unit_status_; -} - -////////////ObLSGroupInfo -bool ObLSGroupInfo::is_valid() const -{ - return OB_INVALID_ID != unit_group_id_ - && OB_INVALID_ID != ls_group_id_; -} - -int ObLSGroupInfo::init(const uint64_t unit_group_id, - const uint64_t ls_group_id) -{ - int ret = OB_SUCCESS; - reset(); - if (OB_UNLIKELY(OB_INVALID_ID == unit_group_id - || OB_INVALID_ID == ls_group_id)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(unit_group_id), K(ls_group_id)); - } else { - unit_group_id_ = unit_group_id; - ls_group_id_ = ls_group_id; - } - return ret; -} - -int ObLSGroupInfo::assign(const ObLSGroupInfo &other) -{ - int ret = OB_SUCCESS; - if (this != &other) { - reset(); - if (OB_FAIL(ls_ids_.assign(other.ls_ids_))) { - LOG_WARN("failed to assign ls ids", KR(ret), K(other)); - } else { - unit_group_id_ = other.unit_group_id_; - ls_group_id_ = other.ls_group_id_; - } - } - return ret; -} - -void ObLSGroupInfo::reset() -{ - ls_group_id_ = OB_INVALID_ID; - unit_group_id_ = OB_INVALID_ID; - ls_ids_.reset(); -} - -int ObLSGroupInfo::remove_ls(const share::ObLSID &ls_id) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!ls_id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("ls id is invalid", KR(ret), K(ls_id)); - } else { - bool remove = false; - for (int64_t i = 0; OB_SUCC(ret) && i < ls_ids_.count(); ++i) { - if (ls_ids_.at(i) == ls_id) { - remove = true; - if (OB_FAIL(ls_ids_.remove(i))) { - LOG_WARN("failed to remove from array", KR(ret), K(i), - K(ls_id), "this", *this); - } - break; - } - } - if (OB_SUCC(ret) && !remove) { - ret = OB_ENTRY_NOT_EXIST; - LOG_WARN("failed to find ls id", KR(ret), - K(ls_id), "this", *this); - } - } - return ret; - -} - -///////////////ObLSStatusMachineParameter -//no need check status valid or ls_status is valid -int ObLSStatusMachineParameter::init(const share::ObLSID &id, - const share::ObLSStatusInfo &status_info, - const share::ObLSStatus &ls_info) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("id is invalid", KR(ret), K(id)); - } else if (OB_FAIL(status_info_.assign(status_info))) { - LOG_WARN("failed to assign status info", KR(ret), K(status_info)); - } else { - ls_info_ = ls_info; - ls_id_ = id; - } - return ret; -} - -void ObLSStatusMachineParameter::reset() -{ - ls_id_.reset(); - ls_info_ = share::OB_LS_EMPTY; - status_info_.reset(); -} - -/////////////ObTenantLSInfo - -void ObTenantLSInfo::reset() -{ - is_load_ = false; - status_array_.reset(); - unit_group_array_.reset(); - ls_group_array_.reset(); - primary_zone_.reset(); - max_ls_id_ = OB_INVALID_ID; - max_ls_group_id_ = OB_INVALID_ID; -} - - -bool ObTenantLSInfo::is_valid() const -{ - return ATOMIC_LOAD(&is_load_); -} - -//Regardless of the tenant being dropped, -//handle the asynchronous operation of status and history table -int ObTenantLSInfo::process_ls_status_missmatch( - const bool lock_sys_ls, - const share::ObTenantSwitchoverStatus &working_sw_status) -{ - int ret = OB_SUCCESS; - share::ObLSStatusInfoArray status_info_array; - share::ObLSAttrArray ls_array; - ObArray status_machine_array; - if (OB_ISNULL(sql_proxy_) || OB_ISNULL(tenant_schema_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy or tenant schema is null", KR(ret), KP(tenant_schema_), KP(sql_proxy_)); - } else if (OB_FAIL(ls_operator_.get_all_ls_by_order(lock_sys_ls, ls_array))) { - LOG_WARN("failed to insert ls operation", KR(ret)); - } else { - const uint64_t tenant_id = tenant_schema_->get_tenant_id(); - if (OB_FAIL(status_operator_.get_all_ls_status_by_order( - tenant_id, status_info_array, *sql_proxy_))) { - LOG_WARN("failed to update ls status", KR(ret)); - } - } - if (FAILEDx(construct_ls_status_machine_(status_info_array, ls_array, - status_machine_array))) { - LOG_WARN("failed to construct ls status machine array", KR(ret), - K(status_info_array), K(ls_array)); - } else { - int tmp_ret = OB_SUCCESS; - ARRAY_FOREACH_NORET(status_machine_array, idx) { - const ObLSStatusMachineParameter &machine = status_machine_array.at(idx); - //ignore error of each ls - //may ls can not create success - if (OB_SUCCESS != (tmp_ret = fix_ls_status_(machine, working_sw_status))) { - LOG_WARN("failed to fix ls status", KR(ret), KR(tmp_ret), K(machine)); - ret = OB_SUCC(ret) ? tmp_ret : ret; - } - } - } - - return ret; -} - -int ObTenantLSInfo::construct_ls_status_machine_( - const share::ObLSStatusInfoIArray &status_info_array, - const share::ObLSAttrIArray &ls_array, - common::ObIArray &status_machine_array) -{ - int ret = OB_SUCCESS; - status_machine_array.reset(); - if (OB_UNLIKELY(0 == status_info_array.count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls array can not be empty", KR(ret), K(ls_array), K(status_info_array)); - } else { - // merge by sort - int64_t status_index = 0; - int64_t ls_index = 0; - ObLSStatusMachineParameter status_machine; - const int64_t status_count = status_info_array.count(); - const int64_t ls_count = ls_array.count(); - share::ObLSStatusInfo status_info; - ObLSID ls_id; - ObLSStatus ls_info_status = share::OB_LS_EMPTY; - while ((status_index < status_count || ls_index < ls_count) && OB_SUCC(ret)) { - status_machine.reset(); - if (status_index == status_count) { - //status already end - ls_id = ls_array.at(ls_index).get_ls_id(); - ls_info_status = ls_array.at(ls_index).get_ls_status(); - status_info.reset(); - ls_index++; - } else if (ls_index == ls_count) { - //ls already end - if (OB_FAIL(status_info.assign(status_info_array.at(status_index)))) { - LOG_WARN("failed to assign status info", KR(ret), K(status_index)); - } else { - status_index++; - ls_id = status_info.ls_id_; - ls_info_status = share::OB_LS_EMPTY; - } - } else { - const share::ObLSStatusInfo &tmp_status_info = status_info_array.at(status_index); - const share::ObLSAttr &tmp_ls_info = ls_array.at(ls_index); - if (tmp_status_info.ls_id_ == tmp_ls_info.get_ls_id()) { - status_index++; - ls_index++; - ls_info_status = tmp_ls_info.get_ls_status(); - ls_id = tmp_status_info.ls_id_; - if (OB_FAIL(status_info.assign(tmp_status_info))) { - LOG_WARN("failed to assign status info", KR(ret), K(tmp_status_info)); - } - } else if (tmp_status_info.ls_id_ > tmp_ls_info.get_ls_id()) { - ls_index++; - ls_id = tmp_ls_info.get_ls_id(); - ls_info_status = tmp_ls_info.get_ls_status(); - status_info.reset(); - } else { - status_index++; - ls_id = tmp_status_info.ls_id_; - ls_info_status = share::OB_LS_EMPTY; - if (OB_FAIL(status_info.assign(tmp_status_info))) { - LOG_WARN("failed to assign status info", KR(ret), K(tmp_status_info)); - } - } - } - if (FAILEDx(status_machine.init(ls_id, status_info, ls_info_status))) { - LOG_WARN("failed to push back status", KR(ret), K(ls_id), K(status_info), K(ls_info_status)); - } else if (OB_FAIL(status_machine_array.push_back(status_machine))) { - LOG_WARN("failed to push back status machine", KR(ret), K(status_machine)); - } - } // end while - - } - return ret; -} - -//for create new ls -//ls_info:creating, status_info:creating, status_info:created, ls_info:normal, status_info:normal - -//for drop ls -//ls_info:dropping, status_info:dropping, ls_info:empty, status_info:wait_offline - -//for drop tenant -//ls_info:tenant_dropping, status_info:tenant_dropping, ls_info:empty, status_info:wait_offline -//the sys ls can be tenant_dropping in status_info after all_users ls were deleted in ls_info -int ObTenantLSInfo::fix_ls_status_(const ObLSStatusMachineParameter &machine, - const ObTenantSwitchoverStatus &working_sw_status) -{ - int ret = OB_SUCCESS; - const share::ObLSStatusInfo &status_info = machine.status_info_; - const uint64_t tenant_id = status_info.tenant_id_; - ObLSLifeAgentManager ls_life_agent(*sql_proxy_); - if (OB_UNLIKELY(!machine.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("status machine is valid", KR(ret), K(machine)); - } else if (OB_ISNULL(sql_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy is null", KR(ret), KP(sql_proxy_)); - } else if (ls_is_empty_status(machine.ls_info_)) { - //the ls not in ls and in status - if (status_info.ls_is_wait_offline()) { - //already offline, status will be deleted by GC - } else if (status_info.ls_is_dropping() - || status_info.ls_is_tenant_dropping()) { - SCN drop_scn; - if (OB_FAIL(ObLSAttrOperator::get_tenant_gts(tenant_id, drop_scn))) { - LOG_WARN("failed to get gts", KR(ret), K(tenant_id)); - } else if (OB_FAIL(ls_life_agent.set_ls_offline(tenant_id, - status_info.ls_id_, status_info.status_, - drop_scn, working_sw_status))) { - LOG_WARN("failed to update ls status", KR(ret), K(status_info), K(tenant_id), K(drop_scn)); - } - } else if (status_info.ls_is_creating() - || status_info.ls_is_created() - || status_info.ls_is_create_abort()) { - //ls may create_abort - if (OB_FAIL(ls_life_agent.drop_ls(tenant_id, machine.ls_id_, working_sw_status))) { - LOG_WARN("failed to delete ls", KR(ret), K(machine)); - } - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("status machine not expected", KR(ret), K(machine)); - } - } else if (ls_is_creating_status(machine.ls_info_)) { - //the ls may is in creating - if (!status_info.is_valid()) { - // TODO fix later - //in primary cluster, can create continue, but no need - if (OB_FAIL(ls_operator_.delete_ls(machine.ls_id_, machine.ls_info_, working_sw_status))) { - LOG_WARN("failed to delete ls", KR(ret), K(machine)); - } - } else if (status_info.ls_is_creating() && working_sw_status.is_normal_status()) { - //TODO check the primary zone or unit group id is valid - ObLSRecoveryStat recovery_stat; - ObLSRecoveryStatOperator ls_recovery_operator; - if (OB_FAIL(ls_recovery_operator.get_ls_recovery_stat(tenant_id, status_info.ls_id_, - false/*for_update*/, recovery_stat, *sql_proxy_))) { - LOG_WARN("failed to get ls recovery stat", KR(ret), K(tenant_id), K(status_info)); - } else if (OB_FAIL(do_create_ls_(status_info, recovery_stat.get_create_scn()))) { - LOG_WARN("failed to create new ls", KR(ret), K(status_info), - K(recovery_stat)); - } - } else if (status_info.ls_is_created()) { - if (OB_FAIL(process_ls_status_after_created_(status_info, working_sw_status))) { - LOG_WARN("failed to create ls", KR(ret), K(status_info)); - } - } else if (status_info.ls_is_create_abort()) { - //drop ls - if (OB_FAIL(ls_operator_.delete_ls(status_info.ls_id_, machine.ls_info_, working_sw_status))) { - LOG_WARN("failed to remove ls not normal", KR(ret), K(machine)); - } - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("status machine not expected", KR(ret), K(machine)); - } - } else if (ls_is_normal_status(machine.ls_info_)) { - //the ls is normal - if (status_info.ls_is_normal()) { - //nothing - } else if (status_info.ls_is_created()) { - if (OB_FAIL(status_operator_.update_ls_status( - tenant_id, status_info.ls_id_, status_info.status_, - share::OB_LS_NORMAL, working_sw_status, *sql_proxy_))) { - LOG_WARN("failed to update ls status", KR(ret), K(status_info), K(tenant_id)); - } - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("status machine not expected", KR(ret), K(machine)); - } - } else if (ls_is_dropping_status(machine.ls_info_)) { - if (FAILEDx(do_drop_ls_(status_info, working_sw_status))) { - LOG_WARN("failed to drop ls", KR(ret), K(status_info)); - } - } else if (ls_is_pre_tenant_dropping_status(machine.ls_info_)) { - if (!machine.ls_id_.is_sys_ls()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("normal ls can not in pre tenant dropping status", KR(ret), K(machine)); - } else if (OB_FAIL(sys_ls_tenant_drop_(status_info, working_sw_status))) { - LOG_WARN("failed to process sys ls", KR(ret), K(status_info)); - } - } else if (ls_is_tenant_dropping_status(machine.ls_info_)) { - if (status_info.ls_is_wait_offline()) { - //sys ls will tenant_dropping and wait offline - } else if (OB_FAIL(do_tenant_drop_ls_(status_info, working_sw_status))) { - LOG_WARN("failed to drop ls", KR(ret), K(machine)); - } - } else { - //other status can not be in __all_ls - //such as created, wait_offline - ret = OB_ERR_UNEXPECTED; - LOG_WARN("the ls not expected in all_ls", KR(ret), K(machine)); - } - return ret; -} - -int ObTenantLSInfo::drop_tenant(const ObTenantSwitchoverStatus &working_sw_status) -{ - int ret = OB_SUCCESS; - share::ObLSAttrArray ls_array; - if (OB_FAIL(ls_operator_.get_all_ls_by_order(ls_array))) { - LOG_WARN("failed to get all ls", KR(ret)); - } else { - //1. set sys_ls to tenant_dropping in __all_ls - //2. set user_ls to tenant_dropping or abort in __all_ls - for (int64_t i = 0; OB_SUCC(ret) && i < ls_array.count(); ++i) { - share::ObLSAttr &attr = ls_array.at(i); - if (attr.get_ls_id().is_sys_ls()) { - if (attr.ls_is_normal()) { - if (OB_FAIL(ls_operator_.update_ls_status(attr.get_ls_id(), - attr.get_ls_status(), share::OB_LS_PRE_TENANT_DROPPING, working_sw_status))) { - LOG_WARN("failed to update ls status", KR(ret), K(attr)); - } - } - } else if (attr.ls_is_creating()) { - //drop the status - if (OB_FAIL(ls_operator_.delete_ls(attr.get_ls_id(), attr.get_ls_status(), working_sw_status))) { - LOG_WARN("failed to remove ls not normal", KR(ret), K(attr)); - } - } else { - //no matter the status is in normal or dropping - //may be the status in status info is created - if (!attr.ls_is_tenant_dropping()) { - if (OB_FAIL(ls_operator_.update_ls_status( - attr.get_ls_id(), attr.get_ls_status(), - share::OB_LS_TENANT_DROPPING, working_sw_status))) { - LOG_WARN("failed to update ls status", KR(ret), K(attr)); - } else { - LOG_INFO("[LS_MGR] set ls to tenant dropping", KR(ret), K(attr)); - } - } - } - }//end for - } - return ret; -} -int ObTenantLSInfo::gather_stat(bool for_recovery) -{ - int ret = OB_SUCCESS; - reset(); - if (OB_ISNULL(tenant_schema_) || OB_ISNULL(sql_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected", KR(ret), KP(tenant_schema_), KP(sql_proxy_)); - } else { - ObUnitTableOperator unit_operator; - common::ObArray units; - const uint64_t tenant_id = tenant_schema_->get_tenant_id(); - if (OB_FAIL(unit_operator.init(*sql_proxy_))) { - LOG_WARN("failed to init unit operator", KR(ret)); - } else if (OB_FAIL(unit_operator.get_units_by_tenant( - tenant_id, units))) { - LOG_WARN("failed to get all tenant unit", KR(ret), K(tenant_id)); - } else { - ObUnitGroupInfo info; - for (int64_t j = 0; OB_SUCC(ret) && j < units.count(); ++j) { - info.reset(); - const ObUnit &unit = units.at(j); - if (OB_FAIL(info.init(unit.unit_group_id_, unit.status_))) { - LOG_WARN("failed to init unit info", KR(ret), K(unit)); - } else if (has_exist_in_array(unit_group_array_, info)) { - //nothing - } else if (OB_FAIL(unit_group_array_.push_back(info))) { - LOG_WARN("fail to push back", KR(ret), K(info)); - } - } - } - if (OB_SUCC(ret)) { - if (for_recovery) { - } else { - //get max ls id and max ls group id - share::ObMaxIdFetcher id_fetcher(*sql_proxy_); - if (OB_FAIL(id_fetcher.fetch_max_id(*sql_proxy_, tenant_id, - share::OB_MAX_USED_LS_GROUP_ID_TYPE, max_ls_group_id_))) { - LOG_WARN("failed to fetch max ls group id", KR(ret), K(tenant_id)); - } else if (OB_FAIL(id_fetcher.fetch_max_id(*sql_proxy_, tenant_id, - share::OB_MAX_USED_LS_ID_TYPE, max_ls_id_))) { - LOG_WARN("failed to fetch max ls id", KR(ret), K(tenant_id)); - } - } - } - if (FAILEDx(gather_all_ls_info_())) { - LOG_WARN("failed to get all ls info", KR(ret), K(tenant_id)); - } else if (OB_FAIL(ObPrimaryZoneUtil::get_tenant_primary_zone_array( - *tenant_schema_, primary_zone_))) { - LOG_WARN("failed to get tenant primary zone array", KR(ret), KPC(tenant_schema_)); - } else { - is_load_ = true; - } - } - LOG_INFO("[LS_MGR] gather stat", KR(ret), K(primary_zone_), K(unit_group_array_), - K(max_ls_id_), K(max_ls_group_id_)); - return ret; -} - -int ObTenantLSInfo::gather_all_ls_info_() -{ - int ret = OB_SUCCESS; - share::ObLSStatusInfoArray status_info_array; - if (OB_ISNULL(sql_proxy_) || OB_ISNULL(tenant_schema_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy or tenant schema is null", KR(ret), KP(tenant_schema_), KP(sql_proxy_)); - } else { - const uint64_t tenant_id = tenant_schema_->get_tenant_id(); - if (OB_FAIL(status_operator_.get_all_ls_status_by_order( - tenant_id, status_info_array, *sql_proxy_))) { - LOG_WARN("failed to update ls status", KR(ret), K(tenant_id)); - } - } - if (OB_FAIL(ret)) { - } else { - const int64_t count = max(1, hash::cal_next_prime(status_info_array.count())); - if (OB_FAIL(status_map_.create(count, "LogStrInfo", "LogStrInfo"))) { - LOG_WARN("failed to create ls map", KR(ret), K(count)); - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < status_info_array.count(); ++i) { - const share::ObLSStatusInfo &info = status_info_array.at(i); - if (info.ls_is_wait_offline()) { - //ls is already offline, no need to process - } else if (FAILEDx(add_ls_status_info_(info))) { - LOG_WARN("failed to add ls status info", KR(ret), K(i), K(info)); - } - }// end for - } - } - return ret; -} - -int ObTenantLSInfo::add_ls_to_ls_group_(const share::ObLSStatusInfo &info) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!info.is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls status is invalid", KR(ret), K(info)); - } else { - int64_t group_index = OB_INVALID_INDEX_INT64; - for (int64_t j = 0; OB_SUCC(ret) && j < ls_group_array_.count(); - ++j) { - const ObLSGroupInfo &group = ls_group_array_.at(j); - if (info.ls_group_id_ == group.ls_group_id_) { - group_index = j; - break; - } - } - if (OB_SUCC(ret) && OB_INVALID_INDEX_INT64 == group_index) { - ObLSGroupInfo group; - group_index = ls_group_array_.count(); - if (OB_FAIL(group.init(info.unit_group_id_, info.ls_group_id_))) { - LOG_WARN("failed to init ls group", KR(ret), K(info)); - } else if (OB_FAIL(ls_group_array_.push_back(group))) { - LOG_WARN("failed to pushback group", KR(ret), K(group)); - } else if (OB_FAIL(add_ls_group_to_unit_group_(group))) { - LOG_WARN("failed to add ls group to unit group", KR(ret), K(group)); - } - } - if (OB_SUCC(ret)) { - if (OB_UNLIKELY(group_index >= ls_group_array_.count() - || OB_INVALID_INDEX_INT64 == group_index)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("group index is invalid", KR(ret), K(group_index), "count", - ls_group_array_.count()); - } else if (OB_FAIL(ls_group_array_.at(group_index) - .ls_ids_.push_back(info.ls_id_))) { - LOG_WARN("failed to push back ls id", KR(ret), K(group_index), - K(info)); - } - } - } - return ret; -} - -int ObTenantLSInfo::add_ls_group_to_unit_group_(const ObLSGroupInfo &group_info) -{ - int ret = OB_SUCCESS; - bool found = false; - if (OB_UNLIKELY(!group_info.is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("group_info is invalid", KR(ret), K(group_info)); - } else { - found = false; - for (int64_t j = 0; OB_SUCC(ret) && j < unit_group_array_.count(); ++j) { - ObUnitGroupInfo &unit_info = unit_group_array_.at(j); - if (group_info.unit_group_id_ == unit_info.unit_group_id_) { - found = true; - if (OB_FAIL(unit_info.ls_group_ids_.push_back( - group_info.ls_group_id_))) { - LOG_WARN("failed to push back ls group id", KR(ret), - K(group_info), K(unit_info)); - } - } - } // end j - if (OB_SUCC(ret) && !found) { - //can found unit_group for a ls group - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to find unit group for ls group", KR(ret), - K(group_info), K(unit_group_array_)); - } - } - - return ret; -} - -int ObTenantLSInfo::check_unit_group_valid_( - const uint64_t unit_group_id, bool &is_valid_info) -{ - int ret = OB_SUCCESS; - is_valid_info = true; - if (OB_UNLIKELY(OB_INVALID_VERSION == unit_group_id)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("ls id is invalid", KR(ret), K(unit_group_id)); - } else if (OB_UNLIKELY(!is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant stat not valid", KR(ret)); - } else { - //check unit group status is valid - ret = OB_ENTRY_NOT_EXIST; - for (int64_t i = 0; OB_FAIL(ret) && i < unit_group_array_.count(); ++i) { - const ObUnitGroupInfo &group_info = unit_group_array_.at(i); - if (unit_group_id == group_info.unit_group_id_) { - ret = OB_SUCCESS; - if (share::ObUnit::UNIT_STATUS_ACTIVE != group_info.unit_status_) { - is_valid_info = false; - } - break; - } - }//end for - if (OB_FAIL(ret)) { - LOG_WARN("failed to found unit group status", KR(ret), K(unit_group_id), - K(unit_group_array_)); - } - } - return ret; -} - -//for standby -int ObTenantLSInfo::create_new_ls_for_recovery( - const share::ObLSID &ls_id, - const uint64_t ls_group_id, - const SCN &create_scn, - ObMySQLTransaction &trans) -{ - int ret = OB_SUCCESS; - const bool is_recovery = true; - int64_t info_index = OB_INVALID_INDEX_INT64; - if (OB_UNLIKELY(!ls_id.is_valid() - || OB_INVALID_ID == ls_group_id - || !create_scn.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("ls id is invalid", KR(ret), K(ls_id), K(ls_group_id), K(create_scn)); - } else if (OB_UNLIKELY(!is_valid()) - || OB_ISNULL(tenant_schema_) - || OB_ISNULL(sql_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant stat not valid", KR(ret), KP(tenant_schema_), KP(sql_proxy_)); - } else { - const uint64_t tenant_id = tenant_schema_->get_tenant_id(); - share::ObLSStatusInfo new_info; - ObLSGroupInfo group_info; - ObZone primary_zone; - ObLSLifeAgentManager ls_life_agent(*sql_proxy_); - ObSqlString zone_priority; - if (OB_SUCC(get_ls_group_info( - ls_group_id, group_info))) { - // need a new primary zone - } else if (OB_ENTRY_NOT_EXIST == ret) { - // need a new unit group - ret = OB_SUCCESS; - int64_t unit_group_index = OB_INVALID_INDEX_INT64; - if (OB_FAIL(get_next_unit_group_(is_recovery, unit_group_index))) { - LOG_WARN("failed to get next unit group", KR(ret), K(is_recovery)); - } else if (OB_UNLIKELY(OB_INVALID_INDEX_INT64 == unit_group_index - || unit_group_index >= unit_group_array_.count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to next unit group", KR(ret), K(unit_group_index)); - } else if (OB_FAIL(group_info.init( - unit_group_array_.at(unit_group_index).unit_group_id_, - ls_group_id))) { - LOG_WARN("failed to init group info", KR(ret), K(unit_group_index), K(ls_group_id)); - } - } else { - LOG_WARN("failed to get ls group info", KR(ret), - K(ls_group_id)); - } - - if (FAILEDx(get_next_primary_zone_(is_recovery, group_info, primary_zone))) { - LOG_WARN("failed to get next primary zone", KR(ret), K(group_info)); - } else if (OB_FAIL(new_info.init(tenant_id, ls_id, - ls_group_id, - share::OB_LS_CREATING, - group_info.unit_group_id_, primary_zone))) { - LOG_WARN("failed to init new info", KR(ret), K(tenant_id), - K(ls_id), K(ls_group_id), K(group_info), K(primary_zone)); - } else if (OB_FAIL(get_zone_priority(primary_zone, *tenant_schema_, zone_priority))) { - LOG_WARN("failed to get normalize primary zone", KR(ret), K(primary_zone), K(zone_priority)); - } else if (OB_FAIL(ls_life_agent.create_new_ls_in_trans(new_info, create_scn, - zone_priority.string(), share::NORMAL_SWITCHOVER_STATUS, trans))) { - LOG_WARN("failed to insert ls info", KR(ret), K(new_info), K(create_scn), K(zone_priority)); - } else if (OB_FAIL(add_ls_status_info_(new_info))) { - LOG_WARN("failed to add new ls info to memory", KR(ret), K(new_info)); - } - } - return ret; -} - -int ObTenantLSInfo::process_ls_stats_for_recovery() -{ - int ret = OB_SUCCESS; - share::ObLSStatusInfoArray status_info_array; - ObLSRecoveryStat recovery_stat; - ObLSRecoveryStatOperator ls_recovery_operator; - if (OB_ISNULL(sql_proxy_) || OB_ISNULL(tenant_schema_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy or tenant schema is null", KR(ret), KP(tenant_schema_), KP(sql_proxy_)); - } else { - const uint64_t tenant_id = tenant_schema_->get_tenant_id(); - int tmp_ret = OB_SUCCESS; - if (OB_FAIL(status_operator_.get_all_ls_status_by_order( - tenant_id, status_info_array, *sql_proxy_))) { - LOG_WARN("failed to update ls status", KR(ret), K(tenant_id)); - } - for (int64_t i = 0; OB_SUCC(ret) && i < status_info_array.count(); ++i) { - const ObLSStatusInfo &status_info = status_info_array.at(i); - recovery_stat.reset(); - if (status_info.ls_is_creating()) { - if (OB_FAIL(ls_recovery_operator.get_ls_recovery_stat(tenant_id, status_info.ls_id_, - false/*for_update*/, recovery_stat, *sql_proxy_))) { - LOG_WARN("failed to get ls recovery stat", KR(ret), K(tenant_id), K(status_info)); - } else if (OB_FAIL(do_create_ls_(status_info, recovery_stat.get_create_scn()))) { - LOG_WARN("failed to create new ls", KR(ret), K(status_info), - K(recovery_stat)); - } - } - }//end for - //adjust primary zone - if (OB_SUCCESS != (tmp_ret = adjust_user_tenant_primary_zone())) { - ret = OB_SUCC(ret) ? tmp_ret : ret; - LOG_WARN("failed to adjust user tenant primary zone", KR(ret), - KR(tmp_ret), K(tenant_schema_)); - } - } - return ret; -} - -int ObTenantLSInfo::add_ls_status_info_( - const share::ObLSStatusInfo &ls_info) -{ - int ret = OB_SUCCESS; - const int64_t index = status_array_.count(); - if (OB_UNLIKELY(!ls_info.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("ls info is invalid", KR(ret), K(ls_info)); - } else if (OB_FAIL(status_array_.push_back(ls_info))) { - LOG_WARN("failed to remove status", KR(ret), K(ls_info)); - } else if (OB_FAIL(status_map_.set_refactored(ls_info.ls_id_, index))) { - LOG_WARN("failed to remove ls from map", KR(ret), K(ls_info), K(index)); - } else if (ls_info.ls_id_.is_sys_ls()) { - //sys ls no ls group - } else if (OB_FAIL(add_ls_to_ls_group_(ls_info))) { - LOG_WARN("failed to add ls info", KR(ret), K(ls_info)); - } - return ret; -} - -int ObTenantLSInfo::get_ls_group_info( - const uint64_t ls_group_id, ObLSGroupInfo &info) const -{ - int ret = OB_SUCCESS; - info.reset(); - if (OB_UNLIKELY(OB_INVALID_ID == ls_group_id)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("ls group id is invalid", KR(ret), K(ls_group_id)); - } else if (OB_UNLIKELY(!is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant stat not valid", KR(ret)); - } else { - bool found = false; - for (int64_t i = 0; OB_SUCC(ret) && i < ls_group_array_.count(); ++i) { - const ObLSGroupInfo group_info = ls_group_array_.at(i); - if (ls_group_id == group_info.ls_group_id_) { - found = true; - if (OB_FAIL(info.assign(group_info))) { - LOG_WARN("failed to assign group info", KR(ret), K(group_info)); - } - break; - } - }//end for - if (OB_SUCC(ret) && !found) { - ret = OB_ENTRY_NOT_EXIST; - LOG_WARN("failed to find ls group info", KR(ret), - K(ls_group_id)); - } - } - return ret; -} - -int ObTenantLSInfo::get_ls_status_info( - const share::ObLSID &ls_id, - share::ObLSStatusInfo &info, - int64_t &info_index) const -{ - int ret = OB_SUCCESS; - info.reset(); - info_index = OB_INVALID_INDEX_INT64; - if (OB_UNLIKELY(!ls_id.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("ls id is invalid", KR(ret), K(ls_id)); - } else if (OB_UNLIKELY(!is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant stat not valid", KR(ret)); - } else if (OB_FAIL(status_map_.get_refactored(ls_id, info_index))) { - if (OB_HASH_NOT_EXIST == ret) { - ret = OB_ENTRY_NOT_EXIST; - } - LOG_WARN("failed to find ls index", KR(ret), K(ls_id)); - } else if (OB_UNLIKELY(info_index < 0 || info_index >= status_array_.count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("info index not valid", KR(ret), K(info_index), K(ls_id), - "status_count", status_array_.count()); - } else if (OB_FAIL(info.assign(status_array_.at(info_index)))) { - LOG_WARN("failed to assign ls info", KR(ret), K(info_index), - "status", status_array_.at(info_index)); - } - return ret; -} - -int ObTenantLSInfo::get_next_unit_group_(const bool is_recovery, int64_t &group_index) -{ - int ret = OB_SUCCESS; - group_index = OB_INVALID_INDEX_INT64; - if (OB_UNLIKELY(!is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant stat not valid", KR(ret)); - } else { - int64_t ls_count = OB_INVALID_COUNT; - // Find the unit group with the least number of log streams - for (int64_t i = 0; OB_SUCC(ret) && i < unit_group_array_.count(); ++i) { - const ObUnitGroupInfo &info = unit_group_array_.at(i); - const int64_t count = info.ls_group_ids_.count(); - if (share::ObUnit::UNIT_STATUS_ACTIVE == info.unit_status_) { - if (OB_INVALID_COUNT == ls_count || ls_count > count) { - ls_count = count; - group_index = i; - if (0 == count) { - //the first has no ls group unit group - break; - } - } - } - } - if (OB_SUCC(ret)) { - if (OB_INVALID_INDEX_INT64 == group_index - || (!is_recovery && 0 != ls_count)) { - ret = OB_ENTRY_NOT_EXIST; - LOG_WARN("failed to find next unit group", KR(ret), - K(unit_group_array_), K(is_recovery), K(ls_count)); - } - } - } - LOG_INFO("get next primary zone", KR(ret), K(group_index), K(is_recovery)); - return ret; -} - -int ObTenantLSInfo::get_next_primary_zone_( - const bool is_recovery, - const ObLSGroupInfo &group_info, - ObZone &primary_zone) -{ - int ret = OB_SUCCESS; - primary_zone.reset(); - if (OB_UNLIKELY(!group_info.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("group info is invalid", KR(ret), K(group_info)); - } else if (OB_UNLIKELY(!is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant ls info not valid", KR(ret)); - } else { - share::ObLSStatusInfo info; - int64_t ls_count = OB_INVALID_COUNT; - int64_t info_index = OB_INVALID_INDEX_INT64; - for (int64_t i = 0; OB_SUCC(ret) && i < primary_zone_.count(); ++i) { - const ObZone &zone = primary_zone_.at(i); - int64_t primary_zone_count = 0; - for (int64_t j = 0; OB_SUCC(ret) && j < group_info.ls_ids_.count(); ++j) { - const share::ObLSID &id = group_info.ls_ids_.at(j); - if (OB_FAIL(get_ls_status_info(id, info, info_index))) { - LOG_WARN("failed to find ls info", KR(ret), K(id), K(j)); - } else if (zone == info.primary_zone_) { - primary_zone_count++; - } - }//end for j - if (OB_SUCC(ret)) { - if (OB_INVALID_COUNT == ls_count || ls_count > primary_zone_count) { - ls_count = primary_zone_count; - if (OB_FAIL(primary_zone.assign(zone))) { - LOG_WARN("failed to assign zone", KR(ret), K(zone)); - } else if (0 == primary_zone_count) { - //the first zone has no primary zone - break; - } - } - } - }//end for i - if (OB_SUCC(ret)) { - if (primary_zone.is_empty() || (!is_recovery && 0 != ls_count)) { - ret = OB_ENTRY_NOT_EXIST; - LOG_WARN("failed to find next primary zone", KR(ret), K(primary_zone_), - K(group_info), K(is_recovery), K(ls_count)); - } - } - LOG_INFO("get next primary zone", KR(ret), K(group_info), K(primary_zone)); - } - return ret; -} - -//need create or delete new ls group -int ObTenantLSInfo::check_ls_group_match_unitnum() -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("tenant stat not valid", KR(ret)); - } - for (int64_t i = 0; OB_SUCC(ret) && i < unit_group_array_.count(); ++i) { - const ObUnitGroupInfo &info = unit_group_array_.at(i); - if (share::ObUnit::UNIT_STATUS_MAX == info.unit_status_) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unit status not valid", KR(ret), K(info)); - } else if (share::ObUnit::UNIT_STATUS_DELETING == info.unit_status_) { - // drop logstream, - if (0 == info.ls_group_ids_.count()) { - //nothing - } else if (OB_FAIL(try_drop_ls_of_deleting_unit_group_(info))) { - LOG_WARN("failed to drop ls of the unit group", KR(ret), K(info)); - } - } else if (0 == info.ls_group_ids_.count()) { - //create ls - if (OB_FAIL(create_new_ls_for_empty_unit_group_(info.unit_group_id_))) { - LOG_WARN("failed to create new ls for empty unit_group", KR(ret), - K(info)); - } - } - } - return ret; -} - -int ObTenantLSInfo::create_new_ls_for_empty_unit_group_(const uint64_t unit_group_id) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(OB_INVALID_INDEX_INT64 == unit_group_id)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("group id is invalid", KR(ret), K(unit_group_id)); - } else if (OB_UNLIKELY(!is_valid()) - || OB_ISNULL(tenant_schema_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant ls info not valid", KR(ret), KP(tenant_schema_)); - } else if (OB_UNLIKELY(0 == primary_zone_.count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("primary zone is invalid", KR(ret)); - } else { - const uint64_t tenant_id = tenant_schema_->get_tenant_id(); - share::ObLSID new_id; - share::ObLSStatusInfo new_info; - uint64_t new_ls_group_id = OB_INVALID_INDEX_INT64; - if (OB_FAIL(fetch_new_ls_group_id(tenant_id, new_ls_group_id))) { - LOG_WARN("failed to fetch new id", KR(ret), K(tenant_id)); - } - for (int64_t i = 0; OB_SUCC(ret) && i < primary_zone_.count(); ++i) { - const ObZone &zone = primary_zone_.at(i); - new_info.reset(); - if (OB_FAIL(fetch_new_ls_id(tenant_id, new_id))) { - LOG_WARN("failed to get new id", KR(ret), K(tenant_id)); - } else if (OB_FAIL(new_info.init(tenant_id, new_id, new_ls_group_id, - share::OB_LS_CREATING, unit_group_id, - zone))) { - LOG_WARN("failed to init new info", KR(ret), K(new_id), - K(new_ls_group_id), K(unit_group_id), K(zone), K(tenant_id)); - } else if (OB_FAIL(create_new_ls_(new_info, share::NORMAL_SWITCHOVER_STATUS))) { - LOG_WARN("failed to add ls info", KR(ret), K(new_info)); - } - LOG_INFO("[LS_MGR] create new ls for empty unit group", KR(ret), K(new_info)); - }//end for - } - return ret; -} - -int ObTenantLSInfo::try_drop_ls_of_deleting_unit_group_(const ObUnitGroupInfo &info) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(share::ObUnit::UNIT_STATUS_DELETING != info.unit_status_)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("unit group status is invalid", KR(ret), K(info)); - } else if (OB_UNLIKELY(!is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant ls info not valid", KR(ret)); - } else { - ARRAY_FOREACH_X(info.ls_group_ids_, idx, cnt, OB_SUCC(ret)) { - ObLSGroupInfo ls_group_info; - const int64_t ls_group_id = info.ls_group_ids_.at(idx); - if (OB_FAIL(get_ls_group_info(ls_group_id, ls_group_info))) { - LOG_WARN("failed to get ls group info", KR(ret), K(idx), - K(ls_group_id)); - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < ls_group_info.ls_ids_.count(); ++i) { - const share::ObLSID &ls_id = ls_group_info.ls_ids_.at(i); - share::ObLSStatusInfo info; - int64_t info_index = 0; - if (OB_FAIL(get_ls_status_info(ls_id, info, info_index))) { - LOG_WARN("failed to get ls status info", KR(ret), K(ls_id)); - } else if (OB_FAIL(drop_ls_(info, share::NORMAL_SWITCHOVER_STATUS))) { - LOG_WARN("failed to drop ls", KR(ret), K(info)); - } - LOG_INFO("[LS_MGR] drop ls for deleting unit group", KR(ret), - K(ls_id), K(info)); - }//end for each ls in ls group info - } - }//end for each ls group in unit group info - } - return ret; -} - -int ObTenantLSInfo::drop_ls_(const share::ObLSStatusInfo &info, - const ObTenantSwitchoverStatus &working_sw_status) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("tenant stat not valid", KR(ret)); - } else if (OB_UNLIKELY(!info.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("ls info is invalid", KR(ret), K(info)); - } else if (info.ls_is_normal()) { - //try to set ls to dropping - if (OB_FAIL(ls_operator_.update_ls_status(info.ls_id_, - share::OB_LS_NORMAL, share::OB_LS_DROPPING, working_sw_status))) { - LOG_WARN("failed to update ls status", KR(ret), K(info)); - } - } else if (info.ls_is_created()) { - //if ls_status is created, the ls_info may creating or normal - //creating can not to dropping, so need retry after fix_status_machine. - ret = OB_NEED_RETRY; - LOG_WARN("not certain status in ls info, need retry", KR(ret), K(info)); - } else if (info.ls_is_dropping() - || info.ls_is_wait_offline()) { - //nothing - } else if (info.ls_is_creating()) { - //if status is in creating, the ls must in creating too - if (OB_FAIL(ls_operator_.delete_ls(info.ls_id_, share::OB_LS_CREATING, working_sw_status))) { - LOG_WARN("failed to process creating info", KR(ret), K(info)); - } - } else { - //other ls status not expected - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls status not expected for dropping ls", KR(ret), K(info)); - } - return ret; -} - -//need create or delete new ls -//TODO for primary cluster, The primary_zone of multiple LSs may be the same, -//the redundant ones need to be deleted, and only one is kept. -int ObTenantLSInfo::check_ls_match_primary_zone() -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!is_valid()) || OB_ISNULL(tenant_schema_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant ls info not valid", KR(ret), KP(tenant_schema_)); - } else { - const uint64_t tenant_id = tenant_schema_->get_tenant_id(); - ObZone zone; - share::ObLSID new_id; - share::ObLSStatusInfo new_info; - for (int64_t i = 0; OB_SUCC(ret) && i < ls_group_array_.count(); ++i) { - ObLSGroupInfo &group_info = ls_group_array_.at(i); - //check the unit group is active - //if unit group is in deleting, no need check primary zone - bool is_valid = false; - int64_t group_id_count = group_info.ls_ids_.count(); - if (OB_FAIL(check_unit_group_valid_(group_info.unit_group_id_, is_valid))) { - LOG_WARN("failed to check unit group valid", KR(ret), K(group_info)); - } - while (OB_SUCC(ret) && is_valid - && primary_zone_.count() > group_id_count) { - group_id_count++; - if (OB_FAIL(get_next_primary_zone_(false/*is_recovery*/, group_info, zone))) { - LOG_WARN("failed to get next primary zone", K(group_info)); - } else if (OB_FAIL(fetch_new_ls_id(tenant_id, new_id))) { - LOG_WARN("failed to get new id", KR(ret), K(tenant_id)); - } else if (OB_FAIL(new_info.init(tenant_id, new_id, group_info.ls_group_id_, - share::OB_LS_CREATING, - group_info.unit_group_id_, zone))) { - LOG_WARN("failed to init new info", KR(ret), K(new_id), - K(group_info), K(zone), K(tenant_id)); - } else if (OB_FAIL(create_new_ls_(new_info, share::NORMAL_SWITCHOVER_STATUS))) { - LOG_WARN("failed to create new ls", KR(ret), K(new_info)); - } - LOG_INFO("[LS_MGR] create new ls of primary zone", KR(ret), K(new_info), - K(group_id_count)); - } - } - } - return ret; -} - -int ObTenantLSInfo::get_zone_priority(const ObZone &primary_zone, - const share::schema::ObTenantSchema &tenant_schema, - ObSqlString &primary_zone_str) -{ - int ret = OB_SUCCESS; - primary_zone_str.reset(); - if (OB_UNLIKELY(!tenant_schema.is_valid() || primary_zone.is_empty())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(primary_zone), K(tenant_schema)); - } else if (OB_FAIL(ObPrimaryZoneUtil::get_ls_primary_zone_priority(primary_zone, - tenant_schema, primary_zone_str))) { - LOG_WARN("failed to get ls primary zone priority", KR(ret), K(primary_zone), K(tenant_schema)); - } - LOG_DEBUG("get zone priority", KR(ret), K(primary_zone_str), K(tenant_schema)); - return ret; -} - - -int ObTenantLSInfo::create_new_ls_(const share::ObLSStatusInfo &status_info, - const share::ObTenantSwitchoverStatus &working_sw_status) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("tenant stat not valid", KR(ret)); - } else if (OB_UNLIKELY(!status_info.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("ls info is invalid", KR(ret), K(status_info)); - } else if (OB_ISNULL(sql_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy is null", KR(ret)); - } else { - share::ObLSLifeAgentManager ls_life_agent(*sql_proxy_); - share::ObLSAttr ls_info; - share::ObLSFlag flag = share::OB_LS_FLAG_NORMAL;//TODO - SCN create_scn; - ObSqlString zone_priority; - if (OB_FAIL(ObLSAttrOperator::get_tenant_gts(status_info.tenant_id_, create_scn))) { - LOG_WARN("failed to get tenant gts", KR(ret), K(status_info)); - } else if (OB_FAIL(ls_info.init(status_info.ls_id_, status_info.ls_group_id_, flag, - share::OB_LS_CREATING, share::OB_LS_OP_CREATE_PRE, create_scn))) { - LOG_WARN("failed to init new operation", KR(ret), K(status_info), K(create_scn)); - } else if (OB_FAIL(ls_operator_.insert_ls(ls_info, max_ls_group_id_, working_sw_status))) { - LOG_WARN("failed to insert new operation", KR(ret), K(ls_info), K(max_ls_group_id_)); - } else if (OB_FAIL(get_zone_priority(status_info.primary_zone_, - *tenant_schema_, zone_priority))) { - LOG_WARN("failed to get normalize primary zone", KR(ret), K(status_info), - K(zone_priority), K(tenant_schema_)); - } else if (OB_FAIL(ls_life_agent.create_new_ls(status_info, create_scn, - zone_priority.string(), working_sw_status))) { - LOG_WARN("failed to create new ls", KR(ret), K(status_info), K(create_scn), - K(zone_priority)); - } else if (OB_FAIL(do_create_ls_(status_info, create_scn))) { - LOG_WARN("failed to create ls", KR(ret), K(status_info), K(create_scn)); - } else if (OB_FAIL(process_ls_status_after_created_(status_info, working_sw_status))) { - LOG_WARN("failed to update ls status", KR(ret), K(status_info)); - } - LOG_INFO("[LS_MGR] create new ls", KR(ret), K(status_info)); - } - return ret; -} - -int ObTenantLSInfo::fetch_new_ls_group_id( - const uint64_t tenant_id, - uint64_t &ls_group_id) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("tenant id is invalid", KR(ret), K(tenant_id)); - } else if (OB_ISNULL(sql_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy is null", KR(ret)); - } else { - ls_group_id = OB_INVALID_ID; - share::ObMaxIdFetcher id_fetcher(*sql_proxy_); - if (OB_FAIL(id_fetcher.fetch_new_max_id( - tenant_id, share::OB_MAX_USED_LS_GROUP_ID_TYPE, ls_group_id))) { - LOG_WARN("fetch_max_id failed", KR(ret), K(tenant_id), "id_type", - share::OB_MAX_USED_LS_GROUP_ID_TYPE); - } else { - if (ls_group_id != ++max_ls_group_id_) { - ret = OB_EAGAIN; - LOG_WARN("ls may create concurrency, need retry", KR(ret), K(ls_group_id), - K(max_ls_group_id_)); - } - } - } - return ret; -} - -int ObTenantLSInfo::fetch_new_ls_id(const uint64_t tenant_id, share::ObLSID &id) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("tenant id is invalid", KR(ret), K(tenant_id)); - } else if (OB_ISNULL(sql_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy is null", KR(ret)); - } else { - share::ObMaxIdFetcher id_fetcher(*sql_proxy_); - uint64_t id_value = OB_INVALID_ID; - if (OB_FAIL(id_fetcher.fetch_new_max_id( - tenant_id, share::OB_MAX_USED_LS_ID_TYPE, id_value))) { - LOG_WARN("fetch_max_id failed", KR(ret), K(tenant_id), "id_type", - share::OB_MAX_USED_LS_ID_TYPE); - } else { - if (id_value != ++max_ls_id_) { - ret = OB_EAGAIN; - LOG_WARN("ls may create concurrency, need retry", KR(ret), K(id_value), - K(max_ls_id_)); - } else { - share::ObLSID new_id(id_value); - id = new_id; - } - } - } - return ret; -} - -int ObTenantLSInfo::create_ls_with_palf( - const share::ObLSStatusInfo &info, - const SCN &create_scn, - const bool create_ls_with_palf, - const palf::PalfBaseInfo &palf_base_info) -{ - int ret = OB_SUCCESS; - LOG_INFO("[LS_EXEC] start to create ls", K(info)); - const int64_t start_time = ObTimeUtility::fast_current_time(); - if (OB_UNLIKELY(!info.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("info not valid", KR(ret), K(info)); - } else if (OB_ISNULL(rpc_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("rpc proxy is null", KR(ret)); - } else if (OB_UNLIKELY(!info.ls_is_created() && !info.ls_is_creating())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("info status not expected", KR(ret), K(info)); - } else if (info.ls_is_creating()) { - common::ObArray locality_array; - int64_t paxos_replica_num = 0; - ObSchemaGetterGuard guard;//nothing - if (OB_ISNULL(tenant_schema_)) { - ret = OB_TENANT_NOT_EXIST; - LOG_WARN("tenant not exist", KR(ret), K(info)); - } else if (tenant_schema_->is_dropping()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant is in dropping, no need create ls", KR(ret), K(tenant_schema_)); - } else if (OB_FAIL(tenant_schema_->get_zone_replica_attr_array( - locality_array))) { - LOG_WARN("failed to get zone locality array", KR(ret)); - } else if (OB_FAIL(tenant_schema_->get_paxos_replica_num( - guard, paxos_replica_num))) { - LOG_WARN("failed to get paxos replica num", KR(ret)); - } else { - ObLSCreator creator(*rpc_proxy_, info.tenant_id_, - info.ls_id_, sql_proxy_); - if (OB_FAIL(creator.create_user_ls(info, paxos_replica_num, - locality_array, create_scn, - tenant_schema_->get_compatibility_mode(), - create_ls_with_palf, - palf_base_info))) { - LOG_WARN("failed to create user ls", KR(ret), K(info), K(locality_array), K(create_scn), - K(palf_base_info), K(create_ls_with_palf)); - } - } - } - const int64_t cost = ObTimeUtility::fast_current_time() - start_time; - LOG_INFO("[LS_EXEC] end to create ls", KR(ret), K(info), K(cost)); - return ret; -} - -int ObTenantLSInfo::do_create_ls_(const share::ObLSStatusInfo &info, - const SCN &create_scn) -{ - int ret = OB_SUCCESS; - const int64_t start_time = ObTimeUtility::fast_current_time(); - if (OB_UNLIKELY(!info.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("info not valid", KR(ret), K(info)); - } else { - palf::PalfBaseInfo palf_base_info; - if (OB_FAIL(create_ls_with_palf(info, create_scn, false, palf_base_info))) { - LOG_WARN("failed to create ls with palf", KR(ret), K(info), K(create_scn), K(palf_base_info)); - } - } - return ret; -} - -int ObTenantLSInfo::process_ls_status_after_created_(const share::ObLSStatusInfo &status_info, - const ObTenantSwitchoverStatus &working_sw_status) -{ - //do not check ls status - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!status_info.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("info not valid", KR(ret), K(status_info)); - } else if (OB_ISNULL(sql_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy is null", KR(ret)); - } else if (OB_FAIL(ls_operator_.update_ls_status( - status_info.ls_id_, share::OB_LS_CREATING, share::OB_LS_NORMAL, working_sw_status))) { - LOG_WARN("failed to update ls status", KR(ret), K(status_info)); - } else if (OB_FAIL(status_operator_.update_ls_status( - status_info.tenant_id_, status_info.ls_id_, share::OB_LS_CREATED, - share::OB_LS_NORMAL, working_sw_status, *sql_proxy_))) { - LOG_WARN("failed to update ls status", KR(ret), K(status_info)); - } - return ret; -} - -int ObTenantLSInfo::do_tenant_drop_ls_(const share::ObLSStatusInfo &status_info, - const ObTenantSwitchoverStatus &working_sw_status) -{ - int ret = OB_SUCCESS; - LOG_INFO("[LS_EXEC] start to tenant drop ls", K(status_info)); - const int64_t start_time = ObTimeUtility::fast_current_time(); - bool can_offline = false; - if (OB_UNLIKELY(!status_info.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("info not valid", KR(ret), K(status_info)); - } else if (OB_ISNULL(sql_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy is null", KR(ret)); - } else if (OB_UNLIKELY(status_info.ls_is_creating() || status_info.ls_is_wait_offline())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("status info not expected", KR(ret), K(status_info)); - } else if (!status_info.ls_is_tenant_dropping()) { - //status_info may in created, normal, dropping, tenant_dropping - if (OB_FAIL(status_operator_.update_ls_status( - status_info.tenant_id_, status_info.ls_id_, - status_info.status_, share::OB_LS_TENANT_DROPPING, working_sw_status, - *sql_proxy_))) { - LOG_WARN("failed to update ls status", KR(ret), K(status_info)); - } - } - if (FAILEDx(check_ls_can_offline_by_rpc_(status_info, share::OB_LS_TENANT_DROPPING, can_offline))) { - // send rpc to observer - LOG_WARN("failed to check ls can offline", KR(ret), K(status_info)); - } else if (can_offline) { - ObLSLifeAgentManager ls_life_agent(*sql_proxy_); - SCN drop_scn; - if (!status_info.ls_id_.is_sys_ls()) { - //sys ls cannot delete ls, after ls is in tenant dropping - if (OB_FAIL(ls_operator_.delete_ls(status_info.ls_id_, share::OB_LS_TENANT_DROPPING, - working_sw_status))) { - LOG_WARN("failed to delete ls", KR(ret), K(status_info)); - } else if (OB_FAIL(ObLSAttrOperator::get_tenant_gts(status_info.tenant_id_, drop_scn))) { - LOG_WARN("failed to get gts", KR(ret), K(status_info)); - } - } else { - //TODO sys ls can not get GTS after tenant_dropping - drop_scn.set_base(); - } - if (FAILEDx(ls_life_agent.set_ls_offline(status_info.tenant_id_, - status_info.ls_id_, status_info.status_, drop_scn, working_sw_status))) { - LOG_WARN("failed to update ls info", KR(ret), K(status_info), K(drop_scn)); - } - } - const int64_t cost = ObTimeUtility::fast_current_time() - start_time; - LOG_INFO("[LS_EXEC] end to tenant drop ls", KR(ret), K(status_info), K(cost)); - return ret; -} - -int ObTenantLSInfo::do_drop_ls_(const share::ObLSStatusInfo &status_info, - const ObTenantSwitchoverStatus &working_sw_status) -{ - int ret = OB_SUCCESS; - LOG_INFO("[LS_EXEC] start to drop ls", K(status_info)); - const int64_t start_time = ObTimeUtility::fast_current_time(); - bool tablet_empty = false; - if (OB_UNLIKELY(!status_info.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("info not valid", KR(ret), K(status_info)); - } else if (OB_ISNULL(sql_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy is null", KR(ret)); - } else if (status_info.ls_is_normal()) { - if (OB_FAIL(status_operator_.update_ls_status( - status_info.tenant_id_, status_info.ls_id_, - status_info.status_, share::OB_LS_DROPPING, - working_sw_status, - *sql_proxy_))) { - LOG_WARN("failed to update ls status", KR(ret), K(status_info)); - } - } else if (status_info.ls_is_dropping()) { - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("status not expected", KR(ret), K(status_info)); - } - if (FAILEDx(check_ls_empty_(status_info, tablet_empty))) { - LOG_WARN("failed to check ls empty", KR(ret), K(status_info)); - } - if (OB_SUCC(ret) && tablet_empty) { - // send rpc to observer - bool can_offline = false; - if (OB_FAIL(check_ls_can_offline_by_rpc_(status_info, share::OB_LS_DROPPING, can_offline))) { - LOG_WARN("failed to check ls can offline", KR(ret), K(status_info)); - } else if (can_offline) { - ObLSLifeAgentManager ls_life_agent(*sql_proxy_); - SCN drop_scn; - if (OB_FAIL(ls_operator_.delete_ls(status_info.ls_id_, share::OB_LS_DROPPING, - working_sw_status))) { - LOG_WARN("failed to delete ls", KR(ret), K(status_info)); - } else if (OB_FAIL(ObLSAttrOperator::get_tenant_gts(status_info.tenant_id_, drop_scn))) { - LOG_WARN("failed to get gts", KR(ret), K(status_info)); - } else if (OB_FAIL(ls_life_agent.set_ls_offline(status_info.tenant_id_, - status_info.ls_id_, status_info.status_, drop_scn, working_sw_status))) { - LOG_WARN("failed to update ls info", KR(ret), K(status_info), K(drop_scn)); - } - } - } - const int64_t cost = ObTimeUtility::fast_current_time() - start_time; - LOG_INFO("[LS_EXEC] end to drop ls", KR(ret), K(status_info), K(cost), K(tablet_empty)); - return ret; -} - - -int ObTenantLSInfo::sys_ls_tenant_drop_(const share::ObLSStatusInfo &info, - const share::ObTenantSwitchoverStatus &working_sw_status) -{ - int ret = OB_SUCCESS; - const ObLSStatus target_status = share::OB_LS_TENANT_DROPPING; - const ObLSStatus pre_status = share::OB_LS_PRE_TENANT_DROPPING; - if (OB_UNLIKELY(!info.is_valid() - || !info.ls_id_.is_sys_ls())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(info)); - } else if (OB_ISNULL(sql_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy is null", KR(ret), KP(sql_proxy_)); - } else if (info.ls_is_normal()) { - if (OB_FAIL(status_operator_.update_ls_status(info.tenant_id_, - info.ls_id_, info.status_, pre_status, working_sw_status, - *sql_proxy_))) { - LOG_WARN("failed to update ls status", KR(ret), K(info), K(pre_status)); - } - } else if (pre_status == info.status_) { - //nothing - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sys ls can not in other status", KR(ret), K(info)); - } - bool can_offline = false; - if (FAILEDx(check_sys_ls_can_offline_(can_offline))) { - LOG_WARN("failed to check sys ls can offline", KR(ret)); - } else if (can_offline) { - if (OB_FAIL(ls_operator_.update_ls_status(info.ls_id_, pre_status, target_status, - working_sw_status))) { - LOG_WARN("failed to update ls status", KR(ret), K(info), K(pre_status), K(target_status)); - } else if (OB_FAIL(status_operator_.update_ls_status(info.tenant_id_, - info.ls_id_, pre_status, target_status, working_sw_status, *sql_proxy_))) { - LOG_WARN("failed to update ls status", KR(ret), K(info), K(pre_status), K(target_status)); - } - } - return ret; -} - -int ObTenantLSInfo::check_sys_ls_can_offline_(bool &can_offline) -{ - int ret = OB_SUCCESS; - share::ObLSStatusInfoArray status_info_array; - can_offline = true; - if (OB_ISNULL(sql_proxy_) || OB_ISNULL(tenant_schema_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy or tenant schema is null", KR(ret), KP(tenant_schema_), KP(sql_proxy_)); - } else { - const uint64_t tenant_id = tenant_schema_->get_tenant_id(); - if (OB_FAIL(status_operator_.get_all_ls_status_by_order( - tenant_id, status_info_array, *sql_proxy_))) { - LOG_WARN("failed to update ls status", KR(ret), K(tenant_id)); - } else if (0 == status_info_array.count()) { - //if has multi ls_mgr - can_offline = true; - } - for (int64_t i = 0; OB_SUCC(ret) && i < status_info_array.count() && can_offline; ++i) { - const share::ObLSStatusInfo &status_info = status_info_array.at(i); - if (status_info.ls_id_.is_sys_ls()) { - } else if (status_info.ls_is_wait_offline()) { - //nothing - } else { - can_offline = false; - LOG_INFO("[LS_MGR] sys ls can not offline", K(status_info)); - break; - } - } - if (OB_SUCC(ret) && can_offline) { - LOG_INFO("[LS_MGR] sys ls can offline", K(status_info_array)); - } - } - return ret; -} - -int ObTenantLSInfo::check_ls_empty_(const share::ObLSStatusInfo &info, bool &empty) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!info.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("info not valid", KR(ret), K(info)); - } else { - ObSqlString sql; - if (OB_FAIL(sql.assign_fmt( - "SELECT * FROM %s where ls_id = %ld", - OB_ALL_TABLET_TO_LS_TNAME, info.ls_id_.id()))) { - LOG_WARN("failed to assign sql", KR(ret), K(sql)); - } else if (OB_ISNULL(sql_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy is null", KR(ret)); - } else { - HEAP_VAR(ObMySQLProxy::MySQLResult, res) { - common::sqlclient::ObMySQLResult *result = NULL; - if (OB_FAIL(sql_proxy_->read(res, info.tenant_id_, sql.ptr()))) { - LOG_WARN("failed to read", KR(ret), K(info), K(sql)); - } else if (OB_ISNULL(result = res.get_result())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to get sql result", KR(ret)); - } else if (OB_SUCC(result->next())) { - empty = false; - } else if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - empty = true; - } else { - LOG_WARN("failed to get next", KR(ret), K(sql)); - } - } - } - } - return ret; -} - -int ObTenantLSInfo::check_ls_can_offline_by_rpc_(const share::ObLSStatusInfo &info, - const share::ObLSStatus ¤t_ls_status, bool &can_offline) -{ - int ret = OB_SUCCESS; - share::ObLSInfo ls_info; - const share::ObLSReplica *replica = NULL; - if (OB_UNLIKELY(!info.is_valid() - || !ls_is_tenant_dropping_status(current_ls_status) && ! ls_is_dropping_status(current_ls_status))) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("info not valid", KR(ret), K(info), K(current_ls_status)); - } else if (OB_ISNULL(lst_operator_) || OB_ISNULL(rpc_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("lst operator or proxy is null", KR(ret), KP(lst_operator_), - KP(rpc_proxy_)); - } else if (OB_FAIL(lst_operator_->get(GCONF.cluster_id, info.tenant_id_, - info.ls_id_, share::ObLSTable::DEFAULT_MODE, ls_info))) { - LOG_WARN("failed to get ls info", KR(ret), K(info)); - } else if (OB_FAIL(ls_info.find_leader(replica))) { - LOG_WARN("failed to find leader", KR(ret), K(ls_info)); - } else if (OB_ISNULL(replica)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("replica is null", KR(ret), K(ls_info)); - } else { - const int64_t timeout = GCONF.rpc_timeout; - obrpc::ObCheckLSCanOfflineArg arg; - if (OB_FAIL(arg.init(info.tenant_id_, info.ls_id_, current_ls_status))) { - LOG_WARN("failed to init arg", KR(ret), K(info), K(current_ls_status)); - } else if (OB_FAIL(rpc_proxy_->to(replica->get_server()) - .by(info.tenant_id_) - .timeout(timeout) - .check_ls_can_offline(arg))) { - can_offline = false; - LOG_WARN("failed to check ls can offline", KR(ret), K(arg), K(info), - K(timeout), K(replica)); - } else { - can_offline = true; - } - } - return ret; -} -int ObTenantLSInfo::adjust_user_tenant_primary_zone() -{ - int ret = OB_SUCCESS; - share::ObLSStatusOperator status_op; - share::ObLSPrimaryZoneInfoArray info_array; - ObArray primary_zone; - if (OB_ISNULL(sql_proxy_) || OB_ISNULL(tenant_schema_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error", KR(ret), KP(sql_proxy_), KP(tenant_schema_)); - } else if (OB_FAIL(ObPrimaryZoneUtil::get_tenant_primary_zone_array( - *tenant_schema_, primary_zone))) { - LOG_WARN("failed to get tenant primary zone array", KR(ret), KPC(tenant_schema_)); - } else { - const uint64_t tenant_id = tenant_schema_->get_tenant_id(); - if (OB_FAIL(status_op.get_tenant_primary_zone_info_array( - tenant_id, info_array, *sql_proxy_))) { - LOG_WARN("failed to get tenant primary zone info array", KR(ret), K(tenant_id)); - } - } - - if (OB_SUCC(ret)) { - uint64_t last_ls_group_id = OB_INVALID_ID; - share::ObLSPrimaryZoneInfoArray tmp_info_array; - for (int64_t i = 0; OB_SUCC(ret) && i < info_array.count(); ++i) { - const ObLSPrimaryZoneInfo &info = info_array.at(i); - if (OB_INVALID_ID == last_ls_group_id) { - last_ls_group_id = info.get_ls_group_id(); - } - if (last_ls_group_id != info.get_ls_group_id()) { - //process the ls group - if (OB_FAIL(adjust_primary_zone_by_ls_group_(primary_zone, tmp_info_array, *tenant_schema_))) { - LOG_WARN("failed to update primary zone of each ls group", KR(ret), - K(tmp_info_array), K(primary_zone), KPC(tenant_schema_)); - } else { - tmp_info_array.reset(); - last_ls_group_id = info.get_ls_group_id(); - } - } - if (FAILEDx(tmp_info_array.push_back(info))) { - LOG_WARN("failed to push back primary info array", KR(ret), K(i)); - } - } - if (OB_SUCC(ret) && 0 < tmp_info_array.count()) { - if (OB_FAIL(adjust_primary_zone_by_ls_group_(primary_zone, tmp_info_array, *tenant_schema_))) { - LOG_WARN("failed to update primary zone of each ls group", KR(ret), - K(tmp_info_array), K(primary_zone), KPC(tenant_schema_)); - } - } - } - - return ret; -} - - -int ObTenantLSInfo::try_update_ls_primary_zone_( - const share::ObLSPrimaryZoneInfo &primary_zone_info, - const common::ObZone &new_primary_zone, - const common::ObSqlString &zone_priority) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!primary_zone_info.is_valid() - || new_primary_zone.is_empty() || zone_priority.empty())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("primary zone info is invalid", KR(ret), K(primary_zone_info), - K(new_primary_zone), K(zone_priority)); - } else if (OB_ISNULL(GCTX.sql_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy is null", KR(ret)); - } else if (new_primary_zone != primary_zone_info.get_primary_zone() - || zone_priority.string() != primary_zone_info.get_zone_priority_str()) { - ObLSLifeAgentManager ls_life_agent(*GCTX.sql_proxy_); - if (OB_FAIL(ls_life_agent.update_ls_primary_zone(primary_zone_info.get_tenant_id(), - primary_zone_info.get_ls_id(), - new_primary_zone, zone_priority.string()))) { - LOG_WARN("failed to update ls primary zone", KR(ret), K(primary_zone_info), - K(new_primary_zone), K(zone_priority)); - } - LOG_INFO("[LS_MGR] update ls primary zone", KR(ret), K(new_primary_zone), - K(zone_priority), K(primary_zone_info)); - } else { - //no need update - } - return ret; -} - -//check every ls has right primary zone in ls group -//if primary_zone of ls not in primary zone, try to choose a right zone, -//if can not find a zone to modify, modify the ls to exist zone, -//load_balancer will choose the right ls to drop -//eg: -//if ls group has z1, z2 and primary zone is : z1. need modify z2 to z1 -//if ls group has z1, z2 and primary zone is : z1, z3. modify z2 to z3; -int ObTenantLSInfo::adjust_primary_zone_by_ls_group_( - const common::ObIArray &primary_zone_array, - const share::ObLSPrimaryZoneInfoArray &primary_zone_infos, - const share::schema::ObTenantSchema &tenant_schema) -{ - int ret = OB_SUCCESS; - const uint64_t tenant_id = tenant_schema.get_tenant_id(); - const int64_t ls_count = primary_zone_infos.count(); - const int64_t primary_zone_count = primary_zone_array.count(); - if (OB_UNLIKELY(0 == primary_zone_count - || 0 == ls_count || !tenant_schema.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(primary_zone_infos), - K(primary_zone_array), K(tenant_schema)); - } else if (OB_ISNULL(GCTX.sql_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy is null", KR(ret)); - } else if (1 == ls_count - && primary_zone_infos.at(0).get_ls_id().is_sys_ls()) { - //user sys ls is equal to meta sys ls - share::ObLSStatusOperator status_op; - share::ObLSPrimaryZoneInfo meta_primary_zone; - const uint64_t meta_tenant_id = gen_meta_tenant_id(tenant_id); - if (OB_FAIL(status_op.get_ls_primary_zone_info(meta_tenant_id, - SYS_LS, meta_primary_zone, *GCTX.sql_proxy_))) { - LOG_WARN("failed to get ls primary_zone info", KR(ret), K(meta_tenant_id)); - } else if (OB_FAIL(try_update_ls_primary_zone_(primary_zone_infos.at(0), - meta_primary_zone.get_primary_zone(), meta_primary_zone.get_zone_priority()))) { - LOG_WARN("failed to update primary zone", KR(ret), K(primary_zone_infos), K(meta_primary_zone)); - } - } else { - //Algorithm Description: - //Assumption: We have 5 ls, 3 primary_zones(z1, z2, z3). - //1. Set the primary zone of ls to tenant's primary zone, - //choose the least number of log streams on the zone is selected for all zones - //2. After all the primary_zone of the ls are in the primary_zonw of the tenant, - //choose the primary_zone with the most and least ls. Adjust a certain number of ls to the smallest zone without exceeding the balance - //while guaranteeing that the number of the zone with the most is no less than the average. - ObArray ls_primary_zone;//is match with primary_zone_infos - ObSEArray count_group_by_zone;//ls count of each primary zone - ObSqlString new_zone_priority; - if (OB_FAIL(set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, - count_group_by_zone))) { - LOG_WARN("failed to set ls to primary zone", KR(ret), K(primary_zone_array), K(primary_zone_infos)); - } else if (OB_FAIL(balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone))) { - LOG_WARN("failed to balance ls primary zone", KR(ret), K(ls_primary_zone), K(count_group_by_zone)); - } - - for (int64_t i = 0; OB_SUCC(ret) && i < ls_count; ++i) { - const ObZone &new_primary_zone = ls_primary_zone.at(i); - if (OB_FAIL(ObTenantLSInfo::get_zone_priority(new_primary_zone, tenant_schema, new_zone_priority))) { - LOG_WARN("failed to get normalize primary zone", KR(ret), K(new_primary_zone)); - } - if (FAILEDx(try_update_ls_primary_zone_(primary_zone_infos.at(i), new_primary_zone, new_zone_priority))) { - LOG_WARN("failed to update ls primary zone", KR(ret), "primary_zone_info", primary_zone_infos.at(i), - K(new_primary_zone), K(new_zone_priority)); - } - } - } - return ret; -} - -int ObTenantLSInfo::set_ls_to_primary_zone( - const common::ObIArray &primary_zone_array, - const share::ObLSPrimaryZoneInfoArray &primary_zone_infos, - common::ObIArray &ls_primary_zone, - common::ObIArray &count_group_by_zone) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(0 == primary_zone_array.count() - || 0 == primary_zone_infos.count())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(primary_zone_array), K(primary_zone_infos)); - } else { - const int64_t primary_zone_count = primary_zone_array.count(); - const int64_t ls_count = primary_zone_infos.count(); - //ls may not in primary zone, record the index of primary_zone_infos not in primary zone - ObSEArray index_not_primary_zone; - int64_t index = 0; - ARRAY_FOREACH_X(primary_zone_array, idx, cnt, OB_SUCC(ret)) { - if (OB_FAIL(count_group_by_zone.push_back(0))) { - LOG_WARN("failed to push back", KR(ret), K(idx)); - } - } - for (int64_t i = 0; OB_SUCC(ret) && i < ls_count; ++i) { - const ObZone ¤t_zone = primary_zone_infos.at(i).get_primary_zone(); - if (has_exist_in_array(primary_zone_array, current_zone, &index)) { - count_group_by_zone.at(index)++; - } else if (OB_FAIL(index_not_primary_zone.push_back(i))) { - LOG_WARN("failed to push back", KR(ret), K(i), K(current_zone)); - } - if (FAILEDx(ls_primary_zone.push_back(current_zone))) { - LOG_WARN("failed to push back current zone", KR(ret), K(i), K(current_zone)); - } - } - //1. take all ls primary zone to tenant primary zone, choose the less primary zone count - int64_t min_count = INT64_MAX; - int64_t min_index = 0; - for (int64_t i = 0; OB_SUCC(ret) && i < index_not_primary_zone.count(); ++i) { - const int64_t ls_index = index_not_primary_zone.at(i); - min_count = INT64_MAX; - ARRAY_FOREACH_X(primary_zone_array, idx, cnt, OB_SUCC(ret)) { - if (min_count > count_group_by_zone.at(idx)) { - min_count = count_group_by_zone.at(idx); - min_index = idx; - } - }//end for search min count - if (OB_FAIL(ret)) { - } else if (min_index >= primary_zone_count) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to found min count", KR(ret), K(min_index), K(primary_zone_array)); - } else if (OB_FAIL(ls_primary_zone.at(ls_index).assign(primary_zone_array.at(min_index)))) { - LOG_WARN("failed to assign primary zone", KR(ret), K(min_index), K(min_count), K(ls_index)); - } else { - count_group_by_zone.at(min_index)++; - } - } - } - return ret; - -} - -int ObTenantLSInfo::balance_ls_primary_zone( - const common::ObIArray &primary_zone_array, - common::ObIArray &ls_primary_zone, - common::ObIArray &count_group_by_zone) -{ - int ret = OB_SUCCESS; - const int64_t ls_count = ls_primary_zone.count(); - const int64_t primary_zone_count = count_group_by_zone.count(); - if (OB_UNLIKELY(0 == primary_zone_count - || 0 == ls_count - || primary_zone_count != primary_zone_array.count())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(count_group_by_zone), K(ls_primary_zone)); - } else { - int64_t max_count = -1, max_index = 0; - int64_t min_count = INT64_MAX, min_index = 0; - const int64_t each_ls_max_count = ceil((double)(ls_count)/primary_zone_count); - do { - max_count = -1, max_index = 0; - min_count = INT64_MAX, min_index = 0; - for (int64_t i = 0; OB_SUCC(ret) && i < primary_zone_count; ++i) { - const int64_t ls_count = count_group_by_zone.at(i); - if (min_count > ls_count) { - min_count = ls_count; - min_index = i; - } - if (max_count < ls_count) { - max_count = ls_count; - max_index = i; - } - }//end for find min and max count - if (OB_UNLIKELY(max_index >= primary_zone_count || min_index >= primary_zone_count - || -1 == max_count || INT64_MAX == min_count)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to get primary zone", KR(ret), K(max_index), K(min_index), - K(min_count), K(max_count), - K(primary_zone_array), K(primary_zone_count), K(ls_primary_zone)); - } else if (max_count - min_count > 1) { - //choose the max count to min count - int64_t need_change = min(each_ls_max_count - min_count, max_count - each_ls_max_count); - for (int64_t i = 0; OB_SUCC(ret) && i < ls_count && need_change > 0; ++i) { - if (ls_primary_zone.at(i) == primary_zone_array.at(max_index)) { - if (OB_FAIL(ls_primary_zone.at(i).assign(primary_zone_array.at(min_index)))) { - LOG_WARN("failed to push back ls primary zone", KR(ret), K(min_index)); - } else { - need_change--; - count_group_by_zone.at(max_index)--; - count_group_by_zone.at(min_index)++; - } - } - } - } - } while (max_count - min_count > 1); - } - return ret; -} - - -//////////////ObTenantThreadHelper -int ObTenantThreadHelper::create( - const char* thread_name, int tg_def_id, ObTenantThreadHelper &tenant_thread) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(is_created_)) { - ret = OB_INIT_TWICE; - LOG_WARN("has inited", KR(ret)); - } else if (OB_ISNULL(thread_name)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("thread name is null", KR(ret)); - } else if (OB_FAIL(TG_CREATE_TENANT(tg_def_id, tg_id_))) { - LOG_ERROR("create tg failed", KR(ret)); - } else if (OB_FAIL(TG_SET_RUNNABLE(tg_id_, *this))) { - LOG_ERROR("set thread runable fail", KR(ret)); - } else if (OB_FAIL(thread_cond_.init(ObWaitEventIds::REENTRANT_THREAD_COND_WAIT))) { - LOG_WARN("fail to init cond, ", KR(ret)); - } else { - thread_name_ = thread_name; - is_created_ = true; - is_first_time_to_start_ = true; - } - return ret; -} - -int ObTenantThreadHelper::start() -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!is_created_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret)); - } else if (is_first_time_to_start_) { - if (OB_FAIL(TG_START(tg_id_))) { - LOG_WARN("fail ed to start at first time", KR(ret), K(tg_id_), K(thread_name_)); - } else { - is_first_time_to_start_ = false; - } - } else if (OB_FAIL(TG_REENTRANT_LOGICAL_START(tg_id_))) { - LOG_WARN("failed to start", KR(ret)); - } - LOG_INFO("[TENANT THREAD] thread start", KR(ret), K(tg_id_), K(thread_name_)); - return ret; -} - -void ObTenantThreadHelper::stop() -{ - LOG_INFO("[TENANT THREAD] thread stop start", K(tg_id_), K(thread_name_)); - if (-1 != tg_id_) { - TG_REENTRANT_LOGICAL_STOP(tg_id_); - } - LOG_INFO("[TENANT THREAD] thread stop finish", K(tg_id_), K(thread_name_)); -} - -void ObTenantThreadHelper::wait() -{ - LOG_INFO("[TENANT THREAD] thread wait start", K(tg_id_), K(thread_name_)); - if (-1 != tg_id_) { - TG_REENTRANT_LOGICAL_WAIT(tg_id_); - } - LOG_INFO("[TENANT THREAD] thread wait finish", K(tg_id_), K(thread_name_)); -} - -void ObTenantThreadHelper::destroy() -{ - LOG_INFO("[TENANT THREAD] thread destory start", K(tg_id_), K(thread_name_)); - if (-1 != tg_id_) { - TG_STOP(tg_id_); - { - ObThreadCondGuard guard(thread_cond_); - thread_cond_.broadcast(); - } - TG_WAIT(tg_id_); - TG_DESTROY(tg_id_); - tg_id_ = -1; - } - is_created_ = false; - is_first_time_to_start_ = true; - LOG_INFO("[TENANT THREAD] thread destory finish", K(tg_id_), K(thread_name_)); -} - -void ObTenantThreadHelper::switch_to_follower_forcedly() -{ - stop(); -} -int ObTenantThreadHelper::switch_to_leader() -{ - int ret = OB_SUCCESS; - LOG_INFO("[TENANT THREAD] thread start", K(tg_id_), K(thread_name_)); - if (OB_FAIL(start())) { - LOG_WARN("failed to start thread", KR(ret)); - } else { - ObThreadCondGuard guard(thread_cond_); - if (OB_FAIL(thread_cond_.broadcast())) { - LOG_WARN("failed to weakup thread cond", KR(ret)); - } - } - LOG_INFO("[TENANT THREAD] thread start finish", K(tg_id_), K(thread_name_)); - return ret; -} -void ObTenantThreadHelper::run1() -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!is_created_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", K(ret)); - } else { - lib::set_thread_name(thread_name_); - LOG_INFO("thread run", K(thread_name_)); - do_work(); - } -} -void ObTenantThreadHelper::idle(const int64_t idle_time_us) -{ - ObThreadCondGuard guard(thread_cond_); - thread_cond_.wait_us(idle_time_us); -} //////////////ObPrimaryLSService int ObPrimaryLSService::mtl_init(ObPrimaryLSService *&ka) @@ -2048,7 +46,7 @@ int ObPrimaryLSService::init() ret = OB_INIT_TWICE; LOG_WARN("has inited", KR(ret)); } else if (OB_FAIL(ObTenantThreadHelper::create("PLSSer", - lib::TGDefIDs::SimpleLSService, *this))) { + lib::TGDefIDs::LSService, *this))) { LOG_WARN("failed to create thread", KR(ret)); } else if (OB_FAIL(ObTenantThreadHelper::start())) { LOG_WARN("fail to start", KR(ret)); @@ -2071,134 +69,49 @@ void ObPrimaryLSService::do_work() if (OB_UNLIKELY(!inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); + } else if (OB_FAIL(wait_tenant_schema_and_version_ready_(tenant_id_, DATA_VERSION_4_1_0_0))) { + LOG_WARN("failed to wait tenant schema version ready", KR(ret), K(tenant_id_), K(DATA_CURRENT_VERSION)); } else { int64_t idle_time_us = 100 * 1000L; int tmp_ret = OB_SUCCESS; + const uint64_t thread_idx = get_thread_idx(); + share::schema::ObTenantSchema tenant_schema; + palf::PalfBufferIterator iterator; + SCN start_scn; while (!has_set_stop()) { - idle_time_us = 1000 * 1000L; - { - ObCurTraceId::init(GCONF.self_addr_); - share::schema::ObSchemaGetterGuard schema_guard; - const share::schema::ObTenantSchema *tenant_schema = NULL; - if (OB_ISNULL(GCTX.schema_service_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error", KR(ret)); - } else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard( - OB_SYS_TENANT_ID, schema_guard))) { - LOG_WARN("fail to get schema guard", KR(ret)); - } else if (OB_FAIL(schema_guard.get_tenant_info(tenant_id_, tenant_schema))) { - LOG_WARN("failed to get tenant ids", KR(ret), K(tenant_id_)); - } else if (OB_ISNULL(tenant_schema)) { - ret = OB_TENANT_NOT_EXIST; - LOG_WARN("tenant not exist", KR(ret), K(tenant_id_)); - } else if (!is_user_tenant(tenant_id_)) { - if (OB_SUCCESS != (tmp_ret = process_meta_tenant_(*tenant_schema))) { - ret = OB_SUCC(ret) ? tmp_ret : ret; - LOG_WARN("failed to process tenant", KR(ret), KR(tmp_ret), K(tenant_id_)); - } - //drop tenant, no need process sys tenant, ignore failuer - if (!is_sys_tenant(tenant_id_)) { - const uint64_t user_tenant_id = gen_user_tenant_id(tenant_id_); - if (OB_SUCCESS != (tmp_ret = try_force_drop_tenant_(user_tenant_id))) { - ret = OB_SUCC(ret) ? tmp_ret : ret; - LOG_WARN("failed to drop tenant", KR(ret), KR(tmp_ret), K(user_tenant_id)); - } - } - } else { - if (OB_SUCCESS !=( tmp_ret = process_user_tenant_(*tenant_schema))) { - ret = OB_SUCC(ret) ? tmp_ret : ret; - LOG_WARN("failed to process tenant", KR(ret), KR(tmp_ret), K(tenant_id_)); - } - - if (OB_SUCCESS != (tmp_ret = report_sys_ls_recovery_stat_())) { - //ignore error of report, no need wakeup - LOG_WARN("failed to report sys ls recovery stat", KR(ret), KR(tmp_ret), K(tenant_id_)); - } + tenant_schema.reset(); + ObCurTraceId::init(GCONF.self_addr_); + if (0 == thread_idx) { + if (OB_FAIL(get_tenant_schema(tenant_id_, tenant_schema))) { + LOG_WARN("failed to get tenant schema", KR(ret), K(tenant_id_)); + } else if (OB_TMP_FAIL(process_user_tenant_thread0_(tenant_schema))) { + ret = OB_SUCC(ret) ? tmp_ret : ret; + LOG_WARN("failed to process user tenant thread0", KR(ret), + KR(tmp_ret), K(tenant_id_)); } - }//for schema guard, must be free - if (OB_FAIL(ret)) { - idle_time_us = 100 * 1000; - } - idle(idle_time_us); - LOG_INFO("[LS_SER] finish one round", KR(ret), K(idle_time_us)); - }// end while - } -} - - -//meta tenant no need process create_new_ls or drop ls -//only need to agjust primary zone of sys_ls -int ObPrimaryLSService::process_meta_tenant_(const share::schema::ObTenantSchema &tenant_schema) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret)); - } else if (OB_UNLIKELY(!tenant_schema.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("tenant schema is invalid", KR(ret), K(tenant_schema)); - } else if (OB_ISNULL(GCTX.sql_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy is null", KR(ret)); - } else if (!share::schema::ObSchemaService::is_formal_version(tenant_schema.get_schema_version()) - || !tenant_schema.is_normal()) { - } else if (tenant_schema.is_dropping()) { - } else { - share::ObLSPrimaryZoneInfo primary_zone_info; - ObArray primary_zone; - share::ObLSStatusOperator status_op; - ObZone new_primary_zone; - ObSqlString new_zone_priority; - if (OB_FAIL(ObPrimaryZoneUtil::get_tenant_primary_zone_array( - tenant_schema, primary_zone))) { - LOG_WARN("failed to get tenant primary zone array", KR(ret), K(tenant_schema)); - } else if (OB_FAIL(status_op.get_ls_primary_zone_info(tenant_id_, - SYS_LS, primary_zone_info, *GCTX.sql_proxy_))) { - LOG_WARN("failed to get ls primary_zone info", KR(ret), K(tenant_id_)); - } else if (OB_UNLIKELY(0 == primary_zone.count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("primary zone is empty", KR(ret), K(tenant_schema)); - } else if (has_exist_in_array(primary_zone, primary_zone_info.get_primary_zone())) { - if (OB_FAIL(new_primary_zone.assign(primary_zone_info.get_primary_zone()))) { - LOG_WARN("failed to assign primary zone", KR(ret), K(primary_zone_info)); - } - } else if (OB_FAIL(new_primary_zone.assign(primary_zone.at(0)))) { - LOG_WARN("failed to assign primary zone", KR(ret), K(primary_zone)); - } - if (OB_FAIL(ret)) { - } else if (is_sys_tenant(tenant_id_)) { - //sys tenant use tenant normalize primary zone - if (OB_FAIL(ObPrimaryZoneUtil::get_tenant_zone_priority( - tenant_schema, new_zone_priority))) { - LOG_WARN("failed to get tenant primary zone array", KR(ret), K(tenant_schema)); - } - } else if (OB_FAIL(ObTenantLSInfo::get_zone_priority(new_primary_zone, tenant_schema, new_zone_priority))) { - LOG_WARN("failed to get normalize primary zone", KR(ret), K(new_primary_zone)); - } - if (FAILEDx(ObTenantLSInfo::try_update_ls_primary_zone_( - primary_zone_info, new_primary_zone, new_zone_priority))) { - LOG_WARN("failed to update ls primary zone", KR(ret), K(primary_zone_info), - K(new_primary_zone), K(new_zone_priority)); - } - - if (OB_SUCC(ret) && !is_sys_tenant(tenant_id_)) { - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = gather_tenant_recovery_stat_())) { + } else if (OB_TMP_FAIL( + process_user_tenant_thread1_(iterator, start_scn))) { ret = OB_SUCC(ret) ? tmp_ret : ret; - LOG_WARN("failed to gather tenant recovery stat", KR(ret), KR(tmp_ret)); + LOG_WARN("failed to process user tenant thread1", KR(ret), KR(tmp_ret), + K(tenant_id_), K(start_scn)); } - } + + tenant_schema.reset(); + idle(idle_time_us); + LOG_INFO("[PRIMARY_LS_SERVICE] finish one round", KR(ret), K(idle_time_us), K(thread_idx)); + } // end while } - return ret; } -int ObPrimaryLSService::process_user_tenant_(const share::schema::ObTenantSchema &tenant_schema) + +int ObPrimaryLSService::process_user_tenant_thread0_(const share::schema::ObTenantSchema &tenant_schema) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; const uint64_t tenant_id = tenant_schema.get_tenant_id(); ObTenantLSInfo tenant_stat(GCTX.sql_proxy_, &tenant_schema, tenant_id, GCTX.srv_rpc_proxy_, GCTX.lst_operator_); + int64_t task_cnt = 0; if (OB_UNLIKELY(!inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", KR(ret)); @@ -2212,159 +125,61 @@ int ObPrimaryLSService::process_user_tenant_(const share::schema::ObTenantSchema //if tenant schema is in dropping //set the creating ls to create_abort, //set the normal or dropping tenant to drop_tennat_pre - if (OB_FAIL(tenant_stat.drop_tenant(share::NORMAL_SWITCHOVER_STATUS))) { + if (OB_FAIL(tenant_stat.drop_tenant())) { LOG_WARN("failed to drop tenant", KR(ret), K(tenant_id)); - } else if (OB_FAIL(tenant_stat.process_ls_status_missmatch(false/* lock_sys_ls */, - share::NORMAL_SWITCHOVER_STATUS))) { - LOG_WARN("failed to process ls status missmatch", KR(ret), KR(tmp_ret)); + } else if (OB_FAIL(tenant_stat.process_next_ls_status(task_cnt))) { + LOG_WARN("failed to process next ls status", KR(ret), KR(tmp_ret)); } } else { //normal tenant //some ls may failed to create ls, but can continue - if (OB_SUCCESS != (tmp_ret = tenant_stat.process_ls_status_missmatch(false/* lock_sys_ls */, - share::NORMAL_SWITCHOVER_STATUS))) { + if (OB_SUCCESS != (tmp_ret = tenant_stat.process_next_ls_status(task_cnt))) { ret = OB_SUCC(ret) ? tmp_ret : ret; - LOG_WARN("failed to process ls status missmatch", KR(ret), KR(tmp_ret)); - } - - //process each ls group and primary zone is matching - //process each unit group has the right ls group - //overwrite ret - if (OB_SUCCESS != (tmp_ret = tenant_stat.gather_stat(false))) { - ret = OB_SUCC(ret) ? tmp_ret : ret; - LOG_WARN("failed to gather stat", KR(ret), KR(tmp_ret), K(tenant_id)); - } else { - if (OB_SUCCESS != (tmp_ret = tenant_stat.check_ls_match_primary_zone())) { + LOG_WARN("failed to process next ls status", KR(ret), KR(tmp_ret)); + } else if (0 == task_cnt) { + //process each ls group and primary zone is matching + //process each unit group has the right ls group + //overwrite ret + if (OB_SUCCESS != (tmp_ret = tenant_stat.gather_stat(false))) { ret = OB_SUCC(ret) ? tmp_ret : ret; - LOG_WARN("failed to check ls match with primary zone", KR(ret), KR(tmp_ret)); - } - if (OB_SUCCESS != (tmp_ret = tenant_stat.check_ls_group_match_unitnum())) { - ret = OB_SUCC(ret) ? tmp_ret : ret; - LOG_WARN("failed to check ls group match unitnum", KR(ret), KR(tmp_ret)); + LOG_WARN("failed to gather stat", KR(ret), KR(tmp_ret), K(tenant_id)); + } else { + if (OB_SUCCESS != (tmp_ret = tenant_stat.check_ls_match_primary_zone())) { + ret = OB_SUCC(ret) ? tmp_ret : ret; + LOG_WARN("failed to check ls match with primary zone", KR(ret), KR(tmp_ret)); + } + if (OB_SUCCESS != (tmp_ret = tenant_stat.check_ls_group_match_unitnum())) { + ret = OB_SUCC(ret) ? tmp_ret : ret; + LOG_WARN("failed to check ls group match unitnum", KR(ret), KR(tmp_ret)); + } } } - if (OB_SUCCESS != (tmp_ret = tenant_stat.adjust_user_tenant_primary_zone())) { - //ignore error - LOG_WARN("failed to adjust user tenant primary zone", KR(ret), KR(tmp_ret)); - } } - LOG_INFO("finish process tenant", KR(ret), KR(tmp_ret), K(tenant_id), K(tenant_schema)); - + LOG_INFO("[PRIMARY_LS_SERVICE_THREAD0] finish process tenant", + KR(ret), KR(tmp_ret), K(tenant_id), K(task_cnt), K(tenant_schema)); return ret; } - -int ObPrimaryLSService::report_sys_ls_recovery_stat_() +int ObPrimaryLSService::process_user_tenant_thread1_(palf::PalfBufferIterator &iterator, + share::SCN &start_scn) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret), K(inited_)); + if (!is_user_tenant(tenant_id_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls recovery thread must run on user tenant", KR(ret), + K(tenant_id_)); } else { - ObLSService *ls_svr = MTL(ObLSService *); - ObLSHandle ls_handle; - if (OB_FAIL(ls_svr->get_ls(SYS_LS, ls_handle, storage::ObLSGetMod::RS_MOD))) { - LOG_WARN("failed to get ls", KR(ret)); - } else if (OB_FAIL(ObTenantRecoveryReportor::update_ls_recovery(ls_handle.get_ls(), GCTX.sql_proxy_))) { - LOG_WARN("failed to update ls recovery", KR(ret)); + ObRecoveryLSHelper recovery_ls(tenant_id_, GCTX.sql_proxy_); + if (OB_FAIL(recovery_ls.do_work(iterator, start_scn))) { + LOG_WARN("failed to recovery ls", KR(ret), K(start_scn)); } } + LOG_INFO("[PRIMARY_LS_SERVICE_THREAD1] finish process tenant", KR(ret), K(start_scn)); + return ret; } -int ObPrimaryLSService::try_force_drop_tenant_( - const uint64_t user_tenant_id) -{ - int ret = OB_SUCCESS; - ObTimeoutCtx ctx; - share::schema::ObSchemaGetterGuard schema_guard; - const share::schema::ObTenantSchema *tenant_schema = NULL; - - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret)); - } else if (OB_ISNULL(GCTX.sql_proxy_) - || OB_ISNULL(GCTX.rs_rpc_proxy_) - || OB_ISNULL(GCTX.schema_service_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ptr is null", KR(ret), KP(GCTX.sql_proxy_), KP(GCTX.rs_rpc_proxy_), - KP(GCTX.schema_service_)); - } else if (OB_UNLIKELY(!is_user_tenant(user_tenant_id))) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("not user tenant", KR(ret), K(user_tenant_id)); - } else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard( - OB_SYS_TENANT_ID, schema_guard))) { - LOG_WARN("fail to get schema guard", KR(ret)); - } else if (OB_FAIL(schema_guard.get_tenant_info(user_tenant_id, tenant_schema))) { - LOG_WARN("failed to get tenant ids", KR(ret), K(user_tenant_id)); - } else if (OB_ISNULL(tenant_schema)) { - ret = OB_TENANT_NOT_EXIST; - LOG_WARN("tenant not exist", KR(ret), K(user_tenant_id)); - } else if (!tenant_schema->is_dropping()) { - } else { - LOG_INFO("try drop tenant", K(user_tenant_id)); - ObLSStatusOperator op; - share::ObLSStatusInfoArray ls_array; - if (OB_FAIL(ObShareUtil::set_default_timeout_ctx(ctx, GCONF.rpc_timeout))) { - LOG_WARN("fail to set timeout ctx", KR(ret), K(user_tenant_id)); - } else if (OB_FAIL(op.get_all_ls_status_by_order(user_tenant_id, ls_array, *GCTX.sql_proxy_))) { - LOG_WARN("fail to get all ls status", KR(ret), K(user_tenant_id)); - } else if (ls_array.count() <= 0) { - obrpc::ObDropTenantArg arg; - arg.exec_tenant_id_ = OB_SYS_TENANT_ID; - arg.tenant_name_ = tenant_schema->get_tenant_name(); - arg.tenant_id_ = tenant_schema->get_tenant_id(); - arg.if_exist_ = true; - arg.delay_to_drop_ = false; - ObSqlString sql; - const int64_t timeout_ts = ctx.get_timeout(); - if (OB_FAIL(sql.append_fmt("DROP TENANT IF EXISTS %s FORCE", arg.tenant_name_.ptr()))) { - LOG_WARN("fail to generate sql", KR(ret), K(arg)); - } else if (FALSE_IT(arg.ddl_stmt_str_ = sql.string())) { - } else if (OB_FAIL(GCTX.rs_rpc_proxy_->timeout(timeout_ts).drop_tenant(arg))) { - LOG_WARN("fail to drop tenant", KR(ret), K(arg), K(timeout_ts)); - } - } else { - // tenant's logstream is still dropping, check next round - } - } - return ret; -} - -int ObPrimaryLSService::gather_tenant_recovery_stat_() -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret)); - } else if (is_sys_tenant(tenant_id_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sys tenant no need gather tenant recovery stat", KR(ret), K(tenant_id_)); - } else if (OB_ISNULL(GCTX.sql_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql proxy is null", KR(ret)); - } else { - ObLSRecoveryStatOperator ls_recovery_op; - ObAllTenantInfoProxy info_proxy; - ObAllTenantInfo tenant_info; - const uint64_t user_tenant_id = gen_user_tenant_id(tenant_id_); - SCN sync_scn; - SCN min_wrs_scn; - DEBUG_SYNC(BLOCK_TENANT_SYNC_SNAPSHOT_INC); - if (OB_FAIL(ls_recovery_op.get_tenant_recovery_stat( - user_tenant_id, *GCTX.sql_proxy_, sync_scn, min_wrs_scn))) { - LOG_WARN("failed to get tenant recovery stat", KR(ret), K(user_tenant_id)); - //TODO replayable_scn is equal to sync_scn - } else if (OB_FAIL(info_proxy.update_tenant_recovery_status( - user_tenant_id, GCTX.sql_proxy_, share::NORMAL_SWITCHOVER_STATUS, sync_scn, - sync_scn, min_wrs_scn))) { - LOG_WARN("failed to update tenant recovery stat", KR(ret), - K(user_tenant_id), K(sync_scn), K(min_wrs_scn)); - } - } - return ret; -} -} +}//end of rootserver } diff --git a/src/rootserver/ob_primary_ls_service.h b/src/rootserver/ob_primary_ls_service.h index 36752d868..5fd742a37 100644 --- a/src/rootserver/ob_primary_ls_service.h +++ b/src/rootserver/ob_primary_ls_service.h @@ -18,9 +18,11 @@ #include "share/ls/ob_ls_status_operator.h"//ObLSStatusOperator #include "share/ls/ob_ls_operator.h" //ObLSAttr #include "share/ob_thread_mgr.h" //OBTGDefIDEnum +#include "logservice/palf/palf_iterator.h" //PalfBufferIterator #include "share/unit/ob_unit_info.h"//ObUnit::Status #include "lib/thread/thread_mgr_interface.h" // TGRunnable #include "lib/lock/ob_thread_cond.h"//ObThreadCond +#include "rootserver/ob_tenant_thread_helper.h"//ObTenantThreadHelper namespace oceanbase @@ -33,10 +35,12 @@ namespace common { class ObMySQLProxy; class ObMySQLTransaction; +class ObClusterVersion; } namespace share { class ObLSTableOperator; +class SCN; namespace schema { class ObMultiVersionSchemaService; @@ -50,262 +54,6 @@ struct PalfBaseInfo; } namespace rootserver { -struct ObUnitGroupInfo -{ - ObUnitGroupInfo() : unit_group_id_(OB_INVALID_ID), unit_status_(share::ObUnit::UNIT_STATUS_MAX), - ls_group_ids_() {} - virtual ~ObUnitGroupInfo() {} - bool is_valid() const; - int init(const uint64_t unit_group_id, - const share::ObUnit::Status &unit_status); - void reset(); - int assign(const ObUnitGroupInfo &other); - int remove_ls_group(const uint64_t ls_group_id); - bool operator==(const ObUnitGroupInfo &other) const; - - uint64_t unit_group_id_; - share::ObUnit::Status unit_status_; - ObArray ls_group_ids_; - TO_STRING_KV(K_(unit_group_id), K_(unit_status), K_(ls_group_ids)); -}; -typedef ObArray ObUnitGroupInfoArray; -typedef ObIArray ObUnitGroupInfoIArray; - -struct ObLSGroupInfo -{ - ObLSGroupInfo() : ls_group_id_(OB_INVALID_ID), unit_group_id_(OB_INVALID_ID), - ls_ids_() {} - virtual ~ObLSGroupInfo() {} - bool is_valid() const; - int init(const uint64_t unit_group_id, const uint64_t ls_group_id); - int assign(const ObLSGroupInfo &other); - void reset(); - int remove_ls(const share::ObLSID &ls_id); - uint64_t ls_group_id_; - uint64_t unit_group_id_; - ObArray ls_ids_; - TO_STRING_KV(K_(ls_group_id), K_(unit_group_id), K_(ls_ids)); -}; - -typedef ObArray ObLSGroupInfoArray; -typedef ObIArray ObLSGroupInfoIArray; - -struct ObLSStatusMachineParameter -{ - ObLSStatusMachineParameter() : ls_id_(), status_info_(), ls_info_(share::OB_LS_EMPTY) {} - virtual ~ObLSStatusMachineParameter() {} - bool is_valid() const - { - return ls_id_.is_valid() - && (share::OB_LS_EMPTY == status_info_.status_ - || status_info_.ls_id_ == ls_id_); - } - int init(const share::ObLSID &id, const share::ObLSStatusInfo &status_info, - const share::ObLSStatus &ls_info); - void reset(); - share::ObLSID ls_id_; - share::ObLSStatusInfo status_info_;//for create ls and status of __all_ls_status - share::ObLSStatus ls_info_;//status in __all_ls - TO_STRING_KV(K_(ls_id), K_(status_info), K_(ls_info)); -}; - -/*descripthin: Tenant log stream status information: Statistical log stream - * status on __all_ls_status and __all_ls, tenant primary_zone and unit_num - * information. Provides location information for the newly created log stream. - * Whether the build needs to create or delete log streams.*/ -class ObTenantLSInfo -{ -public: - ObTenantLSInfo(ObMySQLProxy *sql_proxy, - const share::schema::ObTenantSchema *tenant_schema, - const uint64_t tenant_id, obrpc::ObSrvRpcProxy *rpc_proxy, - share::ObLSTableOperator *lst_operator) - : sql_proxy_(sql_proxy), - tenant_schema_(tenant_schema), - status_operator_(), - ls_operator_(tenant_id, sql_proxy), - status_array_(), - unit_group_array_(), - ls_group_array_(), - primary_zone_(), - rpc_proxy_(rpc_proxy), - lst_operator_(lst_operator), - max_ls_id_(OB_INVALID_ID), - max_ls_group_id_(OB_INVALID_ID) {} - - virtual ~ObTenantLSInfo(){}; - void reset(); - bool is_valid() const; - - int gather_stat(bool for_recovery); - //some ls not in __all_ls, but in __all_ls_status - //it need delete by gc, no need process it anymore. - //check the ls status, and delete no need ls - int process_ls_status_missmatch( - const bool lock_sys_ls, - const share::ObTenantSwitchoverStatus &working_sw_status); - - // need create or delete new ls group - int check_ls_group_match_unitnum(); - - //need create or delete new ls - int check_ls_match_primary_zone(); - - int fetch_new_ls_group_id(const uint64_t tenant_id, uint64_t &ls_group_id); - int fetch_new_ls_id(const uint64_t tenant_id, share::ObLSID &ls_id); - - //get ls group info from ls_group_array_ by ls_group_id - //the interface must used after gather_stat(); - int get_ls_group_info(const uint64_t ls_group_id, ObLSGroupInfo &info) const; - - //get ls status info from status_array_ by ls_id - //the interface must used after gather_stat(); - int get_ls_status_info(const share::ObLSID &id, share::ObLSStatusInfo &info, - int64_t &info_index) const; - //process dropping tenant, set status to tenant_dropping in __all_ls - int drop_tenant(const share::ObTenantSwitchoverStatus &working_sw_status); - //for recovery tenant, create new ls according to ls_id and ls_group_id - int create_new_ls_for_recovery(const share::ObLSID &ls_id, - const uint64_t ls_group_id, - const share::SCN &create_scn, - common::ObMySQLTransaction &trans); - //for recovery tenant, if ls is in creating in __all_ls_status, create the ls - int process_ls_stats_for_recovery(); - - int adjust_user_tenant_primary_zone(); - - /* - * description: create ls with palf base info for recovery - @param[in] info: status info in __all_ls_status_info - @param[in] create_scn : create scn of ls - @param[in] create_ls_with_palf: restore create init ls - @param[in] palf_base_info : palf base info - */ - int create_ls_with_palf(const share::ObLSStatusInfo &info, const share::SCN &create_scn, - const bool create_ls_with_palf, - const palf::PalfBaseInfo &palf_base_info); - - static int get_zone_priority(const ObZone &primary_zone, - const share::schema::ObTenantSchema &tenant_schema, - ObSqlString &primary_zone_str); - static int try_update_ls_primary_zone_(const share::ObLSPrimaryZoneInfo &primary_zone_info, - const common::ObZone &new_primary_zone, - const common::ObSqlString &zone_priority); - static int construct_ls_status_machine_( - const share::ObLSStatusInfoIArray &statua_info_array, - const share::ObLSAttrIArray &ls_array, - common::ObIArray &status_machine_array); - -private: - int fix_ls_status_(const ObLSStatusMachineParameter &status_machine, - const share::ObTenantSwitchoverStatus &working_sw_status); - // get from __all_ls_status and __all_ls - int gather_all_ls_info_(); - - // base on status_array construct ls_array - int add_ls_to_ls_group_(const share::ObLSStatusInfo &info); - - // base on ls_array construct unit_group_array and __all_unit - int add_ls_group_to_unit_group_(const ObLSGroupInfo &group_info); - - int process_creating_ls_(const share::ObLSAttr &ls_operation); - - int check_unit_group_valid_(const uint64_t unit_group_id, bool &is_valid); - int add_ls_status_info_(const share::ObLSStatusInfo &ls_info); - int create_new_ls_for_empty_unit_group_(const uint64_t unit_group_id); - - int get_next_unit_group_(const bool is_recovery, int64_t &group_index); - // get the primary zone not in ls group - int get_next_primary_zone_(const bool is_recovery, const ObLSGroupInfo &group_info, - ObZone &primary_zone); - int create_new_ls_(const share::ObLSStatusInfo &ls_info, - const share::ObTenantSwitchoverStatus &working_sw_status); - - // drop ls group info - int try_drop_ls_of_deleting_unit_group_(const ObUnitGroupInfo &info); - int drop_ls_(const share::ObLSStatusInfo &ls_info, - const share::ObTenantSwitchoverStatus &working_sw_status); - int do_drop_ls_(const share::ObLSStatusInfo &ls_info, - const share::ObTenantSwitchoverStatus &working_sw_status); - int do_tenant_drop_ls_(const share::ObLSStatusInfo &ls_info, - const share::ObTenantSwitchoverStatus &working_sw_status); - int do_create_ls_(const share::ObLSStatusInfo &info, const share::SCN &create_scn); - int sys_ls_tenant_drop_(const share::ObLSStatusInfo &info, - const share::ObTenantSwitchoverStatus &working_sw_status); - int check_sys_ls_can_offline_(bool &can_offline); - int check_ls_empty_(const share::ObLSStatusInfo &info, bool &empty); - int check_ls_can_offline_by_rpc_(const share::ObLSStatusInfo &info, - const share::ObLSStatus ¤t_ls_status, - bool &can_offline); - int process_ls_status_after_created_(const share::ObLSStatusInfo &info, - const share::ObTenantSwitchoverStatus &working_sw_status); - -private: - static int set_ls_to_primary_zone(const common::ObIArray &primary_zone_array, - const share::ObLSPrimaryZoneInfoArray &primary_zone_infos, - common::ObIArray &ls_primary_zone, - common::ObIArray &count_group_by_zone); - static int balance_ls_primary_zone(const common::ObIArray &primary_zone_array, - common::ObIArray &ls_primary_zone, - common::ObIArray &count_group_by_zone); - int adjust_primary_zone_by_ls_group_(const common::ObIArray &primary_zone_array, - const share::ObLSPrimaryZoneInfoArray &primary_zone_infos, - const share::schema::ObTenantSchema &tenant_schema); - -private: - ObMySQLProxy *sql_proxy_; - const share::schema::ObTenantSchema *tenant_schema_; - share::ObLSStatusOperator status_operator_; - share::ObLSAttrOperator ls_operator_; - bool is_load_; - share::ObLSStatusInfoArray status_array_; - common::hash::ObHashMap status_map_; - ObUnitGroupInfoArray unit_group_array_; - ObLSGroupInfoArray ls_group_array_; - ObArray primary_zone_; - obrpc::ObSrvRpcProxy *rpc_proxy_; - share::ObLSTableOperator *lst_operator_; - //ensure ls can not be concurrent created - uint64_t max_ls_id_; - uint64_t max_ls_group_id_; -}; - -class ObTenantThreadHelper : public lib::TGRunnable, - public logservice::ObIRoleChangeSubHandler -{ -public: - ObTenantThreadHelper() : tg_id_(-1), thread_cond_(), is_created_(false), is_first_time_to_start_(true), thread_name_("") {} - virtual ~ObTenantThreadHelper() {} - virtual void do_work() = 0; - virtual void run1() override; - virtual void destroy(); - int start(); - void stop(); - void wait(); - int create(const char* thread_name, int tg_def_id, ObTenantThreadHelper &tenant_thread); - void idle(const int64_t idle_time_us); -public: - virtual void switch_to_follower_forcedly() override; - - virtual int switch_to_leader() override; - virtual int switch_to_follower_gracefully() override - { - stop(); - return OB_SUCCESS; - } - virtual int resume_leader() override - { - return OB_SUCCESS; - } - -protected: - int tg_id_; -private: - common::ObThreadCond thread_cond_; - bool is_created_; - bool is_first_time_to_start_; - const char* thread_name_; -}; /*description: *Log stream management thread: Started on the leader of the system log stream @@ -342,16 +90,9 @@ public: private: - int process_user_tenant_(const share::schema::ObTenantSchema &tenant_schema); - //no need create ls or drop ls, but need adjust primary_zone - int process_meta_tenant_(const share::schema::ObTenantSchema &tenant_schema); - //for primary cluster, sys ls recovery stat need report, - //standby cluster will be reported in RecoveryLSService - int report_sys_ls_recovery_stat_(); - // force drop user tenant if tenant is in dropping status - int try_force_drop_tenant_( - const uint64_t user_tenant_id); - int gather_tenant_recovery_stat_(); + int process_user_tenant_thread0_(const share::schema::ObTenantSchema &tenant_schema); + int process_user_tenant_thread1_(palf::PalfBufferIterator &iterator, + share::SCN &start_scn); private: bool inited_; uint64_t tenant_id_; diff --git a/src/rootserver/ob_recovery_ls_service.cpp b/src/rootserver/ob_recovery_ls_service.cpp index 42eb0338f..c6751c969 100644 --- a/src/rootserver/ob_recovery_ls_service.cpp +++ b/src/rootserver/ob_recovery_ls_service.cpp @@ -20,28 +20,16 @@ #include "logservice/ob_log_handler.h" //ObLogHandler #include "logservice/palf/log_entry.h" //LogEntry #include "logservice/palf/log_define.h" +#include "logservice/ob_log_service.h" #include "share/scn.h"//SCN -#include "logservice/ob_garbage_collector.h"//ObGCLSLog -#include "logservice/restoreservice/ob_log_restore_handler.h"//ObLogRestoreHandler #include "observer/ob_server_struct.h" //GCTX -#include "rootserver/ob_primary_ls_service.h" //ObTenantLSInfo -#include "rootserver/ob_tenant_recovery_reportor.h" //ObTenantRecoveryReportor -#include "rootserver/ob_tenant_info_loader.h" // ObTenantInfoLoader -#include "share/ls/ob_ls_life_manager.h" //ObLSLifeManger -#include "share/ls/ob_ls_operator.h" //ObLSAttr -#include "share/ls/ob_ls_recovery_stat_operator.h" //ObLSRecoveryLSStatOperator -#include "share/ob_errno.h" #include "share/ob_share_util.h" //ObShareUtil #include "share/schema/ob_multi_version_schema_service.h" //ObMultiSchemaService -#include "share/restore/ob_physical_restore_info.h" //restore_status -#include "share/restore/ob_physical_restore_table_operator.h"//ObPhysicalRestoreTableOperator -#include "share/ob_primary_standby_service.h" // ObPrimaryStandbyService #include "share/ob_standby_upgrade.h" // ObStandbyUpgrade #include "share/ob_upgrade_utils.h" // ObUpgradeChecker #include "share/ob_global_stat_proxy.h" // ObGlobalStatProxy -#include "storage/tx/ob_tx_log.h" //ObTxLogHeader -#include "storage/tx_storage/ob_ls_service.h" //ObLSService -#include "storage/tx_storage/ob_ls_handle.h" //ObLSHandle +#include "share/ob_cluster_version.h"//GET_TENANT_DATA_VERSION +#include "rootserver/ob_ls_service_helper.h"//ObRecoveryLSHelper namespace oceanbase { @@ -64,7 +52,7 @@ int ObRecoveryLSService::init() ret = OB_INIT_TWICE; LOG_WARN("has inited", KR(ret)); } else if (OB_FAIL(ObTenantThreadHelper::create("RecLSSer", - lib::TGDefIDs::LSService, *this))) { + lib::TGDefIDs::SimpleLSService, *this))) { LOG_WARN("failed to create thread", KR(ret)); } else if (OB_FAIL(ObTenantThreadHelper::start())) { LOG_WARN("failed to start thread", KR(ret)); @@ -92,318 +80,40 @@ void ObRecoveryLSService::do_work() if (OB_UNLIKELY(!inited_) || OB_ISNULL(proxy_)) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret), K(inited_), KP(proxy_)); + } else if (OB_FAIL(wait_tenant_data_version_ready_(tenant_id_, DATA_VERSION_4_1_0_0))) { + LOG_WARN("failed to wait tenant and version ready", KR(ret), K(tenant_id_)); } else { - ObLSRecoveryStatOperator ls_recovery; palf::PalfBufferIterator iterator; - int tmp_ret = OB_SUCCESS; - const int64_t idle_time_us = 100 * 1000L; + int64_t idle_time_us = 100 * 1000L; SCN start_scn; + while (!has_set_stop()) { ObCurTraceId::init(GCONF.self_addr_); - uint64_t thread_idx = get_thread_idx(); - ObTenantInfoLoader *tenant_info_loader = MTL(ObTenantInfoLoader*); - ObAllTenantInfo tenant_info; - //two thread for seed log and recovery_ls_manager + // two thread for seed log and recovery_ls_manager if (!is_user_tenant(tenant_id_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("ls recovery thread must run on user tenant", KR(ret), K(tenant_id_)); - } else if (OB_ISNULL(tenant_info_loader)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant report is null", KR(ret), K(tenant_id_)); - } else if (OB_FAIL(tenant_info_loader->get_tenant_info(tenant_info))) { - LOG_WARN("failed to get tenant info", KR(ret)); - } else if (OB_FAIL(check_can_do_recovery_(tenant_info))) { - LOG_WARN("failed to check do recovery", KR(ret), K(tenant_info)); - } else if (0 == thread_idx) { - if (OB_SUCCESS != (tmp_ret = process_recovery_ls_manager())) { - ret = OB_SUCC(ret) ? tmp_ret : ret; - LOG_WARN("failed to process recovery ls manager", KR(ret), KR(tmp_ret)); - } } else { - if (!start_scn.is_valid()) { - ObLSRecoveryStat ls_recovery_stat; - if (OB_FAIL(ls_recovery.get_ls_recovery_stat(tenant_id_, - SYS_LS, false, ls_recovery_stat, *proxy_))) { - LOG_WARN("failed to load sys recovery stat", KR(ret), K(tenant_id_)); - } else if (SCN::base_scn() == ls_recovery_stat.get_sync_scn()) { - //等待物理恢复设置完系统日志流初始同步点之后开始迭代 - ret = OB_EAGAIN; - LOG_WARN("restore init sync scn has not setted, need wait", KR(ret), K(ls_recovery_stat)); - } else if (OB_FAIL(report_sys_ls_recovery_stat_(ls_recovery_stat.get_sync_scn()))) { - //may recovery end, but readable scn need report - LOG_WARN("failed to report ls recovery stat", KR(ret), K(ls_recovery_stat)); - } else if (tenant_info.get_recovery_until_scn() == ls_recovery_stat.get_sync_scn()) { - ret = OB_EAGAIN; - if (REACH_TIME_INTERVAL(60 * 1000 * 1000)) { // every minute - LOG_INFO("has recovered to recovery_until_scn", KR(ret), K(ls_recovery_stat), K(tenant_info)); - } - } else if (OB_FAIL(seek_log_iterator_(ls_recovery_stat.get_sync_scn(), iterator))) { - LOG_WARN("failed to seek log iterator", KR(ret), K(ls_recovery_stat)); - } else { - start_scn = ls_recovery_stat.get_sync_scn(); - LOG_INFO("start to seek at", K(start_scn)); - } + ObRecoveryLSHelper recovery_ls(tenant_id_, proxy_); + if (OB_FAIL(recovery_ls.do_work(iterator, start_scn))) { + LOG_WARN("failed to recovery ls", KR(ret), K(start_scn)); } - if (OB_FAIL(ret)) { - } else if (start_scn.is_valid() && OB_FAIL(process_ls_log_(tenant_info, start_scn, iterator))) { - if (OB_ITER_STOP != ret) { - LOG_WARN("failed to process ls log", KR(ret), K(start_scn), K(tenant_info)); - } - } - if (OB_FAIL(ret)) { - start_scn.reset(); - } - - if (REACH_TIME_INTERVAL(10 * 1000 * 1000)) { // every 10 second - (void)try_tenant_upgrade_end_(); + if (OB_ITER_STOP == ret) { + //standby or recovery may restore finish, + //so when the error code is OB_ITER_STOP, adjust idle_time to 10s + idle_time_us = 10 * 1000 * 1000;//10s } } - LOG_INFO("[LS_RECOVERY] finish one round", KR(ret), KR(tmp_ret), K(start_scn), K(thread_idx), K(tenant_info)); + if (REACH_TIME_INTERVAL(10 * 1000 * 1000)) { // every 10 second + (void)try_tenant_upgrade_end_(); + } + LOG_INFO("[RECOVERY_LS_SERVICE] finish one round", KR(ret), + K(start_scn)); idle(idle_time_us); } } } -int ObRecoveryLSService::seek_log_iterator_(const SCN &sync_scn, PalfBufferIterator &iterator) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret), K(inited_)); - } else if (OB_UNLIKELY(!sync_scn.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("sync scn is invalid", KR(ret), K(sync_scn)); - } else { - ObLSService *ls_svr = MTL(ObLSService *); - ObLSHandle ls_handle; - if (OB_FAIL(ls_svr->get_ls(SYS_LS, ls_handle, storage::ObLSGetMod::RS_MOD))) { - LOG_WARN("failed to get ls", KR(ret)); - } else { - ObLogHandler *log_handler = NULL; - ObLS *ls = NULL; - palf::LSN start_lsn; - if (OB_ISNULL(ls = ls_handle.get_ls()) - || OB_ISNULL(log_handler = ls->get_log_handler())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls or log handle is null", KR(ret), KP(ls), - KP(log_handler)); - } else if (OB_FAIL(log_handler->locate_by_scn_coarsely(sync_scn, start_lsn))) { - LOG_WARN("failed to locate lsn", KR(ret), K(sync_scn)); - } else if (OB_FAIL(log_handler->seek(start_lsn, iterator))) { - LOG_WARN("failed to seek iterator", KR(ret), K(sync_scn)); - } - } - } - return ret; -} - -int ObRecoveryLSService::process_ls_log_( - const ObAllTenantInfo &tenant_info, - SCN &start_scn, - PalfBufferIterator &iterator) -{ - int ret = OB_SUCCESS; - palf::LogEntry log_entry; - palf::LSN target_lsn; - SCN sync_scn; - SCN last_sync_scn; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret), K(inited_)); - } else if (OB_UNLIKELY(!tenant_info.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("tenant_info is invalid", KR(ret), K(tenant_info)); - } - while (OB_SUCC(ret) && OB_SUCC(iterator.next())) { - if (OB_FAIL(iterator.get_entry(log_entry, target_lsn))) { - LOG_WARN("failed to get log", KR(ret), K(log_entry)); - } else if (OB_ISNULL(log_entry.get_data_buf())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("log entry is null", KR(ret)); - } else { - LOG_DEBUG("get log", K(log_entry), K(target_lsn), K(start_scn)); - sync_scn = log_entry.get_scn(); - const char *log_buf = log_entry.get_data_buf(); - const int64_t log_length = log_entry.get_data_len(); - logservice::ObLogBaseHeader header; - const int64_t HEADER_SIZE = header.get_serialize_size(); - int64_t log_pos = 0; - if (OB_UNLIKELY(sync_scn <= start_scn)) { - //通过scn定位的LSN是不准确的，可能会获取多余的数据，所以需要把小于等于sync_scn的过滤掉 - continue; - } else if (tenant_info.get_recovery_until_scn() < sync_scn) { - if (OB_FAIL(report_sys_ls_recovery_stat_(tenant_info.get_recovery_until_scn()))) { - LOG_WARN("failed to report_sys_ls_recovery_stat_", KR(ret), K(sync_scn), K(tenant_info), - K(log_entry), K(target_lsn), K(start_scn)); - // SYS LS has recovered to the recovery_until_scn, need stop iterate SYS LS log and reset start_scn - } else if (OB_FAIL(seek_log_iterator_(tenant_info.get_recovery_until_scn(), iterator))) { - LOG_WARN("failed to seek log iterator", KR(ret), K(sync_scn), K(tenant_info), - K(log_entry), K(target_lsn), K(start_scn)); - } else { - ret = OB_ITER_STOP; - LOG_WARN("SYS LS has recovered to the recovery_until_scn, need stop iterate SYS LS log", - KR(ret), K(sync_scn), K(tenant_info), K(log_entry), K(target_lsn), K(start_scn)); - start_scn.reset(); - } - } else if (OB_FAIL(header.deserialize(log_buf, HEADER_SIZE, log_pos))) { - LOG_WARN("failed to deserialize", KR(ret), K(HEADER_SIZE)); - } else if (OB_UNLIKELY(log_pos >= log_length)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("log pos is not expected", KR(ret), K(log_pos), K(log_length)); - } else if (logservice::GC_LS_LOG_BASE_TYPE == header.get_log_type()) { - ObGCLSLog gc_log; - if (OB_FAIL(gc_log.deserialize(log_buf, log_length, log_pos))) { - LOG_WARN("failed to deserialize gc log", KR(ret), K(log_length)); - } else if (ObGCLSLOGType::OFFLINE_LS == gc_log.get_log_type()) { - //set sys ls offline - if (OB_FAIL(process_gc_log_(gc_log, sync_scn))) { - LOG_WARN("failed to process gc log", KR(ret), K(sync_scn)); - } - } - // nothing - } else if (logservice::TRANS_SERVICE_LOG_BASE_TYPE == header.get_log_type()) { - ObTxLogBlock tx_log_block; - ObTxLogBlockHeader tx_block_header; - if (OB_FAIL(tx_log_block.init(log_buf, log_length, log_pos, tx_block_header))) { - LOG_WARN("failed to init tx log block", KR(ret), K(log_length)); - } else if (OB_FAIL(process_ls_tx_log_(tx_log_block, sync_scn))) { - LOG_WARN("failed to process ls tx log", KR(ret), K(tx_log_block), K(sync_scn)); - } - } else {} - if (OB_SUCC(ret)) { - last_sync_scn = sync_scn; - } else if (last_sync_scn.is_valid()) { - //if ls_operator can not process, need to report last sync scn - int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(report_sys_ls_recovery_stat_(last_sync_scn))) { - LOG_WARN("failed to report ls recovery stat", KR(ret), KR(tmp_ret), K(last_sync_scn)); - } - } - } - }//end for each log - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - if (OB_FAIL(report_sys_ls_recovery_stat_(sync_scn))) { - LOG_WARN("failed to report ls recovery stat", KR(ret), K(sync_scn)); - } - } else if (OB_SUCC(ret)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("iterator must be end", KR(ret)); - } else { - LOG_WARN("failed to get next log", KR(ret)); - } - - return ret; -} - -int ObRecoveryLSService::process_ls_tx_log_(ObTxLogBlock &tx_log_block, const SCN &sync_scn) -{ - int ret = OB_SUCCESS; - bool has_operation = false; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret), K(inited_)); - } - while (OB_SUCC(ret)) { - transaction::ObTxLogHeader tx_header; - if (OB_FAIL(tx_log_block.get_next_log(tx_header))) { - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - break; - } else { - LOG_WARN("failed to get next log", KR(ret)); - } - } else if (transaction::ObTxLogType::TX_COMMIT_LOG != - tx_header.get_tx_log_type()) { - // nothing - } else { - ObTxCommitLogTempRef temp_ref; - ObTxCommitLog commit_log(temp_ref); - const int64_t COMMIT_SIZE = commit_log.get_serialize_size(); - //TODO commit log may too large - if (OB_FAIL(tx_log_block.deserialize_log_body(commit_log))) { - LOG_WARN("failed to deserialize", KR(ret)); - } else { - const ObTxBufferNodeArray &source_data = - commit_log.get_multi_source_data(); - for (int64_t i = 0; OB_SUCC(ret) && i < source_data.count(); ++i) { - const ObTxBufferNode &node = source_data.at(i); - if (ObTxDataSourceType::STANDBY_UPGRADE == node.get_data_source_type()) { - if (OB_FAIL(process_upgrade_log_(node))) { - LOG_WARN("failed to process_upgrade_log_", KR(ret), K(node)); - } - } else if (ObTxDataSourceType::LS_TABLE != node.get_data_source_type()) { - // nothing - } else if (has_operation) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("one clog has more than one operation", KR(ret), K(commit_log)); - } else { - has_operation = true; - ObLSAttr ls_attr; - const int64_t LS_SIZE = ls_attr.get_serialize_size(); - int64_t pos = 0; - if (OB_FAIL(ls_attr.deserialize(node.get_data_buf().ptr(), LS_SIZE, - pos))) { - LOG_WARN("failed to deserialize", KR(ret), K(node), K(LS_SIZE)); - } else if (OB_UNLIKELY(pos > LS_SIZE)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to get ls attr", KR(ret), K(pos), K(LS_SIZE)); - } else { - LOG_INFO("get ls operation", K(ls_attr), K(sync_scn)); - //TODO ls recovery is too fast for ls manager, so it maybe failed, while change ls status - //consider how to retry - if (OB_FAIL(process_ls_operator_(ls_attr, - sync_scn))) { - LOG_WARN("failed to process ls operator", KR(ret), K(ls_attr), - K(sync_scn)); - } - } - } - }// end for - } - } - } // end while for each tx_log - - return ret; -} - -int ObRecoveryLSService::process_upgrade_log_(const ObTxBufferNode &node) -{ - int ret = OB_SUCCESS; - uint64_t standby_data_version = 0; - - if (!node.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("node is invalid", KR(ret), K(node)); - } else { - ObStandbyUpgrade primary_data_version; - int64_t pos = 0; - if (OB_FAIL(primary_data_version.deserialize(node.get_data_buf().ptr(), node.get_data_buf().length(), pos))) { - LOG_WARN("failed to deserialize", KR(ret), K(node), KPHEX(node.get_data_buf().ptr(), node.get_data_buf().length())); - } else if (OB_UNLIKELY(pos > node.get_data_buf().length())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to get primary_data_version", KR(ret), K(pos), K(node.get_data_buf().length())); - } else { - LOG_INFO("get primary_data_version", K(primary_data_version)); - if (!primary_data_version.is_valid()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("primary_data_version not valid", KR(ret), K(primary_data_version)); - } else if (OB_FAIL(ObUpgradeChecker::get_data_version_by_cluster_version( - GET_MIN_CLUSTER_VERSION(), - standby_data_version))) { - LOG_WARN("failed to get_data_version_by_cluster_version", KR(ret), K(GET_MIN_CLUSTER_VERSION())); - } else if (primary_data_version.get_data_version() > standby_data_version) { - ret = OB_EAGAIN; - if (REACH_TIME_INTERVAL(30 * 60 * 1000 * 1000)) { // 30min - LOG_ERROR("standby version is not new enough to recover primary clog", KR(ret), - K(primary_data_version), K(standby_data_version)); - } - } - } - } - return ret; -} - int ObRecoveryLSService::get_min_data_version_(uint64_t &compatible) { int ret = OB_SUCCESS; @@ -506,331 +216,6 @@ void ObRecoveryLSService::try_tenant_upgrade_end_() } } -int ObRecoveryLSService::process_gc_log_(logservice::ObGCLSLog &gc_log, const SCN &sync_scn) -{ - int ret = OB_SUCCESS; - common::ObMySQLTransaction trans; - const uint64_t meta_tenant_id = gen_meta_tenant_id(tenant_id_); - ObLSLifeAgentManager ls_life_agent(*proxy_); - if (OB_ISNULL(proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("proxy is null", KR(ret)); - } else if (OB_FAIL(trans.start(proxy_, meta_tenant_id))) { - LOG_WARN("failed to start trans", KR(ret), K(meta_tenant_id)); - } else if (OB_FAIL(ls_life_agent.set_ls_offline_in_trans( - tenant_id_, SYS_LS, share::OB_LS_TENANT_DROPPING, sync_scn, share::NORMAL_SWITCHOVER_STATUS, - trans))) { - LOG_WARN("failed to set offline", KR(ret), K(tenant_id_), K(sync_scn)); - } else { - ObLSRecoveryStatOperator ls_recovery; - ObLSRecoveryStat ls_recovery_stat; - if (OB_FAIL(construct_ls_recovery_stat(sync_scn, ls_recovery_stat))) { - LOG_WARN("failed to construct ls recovery stat", KR(ret), K(sync_scn)); - } else if (OB_FAIL(ls_recovery.update_ls_recovery_stat_in_trans( - ls_recovery_stat, trans))) { - LOG_WARN("failed to update ls recovery stat", KR(ret), K(ls_recovery_stat)); - } - } - if (trans.is_started()) { - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) { - ret = OB_SUCC(ret) ? tmp_ret : ret; - LOG_WARN("failed to end trans", KR(ret), K(tmp_ret)); - } - } - return ret; -} - -int ObRecoveryLSService::construct_ls_recovery_stat(const SCN &sync_scn, - ObLSRecoveryStat &ls_stat) -{ - int ret = OB_SUCCESS; - ls_stat.reset(); - SCN readable_scn; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret), K(inited_)); - } else if (OB_FAIL(ObTenantRecoveryReportor::get_readable_scn(SYS_LS, readable_scn))) { - LOG_WARN("failed to get readable scn", KR(ret)); - } else if (OB_FAIL(ls_stat.init_only_recovery_stat( - tenant_id_, SYS_LS, sync_scn, readable_scn))) { - LOG_WARN("failed to init ls recovery stat", KR(ret), K(tenant_id_), - K(sync_scn), K(readable_scn)); - } - return ret; -} - -int ObRecoveryLSService::process_ls_operator_(const share::ObLSAttr &ls_attr, const SCN &sync_scn) -{ - int ret = OB_SUCCESS; - common::ObMySQLTransaction trans; - const uint64_t meta_tenant_id = gen_meta_tenant_id(tenant_id_); - if (OB_UNLIKELY(!ls_attr.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("ls attr is invalid", KR(ret), K(ls_attr)); - } else if (OB_ISNULL(proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("proxy is null", KR(ret)); - } else if (OB_FAIL(check_valid_to_operator_ls_(ls_attr, sync_scn))) { - LOG_WARN("failed to check valid to operator ls", KR(ret), K(sync_scn), K(ls_attr)); - } else if (OB_FAIL(trans.start(proxy_, meta_tenant_id))) { - LOG_WARN("failed to start trans", KR(ret), K(meta_tenant_id)); - } else if (OB_FAIL(process_ls_operator_in_trans_(ls_attr, sync_scn, trans))) { - LOG_WARN("failed to process ls operator in trans", KR(ret), K(ls_attr), K(sync_scn)); - } - - if (OB_SUCC(ret)) { - ObLSRecoveryStat ls_recovery_stat; - ObLSRecoveryStatOperator ls_recovery; - if (OB_FAIL(construct_ls_recovery_stat(sync_scn, ls_recovery_stat))) { - LOG_WARN("failed to construct ls recovery stat", KR(ret), K(sync_scn)); - } else if (OB_FAIL(ls_recovery.update_ls_recovery_stat_in_trans( - ls_recovery_stat, trans))) { - LOG_WARN("failed to update ls recovery stat", KR(ret), K(ls_recovery_stat)); - } - } - if (trans.is_started()) { - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) { - ret = OB_SUCC(ret) ? tmp_ret : ret; - LOG_WARN("failed to end trans", KR(ret), K(tmp_ret)); - } - } - return ret; -} - -int ObRecoveryLSService::check_valid_to_operator_ls_( - const share::ObLSAttr &ls_attr, const SCN &sync_scn) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret), K(inited_)); - } else if (OB_ISNULL(proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("proxy is null", KR(ret)); - } else if (OB_UNLIKELY(!sync_scn.is_valid() || !ls_attr.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("syns scn is invalid", KR(ret), K(sync_scn), K(ls_attr)); - } else if (share::is_ls_tenant_drop_pre_op(ls_attr.get_ls_operatin_type())) { - ret = OB_ITER_STOP; - LOG_WARN("can not process ls operator after tenant dropping", K(ls_attr)); - } else if (share::is_ls_tenant_drop_op(ls_attr.get_ls_operatin_type())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls recovery must stop while pre tenant dropping", KR(ret), K(ls_attr)); - } else { - SCN user_scn; - ObLSRecoveryStatOperator ls_recovery; - if (OB_FAIL(ls_recovery.get_user_ls_sync_scn(tenant_id_, *proxy_, user_scn))) { - LOG_WARN("failed to get user scn", KR(ret), K(tenant_id_)); - } else if (user_scn >= sync_scn) { - //other ls has larger sync scn, ls can operator - } else { - ret = OB_NEED_WAIT; - LOG_WARN("can not process ls operator, need wait other ls sync", KR(ret), - K(user_scn), K(sync_scn)); - } - } - - return ret; -} - -int ObRecoveryLSService::process_ls_operator_in_trans_( - const share::ObLSAttr &ls_attr, const SCN &sync_scn, ObMySQLTransaction &trans) -{ - int ret = OB_SUCCESS; - const uint64_t meta_tenant_id = gen_meta_tenant_id(tenant_id_); - if (OB_UNLIKELY(!ls_attr.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("ls attr is invalid", KR(ret), K(ls_attr)); - } else if (OB_ISNULL(proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("proxy is null", KR(ret)); - } else { - ObLSStatusOperator ls_operator; - share::ObLSStatusInfo ls_status; - ObLSLifeAgentManager ls_life_agent(*proxy_); - if (OB_UNLIKELY(!ls_attr.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("ls attr is invalid", KR(ret), K(ls_attr)); - } else if (share::is_ls_create_pre_op(ls_attr.get_ls_operatin_type())) { - //create new ls; - if (OB_FAIL(create_new_ls_(ls_attr, sync_scn, trans))) { - LOG_WARN("failed to create new ls", KR(ret), K(sync_scn), K(ls_attr)); - } - } else if (share::is_ls_create_abort_op(ls_attr.get_ls_operatin_type())) { - if (OB_FAIL(ls_life_agent.drop_ls_in_trans(tenant_id_, ls_attr.get_ls_id(), share::NORMAL_SWITCHOVER_STATUS, trans))) { - LOG_WARN("failed to drop ls", KR(ret), K(tenant_id_), K(ls_attr)); - } - } else if (share::is_ls_drop_end_op(ls_attr.get_ls_operatin_type())) { - if (OB_FAIL(ls_life_agent.set_ls_offline_in_trans(tenant_id_, ls_attr.get_ls_id(), - ls_attr.get_ls_status(), sync_scn, share::NORMAL_SWITCHOVER_STATUS, trans))) { - LOG_WARN("failed to set offline", KR(ret), K(tenant_id_), K(ls_attr), K(sync_scn)); - } - } else { - ObLSStatus target_status = share::OB_LS_EMPTY; - if (OB_FAIL(ls_operator.get_ls_status_info(tenant_id_, ls_attr.get_ls_id(), - ls_status, trans))) { - LOG_WARN("failed to get ls status", KR(ret), K(tenant_id_), K(ls_attr)); - } else if (ls_status.ls_is_creating()) { - ret = OB_EAGAIN; - LOG_WARN("ls not created, need wait", KR(ret), K(ls_status)); - } else if (share::is_ls_create_end_op(ls_attr.get_ls_operatin_type())) { - // set ls to normal - target_status = share::OB_LS_NORMAL; - } else if (share::is_ls_tenant_drop_op(ls_attr.get_ls_operatin_type())) { - target_status = share::OB_LS_TENANT_DROPPING; - } else if (share::is_ls_drop_pre_op(ls_attr.get_ls_operatin_type())) { - target_status = share::OB_LS_DROPPING; - } else if (share::is_ls_tenant_drop_pre_op(ls_attr.get_ls_operatin_type())) { - target_status = share::OB_LS_PRE_TENANT_DROPPING; - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected operation type", KR(ret), K(ls_attr)); - } - if (FAILEDx(ls_operator.update_ls_status(tenant_id_, ls_attr.get_ls_id(), - ls_status.status_, target_status, - share::NORMAL_SWITCHOVER_STATUS, trans))) { - LOG_WARN("failed to update ls status", KR(ret), K(tenant_id_), K(ls_attr), - K(ls_status), K(target_status)); - } - LOG_INFO("[LS_RECOVERY] update ls status", KR(ret), K(ls_attr), K(target_status)); - } - } - - return ret; -} -int ObRecoveryLSService::create_new_ls_(const share::ObLSAttr &ls_attr, - const SCN &sync_scn, - common::ObMySQLTransaction &trans) -{ - int ret = OB_SUCCESS; - if (!share::is_ls_create_pre_op(ls_attr.get_ls_operatin_type())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("ls not create pre operation", KR(ret), K(ls_attr)); - } else { - //create new ls; - share::schema::ObSchemaGetterGuard schema_guard; - const share::schema::ObTenantSchema *tenant_schema = NULL; - if (OB_ISNULL(GCTX.schema_service_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error", KR(ret)); - } else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard( - OB_SYS_TENANT_ID, schema_guard))) { - LOG_WARN("fail to get schema guard", KR(ret)); - } else if (OB_FAIL(schema_guard.get_tenant_info(tenant_id_, tenant_schema))) { - LOG_WARN("failed to get tenant ids", KR(ret), K(tenant_id_)); - } else if (OB_ISNULL(tenant_schema)) { - ret = OB_TENANT_NOT_EXIST; - LOG_WARN("tenant not exist", KR(ret), K(tenant_id_)); - } else { - ObTenantLSInfo tenant_stat(GCTX.sql_proxy_, tenant_schema, tenant_id_, - GCTX.srv_rpc_proxy_, GCTX.lst_operator_); - if (OB_FAIL(tenant_stat.gather_stat(true))) { - LOG_WARN("failed to gather stat", KR(ret)); - } else if (OB_FAIL(tenant_stat.create_new_ls_for_recovery(ls_attr.get_ls_id(), - ls_attr.get_ls_group_id(), ls_attr.get_create_scn(), trans))) { - LOG_WARN("failed to add new ls status info", KR(ret), K(ls_attr), K(sync_scn)); - } - } - LOG_INFO("[LS_RECOVERY] create new ls", KR(ret), K(ls_attr)); - } - return ret; -} - - -int ObRecoveryLSService::process_recovery_ls_manager() -{ - int ret = OB_SUCCESS; - share::schema::ObSchemaGetterGuard schema_guard; - const share::schema::ObTenantSchema *tenant_schema = NULL; - - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", K(ret), K(inited_)); - } else if (OB_ISNULL(GCTX.schema_service_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error", KR(ret)); - } else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard( - OB_SYS_TENANT_ID, schema_guard))) { - LOG_WARN("fail to get schema guard", KR(ret)); - } else if (OB_FAIL(schema_guard.get_tenant_info(tenant_id_, tenant_schema))) { - LOG_WARN("failed to get tenant ids", KR(ret), K(tenant_id_)); - } else if (OB_ISNULL(tenant_schema)) { - ret = OB_TENANT_NOT_EXIST; - LOG_WARN("tenant not exist", KR(ret), K(tenant_id_)); - } else { - ObTenantLSInfo tenant_stat(GCTX.sql_proxy_, tenant_schema, tenant_id_, - GCTX.srv_rpc_proxy_, GCTX.lst_operator_); - if (OB_FAIL(tenant_stat.process_ls_stats_for_recovery())) { - LOG_WARN("failed to do process in standby", KR(ret), K(tenant_id_)); - } - } - return ret; -} - -int ObRecoveryLSService::check_can_do_recovery_(const ObAllTenantInfo &tenant_info) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", K(ret), K(inited_)); - } else if (OB_UNLIKELY(!tenant_info.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("tenant_info is invalid", KR(ret), K(tenant_info)); - } else if (tenant_info.is_primary()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("primary tenant can not do recovery", KR(ret), K(tenant_info)); - } else if (tenant_info.is_standby()) { - //standby can do recovery - } else if (tenant_info.is_restore()) { - //need to check success to create init ls - share::ObPhysicalRestoreTableOperator restore_table_operator; - share::ObPhysicalRestoreJob job_info; - if (OB_ISNULL(proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql can't null", K(ret), K(proxy_)); - } else if (OB_FAIL(restore_table_operator.init(proxy_, tenant_id_))) { - LOG_WARN("fail to init restore table operator", KR(ret), K(tenant_id_)); - } else if (OB_FAIL(restore_table_operator.get_job_by_tenant_id(tenant_id_, - job_info))) { - LOG_WARN("fail to get restore job", K(ret), K(tenant_id_)); - } else if (job_info.is_valid_status_to_recovery()) { - //can do recovery - } else { - ret = OB_NEED_WAIT; - LOG_WARN("restore tenant not valid to recovery", KR(ret), K(job_info)); - } - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected tenant role", KR(ret), K(tenant_info)); - } - return ret; -} - -int ObRecoveryLSService::report_sys_ls_recovery_stat_(const SCN &sync_scn) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret), K(inited_)); - } else if (OB_ISNULL(proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql can't null", K(ret), K(proxy_)); - } else if (SCN::base_scn() != sync_scn) { - ObLSRecoveryStatOperator ls_recovery; - ObLSRecoveryStat ls_recovery_stat; - if (OB_FAIL(construct_ls_recovery_stat(sync_scn, ls_recovery_stat))) { - LOG_WARN("failed to construct ls recovery stat", KR(ret), K(sync_scn)); - } else if (OB_FAIL(ls_recovery.update_ls_recovery_stat(ls_recovery_stat, - *proxy_))) { - LOG_WARN("failed to update ls recovery stat", KR(ret), - K(ls_recovery_stat)); - } - } - return ret; -} - }//end of namespace rootserver }//end of namespace oceanbase diff --git a/src/rootserver/ob_recovery_ls_service.h b/src/rootserver/ob_recovery_ls_service.h index 97fc607a5..3d57426a6 100644 --- a/src/rootserver/ob_recovery_ls_service.h +++ b/src/rootserver/ob_recovery_ls_service.h @@ -14,11 +14,7 @@ #define OCEANBASE_ROOTSERVER_OB_RECCOVERY_LS_SERVICE_H #include "lib/thread/ob_reentrant_thread.h"//ObRsReentrantThread #include "logservice/ob_log_base_type.h"//ObIRoleChangeSubHandler ObICheckpointSubHandler ObIReplaySubHandler -#include "logservice/palf/lsn.h"//palf::LSN -#include "logservice/palf/palf_iterator.h" //PalfBufferIterator -#include "ob_primary_ls_service.h" //ObTenantThreadHelper -#include "lib/lock/ob_spin_lock.h" //ObSpinLock -#include "storage/tx/ob_multi_data_source.h" //ObTxBufferNode +#include "ob_tenant_thread_helper.h" //ObTenantThreadHelper namespace oceanbase { @@ -32,35 +28,11 @@ class ObMySQLProxy; class ObISQLClient; class ObMySQLTransaction; } -namespace share -{ -class ObLSTableOperator; -struct ObLSAttr; -struct ObLSRecoveryStat; -class SCN; -namespace schema -{ -class ObMultiVersionSchemaService; -class ObTenantSchema; -} -} -namespace logservice -{ -class ObLogHandler; -class ObGCLSLog; -} -namespace transaction -{ -class ObTxLogBlock; -} + + + namespace rootserver { -/*description: - *Restores the status of each log stream according to the logs to which the - *system log stream is synchronized, and updates the recovery progress of the - *system log stream. This thread should only exist in the standby database or - *the recovery process, and needs to be registered in the RestoreHandler. - *This thread is only active on the leader of the system log stream under the user tenant*/ class ObRecoveryLSService : public ObTenantThreadHelper { public: @@ -72,35 +44,8 @@ public: void destroy(); virtual void do_work() override; private: - //get log iterator by start_scn - int seek_log_iterator_(const share::SCN &syn_scn, - palf::PalfBufferIterator &iterator); - int process_ls_log_(const ObAllTenantInfo &tenant_info, - share::SCN &start_scn, - palf::PalfBufferIterator &iterator); - int process_upgrade_log_(const transaction::ObTxBufferNode &node); - int process_gc_log_(logservice::ObGCLSLog &gc_log, - const share::SCN &syn_scn); - int process_ls_tx_log_(transaction::ObTxLogBlock &tx_log, - const share::SCN &syn_scn); - int process_ls_operator_(const share::ObLSAttr &ls_attr, - const share::SCN &syn_scn); - int create_new_ls_(const share::ObLSAttr &ls_attr, - const share::SCN &syn_scn, - common::ObMySQLTransaction &trans); - int process_recovery_ls_manager(); - int construct_ls_recovery_stat(const share::SCN &syn_scn, - share::ObLSRecoveryStat &ls_stat); - //wait other ls is larger than sycn ts - int check_valid_to_operator_ls_(const share::ObLSAttr &ls_attr, - const share::SCN &syn_scn); - int check_can_do_recovery_(const ObAllTenantInfo &tenant_info); - //readable scn need report - int report_sys_ls_recovery_stat_(const share::SCN &sync_scn); void try_tenant_upgrade_end_(); int get_min_data_version_(uint64_t &compatible); - int process_ls_operator_in_trans_(const share::ObLSAttr &ls_attr, - const share::SCN &sync_scn, common::ObMySQLTransaction &trans); private: bool inited_; uint64_t tenant_id_; diff --git a/src/rootserver/ob_root_utils.cpp b/src/rootserver/ob_root_utils.cpp index 386d5b4be..c9dd75660 100644 --- a/src/rootserver/ob_root_utils.cpp +++ b/src/rootserver/ob_root_utils.cpp @@ -1773,7 +1773,6 @@ int ObRootUtils::get_tenant_intersection(ObUnitManager &unit_mgr, return ret; } - template bool ObRootUtils::has_intersection(const common::ObIArray &this_array, const common::ObIArray &other_array) diff --git a/src/rootserver/ob_root_utils.h b/src/rootserver/ob_root_utils.h index 11fdf9d3e..8069cfcbb 100644 --- a/src/rootserver/ob_root_utils.h +++ b/src/rootserver/ob_root_utils.h @@ -642,7 +642,6 @@ public: common::ObIArray &this_server_list, common::ObIArray &other_server_list, common::ObIArray &tenant_ids); - }; class ObClusterInfoGetter diff --git a/src/rootserver/ob_tenant_info_loader.cpp b/src/rootserver/ob_tenant_info_loader.cpp index 8f9b858c3..11d230047 100644 --- a/src/rootserver/ob_tenant_info_loader.cpp +++ b/src/rootserver/ob_tenant_info_loader.cpp @@ -267,5 +267,20 @@ int ObAllTenantInfoCache::get_tenant_info(share::ObAllTenantInfo &tenant_info) return ret; } +int ObTenantInfoLoader::get_tenant_readable_scn(SCN &readable_scn) +{ + int ret = OB_SUCCESS; + share::ObAllTenantInfo tenant_info; + if (OB_FAIL(get_tenant_info(tenant_info))) { + LOG_WARN("get_tenant_info failed", K(ret)); + } else if (OB_UNLIKELY(! tenant_info.is_valid())) { + ret = OB_EAGAIN; + LOG_WARN("tenant info not valid", K(ret), K(tenant_info)); + } else { + readable_scn = tenant_info.get_standby_scn(); + } + return ret; +} + } } diff --git a/src/rootserver/ob_tenant_info_loader.h b/src/rootserver/ob_tenant_info_loader.h index 1d1149703..7e17cbe7d 100644 --- a/src/rootserver/ob_tenant_info_loader.h +++ b/src/rootserver/ob_tenant_info_loader.h @@ -78,6 +78,7 @@ public: } virtual void run2() override; int get_tenant_info(share::ObAllTenantInfo &tenant_info); + int get_tenant_readable_scn(share::SCN &readable_scn); /** * @description: * get valid sts, only return sts refreshed after specified_time @@ -107,4 +108,4 @@ private: } // namespace rootserver } // namespace oceanbase -#endif /* !OCEANBASE_ROOTSERVER_OB_TENANT_INFO_LOADER_H */ \ No newline at end of file +#endif /* !OCEANBASE_ROOTSERVER_OB_TENANT_INFO_LOADER_H */ diff --git a/src/rootserver/ob_tenant_info_report.cpp b/src/rootserver/ob_tenant_info_report.cpp new file mode 100755 index 000000000..39948d395 --- /dev/null +++ b/src/rootserver/ob_tenant_info_report.cpp @@ -0,0 +1,114 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX RS +#include "ob_tenant_info_report.h" +#include "lib/profile/ob_trace_id.h" +#include "share/ob_errno.h" +#include "share/ob_share_util.h"//ObShareUtil +#include "observer/ob_server_struct.h"//GCTX +#include "share/ls/ob_ls_recovery_stat_operator.h"//ObLSRecoveryStatOperator + +namespace oceanbase +{ +using namespace common; +using namespace share; +using namespace transaction; +using namespace palf; +namespace rootserver +{ +//////////////ObTenantInfoReportor +int ObTenantInfoReportor::mtl_init(ObTenantInfoReportor *&ka) +{ + return ka->init(); +} + +int ObTenantInfoReportor::init() +{ + int ret = OB_SUCCESS; + tenant_id_ = MTL_ID(); + if (OB_UNLIKELY(inited_)) { + ret = OB_INIT_TWICE; + LOG_WARN("has inited", KR(ret)); + } else if (OB_FAIL(ObTenantThreadHelper::create("TenInfoRep", + lib::TGDefIDs::SimpleLSService, *this))) { + LOG_WARN("failed to create thread", KR(ret)); + } else if (OB_FAIL(ObTenantThreadHelper::start())) { + LOG_WARN("fail to start", KR(ret)); + } else { + inited_ = true; + } + return ret; +} + +void ObTenantInfoReportor::destroy() +{ + ObTenantThreadHelper::destroy(); + tenant_id_ = OB_INVALID_TENANT_ID; + inited_ = false; +} + +void ObTenantInfoReportor::do_work() +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else { + int64_t idle_time_us = 100 * 1000L; + while (!has_set_stop()) { + ObCurTraceId::init(GCONF.self_addr_); + if (is_sys_tenant(tenant_id_)) { + idle_time_us = 3600 * 1000 * 1000L; + } else if (OB_FAIL(gather_tenant_recovery_stat_())) { + LOG_WARN("failed to gather tenant recovery stat", KR(ret), K(tenant_id_)); + } + idle(idle_time_us); + }// end while + } +} + +int ObTenantInfoReportor::gather_tenant_recovery_stat_() +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (!is_meta_tenant(tenant_id_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sys or user tenant no need gather tenant recovery stat", KR(ret), K(tenant_id_)); + } else if (OB_ISNULL(GCTX.sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy is null", KR(ret)); + } else { + ObLSRecoveryStatOperator ls_recovery_op; + ObAllTenantInfoProxy info_proxy; + ObAllTenantInfo tenant_info; + const uint64_t user_tenant_id = gen_user_tenant_id(tenant_id_); + SCN sync_scn; + SCN readable_scn; + DEBUG_SYNC(BLOCK_TENANT_SYNC_SNAPSHOT_INC); + if (OB_FAIL(ls_recovery_op.get_tenant_recovery_stat( + user_tenant_id, *GCTX.sql_proxy_, sync_scn, readable_scn))) { + LOG_WARN("failed to get tenant recovery stat", KR(ret), K(user_tenant_id)); + //TODO replayable_scn is equal to sync_scn + } else if (OB_FAIL(info_proxy.update_tenant_recovery_status( + user_tenant_id, GCTX.sql_proxy_, sync_scn, + sync_scn, readable_scn))) { + LOG_WARN("failed to update tenant recovery stat", KR(ret), + K(user_tenant_id), K(sync_scn), K(readable_scn)); + } + } + return ret; +} +} +} diff --git a/src/rootserver/ob_tenant_info_report.h b/src/rootserver/ob_tenant_info_report.h new file mode 100644 index 000000000..40f546535 --- /dev/null +++ b/src/rootserver/ob_tenant_info_report.h @@ -0,0 +1,84 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_ROOTSERVER_OB_TENANT_INFO_REPORT_H +#define OCEANBASE_ROOTSERVER_OB_TENANT_INFO_REPORT_H +#include "lib/thread/ob_reentrant_thread.h"//ObRsReentrantThread +#include "logservice/ob_log_base_type.h" +#include "share/scn.h"//SCN +#include "share/ob_thread_mgr.h" //OBTGDefIDEnum +#include "lib/thread/thread_mgr_interface.h" // TGRunnable +#include "lib/lock/ob_thread_cond.h"//ObThreadCond +#include "rootserver/ob_tenant_thread_helper.h"//ObTenantThreadHelper + + +namespace oceanbase +{ +namespace obrpc +{ +class ObSrvRpcProxy; +} +namespace common +{ +class ObMySQLProxy; +class ObMySQLTransaction; +} +namespace share +{ +class ObLSTableOperator; +class SCN; + +} +namespace palf +{ +struct PalfBaseInfo; +} +namespace rootserver +{ +/*description: + * report __all_tenant_info + */ +class ObTenantInfoReportor : public ObTenantThreadHelper, + public logservice::ObICheckpointSubHandler, + public logservice::ObIReplaySubHandler +{ +public: + ObTenantInfoReportor():inited_(false), tenant_id_(OB_INVALID_TENANT_ID) {} + virtual ~ObTenantInfoReportor() {} + static int mtl_init(ObTenantInfoReportor *&ka); + int init(); + void destroy(); + virtual void do_work() override; + +public: + virtual share::SCN get_rec_scn() override { return share::SCN::max_scn();} + virtual int flush(share::SCN &scn) override { return OB_SUCCESS; } + int replay(const void *buffer, const int64_t nbytes, const palf::LSN &lsn, const share::SCN &scn) override + { + UNUSED(buffer); + UNUSED(nbytes); + UNUSED(lsn); + UNUSED(scn); + return OB_SUCCESS; + } +private: + int gather_tenant_recovery_stat_(); +private: + bool inited_; + uint64_t tenant_id_; + +}; +} +} + + +#endif /* !OCEANBASE_ROOTSERVER_OB_TENANT_INFO_REPORT_H */ diff --git a/src/rootserver/ob_tenant_role_transition_service.cpp b/src/rootserver/ob_tenant_role_transition_service.cpp index 9416f3170..46185b672 100644 --- a/src/rootserver/ob_tenant_role_transition_service.cpp +++ b/src/rootserver/ob_tenant_role_transition_service.cpp @@ -309,8 +309,6 @@ int ObTenantRoleTransitionService::do_prepare_flashback_for_failover_to_primary_ } else if (OB_UNLIKELY(switchover_epoch_ != tenant_info.get_switchover_epoch())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("tenant switchover status not valid", KR(ret), K(tenant_info), K_(switchover_epoch)); - } else if (OB_FAIL(update_tenant_stat_info_())) { - LOG_WARN("failed to update tenant stat info", KR(ret), K(tenant_info), K_(switchover_epoch)); } else if (OB_FAIL(OB_PRIMARY_STANDBY_SERVICE.do_recover_tenant(tenant_id_, share::PREPARE_FLASHBACK_FOR_FAILOVER_TO_PRIMARY_SWITCHOVER_STATUS, obrpc::ObRecoverTenantArg::RecoverType::CANCEL, @@ -338,15 +336,20 @@ int ObTenantRoleTransitionService::do_switch_access_mode_to_flashback( const share::ObAllTenantInfo &tenant_info) { int ret = OB_SUCCESS; - + ObLSStatusOperator ls_status_op; + share::ObLSStatusInfoArray all_ls_status_array; if (OB_FAIL(check_inner_stat())) { LOG_WARN("error unexpected", KR(ret), K(tenant_id_), KP(sql_proxy_), KP(rpc_proxy_)); } else if (OB_UNLIKELY(!tenant_info.is_flashback_status() || switchover_epoch_ != tenant_info.get_switchover_epoch())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("tenant switchover status not valid", KR(ret), K(tenant_info), K(switchover_epoch_)); - } else if (OB_FAIL(change_ls_access_mode_(palf::AccessMode::FLASHBACK, SCN::base_scn()))) { - LOG_WARN("failed to get access mode", KR(ret), K(tenant_info)); + } else if (OB_FAIL(ls_status_op.get_all_ls_status_by_order_for_switch_tenant(tenant_id_, + false/* ignore_need_create_abort */, all_ls_status_array, *sql_proxy_))) { + LOG_WARN("failed to get_all_ls_status_by_order", KR(ret), K(tenant_id_)); + } else if (OB_FAIL(change_ls_access_mode_(all_ls_status_array, + palf::AccessMode::FLASHBACK, SCN::base_scn()))) { + LOG_WARN("failed to get access mode", KR(ret), K(tenant_info), K(all_ls_status_array)); } return ret; } @@ -400,7 +403,10 @@ int ObTenantRoleTransitionService::do_switch_access_mode_to_append( { int ret = OB_SUCCESS; palf::AccessMode access_mode = logservice::ObLogService::get_palf_access_mode(target_tenant_role); - const SCN &ref_scn = tenant_info.get_sync_scn(); + SCN ref_scn; + ObLSRecoveryStatOperator ls_recovery_op; + ObLSStatusOperator status_op; + ObLSStatusInfoArray status_info_array; if (OB_FAIL(check_inner_stat())) { LOG_WARN("error unexpected", KR(ret), K(tenant_id_), KP(sql_proxy_), KP(rpc_proxy_)); } else if (OB_UNLIKELY(!tenant_info.is_switching_to_primary_status() @@ -410,8 +416,25 @@ int ObTenantRoleTransitionService::do_switch_access_mode_to_append( LOG_WARN("tenant switchover status not valid", KR(ret), K(tenant_info), K(target_tenant_role), K(switchover_epoch_)); //TODO(yaoying):xianming - } else if (OB_FAIL(change_ls_access_mode_(access_mode, ref_scn))) { - LOG_WARN("failed to get access mode", KR(ret), K(access_mode), K(ref_scn), K(tenant_info)); + } else if (OB_ISNULL(sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy is null", KR(ret)); + } else if (OB_FAIL(ls_recovery_op.get_tenant_max_sync_scn(tenant_id_, *sql_proxy_, ref_scn))) { + LOG_WARN("failed to get tenant max sync scn", KR(ret), K(tenant_id_)); + } else { + // After flashback, it is necessary to ensure that the newly written log + // must be greater than all the sync_scn that have been reported, so ref_ts + // will be set as the largest sync_scn of all ls. Since sys_recovery_scn is + // reported independently, the maximum value of these two values should be taken . + ref_scn = SCN::max(tenant_info.get_sys_recovery_scn(), ref_scn); + } + + if (FAILEDx(status_op.get_all_ls_status_by_order_for_switch_tenant( + tenant_id_, false/*ignore need create abort*/, status_info_array, *sql_proxy_))) { + LOG_WARN("failed to get all ls status info", KR(ret), K(tenant_id_)); + } else if (OB_FAIL(change_ls_access_mode_(status_info_array, access_mode, ref_scn))) { + LOG_WARN("failed to get access mode", KR(ret), K(access_mode), K(ref_scn), + K(tenant_info), K(status_info_array)); } else { common::ObMySQLTransaction trans; share::ObAllTenantInfo cur_tenant_info; @@ -448,12 +471,14 @@ int ObTenantRoleTransitionService::do_switch_access_mode_to_append( return ret; } -int ObTenantRoleTransitionService::do_switch_access_mode_to_raw_rw( - const share::ObAllTenantInfo &tenant_info) + +int ObTenantRoleTransitionService::switchover_to_standby(const share::ObAllTenantInfo &tenant_info) { int ret = OB_SUCCESS; palf::AccessMode access_mode = logservice::ObLogService::get_palf_access_mode(STANDBY_TENANT_ROLE); ObLSStatusOperator status_op; + ObLSStatusInfoArray status_info_array; + share::ObLSStatusInfo sys_ls; if (OB_FAIL(check_inner_stat())) { LOG_WARN("error unexpected", KR(ret), K(tenant_id_), KP(sql_proxy_), KP(rpc_proxy_)); @@ -463,30 +488,69 @@ int ObTenantRoleTransitionService::do_switch_access_mode_to_raw_rw( ret = OB_INVALID_ARGUMENT; LOG_WARN("tenant switchover status not valid", KR(ret), K(tenant_info), K(switchover_epoch_)); + } else if (OB_FAIL(status_op.get_ls_status_info(tenant_id_, SYS_LS, sys_ls, *sql_proxy_))) { + LOG_WARN("failed to get sys ls status info", KR(ret), K(tenant_id_)); + } else if (OB_FAIL(status_info_array.push_back(sys_ls))) { + LOG_WARN("failed to push back", KR(ret), K(sys_ls)); + } else if (OB_FAIL(change_ls_access_mode_(status_info_array, access_mode, SCN::base_scn()))) { + LOG_WARN("failed to switch sys ls access mode", KR(ret), K(status_info_array), K(access_mode)); + } else if (OB_FAIL(wait_sys_recovery_finish_(status_info_array))) { + LOG_WARN("faild to wait sys recovery finish", KR(ret), K(status_info_array)); } else if (OB_FAIL(status_op.create_abort_ls_in_switch_tenant( tenant_id_, tenant_info.get_switchover_status(), tenant_info.get_switchover_epoch(), *sql_proxy_))) { LOG_WARN("failed to create abort ls", KR(ret), K(tenant_info)); - } else if (OB_FAIL(change_ls_access_mode_(access_mode, SCN::base_scn()))) { + } else if (OB_FAIL(status_op.get_all_ls_status_by_order_for_switch_tenant( + tenant_id_, false/*ignore need create abort*/, status_info_array, *sql_proxy_))) { + LOG_WARN("failed to get all ls status info", KR(ret), K(tenant_id_)); + } else if (OB_FAIL(change_ls_access_mode_(status_info_array, access_mode, SCN::base_scn()))) { LOG_WARN("failed to get access mode", KR(ret), K(access_mode), K(tenant_info)); } return ret; } -int ObTenantRoleTransitionService::update_tenant_stat_info_() +int ObTenantRoleTransitionService::wait_sys_recovery_finish_(const ObLSStatusInfoIArray &ls_status_array) { int ret = OB_SUCCESS; + ObArray checkpoints; + if (OB_FAIL(check_inner_stat())) { LOG_WARN("error unexpected", KR(ret), K(tenant_id_), KP(sql_proxy_), KP(rpc_proxy_)); + } else if (OB_FAIL(get_checkpoints_by_rpc_(tenant_id_, ls_status_array, false, checkpoints))) { + LOG_WARN("failed to get checkpoints by rpc", KR(ret), K(tenant_id_), K(ls_status_array)); + } else if (OB_UNLIKELY(1 != checkpoints.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("expectect only one checkpoint", KR(ret), K(ls_status_array)); } else { - //TODO, get all ls sync_scn, update __all_tenant_info, - //the new sync_scn cannot larger than recovery_scn and sync_scn of sys_ls + //wait sys recovery scn to latest + ObAllTenantInfo curr_tenant_info; + bool sync_to_latest = false; + const SCN sys_scn = checkpoints.at(0).cur_sync_scn_; + while (!THIS_WORKER.is_timeout() && !sync_to_latest && OB_SUCC(ret)) { + //ignore error + if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(tenant_id_, sql_proxy_, false, curr_tenant_info))) { + LOG_WARN("failed to load tenant info", KR(ret), K(tenant_id_)); + } else if (curr_tenant_info.get_sys_recovery_scn() >= sys_scn) { + sync_to_latest = true; + } else { + LOG_WARN("sys ls not recovery finish, need wait", K(tenant_id_), K(sys_scn), K(curr_tenant_info)); + usleep(10L * 1000L); + } + } + if (THIS_WORKER.is_timeout() || !sync_to_latest) { + // return NOT_ALLOW instead of timeout + ret = OB_OP_NOT_ALLOW; + LOG_WARN("wait sys ls recovery finish timeout, can not swithover to standby", KR(ret)); + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "wait sys ls recovery timeout, switchover to standby"); + } else if (OB_SUCC(ret)) { + LOG_INFO("finish check sync to latest"); + } } - LOG_INFO("[ROLE_TRANSITION] finish update tenant stat info", KR(ret), K(tenant_id_)); return ret; } -int ObTenantRoleTransitionService::change_ls_access_mode_(palf::AccessMode target_access_mode, +int ObTenantRoleTransitionService::change_ls_access_mode_(const share::ObLSStatusInfoIArray &status_info_array, + palf::AccessMode target_access_mode, const SCN &ref_scn) { int ret = OB_SUCCESS; @@ -513,8 +577,11 @@ int ObTenantRoleTransitionService::change_ls_access_mode_(palf::AccessMode targe need_retry = false; ret = OB_TIMEOUT; LOG_WARN("already timeout", KR(ret)); - } else if (OB_FAIL(get_ls_access_mode_(ls_mode_info))) { + } else if (OB_FAIL(get_ls_access_mode_(status_info_array, ls_mode_info))) { LOG_WARN("failed to get ls access mode", KR(ret)); + } else if (OB_UNLIKELY(status_info_array.count() != ls_mode_info.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls count not equal to ls mode info count", KR(ret), K(status_info_array), K(ls_mode_info)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < ls_mode_info.count(); ++i) { const LSAccessModeInfo &info = ls_mode_info.at(i); @@ -559,7 +626,8 @@ int ObTenantRoleTransitionService::change_ls_access_mode_(palf::AccessMode targe } -int ObTenantRoleTransitionService::get_ls_access_mode_(ObIArray &ls_access_info) +int ObTenantRoleTransitionService::get_ls_access_mode_(const share::ObLSStatusInfoIArray &status_info_array, + ObIArray &ls_access_info) { int ret = OB_SUCCESS; ls_access_info.reset(); @@ -569,14 +637,10 @@ int ObTenantRoleTransitionService::get_ls_access_mode_(ObIArray= GTS @@ -845,10 +910,11 @@ int ObTenantRoleTransitionService::switchover_update_tenant_status( final_replayable_scn, final_readable_scn, final_recovery_until_scn, + final_sys_recovery_scn, old_switchover_epoch))) { LOG_WARN("failed to update_tenant_status", KR(ret), K(tenant_id), K(new_role), K(old_status), K(new_status), K(final_sync_scn), K(final_replayable_scn), - K(final_readable_scn), K(final_recovery_until_scn), K(old_switchover_epoch)); + K(final_readable_scn), K(final_recovery_until_scn), K(old_switchover_epoch), K(final_sys_recovery_scn)); } else if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info( tenant_id, &trans, false /* for update */, new_tenant_info))) { LOG_WARN("failed to load tenant info", KR(ret), K(tenant_id)); @@ -898,14 +964,17 @@ int ObTenantRoleTransitionService::wait_tenant_sync_to_latest_until_timeout_( if (OB_FAIL(ret) || !has_restore_source) { } else { bool has_sync_to_latest = false; - while (!THIS_WORKER.is_timeout()) { - has_sync_to_latest = false; - if (OB_FAIL(check_sync_to_latest_(tenant_id, tenant_info, has_sync_to_latest))) { + ObAllTenantInfo new_tenant_info; + while (!THIS_WORKER.is_timeout() && !has_sync_to_latest) { + //ignore error + if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(tenant_id_, sql_proxy_, + false, new_tenant_info))) { + LOG_WARN("failed to load tenant info", KR(ret), K(tenant_id_)); + } else if (OB_FAIL(check_sync_to_latest_(tenant_id, new_tenant_info, has_sync_to_latest))) { LOG_WARN("fail to check_sync_to_latest_", KR(ret), K(tenant_id), - K(tenant_info), K(has_sync_to_latest)); + K(new_tenant_info), K(has_sync_to_latest)); } else if (has_sync_to_latest) { LOG_INFO("sync to latest", K(has_sync_to_latest), K(tenant_id)); - break; } else { LOG_WARN("not sync to latest, wait a while", K(tenant_id)); } @@ -944,7 +1013,6 @@ int ObTenantRoleTransitionService::check_sync_to_latest_(const uint64_t tenant_i share::ObLSStatusInfoArray sys_ls_status_array; common::ObArray switchover_checkpoints; ObLSRecoveryStatOperator ls_recovery_operator; - ObLSRecoveryStat sys_ls_recovery_stat; SCN sys_ls_sync_scn = SCN::min_scn(); bool sys_ls_sync_to_latest = false; share::ObLSStatusInfo ls_status; @@ -967,15 +1035,12 @@ int ObTenantRoleTransitionService::check_sync_to_latest_(const uint64_t tenant_i ret = OB_ERR_UNEXPECTED; LOG_WARN("checkpoints count is not 1", KR(ret), K(switchover_checkpoints), K(tenant_id), K(tenant_info), K(sys_ls_status_array)); - } else if (OB_FAIL(ls_recovery_operator.get_ls_recovery_stat(tenant_id, SYS_LS, - false/*for_update*/, sys_ls_recovery_stat, *sql_proxy_))) { - LOG_WARN("failed to get ls recovery stat", KR(ret), K(tenant_id)); } else if (OB_FAIL(get_sys_ls_sync_scn_(switchover_checkpoints, sys_ls_sync_scn, sys_ls_sync_to_latest))) { LOG_WARN("failed to get_sys_ls_sync_scn_", KR(ret), K(switchover_checkpoints)); } else if (!(sys_ls_sync_scn.is_valid_and_not_min() && sys_ls_sync_to_latest - && sys_ls_recovery_stat.get_sync_scn() == sys_ls_sync_scn)) { + && tenant_info.get_sys_recovery_scn() == sys_ls_sync_scn)) { LOG_WARN("sys ls not sync, keep waiting", KR(ret), K(sys_ls_sync_scn), K(sys_ls_sync_to_latest), - K(sys_ls_recovery_stat), K(switchover_checkpoints)); + K(tenant_info), K(switchover_checkpoints)); // SYS LS is sync, check other LS } else if (OB_FAIL(ls_status_op.get_all_ls_status_by_order_for_switch_tenant(tenant_id, true/* ignore_need_create_abort */, all_ls_status_array, *sql_proxy_))) { diff --git a/src/rootserver/ob_tenant_role_transition_service.h b/src/rootserver/ob_tenant_role_transition_service.h index 8529c7d68..277788670 100644 --- a/src/rootserver/ob_tenant_role_transition_service.h +++ b/src/rootserver/ob_tenant_role_transition_service.h @@ -97,10 +97,10 @@ public: switch_optype_(switch_optype) {} virtual ~ObTenantRoleTransitionService() {} int failover_to_primary(); + int switchover_to_standby(const share::ObAllTenantInfo &tenant_info); int check_inner_stat(); int do_switch_access_mode_to_append(const share::ObAllTenantInfo &tenant_info, const share::ObTenantRole &target_tenant_role); - int do_switch_access_mode_to_raw_rw(const share::ObAllTenantInfo &tenant_info); void set_switchover_epoch(const int64_t switchover_epoch) { switchover_epoch_ = switchover_epoch; @@ -156,10 +156,11 @@ private: int do_failover_to_primary_(const share::ObAllTenantInfo &tenant_info); int do_prepare_flashback_(share::ObAllTenantInfo &tenant_info); int do_flashback_(const share::ObAllTenantInfo &tenant_info); - int change_ls_access_mode_(palf::AccessMode target_access_mode, + int change_ls_access_mode_(const share::ObLSStatusInfoIArray &status_info_array, + palf::AccessMode target_access_mode, const share::SCN &ref_scn); - int update_tenant_stat_info_(); - int get_ls_access_mode_(ObIArray &ls_access_info); + int get_ls_access_mode_(const share::ObLSStatusInfoIArray &status_info_array, + ObIArray &ls_access_info); int do_change_ls_access_mode_(const ObIArray &ls_access_info, palf::AccessMode target_access_mode, const share::SCN &ref_scn); @@ -219,6 +220,8 @@ private: int do_prepare_flashback_for_switch_to_primary_(share::ObAllTenantInfo &tenant_info); int do_prepare_flashback_for_failover_to_primary_(share::ObAllTenantInfo &tenant_info); + //switchover + int wait_sys_recovery_finish_(const ObLSStatusInfoIArray &ls_status); private: const static int64_t SEC_UNIT = 1000L * 1000L; const static int64_t PRINT_INTERVAL = 10 * 1000 * 1000L; diff --git a/src/rootserver/ob_tenant_thread_helper.cpp b/src/rootserver/ob_tenant_thread_helper.cpp new file mode 100755 index 000000000..7b36b3ecf --- /dev/null +++ b/src/rootserver/ob_tenant_thread_helper.cpp @@ -0,0 +1,305 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX RS +#include "ob_tenant_thread_helper.h" +#include "lib/profile/ob_trace_id.h" +#include "lib/thread/thread_mgr.h"//TG +#include "share/ob_errno.h" +#include "share/schema/ob_schema_struct.h"//ObTenantInfo +#include "share/schema/ob_schema_service.h"//ObMultiSchemaService +#include "share/schema/ob_schema_getter_guard.h"//ObSchemaGetterGuard +#include "share/ob_share_util.h"//ObShareUtil +#include "share/ob_tenant_info_proxy.h"//ObAllTenantInfo +#include "share/restore/ob_physical_restore_table_operator.h"//restore_job +#include "share/restore/ob_physical_restore_info.h"//restore_info +#include "share/ob_primary_zone_util.h"//get_ls_primary_zone_priority +#include "observer/ob_server_struct.h"//GCTX + +namespace oceanbase +{ +using namespace common; +using namespace share; +using namespace transaction; +using namespace palf; +using namespace lib; +namespace rootserver +{ +//////////////ObTenantThreadHelper +int ObTenantThreadHelper::create( + const char* thread_name, int tg_def_id, ObTenantThreadHelper &tenant_thread) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(is_created_)) { + ret = OB_INIT_TWICE; + LOG_WARN("has inited", KR(ret)); + } else if (OB_ISNULL(thread_name)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("thread name is null", KR(ret)); + } else if (OB_FAIL(TG_CREATE_TENANT(tg_def_id, tg_id_))) { + LOG_ERROR("create tg failed", KR(ret)); + } else if (OB_FAIL(TG_SET_RUNNABLE(tg_id_, *this))) { + LOG_ERROR("set thread runable fail", KR(ret)); + } else if (OB_FAIL(thread_cond_.init(ObWaitEventIds::REENTRANT_THREAD_COND_WAIT))) { + LOG_WARN("fail to init cond, ", KR(ret)); + } else { + thread_name_ = thread_name; + is_created_ = true; + is_first_time_to_start_ = true; + } + return ret; +} + +int ObTenantThreadHelper::start() +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_created_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (is_first_time_to_start_) { + if (OB_FAIL(TG_START(tg_id_))) { + LOG_WARN("fail ed to start at first time", KR(ret), K(tg_id_), K(thread_name_)); + } else { + is_first_time_to_start_ = false; + } + } else if (OB_FAIL(TG_REENTRANT_LOGICAL_START(tg_id_))) { + LOG_WARN("failed to start", KR(ret)); + } + LOG_INFO("[TENANT THREAD] thread start", KR(ret), K(tg_id_), K(thread_name_)); + return ret; +} + +void ObTenantThreadHelper::stop() +{ + LOG_INFO("[TENANT THREAD] thread stop start", K(tg_id_), K(thread_name_)); + if (-1 != tg_id_) { + TG_REENTRANT_LOGICAL_STOP(tg_id_); + } + LOG_INFO("[TENANT THREAD] thread stop finish", K(tg_id_), K(thread_name_)); +} + +void ObTenantThreadHelper::wait() +{ + LOG_INFO("[TENANT THREAD] thread wait start", K(tg_id_), K(thread_name_)); + if (-1 != tg_id_) { + TG_REENTRANT_LOGICAL_WAIT(tg_id_); + } + LOG_INFO("[TENANT THREAD] thread wait finish", K(tg_id_), K(thread_name_)); +} + +void ObTenantThreadHelper::destroy() +{ + LOG_INFO("[TENANT THREAD] thread destory start", K(tg_id_), K(thread_name_)); + if (-1 != tg_id_) { + TG_STOP(tg_id_); + { + ObThreadCondGuard guard(thread_cond_); + thread_cond_.broadcast(); + } + TG_WAIT(tg_id_); + TG_DESTROY(tg_id_); + tg_id_ = -1; + } + is_created_ = false; + is_first_time_to_start_ = true; + LOG_INFO("[TENANT THREAD] thread destory finish", K(tg_id_), K(thread_name_)); +} + +void ObTenantThreadHelper::switch_to_follower_forcedly() +{ + stop(); +} +int ObTenantThreadHelper::switch_to_leader() +{ + int ret = OB_SUCCESS; + LOG_INFO("[TENANT THREAD] thread start", K(tg_id_), K(thread_name_)); + if (OB_FAIL(start())) { + LOG_WARN("failed to start thread", KR(ret)); + } else { + ObThreadCondGuard guard(thread_cond_); + if (OB_FAIL(thread_cond_.broadcast())) { + LOG_WARN("failed to weakup thread cond", KR(ret)); + } + } + LOG_INFO("[TENANT THREAD] thread start finish", K(tg_id_), K(thread_name_)); + return ret; +} +int ObTenantThreadHelper::wait_tenant_data_version_ready_( + const uint64_t tenant_id, const uint64_t &data_version) +{ + int ret = OB_SUCCESS; + bool is_ready = false; + uint64_t tenant_data_version = 0; + while (!is_ready && !has_set_stop()) { + ret = OB_SUCCESS; + + if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, tenant_data_version))) { + LOG_WARN("failed to get min data version", KR(ret), K(tenant_id)); + } else if (tenant_data_version < data_version) { + ret = OB_NEED_WAIT; + LOG_WARN("tenant version not target, need wait", KR(ret), + K(tenant_data_version), K(data_version)); + } else { + is_ready = true; + } + + if (!is_ready) { + idle(10 * 1000 * 1000); + } + } + + if (has_set_stop()) { + LOG_WARN("thread has been stopped", K(is_ready), K(tenant_id)); + ret = OB_IN_STOP_STATE; + } + return ret; +} + +int ObTenantThreadHelper::wait_tenant_schema_and_version_ready_( + const uint64_t tenant_id, const uint64_t &data_version) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(GCTX.schema_service_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("schema ptr is null", KR(ret), KP(GCTX.schema_service_)); + } else if (OB_FAIL(wait_tenant_data_version_ready_(tenant_id, data_version))) { + LOG_WARN("failed to wait tenant data version", KR(ret), K(tenant_id), K(data_version)); + } else { + bool is_ready = false; + share::schema::ObTenantSchema tenant_schema; + while (!is_ready && !has_set_stop()) { + ret = OB_SUCCESS; + if (OB_FAIL(get_tenant_schema(tenant_id, tenant_schema))) { + LOG_WARN("failed to get tenant schema", KR(ret), K(tenant_id)); + } else if (!tenant_schema.is_normal()) { + ret = OB_NEED_WAIT; + LOG_WARN("tenant schema not ready, no need tenant balance", KR(ret), K(tenant_schema)); + } else { + is_ready = true; + } + + if (!is_ready) { + idle(10 * 1000 *1000); + } + } + + if (has_set_stop()) { + LOG_WARN("thread has been stopped", K(is_ready), K(tenant_id)); + ret = OB_IN_STOP_STATE; + } + } + return ret; +} + +void ObTenantThreadHelper::run1() { + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_created_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else { + lib::set_thread_name(thread_name_); + LOG_INFO("thread run", K(thread_name_)); + do_work(); + } +} +void ObTenantThreadHelper::idle(const int64_t idle_time_us) +{ + ObThreadCondGuard guard(thread_cond_); + thread_cond_.wait_us(idle_time_us); +} + +int ObTenantThreadHelper::get_tenant_schema(const uint64_t tenant_id, + share::schema::ObTenantSchema &tenant_schema) +{ + int ret = OB_SUCCESS; + share::schema::ObSchemaGetterGuard schema_guard; + const share::schema::ObTenantSchema *cur_tenant_schema = NULL; + if (!is_valid_tenant_id(tenant_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tenant id is invalid", KR(ret), K(tenant_id)); + } else if (OB_ISNULL(GCTX.schema_service_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error", KR(ret), KP(GCTX.schema_service_)); + } else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard( + OB_SYS_TENANT_ID, schema_guard))) { + LOG_WARN("fail to get schema guard", KR(ret)); + } else if (OB_FAIL(schema_guard.get_tenant_info(tenant_id, + cur_tenant_schema))) { + LOG_WARN("failed to get tenant ids", KR(ret), K(tenant_id)); + } else if (OB_ISNULL(cur_tenant_schema)) { + ret = OB_TENANT_NOT_EXIST; + LOG_WARN("tenant not exist", KR(ret), K(tenant_id)); + } else if (OB_FAIL(tenant_schema.assign(*cur_tenant_schema))) { + LOG_WARN("failed to get cur tenant schema", KR(ret), KP(cur_tenant_schema)); + } + return ret; +} + +int ObTenantThreadHelper::check_can_do_recovery(const ObAllTenantInfo &tenant_info) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!tenant_info.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tenant_info is invalid", KR(ret), K(tenant_info)); + } else if (tenant_info.get_sys_recovery_scn().is_min()) { + //during upgrade, not expected + ret = OB_NEED_RETRY; + LOG_WARN("sys recovery scn can not be min now", KR(ret), K(tenant_info)); + } else if (OB_ISNULL(GCTX.sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sql proxy is null", KR(ret)); + } else if (tenant_info.is_primary()) { + } else if (tenant_info.is_standby()) { + //standby can do recovery + } else if (tenant_info.is_restore()) { + //need to check success to create init ls + const uint64_t tenant_id = tenant_info.get_tenant_id(); + share::ObPhysicalRestoreTableOperator restore_table_operator; + share::ObPhysicalRestoreJob job_info; + if (FAILEDx(restore_table_operator.init(GCTX.sql_proxy_, tenant_id))) { + LOG_WARN("fail to init restore table operator", KR(ret), K(tenant_id)); + } else if (OB_FAIL(restore_table_operator.get_job_by_tenant_id(tenant_id, + job_info))) { + LOG_WARN("fail to get restore job", K(ret), K(tenant_id)); + } else if (job_info.is_valid_status_to_recovery()) { + //can do recovery + } else { + ret = OB_NEED_WAIT; + LOG_WARN("restore tenant not valid to recovery", KR(ret), K(job_info)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected tenant role", KR(ret), K(tenant_info)); + } + return ret; +} + +int ObTenantThreadHelper::get_zone_priority(const ObZone &primary_zone, + const share::schema::ObTenantSchema &tenant_schema, + ObSqlString &primary_zone_str) +{ + int ret = OB_SUCCESS; + primary_zone_str.reset(); + if (OB_UNLIKELY(!tenant_schema.is_valid() || primary_zone.is_empty())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(primary_zone), K(tenant_schema)); + } else if (OB_FAIL(ObPrimaryZoneUtil::get_ls_primary_zone_priority(primary_zone, + tenant_schema, primary_zone_str))) { + LOG_WARN("failed to get ls primary zone priority", KR(ret), K(primary_zone), K(tenant_schema)); + } + LOG_DEBUG("get zone priority", KR(ret), K(primary_zone_str), K(tenant_schema)); + return ret; +} + + +}//end of rootserver +} diff --git a/src/rootserver/ob_tenant_thread_helper.h b/src/rootserver/ob_tenant_thread_helper.h new file mode 100644 index 000000000..95bad8104 --- /dev/null +++ b/src/rootserver/ob_tenant_thread_helper.h @@ -0,0 +1,101 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_ROOTSERVER_OB_TENANT_THREAD_HELPER_H +#define OCEANBASE_ROOTSERVER_OB_TENANT_THREAD_HELPER_H +#include "lib/thread/ob_reentrant_thread.h"//ObRsReentrantThread +#include "logservice/ob_log_base_type.h" +#include "share/ob_thread_mgr.h" //OBTGDefIDEnum +#include "lib/thread/thread_mgr_interface.h" // TGRunnable +#include "lib/lock/ob_thread_cond.h"//ObThreadCond +#include "common/ob_zone.h"//ObZone + + +namespace oceanbase +{ +namespace obrpc +{ +class ObSrvRpcProxy; +} +namespace common +{ +class ObSqlString; +} +namespace share +{ +class ObAllTenantInfo; +namespace schema +{ +class ObTenantSchema; +} +} +namespace logservice +{ +class ObLogHandler; +} + +namespace rootserver +{ +class ObTenantThreadHelper : public lib::TGRunnable, + public logservice::ObIRoleChangeSubHandler +{ +public: + ObTenantThreadHelper() : tg_id_(-1), thread_cond_(), is_created_(false), is_first_time_to_start_(true), thread_name_("") {} + virtual ~ObTenantThreadHelper() {} + virtual void do_work() = 0; + virtual void run1() override; + virtual void destroy(); + int start(); + void stop(); + void wait(); + int create(const char* thread_name, int tg_def_id, ObTenantThreadHelper &tenant_thread); + void idle(const int64_t idle_time_us); +public: + virtual void switch_to_follower_forcedly() override; + + virtual int switch_to_leader() override; + virtual int switch_to_follower_gracefully() override + { + stop(); + return OB_SUCCESS; + } + virtual int resume_leader() override + { + return OB_SUCCESS; + } +public: + static int get_tenant_schema(const uint64_t tenant_id, + share::schema::ObTenantSchema &tenant_schema); + static int check_can_do_recovery(const share::ObAllTenantInfo &tenant_info); + static int get_zone_priority(const ObZone &primary_zone, + const share::schema::ObTenantSchema &tenant_schema, + common::ObSqlString &primary_zone_str); + +protected: + int wait_tenant_schema_and_version_ready_( + const uint64_t tenant_id, const uint64_t &data_version); + int wait_tenant_data_version_ready_( + const uint64_t tenant_id, const uint64_t &data_version); + int tg_id_; +private: + common::ObThreadCond thread_cond_; + bool is_created_; + bool is_first_time_to_start_; + const char* thread_name_; +}; + + +} +} + + +#endif /* !OCEANBASE_ROOTSERVER_OB_TENANT_THREAD_HELPER_H */ diff --git a/src/rootserver/restore/ob_restore_scheduler.cpp b/src/rootserver/restore/ob_restore_scheduler.cpp index 3d9291a3d..00323580b 100644 --- a/src/rootserver/restore/ob_restore_scheduler.cpp +++ b/src/rootserver/restore/ob_restore_scheduler.cpp @@ -15,7 +15,8 @@ #include "ob_restore_scheduler.h" #include "rootserver/ob_rs_event_history_table_operator.h" #include "rootserver/ob_unit_manager.h"//convert_pool_name_lis -#include "rootserver/ob_tenant_role_transition_service.h"//failover_to_primary +#include "rootserver/ob_ls_service_helper.h"//create_new_ls_in_trans +#include "rootserver/ob_common_ls_service.h"//do_create_user_ls #include "share/ob_schema_status_proxy.h" #include "share/schema/ob_schema_utils.h" #include "share/schema/ob_schema_mgr.h" @@ -976,19 +977,16 @@ int ObRestoreService::restore_init_ls(const share::ObPhysicalRestoreJob &job_inf } else if (OB_FAIL(store.read_ls_attr_info(backup_ls_attr))) { LOG_WARN("failed to read ls info", KR(ret)); } else { - const SCN &sync_scn = backup_ls_attr.backup_scn_; - const SCN readable_scn = SCN::base_scn(); - ObLSRecoveryStatOperator ls_recovery; - ObLSRecoveryStat ls_recovery_stat; - LOG_INFO("start to create ls and set sync scn", K(sync_scn), K(backup_ls_attr)); - if (OB_FAIL(ls_recovery_stat.init_only_recovery_stat( - tenant_id_, SYS_LS, sync_scn, readable_scn))) { - LOG_WARN("failed to init ls recovery stat", KR(ret), K(backup_ls_attr.backup_scn_), - K(sync_scn), K(readable_scn)); - } else if (OB_FAIL(ls_recovery.update_ls_recovery_stat(ls_recovery_stat, - *sql_proxy_))) { - LOG_WARN("failed to update ls recovery stat", KR(ret), - K(ls_recovery_stat)); + const SCN &sys_recovery_scn = backup_ls_attr.backup_scn_; + ObAllTenantInfo tenant_info; + LOG_INFO("start to create ls and set sys_recovery_scn", K(sys_recovery_scn), K(backup_ls_attr)); + if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(tenant_id_, sql_proxy_, false, tenant_info))) { + LOG_WARN("failed to update tenant info", KR(ret), K(tenant_id_)); + } else if (tenant_info.get_sys_recovery_scn() >= sys_recovery_scn) { + //no need update sys recovery scn + } else if (OB_FAIL(ObAllTenantInfoProxy::update_tenant_sys_recovery_scn(tenant_id_, sys_recovery_scn, + false, sql_proxy_))) { + LOG_WARN("failed to update tenant sys recovery scn", KR(ret), K(tenant_id_), K(sys_recovery_scn), K(tenant_info)); } } if (FAILEDx(create_all_ls_(*tenant_schema, backup_ls_attr.ls_attr_array_))) { @@ -1056,7 +1054,7 @@ int ObRestoreService::create_all_ls_( LOG_INFO("[RESTORE] ls already exist", K(ls_info), K(tenant_id_)); } else if (OB_ENTRY_NOT_EXIST != ret) { LOG_WARN("failed to get ls status info", KR(ret), K(tenant_id_), K(ls_info)); - } else if (OB_FAIL(tenant_stat.create_new_ls_for_recovery( + } else if (OB_FAIL(tenant_stat.create_new_ls_in_trans( ls_info.get_ls_id(), ls_info.get_ls_group_id(), ls_info.get_create_scn(), trans))) { LOG_WARN("failed to add new ls status info", KR(ret), K(ls_info)); @@ -1096,12 +1094,10 @@ int ObRestoreService::wait_all_ls_created_(const share::schema::ObTenantSchema & ObLSRecoveryStat recovery_stat; ObLSRecoveryStatOperator ls_recovery_operator; - ObTenantLSInfo tenant_stat(sql_proxy_, &tenant_schema, tenant_id_, - srv_rpc_proxy_, GCTX.lst_operator_); if (OB_FAIL(status_op.get_all_ls_status_by_order(tenant_id, ls_array, *sql_proxy_))) { LOG_WARN("failed to get all ls status", KR(ret), K(tenant_id)); - } + } for (int64_t i = 0; OB_SUCC(ret) && i < ls_array.count(); ++i) { const ObLSStatusInfo &info = ls_array.at(i); if (info.ls_is_creating()) { @@ -1113,10 +1109,11 @@ int ObRestoreService::wait_all_ls_created_(const share::schema::ObTenantSchema & job_info, info.ls_id_, palf_base_info))) { LOG_WARN("failed to get restore ls palf info", KR(ret), K(info), K(job_info)); - } else if (OB_FAIL(tenant_stat.create_ls_with_palf(info, recovery_stat.get_create_scn(), - true,/*create with palf*/ - palf_base_info))) { - LOG_WARN("failed to create ls with palf", KR(ret), K(info), + } else if (OB_FAIL(ObCommonLSService::do_create_user_ls( + tenant_schema, info, recovery_stat.get_create_scn(), + true, /*create with palf*/ + palf_base_info))) { + LOG_WARN("failed to create ls with palf", KR(ret), K(info), K(tenant_schema), K(palf_base_info)); } } diff --git a/src/rootserver/restore/ob_restore_scheduler.h b/src/rootserver/restore/ob_restore_scheduler.h index 4ffa13a24..3ddcb5441 100644 --- a/src/rootserver/restore/ob_restore_scheduler.h +++ b/src/rootserver/restore/ob_restore_scheduler.h @@ -15,7 +15,7 @@ #include "share/backup/ob_backup_info_mgr.h" #include "rootserver/restore/ob_restore_util.h" -#include "rootserver/ob_primary_ls_service.h"//ObTenantThreadHelper +#include "rootserver/ob_tenant_thread_helper.h"//ObTenantThreadHelper #include "share/backup/ob_backup_struct.h" #include "share/ob_rpc_struct.h" #include "share/ob_common_rpc_proxy.h" diff --git a/src/share/inner_table/ob_inner_table_schema.12251_12300.cpp b/src/share/inner_table/ob_inner_table_schema.12251_12300.cpp index 7b8008847..34fdda012 100644 --- a/src/share/inner_table/ob_inner_table_schema.12251_12300.cpp +++ b/src/share/inner_table/ob_inner_table_schema.12251_12300.cpp @@ -1377,6 +1377,25 @@ int ObInnerTableSchema::all_virtual_tenant_info_schema(ObTableSchema &table_sche log_mode_default, log_mode_default); //default_value } + + if (OB_SUCC(ret)) { + ObObj sys_recovery_scn_default; + sys_recovery_scn_default.set_uint64(OB_MIN_SCN_TS_NS); + ADD_COLUMN_SCHEMA_T("sys_recovery_scn", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt64Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + sys_recovery_scn_default, + sys_recovery_scn_default); //default_value + } table_schema.set_index_using_type(USING_BTREE); table_schema.set_row_store_type(ENCODING_ROW_STORE); table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); diff --git a/src/share/inner_table/ob_inner_table_schema.15251_15300.cpp b/src/share/inner_table/ob_inner_table_schema.15251_15300.cpp index 38cc7fce7..179ae233f 100644 --- a/src/share/inner_table/ob_inner_table_schema.15251_15300.cpp +++ b/src/share/inner_table/ob_inner_table_schema.15251_15300.cpp @@ -4777,6 +4777,21 @@ int ObInnerTableSchema::all_virtual_tenant_info_ora_schema(ObTableSchema &table_ false, //is_nullable false); //is_autoincrement } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("SYS_RECOVERY_SCN", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } table_schema.set_index_using_type(USING_BTREE); table_schema.set_row_store_type(ENCODING_ROW_STORE); table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); diff --git a/src/share/inner_table/ob_inner_table_schema.21151_21200.cpp b/src/share/inner_table/ob_inner_table_schema.21151_21200.cpp index 160a45577..e40c0ebc3 100644 --- a/src/share/inner_table/ob_inner_table_schema.21151_21200.cpp +++ b/src/share/inner_table/ob_inner_table_schema.21151_21200.cpp @@ -410,7 +410,7 @@ int ObInnerTableSchema::dba_ob_tenants_schema(ObTableSchema &table_schema) table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT A.TENANT_ID, TENANT_NAME, (CASE WHEN A.TENANT_ID = 1 THEN 'SYS' WHEN (A.TENANT_ID & 0x1) = 1 THEN 'META' ELSE 'USER' END) AS TENANT_TYPE, A.gmt_create AS CREATE_TIME, A.gmt_modified AS MODIFY_TIME, PRIMARY_ZONE, LOCALITY, CASE previous_locality WHEN "" THEN NULL ELSE previous_locality END AS PREVIOUS_LOCALITY, CASE compatibility_mode WHEN 0 THEN 'MYSQL' WHEN 1 THEN 'ORACLE' ELSE NULL END AS COMPATIBILITY_MODE, STATUS, CASE in_recyclebin WHEN 0 THEN 'NO' ELSE 'YES' END AS IN_RECYCLEBIN, CASE locked WHEN 0 THEN 'NO' ELSE 'YES' END AS LOCKED, (CASE WHEN A.TENANT_ID = 1 THEN 'PRIMARY' WHEN (A.TENANT_ID & 0x1) = 1 THEN 'PRIMARY' ELSE TENANT_ROLE END) AS TENANT_ROLE, (CASE WHEN A.TENANT_ID = 1 THEN 'NORMAL' WHEN (A.TENANT_ID & 0x1) = 1 THEN 'NORMAL' ELSE SWITCHOVER_STATUS END) AS SWITCHOVER_STATUS, (CASE WHEN A.TENANT_ID = 1 THEN 0 WHEN (A.TENANT_ID & 0x1) = 1 THEN 0 ELSE SWITCHOVER_EPOCH END) AS SWITCHOVER_EPOCH, (CASE WHEN A.TENANT_ID = 1 THEN NULL WHEN (A.TENANT_ID & 0x1) = 1 THEN NULL ELSE SYNC_SCN END) AS SYNC_SCN, (CASE WHEN A.TENANT_ID = 1 THEN NULL WHEN (A.TENANT_ID & 0x1) = 1 THEN NULL ELSE REPLAYABLE_SCN END) AS REPLAYABLE_SCN, (CASE WHEN A.TENANT_ID = 1 THEN NULL WHEN (A.TENANT_ID & 0x1) = 1 THEN NULL ELSE READABLE_SCN END) AS READABLE_SCN, (CASE WHEN A.TENANT_ID = 1 THEN NULL WHEN (A.TENANT_ID & 0x1) = 1 THEN NULL ELSE RECOVERY_UNTIL_SCN END) AS RECOVERY_UNTIL_SCN, (CASE WHEN A.TENANT_ID = 1 THEN 'NOARCHIVELOG' WHEN (A.TENANT_ID & 0x1) = 1 THEN 'NOARCHIVELOG' ELSE LOG_MODE END) AS LOG_MODE, ARBITRATION_SERVICE_STATUS FROM OCEANBASE.__ALL_VIRTUAL_TENANT_MYSQL_SYS_AGENT AS A LEFT JOIN OCEANBASE.__ALL_VIRTUAL_TENANT_INFO AS B ON A.TENANT_ID = B.TENANT_ID )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT A.TENANT_ID, TENANT_NAME, (CASE WHEN A.TENANT_ID = 1 THEN 'SYS' WHEN (A.TENANT_ID & 0x1) = 1 THEN 'META' ELSE 'USER' END) AS TENANT_TYPE, A.gmt_create AS CREATE_TIME, A.gmt_modified AS MODIFY_TIME, PRIMARY_ZONE, LOCALITY, CASE previous_locality WHEN "" THEN NULL ELSE previous_locality END AS PREVIOUS_LOCALITY, CASE compatibility_mode WHEN 0 THEN 'MYSQL' WHEN 1 THEN 'ORACLE' ELSE NULL END AS COMPATIBILITY_MODE, STATUS, CASE in_recyclebin WHEN 0 THEN 'NO' ELSE 'YES' END AS IN_RECYCLEBIN, CASE locked WHEN 0 THEN 'NO' ELSE 'YES' END AS LOCKED, (CASE WHEN A.TENANT_ID = 1 THEN 'PRIMARY' WHEN (A.TENANT_ID & 0x1) = 1 THEN 'PRIMARY' ELSE TENANT_ROLE END) AS TENANT_ROLE, (CASE WHEN A.TENANT_ID = 1 THEN 'NORMAL' WHEN (A.TENANT_ID & 0x1) = 1 THEN 'NORMAL' ELSE SWITCHOVER_STATUS END) AS SWITCHOVER_STATUS, (CASE WHEN A.TENANT_ID = 1 THEN 0 WHEN (A.TENANT_ID & 0x1) = 1 THEN 0 ELSE SWITCHOVER_EPOCH END) AS SWITCHOVER_EPOCH, (CASE WHEN A.TENANT_ID = 1 THEN NULL WHEN (A.TENANT_ID & 0x1) = 1 THEN NULL ELSE SYNC_SCN END) AS SYNC_SCN, (CASE WHEN A.TENANT_ID = 1 THEN NULL WHEN (A.TENANT_ID & 0x1) = 1 THEN NULL ELSE REPLAYABLE_SCN END) AS REPLAYABLE_SCN, (CASE WHEN A.TENANT_ID = 1 THEN NULL WHEN (A.TENANT_ID & 0x1) = 1 THEN NULL ELSE READABLE_SCN END) AS READABLE_SCN, (CASE WHEN A.TENANT_ID = 1 THEN NULL WHEN (A.TENANT_ID & 0x1) = 1 THEN NULL ELSE RECOVERY_UNTIL_SCN END) AS RECOVERY_UNTIL_SCN, (CASE WHEN A.TENANT_ID = 1 THEN NULL WHEN (A.TENANT_ID & 0x1) = 1 THEN NULL ELSE SYS_RECOVERY_SCN END) AS SYS_RECOVERY_SCN, (CASE WHEN A.TENANT_ID = 1 THEN 'NOARCHIVELOG' WHEN (A.TENANT_ID & 0x1) = 1 THEN 'NOARCHIVELOG' ELSE LOG_MODE END) AS LOG_MODE, ARBITRATION_SERVICE_STATUS FROM OCEANBASE.__ALL_VIRTUAL_TENANT_MYSQL_SYS_AGENT AS A LEFT JOIN OCEANBASE.__ALL_VIRTUAL_TENANT_INFO AS B ON A.TENANT_ID = B.TENANT_ID )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema.351_400.cpp b/src/share/inner_table/ob_inner_table_schema.351_400.cpp index b4698cd2e..adfbbd125 100644 --- a/src/share/inner_table/ob_inner_table_schema.351_400.cpp +++ b/src/share/inner_table/ob_inner_table_schema.351_400.cpp @@ -5775,6 +5775,25 @@ int ObInnerTableSchema::all_tenant_info_schema(ObTableSchema &table_schema) log_mode_default, log_mode_default); //default_value } + + if (OB_SUCC(ret)) { + ObObj sys_recovery_scn_default; + sys_recovery_scn_default.set_uint64(OB_MIN_SCN_TS_NS); + ADD_COLUMN_SCHEMA_T("sys_recovery_scn", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObUInt64Type, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(uint64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + sys_recovery_scn_default, + sys_recovery_scn_default); //default_value + } table_schema.set_index_using_type(USING_BTREE); table_schema.set_row_store_type(ENCODING_ROW_STORE); table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index 30f340868..60a997aee 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -4049,6 +4049,7 @@ def_table_schema( ('readable_scn', 'uint'), ('recovery_until_scn', 'uint', 'false', 'OB_MAX_SCN_TS_NS'), ('log_mode', 'varchar:100', 'false', 'NOARCHIVELOG'), + ("sys_recovery_scn", 'uint', 'false', 'OB_MIN_SCN_TS_NS'), ], ) @@ -15446,6 +15447,12 @@ SELECT A.TENANT_ID, ELSE RECOVERY_UNTIL_SCN END) AS RECOVERY_UNTIL_SCN, + (CASE + WHEN A.TENANT_ID = 1 THEN NULL + WHEN (A.TENANT_ID & 0x1) = 1 THEN NULL + ELSE SYS_RECOVERY_SCN + END) AS SYS_RECOVERY_SCN, + (CASE WHEN A.TENANT_ID = 1 THEN 'NOARCHIVELOG' WHEN (A.TENANT_ID & 0x1) = 1 THEN 'NOARCHIVELOG' diff --git a/src/share/ls/ob_ls_i_life_manager.h b/src/share/ls/ob_ls_i_life_manager.h index 47b307d70..99212bc11 100644 --- a/src/share/ls/ob_ls_i_life_manager.h +++ b/src/share/ls/ob_ls_i_life_manager.h @@ -241,7 +241,44 @@ int ObLSTemplateOperator::exec_write(const uint64_t &tenant_id, return ret; } +#define DEFINE_IN_TRANS_FUC(func_name, ...)\ +int func_name ##_in_trans(__VA_ARGS__, ObMySQLTransaction &trans);\ +int func_name(__VA_ARGS__); +#define DEFINE_IN_TRANS_FUC1(func_name, ...)\ +static int func_name ##_in_trans(__VA_ARGS__, ObMySQLTransaction &trans);\ +static int func_name(__VA_ARGS__, ObISQLClient *proxy); + +#define TAKE_IN_TRANS(func_name, proxy, exec_tenant_id, ...)\ +do {\ + ObMySQLTransaction trans; \ + if (FAILEDx(trans.start(proxy, exec_tenant_id))) {\ + SHARE_LOG(WARN, "failed to start trans", KR(ret), K(exec_tenant_id));\ + } else if (OB_FAIL(func_name##_in_trans(__VA_ARGS__, trans))) {\ + SHARE_LOG(WARN, "failed to do it in trans", KR(ret));\ + }\ + if (trans.is_started()) {\ + int tmp_ret = OB_SUCCESS;\ + if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) {\ + SHARE_LOG(WARN, "failed to commit trans", KR(ret), KR(tmp_ret));\ + ret = OB_SUCC(ret) ? tmp_ret : ret;\ + }\ + }\ +} while (0) + +#define START_TRANSACTION(proxy, exec_tenant_id) \ + ObMySQLTransaction trans; \ + if (FAILEDx(trans.start(proxy, exec_tenant_id))) { \ + SHARE_LOG(WARN, "failed to start trans", KR(ret), K(exec_tenant_id)); \ + } +#define END_TRANSACTION(trans)\ + if (trans.is_started()) {\ + int tmp_ret = OB_SUCCESS;\ + if (OB_TMP_FAIL(trans.end(OB_SUCC(ret)))) {\ + ret = OB_SUCC(ret) ? tmp_ret : ret;\ + SHARE_LOG(WARN, "failed to end trans", KR(ret), K(tmp_ret));\ + }\ + } } } diff --git a/src/share/ls/ob_ls_life_manager.cpp b/src/share/ls/ob_ls_life_manager.cpp index bb2c1f0e9..938d21564 100644 --- a/src/share/ls/ob_ls_life_manager.cpp +++ b/src/share/ls/ob_ls_life_manager.cpp @@ -34,21 +34,13 @@ int ObLSLifeAgentManager::create_new_ls( if (OB_UNLIKELY(!ls_info.is_valid() || !create_ls_scn.is_valid() || zone_priority.empty())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(ls_info), K(create_ls_scn), K(zone_priority)); - } else if (OB_FAIL(trans.start(proxy_, exec_tenant_id))) { - LOG_WARN("failed to start trans", KR(ret), K(exec_tenant_id), K(ls_info)); - } else if (OB_FAIL(create_new_ls_in_trans(ls_info, create_ls_scn, zone_priority, working_sw_status, trans))) { - LOG_WARN("failed to create new ls", KR(ret), K(ls_info), K(create_ls_scn), K(zone_priority)); + } else { + TAKE_IN_TRANS(create_new_ls, proxy_, + exec_tenant_id, ls_info, create_ls_scn, zone_priority, working_sw_status); } - if (trans.is_started()) { - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) { - LOG_WARN("failed to commit trans", KR(ret), KR(tmp_ret)); - ret = OB_SUCC(ret) ? tmp_ret : ret; - } - } - return ret; } + int ObLSLifeAgentManager::drop_ls(const uint64_t &tenant_id, const share::ObLSID &ls_id, const ObTenantSwitchoverStatus &working_sw_status) @@ -59,19 +51,10 @@ int ObLSLifeAgentManager::drop_ls(const uint64_t &tenant_id, if (OB_UNLIKELY(!ls_id.is_valid() || OB_INVALID_TENANT_ID == tenant_id)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(ls_id), K(tenant_id)); - } else if (OB_FAIL(trans.start(proxy_, exec_tenant_id))) { - LOG_WARN("failed to start trans", KR(ret), K(exec_tenant_id), K(tenant_id)); - } else if (OB_FAIL(drop_ls_in_trans(tenant_id, ls_id, working_sw_status, trans))) { - LOG_WARN("failed to create new ls", KR(ret), K(tenant_id), K(ls_id)); + } else { + TAKE_IN_TRANS(drop_ls, proxy_, exec_tenant_id, tenant_id, + ls_id, working_sw_status); } - if (trans.is_started()) { - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) { - LOG_WARN("failed to commit trans", KR(ret), KR(tmp_ret)); - ret = OB_SUCC(ret) ? tmp_ret : ret; - } - } - return ret; } @@ -90,22 +73,10 @@ int ObLSLifeAgentManager::set_ls_offline(const uint64_t &tenant_id, || (!ls_is_dropping_status(ls_status) && !ls_is_tenant_dropping_status(ls_status)))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(ls_id), K(tenant_id), K(drop_scn), K(ls_status)); - } else if (OB_FAIL(trans.start(proxy_, exec_tenant_id))) { - LOG_WARN("failed to start trans", KR(ret), K(exec_tenant_id), K(tenant_id)); - } else if (OB_FAIL(set_ls_offline_in_trans(tenant_id, ls_id, ls_status, drop_scn, - working_sw_status, trans))) { - LOG_WARN("failed to create new ls", KR(ret), K(tenant_id), K(ls_id), - K(ls_status), K(drop_scn)); + } else { + TAKE_IN_TRANS(set_ls_offline, proxy_, exec_tenant_id, tenant_id, + ls_id, ls_status, drop_scn, working_sw_status); } - - if (trans.is_started()) { - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) { - LOG_WARN("failed to commit trans", KR(ret), KR(tmp_ret)); - ret = OB_SUCC(ret) ? tmp_ret : ret; - } - } - return ret; } diff --git a/src/share/ls/ob_ls_life_manager.h b/src/share/ls/ob_ls_life_manager.h index 461c9cf39..937e9eb0b 100644 --- a/src/share/ls/ob_ls_life_manager.h +++ b/src/share/ls/ob_ls_life_manager.h @@ -71,19 +71,19 @@ public: * @param[in]zone_priority: the primary_zone of OB_ALL_LS_ELECTION_REFERENCE_INFO * @param[in] working_sw_status only support working on specified switchover status * */ - int create_new_ls(const ObLSStatusInfo &ls_info, - const SCN &create_ls_scn, - const common::ObString &zone_priority, - const share::ObTenantSwitchoverStatus &working_sw_status); + DEFINE_IN_TRANS_FUC(create_new_ls, const ObLSStatusInfo &ls_info,\ + const SCN &create_ls_scn,\ + const common::ObString &zone_priority,\ + const share::ObTenantSwitchoverStatus &working_sw_status) /* * description: for primary cluster and GC of standby, delete ls from each inner_table * @param[in] tenant_id: tenant_id * @param[in] ls_id: need delete ls * @param[in] working_sw_status only support working on specified switchover status * */ - int drop_ls(const uint64_t &tenant_id, - const share::ObLSID &ls_id, - const ObTenantSwitchoverStatus &working_sw_status); + DEFINE_IN_TRANS_FUC(drop_ls, const uint64_t &tenant_id,\ + const share::ObLSID &ls_id,\ + const ObTenantSwitchoverStatus &working_sw_status) /* * description: for primary cluster set ls to wait offline from tenant_dropping or dropping status * @param[in] tenant_id: tenant_id @@ -92,11 +92,11 @@ public: * @param[in] drop_scn: there is no user data after drop_scn except offline * @param[in] working_sw_status only support working on specified switchover status * */ - int set_ls_offline(const uint64_t &tenant_id, - const share::ObLSID &ls_id, - const ObLSStatus &ls_status, - const SCN &drop_scn, - const ObTenantSwitchoverStatus &working_sw_status); + DEFINE_IN_TRANS_FUC(set_ls_offline, const uint64_t &tenant_id,\ + const share::ObLSID &ls_id,\ + const ObLSStatus &ls_status,\ + const SCN &drop_scn,\ + const ObTenantSwitchoverStatus &working_sw_status) /* * description: update ls primary zone, need update __all_ls_status and __all_ls_election_reference * @param[in] tenant_id: tenant_id @@ -110,32 +110,6 @@ public: const common::ObZone &primary_zone, const common::ObString &zone_priority); -public: - /* - * description: for standby cluster, create new ls - */ - int create_new_ls_in_trans(const ObLSStatusInfo &ls_info, - const SCN &create_ls_scn, - const common::ObString &zone_priority, - const share::ObTenantSwitchoverStatus &working_sw_status, - ObMySQLTransaction &trans); - /* - * description: for standby cluster, create new ls - */ - int drop_ls_in_trans(const uint64_t &tenant_id, - const share::ObLSID &ls_id, - const ObTenantSwitchoverStatus &working_sw_status, - ObMySQLTransaction &trans); - /* - * description: for standby cluster set ls to offline - * */ - int set_ls_offline_in_trans(const uint64_t &tenant_id, - const share::ObLSID &ls_id, - const ObLSStatus &ls_status, - const SCN &drop_scn, - const ObTenantSwitchoverStatus &working_sw_status, - ObMySQLTransaction &trans); - private: bool inited_; //table_operator diff --git a/src/share/ls/ob_ls_operator.cpp b/src/share/ls/ob_ls_operator.cpp index 4c461361b..9ca3d8602 100644 --- a/src/share/ls/ob_ls_operator.cpp +++ b/src/share/ls/ob_ls_operator.cpp @@ -116,17 +116,15 @@ bool ObLSAttrOperator::is_valid() const int ObLSAttrOperator::operator_ls_( const ObLSAttr &ls_attr, const common::ObSqlString &sql, - const uint64_t target_max_ls_group_id, - const ObTenantSwitchoverStatus &working_sw_status) + const uint64_t target_max_ls_group_id) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!ls_attr.is_valid() || sql.empty() - || OB_INVALID_ID == target_max_ls_group_id - || !working_sw_status.is_valid())) { + || OB_INVALID_ID == target_max_ls_group_id)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("operation is invalid", KR(ret), K(ls_attr), K(sql), - K(target_max_ls_group_id), K(working_sw_status)); + K(target_max_ls_group_id)); } else { ObMySQLTransaction trans; const bool for_update = true; @@ -146,10 +144,6 @@ int ObLSAttrOperator::operator_ls_( LOG_WARN("failed to load sys ls status", KR(ret)); } else if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(tenant_id_, proxy_, false /* for_update */, tenant_info))) { LOG_WARN("failed to load tenant info", KR(ret), K_(tenant_id)); - } else if (working_sw_status != tenant_info.get_switchover_status()) { - ret = OB_NEED_RETRY; - LOG_WARN("tenant not in specified switchover status", K_(tenant_id), K(working_sw_status), - K(tenant_info)); } else if (OB_LS_OP_CREATE_PRE == ls_attr.get_ls_operation_type()) { if (OB_LS_NORMAL != sys_ls_attr.get_ls_status()) { //for sys ls, need insert_ls, but ls_status is normal @@ -196,8 +190,7 @@ int ObLSAttrOperator::operator_ls_( int ObLSAttrOperator::insert_ls( const ObLSAttr &ls_attr, - const uint64_t max_ls_group_id, - const ObTenantSwitchoverStatus &working_sw_status) + const uint64_t max_ls_group_id) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!ls_attr.is_valid() || OB_INVALID_ID == max_ls_group_id)) { @@ -215,7 +208,7 @@ int ObLSAttrOperator::insert_ls( ObLSStatusOperator::ls_status_to_str(ls_attr.get_ls_status()), "", ls_attr.get_create_scn().get_val_for_inner_table_field()))) { LOG_WARN("failed to assign sql", KR(ret), K(ls_attr), K(sql)); - } else if (OB_FAIL(operator_ls_(ls_attr, sql, max_ls_group_id, working_sw_status))) { + } else if (OB_FAIL(operator_ls_(ls_attr, sql, max_ls_group_id))) { LOG_WARN("failed to operator ls", KR(ret), K(ls_attr), K(sql)); } } @@ -224,8 +217,7 @@ int ObLSAttrOperator::insert_ls( } int ObLSAttrOperator::delete_ls( - const ObLSID &ls_id, const share::ObLSStatus &old_status, - const ObTenantSwitchoverStatus &working_sw_status) + const ObLSID &ls_id, const share::ObLSStatus &old_status) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!ls_id.is_valid() @@ -258,8 +250,7 @@ int ObLSAttrOperator::delete_ls( ls_attr.get_ls_flag(), ls_attr.get_ls_status(), operation_type, SCN::base_scn()))) { LOG_WARN("failed to init new ls attr", KR(ret), K(ls_id), K(ls_attr), K(operation_type)); - } else if (OB_FAIL(operator_ls_(new_ls_attr, sql, new_ls_attr.get_ls_group_id(), - working_sw_status))) { + } else if (OB_FAIL(operator_ls_(new_ls_attr, sql, new_ls_attr.get_ls_group_id()))) { LOG_WARN("failed to operator ls", KR(ret), K(new_ls_attr), K(sql)); } LOG_INFO("[LS_OPERATOR] delete ls", KR(ret), K(ls_id), K(old_status)); @@ -323,8 +314,7 @@ int ObLSAttrOperator::process_sub_trans_(const ObLSAttr &ls_attr, ObMySQLTransac int ObLSAttrOperator::update_ls_status(const ObLSID &id, const share::ObLSStatus &old_status, - const share::ObLSStatus &new_status, - const ObTenantSwitchoverStatus &working_sw_status) + const share::ObLSStatus &new_status) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!id.is_valid() @@ -353,8 +343,7 @@ int ObLSAttrOperator::update_ls_status(const ObLSID &id, operation_type, SCN::base_scn()))) { LOG_WARN("failed to init new ls attr", KR(ret), K(id), K(ls_attr), K(operation_type)); - } else if (OB_FAIL(operator_ls_(new_ls_attr, sql, new_ls_attr.get_ls_group_id(), - working_sw_status))) { + } else if (OB_FAIL(operator_ls_(new_ls_attr, sql, new_ls_attr.get_ls_group_id()))) { LOG_WARN("failed to operator ls", KR(ret), K(new_ls_attr), K(sql)); } LOG_INFO("[LS_OPERATOR] update ls status", KR(ret), K(ls_attr), K(new_ls_attr)); @@ -549,43 +538,6 @@ int ObLSAttrOperator::get_tenant_gts(const uint64_t &tenant_id, SCN >s_scn) return ret; } -int ObLSAttrOperator::get_all_ls_by_order(const bool lock_sys_ls, ObLSAttrIArray &ls_operation_array) -{ - int ret = OB_SUCCESS; - ls_operation_array.reset(); - ObMySQLTransaction trans; - ObLSAttr sys_ls_attr; - - if (OB_UNLIKELY(!is_valid())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("operation is not valid", KR(ret), "operation", *this); - } else if (OB_FAIL(trans.start(proxy_, tenant_id_))) { - LOG_WARN("failed to start transaction", KR(ret), K_(tenant_id)); - /* to get accurate LS list need lock SYS_LS */ - } else if (lock_sys_ls && OB_FAIL(get_ls_attr(SYS_LS, true /* for_update */, trans, sys_ls_attr))) { - LOG_WARN("failed to load sys ls status", KR(ret)); - } else { - ObSqlString sql; - if (OB_FAIL(sql.assign_fmt( - "select * from %s order by ls_id", - OB_ALL_LS_TNAME))) { - LOG_WARN("failed to assign sql", KR(ret), K(sql)); - } else if (OB_FAIL(exec_read(tenant_id_, sql, trans, this, ls_operation_array))) { - LOG_WARN("failed to construct ls attr", KR(ret), K(sql), K_(tenant_id)); - } - } - - if (trans.is_started()) { - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) { - LOG_WARN("failed to end trans", KR(ret), KR(tmp_ret)); - ret = OB_SUCC(ret) ? tmp_ret : ret; - } - } - - return ret; -} - } } diff --git a/src/share/ls/ob_ls_operator.h b/src/share/ls/ob_ls_operator.h index b10b81d6b..dddc42639 100644 --- a/src/share/ls/ob_ls_operator.h +++ b/src/share/ls/ob_ls_operator.h @@ -105,10 +105,6 @@ struct ObLSAttr { return ls_is_normal_status(status_); } - ObLSOperationType get_ls_operatin_type() const - { - return operation_type_; - } ObLSID get_ls_id() const { return id_; @@ -172,24 +168,11 @@ public: bool is_valid() const; int get_all_ls_by_order( ObLSAttrIArray &ls_array); - /** - * @description: - * get ls list from all_ls table, - * if want to get accurate LS list, set lock_sys_ls to true to lock SYS LS in __all_ls table - * to make sure mutual exclusion with load balancing thread - * @param[in] lock_sys_ls whether lock SYS LS in __all_ls table - * @param[out] ls_operation_array ls list - * @return return code - */ - int get_all_ls_by_order(const bool lock_sys_ls, ObLSAttrIArray &ls_operation_array); - int insert_ls(const ObLSAttr &ls_attr, const uint64_t max_ls_group_id, - const ObTenantSwitchoverStatus &working_sw_status); + int insert_ls(const ObLSAttr &ls_attr, const uint64_t max_ls_group_id); //prevent the concurrency of create and drop ls int delete_ls(const ObLSID &id, - const share::ObLSStatus &old_status, - const ObTenantSwitchoverStatus &working_sw_status); - int update_ls_status(const ObLSID &id, const share::ObLSStatus &old_status, const share::ObLSStatus &new_status, - const ObTenantSwitchoverStatus &working_sw_status); + const share::ObLSStatus &old_status); + int update_ls_status(const ObLSID &id, const share::ObLSStatus &old_status, const share::ObLSStatus &new_status); static ObLSOperationType get_ls_operation_by_status(const ObLSStatus &ls_status); int get_ls_attr(const ObLSID &id, const bool for_update, common::ObISQLClient &client, ObLSAttr &ls_attr); /* @@ -202,8 +185,7 @@ public: private: int process_sub_trans_(const ObLSAttr &ls_attr, ObMySQLTransaction &trans); - int operator_ls_(const ObLSAttr &ls_attr, const common::ObSqlString &sql, const uint64_t max_ls_group_id, - const ObTenantSwitchoverStatus &working_sw_status); + int operator_ls_(const ObLSAttr &ls_attr, const common::ObSqlString &sql, const uint64_t max_ls_group_id); private: uint64_t tenant_id_; common::ObMySQLProxy *proxy_; diff --git a/src/share/ls/ob_ls_recovery_stat_operator.cpp b/src/share/ls/ob_ls_recovery_stat_operator.cpp index 53a70cad6..7d81b2c20 100644 --- a/src/share/ls/ob_ls_recovery_stat_operator.cpp +++ b/src/share/ls/ob_ls_recovery_stat_operator.cpp @@ -17,6 +17,7 @@ #include "share/ob_errno.h" #include "share/config/ob_server_config.h" #include "share/inner_table/ob_inner_table_schema.h" +#include "share/ob_tenant_info_proxy.h"//ObAllTenantInfo #include "lib/string/ob_sql_string.h"//ObSqlString #include "lib/mysqlclient/ob_mysql_transaction.h"//ObMySQLTransaction #include "common/ob_timeout_ctx.h" @@ -219,22 +220,18 @@ int ObLSRecoveryStatOperator::update_ls_recovery_stat_in_trans( true, old_ls_recovery, trans))) { LOG_WARN("failed to get ls current recovery stat", KR(ret), K(ls_recovery)); } else if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(ls_recovery.get_tenant_id(), &trans, - true /* for update */, tenant_info))) { + true /* for update */, tenant_info))) { LOG_WARN("failed to load tenant info", KR(ret), K(ls_recovery)); - } else if (OB_UNLIKELY(!tenant_info.is_normal_status())) { - ret = OB_NEED_RETRY; - LOG_WARN("switchover status is not normal, do not update ls recovery", KR(ret), - K(ls_recovery), K(tenant_info)); + } else if (!tenant_info.get_switchover_status().can_report_recovery_status()) { + ret = OB_OP_NOT_ALLOW; + LOG_WARN("status can not report recovery status", KR(ret), K(tenant_info)); } else { const SCN sync_scn = ls_recovery.get_sync_scn() > old_ls_recovery.get_sync_scn() ? ls_recovery.get_sync_scn() : old_ls_recovery.get_sync_scn(); const SCN &readable_scn = ls_recovery.get_readable_scn() > old_ls_recovery.get_readable_scn() ? ls_recovery.get_readable_scn() : old_ls_recovery.get_readable_scn(); common::ObSqlString sql; - if (ls_recovery.get_ls_id() == share::SYS_LS && tenant_info.get_recovery_until_scn() < sync_scn) { - ret = OB_NEED_RETRY; - LOG_WARN("can not recovery bigger than recovery_until_scn", KR(ret), K(sync_scn), K(tenant_info)); - } else if (OB_FAIL(sql.assign_fmt("UPDATE %s SET sync_scn = %lu, readable_scn = " + if (OB_FAIL(sql.assign_fmt("UPDATE %s SET sync_scn = %lu, readable_scn = " "%lu where ls_id = %ld and tenant_id = %lu", OB_ALL_LS_RECOVERY_STAT_TNAME, sync_scn.get_val_for_inner_table_field(), readable_scn.get_val_for_inner_table_field(), @@ -246,6 +243,7 @@ int ObLSRecoveryStatOperator::update_ls_recovery_stat_in_trans( } return ret; } + int ObLSRecoveryStatOperator::set_ls_offline(const uint64_t &tenant_id, const share::ObLSID &ls_id, const ObLSStatus &ls_status, diff --git a/src/share/ob_primary_standby_service.cpp b/src/share/ob_primary_standby_service.cpp index a386fa03f..b62a2d130 100644 --- a/src/share/ob_primary_standby_service.cpp +++ b/src/share/ob_primary_standby_service.cpp @@ -19,7 +19,7 @@ #include "rootserver/ob_rs_event_history_table_operator.h" // ROOTSERVICE_EVENT_ADD #include "rootserver/ob_tenant_role_transition_service.h" // ObTenantRoleTransitionService #include "rootserver/ob_primary_ls_service.h"//ObTenantLSInfo -#include "share/ls/ob_ls_recovery_stat_operator.h"// ObLSRecoveryStatOperator +#include "share/restore/ob_log_restore_source_mgr.h" // ObLogRestoreSourceMgr #include "share/ls/ob_ls_life_manager.h" //ObLSLifeAgentManager #include "share/ls/ob_ls_operator.h" //ObLSAttr #include "storage/tx/ob_timestamp_service.h" // ObTimestampService @@ -281,8 +281,6 @@ int ObPrimaryStandbyService::do_recover_tenant( const uint64_t exec_tenant_id = gen_meta_tenant_id(tenant_id); common::ObMySQLTransaction trans; const ObSimpleTenantSchema *tenant_schema = nullptr; - ObLSRecoveryStatOperator ls_recovery_operator; - ObLSRecoveryStat sys_ls_recovery; if (OB_FAIL(check_inner_stat_())) { LOG_WARN("inner stat error", KR(ret), K_(inited)); } else if (!obrpc::ObRecoverTenantArg::is_valid(recover_type, recovery_until_scn) @@ -313,19 +311,16 @@ int ObPrimaryStandbyService::do_recover_tenant( } else if (tenant_info.get_switchover_status() != working_sw_status) { ret = OB_OP_NOT_ALLOW; LOG_WARN("unexpected tenant switchover status", KR(ret), K(working_sw_status), K(tenant_info)); - } else if (OB_FAIL(ls_recovery_operator.get_ls_recovery_stat(tenant_id, share::SYS_LS, - true /*for_update*/, sys_ls_recovery, trans))) { - LOG_WARN("failed to get ls recovery stat", KR(ret), K(tenant_id)); } else if (obrpc::ObRecoverTenantArg::RecoverType::UNTIL == recover_type && (recovery_until_scn < tenant_info.get_sync_scn() - || recovery_until_scn < sys_ls_recovery.get_sync_scn())) { + || recovery_until_scn < tenant_info.get_sys_recovery_scn())) { ret = OB_OP_NOT_ALLOW; - LOG_WARN("recover before tenant sync_scn or SYS LS sync_scn is not allow", KR(ret), K(tenant_info), - K(tenant_id), K(recover_type), K(recovery_until_scn), K(sys_ls_recovery)); - LOG_USER_ERROR(OB_OP_NOT_ALLOW, "recover before tenant sync_scn or SYS LS sync_scn is"); + LOG_WARN("recover before tenant sync_scn or sys recovery scn is not allow", KR(ret), K(tenant_info), + K(tenant_id), K(recover_type), K(recovery_until_scn)); + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "recover before tenant sync_scn or sys recovery scn is"); } else if (tenant_schema->is_normal()) { const SCN &recovery_until_scn_to_set = obrpc::ObRecoverTenantArg::RecoverType::UNTIL == recover_type ? - recovery_until_scn : SCN::max(tenant_info.get_sync_scn(), sys_ls_recovery.get_sync_scn()); + recovery_until_scn : SCN::max(tenant_info.get_sync_scn(), tenant_info.get_sys_recovery_scn()); if (tenant_info.get_recovery_until_scn() == recovery_until_scn_to_set) { LOG_WARN("recovery_until_scn is same with original", KR(ret), K(tenant_info), K(tenant_id), K(recover_type), K(recovery_until_scn)); @@ -423,23 +418,15 @@ int ObPrimaryStandbyService::switch_to_standby( K(tenant_id)); } } - case share::ObTenantSwitchoverStatus::PREPARE_SWITCHING_TO_STANDBY_STATUS: { - if (OB_FAIL(ret)) { - } else if (OB_FAIL(switch_to_standby_prepare_ls_status_(tenant_id, - tenant_info.get_switchover_status(), - tenant_info.get_switchover_epoch(), - tenant_info))) { - LOG_WARN("failed to switch_to_standby_prepare_ls_status_", KR(ret), K(tenant_id), K(tenant_info)); - } - } case share::ObTenantSwitchoverStatus::SWITCHING_TO_STANDBY_STATUS: { if (OB_FAIL(ret)) { } else { + //change sys ls asscess mode and wait sys_recovery_scn to latest ObTenantRoleTransitionService role_transition_service(tenant_id, sql_proxy_, GCTX.srv_rpc_proxy_, switch_optype); (void)role_transition_service.set_switchover_epoch(tenant_info.get_switchover_epoch()); - if (OB_FAIL(role_transition_service.do_switch_access_mode_to_raw_rw(tenant_info))) { - LOG_WARN("failed to do_switch_access_mode", KR(ret), K(tenant_id), K(tenant_info)); + if (OB_FAIL(role_transition_service.switchover_to_standby(tenant_info))) { + LOG_WARN("failed to switchover to standby", KR(ret), K(tenant_info)); } else if (OB_FAIL(role_transition_service.switchover_update_tenant_status(tenant_id, false /* switch_to_standby */, share::STANDBY_TENANT_ROLE, @@ -508,8 +495,8 @@ int ObPrimaryStandbyService::update_tenant_status_before_sw_to_standby_( LOG_WARN("tenant not expect switchover epoch", KR(ret), K(tenant_info), K(cur_switchover_epoch)); } else if (OB_FAIL(ObAllTenantInfoProxy::update_tenant_role( tenant_id, &trans, cur_switchover_epoch, - PRIMARY_TENANT_ROLE, cur_switchover_status, - share::PREP_SWITCHING_TO_STANDBY_SWITCHOVER_STATUS, new_switchover_ts))) { + STANDBY_TENANT_ROLE, cur_switchover_status, + share::SWITCHING_TO_STANDBY_SWITCHOVER_STATUS, new_switchover_ts))) { LOG_WARN("failed to update tenant role", KR(ret), K(tenant_id), K(cur_switchover_epoch), K(tenant_info)); } else if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info( tenant_id, &trans, true, new_tenant_info))) { @@ -533,68 +520,6 @@ int ObPrimaryStandbyService::update_tenant_status_before_sw_to_standby_( return ret; } -int ObPrimaryStandbyService::switch_to_standby_prepare_ls_status_( - const uint64_t tenant_id, - const ObTenantSwitchoverStatus &status, - const int64_t switchover_epoch, - ObAllTenantInfo &new_tenant_info) -{ - int ret = OB_SUCCESS; - ObMySQLTransaction trans; - ObLSAttr sys_ls_attr; - share::ObLSAttrOperator ls_operator(tenant_id, sql_proxy_); - share::schema::ObSchemaGetterGuard schema_guard; - const share::schema::ObTenantSchema *tenant_schema = NULL; - int64_t new_switchover_epoch = OB_INVALID_VERSION; - - if (!is_user_tenant(tenant_id)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(tenant_id)); - } else if (OB_ISNULL(GCTX.schema_service_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("schema_service_ is NULL", KR(ret)); - } else if (OB_FAIL(check_inner_stat_())) { - LOG_WARN("inner stat error", KR(ret), K_(inited)); - } else if (OB_UNLIKELY(!status.is_prepare_switching_to_standby_status())) { - ret = OB_OP_NOT_ALLOW; - LOG_WARN("switchover status not match, switchover to standby not allow", KR(ret), K(status)); - LOG_USER_ERROR(OB_OP_NOT_ALLOW, "switchover status not match, switchover to standby"); - } else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID, schema_guard))) { - LOG_WARN("fail to get schema guard", KR(ret)); - } else if (OB_FAIL(schema_guard.get_tenant_info(tenant_id, tenant_schema))) { - LOG_WARN("failed to get tenant ids", KR(ret), K(tenant_id)); - } else if (OB_ISNULL(tenant_schema)) { - ret = OB_TENANT_NOT_EXIST; - LOG_WARN("tenant not exist", KR(ret), K(tenant_id)); - } else { - ObTenantLSInfo tenant_stat(GCTX.sql_proxy_, tenant_schema, tenant_id, - GCTX.srv_rpc_proxy_, GCTX.lst_operator_); - /* lock SYS_LS to get accurate LS list, then fix ls status to make ls status consistency - between __all_ls&__all_ls_status. - Refer to ls operator, insert/update/delete of ls table are executed in the SYS_LS lock - and normal switchover status */ - if (OB_FAIL(tenant_stat.process_ls_status_missmatch(true/* lock_sys_ls */, - share::PREP_SWITCHING_TO_STANDBY_SWITCHOVER_STATUS))) { - LOG_WARN("failed to process_ls_status_missmatch", KR(ret)); - } else if (OB_FAIL(ObAllTenantInfoProxy::update_tenant_role( - tenant_id, sql_proxy_, switchover_epoch, - share::STANDBY_TENANT_ROLE, status, - share::SWITCHING_TO_STANDBY_SWITCHOVER_STATUS, new_switchover_epoch))) { - LOG_WARN("failed to update tenant role", KR(ret), K(tenant_id), K(switchover_epoch)); - } else if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info( - tenant_id, sql_proxy_, false, new_tenant_info))) { - LOG_WARN("failed to load tenant info", KR(ret), K(tenant_id)); - } else if (OB_UNLIKELY(new_tenant_info.get_switchover_epoch() != new_switchover_epoch)) { - ret = OB_NEED_RETRY; - LOG_WARN("switchover is concurrency", KR(ret), K(switchover_epoch), K(new_tenant_info)); - } - - DEBUG_SYNC(SWITCHING_TO_STANDBY); - } - - return ret; -} - int ObPrimaryStandbyService::write_upgrade_barrier_log( ObMySQLTransaction &trans, const uint64_t tenant_id, @@ -637,5 +562,5 @@ int ObPrimaryStandbyService::write_upgrade_barrier_log( return ret; } -} +}//end of share } diff --git a/src/share/ob_primary_standby_service.h b/src/share/ob_primary_standby_service.h index 9d6440173..1409baf38 100644 --- a/src/share/ob_primary_standby_service.h +++ b/src/share/ob_primary_standby_service.h @@ -159,21 +159,6 @@ private: const uint64_t tenant_id, ObAllTenantInfo &new_tenant_info); - /** - * @description: - * when switch to standby, prepare ls_status in all_ls and all_ls_status to proper status - * @param[in] tenant_id the tenant id to check - * @param[in] status only prepare in specified switchover status - * @param[in] switchover_epoch only prepare in specified switchover epoch - * @param[out] new_tenant_info return the updated tenant_info - * @return return code - */ - int switch_to_standby_prepare_ls_status_( - const uint64_t tenant_id, - const ObTenantSwitchoverStatus &status, - const int64_t switchover_epoch, - ObAllTenantInfo &new_tenant_info); - private: const static int64_t SEC_UNIT = 1000L * 1000L; const static int64_t PRINT_INTERVAL = 10 * 1000 * 1000L; diff --git a/src/share/ob_tenant_info_proxy.cpp b/src/share/ob_tenant_info_proxy.cpp index d8187adc3..71188b2f9 100644 --- a/src/share/ob_tenant_info_proxy.cpp +++ b/src/share/ob_tenant_info_proxy.cpp @@ -18,7 +18,6 @@ #include "share/config/ob_server_config.h"//GCONF #include "share/inner_table/ob_inner_table_schema.h"//ALL_TENANT_INFO_TNAME #include "share/ls/ob_ls_i_life_manager.h"//TODO SCN VALUE -#include "share/ls/ob_ls_recovery_stat_operator.h"//ObLSRecoveryStatOperator #include "lib/string/ob_sql_string.h"//ObSqlString #include "lib/mysqlclient/ob_mysql_transaction.h"//ObMySQLTrans #include "common/ob_timeout_ctx.h"//ObTimeoutCtx @@ -37,14 +36,22 @@ bool is_valid_tenant_scn( const SCN &sync_scn, const SCN &replayable_scn, const SCN &standby_scn, - const SCN &recovery_until_scn) + const SCN &recovery_until_scn, + const SCN &sys_recovery_scn) { - return standby_scn <= replayable_scn && replayable_scn <= sync_scn && sync_scn <= recovery_until_scn; + return standby_scn <= replayable_scn && replayable_scn <= sync_scn && sync_scn <= recovery_until_scn + && (sys_recovery_scn.is_min() || sync_scn <= sys_recovery_scn) + && (sys_recovery_scn.is_min() || sys_recovery_scn <= recovery_until_scn); } -SCN gen_new_sync_scn(const SCN &cur_sync_scn, const SCN &desired_sync_scn, const SCN &cur_recovery_until_scn) +SCN gen_new_sync_scn(const SCN &cur_sync_scn, const SCN &desired_sync_scn, const SCN &cur_recovery_until_scn, + const SCN &sys_recovery_scn) { - return MIN(MAX(cur_sync_scn, desired_sync_scn), cur_recovery_until_scn); + SCN new_sync_scn = MIN(MAX(cur_sync_scn, desired_sync_scn), cur_recovery_until_scn); + if (!sys_recovery_scn.is_min()) { + new_sync_scn = MIN(new_sync_scn, sys_recovery_scn); + } + return new_sync_scn; } SCN gen_new_replayable_scn(const SCN &cur_replayable_scn, const SCN &desired_replayable_scn, const SCN &new_sync_scn) @@ -69,7 +76,8 @@ bool ObAllTenantInfo::is_valid() const && tenant_role_.is_valid() && switchover_status_.is_valid() && log_mode_.is_valid() - && is_valid_tenant_scn(sync_scn_, replayable_scn_, standby_scn_, recovery_until_scn_); + && sys_recovery_scn_.is_valid() + && is_valid_tenant_scn(sync_scn_, replayable_scn_, standby_scn_, recovery_until_scn_, sys_recovery_scn_); } int ObAllTenantInfo::init( @@ -81,7 +89,8 @@ int ObAllTenantInfo::init( const SCN &replayable_scn, const SCN &standby_scn, const SCN &recovery_until_scn, - const ObArchiveMode &log_mode) + const ObArchiveMode &log_mode, + const SCN &sys_recovery_scn) { int ret = OB_SUCCESS; if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id @@ -93,11 +102,13 @@ int ObAllTenantInfo::init( || !standby_scn.is_valid_and_not_min() || !recovery_until_scn.is_valid_and_not_min() || !log_mode.is_valid() - || !is_valid_tenant_scn(sync_scn, replayable_scn, standby_scn, recovery_until_scn))) { + //sys recovery scn is valid and maybe min during upgrade + || !sys_recovery_scn.is_valid() + || !is_valid_tenant_scn(sync_scn, replayable_scn, standby_scn, recovery_until_scn, sys_recovery_scn))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(tenant_role), K(switchover_status), K(switchover_epoch), K(sync_scn), K(replayable_scn), K(standby_scn), K(recovery_until_scn), - K(log_mode)); + K(log_mode), K(sys_recovery_scn)); } else { tenant_id_ = tenant_id; tenant_role_ = tenant_role; @@ -108,6 +119,7 @@ int ObAllTenantInfo::init( standby_scn_ = standby_scn; recovery_until_scn_ = recovery_until_scn; log_mode_ = log_mode; + sys_recovery_scn_ = sys_recovery_scn; } return ret; } @@ -125,6 +137,7 @@ void ObAllTenantInfo::assign(const ObAllTenantInfo &other) standby_scn_ = other.standby_scn_; recovery_until_scn_ = other.recovery_until_scn_; log_mode_ = other.log_mode_; + sys_recovery_scn_ = other.sys_recovery_scn_; } return ; } @@ -140,10 +153,11 @@ void ObAllTenantInfo::reset() standby_scn_.set_min() ; recovery_until_scn_.set_min(); log_mode_.reset(); + sys_recovery_scn_.set_min(); } OB_SERIALIZE_MEMBER(ObAllTenantInfo, tenant_id_, tenant_role_, switchover_status_, switchover_epoch_, sync_scn_, - replayable_scn_, standby_scn_, recovery_until_scn_, log_mode_); + replayable_scn_, standby_scn_, recovery_until_scn_, log_mode_, sys_recovery_scn_); ObAllTenantInfo& ObAllTenantInfo::operator= (const ObAllTenantInfo &other) { @@ -178,8 +192,8 @@ int ObAllTenantInfoProxy::init_tenant_info( } else if (OB_FAIL(sql.assign_fmt( "insert into %s (tenant_id, tenant_role, " "switchover_status, switchover_epoch, " - "sync_scn, replayable_scn, readable_scn, recovery_until_scn, log_mode) " - "values(%lu, '%s', '%s', %ld, %lu, %lu, %lu, %lu, '%s')", + "sync_scn, replayable_scn, readable_scn, recovery_until_scn, log_mode, sys_recovery_scn) " + "values(%lu, '%s', '%s', %ld, %lu, %lu, %lu, %lu, '%s', %lu)", OB_ALL_TENANT_INFO_TNAME, tenant_info.get_tenant_id(), tenant_info.get_tenant_role().to_str(), tenant_info.get_switchover_status().to_str(), @@ -188,7 +202,8 @@ int ObAllTenantInfoProxy::init_tenant_info( tenant_info.get_replayable_scn().get_val_for_inner_table_field(), tenant_info.get_standby_scn().get_val_for_inner_table_field(), tenant_info.get_recovery_until_scn().get_val_for_inner_table_field(), - tenant_info.get_log_mode().to_str()))) { + tenant_info.get_log_mode().to_str(), + tenant_info.get_sys_recovery_scn().get_val_for_inner_table_field()))) { LOG_WARN("failed to assign sql", KR(ret), K(tenant_info), K(sql)); } else if (OB_FAIL(proxy->write(exec_tenant_id, sql.ptr(), affected_rows))) { LOG_WARN("failed to execute sql", KR(ret), K(exec_tenant_id), K(sql)); @@ -304,8 +319,7 @@ int ObAllTenantInfoProxy::load_tenant_info(const uint64_t tenant_id, } int ObAllTenantInfoProxy::update_tenant_recovery_status( - const uint64_t tenant_id, ObMySQLProxy *proxy, - ObTenantSwitchoverStatus status, const SCN &sync_scn, + const uint64_t tenant_id, ObMySQLProxy *proxy, const SCN &sync_scn, const SCN &replay_scn, const SCN &readable_scn) { int ret = OB_SUCCESS; @@ -315,10 +329,9 @@ int ObAllTenantInfoProxy::update_tenant_recovery_status( common::ObMySQLTransaction trans; ObAllTenantInfo old_tenant_info; ObTimeoutCtx ctx; - if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id || - !status.is_valid())) { + if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("tenant_info is invalid", KR(ret), K(tenant_id), K(status)); + LOG_WARN("tenant_id is invalid", KR(ret), K(tenant_id)); } else if (OB_ISNULL(proxy)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("proxy is null", KR(ret), KP(proxy)); @@ -330,7 +343,8 @@ int ObAllTenantInfoProxy::update_tenant_recovery_status( } else if (OB_FAIL(load_tenant_info(tenant_id, &trans, true, old_tenant_info))) { LOG_WARN("failed to load all tenant info", KR(ret), K(tenant_id)); } else { - SCN new_sync_scn = gen_new_sync_scn(old_tenant_info.get_sync_scn(), sync_scn, old_tenant_info.get_recovery_until_scn()); + SCN new_sync_scn = gen_new_sync_scn(old_tenant_info.get_sync_scn(), sync_scn, + old_tenant_info.get_recovery_until_scn(), old_tenant_info.get_sys_recovery_scn()); SCN new_replay_scn = gen_new_replayable_scn(old_tenant_info.get_replayable_scn(), replay_scn, new_sync_scn); SCN new_scn = gen_new_standby_scn(old_tenant_info.get_standby_scn(), readable_scn, new_replay_scn); @@ -343,14 +357,14 @@ int ObAllTenantInfoProxy::update_tenant_recovery_status( } else if (OB_FAIL(sql.assign_fmt( "update %s set sync_scn = %ld, replayable_scn = %ld, " "readable_scn = %ld where tenant_id = %lu " - "and switchover_status = '%s' and readable_scn <= replayable_scn and " - "replayable_scn <= sync_scn and sync_scn <= recovery_until_scn", OB_ALL_TENANT_INFO_TNAME, + "and readable_scn <= replayable_scn and " + "replayable_scn <= sync_scn and sync_scn <= recovery_until_scn and sync_scn <= %lu", + OB_ALL_TENANT_INFO_TNAME, new_sync_scn.get_val_for_inner_table_field(), new_replay_scn.get_val_for_inner_table_field(), new_scn.get_val_for_inner_table_field(), - tenant_id, status.to_str()))) { - LOG_WARN("failed to assign sql", KR(ret), K(tenant_id), K(status), - K(sql)); + tenant_id, new_sync_scn.get_val_for_inner_table_field()))) { + LOG_WARN("failed to assign sql", KR(ret), K(tenant_id), K(sql)); } else if (OB_FAIL(trans.write(exec_tenant_id, sql.ptr(), affected_rows))) { LOG_WARN("failed to execute sql", KR(ret), K(exec_tenant_id), K(sql)); } else if (!is_single_row(affected_rows)) { @@ -384,12 +398,14 @@ int ObAllTenantInfoProxy::fill_cell(common::sqlclient::ObMySQLResult *result, Ob uint64_t replay_scn_val = OB_INVALID_SCN_VAL; uint64_t sts_scn_val = OB_INVALID_SCN_VAL; uint64_t recovery_until_scn_val = OB_INVALID_SCN_VAL; + uint64_t sys_recovery_scn_val = OB_INVALID_SCN_VAL; ObString log_mode_str; ObString log_mode_default_value("NOARCHIVELOG"); SCN sync_scn; SCN replay_scn; SCN sts_scn; SCN recovery_until_scn; + SCN sys_recovery_scn; EXTRACT_VARCHAR_FIELD_MYSQL(*result, "tenant_role", tenant_role_str); EXTRACT_VARCHAR_FIELD_MYSQL(*result, "switchover_status", status_str); EXTRACT_INT_FIELD_MYSQL(*result, "tenant_id", tenant_id, uint64_t); @@ -398,6 +414,7 @@ int ObAllTenantInfoProxy::fill_cell(common::sqlclient::ObMySQLResult *result, Ob EXTRACT_UINT_FIELD_MYSQL(*result, "replayable_scn", replay_scn_val, uint64_t); EXTRACT_UINT_FIELD_MYSQL(*result, "readable_scn", sts_scn_val, uint64_t); EXTRACT_UINT_FIELD_MYSQL_WITH_DEFAULT_VALUE(*result, "recovery_until_scn", recovery_until_scn_val, uint64_t, false /* skip_null_error */, true /* skip_column_error */, OB_MAX_SCN_TS_NS); + EXTRACT_UINT_FIELD_MYSQL_WITH_DEFAULT_VALUE(*result, "sys_recovery_scn", sys_recovery_scn_val, uint64_t, false /* skip_null_error */, true /* skip_column_error */, OB_MIN_SCN_TS_NS); EXTRACT_VARCHAR_FIELD_MYSQL_WITH_DEFAULT_VALUE(*result, "log_mode", log_mode_str, false /* skip_null_error */, true /* skip_column_error */, log_mode_default_value); ObTenantRole tmp_tenant_role(tenant_role_str); @@ -414,13 +431,15 @@ int ObAllTenantInfoProxy::fill_cell(common::sqlclient::ObMySQLResult *result, Ob LOG_WARN("failed to convert_for_inner_table_field", KR(ret), K(sts_scn_val)); } else if (OB_FAIL(recovery_until_scn.convert_for_inner_table_field(recovery_until_scn_val))) { LOG_WARN("failed to convert_for_inner_table_field", KR(ret), K(recovery_until_scn_val)); + } else if (OB_FAIL(sys_recovery_scn.convert_for_inner_table_field(sys_recovery_scn_val))) { + LOG_WARN("failed to conver for inner table field", KR(ret), K(sys_recovery_scn_val)); } else if (OB_FAIL(tenant_info.init( tenant_id, tmp_tenant_role, tmp_tenant_sw_status, switchover_epoch, - sync_scn, replay_scn, sts_scn, recovery_until_scn, tmp_log_mode))) { + sync_scn, replay_scn, sts_scn, recovery_until_scn, tmp_log_mode, sys_recovery_scn))) { LOG_WARN("failed to init tenant info", KR(ret), K(tenant_id), K(tmp_tenant_role), K(tenant_role_str), K(tmp_tenant_sw_status), K(status_str), K(switchover_epoch), K(sync_scn), K(recovery_until_scn), - K(log_mode_str), K(tmp_log_mode)); + K(log_mode_str), K(tmp_log_mode), K(sys_recovery_scn)); } } return ret; @@ -551,8 +570,6 @@ int ObAllTenantInfoProxy::update_tenant_recovery_until_scn( ObSqlString sql; int64_t affected_rows = 0; ObTimeoutCtx ctx; - ObLSRecoveryStatOperator ls_recovery_operator; - ObLSRecoveryStat sys_ls_recovery; ObLogRestoreSourceMgr restore_source_mgr; uint64_t compat_version = 0; @@ -568,27 +585,26 @@ int ObAllTenantInfoProxy::update_tenant_recovery_until_scn( } else if (OB_UNLIKELY(!recovery_until_scn.is_valid_and_not_min())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("recovery_until_scn invalid", KR(ret), K(recovery_until_scn)); - } else if (OB_FAIL(ls_recovery_operator.get_ls_recovery_stat(tenant_id, share::SYS_LS, - true /*for_update*/, sys_ls_recovery, trans))) { - LOG_WARN("failed to get ls recovery stat", KR(ret), K(tenant_id)); - } else if (recovery_until_scn < sys_ls_recovery.get_sync_scn()) { - ret = OB_OP_NOT_ALLOW; - LOG_WARN("recover before SYS LS sync_scn is not allowed", KR(ret), K(tenant_id), K(recovery_until_scn), K(sys_ls_recovery)); - LOG_USER_ERROR(OB_OP_NOT_ALLOW, "recover before SYS LS sync_scn is"); } else if (OB_FAIL(rootserver::ObRootUtils::get_rs_default_timeout_ctx(ctx))) { LOG_WARN("fail to get timeout ctx", KR(ret), K(ctx)); } else if (OB_FAIL(sql.assign_fmt( - "update %s set recovery_until_scn = %lu where tenant_id = %lu and sync_scn <= %lu and switchover_epoch = %ld", - OB_ALL_TENANT_INFO_TNAME, recovery_until_scn.get_val_for_inner_table_field(), tenant_id, - recovery_until_scn.get_val_for_inner_table_field(), switchover_epoch))) { + "update %s set recovery_until_scn = %lu where tenant_id = %lu " + "and sync_scn <= %lu and switchover_epoch = %ld and sys_recovery_scn <= %lu", + OB_ALL_TENANT_INFO_TNAME, + recovery_until_scn.get_val_for_inner_table_field(), tenant_id, + recovery_until_scn.get_val_for_inner_table_field(), + switchover_epoch, + recovery_until_scn.get_val_for_inner_table_field()))) { LOG_WARN("failed to assign sql", KR(ret), K(tenant_id), K(recovery_until_scn), K(sql)); } else if (OB_FAIL(trans.write(exec_tenant_id, sql.ptr(), affected_rows))) { LOG_WARN("failed to execute sql", KR(ret), K(exec_tenant_id), K(sql)); } else if (0 == affected_rows) { ret = OB_OP_NOT_ALLOW; - LOG_WARN("state changed, check sync_scn and switchover status", KR(ret), K(tenant_id), + LOG_WARN("state changed, check sync_scn, switchover status and sys_recovery_scn", KR(ret), K(tenant_id), K(switchover_epoch), K(recovery_until_scn), K(sql)); - LOG_USER_ERROR(OB_OP_NOT_ALLOW, "state changed, check sync_scn and switchover status, recover is"); + LOG_USER_ERROR(OB_OP_NOT_ALLOW, + "state changed, check sync_scn, sys_recovery_scn and " + "switchover status, recover is"); } else if (!is_single_row(affected_rows)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("expect updating one row", KR(ret), K(affected_rows), @@ -601,10 +617,10 @@ int ObAllTenantInfoProxy::update_tenant_recovery_until_scn( } int64_t cost = ObTimeUtility::current_time() - begin_time; - LOG_INFO("update_recovery_until_scn finish", KR(ret), K(tenant_id), K(sys_ls_recovery), + LOG_INFO("update_recovery_until_scn finish", KR(ret), K(tenant_id), K(recovery_until_scn), K(affected_rows), K(switchover_epoch), K(sql), K(cost)); ROOTSERVICE_EVENT_ADD("tenant_info", "update_recovery_until_scn", K(ret), K(tenant_id), - K(recovery_until_scn), K(affected_rows), K(switchover_epoch), K(sys_ls_recovery)); + K(recovery_until_scn), K(affected_rows), K(switchover_epoch)); return ret; } @@ -618,6 +634,7 @@ int ObAllTenantInfoProxy::update_tenant_status( const share::SCN &replayable_scn, const share::SCN &readable_scn, const share::SCN &recovery_until_scn, + const share::SCN &sys_recovery_scn, const int64_t old_switchover_epoch) { int64_t begin_time = ObTimeUtility::current_time(); @@ -628,6 +645,7 @@ int ObAllTenantInfoProxy::update_tenant_status( ObTimeoutCtx ctx; int64_t new_switchover_epoch = OB_INVALID_VERSION; ObLogRestoreSourceMgr restore_source_mgr; + ObAllTenantInfo tenant_info; if (OB_UNLIKELY(!is_user_tenant(tenant_id) || !new_role.is_valid() @@ -638,15 +656,24 @@ int ObAllTenantInfoProxy::update_tenant_status( || !replayable_scn.is_valid_and_not_min() || !readable_scn.is_valid_and_not_min() || !recovery_until_scn.is_valid_and_not_min() + || !sys_recovery_scn.is_valid_and_not_min() || OB_INVALID_VERSION == old_switchover_epoch)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("tenant_info is invalid", KR(ret), K(tenant_id), K(new_role), K(old_status), K(new_status), K(sync_scn), K(replayable_scn), K(readable_scn), K(recovery_until_scn), K(old_switchover_epoch)); + //use tenant_info check parameters valid } else if (OB_FAIL(get_new_switchover_epoch_(old_switchover_epoch, old_status, new_status, new_switchover_epoch))) { LOG_WARN("fail to get_new_switchover_epoch_", KR(ret), K(old_switchover_epoch), K(old_status), K(new_status)); + } else if (OB_FAIL(tenant_info.init(tenant_id, new_role, new_status, new_switchover_epoch, + sync_scn, replayable_scn, readable_scn, + recovery_until_scn, NOARCHIVE_MODE, sys_recovery_scn))) { + LOG_WARN("failed to init tenant_info", KR(ret), K(tenant_id), K(new_role), + K(new_status), K(new_switchover_epoch), K(sync_scn), + K(replayable_scn), K(readable_scn), K(recovery_until_scn), + K(sys_recovery_scn)); } else if (OB_UNLIKELY(OB_INVALID_VERSION == new_switchover_epoch)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("new_switchover_ts is invalid", KR(ret), K(new_switchover_epoch), @@ -656,20 +683,22 @@ int ObAllTenantInfoProxy::update_tenant_status( } else if (OB_FAIL(sql.assign_fmt( "update %s set tenant_role = '%s', switchover_status = '%s', " "switchover_epoch = %ld, sync_scn = %lu, replayable_scn = %lu, " - "readable_scn = %lu, recovery_until_scn = %lu where tenant_id = %lu " + "readable_scn = %lu, recovery_until_scn = %lu ,sys_recovery_scn = %lu where tenant_id = %lu " "and switchover_status = '%s' and switchover_epoch = %ld " "and readable_scn <= replayable_scn and replayable_scn <= sync_scn and sync_scn <= recovery_until_scn " - "and sync_scn <= %lu and replayable_scn <= %lu and readable_scn <= %lu ", + "and sync_scn <= %lu and replayable_scn <= %lu and readable_scn <= %lu and sys_recovery_scn <= %lu", OB_ALL_TENANT_INFO_TNAME, new_role.to_str(), new_status.to_str(), new_switchover_epoch, sync_scn.get_val_for_inner_table_field(), replayable_scn.get_val_for_inner_table_field(), readable_scn.get_val_for_inner_table_field(), recovery_until_scn.get_val_for_inner_table_field(), + sys_recovery_scn.get_val_for_inner_table_field(), tenant_id, old_status.to_str(), old_switchover_epoch, sync_scn.get_val_for_inner_table_field(), replayable_scn.get_val_for_inner_table_field(), - readable_scn.get_val_for_inner_table_field()))) { + readable_scn.get_val_for_inner_table_field(), + sys_recovery_scn.get_val_for_inner_table_field()))) { LOG_WARN("failed to assign sql", KR(ret), K(tenant_id), K(sql)); } else if (OB_FAIL(trans.write(exec_tenant_id, sql.ptr(), affected_rows))) { LOG_WARN("failed to execute sql", KR(ret), K(exec_tenant_id), K(sql)); @@ -686,22 +715,6 @@ int ObAllTenantInfoProxy::update_tenant_status( LOG_WARN("failed to update_recovery_until_scn", KR(ret), K(tenant_id), K(recovery_until_scn)); } - ObAllTenantInfo tenant_info; - int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = tenant_info.init(tenant_id, - new_role, - new_status, - new_switchover_epoch, - sync_scn, - replayable_scn, - readable_scn, - recovery_until_scn))) { - LOG_WARN("failed to init tenant_info", KR(ret), KR(tmp_ret), K(tenant_id), K(new_role), - K(new_status), K(new_switchover_epoch), K(sync_scn), - K(replayable_scn), K(readable_scn), K(recovery_until_scn)); - ret = OB_SUCC(ret) ? tmp_ret : ret; - } - int64_t cost = ObTimeUtility::current_time() - begin_time; ROOTSERVICE_EVENT_ADD("tenant_info", "update_tenant_role", K(ret), K(tenant_id), K(tenant_info), K(old_status), K(old_switchover_epoch), K(cost)); @@ -770,6 +783,71 @@ int ObAllTenantInfoProxy::update_tenant_log_mode( return ret; } -} +int ObAllTenantInfoProxy::update_tenant_sys_recovery_scn( + const uint64_t tenant_id, + const share::SCN &sys_recovery_scn, + bool update_sync_scn, ObISQLClient *proxy) +{ + int ret = OB_SUCCESS; + const uint64_t exec_tenant_id = gen_meta_tenant_id(tenant_id); + + if (OB_UNLIKELY(!is_user_tenant(tenant_id) + || !sys_recovery_scn.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(sys_recovery_scn)); + } else { + TAKE_IN_TRANS(update_tenant_sys_recovery_scn, proxy, exec_tenant_id, + tenant_id, sys_recovery_scn, update_sync_scn); + } + return ret; +} + +int ObAllTenantInfoProxy::update_tenant_sys_recovery_scn_in_trans( + const uint64_t tenant_id, + const share::SCN &sys_recovery_scn, + bool update_sync_scn, common::ObMySQLTransaction &trans) +{ + int ret = OB_SUCCESS; + const uint64_t exec_tenant_id = gen_meta_tenant_id(tenant_id); + ObSqlString sql; + int64_t affected_rows = 0; + ObTimeoutCtx ctx; + ObAllTenantInfo tenant_info; + + if (OB_UNLIKELY(!is_user_tenant(tenant_id) + || !sys_recovery_scn.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(sys_recovery_scn)); + } else if (OB_FAIL(rootserver::ObRootUtils::get_rs_default_timeout_ctx(ctx))) { + LOG_WARN("fail to get timeout ctx", KR(ret), K(ctx)); + } else if (OB_FAIL(load_tenant_info(tenant_id, &trans, true, tenant_info))) { + LOG_WARN("failed to load tenant info", KR(ret), K(tenant_id)); + } else if (!tenant_info.get_switchover_status().can_report_recovery_status() + || sys_recovery_scn <= tenant_info.get_sys_recovery_scn() + || sys_recovery_scn > tenant_info.get_recovery_until_scn() + || sys_recovery_scn < tenant_info.get_sync_scn()) { + ret = OB_NEED_RETRY; + LOG_WARN("can not report recovery status", KR(ret), + K(tenant_info), K(sys_recovery_scn)); + } else if (OB_FAIL(sql.assign_fmt( + "update %s set sys_recovery_scn = %lu ", + OB_ALL_TENANT_INFO_TNAME, sys_recovery_scn.get_val_for_inner_table_field()))) { + LOG_WARN("failed to assign sql", KR(ret), K(tenant_id), K(sys_recovery_scn), K(sql)); + } else if (update_sync_scn && OB_FAIL(sql.append_fmt(", sync_scn = %lu ", + sys_recovery_scn.get_val_for_inner_table_field()))) { + LOG_WARN("failed to append sql", KR(ret), K(sys_recovery_scn), K(sql)); + } else if (OB_FAIL(trans.write(exec_tenant_id, sql.ptr(), affected_rows))) { + LOG_WARN("failed to execute sql", KR(ret), K(exec_tenant_id), K(sql)); + } else if (!is_single_row(affected_rows)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("expect updating one row", KR(ret), K(affected_rows), K(sql)); + } + return ret; + + LOG_TRACE("update sys_recovery_scn", KR(ret), K(tenant_id), K(sys_recovery_scn), + K(update_sync_scn)); +} + +} // namespace share } diff --git a/src/share/ob_tenant_info_proxy.h b/src/share/ob_tenant_info_proxy.h index bf57ca93c..7428c5f0c 100644 --- a/src/share/ob_tenant_info_proxy.h +++ b/src/share/ob_tenant_info_proxy.h @@ -43,9 +43,12 @@ bool is_valid_tenant_scn( const share::SCN &sync_scn, const share::SCN &replayable_scn, const share::SCN &standby_scn, - const share::SCN &recovery_until_scn); + const share::SCN &recovery_until_scn, + const share::SCN &sys_recovery_scn); -SCN gen_new_sync_scn(const share::SCN &cur_sync_scn, const share::SCN &desired_sync_scn, const share::SCN &cur_recovery_until_scn); +SCN gen_new_sync_scn(const share::SCN &cur_sync_scn, const share::SCN &desired_sync_scn, + const share::SCN &cur_recovery_until_scn, + const share::SCN &sys_recovery_scn); struct ObAllTenantInfo { @@ -64,6 +67,7 @@ public: * @param[in] standby_scn * @param[in] recovery_until_scn * @param[in] log_mode + * @param[in] sys_recovery_scn */ int init(const uint64_t tenant_id, const ObTenantRole &tenant_role, @@ -73,7 +77,8 @@ public: const SCN &replayable_scn = SCN::base_scn(), const SCN &standby_scn = SCN::base_scn(), const SCN &recovery_until_scn = SCN::base_scn(), - const ObArchiveMode &log_mode = NOARCHIVE_MODE); + const ObArchiveMode &log_mode = NOARCHIVE_MODE, + const SCN &sys_recovery_scn = SCN::base_scn()); ObAllTenantInfo &operator=(const ObAllTenantInfo &other); void assign(const ObAllTenantInfo &other); void reset(); @@ -90,12 +95,7 @@ public: * Because STS will be changed when switchover to standby. */ bool is_sts_ready() const { return !(tenant_role_.is_primary() - || tenant_is_switchover_to_standby()); } - - // ************* Functions that describe what tenant is doing ********************* - // tenant is in switchover from primary to standby - bool tenant_is_switchover_to_standby() const { return is_prepare_switching_to_standby_status() - || is_switching_to_standby_status(); } + || is_switching_to_standby_status());} // ObTenantSwitchoverStatus related function #define IS_TENANT_STATUS(STATUS) \ @@ -106,13 +106,12 @@ IS_TENANT_STATUS(switching_to_primary) IS_TENANT_STATUS(prepare_flashback_for_failover_to_primary) IS_TENANT_STATUS(flashback) IS_TENANT_STATUS(switching_to_standby) -IS_TENANT_STATUS(prepare_switching_to_standby) IS_TENANT_STATUS(prepare_flashback_for_switch_to_primary) #undef IS_TENANT_STATUS TO_STRING_KV(K_(tenant_id), K_(tenant_role), K_(switchover_status), K_(switchover_epoch), K_(sync_scn), K_(replayable_scn), - K_(standby_scn), K_(recovery_until_scn), K_(log_mode)); + K_(standby_scn), K_(recovery_until_scn), K_(log_mode), K_(sys_recovery_scn)); // Getter&Setter const ObTenantRole &get_tenant_role() const { return tenant_role_; } @@ -132,6 +131,7 @@ public:\ Property_declare_var(share::SCN, standby_scn) Property_declare_var(share::SCN, recovery_until_scn) Property_declare_var(ObArchiveMode, log_mode) + Property_declare_var(share::SCN, sys_recovery_scn) #undef Property_declare_var private: ObTenantRole tenant_role_; @@ -169,14 +169,12 @@ public: * @description: update tenant recovery status * @param[in] tenant_id * @param[in] proxy - * @param[in] status: the target status while update recovery status * @param[in] sync_scn : sync point * @param[in] replay_scn : max replay point * @param[in] reabable_scn : standby readable scn */ static int update_tenant_recovery_status(const uint64_t tenant_id, ObMySQLProxy *proxy, - ObTenantSwitchoverStatus status, const SCN &sync_scn, const SCN &replay_scn, const SCN &reabable_scn); @@ -219,6 +217,7 @@ public: const share::SCN &replayable_scn, const share::SCN &readable_scn, const share::SCN &recovery_until_scn, + const share::SCN &sys_recovery_scn, const int64_t old_switchover_epoch); /** @@ -282,6 +281,23 @@ public: const ObArchiveMode &old_log_mode, const ObArchiveMode &new_log_mode); + /** + * @description: update tenant sys ls log sys recovery scn + * @param[in] tenant_id + * @param[in] trans + * @param[in] sys_recovery_scn new sys recovery scn + * @param[in] update_sync_scn : When iterating to the LS operations, these operations can be executed provided that + * the sync_scn of other ls have reached the LS operation's point, + * so when updating the sys_recovery_scn, can the sync_scn be updated at the same time. + * return : + * OB_SUCCESS update tenant sys recovery scn successfully + * OB_NEED_RETRY sys recovery scn backoff, need retry + */ + DEFINE_IN_TRANS_FUC1(update_tenant_sys_recovery_scn, + const uint64_t tenant_id, + const share::SCN &sys_recovery_scn, + bool update_sync_scn); + }; } } diff --git a/src/share/ob_tenant_switchover_status.cpp b/src/share/ob_tenant_switchover_status.cpp index ccbe39469..851311037 100644 --- a/src/share/ob_tenant_switchover_status.cpp +++ b/src/share/ob_tenant_switchover_status.cpp @@ -27,7 +27,6 @@ static const char* TENANT_SWITCHOVER_ARRAY[] = "SWITCHING TO PRIMARY", "PREPARE FLASHBACK", "FLASHBACK", - "PREPARE SWITCHING TO STANDBY", "SWITCHING TO STANDBY", "PREPARE SWITCHING TO PRIMARY", }; diff --git a/src/share/ob_tenant_switchover_status.h b/src/share/ob_tenant_switchover_status.h index ba3f08a5b..1c712c0db 100644 --- a/src/share/ob_tenant_switchover_status.h +++ b/src/share/ob_tenant_switchover_status.h @@ -33,10 +33,9 @@ public: SWITCHING_TO_PRIMARY_STATUS = 2, PREPARE_FLASHBACK_FOR_FAILOVER_TO_PRIMARY_STATUS = 3, FLASHBACK_STATUS = 4, - PREPARE_SWITCHING_TO_STANDBY_STATUS = 5, - SWITCHING_TO_STANDBY_STATUS = 6, - PREPARE_FLASHBACK_FOR_SWITCH_TO_PRIMARY_STATUS = 7, - MAX_STATUS = 8 + SWITCHING_TO_STANDBY_STATUS = 5, + PREPARE_FLASHBACK_FOR_SWITCH_TO_PRIMARY_STATUS = 6, + MAX_STATUS = 7 }; public: ObTenantSwitchoverStatus() : value_(INVALID_STATUS) {} @@ -49,6 +48,13 @@ public: bool is_valid() const { return INVALID_STATUS != value_; } ObTenantSwitchoverStatus::Status value() const { return value_; } const char* to_str() const; + //In the flashback status, the log needs to be truncated, + //and sync_scn will fall back, so at this stage, + //__all_ls_recovery_stat/__all_tenant_info will not be reported + bool can_report_recovery_status() const + { + return FLASHBACK_STATUS != value_; + } // compare operator bool operator == (const ObTenantSwitchoverStatus &other) const { return value_ == other.value_; } @@ -69,7 +75,6 @@ IS_TENANT_STATUS(NORMAL_STATUS, normal) IS_TENANT_STATUS(SWITCHING_TO_PRIMARY_STATUS, switching_to_primary) IS_TENANT_STATUS(PREPARE_FLASHBACK_FOR_FAILOVER_TO_PRIMARY_STATUS, prepare_flashback_for_failover_to_primary) IS_TENANT_STATUS(FLASHBACK_STATUS, flashback) -IS_TENANT_STATUS(PREPARE_SWITCHING_TO_STANDBY_STATUS, prepare_switching_to_standby) IS_TENANT_STATUS(SWITCHING_TO_STANDBY_STATUS, switching_to_standby) IS_TENANT_STATUS(PREPARE_FLASHBACK_FOR_SWITCH_TO_PRIMARY_STATUS, prepare_flashback_for_switch_to_primary) #undef IS_TENANT_STATUS @@ -84,7 +89,6 @@ static const ObTenantSwitchoverStatus NORMAL_SWITCHOVER_STATUS(ObTenantSwitchove static const ObTenantSwitchoverStatus PREPARE_FLASHBACK_FOR_FAILOVER_TO_PRIMARY_SWITCHOVER_STATUS(ObTenantSwitchoverStatus::PREPARE_FLASHBACK_FOR_FAILOVER_TO_PRIMARY_STATUS); static const ObTenantSwitchoverStatus FLASHBACK_SWITCHOVER_STATUS(ObTenantSwitchoverStatus::FLASHBACK_STATUS); static const ObTenantSwitchoverStatus SWITCHING_TO_PRIMARY_SWITCHOVER_STATUS(ObTenantSwitchoverStatus::SWITCHING_TO_PRIMARY_STATUS); -static const ObTenantSwitchoverStatus PREP_SWITCHING_TO_STANDBY_SWITCHOVER_STATUS(ObTenantSwitchoverStatus::PREPARE_SWITCHING_TO_STANDBY_STATUS); static const ObTenantSwitchoverStatus SWITCHING_TO_STANDBY_SWITCHOVER_STATUS(ObTenantSwitchoverStatus::SWITCHING_TO_STANDBY_STATUS); static const ObTenantSwitchoverStatus PREPARE_FLASHBACK_FOR_SWITCH_TO_PRIMARY_SWITCHOVER_STATUS(ObTenantSwitchoverStatus::PREPARE_FLASHBACK_FOR_SWITCH_TO_PRIMARY_STATUS); diff --git a/src/share/ob_upgrade_utils.cpp b/src/share/ob_upgrade_utils.cpp index 09f08f3a1..a14a2c313 100644 --- a/src/share/ob_upgrade_utils.cpp +++ b/src/share/ob_upgrade_utils.cpp @@ -21,6 +21,10 @@ #include "rootserver/ob_root_service.h" #include "sql/resolver/expr/ob_raw_expr_util.h" #include "share/ob_rpc_struct.h" +#include "share/ob_tenant_info_proxy.h"//AllTenantInfo +#include "share/ls/ob_ls_i_life_manager.h"//start_transaction +#include "rootserver/ob_ls_service_helper.h" +#include "ob_upgrade_utils.h" namespace oceanbase { @@ -791,6 +795,8 @@ int ObUpgradeFor4100Processor::post_upgrade() LOG_WARN("fail to check inner stat", KR(ret)); } else if (OB_FAIL(recompile_all_views_and_synonyms(tenant_id))) { LOG_WARN("fail to init rewrite rule version", K(ret), K(tenant_id)); + } else if (OB_FAIL(init_tenant_sys_recovery_scn(tenant_id))) { + LOG_WARN("failed to init tenant sys recovery scn", KR(ret), K(tenant_id)); } return ret; } @@ -924,6 +930,106 @@ int ObUpgradeFor4100Processor::recompile_all_views_and_synonyms(const uint64_t t return ret; } +int ObUpgradeFor4100Processor::init_tenant_sys_recovery_scn(const uint64_t tenant_id) +{ + int ret = OB_SUCCESS; + int64_t start = ObTimeUtility::current_time(); + if (!is_user_tenant(tenant_id)) { + //no need to update meta tenant or sys tenant sys recovery scn + } else { + //check sys recovery scn, make status to equal and set sys_recovery_scn + const uint64_t exec_tenant_id = gen_meta_tenant_id(tenant_id); + ObAllTenantInfo tenant_info; + SCN sync_scn; + ObTenantSchema tenant_schema; + if (OB_FAIL(rootserver::ObTenantThreadHelper::get_tenant_schema(tenant_id, tenant_schema))) { + LOG_WARN("failed to get tenant schema", KR(ret), K(tenant_id)); + } + START_TRANSACTION(sql_proxy_, exec_tenant_id); + rootserver::ObTenantLSInfo tenant_stat(sql_proxy_, &tenant_schema, + tenant_id, GCTX.srv_rpc_proxy_, + GCTX.lst_operator_); + if (FAILEDx(ObAllTenantInfoProxy::load_tenant_info(tenant_id, &trans, + true, tenant_info))) { + LOG_WARN("failed to load tenant info", KR(ret), K(tenant_id)); + } else if (!tenant_info.get_sys_recovery_scn().is_min()) { + LOG_INFO("tenant sys recovery scn already update, no need to init", K(tenant_info)); + } else if (OB_FAIL(tenant_stat.revision_to_equal_status(trans))) { + LOG_WARN("failed to make revision status", KR(ret), K(tenant_id)); + } else if (OB_FAIL(get_sys_ls_max_sync_scn_(tenant_id, sync_scn))) { + LOG_WARN("failed to get sys ls max sync scn", KR(ret), K(tenant_id)); + } else if (OB_FAIL(ObAllTenantInfoProxy::update_tenant_sys_recovery_scn_in_trans( + tenant_id, sync_scn, false, trans))) { + LOG_WARN("failed to update tenant sys recovery scn", KR(ret), K(tenant_id), + K(sync_scn), K(tenant_info)); + } else { + LOG_INFO("init tenant sys recovery scn success", K(tenant_id), K(sync_scn)); + } + END_TRANSACTION(trans); + } + + LOG_INFO("add tenant sys recovery scn finish", K(ret), K(tenant_id_), "cost", ObTimeUtility::current_time() - start); + return ret; +} + +int ObUpgradeFor4100Processor::get_sys_ls_max_sync_scn_(const uint64_t tenant_id, SCN &sync_scn) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(GCTX.sql_proxy_) || OB_ISNULL(GCTX.srv_rpc_proxy_) || + OB_ISNULL(GCTX.location_service_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("error unexpcted", KR(ret), KP(GCTX.sql_proxy_), KP(GCTX.srv_rpc_proxy_), + KP(GCTX.location_service_)); + } else { + ObTimeoutCtx ctx; + ObAddr leader; + obrpc::ObGetLSSyncScnArg arg; + ObGetLSSyncScnRes res; + int tmp_ret = OB_SUCCESS; + rootserver::ObGetLSSyncScnProxy proxy( + *GCTX.srv_rpc_proxy_, &obrpc::ObSrvRpcProxy::get_ls_sync_scn); + const int64_t timeout = GCONF.internal_sql_execute_timeout; + if (FAILEDx(arg.init(tenant_id, SYS_LS, false))) { + LOG_WARN("failed to init arg", KR(ret), K(tenant_id)); + } else if (OB_FAIL(ObShareUtil::set_default_timeout_ctx(ctx, timeout))) { + LOG_WARN("fail to set timeout ctx", KR(ret), K(timeout)); + } + while (OB_SUCC(ret)) { + proxy.reuse(); + if (ctx.is_timeouted()) { + ret = OB_TIMEOUT; + LOG_WARN("already timeout, failed to get sys ls sync scn", KR(ret), K(ctx), K(tenant_id)); + } else { + if (OB_FAIL(GCTX.location_service_->get_leader( + GCONF.cluster_id, tenant_id, SYS_LS, true, leader))) { + LOG_WARN("failed to get leader", KR(ret), K(tenant_id)); + } else if (OB_FAIL(proxy.call(leader, ctx.get_timeout(), tenant_id, arg))) { + LOG_WARN("failed to get ls sync scn", KR(ret), K(arg), K(ctx), + K(leader)); + } + if (OB_TMP_FAIL(proxy.wait())) { + ret = OB_SUCC(ret) ? tmp_ret : ret; + LOG_WARN("failed to wait all", KR(ret), KR(tmp_ret)); + } else if (OB_FAIL(ret)) { + } else if (1 != proxy.get_results().count() || + OB_ISNULL(proxy.get_results().at(0))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get rpc result", KR(ret), "proxy result", proxy.get_results()); + } else { + sync_scn = proxy.get_results().at(0)->get_cur_sync_scn(); + break; + } + if (OB_FAIL(ret)) { + //ignore error of each rpc until timeout + LOG_WARN("failed to get sys ls sync scn, sleep and try again", KR(ret)); + ret = OB_SUCCESS; + usleep(100 * 1000L); + } + } + } + } + return ret; +} /* =========== 4100 upgrade processor end ============= */ /* =========== special upgrade processor end ============= */ } // end share diff --git a/src/share/ob_upgrade_utils.h b/src/share/ob_upgrade_utils.h index bd4f32988..90c69cc16 100644 --- a/src/share/ob_upgrade_utils.h +++ b/src/share/ob_upgrade_utils.h @@ -22,9 +22,13 @@ namespace oceanbase { +namespace common +{ +class ObTimeoutCtx; +} namespace share { - +class SCN; static const int64_t UPGRADE_JOB_TYPE_COUNT = 1; static const rootserver::ObRsJobType upgrade_job_type_array[UPGRADE_JOB_TYPE_COUNT] = { rootserver::JOB_TYPE_INVALID, @@ -186,6 +190,8 @@ private: int post_upgrade_for_srs(); int init_rewrite_rule_version(const uint64_t tenant_id); static int recompile_all_views_and_synonyms(const uint64_t tenant_id); + int init_tenant_sys_recovery_scn(const uint64_t tenant_id); + int get_sys_ls_max_sync_scn_(const uint64_t tenant_id, share::SCN &sys_scn); }; /* =========== special upgrade processor end ============= */ diff --git a/src/share/rc/ob_tenant_base.h b/src/share/rc/ob_tenant_base.h index 77866cc0c..76216b464 100644 --- a/src/share/rc/ob_tenant_base.h +++ b/src/share/rc/ob_tenant_base.h @@ -124,9 +124,11 @@ namespace rootserver { class ObPrimaryMajorFreezeService; class ObRestoreMajorFreezeService; - class ObTenantRecoveryReportor; + class ObLSRecoveryReportor; class ObTenantInfoLoader; + class ObTenantInfoReportor; class ObPrimaryLSService; + class ObCommonLSService; class ObRestoreService; class ObRecoveryLSService; class ObArbitrationService; @@ -198,8 +200,10 @@ using ObPartTransCtxObjPool = common::ObServerObjectPoolset_cache_ls(this); } - if (OB_SUCC(ret) && ls_id.is_sys_ls()) { - //meta tenant need create thread for ls manager + if (OB_SUCC(ret) && ls_id.is_sys_ls() && is_user_tenant(tenant_id)) { REGISTER_TO_LOGSERVICE(logservice::PRIMARY_LS_SERVICE_LOG_BASE_TYPE, MTL(rootserver::ObPrimaryLSService *)); - LOG_INFO("primary ls manager registre to logservice success"); + LOG_INFO("primary ls manager register to logservice success"); + } + if (OB_SUCC(ret) && ls_id.is_sys_ls() && !is_user_tenant(tenant_id)) { + REGISTER_TO_LOGSERVICE(logservice::COMMON_LS_SERVICE_LOG_BASE_TYPE, MTL(rootserver::ObCommonLSService *)); + LOG_INFO("common ls manager register to logservice success"); + } + if (OB_SUCC(ret) && ls_id.is_sys_ls() && is_meta_tenant(tenant_id)) { + //sys no need to report + REGISTER_TO_LOGSERVICE(logservice::TENANT_INFO_REPORTOR_LOG_BASE_TYPE, MTL(rootserver::ObTenantInfoReportor *)); + LOG_INFO("tenant info report register to logservice success"); + } if (OB_SUCC(ret) && is_user_tenant(tenant_id) && ls_id.is_sys_ls()) { // only user tenant need dump datadict REGISTER_TO_LOGSERVICE(logservice::DATA_DICT_LOG_BASE_TYPE, MTL(datadict::ObDataDictService *)); //only user table need recovery REGISTER_TO_RESTORESERVICE(logservice::RECOVERY_LS_SERVICE_LOG_BASE_TYPE, MTL(rootserver::ObRecoveryLSService *)); - LOG_INFO("recovery ls manager registre to restoreservice success"); + LOG_INFO("recovery ls manager register to restoreservice success"); } if (OB_SUCC(ret) && !is_user_tenant(tenant_id) && ls_id.is_sys_ls()) { @@ -632,20 +643,24 @@ void ObLS::destroy() UNREGISTER_FROM_LOGSERVICE(logservice::GAIS_LOG_BASE_TYPE, MTL(share::ObGlobalAutoIncService *)); MTL(share::ObGlobalAutoIncService *)->set_cache_ls(nullptr); } - if (OB_SUCC(ret) && ls_meta_.ls_id_.is_sys_ls()) { + if (ls_meta_.ls_id_.is_sys_ls() && is_user_tenant(MTL_ID())) { rootserver::ObPrimaryLSService* ls_service = MTL(rootserver::ObPrimaryLSService*); UNREGISTER_FROM_LOGSERVICE(logservice::PRIMARY_LS_SERVICE_LOG_BASE_TYPE, ls_service); } - - if (OB_SUCC(ret) && is_user_tenant(MTL_ID()) && ls_meta_.ls_id_.is_sys_ls()) { + if (ls_meta_.ls_id_.is_sys_ls() && !is_user_tenant(MTL_ID())) { + rootserver::ObCommonLSService *ls_service = MTL(rootserver::ObCommonLSService*); + UNREGISTER_FROM_LOGSERVICE(logservice::COMMON_LS_SERVICE_LOG_BASE_TYPE, ls_service); + rootserver::ObRestoreService * restore_service = MTL(rootserver::ObRestoreService*); + UNREGISTER_FROM_LOGSERVICE(logservice::RESTORE_SERVICE_LOG_BASE_TYPE, restore_service); + } + if (ls_meta_.ls_id_.is_sys_ls() && is_meta_tenant(MTL_ID())) { + UNREGISTER_FROM_LOGSERVICE(logservice::TENANT_INFO_REPORTOR_LOG_BASE_TYPE, MTL(rootserver::ObTenantInfoReportor*)); + } + if (is_user_tenant(MTL_ID()) && ls_meta_.ls_id_.is_sys_ls()) { UNREGISTER_FROM_LOGSERVICE(logservice::DATA_DICT_LOG_BASE_TYPE, MTL(datadict::ObDataDictService *)); rootserver::ObRecoveryLSService* ls_service = MTL(rootserver::ObRecoveryLSService*); UNREGISTER_FROM_RESTORESERVICE(logservice::RECOVERY_LS_SERVICE_LOG_BASE_TYPE, ls_service); } - if (OB_SUCC(ret) && !is_user_tenant(MTL_ID()) && ls_meta_.ls_id_.is_sys_ls()) { - rootserver::ObRestoreService * restore_service = MTL(rootserver::ObRestoreService*); - UNREGISTER_FROM_LOGSERVICE(logservice::RESTORE_SERVICE_LOG_BASE_TYPE, restore_service); - } tx_table_.destroy(); lock_table_.destroy(); ls_tablet_svr_.destroy(); diff --git a/src/storage/tx/ob_trans_service_v4.cpp b/src/storage/tx/ob_trans_service_v4.cpp index 56c54d26d..5b346b611 100644 --- a/src/storage/tx/ob_trans_service_v4.cpp +++ b/src/storage/tx/ob_trans_service_v4.cpp @@ -39,7 +39,6 @@ #include "storage/tx_storage/ob_ls_handle.h" #include "storage/ls/ob_ls.h" #include "ob_xa_service.h" -#include "rootserver/ob_tenant_recovery_reportor.h" /* interface(s) */ namespace oceanbase { diff --git a/src/storage/tx_storage/ob_checkpoint_service.cpp b/src/storage/tx_storage/ob_checkpoint_service.cpp index 8438d4893..db9d35a80 100644 --- a/src/storage/tx_storage/ob_checkpoint_service.cpp +++ b/src/storage/tx_storage/ob_checkpoint_service.cpp @@ -25,7 +25,11 @@ #include "logservice/palf/palf_env.h" #include "logservice/palf/lsn.h" #include "logservice/archiveservice/ob_archive_service.h" +#include "logservice/ob_log_handler.h" //ObLogHandler #include "storage/tx_storage/ob_ls_handle.h" +#include "rootserver/ob_tenant_info_loader.h"//get_tenant_info +#include "share/ob_cluster_version.h"//get_tenant_data_version +#include "share/ob_tenant_info_proxy.h"//ObAllTenantInfo namespace oceanbase { @@ -177,13 +181,29 @@ void ObCheckPointService::ObCheckpointTask::runTimerTask() K(archive_lsn), K(checkpoint_lsn), KPC(ls)); checkpoint_lsn = archive_lsn; } - if (OB_FAIL(ls->get_log_handler()->advance_base_lsn(checkpoint_lsn))) { - ARCHIVE_LOG(WARN, "advance base lsn failed", K(ret), K(checkpoint_lsn)); - } else { - FLOG_INFO("[CHECKPOINT] advance palf base lsn successfully", - K(checkpoint_lsn), K(ls->get_ls_id())); - } } + if (OB_SUCC(ret) && ls->get_ls_id().is_sys_ls()) { + //The check_point point of the sys ls refers to the sys_recovery_scn of __all_tenant_info + const uint64_t tenant_id = ls->get_tenant_id(); + palf::LSN sys_recovery_lsn; + if (!is_user_tenant(tenant_id)) { + //nothing todo + } else if (OB_FAIL(get_sys_ls_recovery_lsn_(ls->get_log_handler(), sys_recovery_lsn))) { + LOG_WARN("failed to get sys ls recovery lsn", KR(ret)); + } else if (sys_recovery_lsn < checkpoint_lsn) { + LOG_TRACE( + "sys ls recovery lsn is small than checkpoint_lsn, set base_lsn with sys recovery lsn", + K(sys_recovery_lsn), K(checkpoint_lsn)); + checkpoint_lsn = sys_recovery_lsn; + } // end for user tenant + } + if (FAILEDx(ls->get_log_handler()->advance_base_lsn(checkpoint_lsn))) { + ARCHIVE_LOG(WARN, "advance base lsn failed", K(ret), K(checkpoint_lsn)); + } else { + FLOG_INFO("[CHECKPOINT] advance palf base lsn successfully", + K(checkpoint_lsn), K(ls->get_ls_id())); + } + } if (ret == OB_ITER_END) { ret = OB_SUCCESS; @@ -196,6 +216,32 @@ void ObCheckPointService::ObCheckpointTask::runTimerTask() } } +int ObCheckPointService::ObCheckpointTask::get_sys_ls_recovery_lsn_( + logservice::ObLogHandler *log_handler, palf::LSN &recovery_lsn) +{ + int ret = OB_SUCCESS; + share::ObAllTenantInfo tenant_info; + SCN recovery_scn; + recovery_lsn.reset(); + if (OB_FAIL(MTL(rootserver::ObTenantInfoLoader *)->get_tenant_info(tenant_info))) { + LOG_WARN("failed to get tenant info", KR(ret)); + } else if (tenant_info.get_sys_recovery_scn().is_min()) { + // min is during upgrade, use sync scn instead + recovery_scn = tenant_info.get_sync_scn(); + } else { + recovery_scn = tenant_info.get_sys_recovery_scn(); + } + if (OB_FAIL(ret)) { + } else if (OB_ISNULL(log_handler)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("log handle is null", KR(ret), K(tenant_info)); + } else if (OB_FAIL(log_handler->locate_by_scn_coarsely(recovery_scn, + recovery_lsn))) { + LOG_WARN("failed to locate scn", KR(ret), K(tenant_info)); + } + return ret; +} + bool ObCheckPointService::get_disk_usage_threshold_(int64_t &threshold) { int ret = OB_SUCCESS; diff --git a/src/storage/tx_storage/ob_checkpoint_service.h b/src/storage/tx_storage/ob_checkpoint_service.h index 720c1d809..5a6a4b059 100644 --- a/src/storage/tx_storage/ob_checkpoint_service.h +++ b/src/storage/tx_storage/ob_checkpoint_service.h @@ -19,6 +19,13 @@ namespace oceanbase { +namespace logservice +{ +class ObLogHandler; +} +namespace palf { +struct LSN; +} namespace storage { namespace checkpoint @@ -75,6 +82,8 @@ private: virtual ~ObCheckpointTask() {} virtual void runTimerTask(); + private: + int get_sys_ls_recovery_lsn_(logservice::ObLogHandler *log_handler, palf::LSN &recovery_lsn); }; class ObTraversalFlushTask : public common::ObTimerTask diff --git a/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp b/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp index ec5510101..bed96c43e 100644 --- a/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp +++ b/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp @@ -2152,7 +2152,7 @@ int ObAdminDumpBackupDataExecutor::dump_ls_attr_info_(const share::ObLSAttr &ls_ PrintHelper::print_dump_line("ls_group_id", ls_attr.get_ls_group_id()); PrintHelper::print_dump_line("flag", ls_attr.get_ls_flag()); PrintHelper::print_dump_line("status", ls_attr.get_ls_status()); - PrintHelper::print_dump_line("operation_type", ls_attr.get_ls_operatin_type()); + PrintHelper::print_dump_line("operation_type", ls_attr.get_ls_operation_type()); return ret; } diff --git a/unittest/rootserver/test_primary_ls_service.cpp b/unittest/rootserver/test_primary_ls_service.cpp index 576797bb9..c3e560b39 100644 --- a/unittest/rootserver/test_primary_ls_service.cpp +++ b/unittest/rootserver/test_primary_ls_service.cpp @@ -14,7 +14,7 @@ #include #include #define private public -#include "rootserver/ob_primary_ls_service.h" +#include "rootserver/ob_common_ls_service.h" namespace oceanbase { using namespace common; using namespace share; @@ -57,12 +57,12 @@ TEST_F(TestPrimaryLSService, zone_balance) ASSERT_EQ(ret, OB_SUCCESS); ret = primary_zone_array.push_back(z1); ASSERT_EQ(ret, OB_SUCCESS); - ret = ObTenantLSInfo::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone); + ret = ObCommonLSService::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone); ASSERT_EQ(ret, OB_SUCCESS); LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone)); ASSERT_EQ(z1, ls_primary_zone.at(0)); ASSERT_EQ(1, count_group_by_zone.at(0)); - ret = ObTenantLSInfo::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone); + ret = ObCommonLSService::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone); ASSERT_EQ(ret, OB_SUCCESS); ASSERT_EQ(z1, ls_primary_zone.at(0)); ASSERT_EQ(1, count_group_by_zone.at(0)); @@ -74,14 +74,14 @@ TEST_F(TestPrimaryLSService, zone_balance) ASSERT_EQ(ret, OB_SUCCESS); ret = primary_zone_array.push_back(z2); ASSERT_EQ(ret, OB_SUCCESS); - ret = ObTenantLSInfo::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone); + ret = ObCommonLSService::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone); LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone)); ASSERT_EQ(ret, OB_SUCCESS); ASSERT_EQ(z2, ls_primary_zone.at(0)); ASSERT_EQ(z2, ls_primary_zone.at(1)); ASSERT_EQ(0, count_group_by_zone.at(0)); ASSERT_EQ(2, count_group_by_zone.at(1)); - ret = ObTenantLSInfo::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone); + ret = ObCommonLSService::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone); LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone)); ASSERT_EQ(ret, OB_SUCCESS); ASSERT_EQ(z1, ls_primary_zone.at(0)); @@ -97,14 +97,14 @@ TEST_F(TestPrimaryLSService, zone_balance) ASSERT_EQ(ret, OB_SUCCESS); ret = primary_zone_array.push_back(z3); ASSERT_EQ(ret, OB_SUCCESS); - ret = ObTenantLSInfo::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone); + ret = ObCommonLSService::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone); LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone)); ASSERT_EQ(ret, OB_SUCCESS); ASSERT_EQ(z1, ls_primary_zone.at(0)); ASSERT_EQ(z3, ls_primary_zone.at(1)); ASSERT_EQ(1, count_group_by_zone.at(0)); ASSERT_EQ(1, count_group_by_zone.at(1)); - ret = ObTenantLSInfo::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone); + ret = ObCommonLSService::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone); LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone)); ASSERT_EQ(ret, OB_SUCCESS); ASSERT_EQ(z1, ls_primary_zone.at(0)); @@ -117,7 +117,7 @@ TEST_F(TestPrimaryLSService, zone_balance) count_group_by_zone.reset(); ret = primary_zone_array.push_back(z2); ASSERT_EQ(ret, OB_SUCCESS); - ret = ObTenantLSInfo::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone); + ret = ObCommonLSService::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone); LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone)); ASSERT_EQ(ret, OB_SUCCESS); ASSERT_EQ(z2, ls_primary_zone.at(0)); @@ -125,7 +125,7 @@ TEST_F(TestPrimaryLSService, zone_balance) ASSERT_EQ(0, count_group_by_zone.at(0)); ASSERT_EQ(0, count_group_by_zone.at(1)); ASSERT_EQ(2, count_group_by_zone.at(2)); - ret = ObTenantLSInfo::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone); + ret = ObCommonLSService::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone); LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone)); ASSERT_EQ(ret, OB_SUCCESS); ASSERT_EQ(z1, ls_primary_zone.at(0)); @@ -145,7 +145,7 @@ TEST_F(TestPrimaryLSService, zone_balance) ASSERT_EQ(ret, OB_SUCCESS); ret = primary_zone_infos.push_back(info); ASSERT_EQ(ret, OB_SUCCESS); - ret = ObTenantLSInfo::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone); + ret = ObCommonLSService::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone); LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone)); ASSERT_EQ(ret, OB_SUCCESS); ASSERT_EQ(z2, ls_primary_zone.at(0)); @@ -155,7 +155,7 @@ TEST_F(TestPrimaryLSService, zone_balance) ASSERT_EQ(1, count_group_by_zone.at(0)); ASSERT_EQ(0, count_group_by_zone.at(1)); ASSERT_EQ(3, count_group_by_zone.at(2)); - ret = ObTenantLSInfo::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone); + ret = ObCommonLSService::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone); LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone)); ASSERT_EQ(ret, OB_SUCCESS); ASSERT_EQ(z3, ls_primary_zone.at(0)); @@ -184,7 +184,7 @@ TEST_F(TestPrimaryLSService, zone_balance) ASSERT_EQ(ret, OB_SUCCESS); ret = primary_zone_infos.push_back(info); ASSERT_EQ(ret, OB_SUCCESS); - ret = ObTenantLSInfo::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone); + ret = ObCommonLSService::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone); LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone)); ASSERT_EQ(ret, OB_SUCCESS); ASSERT_EQ(z1, ls_primary_zone.at(0)); @@ -195,7 +195,7 @@ TEST_F(TestPrimaryLSService, zone_balance) ASSERT_EQ(3, count_group_by_zone.at(0)); ASSERT_EQ(1, count_group_by_zone.at(1)); ASSERT_EQ(1, count_group_by_zone.at(2)); - ret = ObTenantLSInfo::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone); + ret = ObCommonLSService::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone); LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone)); ASSERT_EQ(z3, ls_primary_zone.at(0)); ASSERT_EQ(z1, ls_primary_zone.at(1)); @@ -232,7 +232,7 @@ TEST_F(TestPrimaryLSService, zone_balance) ASSERT_EQ(ret, OB_SUCCESS); ret = primary_zone_infos.push_back(info); ASSERT_EQ(ret, OB_SUCCESS); - ret = ObTenantLSInfo::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone); + ret = ObCommonLSService::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone); LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone)); ASSERT_EQ(ret, OB_SUCCESS); ASSERT_EQ(z1, ls_primary_zone.at(0)); @@ -246,7 +246,7 @@ TEST_F(TestPrimaryLSService, zone_balance) ASSERT_EQ(4, count_group_by_zone.at(0)); ASSERT_EQ(0, count_group_by_zone.at(1)); ASSERT_EQ(4, count_group_by_zone.at(2)); - ret = ObTenantLSInfo::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone); + ret = ObCommonLSService::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone); LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone)); ASSERT_EQ(z3, ls_primary_zone.at(0)); ASSERT_EQ(z1, ls_primary_zone.at(1));