hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

add sys_recovery_scn to __all_tenant_info && primary tenant iter log

Browse Source
This commit is contained in:
maosy 2023-02-27 10:41:19 +00:00 committed by ob-robot
parent 41c067141f
commit 09f099c4e2
55 changed files with 4347 additions and 3733 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/logservice/data_dictionary/ob_data_dict_struct.cpp
View File

@ -389,7 +389,7 @@ int ObDictTenantMeta::incremental_data_update(const share::ObLSAttr &ls_attr)
if (OB_UNLIKELY(! ls_attr.is_valid())) {
ret = OB_INVALID_ARGUMENT;
DDLOG(WARN, "ls attr is invalid", KR(ret), K(ls_attr));
} else if (share::is_ls_create_end_op(ls_attr.get_ls_operatin_type())) {
} else if (share::is_ls_create_end_op(ls_attr.get_ls_operation_type())) {
if (OB_FAIL(ls_arr_.push_back(ls_attr.get_ls_id()))) {
DDLOG(WARN, "ls_arr_ push_back failed", KR(ret), K(ls_attr), K(ls_arr_));
}

14
src/logservice/libobcdc/src/ob_log_ls_op_processor.cpp
View File

@ -38,20 +38,20 @@ int ObLogLSOpProcessor::process_ls_op(
} else if (OB_ISNULL(tenant = guard.get_tenant())) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("get tenant fail, tenant is NULL", KR(ret), K(tenant_id));
} else if (share::is_ls_create_pre_op(ls_attr.get_ls_operatin_type())) {
} else if (share::is_ls_create_pre_op(ls_attr.get_ls_operation_type())) {
LOG_INFO("[LS_PROCESSOR] ls create pre operation", K(tenant_id), K(lsn), K(start_tstamp_ns), K(ls_attr));
} else if (share::is_ls_create_end_op(ls_attr.get_ls_operatin_type())) {
} else if (share::is_ls_create_end_op(ls_attr.get_ls_operation_type())) {
LOG_INFO("[LS_PROCESSOR] ls create end operation", K(tenant_id), K(lsn), K(start_tstamp_ns), K(ls_attr));
//create new ls;
if (OB_FAIL(create_new_ls_(tenant, start_tstamp_ns, ls_attr))) {
LOG_ERROR("failed to create new ls", KR(ret), K(tenant_id), K(lsn), K(start_tstamp_ns), K(ls_attr));
}
} else if (share::is_ls_create_abort_op(ls_attr.get_ls_operatin_type())) {
} else if (share::is_ls_drop_pre_op(ls_attr.get_ls_operatin_type())) {
} else if (share::is_ls_drop_end_op(ls_attr.get_ls_operatin_type())) {
} else if (share::is_ls_tenant_drop_pre_op(ls_attr.get_ls_operatin_type())) {
} else if (share::is_ls_create_abort_op(ls_attr.get_ls_operation_type())) {
} else if (share::is_ls_drop_pre_op(ls_attr.get_ls_operation_type())) {
} else if (share::is_ls_drop_end_op(ls_attr.get_ls_operation_type())) {
} else if (share::is_ls_tenant_drop_pre_op(ls_attr.get_ls_operation_type())) {
//only sys ls
} else if (share::is_ls_tenant_drop_op(ls_attr.get_ls_operatin_type())) {
} else if (share::is_ls_tenant_drop_op(ls_attr.get_ls_operation_type())) {
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected operation type", KR(ret), K(tenant_id), K(lsn), K(start_tstamp_ns), K(ls_attr));

4
src/logservice/ob_garbage_collector.cpp
View File

@ -29,7 +29,7 @@
#include "share/rc/ob_tenant_base.h"
#include "share/ls/ob_ls_life_manager.h"
#include "storage/tx_storage/ob_ls_handle.h"
#include "rootserver/ob_tenant_recovery_reportor.h" // ObTenantRecoveryReportor
#include "rootserver/ob_tenant_info_loader.h" // ObTenantInfoLoader
#include "share/ob_occam_time_guard.h"
namespace oceanbase
@ -651,7 +651,7 @@ void ObGCHandler::try_check_and_set_wait_gc_(ObGarbageCollector::LSStatus &ls_st
int ObGCHandler::get_tenant_readable_scn_(SCN &readable_scn)
{
return MTL(rootserver::ObTenantRecoveryReportor *)->get_tenant_readable_scn(readable_scn);
return MTL(rootserver::ObTenantInfoLoader *)->get_tenant_readable_scn(readable_scn);
}
// 由于日志流GC导致的归档日志不完整是无法被归档检查出来的异常, 因此需要保证GC与归档状态互斥;

8
src/logservice/ob_log_base_type.h
View File

@ -75,6 +75,10 @@ enum ObLogBaseType
// for arbitration service
ARBITRATION_SERVICE_LOG_BASE_TYPE = 21,
TENANT_INFO_REPORTOR_LOG_BASE_TYPE = 22,
COMMON_LS_SERVICE_LOG_BASE_TYPE = 23,
// pay attention!!!
// add log type in log_base_type_to_string
// max value
@ -131,6 +135,10 @@ int log_base_type_to_string(const ObLogBaseType log_type,
strncpy(str ,"DATA_DICTIONARY_SERVICE", str_len);
} else if (log_type == ARBITRATION_SERVICE_LOG_BASE_TYPE) {
strncpy(str ,"ARBITRATION_SERVICE", str_len);
} else if (log_type == TENANT_INFO_REPORTOR_LOG_BASE_TYPE) {
strncpy(str ,"TENANT_INFO_REPORTOR", str_len);
} else if (log_type == COMMON_LS_SERVICE_LOG_BASE_TYPE) {
strncpy(str ,"COMMON_LS_SERVICE", str_len);
} else {
ret = OB_INVALID_ARGUMENT;
}

8
src/observer/omt/ob_multi_tenant.cpp
View File

@ -85,10 +85,12 @@
#include "observer/omt/ob_tenant_config_mgr.h"
#include "observer/report/ob_tenant_meta_checker.h"
#include "storage/high_availability/ob_storage_ha_service.h"
#include "rootserver/ob_tenant_recovery_reportor.h"//ObTenantRecoveryReportor
#include "rootserver/ob_ls_recovery_reportor.h"//ObLSRecoveryReportor
#include "rootserver/ob_tenant_info_loader.h"//ObTenantInfoLoader
#include "rootserver/ob_primary_ls_service.h"//ObLSService
#include "rootserver/ob_recovery_ls_service.h"//ObRecoveryLSService
#include "rootserver/ob_common_ls_service.h"//ObCommonLSService
#include "rootserver/ob_tenant_info_report.h"//ObTenantInfoReportor
#include "rootserver/restore/ob_restore_scheduler.h" //ObRestoreService
#include "logservice/leader_coordinator/ob_leader_coordinator.h"
#include "storage/lob/ob_lob_manager.h"
@ -401,9 +403,11 @@ int ObMultiTenant::init(ObAddr myaddr,
MTL_BIND2(mtl_new_default, rootserver::ObPrimaryMajorFreezeService::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default);
MTL_BIND2(mtl_new_default, rootserver::ObRestoreMajorFreezeService::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default);
MTL_BIND2(mtl_new_default, ObTenantMetaChecker::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default);
MTL_BIND2(mtl_new_default, rootserver::ObTenantRecoveryReportor::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default);
MTL_BIND2(mtl_new_default, rootserver::ObLSRecoveryReportor::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default);
MTL_BIND2(mtl_new_default, rootserver::ObTenantInfoLoader::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default);
MTL_BIND2(mtl_new_default, rootserver::ObPrimaryLSService::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default);
MTL_BIND2(mtl_new_default, rootserver::ObCommonLSService::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default);
MTL_BIND2(mtl_new_default, rootserver::ObTenantInfoReportor::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default);
MTL_BIND2(mtl_new_default, rootserver::ObRecoveryLSService::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default);
MTL_BIND2(mtl_new_default, rootserver::ObRestoreService::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default);
MTL_BIND2(mtl_new_default, coordinator::ObLeaderCoordinator::mtl_init, coordinator::ObLeaderCoordinator::mtl_start, coordinator::ObLeaderCoordinator::mtl_stop, coordinator::ObLeaderCoordinator::mtl_wait, mtl_destroy_default);

6
src/rootserver/CMakeLists.txt
View File

@ -66,10 +66,14 @@ ob_set_subtarget(ob_rootserver common
ob_vtable_location_getter.cpp
ob_zone_manager.cpp
ob_zone_unit_provider.cpp
ob_tenant_recovery_reportor.cpp
ob_tenant_thread_helper.cpp
ob_ls_recovery_reportor.cpp
ob_tenant_info_loader.cpp
ob_tenant_info_report.cpp
ob_primary_ls_service.cpp
ob_recovery_ls_service.cpp
ob_common_ls_service.cpp
ob_ls_service_helper.cpp
ob_tenant_role_transition_service.cpp
ob_server_balancer.cpp
ob_system_admin_util.cpp

602
src/rootserver/ob_common_ls_service.cpp Executable file
View File

@ -0,0 +1,602 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX RS
#include "ob_common_ls_service.h"
#include "lib/profile/ob_trace_id.h"
#include "share/ob_errno.h"
#include "share/ob_max_id_fetcher.h"
#include "share/schema/ob_schema_struct.h"//ObTenantInfo
#include "share/ls/ob_ls_creator.h" //ObLSCreator
#include "share/ls/ob_ls_life_manager.h"//ObLSLifeAgentManager
#include "share/ob_primary_zone_util.h"//ObPrimaryZoneUtil
#include "share/ob_share_util.h"//ObShareUtil
#include "share/ob_tenant_info_proxy.h"//ObAllTenantInfo
#include "share/ob_common_rpc_proxy.h"//common_rpc_proxy
#include "observer/ob_server_struct.h"//GCTX
#include "rootserver/ob_tenant_info_loader.h"//get_tenant_info
#include "logservice/palf/palf_base_info.h"//PalfBaseInfo
namespace oceanbase
{
using namespace common;
using namespace share;
using namespace transaction;
using namespace palf;
namespace rootserver
{
//////////////ObCommonLSService
int ObCommonLSService::mtl_init(ObCommonLSService *&ka)
{
return ka->init();
}
int ObCommonLSService::init()
{
int ret = OB_SUCCESS;
tenant_id_ = MTL_ID();
if (OB_UNLIKELY(inited_)) {
ret = OB_INIT_TWICE;
LOG_WARN("has inited", KR(ret));
} else if (OB_FAIL(ObTenantThreadHelper::create("COMMONLSSe",
lib::TGDefIDs::SimpleLSService, *this))) {
LOG_WARN("failed to create thread", KR(ret));
} else if (OB_FAIL(ObTenantThreadHelper::start())) {
LOG_WARN("fail to start", KR(ret));
} else {
inited_ = true;
}
return ret;
}
void ObCommonLSService::destroy()
{
ObTenantThreadHelper::destroy();
tenant_id_ = OB_INVALID_TENANT_ID;
inited_ = false;
}
void ObCommonLSService::do_work()
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (OB_FAIL(wait_tenant_schema_and_version_ready_(tenant_id_, DATA_VERSION_4_1_0_0))) {
LOG_WARN("failed to wait tenant schema version ready", KR(ret), K(tenant_id_), K(DATA_CURRENT_VERSION));
} else {
int64_t idle_time_us = 100 * 1000L;
share::schema::ObTenantSchema user_tenant_schema;
while (!has_set_stop()) {
ret = OB_SUCCESS;
ObCurTraceId::init(GCONF.self_addr_);
if (is_meta_tenant(tenant_id_)) {
const uint64_t user_tenant_id = gen_user_tenant_id(tenant_id_);
if (OB_FAIL(get_tenant_schema(user_tenant_id, user_tenant_schema))) {
LOG_WARN("failed to get user tenant schema", KR(ret), K(user_tenant_id));
} else if (user_tenant_schema.is_dropping()) {
if (OB_FAIL(try_force_drop_tenant_(user_tenant_schema))) {
LOG_WARN("failed to force drop tenant", KR(ret), K(user_tenant_id));
}
} else if (OB_FAIL(try_create_ls_(user_tenant_schema))) {
LOG_WARN("failed to create ls", KR(ret), K(user_tenant_schema));
}
}
if (REACH_TIME_INTERVAL(1 * 1000 * 1000)) {
ret = OB_SUCCESS;//ignore error
if (OB_FAIL(try_update_sys_ls_primary_zone_())) {
LOG_WARN("failed to update sys ls primary zone", KR(ret));
}
if (is_meta_tenant(tenant_id_)) {
if (FAILEDx(try_adjust_user_ls_primary_zone_(user_tenant_schema))) {
LOG_WARN("failed to adjust user ls primary zone", KR(ret), K(user_tenant_schema));
}
}
}
user_tenant_schema.reset();
idle(idle_time_us);
LOG_INFO("[COMMON_LS_SERVICE] finish one round", KR(ret), K(idle_time_us));
} // end while
}
}
int ObCommonLSService::try_update_sys_ls_primary_zone_()
{
int ret = OB_SUCCESS;
share::schema::ObTenantSchema tenant_schema;
if (OB_ISNULL(GCTX.schema_service_) || OB_ISNULL(GCTX.sql_proxy_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected error", KR(ret), KP(GCTX.sql_proxy_), KP(GCTX.schema_service_));
} else if (OB_FAIL(get_tenant_schema(tenant_id_, tenant_schema))) {
LOG_WARN("failed to get tenant schema", KR(ret), K(tenant_id_));
} else {
share::ObLSPrimaryZoneInfo primary_zone_info;
ObArray<common::ObZone> primary_zone;
share::ObLSStatusOperator status_op;
ObZone new_primary_zone;
ObSqlString new_zone_priority;
if (OB_FAIL(ObPrimaryZoneUtil::get_tenant_primary_zone_array(
tenant_schema, primary_zone))) {
LOG_WARN("failed to get tenant primary zone array", KR(ret), K(tenant_schema));
} else if (OB_FAIL(status_op.get_ls_primary_zone_info(tenant_id_,
SYS_LS, primary_zone_info, *GCTX.sql_proxy_))) {
LOG_WARN("failed to get ls primary_zone info", KR(ret), K(tenant_id_));
} else if (OB_UNLIKELY(0 == primary_zone.count())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("primary zone is empty", KR(ret), K(tenant_schema));
} else if (has_exist_in_array(primary_zone, primary_zone_info.get_primary_zone())) {
if (OB_FAIL(new_primary_zone.assign(primary_zone_info.get_primary_zone()))) {
LOG_WARN("failed to assign primary zone", KR(ret), K(primary_zone_info));
}
} else if (OB_FAIL(new_primary_zone.assign(primary_zone.at(0)))) {
LOG_WARN("failed to assign primary zone", KR(ret), K(primary_zone));
}
if (OB_FAIL(ret)) {
} else if (is_sys_tenant(tenant_id_)) {
//sys tenant use tenant normalize primary zone
if (OB_FAIL(ObPrimaryZoneUtil::get_tenant_zone_priority(
tenant_schema, new_zone_priority))) {
LOG_WARN("failed to get tenant primary zone array", KR(ret), K(tenant_schema));
}
} else if (OB_FAIL(get_zone_priority(new_primary_zone,
tenant_schema, new_zone_priority))) {
LOG_WARN("failed to get normalize primary zone", KR(ret), K(new_primary_zone));
}
if (FAILEDx(try_update_ls_primary_zone_(
primary_zone_info, new_primary_zone, new_zone_priority))) {
LOG_WARN("failed to update ls primary zone", KR(ret), K(primary_zone_info),
K(new_primary_zone), K(new_zone_priority));
} else if (is_meta_tenant(tenant_id_)) {
//user sys ls has same primary zone with meta sys ls
share::ObLSPrimaryZoneInfo user_primary_zone_info;
const uint64_t user_tenant_id = gen_user_tenant_id(tenant_id_);
if (OB_FAIL(status_op.get_ls_primary_zone_info(user_tenant_id, SYS_LS,
user_primary_zone_info, *GCTX.sql_proxy_))) {
LOG_WARN("failed to get ls primary_zone info", KR(ret), K(tenant_id_), K(user_tenant_id));
} else if (OB_FAIL(try_update_ls_primary_zone_(user_primary_zone_info, new_primary_zone, new_zone_priority))) {
LOG_WARN("failed to update ls primary zone", KR(ret), K(user_primary_zone_info),
K(new_primary_zone), K(new_zone_priority));
}
}
}
return ret;
}
int ObCommonLSService::try_adjust_user_ls_primary_zone_(const share::schema::ObTenantSchema &tenant_schema)
{
int ret = OB_SUCCESS;
share::ObLSStatusOperator status_op;
share::ObLSPrimaryZoneInfoArray info_array;
ObArray<common::ObZone> primary_zone;
const uint64_t tenant_id = tenant_schema.get_tenant_id();
if (OB_ISNULL(GCTX.schema_service_) || OB_ISNULL(GCTX.sql_proxy_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected error", KR(ret), KP(GCTX.sql_proxy_), KP(GCTX.schema_service_));
} else if (!is_user_tenant(tenant_id) || !tenant_schema.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("tenant iS invalid", KR(ret), K(tenant_id), K(tenant_schema));
} else if (OB_FAIL(ObPrimaryZoneUtil::get_tenant_primary_zone_array(
tenant_schema, primary_zone))) {
LOG_WARN("failed to get tenant primary zone array", KR(ret), K(tenant_schema));
} else {
if (OB_FAIL(status_op.get_tenant_primary_zone_info_array(
tenant_id, info_array, *GCTX.sql_proxy_))) {
LOG_WARN("failed to get tenant primary zone info array", KR(ret), K(tenant_id));
}
}
if (OB_SUCC(ret)) {
uint64_t last_ls_group_id = OB_INVALID_ID;
share::ObLSPrimaryZoneInfoArray tmp_info_array;
for (int64_t i = 0; OB_SUCC(ret) && i < info_array.count(); ++i) {
//TODO ls group id maybe zero
const ObLSPrimaryZoneInfo &info = info_array.at(i);
if (!info.get_ls_id().is_sys_ls()) {
if (OB_INVALID_ID == last_ls_group_id) {
last_ls_group_id = info.get_ls_group_id();
}
if (last_ls_group_id != info.get_ls_group_id()) {
//process the ls group
if (OB_FAIL(adjust_primary_zone_by_ls_group_(primary_zone, tmp_info_array, tenant_schema))) {
LOG_WARN("failed to update primary zone of each ls group", KR(ret),
K(tmp_info_array), K(primary_zone), K(tenant_schema));
} else {
tmp_info_array.reset();
last_ls_group_id = info.get_ls_group_id();
}
}
if (FAILEDx(tmp_info_array.push_back(info))) {
LOG_WARN("failed to push back primary info array", KR(ret), K(i));
}
}
}
if (OB_SUCC(ret) && 0 < tmp_info_array.count()) {
if (OB_FAIL(adjust_primary_zone_by_ls_group_(primary_zone, tmp_info_array, tenant_schema))) {
LOG_WARN("failed to update primary zone of each ls group", KR(ret),
K(tmp_info_array), K(primary_zone), K(tenant_schema));
}
}
}
return ret;
}
//check every ls has right primary zone in ls group
//if primary_zone of ls not in primary zone, try to choose a right zone,
//if can not find a zone to modify, modify the ls to exist zone,
//load_balancer will choose the right ls to drop
//eg:
//if ls group has z1, z2 and primary zone is : z1. need modify z2 to z1
//if ls group has z1, z2 and primary zone is : z1, z3. modify z2 to z3;
int ObCommonLSService::adjust_primary_zone_by_ls_group_(
const common::ObIArray<common::ObZone> &primary_zone_array,
const share::ObLSPrimaryZoneInfoArray &primary_zone_infos,
const share::schema::ObTenantSchema &tenant_schema)
{
int ret = OB_SUCCESS;
const uint64_t tenant_id = tenant_schema.get_tenant_id();
const int64_t ls_count = primary_zone_infos.count();
const int64_t primary_zone_count = primary_zone_array.count();
if (OB_UNLIKELY(0 == primary_zone_count
|| 0 == ls_count || !tenant_schema.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KR(ret), K(primary_zone_infos),
K(primary_zone_array), K(tenant_schema));
} else if (OB_ISNULL(GCTX.sql_proxy_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("sql proxy is null", KR(ret));
} else if (1 == ls_count
&& primary_zone_infos.at(0).get_ls_id().is_sys_ls()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("can not be sys ls", KR(ret), K(ls_count), K(primary_zone_infos));
} else {
//Algorithm Description:
//Assumption: We have 5 ls, 3 primary_zones(z1, z2, z3).
//1. Set the primary zone of ls to tenant's primary zone,
//choose the least number of log streams on the zone is selected for all zones
//2. After all the primary_zone of the ls are in the primary_zonw of the tenant,
//choose the primary_zone with the most and least ls. Adjust a certain number of ls to the smallest zone without exceeding the balance
//while guaranteeing that the number of the zone with the most is no less than the average.
ObArray<ObZone> ls_primary_zone;//is match with primary_zone_infos
ObSEArray<uint64_t, 3> count_group_by_zone;//ls count of each primary zone
ObSqlString new_zone_priority;
if (OB_FAIL(set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone,
count_group_by_zone))) {
LOG_WARN("failed to set ls to primary zone", KR(ret), K(primary_zone_array), K(primary_zone_infos));
} else if (OB_FAIL(balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone))) {
LOG_WARN("failed to balance ls primary zone", KR(ret), K(ls_primary_zone), K(count_group_by_zone));
}
for (int64_t i = 0; OB_SUCC(ret) && i < ls_count; ++i) {
const ObZone &new_primary_zone = ls_primary_zone.at(i);
if (OB_FAIL(get_zone_priority(new_primary_zone, tenant_schema, new_zone_priority))) {
LOG_WARN("failed to get normalize primary zone", KR(ret), K(new_primary_zone));
}
if (FAILEDx(try_update_ls_primary_zone_(primary_zone_infos.at(i), new_primary_zone, new_zone_priority))) {
LOG_WARN("failed to update ls primary zone", KR(ret), "primary_zone_info", primary_zone_infos.at(i),
K(new_primary_zone), K(new_zone_priority));
}
}
}
return ret;
}
int ObCommonLSService::set_ls_to_primary_zone(
const common::ObIArray<ObZone> &primary_zone_array,
const share::ObLSPrimaryZoneInfoArray &primary_zone_infos,
common::ObIArray<common::ObZone> &ls_primary_zone,
common::ObIArray<uint64_t> &count_group_by_zone)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(0 == primary_zone_array.count()
|| 0 == primary_zone_infos.count())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KR(ret), K(primary_zone_array), K(primary_zone_infos));
} else {
const int64_t primary_zone_count = primary_zone_array.count();
const int64_t ls_count = primary_zone_infos.count();
//ls may not in primary zone, record the index of primary_zone_infos not in primary zone
ObSEArray<int64_t, 3> index_not_primary_zone;
int64_t index = 0;
ARRAY_FOREACH_X(primary_zone_array, idx, cnt, OB_SUCC(ret)) {
if (OB_FAIL(count_group_by_zone.push_back(0))) {
LOG_WARN("failed to push back", KR(ret), K(idx));
}
}
for (int64_t i = 0; OB_SUCC(ret) && i < ls_count; ++i) {
const ObZone &current_zone = primary_zone_infos.at(i).get_primary_zone();
if (has_exist_in_array(primary_zone_array, current_zone, &index)) {
count_group_by_zone.at(index)++;
} else if (OB_FAIL(index_not_primary_zone.push_back(i))) {
LOG_WARN("failed to push back", KR(ret), K(i), K(current_zone));
}
if (FAILEDx(ls_primary_zone.push_back(current_zone))) {
LOG_WARN("failed to push back current zone", KR(ret), K(i), K(current_zone));
}
}
//1. take all ls primary zone to tenant primary zone, choose the less primary zone count
int64_t min_count = INT64_MAX;
int64_t min_index = 0;
for (int64_t i = 0; OB_SUCC(ret) && i < index_not_primary_zone.count(); ++i) {
const int64_t ls_index = index_not_primary_zone.at(i);
min_count = INT64_MAX;
ARRAY_FOREACH_X(primary_zone_array, idx, cnt, OB_SUCC(ret)) {
if (min_count > count_group_by_zone.at(idx)) {
min_count = count_group_by_zone.at(idx);
min_index = idx;
}
}//end for search min count
if (OB_FAIL(ret)) {
} else if (min_index >= primary_zone_count) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to found min count", KR(ret), K(min_index), K(primary_zone_array));
} else if (OB_FAIL(ls_primary_zone.at(ls_index).assign(primary_zone_array.at(min_index)))) {
LOG_WARN("failed to assign primary zone", KR(ret), K(min_index), K(min_count), K(ls_index));
} else {
count_group_by_zone.at(min_index)++;
}
}
}
return ret;
}
int ObCommonLSService::balance_ls_primary_zone(
const common::ObIArray<common::ObZone> &primary_zone_array,
common::ObIArray<common::ObZone> &ls_primary_zone,
common::ObIArray<uint64_t> &count_group_by_zone)
{
int ret = OB_SUCCESS;
const int64_t ls_count = ls_primary_zone.count();
const int64_t primary_zone_count = count_group_by_zone.count();
if (OB_UNLIKELY(0 == primary_zone_count
|| 0 == ls_count
|| primary_zone_count != primary_zone_array.count())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KR(ret), K(count_group_by_zone), K(ls_primary_zone));
} else {
int64_t max_count = -1, max_index = 0;
int64_t min_count = INT64_MAX, min_index = 0;
do {
max_count = -1, max_index = 0;
min_count = INT64_MAX, min_index = 0;
for (int64_t i = 0; OB_SUCC(ret) && i < primary_zone_count; ++i) {
const int64_t ls_count = count_group_by_zone.at(i);
if (min_count > ls_count) {
min_count = ls_count;
min_index = i;
}
if (max_count < ls_count) {
max_count = ls_count;
max_index = i;
}
}//end for find min and max count
if (OB_UNLIKELY(max_index >= primary_zone_count || min_index >= primary_zone_count
|| -1 == max_count || INT64_MAX == min_count)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to get primary zone", KR(ret), K(max_index), K(min_index),
K(min_count), K(max_count),
K(primary_zone_array), K(primary_zone_count), K(ls_primary_zone));
} else if (max_count - min_count > 1) {
for (int64_t i = 0; OB_SUCC(ret) && i < ls_count; ++i) {
if (ls_primary_zone.at(i) == primary_zone_array.at(max_index)) {
if (OB_FAIL(ls_primary_zone.at(i).assign(primary_zone_array.at(min_index)))) {
LOG_WARN("failed to push back ls primary zone", KR(ret), K(min_index));
} else {
count_group_by_zone.at(max_index)--;
count_group_by_zone.at(min_index)++;
}
break;//only change one by one
}
}
}
} while (max_count - min_count > 1);
}
return ret;
}
int ObCommonLSService::try_update_ls_primary_zone_(
const share::ObLSPrimaryZoneInfo &primary_zone_info,
const common::ObZone &new_primary_zone,
const common::ObSqlString &zone_priority)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!primary_zone_info.is_valid()
|| new_primary_zone.is_empty() || zone_priority.empty())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("primary zone info is invalid", KR(ret), K(primary_zone_info),
K(new_primary_zone), K(zone_priority));
} else if (OB_ISNULL(GCTX.sql_proxy_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("sql proxy is null", KR(ret));
} else if (new_primary_zone != primary_zone_info.get_primary_zone()
|| zone_priority.string() != primary_zone_info.get_zone_priority_str()) {
ObLSLifeAgentManager ls_life_agent(*GCTX.sql_proxy_);
if (OB_FAIL(ls_life_agent.update_ls_primary_zone(primary_zone_info.get_tenant_id(),
primary_zone_info.get_ls_id(),
new_primary_zone, zone_priority.string()))) {
LOG_WARN("failed to update ls primary zone", KR(ret), K(primary_zone_info),
K(new_primary_zone), K(zone_priority));
}
LOG_INFO("[COMMON_LS_SERVICE] update ls primary zone", KR(ret), K(new_primary_zone),
K(zone_priority), K(primary_zone_info));
} else {
//no need update
}
return ret;
}
int ObCommonLSService::try_create_ls_(const share::schema::ObTenantSchema &tenant_schema)
{
int ret = OB_SUCCESS;
const uint64_t tenant_id = tenant_schema.get_tenant_id();
if (OB_ISNULL(GCTX.sql_proxy_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("sql proxy is null", KR(ret), KP(GCTX.sql_proxy_));
} else if (!is_user_tenant(tenant_id) || !tenant_schema.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("tenant is invalid", KR(ret), K(tenant_id), K(tenant_schema));
} else {
share::ObLSStatusInfoArray status_info_array;
share::ObLSStatusOperator ls_op;
ObLSRecoveryStat recovery_stat;
ObLSRecoveryStatOperator ls_recovery_operator;
palf::PalfBaseInfo palf_base_info;
int tmp_ret = OB_SUCCESS;
if (OB_FAIL(ls_op.get_all_ls_status_by_order(
tenant_id, status_info_array, *GCTX.sql_proxy_))) {
LOG_WARN("failed to update ls status", KR(ret), K(tenant_id));
}
for (int64_t i = 0; OB_SUCC(ret) && i < status_info_array.count(); ++i) {
const ObLSStatusInfo &status_info = status_info_array.at(i);
if (status_info.ls_is_creating()) {
recovery_stat.reset();
if (OB_FAIL(check_can_create_ls_(tenant_schema))) {
LOG_WARN("failed to wait can do recovery", KR(ret), K(tenant_schema));
} else if (OB_FAIL(ls_recovery_operator.get_ls_recovery_stat(
tenant_id, status_info.ls_id_, false /*for_update*/,
recovery_stat, *GCTX.sql_proxy_))) {
LOG_WARN("failed to get ls recovery stat", KR(ret), K(tenant_id),
K(status_info));
} else if (OB_FAIL(do_create_user_ls(tenant_schema, status_info,
recovery_stat.get_create_scn(),
false, palf_base_info))) {
LOG_WARN("failed to create new ls", KR(ret), K(status_info),
K(recovery_stat));
}
}
} // end for
}
return ret;
}
int ObCommonLSService::check_can_create_ls_(const share::schema::ObTenantSchema &tenant_schema)
{
int ret = OB_SUCCESS;
const uint64_t tenant_id = tenant_schema.get_tenant_id();
if (OB_ISNULL(GCTX.sql_proxy_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("sql proxy is null", KR(ret), KP(GCTX.sql_proxy_));
} else if (!is_user_tenant(tenant_id) || !tenant_schema.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("tenant is invalid", KR(ret), K(tenant_id), K(tenant_schema));
} else if (tenant_schema.is_restore()) {
//meta and user tenant unit must in the same observer
MTL_SWITCH(tenant_id) {
ObAllTenantInfo tenant_info;
ObTenantInfoLoader* tenant_loader = MTL(rootserver::ObTenantInfoLoader*);
if (OB_ISNULL(tenant_loader)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("tenant report is null", KR(ret), K(tenant_id_));
} else if (OB_FAIL(tenant_loader->get_tenant_info(tenant_info))) {
LOG_WARN("failed to get tenant info", KR(ret), K(tenant_id));
} else if (OB_FAIL(check_can_do_recovery(tenant_info))) {
LOG_WARN("failed to wait can do recovery", KR(ret), K(tenant_info), K(tenant_id));
}
}
}
return ret;
}
int ObCommonLSService::do_create_user_ls(
const share::schema::ObTenantSchema &tenant_schema,
const share::ObLSStatusInfo &info, const SCN &create_scn,
bool create_with_palf, const palf::PalfBaseInfo &palf_base_info)
{
int ret = OB_SUCCESS;
LOG_INFO("[COMMON_LS_SERVICE] start to create ls", K(info), K(create_scn));
const int64_t start_time = ObTimeUtility::fast_current_time();
if (OB_UNLIKELY(!info.is_valid() || !info.ls_is_creating())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("info not valid", KR(ret), K(info));
} else {
CK(OB_NOT_NULL(GCTX.sql_proxy_), OB_NOT_NULL(GCTX.srv_rpc_proxy_))
common::ObArray<share::ObZoneReplicaAttrSet> locality_array;
int64_t paxos_replica_num = 0;
ObSchemaGetterGuard guard;//nothing
if (FAILEDx(tenant_schema.get_zone_replica_attr_array(
locality_array))) {
LOG_WARN("failed to get zone locality array", KR(ret));
} else if (OB_FAIL(tenant_schema.get_paxos_replica_num(
guard, paxos_replica_num))) {
LOG_WARN("failed to get paxos replica num", KR(ret));
} else {
ObLSCreator creator(*GCTX.srv_rpc_proxy_, info.tenant_id_,
info.ls_id_, GCTX.sql_proxy_);
if (OB_FAIL(creator.create_user_ls(info, paxos_replica_num,
locality_array, create_scn,
tenant_schema.get_compatibility_mode(),
create_with_palf,
palf_base_info))) {
LOG_WARN("failed to create user ls", KR(ret), K(info), K(locality_array), K(create_scn),
K(palf_base_info), K(create_with_palf));
}
}
}
const int64_t cost = ObTimeUtility::fast_current_time() - start_time;
LOG_INFO("[COMMON_LS_SERVICE] end to create ls", KR(ret), K(info), K(cost));
return ret;
}
int ObCommonLSService::try_force_drop_tenant_(
const share::schema::ObTenantSchema &tenant_schema)
{
int ret = OB_SUCCESS;
ObTimeoutCtx ctx;
const uint64_t tenant_id = tenant_schema.get_tenant_id();
if (OB_UNLIKELY(!inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", KR(ret));
} else if (OB_ISNULL(GCTX.sql_proxy_)
|| OB_ISNULL(GCTX.rs_rpc_proxy_)
|| OB_ISNULL(GCTX.schema_service_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ptr is null", KR(ret), KP(GCTX.sql_proxy_), KP(GCTX.rs_rpc_proxy_),
KP(GCTX.schema_service_));
} else if (!is_user_tenant(tenant_id) || !tenant_schema.is_valid()
|| !tenant_schema.is_dropping()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("tenant is invalid", KR(ret), K(tenant_id), K(tenant_schema));
} else {
LOG_INFO("try drop tenant", K(tenant_id));
ObLSStatusOperator op;
share::ObLSStatusInfoArray ls_array;
if (OB_FAIL(ObShareUtil::set_default_timeout_ctx(ctx, GCONF.rpc_timeout))) {
LOG_WARN("fail to set timeout ctx", KR(ret), K(tenant_id));
} else if (OB_FAIL(op.get_all_ls_status_by_order(tenant_id, ls_array, *GCTX.sql_proxy_))) {
LOG_WARN("fail to get all ls status", KR(ret), K(tenant_id));
} else if (ls_array.count() <= 0) {
obrpc::ObDropTenantArg arg;
arg.exec_tenant_id_ = OB_SYS_TENANT_ID;
arg.tenant_name_ = tenant_schema.get_tenant_name();
arg.tenant_id_ = tenant_schema.get_tenant_id();
arg.if_exist_ = true;
arg.delay_to_drop_ = false;
ObSqlString sql;
const int64_t timeout_ts = ctx.get_timeout();
if (OB_FAIL(sql.append_fmt("DROP TENANT IF EXISTS %s FORCE", arg.tenant_name_.ptr()))) {
LOG_WARN("fail to generate sql", KR(ret), K(arg));
} else if (FALSE_IT(arg.ddl_stmt_str_ = sql.string())) {
} else if (OB_FAIL(GCTX.rs_rpc_proxy_->timeout(timeout_ts).drop_tenant(arg))) {
LOG_WARN("fail to drop tenant", KR(ret), K(arg), K(timeout_ts));
}
} else {
// tenant's logstream is still dropping, check next round
}
}
return ret;
}
}//end of rootserver
}

126
src/rootserver/ob_common_ls_service.h Normal file
View File

@ -0,0 +1,126 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_ROOTSERVER_OB_COMMON_LS_SERVICE_H
#define OCEANBASE_ROOTSERVER_OB_COMMON_LS_SERVICE_H
#include "lib/thread/ob_reentrant_thread.h"//ObRsReentrantThread
#include "logservice/ob_log_base_type.h"
#include "share/scn.h"//SCN
#include "share/ls/ob_ls_status_operator.h"//ObLSStatusOperator
#include "share/ls/ob_ls_operator.h" //ObLSAttr
#include "share/ob_thread_mgr.h" //OBTGDefIDEnum
#include "logservice/palf/palf_iterator.h" //PalfBufferIterator
#include "share/unit/ob_unit_info.h"//ObUnit::Status
#include "lib/thread/thread_mgr_interface.h" // TGRunnable
#include "lib/lock/ob_thread_cond.h"//ObThreadCond
#include "rootserver/ob_tenant_thread_helper.h"//ObTenantThreadHelper
namespace oceanbase
{
namespace obrpc
{
class ObSrvRpcProxy;
}
namespace common
{
class ObMySQLProxy;
class ObMySQLTransaction;
}
namespace share
{
namespace schema
{
class ObMultiVersionSchemaService;
class ObTenantSchema;
}
}
namespace palf
{
struct PalfBaseInfo;
}
namespace rootserver
{
/*description:
*COMMON_LS_SERVICE thread: Started on the leader of the meta tenant sys ls
* 1. adjust sys ls primary zone of meta and user tenant
* 2. adjust user ls primary zone
* 3. make ls status from creating to created of __all_ls_status
* 4. check dropping tenant need drop tenant force;
* */
class ObCommonLSService : public ObTenantThreadHelper,
public logservice::ObICheckpointSubHandler,
public logservice::ObIReplaySubHandler
{
public:
ObCommonLSService():inited_(false), tenant_id_(OB_INVALID_TENANT_ID) {}
virtual ~ObCommonLSService() {}
static int mtl_init(ObCommonLSService *&ka);
int init();
void destroy();
virtual void do_work() override;
public:
virtual share::SCN get_rec_scn() override { return share::SCN::max_scn();}
virtual int flush(share::SCN &scn) override { return OB_SUCCESS; }
int replay(const void *buffer, const int64_t nbytes, const palf::LSN &lsn, const share::SCN &scn) override
{
UNUSED(buffer);
UNUSED(nbytes);
UNUSED(lsn);
UNUSED(scn);
return OB_SUCCESS;
}
private:
int try_update_sys_ls_primary_zone_();
int try_adjust_user_ls_primary_zone_(const share::schema::ObTenantSchema &tenant_schema);
int try_update_ls_primary_zone_(
const share::ObLSPrimaryZoneInfo &primary_zone_info,
const common::ObZone &new_primary_zone,
const common::ObSqlString &zone_priority);
static int set_ls_to_primary_zone(const common::ObIArray<common::ObZone> &primary_zone_array,
const share::ObLSPrimaryZoneInfoArray &primary_zone_infos,
common::ObIArray<common::ObZone> &ls_primary_zone,
common::ObIArray<uint64_t> &count_group_by_zone);
static int balance_ls_primary_zone(const common::ObIArray<common::ObZone> &primary_zone_array,
common::ObIArray<common::ObZone> &ls_primary_zone,
common::ObIArray<uint64_t> &count_group_by_zone);
int adjust_primary_zone_by_ls_group_(const common::ObIArray<common::ObZone> &primary_zone_array,
const share::ObLSPrimaryZoneInfoArray &primary_zone_infos,
const share::schema::ObTenantSchema &tenant_schema);
// force drop user tenant if tenant is in dropping status
int try_force_drop_tenant_(
const share::schema::ObTenantSchema &tenant_schema);
int try_create_ls_(const share::schema::ObTenantSchema &tenant_schema);
//restore tenant can not create ls before "CREATE_INIT_LS" status
int check_can_create_ls_(const share::schema::ObTenantSchema &tenant_schema);
public:
//restore_service need create init ls too
static int do_create_user_ls(const share::schema::ObTenantSchema &tenant_schema,
const share::ObLSStatusInfo &info,
const SCN &create_scn,
bool create_with_palf,
const palf::PalfBaseInfo &palf_base_info);
private:
bool inited_;
uint64_t tenant_id_;
};
}
}
#endif /* !OCEANBASE_ROOTSERVER_OB_COMMON_LS_SERVICE_H */

6
src/rootserver/ob_ddl_service.cpp
View File

@ -72,7 +72,7 @@
#include "rootserver/ddl_task/ob_ddl_retry_task.h"
#include "rootserver/freeze/ob_freeze_info_manager.h"
#include "rootserver/freeze/ob_major_freeze_helper.h"
#include "rootserver/ob_primary_ls_service.h"//ObTenantLSInfo
#include "rootserver/ob_tenant_thread_helper.h"//get_zone_priority
#include "lib/utility/ob_tracepoint.h"
#include "observer/ob_server_struct.h"
#include "storage/tx/ob_ts_mgr.h"
@ -20702,7 +20702,7 @@ int ObDDLService::create_tenant_sys_ls(
} else if (OB_UNLIKELY(0 == primary_zone_list.count())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("primary zone is empty", KR(ret), K(tenant_schema));
} else if (OB_FAIL(ObTenantLSInfo::get_zone_priority(
} else if (OB_FAIL(ObTenantThreadHelper::get_zone_priority(
primary_zone_list.at(0), tenant_schema, zone_priority))) {
LOG_WARN("failed to get zone priority", KR(ret), K(primary_zone_list), K(tenant_schema));
} else if (OB_FAIL(ls_creator.create_tenant_sys_ls(
@ -21169,7 +21169,7 @@ int ObDDLService::set_sys_ls_status(const uint64_t tenant_id)
if (OB_FAIL(new_ls.init(SYS_LS, ls_group_id, flag,
share::OB_LS_NORMAL, share::OB_LS_OP_CREATE_END, create_scn))) {
LOG_WARN("failed to init new operation", KR(ret), K(flag), K(create_scn));
} else if (OB_FAIL(ls_operator.insert_ls(new_ls, ls_group_id, share::NORMAL_SWITCHOVER_STATUS))) {
} else if (OB_FAIL(ls_operator.insert_ls(new_ls, ls_group_id))) {
LOG_WARN("failed to insert new ls", KR(ret), K(new_ls), K(ls_group_id));
}
}

89
src/rootserver/ob_tenant_recovery_reportor.cpp → src/rootserver/ob_ls_recovery_reportor.cpp
View File

@ -12,7 +12,7 @@
#define USING_LOG_PREFIX RS
#include "rootserver/ob_tenant_recovery_reportor.h"
#include "rootserver/ob_ls_recovery_reportor.h"
#include "rootserver/ob_tenant_info_loader.h"
#include "rootserver/ob_tenant_role_transition_service.h"//ObTenantRoleTransitionConstants
#include "storage/tx_storage/ob_ls_service.h" //ObLSService
@ -36,11 +36,11 @@ using namespace storage;
using namespace palf;
namespace rootserver
{
int ObTenantRecoveryReportor::mtl_init(ObTenantRecoveryReportor *&ka)
int ObLSRecoveryReportor::mtl_init(ObLSRecoveryReportor *&ka)
{
return ka->init();
}
int ObTenantRecoveryReportor::init()
int ObLSRecoveryReportor::init()
{
int ret = OB_SUCCESS;
lib::ThreadPool::set_run_wrapper(MTL_CTX());
@ -66,7 +66,7 @@ int ObTenantRecoveryReportor::init()
}
return ret;
}
void ObTenantRecoveryReportor::destroy()
void ObLSRecoveryReportor::destroy()
{
LOG_INFO("tenant recovery service destory", KPC(this));
stop();
@ -76,7 +76,7 @@ void ObTenantRecoveryReportor::destroy()
sql_proxy_ = NULL;
}
int ObTenantRecoveryReportor::start()
int ObLSRecoveryReportor::start()
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
@ -94,16 +94,16 @@ int ObTenantRecoveryReportor::start()
return ret;
}
void ObTenantRecoveryReportor::stop()
void ObLSRecoveryReportor::stop()
{
logical_stop();
}
void ObTenantRecoveryReportor::wait()
void ObLSRecoveryReportor::wait()
{
logical_wait();
}
void ObTenantRecoveryReportor::wakeup()
void ObLSRecoveryReportor::wakeup()
{
if (OB_NOT_INIT) {
} else {
@ -112,7 +112,7 @@ void ObTenantRecoveryReportor::wakeup()
}
}
void ObTenantRecoveryReportor::run2()
void ObLSRecoveryReportor::run2()
{
int ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS;
@ -156,7 +156,7 @@ void ObTenantRecoveryReportor::run2()
}
}
int ObTenantRecoveryReportor::submit_tenant_refresh_schema_task_()
int ObLSRecoveryReportor::submit_tenant_refresh_schema_task_()
{
int ret = OB_SUCCESS;
ObAllTenantInfo tenant_info;
@ -200,7 +200,7 @@ int ObTenantRecoveryReportor::submit_tenant_refresh_schema_task_()
}
return ret;
}
int ObTenantRecoveryReportor::update_ls_recovery_stat_()
int ObLSRecoveryReportor::update_ls_recovery_stat_()
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
@ -235,10 +235,7 @@ int ObTenantRecoveryReportor::update_ls_recovery_stat_()
LOG_WARN("failed to update_ls_replayable_point", KR(tmp_ret), KPC(ls), K(tenant_info));
}
if (ls->is_sys_ls()) {
// nothing todo
// sys ls of user tenant is in ls_recovery
} else if (OB_FAIL(update_ls_recovery(ls, sql_proxy_))) {
if (OB_FAIL(update_ls_recovery(ls, sql_proxy_))) {
LOG_WARN("failed to update ls recovery", KR(ret), KPC(ls));
}
}
@ -256,7 +253,7 @@ int ObTenantRecoveryReportor::update_ls_recovery_stat_()
return ret;
}
int ObTenantRecoveryReportor::update_ls_recovery(ObLS *ls, common::ObMySQLProxy *sql_proxy)
int ObLSRecoveryReportor::update_ls_recovery(ObLS *ls, common::ObMySQLProxy *sql_proxy)
{
int ret = OB_SUCCESS;
logservice::ObLogService *ls_svr = MTL(logservice::ObLogService*);
@ -272,8 +269,9 @@ int ObTenantRecoveryReportor::update_ls_recovery(ObLS *ls, common::ObMySQLProxy
ObLSRecoveryStat ls_recovery_stat;
const ObLSID ls_id = ls->get_ls_id();
const uint64_t tenant_id = MTL_ID();
ObMySQLTransaction trans;
ObLSRecoveryStatOperator ls_recovery;
const uint64_t exec_tenant_id = ls_recovery.get_exec_tenant_id(tenant_id);
if (OB_FAIL(ls_svr->get_palf_role(ls_id, role, first_proposal_id))) {
LOG_WARN("failed to get first role", KR(ret), K(ls_id), KPC(ls));
} else if (!is_strong_leader(role)) {
@ -294,10 +292,31 @@ int ObTenantRecoveryReportor::update_ls_recovery(ObLS *ls, common::ObMySQLProxy
ret = OB_EAGAIN;
LOG_INFO("role change, try again", KR(ret), K(role),
K(first_proposal_id), K(second_proposal_id), KPC(ls));
} else if (OB_FAIL(ls_recovery.update_ls_recovery_stat(ls_recovery_stat,
*sql_proxy))) {
} else if (OB_FAIL(trans.start(sql_proxy, exec_tenant_id))) {
LOG_WARN("failed to start trans", KR(ret), K(exec_tenant_id), K(ls_recovery_stat));
} else if (OB_FAIL(ls_recovery.update_ls_recovery_stat_in_trans(ls_recovery_stat,
trans))) {
LOG_WARN("failed to update ls recovery stat", KR(ret),
K(ls_recovery_stat));
} else {
//double check sync can not fallback
palf::PalfHandleGuard palf_handle_guard;
SCN new_scn;
if (OB_FAIL(ls_svr->open_palf(ls_id, palf_handle_guard))) {
LOG_WARN("failed to open palf", KR(ret), K(ls_id));
} else if (OB_FAIL(palf_handle_guard.get_end_scn(new_scn))) {
LOG_WARN("failed to get end ts", KR(ret), K(ls_id));
} else if (new_scn < sync_scn) {
ret = OB_OP_NOT_ALLOW;
LOG_WARN("maybe flashback, can not report sync scn", KR(ret), K(ls_recovery_stat), K(new_scn));
}
}
if (trans.is_started()) {
int tmp_ret = OB_SUCCESS;
if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) {
LOG_WARN("failed to commit trans", KR(ret), KR(tmp_ret));
ret = OB_SUCC(ret) ? tmp_ret : ret;
}
}
}
const int64_t PRINT_INTERVAL = 10 * 1000 * 1000L;
@ -315,27 +334,7 @@ int ObTenantRecoveryReportor::update_ls_recovery(ObLS *ls, common::ObMySQLProxy
}
int ObTenantRecoveryReportor::get_tenant_readable_scn(SCN &readable_scn)
{
int ret = OB_SUCCESS;
share::ObAllTenantInfo tenant_info;
rootserver::ObTenantInfoLoader *tenant_info_loader = MTL(rootserver::ObTenantInfoLoader*);
if (OB_ISNULL(tenant_info_loader)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("mtl pointer is null", KR(ret), KP(tenant_info_loader));
} else if (OB_FAIL(tenant_info_loader->get_tenant_info(tenant_info))) {
LOG_WARN("get_tenant_info failed", K(ret));
} else if (OB_UNLIKELY(! tenant_info.is_valid())) {
ret = OB_EAGAIN;
LOG_WARN("tenant info not valid", K(ret), K(tenant_info));
} else {
readable_scn = tenant_info.get_standby_scn();
}
return ret;
}
int ObTenantRecoveryReportor::update_replayable_point_()
int ObLSRecoveryReportor::update_replayable_point_()
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
@ -352,7 +351,7 @@ int ObTenantRecoveryReportor::update_replayable_point_()
return ret;
}
int ObTenantRecoveryReportor::update_replayable_point_from_tenant_info_()
int ObLSRecoveryReportor::update_replayable_point_from_tenant_info_()
{
int ret = OB_SUCCESS;
logservice::ObLogService *log_service = MTL(logservice::ObLogService*);
@ -373,7 +372,7 @@ int ObTenantRecoveryReportor::update_replayable_point_from_tenant_info_()
return ret;
}
int ObTenantRecoveryReportor::update_replayable_point_from_meta_()
int ObLSRecoveryReportor::update_replayable_point_from_meta_()
{
int ret = OB_SUCCESS;
SCN replayable_point;
@ -413,12 +412,10 @@ int ObTenantRecoveryReportor::update_replayable_point_from_meta_()
return ret;
}
int ObTenantRecoveryReportor::get_sync_point_(const share::ObLSID &id,
int ObLSRecoveryReportor::get_sync_point_(const share::ObLSID &id,
SCN &sync_scn, SCN &read_scn)
{
int ret = OB_SUCCESS;
palf::AccessMode access_mode;
int64_t unused_mode_version;
palf::PalfHandleGuard palf_handle_guard;
if (OB_FAIL(MTL(logservice::ObLogService*)->open_palf(id, palf_handle_guard))) {
LOG_WARN("failed to open palf", KR(ret), K(id));
@ -432,7 +429,7 @@ int ObTenantRecoveryReportor::get_sync_point_(const share::ObLSID &id,
}
int ObTenantRecoveryReportor::get_readable_scn(const share::ObLSID &id, SCN &readable_scn)
int ObLSRecoveryReportor::get_readable_scn(const share::ObLSID &id, SCN &readable_scn)
{
int ret = OB_SUCCESS;
storage::ObLSHandle ls_handle;

17
src/rootserver/ob_tenant_recovery_reportor.h → src/rootserver/ob_ls_recovery_reportor.h
View File

@ -10,8 +10,8 @@
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_ROOTSERVER_OB_TENANT_RECOVERY_SERVICE_H
#define OCEANBASE_ROOTSERVER_OB_TENANT_RECOVERY_SERVICE_H
#ifndef OCEANBASE_ROOTSERVER_OB_LS_RECOVERY_SERVICE_H
#define OCEANBASE_ROOTSERVER_OB_LS_RECOVERY_SERVICE_H
#include "lib/thread/ob_reentrant_thread.h"//ObRsReentrantThread
#include "lib/utility/ob_print_utils.h" //TO_STRING_KV
@ -40,15 +40,15 @@ namespace rootserver
* standby machine-readable timestamp of the majority, the minimum standby
* machine-readable timestamp of all replicas, the synchronization point, etc.
* Statistics for the syslog stream are not in this thread.*/
class ObTenantRecoveryReportor : public share::ObReentrantThread
class ObLSRecoveryReportor : public share::ObReentrantThread
{
public:
ObTenantRecoveryReportor()
ObLSRecoveryReportor()
: is_inited_(false),
tenant_id_(common::OB_INVALID_TENANT_ID),
sql_proxy_(nullptr) {}
~ObTenantRecoveryReportor() {}
static int mtl_init(ObTenantRecoveryReportor *&ka);
~ObLSRecoveryReportor() {}
static int mtl_init(ObLSRecoveryReportor *&ka);
int init();
void destroy();
int start();
@ -62,7 +62,6 @@ public:
//description: update ls recovery
static int update_ls_recovery(storage::ObLS *ls, common::ObMySQLProxy *sql_proxy);
int get_tenant_readable_scn(share::SCN &readable_scn);
static int get_readable_scn(const share::ObLSID &id, share::SCN &read_scn);
private:
static int get_sync_point_(const share::ObLSID &id, share::SCN &scn, share::SCN &read_scn);
@ -80,11 +79,11 @@ private:
int update_replayable_point_from_tenant_info_();
int update_replayable_point_from_meta_();
int submit_tenant_refresh_schema_task_();
DISALLOW_COPY_AND_ASSIGN(ObTenantRecoveryReportor);
DISALLOW_COPY_AND_ASSIGN(ObLSRecoveryReportor);
};
} // namespace rootserver
} // namespace oceanbase
#endif /* !OCEANBASE_ROOTSERVER_OB_TENANT_RECOVERY_SERVICE_H */
#endif /* !OCEANBASE_ROOTSERVER_OB_LS_RECOVERY_SERVICE_H */

1860
src/rootserver/ob_ls_service_helper.cpp Executable file
View File

File diff suppressed because it is too large Load Diff

279
src/rootserver/ob_ls_service_helper.h Normal file
View File

@ -0,0 +1,279 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_ROOTSERVER_OB_LS_SERVICE_HELPER_H
#define OCEANBASE_ROOTSERVER_OB_LS_SERVICE_HELPER_H
#include "lib/thread/ob_reentrant_thread.h"//ObRsReentrantThread
#include "logservice/ob_log_base_type.h"
#include "share/scn.h"//SCN
#include "share/ls/ob_ls_status_operator.h"//ObLSStatusOperator
#include "share/ls/ob_ls_operator.h" //ObLSAttr
#include "share/ob_thread_mgr.h" //OBTGDefIDEnum
#include "logservice/palf/palf_iterator.h" //PalfBufferIterator
#include "share/unit/ob_unit_info.h"//ObUnit::Status
#include "lib/thread/thread_mgr_interface.h" // TGRunnable
#include "lib/lock/ob_thread_cond.h"//ObThreadCond
namespace oceanbase
{
namespace obrpc
{
class ObSrvRpcProxy;
}
namespace common
{
class ObMySQLProxy;
class ObISQLClient;
class ObMySQLTransaction;
class ObClusterVersion;
}
namespace share
{
class ObLSTableOperator;
class SCN;
namespace schema
{
class ObMultiVersionSchemaService;
class ObTenantSchema;
}
}
namespace logservice
{
class ObLogHandler;
}
namespace transaction
{
class ObTxLogBlock;
class ObTxBufferNode;
}
namespace palf
{
struct PalfBaseInfo;
}
namespace rootserver
{
struct ObUnitGroupInfo
{
ObUnitGroupInfo() : unit_group_id_(OB_INVALID_ID), unit_status_(share::ObUnit::UNIT_STATUS_MAX),
ls_group_ids_() {}
virtual ~ObUnitGroupInfo() {}
bool is_valid() const;
int init(const uint64_t unit_group_id,
const share::ObUnit::Status &unit_status);
void reset();
int assign(const ObUnitGroupInfo &other);
int remove_ls_group(const uint64_t ls_group_id);
bool operator==(const ObUnitGroupInfo &other) const;
uint64_t unit_group_id_;
share::ObUnit::Status unit_status_;
ObArray<uint64_t> ls_group_ids_;
TO_STRING_KV(K_(unit_group_id), K_(unit_status), K_(ls_group_ids));
};
typedef ObArray<ObUnitGroupInfo> ObUnitGroupInfoArray;
typedef ObIArray<ObUnitGroupInfo> ObUnitGroupInfoIArray;
struct ObLSGroupInfo
{
ObLSGroupInfo() : ls_group_id_(OB_INVALID_ID), unit_group_id_(OB_INVALID_ID),
ls_ids_() {}
virtual ~ObLSGroupInfo() {}
bool is_valid() const;
int init(const uint64_t unit_group_id, const uint64_t ls_group_id);
int assign(const ObLSGroupInfo &other);
void reset();
int remove_ls(const share::ObLSID &ls_id);
uint64_t ls_group_id_;
uint64_t unit_group_id_;
ObArray<share::ObLSID> ls_ids_;
TO_STRING_KV(K_(ls_group_id), K_(unit_group_id), K_(ls_ids));
};
typedef ObArray<ObLSGroupInfo> ObLSGroupInfoArray;
typedef ObIArray<ObLSGroupInfo> ObLSGroupInfoIArray;
struct ObLSStatusMachineParameter
{
ObLSStatusMachineParameter() : ls_id_(), status_info_(), ls_info_() {}
virtual ~ObLSStatusMachineParameter() {}
bool is_valid() const
{
return ls_id_.is_valid()
&& (share::OB_LS_EMPTY == status_info_.status_
|| status_info_.ls_id_ == ls_id_);
}
int init(const share::ObLSID &id, const share::ObLSStatusInfo &status_info,
const share::ObLSAttr &ls_info);
void reset();
share::ObLSID ls_id_;
share::ObLSStatusInfo status_info_;//for create ls and status of __all_ls_status
share::ObLSAttr ls_info_;
TO_STRING_KV(K_(ls_id), K_(status_info), K_(ls_info));
};
/*descripthin: Tenant log stream status information: Statistical log stream
* status on __all_ls_status and __all_ls, tenant primary_zone and unit_num
* information. Provides location information for the newly created log stream.
* Whether the build needs to create or delete log streams.*/
class ObTenantLSInfo
{
public:
ObTenantLSInfo(ObMySQLProxy *sql_proxy,
const share::schema::ObTenantSchema *tenant_schema,
const uint64_t tenant_id, obrpc::ObSrvRpcProxy *rpc_proxy,
share::ObLSTableOperator *lst_operator)
: sql_proxy_(sql_proxy),
tenant_schema_(tenant_schema),
status_operator_(),
ls_operator_(tenant_id, sql_proxy),
status_array_(),
unit_group_array_(),
ls_group_array_(),
primary_zone_(),
rpc_proxy_(rpc_proxy),
lst_operator_(lst_operator),
max_ls_id_(OB_INVALID_ID),
max_ls_group_id_(OB_INVALID_ID) {}
virtual ~ObTenantLSInfo(){};
void reset();
bool is_valid() const;
int gather_stat(bool for_recovery);
int process_next_ls_status(int64_t &task_cnt);
// need create or delete new ls group
int check_ls_group_match_unitnum();
//need create or delete new ls
int check_ls_match_primary_zone();
int fetch_new_ls_group_id(const uint64_t tenant_id, uint64_t &ls_group_id);
int fetch_new_ls_id(const uint64_t tenant_id, share::ObLSID &ls_id);
//get ls group info from ls_group_array_ by ls_group_id
//the interface must used after gather_stat();
int get_ls_group_info(const uint64_t ls_group_id, ObLSGroupInfo &info) const;
//get ls status info from status_array_ by ls_id
//the interface must used after gather_stat();
int get_ls_status_info(const share::ObLSID &id, share::ObLSStatusInfo &info,
int64_t &info_index) const;
//process dropping tenant, set status to tenant_dropping in __all_ls
int drop_tenant();
//for recovery tenant, create new ls according to ls_id and ls_group_id
int create_new_ls_in_trans(const share::ObLSID &ls_id,
const uint64_t ls_group_id,
const share::SCN &create_scn,
common::ObMySQLTransaction &trans);
int construct_ls_status_machine_(
common::ObIArray<ObLSStatusMachineParameter> &status_machine_array);
//only for upgrade, before 4.1, make __all_ls_status equal to __all_ls
int revision_to_equal_status(common::ObMySQLTransaction &trans);
int check_ls_can_offline_by_rpc(const share::ObLSStatusInfo &info,
bool &can_offline);
private:
// get from __all_ls_status and __all_ls
int gather_all_ls_info_();
// base on status_array construct ls_array
int add_ls_to_ls_group_(const share::ObLSStatusInfo &info);
// base on ls_array construct unit_group_array and __all_unit
int add_ls_group_to_unit_group_(const ObLSGroupInfo &group_info);
int check_unit_group_valid_(const uint64_t unit_group_id, bool &is_valid);
int add_ls_status_info_(const share::ObLSStatusInfo &ls_info);
int create_new_ls_for_empty_unit_group_(const uint64_t unit_group_id);
int get_next_unit_group_(const bool is_recovery, int64_t &group_index);
// get the primary zone not in ls group
int get_next_primary_zone_(const bool is_recovery, const ObLSGroupInfo &group_info,
ObZone &primary_zone);
int create_new_ls_(const share::ObLSStatusInfo &ls_info);
// drop ls group info
int do_tenant_drop_ls_(const share::ObLSStatusInfo &ls_info);
int sys_ls_tenant_drop_(const share::ObLSStatusInfo &info);
int check_sys_ls_can_offline_(bool &can_offline);
private:
//modify __all_ls
int process_next_ls_status_(const ObLSStatusMachineParameter &status_machine, bool &is_steady);
private:
ObMySQLProxy *sql_proxy_;
const share::schema::ObTenantSchema *tenant_schema_;
share::ObLSStatusOperator status_operator_;
share::ObLSAttrOperator ls_operator_;
bool is_load_;
share::ObLSStatusInfoArray status_array_;
common::hash::ObHashMap<share::ObLSID, int64_t> status_map_;
ObUnitGroupInfoArray unit_group_array_;
ObLSGroupInfoArray ls_group_array_;
ObArray<common::ObZone> primary_zone_;
obrpc::ObSrvRpcProxy *rpc_proxy_;
share::ObLSTableOperator *lst_operator_;
//ensure ls can not be concurrent created
uint64_t max_ls_id_;
uint64_t max_ls_group_id_;
};
/*description:
*only active on the leader of the system log stream under the user tenant*/
class ObRecoveryLSHelper
{
public:
ObRecoveryLSHelper(const uint64_t tenant_id,
common::ObMySQLProxy *sql_proxy) :
tenant_id_(tenant_id), proxy_(sql_proxy) {}
virtual ~ObRecoveryLSHelper() {}
int do_work(palf::PalfBufferIterator &iterator,
share::SCN &start_scn);
private:
//get log iterator by start_scn
int seek_log_iterator_(const share::SCN &syn_scn,
palf::PalfBufferIterator &iterator);
int process_ls_log_(const ObAllTenantInfo &tenant_info,
share::SCN &start_scn,
palf::PalfBufferIterator &iterator);
int process_upgrade_log_(const transaction::ObTxBufferNode &node);
int process_ls_tx_log_(transaction::ObTxLogBlock &tx_log,
const share::SCN &syn_scn);
int process_ls_operator_(const share::ObLSAttr &ls_attr,
const share::SCN &syn_scn);
int create_new_ls_(const share::ObLSAttr &ls_attr,
const share::SCN &syn_scn,
common::ObMySQLTransaction &trans);
//wait other ls is larger than sycn ts
int check_valid_to_operator_ls_(const share::ObLSAttr &ls_attr,
const share::SCN &syn_scn);
//readable scn need report
int report_tenant_sys_recovery_scn_(const share::SCN &sys_recovery_scn, const bool update_sys_recovery_scn,
ObISQLClient *proxy);
int report_tenant_sys_recovery_scn_trans_(const share::SCN &sys_recovery_scn, const bool update_sys_recovery_scn,
common::ObMySQLTransaction &trans);
int process_ls_operator_in_trans_(const share::ObLSAttr &ls_attr,
const share::SCN &sys_recovery_scn, common::ObMySQLTransaction &trans);
int try_to_process_sys_ls_offline_();
private:
uint64_t tenant_id_;
common::ObMySQLProxy *proxy_;
};
}
}
#endif /* !OCEANBASE_ROOTSERVER_OB_LS_SERVICE_HELPER_H */

2311
src/rootserver/ob_primary_ls_service.cpp
View File

File diff suppressed because it is too large Load Diff

273
src/rootserver/ob_primary_ls_service.h
View File

@ -18,9 +18,11 @@
#include "share/ls/ob_ls_status_operator.h"//ObLSStatusOperator
#include "share/ls/ob_ls_operator.h" //ObLSAttr
#include "share/ob_thread_mgr.h" //OBTGDefIDEnum
#include "logservice/palf/palf_iterator.h" //PalfBufferIterator
#include "share/unit/ob_unit_info.h"//ObUnit::Status
#include "lib/thread/thread_mgr_interface.h" // TGRunnable
#include "lib/lock/ob_thread_cond.h"//ObThreadCond
#include "rootserver/ob_tenant_thread_helper.h"//ObTenantThreadHelper
namespace oceanbase
@ -33,10 +35,12 @@ namespace common
{
class ObMySQLProxy;
class ObMySQLTransaction;
class ObClusterVersion;
}
namespace share
{
class ObLSTableOperator;
class SCN;
namespace schema
{
class ObMultiVersionSchemaService;
@ -50,262 +54,6 @@ struct PalfBaseInfo;
}
namespace rootserver
{
struct ObUnitGroupInfo
{
ObUnitGroupInfo() : unit_group_id_(OB_INVALID_ID), unit_status_(share::ObUnit::UNIT_STATUS_MAX),
ls_group_ids_() {}
virtual ~ObUnitGroupInfo() {}
bool is_valid() const;
int init(const uint64_t unit_group_id,
const share::ObUnit::Status &unit_status);
void reset();
int assign(const ObUnitGroupInfo &other);
int remove_ls_group(const uint64_t ls_group_id);
bool operator==(const ObUnitGroupInfo &other) const;
uint64_t unit_group_id_;
share::ObUnit::Status unit_status_;
ObArray<uint64_t> ls_group_ids_;
TO_STRING_KV(K_(unit_group_id), K_(unit_status), K_(ls_group_ids));
};
typedef ObArray<ObUnitGroupInfo> ObUnitGroupInfoArray;
typedef ObIArray<ObUnitGroupInfo> ObUnitGroupInfoIArray;
struct ObLSGroupInfo
{
ObLSGroupInfo() : ls_group_id_(OB_INVALID_ID), unit_group_id_(OB_INVALID_ID),
ls_ids_() {}
virtual ~ObLSGroupInfo() {}
bool is_valid() const;
int init(const uint64_t unit_group_id, const uint64_t ls_group_id);
int assign(const ObLSGroupInfo &other);
void reset();
int remove_ls(const share::ObLSID &ls_id);
uint64_t ls_group_id_;
uint64_t unit_group_id_;
ObArray<share::ObLSID> ls_ids_;
TO_STRING_KV(K_(ls_group_id), K_(unit_group_id), K_(ls_ids));
};
typedef ObArray<ObLSGroupInfo> ObLSGroupInfoArray;
typedef ObIArray<ObLSGroupInfo> ObLSGroupInfoIArray;
struct ObLSStatusMachineParameter
{
ObLSStatusMachineParameter() : ls_id_(), status_info_(), ls_info_(share::OB_LS_EMPTY) {}
virtual ~ObLSStatusMachineParameter() {}
bool is_valid() const
{
return ls_id_.is_valid()
&& (share::OB_LS_EMPTY == status_info_.status_
|| status_info_.ls_id_ == ls_id_);
}
int init(const share::ObLSID &id, const share::ObLSStatusInfo &status_info,
const share::ObLSStatus &ls_info);
void reset();
share::ObLSID ls_id_;
share::ObLSStatusInfo status_info_;//for create ls and status of __all_ls_status
share::ObLSStatus ls_info_;//status in __all_ls
TO_STRING_KV(K_(ls_id), K_(status_info), K_(ls_info));
};
/*descripthin: Tenant log stream status information: Statistical log stream
* status on __all_ls_status and __all_ls, tenant primary_zone and unit_num
* information. Provides location information for the newly created log stream.
* Whether the build needs to create or delete log streams.*/
class ObTenantLSInfo
{
public:
ObTenantLSInfo(ObMySQLProxy *sql_proxy,
const share::schema::ObTenantSchema *tenant_schema,
const uint64_t tenant_id, obrpc::ObSrvRpcProxy *rpc_proxy,
share::ObLSTableOperator *lst_operator)
: sql_proxy_(sql_proxy),
tenant_schema_(tenant_schema),
status_operator_(),
ls_operator_(tenant_id, sql_proxy),
status_array_(),
unit_group_array_(),
ls_group_array_(),
primary_zone_(),
rpc_proxy_(rpc_proxy),
lst_operator_(lst_operator),
max_ls_id_(OB_INVALID_ID),
max_ls_group_id_(OB_INVALID_ID) {}
virtual ~ObTenantLSInfo(){};
void reset();
bool is_valid() const;
int gather_stat(bool for_recovery);
//some ls not in __all_ls, but in __all_ls_status
//it need delete by gc, no need process it anymore.
//check the ls status, and delete no need ls
int process_ls_status_missmatch(
const bool lock_sys_ls,
const share::ObTenantSwitchoverStatus &working_sw_status);
// need create or delete new ls group
int check_ls_group_match_unitnum();
//need create or delete new ls
int check_ls_match_primary_zone();
int fetch_new_ls_group_id(const uint64_t tenant_id, uint64_t &ls_group_id);
int fetch_new_ls_id(const uint64_t tenant_id, share::ObLSID &ls_id);
//get ls group info from ls_group_array_ by ls_group_id
//the interface must used after gather_stat();
int get_ls_group_info(const uint64_t ls_group_id, ObLSGroupInfo &info) const;
//get ls status info from status_array_ by ls_id
//the interface must used after gather_stat();
int get_ls_status_info(const share::ObLSID &id, share::ObLSStatusInfo &info,
int64_t &info_index) const;
//process dropping tenant, set status to tenant_dropping in __all_ls
int drop_tenant(const share::ObTenantSwitchoverStatus &working_sw_status);
//for recovery tenant, create new ls according to ls_id and ls_group_id
int create_new_ls_for_recovery(const share::ObLSID &ls_id,
const uint64_t ls_group_id,
const share::SCN &create_scn,
common::ObMySQLTransaction &trans);
//for recovery tenant, if ls is in creating in __all_ls_status, create the ls
int process_ls_stats_for_recovery();
int adjust_user_tenant_primary_zone();
/*
* description: create ls with palf base info for recovery
@param[in] info: status info in __all_ls_status_info
@param[in] create_scn : create scn of ls
@param[in] create_ls_with_palf: restore create init ls
@param[in] palf_base_info : palf base info
*/
int create_ls_with_palf(const share::ObLSStatusInfo &info, const share::SCN &create_scn,
const bool create_ls_with_palf,
const palf::PalfBaseInfo &palf_base_info);
static int get_zone_priority(const ObZone &primary_zone,
const share::schema::ObTenantSchema &tenant_schema,
ObSqlString &primary_zone_str);
static int try_update_ls_primary_zone_(const share::ObLSPrimaryZoneInfo &primary_zone_info,
const common::ObZone &new_primary_zone,
const common::ObSqlString &zone_priority);
static int construct_ls_status_machine_(
const share::ObLSStatusInfoIArray &statua_info_array,
const share::ObLSAttrIArray &ls_array,
common::ObIArray<ObLSStatusMachineParameter> &status_machine_array);
private:
int fix_ls_status_(const ObLSStatusMachineParameter &status_machine,
const share::ObTenantSwitchoverStatus &working_sw_status);
// get from __all_ls_status and __all_ls
int gather_all_ls_info_();
// base on status_array construct ls_array
int add_ls_to_ls_group_(const share::ObLSStatusInfo &info);
// base on ls_array construct unit_group_array and __all_unit
int add_ls_group_to_unit_group_(const ObLSGroupInfo &group_info);
int process_creating_ls_(const share::ObLSAttr &ls_operation);
int check_unit_group_valid_(const uint64_t unit_group_id, bool &is_valid);
int add_ls_status_info_(const share::ObLSStatusInfo &ls_info);
int create_new_ls_for_empty_unit_group_(const uint64_t unit_group_id);
int get_next_unit_group_(const bool is_recovery, int64_t &group_index);
// get the primary zone not in ls group
int get_next_primary_zone_(const bool is_recovery, const ObLSGroupInfo &group_info,
ObZone &primary_zone);
int create_new_ls_(const share::ObLSStatusInfo &ls_info,
const share::ObTenantSwitchoverStatus &working_sw_status);
// drop ls group info
int try_drop_ls_of_deleting_unit_group_(const ObUnitGroupInfo &info);
int drop_ls_(const share::ObLSStatusInfo &ls_info,
const share::ObTenantSwitchoverStatus &working_sw_status);
int do_drop_ls_(const share::ObLSStatusInfo &ls_info,
const share::ObTenantSwitchoverStatus &working_sw_status);
int do_tenant_drop_ls_(const share::ObLSStatusInfo &ls_info,
const share::ObTenantSwitchoverStatus &working_sw_status);
int do_create_ls_(const share::ObLSStatusInfo &info, const share::SCN &create_scn);
int sys_ls_tenant_drop_(const share::ObLSStatusInfo &info,
const share::ObTenantSwitchoverStatus &working_sw_status);
int check_sys_ls_can_offline_(bool &can_offline);
int check_ls_empty_(const share::ObLSStatusInfo &info, bool &empty);
int check_ls_can_offline_by_rpc_(const share::ObLSStatusInfo &info,
const share::ObLSStatus &current_ls_status,
bool &can_offline);
int process_ls_status_after_created_(const share::ObLSStatusInfo &info,
const share::ObTenantSwitchoverStatus &working_sw_status);
private:
static int set_ls_to_primary_zone(const common::ObIArray<common::ObZone> &primary_zone_array,
const share::ObLSPrimaryZoneInfoArray &primary_zone_infos,
common::ObIArray<common::ObZone> &ls_primary_zone,
common::ObIArray<uint64_t> &count_group_by_zone);
static int balance_ls_primary_zone(const common::ObIArray<common::ObZone> &primary_zone_array,
common::ObIArray<common::ObZone> &ls_primary_zone,
common::ObIArray<uint64_t> &count_group_by_zone);
int adjust_primary_zone_by_ls_group_(const common::ObIArray<common::ObZone> &primary_zone_array,
const share::ObLSPrimaryZoneInfoArray &primary_zone_infos,
const share::schema::ObTenantSchema &tenant_schema);
private:
ObMySQLProxy *sql_proxy_;
const share::schema::ObTenantSchema *tenant_schema_;
share::ObLSStatusOperator status_operator_;
share::ObLSAttrOperator ls_operator_;
bool is_load_;
share::ObLSStatusInfoArray status_array_;
common::hash::ObHashMap<share::ObLSID, int64_t> status_map_;
ObUnitGroupInfoArray unit_group_array_;
ObLSGroupInfoArray ls_group_array_;
ObArray<common::ObZone> primary_zone_;
obrpc::ObSrvRpcProxy *rpc_proxy_;
share::ObLSTableOperator *lst_operator_;
//ensure ls can not be concurrent created
uint64_t max_ls_id_;
uint64_t max_ls_group_id_;
};
class ObTenantThreadHelper : public lib::TGRunnable,
public logservice::ObIRoleChangeSubHandler
{
public:
ObTenantThreadHelper() : tg_id_(-1), thread_cond_(), is_created_(false), is_first_time_to_start_(true), thread_name_("") {}
virtual ~ObTenantThreadHelper() {}
virtual void do_work() = 0;
virtual void run1() override;
virtual void destroy();
int start();
void stop();
void wait();
int create(const char* thread_name, int tg_def_id, ObTenantThreadHelper &tenant_thread);
void idle(const int64_t idle_time_us);
public:
virtual void switch_to_follower_forcedly() override;
virtual int switch_to_leader() override;
virtual int switch_to_follower_gracefully() override
{
stop();
return OB_SUCCESS;
}
virtual int resume_leader() override
{
return OB_SUCCESS;
}
protected:
int tg_id_;
private:
common::ObThreadCond thread_cond_;
bool is_created_;
bool is_first_time_to_start_;
const char* thread_name_;
};
/*description:
*Log stream management thread: Started on the leader of the system log stream
@ -342,16 +90,9 @@ public:
private:
int process_user_tenant_(const share::schema::ObTenantSchema &tenant_schema);
//no need create ls or drop ls, but need adjust primary_zone
int process_meta_tenant_(const share::schema::ObTenantSchema &tenant_schema);
//for primary cluster, sys ls recovery stat need report,
//standby cluster will be reported in RecoveryLSService
int report_sys_ls_recovery_stat_();
// force drop user tenant if tenant is in dropping status
int try_force_drop_tenant_(
const uint64_t user_tenant_id);
int gather_tenant_recovery_stat_();
int process_user_tenant_thread0_(const share::schema::ObTenantSchema &tenant_schema);
int process_user_tenant_thread1_(palf::PalfBufferIterator &iterator,
share::SCN &start_scn);
private:
bool inited_;
uint64_t tenant_id_;

657
src/rootserver/ob_recovery_ls_service.cpp
View File

@ -20,28 +20,16 @@
#include "logservice/ob_log_handler.h" //ObLogHandler
#include "logservice/palf/log_entry.h" //LogEntry
#include "logservice/palf/log_define.h"
#include "logservice/ob_log_service.h"
#include "share/scn.h"//SCN
#include "logservice/ob_garbage_collector.h"//ObGCLSLog
#include "logservice/restoreservice/ob_log_restore_handler.h"//ObLogRestoreHandler
#include "observer/ob_server_struct.h" //GCTX
#include "rootserver/ob_primary_ls_service.h" //ObTenantLSInfo
#include "rootserver/ob_tenant_recovery_reportor.h" //ObTenantRecoveryReportor
#include "rootserver/ob_tenant_info_loader.h" // ObTenantInfoLoader
#include "share/ls/ob_ls_life_manager.h" //ObLSLifeManger
#include "share/ls/ob_ls_operator.h" //ObLSAttr
#include "share/ls/ob_ls_recovery_stat_operator.h" //ObLSRecoveryLSStatOperator
#include "share/ob_errno.h"
#include "share/ob_share_util.h" //ObShareUtil
#include "share/schema/ob_multi_version_schema_service.h" //ObMultiSchemaService
#include "share/restore/ob_physical_restore_info.h" //restore_status
#include "share/restore/ob_physical_restore_table_operator.h"//ObPhysicalRestoreTableOperator
#include "share/ob_primary_standby_service.h" // ObPrimaryStandbyService
#include "share/ob_standby_upgrade.h" // ObStandbyUpgrade
#include "share/ob_upgrade_utils.h" // ObUpgradeChecker
#include "share/ob_global_stat_proxy.h" // ObGlobalStatProxy
#include "storage/tx/ob_tx_log.h" //ObTxLogHeader
#include "storage/tx_storage/ob_ls_service.h" //ObLSService
#include "storage/tx_storage/ob_ls_handle.h" //ObLSHandle
#include "share/ob_cluster_version.h"//GET_TENANT_DATA_VERSION
#include "rootserver/ob_ls_service_helper.h"//ObRecoveryLSHelper
namespace oceanbase
{
@ -64,7 +52,7 @@ int ObRecoveryLSService::init()
ret = OB_INIT_TWICE;
LOG_WARN("has inited", KR(ret));
} else if (OB_FAIL(ObTenantThreadHelper::create("RecLSSer",
lib::TGDefIDs::LSService, *this))) {
lib::TGDefIDs::SimpleLSService, *this))) {
LOG_WARN("failed to create thread", KR(ret));
} else if (OB_FAIL(ObTenantThreadHelper::start())) {
LOG_WARN("failed to start thread", KR(ret));
@ -92,318 +80,40 @@ void ObRecoveryLSService::do_work()
if (OB_UNLIKELY(!inited_) || OB_ISNULL(proxy_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret), K(inited_), KP(proxy_));
} else if (OB_FAIL(wait_tenant_data_version_ready_(tenant_id_, DATA_VERSION_4_1_0_0))) {
LOG_WARN("failed to wait tenant and version ready", KR(ret), K(tenant_id_));
} else {
ObLSRecoveryStatOperator ls_recovery;
palf::PalfBufferIterator iterator;
int tmp_ret = OB_SUCCESS;
const int64_t idle_time_us = 100 * 1000L;
int64_t idle_time_us = 100 * 1000L;
SCN start_scn;
while (!has_set_stop()) {
ObCurTraceId::init(GCONF.self_addr_);
uint64_t thread_idx = get_thread_idx();
ObTenantInfoLoader *tenant_info_loader = MTL(ObTenantInfoLoader*);
ObAllTenantInfo tenant_info;
//two thread for seed log and recovery_ls_manager
// two thread for seed log and recovery_ls_manager
if (!is_user_tenant(tenant_id_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ls recovery thread must run on user tenant", KR(ret), K(tenant_id_));
} else if (OB_ISNULL(tenant_info_loader)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("tenant report is null", KR(ret), K(tenant_id_));
} else if (OB_FAIL(tenant_info_loader->get_tenant_info(tenant_info))) {
LOG_WARN("failed to get tenant info", KR(ret));
} else if (OB_FAIL(check_can_do_recovery_(tenant_info))) {
LOG_WARN("failed to check do recovery", KR(ret), K(tenant_info));
} else if (0 == thread_idx) {
if (OB_SUCCESS != (tmp_ret = process_recovery_ls_manager())) {
ret = OB_SUCC(ret) ? tmp_ret : ret;
LOG_WARN("failed to process recovery ls manager", KR(ret), KR(tmp_ret));
}
} else {
if (!start_scn.is_valid()) {
ObLSRecoveryStat ls_recovery_stat;
if (OB_FAIL(ls_recovery.get_ls_recovery_stat(tenant_id_,
SYS_LS, false, ls_recovery_stat, *proxy_))) {
LOG_WARN("failed to load sys recovery stat", KR(ret), K(tenant_id_));
} else if (SCN::base_scn() == ls_recovery_stat.get_sync_scn()) {
//等待物理恢复设置完系统日志流初始同步点之后开始迭代
ret = OB_EAGAIN;
LOG_WARN("restore init sync scn has not setted, need wait", KR(ret), K(ls_recovery_stat));
} else if (OB_FAIL(report_sys_ls_recovery_stat_(ls_recovery_stat.get_sync_scn()))) {
//may recovery end, but readable scn need report
LOG_WARN("failed to report ls recovery stat", KR(ret), K(ls_recovery_stat));
} else if (tenant_info.get_recovery_until_scn() == ls_recovery_stat.get_sync_scn()) {
ret = OB_EAGAIN;
if (REACH_TIME_INTERVAL(60 * 1000 * 1000)) { // every minute
LOG_INFO("has recovered to recovery_until_scn", KR(ret), K(ls_recovery_stat), K(tenant_info));
}
} else if (OB_FAIL(seek_log_iterator_(ls_recovery_stat.get_sync_scn(), iterator))) {
LOG_WARN("failed to seek log iterator", KR(ret), K(ls_recovery_stat));
} else {
start_scn = ls_recovery_stat.get_sync_scn();
LOG_INFO("start to seek at", K(start_scn));
}
ObRecoveryLSHelper recovery_ls(tenant_id_, proxy_);
if (OB_FAIL(recovery_ls.do_work(iterator, start_scn))) {
LOG_WARN("failed to recovery ls", KR(ret), K(start_scn));
}
if (OB_FAIL(ret)) {
} else if (start_scn.is_valid() && OB_FAIL(process_ls_log_(tenant_info, start_scn, iterator))) {
if (OB_ITER_STOP != ret) {
LOG_WARN("failed to process ls log", KR(ret), K(start_scn), K(tenant_info));
}
}
if (OB_FAIL(ret)) {
start_scn.reset();
}
if (REACH_TIME_INTERVAL(10 * 1000 * 1000)) { // every 10 second
(void)try_tenant_upgrade_end_();
if (OB_ITER_STOP == ret) {
//standby or recovery may restore finish,
//so when the error code is OB_ITER_STOP, adjust idle_time to 10s
idle_time_us = 10 * 1000 * 1000;//10s
}
}
LOG_INFO("[LS_RECOVERY] finish one round", KR(ret), KR(tmp_ret), K(start_scn), K(thread_idx), K(tenant_info));
if (REACH_TIME_INTERVAL(10 * 1000 * 1000)) { // every 10 second
(void)try_tenant_upgrade_end_();
}
LOG_INFO("[RECOVERY_LS_SERVICE] finish one round", KR(ret),
K(start_scn));
idle(idle_time_us);
}
}
}
int ObRecoveryLSService::seek_log_iterator_(const SCN &sync_scn, PalfBufferIterator &iterator)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", KR(ret), K(inited_));
} else if (OB_UNLIKELY(!sync_scn.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("sync scn is invalid", KR(ret), K(sync_scn));
} else {
ObLSService *ls_svr = MTL(ObLSService *);
ObLSHandle ls_handle;
if (OB_FAIL(ls_svr->get_ls(SYS_LS, ls_handle, storage::ObLSGetMod::RS_MOD))) {
LOG_WARN("failed to get ls", KR(ret));
} else {
ObLogHandler *log_handler = NULL;
ObLS *ls = NULL;
palf::LSN start_lsn;
if (OB_ISNULL(ls = ls_handle.get_ls())
|| OB_ISNULL(log_handler = ls->get_log_handler())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ls or log handle is null", KR(ret), KP(ls),
KP(log_handler));
} else if (OB_FAIL(log_handler->locate_by_scn_coarsely(sync_scn, start_lsn))) {
LOG_WARN("failed to locate lsn", KR(ret), K(sync_scn));
} else if (OB_FAIL(log_handler->seek(start_lsn, iterator))) {
LOG_WARN("failed to seek iterator", KR(ret), K(sync_scn));
}
}
}
return ret;
}
int ObRecoveryLSService::process_ls_log_(
const ObAllTenantInfo &tenant_info,
SCN &start_scn,
PalfBufferIterator &iterator)
{
int ret = OB_SUCCESS;
palf::LogEntry log_entry;
palf::LSN target_lsn;
SCN sync_scn;
SCN last_sync_scn;
if (OB_UNLIKELY(!inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", KR(ret), K(inited_));
} else if (OB_UNLIKELY(!tenant_info.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("tenant_info is invalid", KR(ret), K(tenant_info));
}
while (OB_SUCC(ret) && OB_SUCC(iterator.next())) {
if (OB_FAIL(iterator.get_entry(log_entry, target_lsn))) {
LOG_WARN("failed to get log", KR(ret), K(log_entry));
} else if (OB_ISNULL(log_entry.get_data_buf())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("log entry is null", KR(ret));
} else {
LOG_DEBUG("get log", K(log_entry), K(target_lsn), K(start_scn));
sync_scn = log_entry.get_scn();
const char *log_buf = log_entry.get_data_buf();
const int64_t log_length = log_entry.get_data_len();
logservice::ObLogBaseHeader header;
const int64_t HEADER_SIZE = header.get_serialize_size();
int64_t log_pos = 0;
if (OB_UNLIKELY(sync_scn <= start_scn)) {
//通过scn定位的LSN是不准确的，可能会获取多余的数据，所以需要把小于等于sync_scn的过滤掉
continue;
} else if (tenant_info.get_recovery_until_scn() < sync_scn) {
if (OB_FAIL(report_sys_ls_recovery_stat_(tenant_info.get_recovery_until_scn()))) {
LOG_WARN("failed to report_sys_ls_recovery_stat_", KR(ret), K(sync_scn), K(tenant_info),
K(log_entry), K(target_lsn), K(start_scn));
// SYS LS has recovered to the recovery_until_scn, need stop iterate SYS LS log and reset start_scn
} else if (OB_FAIL(seek_log_iterator_(tenant_info.get_recovery_until_scn(), iterator))) {
LOG_WARN("failed to seek log iterator", KR(ret), K(sync_scn), K(tenant_info),
K(log_entry), K(target_lsn), K(start_scn));
} else {
ret = OB_ITER_STOP;
LOG_WARN("SYS LS has recovered to the recovery_until_scn, need stop iterate SYS LS log",
KR(ret), K(sync_scn), K(tenant_info), K(log_entry), K(target_lsn), K(start_scn));
start_scn.reset();
}
} else if (OB_FAIL(header.deserialize(log_buf, HEADER_SIZE, log_pos))) {
LOG_WARN("failed to deserialize", KR(ret), K(HEADER_SIZE));
} else if (OB_UNLIKELY(log_pos >= log_length)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("log pos is not expected", KR(ret), K(log_pos), K(log_length));
} else if (logservice::GC_LS_LOG_BASE_TYPE == header.get_log_type()) {
ObGCLSLog gc_log;
if (OB_FAIL(gc_log.deserialize(log_buf, log_length, log_pos))) {
LOG_WARN("failed to deserialize gc log", KR(ret), K(log_length));
} else if (ObGCLSLOGType::OFFLINE_LS == gc_log.get_log_type()) {
//set sys ls offline
if (OB_FAIL(process_gc_log_(gc_log, sync_scn))) {
LOG_WARN("failed to process gc log", KR(ret), K(sync_scn));
}
}
// nothing
} else if (logservice::TRANS_SERVICE_LOG_BASE_TYPE == header.get_log_type()) {
ObTxLogBlock tx_log_block;
ObTxLogBlockHeader tx_block_header;
if (OB_FAIL(tx_log_block.init(log_buf, log_length, log_pos, tx_block_header))) {
LOG_WARN("failed to init tx log block", KR(ret), K(log_length));
} else if (OB_FAIL(process_ls_tx_log_(tx_log_block, sync_scn))) {
LOG_WARN("failed to process ls tx log", KR(ret), K(tx_log_block), K(sync_scn));
}
} else {}
if (OB_SUCC(ret)) {
last_sync_scn = sync_scn;
} else if (last_sync_scn.is_valid()) {
//if ls_operator can not process, need to report last sync scn
int tmp_ret = OB_SUCCESS;
if (OB_TMP_FAIL(report_sys_ls_recovery_stat_(last_sync_scn))) {
LOG_WARN("failed to report ls recovery stat", KR(ret), KR(tmp_ret), K(last_sync_scn));
}
}
}
}//end for each log
if (OB_ITER_END == ret) {
ret = OB_SUCCESS;
if (OB_FAIL(report_sys_ls_recovery_stat_(sync_scn))) {
LOG_WARN("failed to report ls recovery stat", KR(ret), K(sync_scn));
}
} else if (OB_SUCC(ret)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("iterator must be end", KR(ret));
} else {
LOG_WARN("failed to get next log", KR(ret));
}
return ret;
}
int ObRecoveryLSService::process_ls_tx_log_(ObTxLogBlock &tx_log_block, const SCN &sync_scn)
{
int ret = OB_SUCCESS;
bool has_operation = false;
if (OB_UNLIKELY(!inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", KR(ret), K(inited_));
}
while (OB_SUCC(ret)) {
transaction::ObTxLogHeader tx_header;
if (OB_FAIL(tx_log_block.get_next_log(tx_header))) {
if (OB_ITER_END == ret) {
ret = OB_SUCCESS;
break;
} else {
LOG_WARN("failed to get next log", KR(ret));
}
} else if (transaction::ObTxLogType::TX_COMMIT_LOG !=
tx_header.get_tx_log_type()) {
// nothing
} else {
ObTxCommitLogTempRef temp_ref;
ObTxCommitLog commit_log(temp_ref);
const int64_t COMMIT_SIZE = commit_log.get_serialize_size();
//TODO commit log may too large
if (OB_FAIL(tx_log_block.deserialize_log_body(commit_log))) {
LOG_WARN("failed to deserialize", KR(ret));
} else {
const ObTxBufferNodeArray &source_data =
commit_log.get_multi_source_data();
for (int64_t i = 0; OB_SUCC(ret) && i < source_data.count(); ++i) {
const ObTxBufferNode &node = source_data.at(i);
if (ObTxDataSourceType::STANDBY_UPGRADE == node.get_data_source_type()) {
if (OB_FAIL(process_upgrade_log_(node))) {
LOG_WARN("failed to process_upgrade_log_", KR(ret), K(node));
}
} else if (ObTxDataSourceType::LS_TABLE != node.get_data_source_type()) {
// nothing
} else if (has_operation) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("one clog has more than one operation", KR(ret), K(commit_log));
} else {
has_operation = true;
ObLSAttr ls_attr;
const int64_t LS_SIZE = ls_attr.get_serialize_size();
int64_t pos = 0;
if (OB_FAIL(ls_attr.deserialize(node.get_data_buf().ptr(), LS_SIZE,
pos))) {
LOG_WARN("failed to deserialize", KR(ret), K(node), K(LS_SIZE));
} else if (OB_UNLIKELY(pos > LS_SIZE)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to get ls attr", KR(ret), K(pos), K(LS_SIZE));
} else {
LOG_INFO("get ls operation", K(ls_attr), K(sync_scn));
//TODO ls recovery is too fast for ls manager, so it maybe failed, while change ls status
//consider how to retry
if (OB_FAIL(process_ls_operator_(ls_attr,
sync_scn))) {
LOG_WARN("failed to process ls operator", KR(ret), K(ls_attr),
K(sync_scn));
}
}
}
}// end for
}
}
} // end while for each tx_log
return ret;
}
int ObRecoveryLSService::process_upgrade_log_(const ObTxBufferNode &node)
{
int ret = OB_SUCCESS;
uint64_t standby_data_version = 0;
if (!node.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("node is invalid", KR(ret), K(node));
} else {
ObStandbyUpgrade primary_data_version;
int64_t pos = 0;
if (OB_FAIL(primary_data_version.deserialize(node.get_data_buf().ptr(), node.get_data_buf().length(), pos))) {
LOG_WARN("failed to deserialize", KR(ret), K(node), KPHEX(node.get_data_buf().ptr(), node.get_data_buf().length()));
} else if (OB_UNLIKELY(pos > node.get_data_buf().length())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to get primary_data_version", KR(ret), K(pos), K(node.get_data_buf().length()));
} else {
LOG_INFO("get primary_data_version", K(primary_data_version));
if (!primary_data_version.is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("primary_data_version not valid", KR(ret), K(primary_data_version));
} else if (OB_FAIL(ObUpgradeChecker::get_data_version_by_cluster_version(
GET_MIN_CLUSTER_VERSION(),
standby_data_version))) {
LOG_WARN("failed to get_data_version_by_cluster_version", KR(ret), K(GET_MIN_CLUSTER_VERSION()));
} else if (primary_data_version.get_data_version() > standby_data_version) {
ret = OB_EAGAIN;
if (REACH_TIME_INTERVAL(30 * 60 * 1000 * 1000)) { // 30min
LOG_ERROR("standby version is not new enough to recover primary clog", KR(ret),
K(primary_data_version), K(standby_data_version));
}
}
}
}
return ret;
}
int ObRecoveryLSService::get_min_data_version_(uint64_t &compatible)
{
int ret = OB_SUCCESS;
@ -506,331 +216,6 @@ void ObRecoveryLSService::try_tenant_upgrade_end_()
}
}
int ObRecoveryLSService::process_gc_log_(logservice::ObGCLSLog &gc_log, const SCN &sync_scn)
{
int ret = OB_SUCCESS;
common::ObMySQLTransaction trans;
const uint64_t meta_tenant_id = gen_meta_tenant_id(tenant_id_);
ObLSLifeAgentManager ls_life_agent(*proxy_);
if (OB_ISNULL(proxy_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("proxy is null", KR(ret));
} else if (OB_FAIL(trans.start(proxy_, meta_tenant_id))) {
LOG_WARN("failed to start trans", KR(ret), K(meta_tenant_id));
} else if (OB_FAIL(ls_life_agent.set_ls_offline_in_trans(
tenant_id_, SYS_LS, share::OB_LS_TENANT_DROPPING, sync_scn, share::NORMAL_SWITCHOVER_STATUS,
trans))) {
LOG_WARN("failed to set offline", KR(ret), K(tenant_id_), K(sync_scn));
} else {
ObLSRecoveryStatOperator ls_recovery;
ObLSRecoveryStat ls_recovery_stat;
if (OB_FAIL(construct_ls_recovery_stat(sync_scn, ls_recovery_stat))) {
LOG_WARN("failed to construct ls recovery stat", KR(ret), K(sync_scn));
} else if (OB_FAIL(ls_recovery.update_ls_recovery_stat_in_trans(
ls_recovery_stat, trans))) {
LOG_WARN("failed to update ls recovery stat", KR(ret), K(ls_recovery_stat));
}
}
if (trans.is_started()) {
int tmp_ret = OB_SUCCESS;
if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) {
ret = OB_SUCC(ret) ? tmp_ret : ret;
LOG_WARN("failed to end trans", KR(ret), K(tmp_ret));
}
}
return ret;
}
int ObRecoveryLSService::construct_ls_recovery_stat(const SCN &sync_scn,
ObLSRecoveryStat &ls_stat)
{
int ret = OB_SUCCESS;
ls_stat.reset();
SCN readable_scn;
if (OB_UNLIKELY(!inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", KR(ret), K(inited_));
} else if (OB_FAIL(ObTenantRecoveryReportor::get_readable_scn(SYS_LS, readable_scn))) {
LOG_WARN("failed to get readable scn", KR(ret));
} else if (OB_FAIL(ls_stat.init_only_recovery_stat(
tenant_id_, SYS_LS, sync_scn, readable_scn))) {
LOG_WARN("failed to init ls recovery stat", KR(ret), K(tenant_id_),
K(sync_scn), K(readable_scn));
}
return ret;
}
int ObRecoveryLSService::process_ls_operator_(const share::ObLSAttr &ls_attr, const SCN &sync_scn)
{
int ret = OB_SUCCESS;
common::ObMySQLTransaction trans;
const uint64_t meta_tenant_id = gen_meta_tenant_id(tenant_id_);
if (OB_UNLIKELY(!ls_attr.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("ls attr is invalid", KR(ret), K(ls_attr));
} else if (OB_ISNULL(proxy_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("proxy is null", KR(ret));
} else if (OB_FAIL(check_valid_to_operator_ls_(ls_attr, sync_scn))) {
LOG_WARN("failed to check valid to operator ls", KR(ret), K(sync_scn), K(ls_attr));
} else if (OB_FAIL(trans.start(proxy_, meta_tenant_id))) {
LOG_WARN("failed to start trans", KR(ret), K(meta_tenant_id));
} else if (OB_FAIL(process_ls_operator_in_trans_(ls_attr, sync_scn, trans))) {
LOG_WARN("failed to process ls operator in trans", KR(ret), K(ls_attr), K(sync_scn));
}
if (OB_SUCC(ret)) {
ObLSRecoveryStat ls_recovery_stat;
ObLSRecoveryStatOperator ls_recovery;
if (OB_FAIL(construct_ls_recovery_stat(sync_scn, ls_recovery_stat))) {
LOG_WARN("failed to construct ls recovery stat", KR(ret), K(sync_scn));
} else if (OB_FAIL(ls_recovery.update_ls_recovery_stat_in_trans(
ls_recovery_stat, trans))) {
LOG_WARN("failed to update ls recovery stat", KR(ret), K(ls_recovery_stat));
}
}
if (trans.is_started()) {
int tmp_ret = OB_SUCCESS;
if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) {
ret = OB_SUCC(ret) ? tmp_ret : ret;
LOG_WARN("failed to end trans", KR(ret), K(tmp_ret));
}
}
return ret;
}
int ObRecoveryLSService::check_valid_to_operator_ls_(
const share::ObLSAttr &ls_attr, const SCN &sync_scn)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", KR(ret), K(inited_));
} else if (OB_ISNULL(proxy_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("proxy is null", KR(ret));
} else if (OB_UNLIKELY(!sync_scn.is_valid() || !ls_attr.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("syns scn is invalid", KR(ret), K(sync_scn), K(ls_attr));
} else if (share::is_ls_tenant_drop_pre_op(ls_attr.get_ls_operatin_type())) {
ret = OB_ITER_STOP;
LOG_WARN("can not process ls operator after tenant dropping", K(ls_attr));
} else if (share::is_ls_tenant_drop_op(ls_attr.get_ls_operatin_type())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ls recovery must stop while pre tenant dropping", KR(ret), K(ls_attr));
} else {
SCN user_scn;
ObLSRecoveryStatOperator ls_recovery;
if (OB_FAIL(ls_recovery.get_user_ls_sync_scn(tenant_id_, *proxy_, user_scn))) {
LOG_WARN("failed to get user scn", KR(ret), K(tenant_id_));
} else if (user_scn >= sync_scn) {
//other ls has larger sync scn, ls can operator
} else {
ret = OB_NEED_WAIT;
LOG_WARN("can not process ls operator, need wait other ls sync", KR(ret),
K(user_scn), K(sync_scn));
}
}
return ret;
}
int ObRecoveryLSService::process_ls_operator_in_trans_(
const share::ObLSAttr &ls_attr, const SCN &sync_scn, ObMySQLTransaction &trans)
{
int ret = OB_SUCCESS;
const uint64_t meta_tenant_id = gen_meta_tenant_id(tenant_id_);
if (OB_UNLIKELY(!ls_attr.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("ls attr is invalid", KR(ret), K(ls_attr));
} else if (OB_ISNULL(proxy_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("proxy is null", KR(ret));
} else {
ObLSStatusOperator ls_operator;
share::ObLSStatusInfo ls_status;
ObLSLifeAgentManager ls_life_agent(*proxy_);
if (OB_UNLIKELY(!ls_attr.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("ls attr is invalid", KR(ret), K(ls_attr));
} else if (share::is_ls_create_pre_op(ls_attr.get_ls_operatin_type())) {
//create new ls;
if (OB_FAIL(create_new_ls_(ls_attr, sync_scn, trans))) {
LOG_WARN("failed to create new ls", KR(ret), K(sync_scn), K(ls_attr));
}
} else if (share::is_ls_create_abort_op(ls_attr.get_ls_operatin_type())) {
if (OB_FAIL(ls_life_agent.drop_ls_in_trans(tenant_id_, ls_attr.get_ls_id(), share::NORMAL_SWITCHOVER_STATUS, trans))) {
LOG_WARN("failed to drop ls", KR(ret), K(tenant_id_), K(ls_attr));
}
} else if (share::is_ls_drop_end_op(ls_attr.get_ls_operatin_type())) {
if (OB_FAIL(ls_life_agent.set_ls_offline_in_trans(tenant_id_, ls_attr.get_ls_id(),
ls_attr.get_ls_status(), sync_scn, share::NORMAL_SWITCHOVER_STATUS, trans))) {
LOG_WARN("failed to set offline", KR(ret), K(tenant_id_), K(ls_attr), K(sync_scn));
}
} else {
ObLSStatus target_status = share::OB_LS_EMPTY;
if (OB_FAIL(ls_operator.get_ls_status_info(tenant_id_, ls_attr.get_ls_id(),
ls_status, trans))) {
LOG_WARN("failed to get ls status", KR(ret), K(tenant_id_), K(ls_attr));
} else if (ls_status.ls_is_creating()) {
ret = OB_EAGAIN;
LOG_WARN("ls not created, need wait", KR(ret), K(ls_status));
} else if (share::is_ls_create_end_op(ls_attr.get_ls_operatin_type())) {
// set ls to normal
target_status = share::OB_LS_NORMAL;
} else if (share::is_ls_tenant_drop_op(ls_attr.get_ls_operatin_type())) {
target_status = share::OB_LS_TENANT_DROPPING;
} else if (share::is_ls_drop_pre_op(ls_attr.get_ls_operatin_type())) {
target_status = share::OB_LS_DROPPING;
} else if (share::is_ls_tenant_drop_pre_op(ls_attr.get_ls_operatin_type())) {
target_status = share::OB_LS_PRE_TENANT_DROPPING;
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected operation type", KR(ret), K(ls_attr));
}
if (FAILEDx(ls_operator.update_ls_status(tenant_id_, ls_attr.get_ls_id(),
ls_status.status_, target_status,
share::NORMAL_SWITCHOVER_STATUS, trans))) {
LOG_WARN("failed to update ls status", KR(ret), K(tenant_id_), K(ls_attr),
K(ls_status), K(target_status));
}
LOG_INFO("[LS_RECOVERY] update ls status", KR(ret), K(ls_attr), K(target_status));
}
}
return ret;
}
int ObRecoveryLSService::create_new_ls_(const share::ObLSAttr &ls_attr,
const SCN &sync_scn,
common::ObMySQLTransaction &trans)
{
int ret = OB_SUCCESS;
if (!share::is_ls_create_pre_op(ls_attr.get_ls_operatin_type())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("ls not create pre operation", KR(ret), K(ls_attr));
} else {
//create new ls;
share::schema::ObSchemaGetterGuard schema_guard;
const share::schema::ObTenantSchema *tenant_schema = NULL;
if (OB_ISNULL(GCTX.schema_service_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected error", KR(ret));
} else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(
OB_SYS_TENANT_ID, schema_guard))) {
LOG_WARN("fail to get schema guard", KR(ret));
} else if (OB_FAIL(schema_guard.get_tenant_info(tenant_id_, tenant_schema))) {
LOG_WARN("failed to get tenant ids", KR(ret), K(tenant_id_));
} else if (OB_ISNULL(tenant_schema)) {
ret = OB_TENANT_NOT_EXIST;
LOG_WARN("tenant not exist", KR(ret), K(tenant_id_));
} else {
ObTenantLSInfo tenant_stat(GCTX.sql_proxy_, tenant_schema, tenant_id_,
GCTX.srv_rpc_proxy_, GCTX.lst_operator_);
if (OB_FAIL(tenant_stat.gather_stat(true))) {
LOG_WARN("failed to gather stat", KR(ret));
} else if (OB_FAIL(tenant_stat.create_new_ls_for_recovery(ls_attr.get_ls_id(),
ls_attr.get_ls_group_id(), ls_attr.get_create_scn(), trans))) {
LOG_WARN("failed to add new ls status info", KR(ret), K(ls_attr), K(sync_scn));
}
}
LOG_INFO("[LS_RECOVERY] create new ls", KR(ret), K(ls_attr));
}
return ret;
}
int ObRecoveryLSService::process_recovery_ls_manager()
{
int ret = OB_SUCCESS;
share::schema::ObSchemaGetterGuard schema_guard;
const share::schema::ObTenantSchema *tenant_schema = NULL;
if (OB_UNLIKELY(!inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret), K(inited_));
} else if (OB_ISNULL(GCTX.schema_service_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected error", KR(ret));
} else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(
OB_SYS_TENANT_ID, schema_guard))) {
LOG_WARN("fail to get schema guard", KR(ret));
} else if (OB_FAIL(schema_guard.get_tenant_info(tenant_id_, tenant_schema))) {
LOG_WARN("failed to get tenant ids", KR(ret), K(tenant_id_));
} else if (OB_ISNULL(tenant_schema)) {
ret = OB_TENANT_NOT_EXIST;
LOG_WARN("tenant not exist", KR(ret), K(tenant_id_));
} else {
ObTenantLSInfo tenant_stat(GCTX.sql_proxy_, tenant_schema, tenant_id_,
GCTX.srv_rpc_proxy_, GCTX.lst_operator_);
if (OB_FAIL(tenant_stat.process_ls_stats_for_recovery())) {
LOG_WARN("failed to do process in standby", KR(ret), K(tenant_id_));
}
}
return ret;
}
int ObRecoveryLSService::check_can_do_recovery_(const ObAllTenantInfo &tenant_info)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret), K(inited_));
} else if (OB_UNLIKELY(!tenant_info.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("tenant_info is invalid", KR(ret), K(tenant_info));
} else if (tenant_info.is_primary()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("primary tenant can not do recovery", KR(ret), K(tenant_info));
} else if (tenant_info.is_standby()) {
//standby can do recovery
} else if (tenant_info.is_restore()) {
//need to check success to create init ls
share::ObPhysicalRestoreTableOperator restore_table_operator;
share::ObPhysicalRestoreJob job_info;
if (OB_ISNULL(proxy_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("sql can't null", K(ret), K(proxy_));
} else if (OB_FAIL(restore_table_operator.init(proxy_, tenant_id_))) {
LOG_WARN("fail to init restore table operator", KR(ret), K(tenant_id_));
} else if (OB_FAIL(restore_table_operator.get_job_by_tenant_id(tenant_id_,
job_info))) {
LOG_WARN("fail to get restore job", K(ret), K(tenant_id_));
} else if (job_info.is_valid_status_to_recovery()) {
//can do recovery
} else {
ret = OB_NEED_WAIT;
LOG_WARN("restore tenant not valid to recovery", KR(ret), K(job_info));
}
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected tenant role", KR(ret), K(tenant_info));
}
return ret;
}
int ObRecoveryLSService::report_sys_ls_recovery_stat_(const SCN &sync_scn)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", KR(ret), K(inited_));
} else if (OB_ISNULL(proxy_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("sql can't null", K(ret), K(proxy_));
} else if (SCN::base_scn() != sync_scn) {
ObLSRecoveryStatOperator ls_recovery;
ObLSRecoveryStat ls_recovery_stat;
if (OB_FAIL(construct_ls_recovery_stat(sync_scn, ls_recovery_stat))) {
LOG_WARN("failed to construct ls recovery stat", KR(ret), K(sync_scn));
} else if (OB_FAIL(ls_recovery.update_ls_recovery_stat(ls_recovery_stat,
*proxy_))) {
LOG_WARN("failed to update ls recovery stat", KR(ret),
K(ls_recovery_stat));
}
}
return ret;
}
}//end of namespace rootserver
}//end of namespace oceanbase

63
src/rootserver/ob_recovery_ls_service.h
View File

@ -14,11 +14,7 @@
#define OCEANBASE_ROOTSERVER_OB_RECCOVERY_LS_SERVICE_H
#include "lib/thread/ob_reentrant_thread.h"//ObRsReentrantThread
#include "logservice/ob_log_base_type.h"//ObIRoleChangeSubHandler ObICheckpointSubHandler ObIReplaySubHandler
#include "logservice/palf/lsn.h"//palf::LSN
#include "logservice/palf/palf_iterator.h" //PalfBufferIterator
#include "ob_primary_ls_service.h" //ObTenantThreadHelper
#include "lib/lock/ob_spin_lock.h" //ObSpinLock
#include "storage/tx/ob_multi_data_source.h" //ObTxBufferNode
#include "ob_tenant_thread_helper.h" //ObTenantThreadHelper
namespace oceanbase
{
@ -32,35 +28,11 @@ class ObMySQLProxy;
class ObISQLClient;
class ObMySQLTransaction;
}
namespace share
{
class ObLSTableOperator;
struct ObLSAttr;
struct ObLSRecoveryStat;
class SCN;
namespace schema
{
class ObMultiVersionSchemaService;
class ObTenantSchema;
}
}
namespace logservice
{
class ObLogHandler;
class ObGCLSLog;
}
namespace transaction
{
class ObTxLogBlock;
}
namespace rootserver
{
/*description:
*Restores the status of each log stream according to the logs to which the
*system log stream is synchronized, and updates the recovery progress of the
*system log stream. This thread should only exist in the standby database or
*the recovery process, and needs to be registered in the RestoreHandler.
*This thread is only active on the leader of the system log stream under the user tenant*/
class ObRecoveryLSService : public ObTenantThreadHelper
{
public:
@ -72,35 +44,8 @@ public:
void destroy();
virtual void do_work() override;
private:
//get log iterator by start_scn
int seek_log_iterator_(const share::SCN &syn_scn,
palf::PalfBufferIterator &iterator);
int process_ls_log_(const ObAllTenantInfo &tenant_info,
share::SCN &start_scn,
palf::PalfBufferIterator &iterator);
int process_upgrade_log_(const transaction::ObTxBufferNode &node);
int process_gc_log_(logservice::ObGCLSLog &gc_log,
const share::SCN &syn_scn);
int process_ls_tx_log_(transaction::ObTxLogBlock &tx_log,
const share::SCN &syn_scn);
int process_ls_operator_(const share::ObLSAttr &ls_attr,
const share::SCN &syn_scn);
int create_new_ls_(const share::ObLSAttr &ls_attr,
const share::SCN &syn_scn,
common::ObMySQLTransaction &trans);
int process_recovery_ls_manager();
int construct_ls_recovery_stat(const share::SCN &syn_scn,
share::ObLSRecoveryStat &ls_stat);
//wait other ls is larger than sycn ts
int check_valid_to_operator_ls_(const share::ObLSAttr &ls_attr,
const share::SCN &syn_scn);
int check_can_do_recovery_(const ObAllTenantInfo &tenant_info);
//readable scn need report
int report_sys_ls_recovery_stat_(const share::SCN &sync_scn);
void try_tenant_upgrade_end_();
int get_min_data_version_(uint64_t &compatible);
int process_ls_operator_in_trans_(const share::ObLSAttr &ls_attr,
const share::SCN &sync_scn, common::ObMySQLTransaction &trans);
private:
bool inited_;
uint64_t tenant_id_;

1
src/rootserver/ob_root_utils.cpp
View File

@ -1773,7 +1773,6 @@ int ObRootUtils::get_tenant_intersection(ObUnitManager &unit_mgr,
return ret;
}
template<class T>
bool ObRootUtils::has_intersection(const common::ObIArray<T> &this_array,
const common::ObIArray<T> &other_array)

1
src/rootserver/ob_root_utils.h
View File

@ -642,7 +642,6 @@ public:
common::ObIArray<common::ObAddr> &this_server_list,
common::ObIArray<common::ObAddr> &other_server_list,
common::ObIArray<uint64_t> &tenant_ids);
};
class ObClusterInfoGetter

15
src/rootserver/ob_tenant_info_loader.cpp
View File

@ -267,5 +267,20 @@ int ObAllTenantInfoCache::get_tenant_info(share::ObAllTenantInfo &tenant_info)
return ret;
}
int ObTenantInfoLoader::get_tenant_readable_scn(SCN &readable_scn)
{
int ret = OB_SUCCESS;
share::ObAllTenantInfo tenant_info;
if (OB_FAIL(get_tenant_info(tenant_info))) {
LOG_WARN("get_tenant_info failed", K(ret));
} else if (OB_UNLIKELY(! tenant_info.is_valid())) {
ret = OB_EAGAIN;
LOG_WARN("tenant info not valid", K(ret), K(tenant_info));
} else {
readable_scn = tenant_info.get_standby_scn();
}
return ret;
}
}
}

3
src/rootserver/ob_tenant_info_loader.h
View File

@ -78,6 +78,7 @@ public:
}
virtual void run2() override;
int get_tenant_info(share::ObAllTenantInfo &tenant_info);
int get_tenant_readable_scn(share::SCN &readable_scn);
/**
* @description:
* get valid sts, only return sts refreshed after specified_time
@ -107,4 +108,4 @@ private:
} // namespace rootserver
} // namespace oceanbase
#endif /* !OCEANBASE_ROOTSERVER_OB_TENANT_INFO_LOADER_H */
#endif /* !OCEANBASE_ROOTSERVER_OB_TENANT_INFO_LOADER_H */

114
src/rootserver/ob_tenant_info_report.cpp Executable file
View File

@ -0,0 +1,114 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX RS
#include "ob_tenant_info_report.h"
#include "lib/profile/ob_trace_id.h"
#include "share/ob_errno.h"
#include "share/ob_share_util.h"//ObShareUtil
#include "observer/ob_server_struct.h"//GCTX
#include "share/ls/ob_ls_recovery_stat_operator.h"//ObLSRecoveryStatOperator
namespace oceanbase
{
using namespace common;
using namespace share;
using namespace transaction;
using namespace palf;
namespace rootserver
{
//////////////ObTenantInfoReportor
int ObTenantInfoReportor::mtl_init(ObTenantInfoReportor *&ka)
{
return ka->init();
}
int ObTenantInfoReportor::init()
{
int ret = OB_SUCCESS;
tenant_id_ = MTL_ID();
if (OB_UNLIKELY(inited_)) {
ret = OB_INIT_TWICE;
LOG_WARN("has inited", KR(ret));
} else if (OB_FAIL(ObTenantThreadHelper::create("TenInfoRep",
lib::TGDefIDs::SimpleLSService, *this))) {
LOG_WARN("failed to create thread", KR(ret));
} else if (OB_FAIL(ObTenantThreadHelper::start())) {
LOG_WARN("fail to start", KR(ret));
} else {
inited_ = true;
}
return ret;
}
void ObTenantInfoReportor::destroy()
{
ObTenantThreadHelper::destroy();
tenant_id_ = OB_INVALID_TENANT_ID;
inited_ = false;
}
void ObTenantInfoReportor::do_work()
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else {
int64_t idle_time_us = 100 * 1000L;
while (!has_set_stop()) {
ObCurTraceId::init(GCONF.self_addr_);
if (is_sys_tenant(tenant_id_)) {
idle_time_us = 3600 * 1000 * 1000L;
} else if (OB_FAIL(gather_tenant_recovery_stat_())) {
LOG_WARN("failed to gather tenant recovery stat", KR(ret), K(tenant_id_));
}
idle(idle_time_us);
}// end while
}
}
int ObTenantInfoReportor::gather_tenant_recovery_stat_()
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", KR(ret));
} else if (!is_meta_tenant(tenant_id_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("sys or user tenant no need gather tenant recovery stat", KR(ret), K(tenant_id_));
} else if (OB_ISNULL(GCTX.sql_proxy_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("sql proxy is null", KR(ret));
} else {
ObLSRecoveryStatOperator ls_recovery_op;
ObAllTenantInfoProxy info_proxy;
ObAllTenantInfo tenant_info;
const uint64_t user_tenant_id = gen_user_tenant_id(tenant_id_);
SCN sync_scn;
SCN readable_scn;
DEBUG_SYNC(BLOCK_TENANT_SYNC_SNAPSHOT_INC);
if (OB_FAIL(ls_recovery_op.get_tenant_recovery_stat(
user_tenant_id, *GCTX.sql_proxy_, sync_scn, readable_scn))) {
LOG_WARN("failed to get tenant recovery stat", KR(ret), K(user_tenant_id));
//TODO replayable_scn is equal to sync_scn
} else if (OB_FAIL(info_proxy.update_tenant_recovery_status(
user_tenant_id, GCTX.sql_proxy_, sync_scn,
sync_scn, readable_scn))) {
LOG_WARN("failed to update tenant recovery stat", KR(ret),
K(user_tenant_id), K(sync_scn), K(readable_scn));
}
}
return ret;
}
}
}

84
src/rootserver/ob_tenant_info_report.h Normal file
View File

@ -0,0 +1,84 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_ROOTSERVER_OB_TENANT_INFO_REPORT_H
#define OCEANBASE_ROOTSERVER_OB_TENANT_INFO_REPORT_H
#include "lib/thread/ob_reentrant_thread.h"//ObRsReentrantThread
#include "logservice/ob_log_base_type.h"
#include "share/scn.h"//SCN
#include "share/ob_thread_mgr.h" //OBTGDefIDEnum
#include "lib/thread/thread_mgr_interface.h" // TGRunnable
#include "lib/lock/ob_thread_cond.h"//ObThreadCond
#include "rootserver/ob_tenant_thread_helper.h"//ObTenantThreadHelper
namespace oceanbase
{
namespace obrpc
{
class ObSrvRpcProxy;
}
namespace common
{
class ObMySQLProxy;
class ObMySQLTransaction;
}
namespace share
{
class ObLSTableOperator;
class SCN;
}
namespace palf
{
struct PalfBaseInfo;
}
namespace rootserver
{
/*description:
* report __all_tenant_info
*/
class ObTenantInfoReportor : public ObTenantThreadHelper,
public logservice::ObICheckpointSubHandler,
public logservice::ObIReplaySubHandler
{
public:
ObTenantInfoReportor():inited_(false), tenant_id_(OB_INVALID_TENANT_ID) {}
virtual ~ObTenantInfoReportor() {}
static int mtl_init(ObTenantInfoReportor *&ka);
int init();
void destroy();
virtual void do_work() override;
public:
virtual share::SCN get_rec_scn() override { return share::SCN::max_scn();}
virtual int flush(share::SCN &scn) override { return OB_SUCCESS; }
int replay(const void *buffer, const int64_t nbytes, const palf::LSN &lsn, const share::SCN &scn) override
{
UNUSED(buffer);
UNUSED(nbytes);
UNUSED(lsn);
UNUSED(scn);
return OB_SUCCESS;
}
private:
int gather_tenant_recovery_stat_();
private:
bool inited_;
uint64_t tenant_id_;
};
}
}
#endif /* !OCEANBASE_ROOTSERVER_OB_TENANT_INFO_REPORT_H */

137
src/rootserver/ob_tenant_role_transition_service.cpp
View File

@ -309,8 +309,6 @@ int ObTenantRoleTransitionService::do_prepare_flashback_for_failover_to_primary_
} else if (OB_UNLIKELY(switchover_epoch_ != tenant_info.get_switchover_epoch())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("tenant switchover status not valid", KR(ret), K(tenant_info), K_(switchover_epoch));
} else if (OB_FAIL(update_tenant_stat_info_())) {
LOG_WARN("failed to update tenant stat info", KR(ret), K(tenant_info), K_(switchover_epoch));
} else if (OB_FAIL(OB_PRIMARY_STANDBY_SERVICE.do_recover_tenant(tenant_id_,
share::PREPARE_FLASHBACK_FOR_FAILOVER_TO_PRIMARY_SWITCHOVER_STATUS,
obrpc::ObRecoverTenantArg::RecoverType::CANCEL,
@ -338,15 +336,20 @@ int ObTenantRoleTransitionService::do_switch_access_mode_to_flashback(
const share::ObAllTenantInfo &tenant_info)
{
int ret = OB_SUCCESS;
ObLSStatusOperator ls_status_op;
share::ObLSStatusInfoArray all_ls_status_array;
if (OB_FAIL(check_inner_stat())) {
LOG_WARN("error unexpected", KR(ret), K(tenant_id_), KP(sql_proxy_), KP(rpc_proxy_));
} else if (OB_UNLIKELY(!tenant_info.is_flashback_status()
|| switchover_epoch_ != tenant_info.get_switchover_epoch())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("tenant switchover status not valid", KR(ret), K(tenant_info), K(switchover_epoch_));
} else if (OB_FAIL(change_ls_access_mode_(palf::AccessMode::FLASHBACK, SCN::base_scn()))) {
LOG_WARN("failed to get access mode", KR(ret), K(tenant_info));
} else if (OB_FAIL(ls_status_op.get_all_ls_status_by_order_for_switch_tenant(tenant_id_,
false/* ignore_need_create_abort */, all_ls_status_array, *sql_proxy_))) {
LOG_WARN("failed to get_all_ls_status_by_order", KR(ret), K(tenant_id_));
} else if (OB_FAIL(change_ls_access_mode_(all_ls_status_array,
palf::AccessMode::FLASHBACK, SCN::base_scn()))) {
LOG_WARN("failed to get access mode", KR(ret), K(tenant_info), K(all_ls_status_array));
}
return ret;
}
@ -400,7 +403,10 @@ int ObTenantRoleTransitionService::do_switch_access_mode_to_append(
{
int ret = OB_SUCCESS;
palf::AccessMode access_mode = logservice::ObLogService::get_palf_access_mode(target_tenant_role);
const SCN &ref_scn = tenant_info.get_sync_scn();
SCN ref_scn;
ObLSRecoveryStatOperator ls_recovery_op;
ObLSStatusOperator status_op;
ObLSStatusInfoArray status_info_array;
if (OB_FAIL(check_inner_stat())) {
LOG_WARN("error unexpected", KR(ret), K(tenant_id_), KP(sql_proxy_), KP(rpc_proxy_));
} else if (OB_UNLIKELY(!tenant_info.is_switching_to_primary_status()
@ -410,8 +416,25 @@ int ObTenantRoleTransitionService::do_switch_access_mode_to_append(
LOG_WARN("tenant switchover status not valid", KR(ret), K(tenant_info),
K(target_tenant_role), K(switchover_epoch_));
//TODO(yaoying):xianming
} else if (OB_FAIL(change_ls_access_mode_(access_mode, ref_scn))) {
LOG_WARN("failed to get access mode", KR(ret), K(access_mode), K(ref_scn), K(tenant_info));
} else if (OB_ISNULL(sql_proxy_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("sql proxy is null", KR(ret));
} else if (OB_FAIL(ls_recovery_op.get_tenant_max_sync_scn(tenant_id_, *sql_proxy_, ref_scn))) {
LOG_WARN("failed to get tenant max sync scn", KR(ret), K(tenant_id_));
} else {
// After flashback, it is necessary to ensure that the newly written log
// must be greater than all the sync_scn that have been reported, so ref_ts
// will be set as the largest sync_scn of all ls. Since sys_recovery_scn is
// reported independently, the maximum value of these two values should be taken .
ref_scn = SCN::max(tenant_info.get_sys_recovery_scn(), ref_scn);
}
if (FAILEDx(status_op.get_all_ls_status_by_order_for_switch_tenant(
tenant_id_, false/*ignore need create abort*/, status_info_array, *sql_proxy_))) {
LOG_WARN("failed to get all ls status info", KR(ret), K(tenant_id_));
} else if (OB_FAIL(change_ls_access_mode_(status_info_array, access_mode, ref_scn))) {
LOG_WARN("failed to get access mode", KR(ret), K(access_mode), K(ref_scn),
K(tenant_info), K(status_info_array));
} else {
common::ObMySQLTransaction trans;
share::ObAllTenantInfo cur_tenant_info;
@ -448,12 +471,14 @@ int ObTenantRoleTransitionService::do_switch_access_mode_to_append(
return ret;
}
int ObTenantRoleTransitionService::do_switch_access_mode_to_raw_rw(
const share::ObAllTenantInfo &tenant_info)
int ObTenantRoleTransitionService::switchover_to_standby(const share::ObAllTenantInfo &tenant_info)
{
int ret = OB_SUCCESS;
palf::AccessMode access_mode = logservice::ObLogService::get_palf_access_mode(STANDBY_TENANT_ROLE);
ObLSStatusOperator status_op;
ObLSStatusInfoArray status_info_array;
share::ObLSStatusInfo sys_ls;
if (OB_FAIL(check_inner_stat())) {
LOG_WARN("error unexpected", KR(ret), K(tenant_id_), KP(sql_proxy_), KP(rpc_proxy_));
@ -463,30 +488,69 @@ int ObTenantRoleTransitionService::do_switch_access_mode_to_raw_rw(
ret = OB_INVALID_ARGUMENT;
LOG_WARN("tenant switchover status not valid", KR(ret), K(tenant_info),
K(switchover_epoch_));
} else if (OB_FAIL(status_op.get_ls_status_info(tenant_id_, SYS_LS, sys_ls, *sql_proxy_))) {
LOG_WARN("failed to get sys ls status info", KR(ret), K(tenant_id_));
} else if (OB_FAIL(status_info_array.push_back(sys_ls))) {
LOG_WARN("failed to push back", KR(ret), K(sys_ls));
} else if (OB_FAIL(change_ls_access_mode_(status_info_array, access_mode, SCN::base_scn()))) {
LOG_WARN("failed to switch sys ls access mode", KR(ret), K(status_info_array), K(access_mode));
} else if (OB_FAIL(wait_sys_recovery_finish_(status_info_array))) {
LOG_WARN("faild to wait sys recovery finish", KR(ret), K(status_info_array));
} else if (OB_FAIL(status_op.create_abort_ls_in_switch_tenant(
tenant_id_, tenant_info.get_switchover_status(),
tenant_info.get_switchover_epoch(), *sql_proxy_))) {
LOG_WARN("failed to create abort ls", KR(ret), K(tenant_info));
} else if (OB_FAIL(change_ls_access_mode_(access_mode, SCN::base_scn()))) {
} else if (OB_FAIL(status_op.get_all_ls_status_by_order_for_switch_tenant(
tenant_id_, false/*ignore need create abort*/, status_info_array, *sql_proxy_))) {
LOG_WARN("failed to get all ls status info", KR(ret), K(tenant_id_));
} else if (OB_FAIL(change_ls_access_mode_(status_info_array, access_mode, SCN::base_scn()))) {
LOG_WARN("failed to get access mode", KR(ret), K(access_mode), K(tenant_info));
}
return ret;
}
int ObTenantRoleTransitionService::update_tenant_stat_info_()
int ObTenantRoleTransitionService::wait_sys_recovery_finish_(const ObLSStatusInfoIArray &ls_status_array)
{
int ret = OB_SUCCESS;
ObArray<obrpc::ObCheckpoint> checkpoints;
if (OB_FAIL(check_inner_stat())) {
LOG_WARN("error unexpected", KR(ret), K(tenant_id_), KP(sql_proxy_), KP(rpc_proxy_));
} else if (OB_FAIL(get_checkpoints_by_rpc_(tenant_id_, ls_status_array, false, checkpoints))) {
LOG_WARN("failed to get checkpoints by rpc", KR(ret), K(tenant_id_), K(ls_status_array));
} else if (OB_UNLIKELY(1 != checkpoints.count())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("expectect only one checkpoint", KR(ret), K(ls_status_array));
} else {
//TODO, get all ls sync_scn, update __all_tenant_info,
//the new sync_scn cannot larger than recovery_scn and sync_scn of sys_ls
//wait sys recovery scn to latest
ObAllTenantInfo curr_tenant_info;
bool sync_to_latest = false;
const SCN sys_scn = checkpoints.at(0).cur_sync_scn_;
while (!THIS_WORKER.is_timeout() && !sync_to_latest && OB_SUCC(ret)) {
//ignore error
if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(tenant_id_, sql_proxy_, false, curr_tenant_info))) {
LOG_WARN("failed to load tenant info", KR(ret), K(tenant_id_));
} else if (curr_tenant_info.get_sys_recovery_scn() >= sys_scn) {
sync_to_latest = true;
} else {
LOG_WARN("sys ls not recovery finish, need wait", K(tenant_id_), K(sys_scn), K(curr_tenant_info));
usleep(10L * 1000L);
}
}
if (THIS_WORKER.is_timeout() || !sync_to_latest) {
// return NOT_ALLOW instead of timeout
ret = OB_OP_NOT_ALLOW;
LOG_WARN("wait sys ls recovery finish timeout, can not swithover to standby", KR(ret));
LOG_USER_ERROR(OB_OP_NOT_ALLOW, "wait sys ls recovery timeout, switchover to standby");
} else if (OB_SUCC(ret)) {
LOG_INFO("finish check sync to latest");
}
}
LOG_INFO("[ROLE_TRANSITION] finish update tenant stat info", KR(ret), K(tenant_id_));
return ret;
}
int ObTenantRoleTransitionService::change_ls_access_mode_(palf::AccessMode target_access_mode,
int ObTenantRoleTransitionService::change_ls_access_mode_(const share::ObLSStatusInfoIArray &status_info_array,
palf::AccessMode target_access_mode,
const SCN &ref_scn)
{
int ret = OB_SUCCESS;
@ -513,8 +577,11 @@ int ObTenantRoleTransitionService::change_ls_access_mode_(palf::AccessMode targe
need_retry = false;
ret = OB_TIMEOUT;
LOG_WARN("already timeout", KR(ret));
} else if (OB_FAIL(get_ls_access_mode_(ls_mode_info))) {
} else if (OB_FAIL(get_ls_access_mode_(status_info_array, ls_mode_info))) {
LOG_WARN("failed to get ls access mode", KR(ret));
} else if (OB_UNLIKELY(status_info_array.count() != ls_mode_info.count())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ls count not equal to ls mode info count", KR(ret), K(status_info_array), K(ls_mode_info));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < ls_mode_info.count(); ++i) {
const LSAccessModeInfo &info = ls_mode_info.at(i);
@ -559,7 +626,8 @@ int ObTenantRoleTransitionService::change_ls_access_mode_(palf::AccessMode targe
}
int ObTenantRoleTransitionService::get_ls_access_mode_(ObIArray<LSAccessModeInfo> &ls_access_info)
int ObTenantRoleTransitionService::get_ls_access_mode_(const share::ObLSStatusInfoIArray &status_info_array,
ObIArray<LSAccessModeInfo> &ls_access_info)
{
int ret = OB_SUCCESS;
ls_access_info.reset();
@ -569,14 +637,10 @@ int ObTenantRoleTransitionService::get_ls_access_mode_(ObIArray<LSAccessModeInfo
ret = OB_ERR_UNEXPECTED;
LOG_WARN("location service is null", KR(ret));
} else {
share::ObLSStatusInfoArray status_info_array;
ObLSStatusOperator status_op;
ObTimeoutCtx ctx;
if (OB_FAIL(ObShareUtil::set_default_timeout_ctx(ctx, GCONF.rpc_timeout))) {
LOG_WARN("fail to set timeout ctx", KR(ret));
} else if (OB_FAIL(status_op.get_all_ls_status_by_order_for_switch_tenant(tenant_id_,
false/* ignore_need_create_abort */, status_info_array, *sql_proxy_))) {
LOG_WARN("fail to get_all_ls_status_by_order", KR(ret), K_(tenant_id), KP(sql_proxy_));
} else {
ObAddr leader;
ObGetLSAccessModeProxy proxy(
@ -800,6 +864,7 @@ int ObTenantRoleTransitionService::switchover_update_tenant_status(
const SCN final_sync_scn = MAX(max_checkpoint_scn, gts_upper_limit);
const SCN final_replayable_scn = final_sync_scn;
const SCN final_sys_recovery_scn = SCN::max(final_sync_scn, max_sys_ls_sync_scn);
SCN final_readable_scn = SCN::min_scn();
SCN final_recovery_until_scn = SCN::min_scn();
if (OB_FAIL(trans.start(sql_proxy_, exec_tenant_id))) {
@ -817,8 +882,8 @@ int ObTenantRoleTransitionService::switchover_update_tenant_status(
// switch_to_primary
// Does not change STS
final_readable_scn = tmp_tenant_info.get_standby_scn();
// To prevent unexpected sync log, set recovery_until_scn = sync_scn
final_recovery_until_scn = final_sync_scn;
// To prevent unexpected sync log or recovery sys ls log, set recovery_until_scn = max(sync_scn, sys_recovery_scn)
final_recovery_until_scn = MAX(final_sync_scn, final_sys_recovery_scn);
} else {
// switch_to_standby
// STS >= GTS
@ -845,10 +910,11 @@ int ObTenantRoleTransitionService::switchover_update_tenant_status(
final_replayable_scn,
final_readable_scn,
final_recovery_until_scn,
final_sys_recovery_scn,
old_switchover_epoch))) {
LOG_WARN("failed to update_tenant_status", KR(ret), K(tenant_id), K(new_role),
K(old_status), K(new_status), K(final_sync_scn), K(final_replayable_scn),
K(final_readable_scn), K(final_recovery_until_scn), K(old_switchover_epoch));
K(final_readable_scn), K(final_recovery_until_scn), K(old_switchover_epoch), K(final_sys_recovery_scn));
} else if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(
tenant_id, &trans, false /* for update */, new_tenant_info))) {
LOG_WARN("failed to load tenant info", KR(ret), K(tenant_id));
@ -898,14 +964,17 @@ int ObTenantRoleTransitionService::wait_tenant_sync_to_latest_until_timeout_(
if (OB_FAIL(ret) || !has_restore_source) {
} else {
bool has_sync_to_latest = false;
while (!THIS_WORKER.is_timeout()) {
has_sync_to_latest = false;
if (OB_FAIL(check_sync_to_latest_(tenant_id, tenant_info, has_sync_to_latest))) {
ObAllTenantInfo new_tenant_info;
while (!THIS_WORKER.is_timeout() && !has_sync_to_latest) {
//ignore error
if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(tenant_id_, sql_proxy_,
false, new_tenant_info))) {
LOG_WARN("failed to load tenant info", KR(ret), K(tenant_id_));
} else if (OB_FAIL(check_sync_to_latest_(tenant_id, new_tenant_info, has_sync_to_latest))) {
LOG_WARN("fail to check_sync_to_latest_", KR(ret), K(tenant_id),
K(tenant_info), K(has_sync_to_latest));
K(new_tenant_info), K(has_sync_to_latest));
} else if (has_sync_to_latest) {
LOG_INFO("sync to latest", K(has_sync_to_latest), K(tenant_id));
break;
} else {
LOG_WARN("not sync to latest, wait a while", K(tenant_id));
}
@ -944,7 +1013,6 @@ int ObTenantRoleTransitionService::check_sync_to_latest_(const uint64_t tenant_i
share::ObLSStatusInfoArray sys_ls_status_array;
common::ObArray<obrpc::ObCheckpoint> switchover_checkpoints;
ObLSRecoveryStatOperator ls_recovery_operator;
ObLSRecoveryStat sys_ls_recovery_stat;
SCN sys_ls_sync_scn = SCN::min_scn();
bool sys_ls_sync_to_latest = false;
share::ObLSStatusInfo ls_status;
@ -967,15 +1035,12 @@ int ObTenantRoleTransitionService::check_sync_to_latest_(const uint64_t tenant_i
ret = OB_ERR_UNEXPECTED;
LOG_WARN("checkpoints count is not 1", KR(ret), K(switchover_checkpoints), K(tenant_id),
K(tenant_info), K(sys_ls_status_array));
} else if (OB_FAIL(ls_recovery_operator.get_ls_recovery_stat(tenant_id, SYS_LS,
false/*for_update*/, sys_ls_recovery_stat, *sql_proxy_))) {
LOG_WARN("failed to get ls recovery stat", KR(ret), K(tenant_id));
} else if (OB_FAIL(get_sys_ls_sync_scn_(switchover_checkpoints, sys_ls_sync_scn, sys_ls_sync_to_latest))) {
LOG_WARN("failed to get_sys_ls_sync_scn_", KR(ret), K(switchover_checkpoints));
} else if (!(sys_ls_sync_scn.is_valid_and_not_min() && sys_ls_sync_to_latest
&& sys_ls_recovery_stat.get_sync_scn() == sys_ls_sync_scn)) {
&& tenant_info.get_sys_recovery_scn() == sys_ls_sync_scn)) {
LOG_WARN("sys ls not sync, keep waiting", KR(ret), K(sys_ls_sync_scn), K(sys_ls_sync_to_latest),
K(sys_ls_recovery_stat), K(switchover_checkpoints));
K(tenant_info), K(switchover_checkpoints));
// SYS LS is sync, check other LS
} else if (OB_FAIL(ls_status_op.get_all_ls_status_by_order_for_switch_tenant(tenant_id,
true/* ignore_need_create_abort */, all_ls_status_array, *sql_proxy_))) {

11
src/rootserver/ob_tenant_role_transition_service.h
View File

@ -97,10 +97,10 @@ public:
switch_optype_(switch_optype) {}
virtual ~ObTenantRoleTransitionService() {}
int failover_to_primary();
int switchover_to_standby(const share::ObAllTenantInfo &tenant_info);
int check_inner_stat();
int do_switch_access_mode_to_append(const share::ObAllTenantInfo &tenant_info,
const share::ObTenantRole &target_tenant_role);
int do_switch_access_mode_to_raw_rw(const share::ObAllTenantInfo &tenant_info);
void set_switchover_epoch(const int64_t switchover_epoch)
{
switchover_epoch_ = switchover_epoch;
@ -156,10 +156,11 @@ private:
int do_failover_to_primary_(const share::ObAllTenantInfo &tenant_info);
int do_prepare_flashback_(share::ObAllTenantInfo &tenant_info);
int do_flashback_(const share::ObAllTenantInfo &tenant_info);
int change_ls_access_mode_(palf::AccessMode target_access_mode,
int change_ls_access_mode_(const share::ObLSStatusInfoIArray &status_info_array,
palf::AccessMode target_access_mode,
const share::SCN &ref_scn);
int update_tenant_stat_info_();
int get_ls_access_mode_(ObIArray<LSAccessModeInfo> &ls_access_info);
int get_ls_access_mode_(const share::ObLSStatusInfoIArray &status_info_array,
ObIArray<LSAccessModeInfo> &ls_access_info);
int do_change_ls_access_mode_(const ObIArray<LSAccessModeInfo> &ls_access_info,
palf::AccessMode target_access_mode,
const share::SCN &ref_scn);
@ -219,6 +220,8 @@ private:
int do_prepare_flashback_for_switch_to_primary_(share::ObAllTenantInfo &tenant_info);
int do_prepare_flashback_for_failover_to_primary_(share::ObAllTenantInfo &tenant_info);
//switchover
int wait_sys_recovery_finish_(const ObLSStatusInfoIArray &ls_status);
private:
const static int64_t SEC_UNIT = 1000L * 1000L;
const static int64_t PRINT_INTERVAL = 10 * 1000 * 1000L;

305
src/rootserver/ob_tenant_thread_helper.cpp Executable file
View File

@ -0,0 +1,305 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX RS
#include "ob_tenant_thread_helper.h"
#include "lib/profile/ob_trace_id.h"
#include "lib/thread/thread_mgr.h"//TG
#include "share/ob_errno.h"
#include "share/schema/ob_schema_struct.h"//ObTenantInfo
#include "share/schema/ob_schema_service.h"//ObMultiSchemaService
#include "share/schema/ob_schema_getter_guard.h"//ObSchemaGetterGuard
#include "share/ob_share_util.h"//ObShareUtil
#include "share/ob_tenant_info_proxy.h"//ObAllTenantInfo
#include "share/restore/ob_physical_restore_table_operator.h"//restore_job
#include "share/restore/ob_physical_restore_info.h"//restore_info
#include "share/ob_primary_zone_util.h"//get_ls_primary_zone_priority
#include "observer/ob_server_struct.h"//GCTX
namespace oceanbase
{
using namespace common;
using namespace share;
using namespace transaction;
using namespace palf;
using namespace lib;
namespace rootserver
{
//////////////ObTenantThreadHelper
int ObTenantThreadHelper::create(
const char* thread_name, int tg_def_id, ObTenantThreadHelper &tenant_thread)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(is_created_)) {
ret = OB_INIT_TWICE;
LOG_WARN("has inited", KR(ret));
} else if (OB_ISNULL(thread_name)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("thread name is null", KR(ret));
} else if (OB_FAIL(TG_CREATE_TENANT(tg_def_id, tg_id_))) {
LOG_ERROR("create tg failed", KR(ret));
} else if (OB_FAIL(TG_SET_RUNNABLE(tg_id_, *this))) {
LOG_ERROR("set thread runable fail", KR(ret));
} else if (OB_FAIL(thread_cond_.init(ObWaitEventIds::REENTRANT_THREAD_COND_WAIT))) {
LOG_WARN("fail to init cond, ", KR(ret));
} else {
thread_name_ = thread_name;
is_created_ = true;
is_first_time_to_start_ = true;
}
return ret;
}
int ObTenantThreadHelper::start()
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_created_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", KR(ret));
} else if (is_first_time_to_start_) {
if (OB_FAIL(TG_START(tg_id_))) {
LOG_WARN("fail ed to start at first time", KR(ret), K(tg_id_), K(thread_name_));
} else {
is_first_time_to_start_ = false;
}
} else if (OB_FAIL(TG_REENTRANT_LOGICAL_START(tg_id_))) {
LOG_WARN("failed to start", KR(ret));
}
LOG_INFO("[TENANT THREAD] thread start", KR(ret), K(tg_id_), K(thread_name_));
return ret;
}
void ObTenantThreadHelper::stop()
{
LOG_INFO("[TENANT THREAD] thread stop start", K(tg_id_), K(thread_name_));
if (-1 != tg_id_) {
TG_REENTRANT_LOGICAL_STOP(tg_id_);
}
LOG_INFO("[TENANT THREAD] thread stop finish", K(tg_id_), K(thread_name_));
}
void ObTenantThreadHelper::wait()
{
LOG_INFO("[TENANT THREAD] thread wait start", K(tg_id_), K(thread_name_));
if (-1 != tg_id_) {
TG_REENTRANT_LOGICAL_WAIT(tg_id_);
}
LOG_INFO("[TENANT THREAD] thread wait finish", K(tg_id_), K(thread_name_));
}
void ObTenantThreadHelper::destroy()
{
LOG_INFO("[TENANT THREAD] thread destory start", K(tg_id_), K(thread_name_));
if (-1 != tg_id_) {
TG_STOP(tg_id_);
{
ObThreadCondGuard guard(thread_cond_);
thread_cond_.broadcast();
}
TG_WAIT(tg_id_);
TG_DESTROY(tg_id_);
tg_id_ = -1;
}
is_created_ = false;
is_first_time_to_start_ = true;
LOG_INFO("[TENANT THREAD] thread destory finish", K(tg_id_), K(thread_name_));
}
void ObTenantThreadHelper::switch_to_follower_forcedly()
{
stop();
}
int ObTenantThreadHelper::switch_to_leader()
{
int ret = OB_SUCCESS;
LOG_INFO("[TENANT THREAD] thread start", K(tg_id_), K(thread_name_));
if (OB_FAIL(start())) {
LOG_WARN("failed to start thread", KR(ret));
} else {
ObThreadCondGuard guard(thread_cond_);
if (OB_FAIL(thread_cond_.broadcast())) {
LOG_WARN("failed to weakup thread cond", KR(ret));
}
}
LOG_INFO("[TENANT THREAD] thread start finish", K(tg_id_), K(thread_name_));
return ret;
}
int ObTenantThreadHelper::wait_tenant_data_version_ready_(
const uint64_t tenant_id, const uint64_t &data_version)
{
int ret = OB_SUCCESS;
bool is_ready = false;
uint64_t tenant_data_version = 0;
while (!is_ready && !has_set_stop()) {
ret = OB_SUCCESS;
if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, tenant_data_version))) {
LOG_WARN("failed to get min data version", KR(ret), K(tenant_id));
} else if (tenant_data_version < data_version) {
ret = OB_NEED_WAIT;
LOG_WARN("tenant version not target, need wait", KR(ret),
K(tenant_data_version), K(data_version));
} else {
is_ready = true;
}
if (!is_ready) {
idle(10 * 1000 * 1000);
}
}
if (has_set_stop()) {
LOG_WARN("thread has been stopped", K(is_ready), K(tenant_id));
ret = OB_IN_STOP_STATE;
}
return ret;
}
int ObTenantThreadHelper::wait_tenant_schema_and_version_ready_(
const uint64_t tenant_id, const uint64_t &data_version)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(GCTX.schema_service_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("schema ptr is null", KR(ret), KP(GCTX.schema_service_));
} else if (OB_FAIL(wait_tenant_data_version_ready_(tenant_id, data_version))) {
LOG_WARN("failed to wait tenant data version", KR(ret), K(tenant_id), K(data_version));
} else {
bool is_ready = false;
share::schema::ObTenantSchema tenant_schema;
while (!is_ready && !has_set_stop()) {
ret = OB_SUCCESS;
if (OB_FAIL(get_tenant_schema(tenant_id, tenant_schema))) {
LOG_WARN("failed to get tenant schema", KR(ret), K(tenant_id));
} else if (!tenant_schema.is_normal()) {
ret = OB_NEED_WAIT;
LOG_WARN("tenant schema not ready, no need tenant balance", KR(ret), K(tenant_schema));
} else {
is_ready = true;
}
if (!is_ready) {
idle(10 * 1000 *1000);
}
}
if (has_set_stop()) {
LOG_WARN("thread has been stopped", K(is_ready), K(tenant_id));
ret = OB_IN_STOP_STATE;
}
}
return ret;
}
void ObTenantThreadHelper::run1() {
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_created_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else {
lib::set_thread_name(thread_name_);
LOG_INFO("thread run", K(thread_name_));
do_work();
}
}
void ObTenantThreadHelper::idle(const int64_t idle_time_us)
{
ObThreadCondGuard guard(thread_cond_);
thread_cond_.wait_us(idle_time_us);
}
int ObTenantThreadHelper::get_tenant_schema(const uint64_t tenant_id,
share::schema::ObTenantSchema &tenant_schema)
{
int ret = OB_SUCCESS;
share::schema::ObSchemaGetterGuard schema_guard;
const share::schema::ObTenantSchema *cur_tenant_schema = NULL;
if (!is_valid_tenant_id(tenant_id)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("tenant id is invalid", KR(ret), K(tenant_id));
} else if (OB_ISNULL(GCTX.schema_service_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected error", KR(ret), KP(GCTX.schema_service_));
} else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(
OB_SYS_TENANT_ID, schema_guard))) {
LOG_WARN("fail to get schema guard", KR(ret));
} else if (OB_FAIL(schema_guard.get_tenant_info(tenant_id,
cur_tenant_schema))) {
LOG_WARN("failed to get tenant ids", KR(ret), K(tenant_id));
} else if (OB_ISNULL(cur_tenant_schema)) {
ret = OB_TENANT_NOT_EXIST;
LOG_WARN("tenant not exist", KR(ret), K(tenant_id));
} else if (OB_FAIL(tenant_schema.assign(*cur_tenant_schema))) {
LOG_WARN("failed to get cur tenant schema", KR(ret), KP(cur_tenant_schema));
}
return ret;
}
int ObTenantThreadHelper::check_can_do_recovery(const ObAllTenantInfo &tenant_info)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!tenant_info.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("tenant_info is invalid", KR(ret), K(tenant_info));
} else if (tenant_info.get_sys_recovery_scn().is_min()) {
//during upgrade, not expected
ret = OB_NEED_RETRY;
LOG_WARN("sys recovery scn can not be min now", KR(ret), K(tenant_info));
} else if (OB_ISNULL(GCTX.sql_proxy_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("sql proxy is null", KR(ret));
} else if (tenant_info.is_primary()) {
} else if (tenant_info.is_standby()) {
//standby can do recovery
} else if (tenant_info.is_restore()) {
//need to check success to create init ls
const uint64_t tenant_id = tenant_info.get_tenant_id();
share::ObPhysicalRestoreTableOperator restore_table_operator;
share::ObPhysicalRestoreJob job_info;
if (FAILEDx(restore_table_operator.init(GCTX.sql_proxy_, tenant_id))) {
LOG_WARN("fail to init restore table operator", KR(ret), K(tenant_id));
} else if (OB_FAIL(restore_table_operator.get_job_by_tenant_id(tenant_id,
job_info))) {
LOG_WARN("fail to get restore job", K(ret), K(tenant_id));
} else if (job_info.is_valid_status_to_recovery()) {
//can do recovery
} else {
ret = OB_NEED_WAIT;
LOG_WARN("restore tenant not valid to recovery", KR(ret), K(job_info));
}
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected tenant role", KR(ret), K(tenant_info));
}
return ret;
}
int ObTenantThreadHelper::get_zone_priority(const ObZone &primary_zone,
const share::schema::ObTenantSchema &tenant_schema,
ObSqlString &primary_zone_str)
{
int ret = OB_SUCCESS;
primary_zone_str.reset();
if (OB_UNLIKELY(!tenant_schema.is_valid() || primary_zone.is_empty())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KR(ret), K(primary_zone), K(tenant_schema));
} else if (OB_FAIL(ObPrimaryZoneUtil::get_ls_primary_zone_priority(primary_zone,
tenant_schema, primary_zone_str))) {
LOG_WARN("failed to get ls primary zone priority", KR(ret), K(primary_zone), K(tenant_schema));
}
LOG_DEBUG("get zone priority", KR(ret), K(primary_zone_str), K(tenant_schema));
return ret;
}
}//end of rootserver
}

101
src/rootserver/ob_tenant_thread_helper.h Normal file
View File

@ -0,0 +1,101 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_ROOTSERVER_OB_TENANT_THREAD_HELPER_H
#define OCEANBASE_ROOTSERVER_OB_TENANT_THREAD_HELPER_H
#include "lib/thread/ob_reentrant_thread.h"//ObRsReentrantThread
#include "logservice/ob_log_base_type.h"
#include "share/ob_thread_mgr.h" //OBTGDefIDEnum
#include "lib/thread/thread_mgr_interface.h" // TGRunnable
#include "lib/lock/ob_thread_cond.h"//ObThreadCond
#include "common/ob_zone.h"//ObZone
namespace oceanbase
{
namespace obrpc
{
class ObSrvRpcProxy;
}
namespace common
{
class ObSqlString;
}
namespace share
{
class ObAllTenantInfo;
namespace schema
{
class ObTenantSchema;
}
}
namespace logservice
{
class ObLogHandler;
}
namespace rootserver
{
class ObTenantThreadHelper : public lib::TGRunnable,
public logservice::ObIRoleChangeSubHandler
{
public:
ObTenantThreadHelper() : tg_id_(-1), thread_cond_(), is_created_(false), is_first_time_to_start_(true), thread_name_("") {}
virtual ~ObTenantThreadHelper() {}
virtual void do_work() = 0;
virtual void run1() override;
virtual void destroy();
int start();
void stop();
void wait();
int create(const char* thread_name, int tg_def_id, ObTenantThreadHelper &tenant_thread);
void idle(const int64_t idle_time_us);
public:
virtual void switch_to_follower_forcedly() override;
virtual int switch_to_leader() override;
virtual int switch_to_follower_gracefully() override
{
stop();
return OB_SUCCESS;
}
virtual int resume_leader() override
{
return OB_SUCCESS;
}
public:
static int get_tenant_schema(const uint64_t tenant_id,
share::schema::ObTenantSchema &tenant_schema);
static int check_can_do_recovery(const share::ObAllTenantInfo &tenant_info);
static int get_zone_priority(const ObZone &primary_zone,
const share::schema::ObTenantSchema &tenant_schema,
common::ObSqlString &primary_zone_str);
protected:
int wait_tenant_schema_and_version_ready_(
const uint64_t tenant_id, const uint64_t &data_version);
int wait_tenant_data_version_ready_(
const uint64_t tenant_id, const uint64_t &data_version);
int tg_id_;
private:
common::ObThreadCond thread_cond_;
bool is_created_;
bool is_first_time_to_start_;
const char* thread_name_;
};
}
}
#endif /* !OCEANBASE_ROOTSERVER_OB_TENANT_THREAD_HELPER_H */

41
src/rootserver/restore/ob_restore_scheduler.cpp
View File

@ -15,7 +15,8 @@
#include "ob_restore_scheduler.h"
#include "rootserver/ob_rs_event_history_table_operator.h"
#include "rootserver/ob_unit_manager.h"//convert_pool_name_lis
#include "rootserver/ob_tenant_role_transition_service.h"//failover_to_primary
#include "rootserver/ob_ls_service_helper.h"//create_new_ls_in_trans
#include "rootserver/ob_common_ls_service.h"//do_create_user_ls
#include "share/ob_schema_status_proxy.h"
#include "share/schema/ob_schema_utils.h"
#include "share/schema/ob_schema_mgr.h"
@ -976,19 +977,16 @@ int ObRestoreService::restore_init_ls(const share::ObPhysicalRestoreJob &job_inf
} else if (OB_FAIL(store.read_ls_attr_info(backup_ls_attr))) {
LOG_WARN("failed to read ls info", KR(ret));
} else {
const SCN &sync_scn = backup_ls_attr.backup_scn_;
const SCN readable_scn = SCN::base_scn();
ObLSRecoveryStatOperator ls_recovery;
ObLSRecoveryStat ls_recovery_stat;
LOG_INFO("start to create ls and set sync scn", K(sync_scn), K(backup_ls_attr));
if (OB_FAIL(ls_recovery_stat.init_only_recovery_stat(
tenant_id_, SYS_LS, sync_scn, readable_scn))) {
LOG_WARN("failed to init ls recovery stat", KR(ret), K(backup_ls_attr.backup_scn_),
K(sync_scn), K(readable_scn));
} else if (OB_FAIL(ls_recovery.update_ls_recovery_stat(ls_recovery_stat,
*sql_proxy_))) {
LOG_WARN("failed to update ls recovery stat", KR(ret),
K(ls_recovery_stat));
const SCN &sys_recovery_scn = backup_ls_attr.backup_scn_;
ObAllTenantInfo tenant_info;
LOG_INFO("start to create ls and set sys_recovery_scn", K(sys_recovery_scn), K(backup_ls_attr));
if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(tenant_id_, sql_proxy_, false, tenant_info))) {
LOG_WARN("failed to update tenant info", KR(ret), K(tenant_id_));
} else if (tenant_info.get_sys_recovery_scn() >= sys_recovery_scn) {
//no need update sys recovery scn
} else if (OB_FAIL(ObAllTenantInfoProxy::update_tenant_sys_recovery_scn(tenant_id_, sys_recovery_scn,
false, sql_proxy_))) {
LOG_WARN("failed to update tenant sys recovery scn", KR(ret), K(tenant_id_), K(sys_recovery_scn), K(tenant_info));
}
}
if (FAILEDx(create_all_ls_(*tenant_schema, backup_ls_attr.ls_attr_array_))) {
@ -1056,7 +1054,7 @@ int ObRestoreService::create_all_ls_(
LOG_INFO("[RESTORE] ls already exist", K(ls_info), K(tenant_id_));
} else if (OB_ENTRY_NOT_EXIST != ret) {
LOG_WARN("failed to get ls status info", KR(ret), K(tenant_id_), K(ls_info));
} else if (OB_FAIL(tenant_stat.create_new_ls_for_recovery(
} else if (OB_FAIL(tenant_stat.create_new_ls_in_trans(
ls_info.get_ls_id(), ls_info.get_ls_group_id(), ls_info.get_create_scn(),
trans))) {
LOG_WARN("failed to add new ls status info", KR(ret), K(ls_info));
@ -1096,12 +1094,10 @@ int ObRestoreService::wait_all_ls_created_(const share::schema::ObTenantSchema &
ObLSRecoveryStat recovery_stat;
ObLSRecoveryStatOperator ls_recovery_operator;
ObTenantLSInfo tenant_stat(sql_proxy_, &tenant_schema, tenant_id_,
srv_rpc_proxy_, GCTX.lst_operator_);
if (OB_FAIL(status_op.get_all_ls_status_by_order(tenant_id, ls_array,
*sql_proxy_))) {
LOG_WARN("failed to get all ls status", KR(ret), K(tenant_id));
}
}
for (int64_t i = 0; OB_SUCC(ret) && i < ls_array.count(); ++i) {
const ObLSStatusInfo &info = ls_array.at(i);
if (info.ls_is_creating()) {
@ -1113,10 +1109,11 @@ int ObRestoreService::wait_all_ls_created_(const share::schema::ObTenantSchema &
job_info, info.ls_id_, palf_base_info))) {
LOG_WARN("failed to get restore ls palf info", KR(ret), K(info),
K(job_info));
} else if (OB_FAIL(tenant_stat.create_ls_with_palf(info, recovery_stat.get_create_scn(),
true,/*create with palf*/
palf_base_info))) {
LOG_WARN("failed to create ls with palf", KR(ret), K(info),
} else if (OB_FAIL(ObCommonLSService::do_create_user_ls(
tenant_schema, info, recovery_stat.get_create_scn(),
true, /*create with palf*/
palf_base_info))) {
LOG_WARN("failed to create ls with palf", KR(ret), K(info), K(tenant_schema),
K(palf_base_info));
}
}

2
src/rootserver/restore/ob_restore_scheduler.h
View File

@ -15,7 +15,7 @@
#include "share/backup/ob_backup_info_mgr.h"
#include "rootserver/restore/ob_restore_util.h"
#include "rootserver/ob_primary_ls_service.h"//ObTenantThreadHelper
#include "rootserver/ob_tenant_thread_helper.h"//ObTenantThreadHelper
#include "share/backup/ob_backup_struct.h"
#include "share/ob_rpc_struct.h"
#include "share/ob_common_rpc_proxy.h"

19
src/share/inner_table/ob_inner_table_schema.12251_12300.cpp
View File

@ -1377,6 +1377,25 @@ int ObInnerTableSchema::all_virtual_tenant_info_schema(ObTableSchema &table_sche
log_mode_default,
log_mode_default); //default_value
}
if (OB_SUCC(ret)) {
ObObj sys_recovery_scn_default;
sys_recovery_scn_default.set_uint64(OB_MIN_SCN_TS_NS);
ADD_COLUMN_SCHEMA_T("sys_recovery_scn", //column_name
++column_id, //column_id
0, //rowkey_id
0, //index_id
0, //part_key_pos
ObUInt64Type, //column_type
CS_TYPE_INVALID, //column_collation_type
sizeof(uint64_t), //column_length
-1, //column_precision
-1, //column_scale
false, //is_nullable
false, //is_autoincrement
sys_recovery_scn_default,
sys_recovery_scn_default); //default_value
}
table_schema.set_index_using_type(USING_BTREE);
table_schema.set_row_store_type(ENCODING_ROW_STORE);
table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL);

15
src/share/inner_table/ob_inner_table_schema.15251_15300.cpp
View File

@ -4777,6 +4777,21 @@ int ObInnerTableSchema::all_virtual_tenant_info_ora_schema(ObTableSchema &table_
false, //is_nullable
false); //is_autoincrement
}
if (OB_SUCC(ret)) {
ADD_COLUMN_SCHEMA("SYS_RECOVERY_SCN", //column_name
++column_id, //column_id
0, //rowkey_id
0, //index_id
0, //part_key_pos
ObNumberType, //column_type
CS_TYPE_INVALID, //column_collation_type
38, //column_length
38, //column_precision
0, //column_scale
false, //is_nullable
false); //is_autoincrement
}
table_schema.set_index_using_type(USING_BTREE);
table_schema.set_row_store_type(ENCODING_ROW_STORE);
table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL);

2
src/share/inner_table/ob_inner_table_schema.21151_21200.cpp
View File

@ -410,7 +410,7 @@ int ObInnerTableSchema::dba_ob_tenants_schema(ObTableSchema &table_schema)
table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
if (OB_SUCC(ret)) {
if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT A.TENANT_ID, TENANT_NAME, (CASE WHEN A.TENANT_ID = 1 THEN 'SYS' WHEN (A.TENANT_ID & 0x1) = 1 THEN 'META' ELSE 'USER' END) AS TENANT_TYPE, A.gmt_create AS CREATE_TIME, A.gmt_modified AS MODIFY_TIME, PRIMARY_ZONE, LOCALITY, CASE previous_locality WHEN "" THEN NULL ELSE previous_locality END AS PREVIOUS_LOCALITY, CASE compatibility_mode WHEN 0 THEN 'MYSQL' WHEN 1 THEN 'ORACLE' ELSE NULL END AS COMPATIBILITY_MODE, STATUS, CASE in_recyclebin WHEN 0 THEN 'NO' ELSE 'YES' END AS IN_RECYCLEBIN, CASE locked WHEN 0 THEN 'NO' ELSE 'YES' END AS LOCKED, (CASE WHEN A.TENANT_ID = 1 THEN 'PRIMARY' WHEN (A.TENANT_ID & 0x1) = 1 THEN 'PRIMARY' ELSE TENANT_ROLE END) AS TENANT_ROLE, (CASE WHEN A.TENANT_ID = 1 THEN 'NORMAL' WHEN (A.TENANT_ID & 0x1) = 1 THEN 'NORMAL' ELSE SWITCHOVER_STATUS END) AS SWITCHOVER_STATUS, (CASE WHEN A.TENANT_ID = 1 THEN 0 WHEN (A.TENANT_ID & 0x1) = 1 THEN 0 ELSE SWITCHOVER_EPOCH END) AS SWITCHOVER_EPOCH, (CASE WHEN A.TENANT_ID = 1 THEN NULL WHEN (A.TENANT_ID & 0x1) = 1 THEN NULL ELSE SYNC_SCN END) AS SYNC_SCN, (CASE WHEN A.TENANT_ID = 1 THEN NULL WHEN (A.TENANT_ID & 0x1) = 1 THEN NULL ELSE REPLAYABLE_SCN END) AS REPLAYABLE_SCN, (CASE WHEN A.TENANT_ID = 1 THEN NULL WHEN (A.TENANT_ID & 0x1) = 1 THEN NULL ELSE READABLE_SCN END) AS READABLE_SCN, (CASE WHEN A.TENANT_ID = 1 THEN NULL WHEN (A.TENANT_ID & 0x1) = 1 THEN NULL ELSE RECOVERY_UNTIL_SCN END) AS RECOVERY_UNTIL_SCN, (CASE WHEN A.TENANT_ID = 1 THEN 'NOARCHIVELOG' WHEN (A.TENANT_ID & 0x1) = 1 THEN 'NOARCHIVELOG' ELSE LOG_MODE END) AS LOG_MODE, ARBITRATION_SERVICE_STATUS FROM OCEANBASE.__ALL_VIRTUAL_TENANT_MYSQL_SYS_AGENT AS A LEFT JOIN OCEANBASE.__ALL_VIRTUAL_TENANT_INFO AS B ON A.TENANT_ID = B.TENANT_ID )__"))) {
if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT A.TENANT_ID, TENANT_NAME, (CASE WHEN A.TENANT_ID = 1 THEN 'SYS' WHEN (A.TENANT_ID & 0x1) = 1 THEN 'META' ELSE 'USER' END) AS TENANT_TYPE, A.gmt_create AS CREATE_TIME, A.gmt_modified AS MODIFY_TIME, PRIMARY_ZONE, LOCALITY, CASE previous_locality WHEN "" THEN NULL ELSE previous_locality END AS PREVIOUS_LOCALITY, CASE compatibility_mode WHEN 0 THEN 'MYSQL' WHEN 1 THEN 'ORACLE' ELSE NULL END AS COMPATIBILITY_MODE, STATUS, CASE in_recyclebin WHEN 0 THEN 'NO' ELSE 'YES' END AS IN_RECYCLEBIN, CASE locked WHEN 0 THEN 'NO' ELSE 'YES' END AS LOCKED, (CASE WHEN A.TENANT_ID = 1 THEN 'PRIMARY' WHEN (A.TENANT_ID & 0x1) = 1 THEN 'PRIMARY' ELSE TENANT_ROLE END) AS TENANT_ROLE, (CASE WHEN A.TENANT_ID = 1 THEN 'NORMAL' WHEN (A.TENANT_ID & 0x1) = 1 THEN 'NORMAL' ELSE SWITCHOVER_STATUS END) AS SWITCHOVER_STATUS, (CASE WHEN A.TENANT_ID = 1 THEN 0 WHEN (A.TENANT_ID & 0x1) = 1 THEN 0 ELSE SWITCHOVER_EPOCH END) AS SWITCHOVER_EPOCH, (CASE WHEN A.TENANT_ID = 1 THEN NULL WHEN (A.TENANT_ID & 0x1) = 1 THEN NULL ELSE SYNC_SCN END) AS SYNC_SCN, (CASE WHEN A.TENANT_ID = 1 THEN NULL WHEN (A.TENANT_ID & 0x1) = 1 THEN NULL ELSE REPLAYABLE_SCN END) AS REPLAYABLE_SCN, (CASE WHEN A.TENANT_ID = 1 THEN NULL WHEN (A.TENANT_ID & 0x1) = 1 THEN NULL ELSE READABLE_SCN END) AS READABLE_SCN, (CASE WHEN A.TENANT_ID = 1 THEN NULL WHEN (A.TENANT_ID & 0x1) = 1 THEN NULL ELSE RECOVERY_UNTIL_SCN END) AS RECOVERY_UNTIL_SCN, (CASE WHEN A.TENANT_ID = 1 THEN NULL WHEN (A.TENANT_ID & 0x1) = 1 THEN NULL ELSE SYS_RECOVERY_SCN END) AS SYS_RECOVERY_SCN, (CASE WHEN A.TENANT_ID = 1 THEN 'NOARCHIVELOG' WHEN (A.TENANT_ID & 0x1) = 1 THEN 'NOARCHIVELOG' ELSE LOG_MODE END) AS LOG_MODE, ARBITRATION_SERVICE_STATUS FROM OCEANBASE.__ALL_VIRTUAL_TENANT_MYSQL_SYS_AGENT AS A LEFT JOIN OCEANBASE.__ALL_VIRTUAL_TENANT_INFO AS B ON A.TENANT_ID = B.TENANT_ID )__"))) {
LOG_ERROR("fail to set view_definition", K(ret));
}
}

19
src/share/inner_table/ob_inner_table_schema.351_400.cpp
View File

@ -5775,6 +5775,25 @@ int ObInnerTableSchema::all_tenant_info_schema(ObTableSchema &table_schema)
log_mode_default,
log_mode_default); //default_value
}
if (OB_SUCC(ret)) {
ObObj sys_recovery_scn_default;
sys_recovery_scn_default.set_uint64(OB_MIN_SCN_TS_NS);
ADD_COLUMN_SCHEMA_T("sys_recovery_scn", //column_name
++column_id, //column_id
0, //rowkey_id
0, //index_id
0, //part_key_pos
ObUInt64Type, //column_type
CS_TYPE_INVALID, //column_collation_type
sizeof(uint64_t), //column_length
-1, //column_precision
-1, //column_scale
false, //is_nullable
false, //is_autoincrement
sys_recovery_scn_default,
sys_recovery_scn_default); //default_value
}
table_schema.set_index_using_type(USING_BTREE);
table_schema.set_row_store_type(ENCODING_ROW_STORE);
table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL);

7
src/share/inner_table/ob_inner_table_schema_def.py
View File

@ -4049,6 +4049,7 @@ def_table_schema(
('readable_scn', 'uint'),
('recovery_until_scn', 'uint', 'false', 'OB_MAX_SCN_TS_NS'),
('log_mode', 'varchar:100', 'false', 'NOARCHIVELOG'),
("sys_recovery_scn", 'uint', 'false', 'OB_MIN_SCN_TS_NS'),
],
)
@ -15446,6 +15447,12 @@ SELECT A.TENANT_ID,
ELSE RECOVERY_UNTIL_SCN
END) AS RECOVERY_UNTIL_SCN,
(CASE
WHEN A.TENANT_ID = 1 THEN NULL
WHEN (A.TENANT_ID & 0x1) = 1 THEN NULL
ELSE SYS_RECOVERY_SCN
END) AS SYS_RECOVERY_SCN,
(CASE
WHEN A.TENANT_ID = 1 THEN 'NOARCHIVELOG'
WHEN (A.TENANT_ID & 0x1) = 1 THEN 'NOARCHIVELOG'

37
src/share/ls/ob_ls_i_life_manager.h
View File

@ -241,7 +241,44 @@ int ObLSTemplateOperator::exec_write(const uint64_t &tenant_id,
return ret;
}
#define DEFINE_IN_TRANS_FUC(func_name, ...)\
int func_name ##_in_trans(__VA_ARGS__, ObMySQLTransaction &trans);\
int func_name(__VA_ARGS__);
#define DEFINE_IN_TRANS_FUC1(func_name, ...)\
static int func_name ##_in_trans(__VA_ARGS__, ObMySQLTransaction &trans);\
static int func_name(__VA_ARGS__, ObISQLClient *proxy);
#define TAKE_IN_TRANS(func_name, proxy, exec_tenant_id, ...)\
do {\
ObMySQLTransaction trans; \
if (FAILEDx(trans.start(proxy, exec_tenant_id))) {\
SHARE_LOG(WARN, "failed to start trans", KR(ret), K(exec_tenant_id));\
} else if (OB_FAIL(func_name##_in_trans(__VA_ARGS__, trans))) {\
SHARE_LOG(WARN, "failed to do it in trans", KR(ret));\
}\
if (trans.is_started()) {\
int tmp_ret = OB_SUCCESS;\
if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) {\
SHARE_LOG(WARN, "failed to commit trans", KR(ret), KR(tmp_ret));\
ret = OB_SUCC(ret) ? tmp_ret : ret;\
}\
}\
} while (0)
#define START_TRANSACTION(proxy, exec_tenant_id) \
ObMySQLTransaction trans; \
if (FAILEDx(trans.start(proxy, exec_tenant_id))) { \
SHARE_LOG(WARN, "failed to start trans", KR(ret), K(exec_tenant_id)); \
}
#define END_TRANSACTION(trans)\
if (trans.is_started()) {\
int tmp_ret = OB_SUCCESS;\
if (OB_TMP_FAIL(trans.end(OB_SUCC(ret)))) {\
ret = OB_SUCC(ret) ? tmp_ret : ret;\
SHARE_LOG(WARN, "failed to end trans", KR(ret), K(tmp_ret));\
}\
}
}
}

49
src/share/ls/ob_ls_life_manager.cpp
View File

@ -34,21 +34,13 @@ int ObLSLifeAgentManager::create_new_ls(
if (OB_UNLIKELY(!ls_info.is_valid() || !create_ls_scn.is_valid() || zone_priority.empty())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KR(ret), K(ls_info), K(create_ls_scn), K(zone_priority));
} else if (OB_FAIL(trans.start(proxy_, exec_tenant_id))) {
LOG_WARN("failed to start trans", KR(ret), K(exec_tenant_id), K(ls_info));
} else if (OB_FAIL(create_new_ls_in_trans(ls_info, create_ls_scn, zone_priority, working_sw_status, trans))) {
LOG_WARN("failed to create new ls", KR(ret), K(ls_info), K(create_ls_scn), K(zone_priority));
} else {
TAKE_IN_TRANS(create_new_ls, proxy_,
exec_tenant_id, ls_info, create_ls_scn, zone_priority, working_sw_status);
}
if (trans.is_started()) {
int tmp_ret = OB_SUCCESS;
if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) {
LOG_WARN("failed to commit trans", KR(ret), KR(tmp_ret));
ret = OB_SUCC(ret) ? tmp_ret : ret;
}
}
return ret;
}
int ObLSLifeAgentManager::drop_ls(const uint64_t &tenant_id,
const share::ObLSID &ls_id,
const ObTenantSwitchoverStatus &working_sw_status)
@ -59,19 +51,10 @@ int ObLSLifeAgentManager::drop_ls(const uint64_t &tenant_id,
if (OB_UNLIKELY(!ls_id.is_valid() || OB_INVALID_TENANT_ID == tenant_id)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KR(ret), K(ls_id), K(tenant_id));
} else if (OB_FAIL(trans.start(proxy_, exec_tenant_id))) {
LOG_WARN("failed to start trans", KR(ret), K(exec_tenant_id), K(tenant_id));
} else if (OB_FAIL(drop_ls_in_trans(tenant_id, ls_id, working_sw_status, trans))) {
LOG_WARN("failed to create new ls", KR(ret), K(tenant_id), K(ls_id));
} else {
TAKE_IN_TRANS(drop_ls, proxy_, exec_tenant_id, tenant_id,
ls_id, working_sw_status);
}
if (trans.is_started()) {
int tmp_ret = OB_SUCCESS;
if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) {
LOG_WARN("failed to commit trans", KR(ret), KR(tmp_ret));
ret = OB_SUCC(ret) ? tmp_ret : ret;
}
}
return ret;
}
@ -90,22 +73,10 @@ int ObLSLifeAgentManager::set_ls_offline(const uint64_t &tenant_id,
|| (!ls_is_dropping_status(ls_status) && !ls_is_tenant_dropping_status(ls_status)))) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KR(ret), K(ls_id), K(tenant_id), K(drop_scn), K(ls_status));
} else if (OB_FAIL(trans.start(proxy_, exec_tenant_id))) {
LOG_WARN("failed to start trans", KR(ret), K(exec_tenant_id), K(tenant_id));
} else if (OB_FAIL(set_ls_offline_in_trans(tenant_id, ls_id, ls_status, drop_scn,
working_sw_status, trans))) {
LOG_WARN("failed to create new ls", KR(ret), K(tenant_id), K(ls_id),
K(ls_status), K(drop_scn));
} else {
TAKE_IN_TRANS(set_ls_offline, proxy_, exec_tenant_id, tenant_id,
ls_id, ls_status, drop_scn, working_sw_status);
}
if (trans.is_started()) {
int tmp_ret = OB_SUCCESS;
if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) {
LOG_WARN("failed to commit trans", KR(ret), KR(tmp_ret));
ret = OB_SUCC(ret) ? tmp_ret : ret;
}
}
return ret;
}

50
src/share/ls/ob_ls_life_manager.h
View File

@ -71,19 +71,19 @@ public:
* @param[in]zone_priority: the primary_zone of OB_ALL_LS_ELECTION_REFERENCE_INFO
* @param[in] working_sw_status only support working on specified switchover status
* */
int create_new_ls(const ObLSStatusInfo &ls_info,
const SCN &create_ls_scn,
const common::ObString &zone_priority,
const share::ObTenantSwitchoverStatus &working_sw_status);
DEFINE_IN_TRANS_FUC(create_new_ls, const ObLSStatusInfo &ls_info,\
const SCN &create_ls_scn,\
const common::ObString &zone_priority,\
const share::ObTenantSwitchoverStatus &working_sw_status)
/*
* description: for primary cluster and GC of standby, delete ls from each inner_table
* @param[in] tenant_id: tenant_id
* @param[in] ls_id: need delete ls
* @param[in] working_sw_status only support working on specified switchover status
* */
int drop_ls(const uint64_t &tenant_id,
const share::ObLSID &ls_id,
const ObTenantSwitchoverStatus &working_sw_status);
DEFINE_IN_TRANS_FUC(drop_ls, const uint64_t &tenant_id,\
const share::ObLSID &ls_id,\
const ObTenantSwitchoverStatus &working_sw_status)
/*
* description: for primary cluster set ls to wait offline from tenant_dropping or dropping status
* @param[in] tenant_id: tenant_id
@ -92,11 +92,11 @@ public:
* @param[in] drop_scn: there is no user data after drop_scn except offline
* @param[in] working_sw_status only support working on specified switchover status
* */
int set_ls_offline(const uint64_t &tenant_id,
const share::ObLSID &ls_id,
const ObLSStatus &ls_status,
const SCN &drop_scn,
const ObTenantSwitchoverStatus &working_sw_status);
DEFINE_IN_TRANS_FUC(set_ls_offline, const uint64_t &tenant_id,\
const share::ObLSID &ls_id,\
const ObLSStatus &ls_status,\
const SCN &drop_scn,\
const ObTenantSwitchoverStatus &working_sw_status)
/*
* description: update ls primary zone, need update __all_ls_status and __all_ls_election_reference
* @param[in] tenant_id: tenant_id
@ -110,32 +110,6 @@ public:
const common::ObZone &primary_zone,
const common::ObString &zone_priority);
public:
/*
* description: for standby cluster, create new ls
*/
int create_new_ls_in_trans(const ObLSStatusInfo &ls_info,
const SCN &create_ls_scn,
const common::ObString &zone_priority,
const share::ObTenantSwitchoverStatus &working_sw_status,
ObMySQLTransaction &trans);
/*
* description: for standby cluster, create new ls
*/
int drop_ls_in_trans(const uint64_t &tenant_id,
const share::ObLSID &ls_id,
const ObTenantSwitchoverStatus &working_sw_status,
ObMySQLTransaction &trans);
/*
* description: for standby cluster set ls to offline
* */
int set_ls_offline_in_trans(const uint64_t &tenant_id,
const share::ObLSID &ls_id,
const ObLSStatus &ls_status,
const SCN &drop_scn,
const ObTenantSwitchoverStatus &working_sw_status,
ObMySQLTransaction &trans);
private:
bool inited_;
//table_operator

66
src/share/ls/ob_ls_operator.cpp
View File

@ -116,17 +116,15 @@ bool ObLSAttrOperator::is_valid() const
int ObLSAttrOperator::operator_ls_(
const ObLSAttr &ls_attr, const common::ObSqlString &sql,
const uint64_t target_max_ls_group_id,
const ObTenantSwitchoverStatus &working_sw_status)
const uint64_t target_max_ls_group_id)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!ls_attr.is_valid()
|| sql.empty()
|| OB_INVALID_ID == target_max_ls_group_id
|| !working_sw_status.is_valid())) {
|| OB_INVALID_ID == target_max_ls_group_id)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("operation is invalid", KR(ret), K(ls_attr), K(sql),
K(target_max_ls_group_id), K(working_sw_status));
K(target_max_ls_group_id));
} else {
ObMySQLTransaction trans;
const bool for_update = true;
@ -146,10 +144,6 @@ int ObLSAttrOperator::operator_ls_(
LOG_WARN("failed to load sys ls status", KR(ret));
} else if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(tenant_id_, proxy_, false /* for_update */, tenant_info))) {
LOG_WARN("failed to load tenant info", KR(ret), K_(tenant_id));
} else if (working_sw_status != tenant_info.get_switchover_status()) {
ret = OB_NEED_RETRY;
LOG_WARN("tenant not in specified switchover status", K_(tenant_id), K(working_sw_status),
K(tenant_info));
} else if (OB_LS_OP_CREATE_PRE == ls_attr.get_ls_operation_type()) {
if (OB_LS_NORMAL != sys_ls_attr.get_ls_status()) {
//for sys ls, need insert_ls, but ls_status is normal
@ -196,8 +190,7 @@ int ObLSAttrOperator::operator_ls_(
int ObLSAttrOperator::insert_ls(
const ObLSAttr &ls_attr,
const uint64_t max_ls_group_id,
const ObTenantSwitchoverStatus &working_sw_status)
const uint64_t max_ls_group_id)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!ls_attr.is_valid() || OB_INVALID_ID == max_ls_group_id)) {
@ -215,7 +208,7 @@ int ObLSAttrOperator::insert_ls(
ObLSStatusOperator::ls_status_to_str(ls_attr.get_ls_status()), "",
ls_attr.get_create_scn().get_val_for_inner_table_field()))) {
LOG_WARN("failed to assign sql", KR(ret), K(ls_attr), K(sql));
} else if (OB_FAIL(operator_ls_(ls_attr, sql, max_ls_group_id, working_sw_status))) {
} else if (OB_FAIL(operator_ls_(ls_attr, sql, max_ls_group_id))) {
LOG_WARN("failed to operator ls", KR(ret), K(ls_attr), K(sql));
}
}
@ -224,8 +217,7 @@ int ObLSAttrOperator::insert_ls(
}
int ObLSAttrOperator::delete_ls(
const ObLSID &ls_id, const share::ObLSStatus &old_status,
const ObTenantSwitchoverStatus &working_sw_status)
const ObLSID &ls_id, const share::ObLSStatus &old_status)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!ls_id.is_valid()
@ -258,8 +250,7 @@ int ObLSAttrOperator::delete_ls(
ls_attr.get_ls_flag(), ls_attr.get_ls_status(),
operation_type, SCN::base_scn()))) {
LOG_WARN("failed to init new ls attr", KR(ret), K(ls_id), K(ls_attr), K(operation_type));
} else if (OB_FAIL(operator_ls_(new_ls_attr, sql, new_ls_attr.get_ls_group_id(),
working_sw_status))) {
} else if (OB_FAIL(operator_ls_(new_ls_attr, sql, new_ls_attr.get_ls_group_id()))) {
LOG_WARN("failed to operator ls", KR(ret), K(new_ls_attr), K(sql));
}
LOG_INFO("[LS_OPERATOR] delete ls", KR(ret), K(ls_id), K(old_status));
@ -323,8 +314,7 @@ int ObLSAttrOperator::process_sub_trans_(const ObLSAttr &ls_attr, ObMySQLTransac
int ObLSAttrOperator::update_ls_status(const ObLSID &id,
const share::ObLSStatus &old_status,
const share::ObLSStatus &new_status,
const ObTenantSwitchoverStatus &working_sw_status)
const share::ObLSStatus &new_status)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!id.is_valid()
@ -353,8 +343,7 @@ int ObLSAttrOperator::update_ls_status(const ObLSID &id,
operation_type, SCN::base_scn()))) {
LOG_WARN("failed to init new ls attr", KR(ret), K(id), K(ls_attr),
K(operation_type));
} else if (OB_FAIL(operator_ls_(new_ls_attr, sql, new_ls_attr.get_ls_group_id(),
working_sw_status))) {
} else if (OB_FAIL(operator_ls_(new_ls_attr, sql, new_ls_attr.get_ls_group_id()))) {
LOG_WARN("failed to operator ls", KR(ret), K(new_ls_attr), K(sql));
}
LOG_INFO("[LS_OPERATOR] update ls status", KR(ret), K(ls_attr), K(new_ls_attr));
@ -549,43 +538,6 @@ int ObLSAttrOperator::get_tenant_gts(const uint64_t &tenant_id, SCN &gts_scn)
return ret;
}
int ObLSAttrOperator::get_all_ls_by_order(const bool lock_sys_ls, ObLSAttrIArray &ls_operation_array)
{
int ret = OB_SUCCESS;
ls_operation_array.reset();
ObMySQLTransaction trans;
ObLSAttr sys_ls_attr;
if (OB_UNLIKELY(!is_valid())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("operation is not valid", KR(ret), "operation", *this);
} else if (OB_FAIL(trans.start(proxy_, tenant_id_))) {
LOG_WARN("failed to start transaction", KR(ret), K_(tenant_id));
/* to get accurate LS list need lock SYS_LS */
} else if (lock_sys_ls && OB_FAIL(get_ls_attr(SYS_LS, true /* for_update */, trans, sys_ls_attr))) {
LOG_WARN("failed to load sys ls status", KR(ret));
} else {
ObSqlString sql;
if (OB_FAIL(sql.assign_fmt(
"select * from %s order by ls_id",
OB_ALL_LS_TNAME))) {
LOG_WARN("failed to assign sql", KR(ret), K(sql));
} else if (OB_FAIL(exec_read(tenant_id_, sql, trans, this, ls_operation_array))) {
LOG_WARN("failed to construct ls attr", KR(ret), K(sql), K_(tenant_id));
}
}
if (trans.is_started()) {
int tmp_ret = OB_SUCCESS;
if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) {
LOG_WARN("failed to end trans", KR(ret), KR(tmp_ret));
ret = OB_SUCC(ret) ? tmp_ret : ret;
}
}
return ret;
}
}
}

26
src/share/ls/ob_ls_operator.h
View File

@ -105,10 +105,6 @@ struct ObLSAttr
{
return ls_is_normal_status(status_);
}
ObLSOperationType get_ls_operatin_type() const
{
return operation_type_;
}
ObLSID get_ls_id() const
{
return id_;
@ -172,24 +168,11 @@ public:
bool is_valid() const;
int get_all_ls_by_order(
ObLSAttrIArray &ls_array);
/**
* @description:
* get ls list from all_ls table,
* if want to get accurate LS list, set lock_sys_ls to true to lock SYS LS in __all_ls table
* to make sure mutual exclusion with load balancing thread
* @param[in] lock_sys_ls whether lock SYS LS in __all_ls table
* @param[out] ls_operation_array ls list
* @return return code
*/
int get_all_ls_by_order(const bool lock_sys_ls, ObLSAttrIArray &ls_operation_array);
int insert_ls(const ObLSAttr &ls_attr, const uint64_t max_ls_group_id,
const ObTenantSwitchoverStatus &working_sw_status);
int insert_ls(const ObLSAttr &ls_attr, const uint64_t max_ls_group_id);
//prevent the concurrency of create and drop ls
int delete_ls(const ObLSID &id,
const share::ObLSStatus &old_status,
const ObTenantSwitchoverStatus &working_sw_status);
int update_ls_status(const ObLSID &id, const share::ObLSStatus &old_status, const share::ObLSStatus &new_status,
const ObTenantSwitchoverStatus &working_sw_status);
const share::ObLSStatus &old_status);
int update_ls_status(const ObLSID &id, const share::ObLSStatus &old_status, const share::ObLSStatus &new_status);
static ObLSOperationType get_ls_operation_by_status(const ObLSStatus &ls_status);
int get_ls_attr(const ObLSID &id, const bool for_update, common::ObISQLClient &client, ObLSAttr &ls_attr);
/*
@ -202,8 +185,7 @@ public:
private:
int process_sub_trans_(const ObLSAttr &ls_attr, ObMySQLTransaction &trans);
int operator_ls_(const ObLSAttr &ls_attr, const common::ObSqlString &sql, const uint64_t max_ls_group_id,
const ObTenantSwitchoverStatus &working_sw_status);
int operator_ls_(const ObLSAttr &ls_attr, const common::ObSqlString &sql, const uint64_t max_ls_group_id);
private:
uint64_t tenant_id_;
common::ObMySQLProxy *proxy_;

16
src/share/ls/ob_ls_recovery_stat_operator.cpp
View File

@ -17,6 +17,7 @@
#include "share/ob_errno.h"
#include "share/config/ob_server_config.h"
#include "share/inner_table/ob_inner_table_schema.h"
#include "share/ob_tenant_info_proxy.h"//ObAllTenantInfo
#include "lib/string/ob_sql_string.h"//ObSqlString
#include "lib/mysqlclient/ob_mysql_transaction.h"//ObMySQLTransaction
#include "common/ob_timeout_ctx.h"
@ -219,22 +220,18 @@ int ObLSRecoveryStatOperator::update_ls_recovery_stat_in_trans(
true, old_ls_recovery, trans))) {
LOG_WARN("failed to get ls current recovery stat", KR(ret), K(ls_recovery));
} else if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(ls_recovery.get_tenant_id(), &trans,
true /* for update */, tenant_info))) {
true /* for update */, tenant_info))) {
LOG_WARN("failed to load tenant info", KR(ret), K(ls_recovery));
} else if (OB_UNLIKELY(!tenant_info.is_normal_status())) {
ret = OB_NEED_RETRY;
LOG_WARN("switchover status is not normal, do not update ls recovery", KR(ret),
K(ls_recovery), K(tenant_info));
} else if (!tenant_info.get_switchover_status().can_report_recovery_status()) {
ret = OB_OP_NOT_ALLOW;
LOG_WARN("status can not report recovery status", KR(ret), K(tenant_info));
} else {
const SCN sync_scn = ls_recovery.get_sync_scn() > old_ls_recovery.get_sync_scn() ?
ls_recovery.get_sync_scn() : old_ls_recovery.get_sync_scn();
const SCN &readable_scn = ls_recovery.get_readable_scn() > old_ls_recovery.get_readable_scn() ?
ls_recovery.get_readable_scn() : old_ls_recovery.get_readable_scn();
common::ObSqlString sql;
if (ls_recovery.get_ls_id() == share::SYS_LS && tenant_info.get_recovery_until_scn() < sync_scn) {
ret = OB_NEED_RETRY;
LOG_WARN("can not recovery bigger than recovery_until_scn", KR(ret), K(sync_scn), K(tenant_info));
} else if (OB_FAIL(sql.assign_fmt("UPDATE %s SET sync_scn = %lu, readable_scn = "
if (OB_FAIL(sql.assign_fmt("UPDATE %s SET sync_scn = %lu, readable_scn = "
"%lu where ls_id = %ld and tenant_id = %lu",
OB_ALL_LS_RECOVERY_STAT_TNAME, sync_scn.get_val_for_inner_table_field(),
readable_scn.get_val_for_inner_table_field(),
@ -246,6 +243,7 @@ int ObLSRecoveryStatOperator::update_ls_recovery_stat_in_trans(
}
return ret;
}
int ObLSRecoveryStatOperator::set_ls_offline(const uint64_t &tenant_id,
const share::ObLSID &ls_id,
const ObLSStatus &ls_status,

99
src/share/ob_primary_standby_service.cpp
View File

@ -19,7 +19,7 @@
#include "rootserver/ob_rs_event_history_table_operator.h" // ROOTSERVICE_EVENT_ADD
#include "rootserver/ob_tenant_role_transition_service.h" // ObTenantRoleTransitionService
#include "rootserver/ob_primary_ls_service.h"//ObTenantLSInfo
#include "share/ls/ob_ls_recovery_stat_operator.h"// ObLSRecoveryStatOperator
#include "share/restore/ob_log_restore_source_mgr.h" // ObLogRestoreSourceMgr
#include "share/ls/ob_ls_life_manager.h" //ObLSLifeAgentManager
#include "share/ls/ob_ls_operator.h" //ObLSAttr
#include "storage/tx/ob_timestamp_service.h" // ObTimestampService
@ -281,8 +281,6 @@ int ObPrimaryStandbyService::do_recover_tenant(
const uint64_t exec_tenant_id = gen_meta_tenant_id(tenant_id);
common::ObMySQLTransaction trans;
const ObSimpleTenantSchema *tenant_schema = nullptr;
ObLSRecoveryStatOperator ls_recovery_operator;
ObLSRecoveryStat sys_ls_recovery;
if (OB_FAIL(check_inner_stat_())) {
LOG_WARN("inner stat error", KR(ret), K_(inited));
} else if (!obrpc::ObRecoverTenantArg::is_valid(recover_type, recovery_until_scn)
@ -313,19 +311,16 @@ int ObPrimaryStandbyService::do_recover_tenant(
} else if (tenant_info.get_switchover_status() != working_sw_status) {
ret = OB_OP_NOT_ALLOW;
LOG_WARN("unexpected tenant switchover status", KR(ret), K(working_sw_status), K(tenant_info));
} else if (OB_FAIL(ls_recovery_operator.get_ls_recovery_stat(tenant_id, share::SYS_LS,
true /*for_update*/, sys_ls_recovery, trans))) {
LOG_WARN("failed to get ls recovery stat", KR(ret), K(tenant_id));
} else if (obrpc::ObRecoverTenantArg::RecoverType::UNTIL == recover_type
&& (recovery_until_scn < tenant_info.get_sync_scn()
|| recovery_until_scn < sys_ls_recovery.get_sync_scn())) {
|| recovery_until_scn < tenant_info.get_sys_recovery_scn())) {
ret = OB_OP_NOT_ALLOW;
LOG_WARN("recover before tenant sync_scn or SYS LS sync_scn is not allow", KR(ret), K(tenant_info),
K(tenant_id), K(recover_type), K(recovery_until_scn), K(sys_ls_recovery));
LOG_USER_ERROR(OB_OP_NOT_ALLOW, "recover before tenant sync_scn or SYS LS sync_scn is");
LOG_WARN("recover before tenant sync_scn or sys recovery scn is not allow", KR(ret), K(tenant_info),
K(tenant_id), K(recover_type), K(recovery_until_scn));
LOG_USER_ERROR(OB_OP_NOT_ALLOW, "recover before tenant sync_scn or sys recovery scn is");
} else if (tenant_schema->is_normal()) {
const SCN &recovery_until_scn_to_set = obrpc::ObRecoverTenantArg::RecoverType::UNTIL == recover_type ?
recovery_until_scn : SCN::max(tenant_info.get_sync_scn(), sys_ls_recovery.get_sync_scn());
recovery_until_scn : SCN::max(tenant_info.get_sync_scn(), tenant_info.get_sys_recovery_scn());
if (tenant_info.get_recovery_until_scn() == recovery_until_scn_to_set) {
LOG_WARN("recovery_until_scn is same with original", KR(ret), K(tenant_info), K(tenant_id),
K(recover_type), K(recovery_until_scn));
@ -423,23 +418,15 @@ int ObPrimaryStandbyService::switch_to_standby(
K(tenant_id));
}
}
case share::ObTenantSwitchoverStatus::PREPARE_SWITCHING_TO_STANDBY_STATUS: {
if (OB_FAIL(ret)) {
} else if (OB_FAIL(switch_to_standby_prepare_ls_status_(tenant_id,
tenant_info.get_switchover_status(),
tenant_info.get_switchover_epoch(),
tenant_info))) {
LOG_WARN("failed to switch_to_standby_prepare_ls_status_", KR(ret), K(tenant_id), K(tenant_info));
}
}
case share::ObTenantSwitchoverStatus::SWITCHING_TO_STANDBY_STATUS: {
if (OB_FAIL(ret)) {
} else {
//change sys ls asscess mode and wait sys_recovery_scn to latest
ObTenantRoleTransitionService role_transition_service(tenant_id, sql_proxy_, GCTX.srv_rpc_proxy_, switch_optype);
(void)role_transition_service.set_switchover_epoch(tenant_info.get_switchover_epoch());
if (OB_FAIL(role_transition_service.do_switch_access_mode_to_raw_rw(tenant_info))) {
LOG_WARN("failed to do_switch_access_mode", KR(ret), K(tenant_id), K(tenant_info));
if (OB_FAIL(role_transition_service.switchover_to_standby(tenant_info))) {
LOG_WARN("failed to switchover to standby", KR(ret), K(tenant_info));
} else if (OB_FAIL(role_transition_service.switchover_update_tenant_status(tenant_id,
false /* switch_to_standby */,
share::STANDBY_TENANT_ROLE,
@ -508,8 +495,8 @@ int ObPrimaryStandbyService::update_tenant_status_before_sw_to_standby_(
LOG_WARN("tenant not expect switchover epoch", KR(ret), K(tenant_info), K(cur_switchover_epoch));
} else if (OB_FAIL(ObAllTenantInfoProxy::update_tenant_role(
tenant_id, &trans, cur_switchover_epoch,
PRIMARY_TENANT_ROLE, cur_switchover_status,
share::PREP_SWITCHING_TO_STANDBY_SWITCHOVER_STATUS, new_switchover_ts))) {
STANDBY_TENANT_ROLE, cur_switchover_status,
share::SWITCHING_TO_STANDBY_SWITCHOVER_STATUS, new_switchover_ts))) {
LOG_WARN("failed to update tenant role", KR(ret), K(tenant_id), K(cur_switchover_epoch), K(tenant_info));
} else if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(
tenant_id, &trans, true, new_tenant_info))) {
@ -533,68 +520,6 @@ int ObPrimaryStandbyService::update_tenant_status_before_sw_to_standby_(
return ret;
}
int ObPrimaryStandbyService::switch_to_standby_prepare_ls_status_(
const uint64_t tenant_id,
const ObTenantSwitchoverStatus &status,
const int64_t switchover_epoch,
ObAllTenantInfo &new_tenant_info)
{
int ret = OB_SUCCESS;
ObMySQLTransaction trans;
ObLSAttr sys_ls_attr;
share::ObLSAttrOperator ls_operator(tenant_id, sql_proxy_);
share::schema::ObSchemaGetterGuard schema_guard;
const share::schema::ObTenantSchema *tenant_schema = NULL;
int64_t new_switchover_epoch = OB_INVALID_VERSION;
if (!is_user_tenant(tenant_id)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KR(ret), K(tenant_id));
} else if (OB_ISNULL(GCTX.schema_service_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("schema_service_ is NULL", KR(ret));
} else if (OB_FAIL(check_inner_stat_())) {
LOG_WARN("inner stat error", KR(ret), K_(inited));
} else if (OB_UNLIKELY(!status.is_prepare_switching_to_standby_status())) {
ret = OB_OP_NOT_ALLOW;
LOG_WARN("switchover status not match, switchover to standby not allow", KR(ret), K(status));
LOG_USER_ERROR(OB_OP_NOT_ALLOW, "switchover status not match, switchover to standby");
} else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID, schema_guard))) {
LOG_WARN("fail to get schema guard", KR(ret));
} else if (OB_FAIL(schema_guard.get_tenant_info(tenant_id, tenant_schema))) {
LOG_WARN("failed to get tenant ids", KR(ret), K(tenant_id));
} else if (OB_ISNULL(tenant_schema)) {
ret = OB_TENANT_NOT_EXIST;
LOG_WARN("tenant not exist", KR(ret), K(tenant_id));
} else {
ObTenantLSInfo tenant_stat(GCTX.sql_proxy_, tenant_schema, tenant_id,
GCTX.srv_rpc_proxy_, GCTX.lst_operator_);
/* lock SYS_LS to get accurate LS list, then fix ls status to make ls status consistency
between __all_ls&__all_ls_status.
Refer to ls operator, insert/update/delete of ls table are executed in the SYS_LS lock
and normal switchover status */
if (OB_FAIL(tenant_stat.process_ls_status_missmatch(true/* lock_sys_ls */,
share::PREP_SWITCHING_TO_STANDBY_SWITCHOVER_STATUS))) {
LOG_WARN("failed to process_ls_status_missmatch", KR(ret));
} else if (OB_FAIL(ObAllTenantInfoProxy::update_tenant_role(
tenant_id, sql_proxy_, switchover_epoch,
share::STANDBY_TENANT_ROLE, status,
share::SWITCHING_TO_STANDBY_SWITCHOVER_STATUS, new_switchover_epoch))) {
LOG_WARN("failed to update tenant role", KR(ret), K(tenant_id), K(switchover_epoch));
} else if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(
tenant_id, sql_proxy_, false, new_tenant_info))) {
LOG_WARN("failed to load tenant info", KR(ret), K(tenant_id));
} else if (OB_UNLIKELY(new_tenant_info.get_switchover_epoch() != new_switchover_epoch)) {
ret = OB_NEED_RETRY;
LOG_WARN("switchover is concurrency", KR(ret), K(switchover_epoch), K(new_tenant_info));
}
DEBUG_SYNC(SWITCHING_TO_STANDBY);
}
return ret;
}
int ObPrimaryStandbyService::write_upgrade_barrier_log(
ObMySQLTransaction &trans,
const uint64_t tenant_id,
@ -637,5 +562,5 @@ int ObPrimaryStandbyService::write_upgrade_barrier_log(
return ret;
}
}
}//end of share
}

15
src/share/ob_primary_standby_service.h
View File

@ -159,21 +159,6 @@ private:
const uint64_t tenant_id,
ObAllTenantInfo &new_tenant_info);
/**
* @description:
* when switch to standby, prepare ls_status in all_ls and all_ls_status to proper status
* @param[in] tenant_id the tenant id to check
* @param[in] status only prepare in specified switchover status
* @param[in] switchover_epoch only prepare in specified switchover epoch
* @param[out] new_tenant_info return the updated tenant_info
* @return return code
*/
int switch_to_standby_prepare_ls_status_(
const uint64_t tenant_id,
const ObTenantSwitchoverStatus &status,
const int64_t switchover_epoch,
ObAllTenantInfo &new_tenant_info);
private:
const static int64_t SEC_UNIT = 1000L * 1000L;
const static int64_t PRINT_INTERVAL = 10 * 1000 * 1000L;

202
src/share/ob_tenant_info_proxy.cpp
View File

@ -18,7 +18,6 @@
#include "share/config/ob_server_config.h"//GCONF
#include "share/inner_table/ob_inner_table_schema.h"//ALL_TENANT_INFO_TNAME
#include "share/ls/ob_ls_i_life_manager.h"//TODO SCN VALUE
#include "share/ls/ob_ls_recovery_stat_operator.h"//ObLSRecoveryStatOperator
#include "lib/string/ob_sql_string.h"//ObSqlString
#include "lib/mysqlclient/ob_mysql_transaction.h"//ObMySQLTrans
#include "common/ob_timeout_ctx.h"//ObTimeoutCtx
@ -37,14 +36,22 @@ bool is_valid_tenant_scn(
const SCN &sync_scn,
const SCN &replayable_scn,
const SCN &standby_scn,
const SCN &recovery_until_scn)
const SCN &recovery_until_scn,
const SCN &sys_recovery_scn)
{
return standby_scn <= replayable_scn && replayable_scn <= sync_scn && sync_scn <= recovery_until_scn;
return standby_scn <= replayable_scn && replayable_scn <= sync_scn && sync_scn <= recovery_until_scn
&& (sys_recovery_scn.is_min() || sync_scn <= sys_recovery_scn)
&& (sys_recovery_scn.is_min() || sys_recovery_scn <= recovery_until_scn);
}
SCN gen_new_sync_scn(const SCN &cur_sync_scn, const SCN &desired_sync_scn, const SCN &cur_recovery_until_scn)
SCN gen_new_sync_scn(const SCN &cur_sync_scn, const SCN &desired_sync_scn, const SCN &cur_recovery_until_scn,
const SCN &sys_recovery_scn)
{
return MIN(MAX(cur_sync_scn, desired_sync_scn), cur_recovery_until_scn);
SCN new_sync_scn = MIN(MAX(cur_sync_scn, desired_sync_scn), cur_recovery_until_scn);
if (!sys_recovery_scn.is_min()) {
new_sync_scn = MIN(new_sync_scn, sys_recovery_scn);
}
return new_sync_scn;
}
SCN gen_new_replayable_scn(const SCN &cur_replayable_scn, const SCN &desired_replayable_scn, const SCN &new_sync_scn)
@ -69,7 +76,8 @@ bool ObAllTenantInfo::is_valid() const
&& tenant_role_.is_valid()
&& switchover_status_.is_valid()
&& log_mode_.is_valid()
&& is_valid_tenant_scn(sync_scn_, replayable_scn_, standby_scn_, recovery_until_scn_);
&& sys_recovery_scn_.is_valid()
&& is_valid_tenant_scn(sync_scn_, replayable_scn_, standby_scn_, recovery_until_scn_, sys_recovery_scn_);
}
int ObAllTenantInfo::init(
@ -81,7 +89,8 @@ int ObAllTenantInfo::init(
const SCN &replayable_scn,
const SCN &standby_scn,
const SCN &recovery_until_scn,
const ObArchiveMode &log_mode)
const ObArchiveMode &log_mode,
const SCN &sys_recovery_scn)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id
@ -93,11 +102,13 @@ int ObAllTenantInfo::init(
|| !standby_scn.is_valid_and_not_min()
|| !recovery_until_scn.is_valid_and_not_min()
|| !log_mode.is_valid()
|| !is_valid_tenant_scn(sync_scn, replayable_scn, standby_scn, recovery_until_scn))) {
//sys recovery scn is valid and maybe min during upgrade
|| !sys_recovery_scn.is_valid()
|| !is_valid_tenant_scn(sync_scn, replayable_scn, standby_scn, recovery_until_scn, sys_recovery_scn))) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(tenant_role), K(switchover_status),
K(switchover_epoch), K(sync_scn), K(replayable_scn), K(standby_scn), K(recovery_until_scn),
K(log_mode));
K(log_mode), K(sys_recovery_scn));
} else {
tenant_id_ = tenant_id;
tenant_role_ = tenant_role;
@ -108,6 +119,7 @@ int ObAllTenantInfo::init(
standby_scn_ = standby_scn;
recovery_until_scn_ = recovery_until_scn;
log_mode_ = log_mode;
sys_recovery_scn_ = sys_recovery_scn;
}
return ret;
}
@ -125,6 +137,7 @@ void ObAllTenantInfo::assign(const ObAllTenantInfo &other)
standby_scn_ = other.standby_scn_;
recovery_until_scn_ = other.recovery_until_scn_;
log_mode_ = other.log_mode_;
sys_recovery_scn_ = other.sys_recovery_scn_;
}
return ;
}
@ -140,10 +153,11 @@ void ObAllTenantInfo::reset()
standby_scn_.set_min() ;
recovery_until_scn_.set_min();
log_mode_.reset();
sys_recovery_scn_.set_min();
}
OB_SERIALIZE_MEMBER(ObAllTenantInfo, tenant_id_, tenant_role_,
switchover_status_, switchover_epoch_, sync_scn_,
replayable_scn_, standby_scn_, recovery_until_scn_, log_mode_);
replayable_scn_, standby_scn_, recovery_until_scn_, log_mode_, sys_recovery_scn_);
ObAllTenantInfo& ObAllTenantInfo::operator= (const ObAllTenantInfo &other)
{
@ -178,8 +192,8 @@ int ObAllTenantInfoProxy::init_tenant_info(
} else if (OB_FAIL(sql.assign_fmt(
"insert into %s (tenant_id, tenant_role, "
"switchover_status, switchover_epoch, "
"sync_scn, replayable_scn, readable_scn, recovery_until_scn, log_mode) "
"values(%lu, '%s', '%s', %ld, %lu, %lu, %lu, %lu, '%s')",
"sync_scn, replayable_scn, readable_scn, recovery_until_scn, log_mode, sys_recovery_scn) "
"values(%lu, '%s', '%s', %ld, %lu, %lu, %lu, %lu, '%s', %lu)",
OB_ALL_TENANT_INFO_TNAME, tenant_info.get_tenant_id(),
tenant_info.get_tenant_role().to_str(),
tenant_info.get_switchover_status().to_str(),
@ -188,7 +202,8 @@ int ObAllTenantInfoProxy::init_tenant_info(
tenant_info.get_replayable_scn().get_val_for_inner_table_field(),
tenant_info.get_standby_scn().get_val_for_inner_table_field(),
tenant_info.get_recovery_until_scn().get_val_for_inner_table_field(),
tenant_info.get_log_mode().to_str()))) {
tenant_info.get_log_mode().to_str(),
tenant_info.get_sys_recovery_scn().get_val_for_inner_table_field()))) {
LOG_WARN("failed to assign sql", KR(ret), K(tenant_info), K(sql));
} else if (OB_FAIL(proxy->write(exec_tenant_id, sql.ptr(), affected_rows))) {
LOG_WARN("failed to execute sql", KR(ret), K(exec_tenant_id), K(sql));
@ -304,8 +319,7 @@ int ObAllTenantInfoProxy::load_tenant_info(const uint64_t tenant_id,
}
int ObAllTenantInfoProxy::update_tenant_recovery_status(
const uint64_t tenant_id, ObMySQLProxy *proxy,
ObTenantSwitchoverStatus status, const SCN &sync_scn,
const uint64_t tenant_id, ObMySQLProxy *proxy, const SCN &sync_scn,
const SCN &replay_scn, const SCN &readable_scn)
{
int ret = OB_SUCCESS;
@ -315,10 +329,9 @@ int ObAllTenantInfoProxy::update_tenant_recovery_status(
common::ObMySQLTransaction trans;
ObAllTenantInfo old_tenant_info;
ObTimeoutCtx ctx;
if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id ||
!status.is_valid())) {
if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("tenant_info is invalid", KR(ret), K(tenant_id), K(status));
LOG_WARN("tenant_id is invalid", KR(ret), K(tenant_id));
} else if (OB_ISNULL(proxy)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("proxy is null", KR(ret), KP(proxy));
@ -330,7 +343,8 @@ int ObAllTenantInfoProxy::update_tenant_recovery_status(
} else if (OB_FAIL(load_tenant_info(tenant_id, &trans, true, old_tenant_info))) {
LOG_WARN("failed to load all tenant info", KR(ret), K(tenant_id));
} else {
SCN new_sync_scn = gen_new_sync_scn(old_tenant_info.get_sync_scn(), sync_scn, old_tenant_info.get_recovery_until_scn());
SCN new_sync_scn = gen_new_sync_scn(old_tenant_info.get_sync_scn(), sync_scn,
old_tenant_info.get_recovery_until_scn(), old_tenant_info.get_sys_recovery_scn());
SCN new_replay_scn = gen_new_replayable_scn(old_tenant_info.get_replayable_scn(), replay_scn, new_sync_scn);
SCN new_scn = gen_new_standby_scn(old_tenant_info.get_standby_scn(), readable_scn, new_replay_scn);
@ -343,14 +357,14 @@ int ObAllTenantInfoProxy::update_tenant_recovery_status(
} else if (OB_FAIL(sql.assign_fmt(
"update %s set sync_scn = %ld, replayable_scn = %ld, "
"readable_scn = %ld where tenant_id = %lu "
"and switchover_status = '%s' and readable_scn <= replayable_scn and "
"replayable_scn <= sync_scn and sync_scn <= recovery_until_scn", OB_ALL_TENANT_INFO_TNAME,
"and readable_scn <= replayable_scn and "
"replayable_scn <= sync_scn and sync_scn <= recovery_until_scn and sync_scn <= %lu",
OB_ALL_TENANT_INFO_TNAME,
new_sync_scn.get_val_for_inner_table_field(),
new_replay_scn.get_val_for_inner_table_field(),
new_scn.get_val_for_inner_table_field(),
tenant_id, status.to_str()))) {
LOG_WARN("failed to assign sql", KR(ret), K(tenant_id), K(status),
K(sql));
tenant_id, new_sync_scn.get_val_for_inner_table_field()))) {
LOG_WARN("failed to assign sql", KR(ret), K(tenant_id), K(sql));
} else if (OB_FAIL(trans.write(exec_tenant_id, sql.ptr(), affected_rows))) {
LOG_WARN("failed to execute sql", KR(ret), K(exec_tenant_id), K(sql));
} else if (!is_single_row(affected_rows)) {
@ -384,12 +398,14 @@ int ObAllTenantInfoProxy::fill_cell(common::sqlclient::ObMySQLResult *result, Ob
uint64_t replay_scn_val = OB_INVALID_SCN_VAL;
uint64_t sts_scn_val = OB_INVALID_SCN_VAL;
uint64_t recovery_until_scn_val = OB_INVALID_SCN_VAL;
uint64_t sys_recovery_scn_val = OB_INVALID_SCN_VAL;
ObString log_mode_str;
ObString log_mode_default_value("NOARCHIVELOG");
SCN sync_scn;
SCN replay_scn;
SCN sts_scn;
SCN recovery_until_scn;
SCN sys_recovery_scn;
EXTRACT_VARCHAR_FIELD_MYSQL(*result, "tenant_role", tenant_role_str);
EXTRACT_VARCHAR_FIELD_MYSQL(*result, "switchover_status", status_str);
EXTRACT_INT_FIELD_MYSQL(*result, "tenant_id", tenant_id, uint64_t);
@ -398,6 +414,7 @@ int ObAllTenantInfoProxy::fill_cell(common::sqlclient::ObMySQLResult *result, Ob
EXTRACT_UINT_FIELD_MYSQL(*result, "replayable_scn", replay_scn_val, uint64_t);
EXTRACT_UINT_FIELD_MYSQL(*result, "readable_scn", sts_scn_val, uint64_t);
EXTRACT_UINT_FIELD_MYSQL_WITH_DEFAULT_VALUE(*result, "recovery_until_scn", recovery_until_scn_val, uint64_t, false /* skip_null_error */, true /* skip_column_error */, OB_MAX_SCN_TS_NS);
EXTRACT_UINT_FIELD_MYSQL_WITH_DEFAULT_VALUE(*result, "sys_recovery_scn", sys_recovery_scn_val, uint64_t, false /* skip_null_error */, true /* skip_column_error */, OB_MIN_SCN_TS_NS);
EXTRACT_VARCHAR_FIELD_MYSQL_WITH_DEFAULT_VALUE(*result, "log_mode", log_mode_str,
false /* skip_null_error */, true /* skip_column_error */, log_mode_default_value);
ObTenantRole tmp_tenant_role(tenant_role_str);
@ -414,13 +431,15 @@ int ObAllTenantInfoProxy::fill_cell(common::sqlclient::ObMySQLResult *result, Ob
LOG_WARN("failed to convert_for_inner_table_field", KR(ret), K(sts_scn_val));
} else if (OB_FAIL(recovery_until_scn.convert_for_inner_table_field(recovery_until_scn_val))) {
LOG_WARN("failed to convert_for_inner_table_field", KR(ret), K(recovery_until_scn_val));
} else if (OB_FAIL(sys_recovery_scn.convert_for_inner_table_field(sys_recovery_scn_val))) {
LOG_WARN("failed to conver for inner table field", KR(ret), K(sys_recovery_scn_val));
} else if (OB_FAIL(tenant_info.init(
tenant_id, tmp_tenant_role,
tmp_tenant_sw_status, switchover_epoch,
sync_scn, replay_scn, sts_scn, recovery_until_scn, tmp_log_mode))) {
sync_scn, replay_scn, sts_scn, recovery_until_scn, tmp_log_mode, sys_recovery_scn))) {
LOG_WARN("failed to init tenant info", KR(ret), K(tenant_id), K(tmp_tenant_role), K(tenant_role_str),
K(tmp_tenant_sw_status), K(status_str), K(switchover_epoch), K(sync_scn), K(recovery_until_scn),
K(log_mode_str), K(tmp_log_mode));
K(log_mode_str), K(tmp_log_mode), K(sys_recovery_scn));
}
}
return ret;
@ -551,8 +570,6 @@ int ObAllTenantInfoProxy::update_tenant_recovery_until_scn(
ObSqlString sql;
int64_t affected_rows = 0;
ObTimeoutCtx ctx;
ObLSRecoveryStatOperator ls_recovery_operator;
ObLSRecoveryStat sys_ls_recovery;
ObLogRestoreSourceMgr restore_source_mgr;
uint64_t compat_version = 0;
@ -568,27 +585,26 @@ int ObAllTenantInfoProxy::update_tenant_recovery_until_scn(
} else if (OB_UNLIKELY(!recovery_until_scn.is_valid_and_not_min())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("recovery_until_scn invalid", KR(ret), K(recovery_until_scn));
} else if (OB_FAIL(ls_recovery_operator.get_ls_recovery_stat(tenant_id, share::SYS_LS,
true /*for_update*/, sys_ls_recovery, trans))) {
LOG_WARN("failed to get ls recovery stat", KR(ret), K(tenant_id));
} else if (recovery_until_scn < sys_ls_recovery.get_sync_scn()) {
ret = OB_OP_NOT_ALLOW;
LOG_WARN("recover before SYS LS sync_scn is not allowed", KR(ret), K(tenant_id), K(recovery_until_scn), K(sys_ls_recovery));
LOG_USER_ERROR(OB_OP_NOT_ALLOW, "recover before SYS LS sync_scn is");
} else if (OB_FAIL(rootserver::ObRootUtils::get_rs_default_timeout_ctx(ctx))) {
LOG_WARN("fail to get timeout ctx", KR(ret), K(ctx));
} else if (OB_FAIL(sql.assign_fmt(
"update %s set recovery_until_scn = %lu where tenant_id = %lu and sync_scn <= %lu and switchover_epoch = %ld",
OB_ALL_TENANT_INFO_TNAME, recovery_until_scn.get_val_for_inner_table_field(), tenant_id,
recovery_until_scn.get_val_for_inner_table_field(), switchover_epoch))) {
"update %s set recovery_until_scn = %lu where tenant_id = %lu "
"and sync_scn <= %lu and switchover_epoch = %ld and sys_recovery_scn <= %lu",
OB_ALL_TENANT_INFO_TNAME,
recovery_until_scn.get_val_for_inner_table_field(), tenant_id,
recovery_until_scn.get_val_for_inner_table_field(),
switchover_epoch,
recovery_until_scn.get_val_for_inner_table_field()))) {
LOG_WARN("failed to assign sql", KR(ret), K(tenant_id), K(recovery_until_scn), K(sql));
} else if (OB_FAIL(trans.write(exec_tenant_id, sql.ptr(), affected_rows))) {
LOG_WARN("failed to execute sql", KR(ret), K(exec_tenant_id), K(sql));
} else if (0 == affected_rows) {
ret = OB_OP_NOT_ALLOW;
LOG_WARN("state changed, check sync_scn and switchover status", KR(ret), K(tenant_id),
LOG_WARN("state changed, check sync_scn, switchover status and sys_recovery_scn", KR(ret), K(tenant_id),
K(switchover_epoch), K(recovery_until_scn), K(sql));
LOG_USER_ERROR(OB_OP_NOT_ALLOW, "state changed, check sync_scn and switchover status, recover is");
LOG_USER_ERROR(OB_OP_NOT_ALLOW,
"state changed, check sync_scn, sys_recovery_scn and "
"switchover status, recover is");
} else if (!is_single_row(affected_rows)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("expect updating one row", KR(ret), K(affected_rows),
@ -601,10 +617,10 @@ int ObAllTenantInfoProxy::update_tenant_recovery_until_scn(
}
int64_t cost = ObTimeUtility::current_time() - begin_time;
LOG_INFO("update_recovery_until_scn finish", KR(ret), K(tenant_id), K(sys_ls_recovery),
LOG_INFO("update_recovery_until_scn finish", KR(ret), K(tenant_id),
K(recovery_until_scn), K(affected_rows), K(switchover_epoch), K(sql), K(cost));
ROOTSERVICE_EVENT_ADD("tenant_info", "update_recovery_until_scn", K(ret), K(tenant_id),
K(recovery_until_scn), K(affected_rows), K(switchover_epoch), K(sys_ls_recovery));
K(recovery_until_scn), K(affected_rows), K(switchover_epoch));
return ret;
}
@ -618,6 +634,7 @@ int ObAllTenantInfoProxy::update_tenant_status(
const share::SCN &replayable_scn,
const share::SCN &readable_scn,
const share::SCN &recovery_until_scn,
const share::SCN &sys_recovery_scn,
const int64_t old_switchover_epoch)
{
int64_t begin_time = ObTimeUtility::current_time();
@ -628,6 +645,7 @@ int ObAllTenantInfoProxy::update_tenant_status(
ObTimeoutCtx ctx;
int64_t new_switchover_epoch = OB_INVALID_VERSION;
ObLogRestoreSourceMgr restore_source_mgr;
ObAllTenantInfo tenant_info;
if (OB_UNLIKELY(!is_user_tenant(tenant_id)
|| !new_role.is_valid()
@ -638,15 +656,24 @@ int ObAllTenantInfoProxy::update_tenant_status(
|| !replayable_scn.is_valid_and_not_min()
|| !readable_scn.is_valid_and_not_min()
|| !recovery_until_scn.is_valid_and_not_min()
|| !sys_recovery_scn.is_valid_and_not_min()
|| OB_INVALID_VERSION == old_switchover_epoch)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("tenant_info is invalid", KR(ret), K(tenant_id), K(new_role), K(old_status),
K(new_status), K(sync_scn), K(replayable_scn), K(readable_scn), K(recovery_until_scn),
K(old_switchover_epoch));
//use tenant_info check parameters valid
} else if (OB_FAIL(get_new_switchover_epoch_(old_switchover_epoch, old_status, new_status,
new_switchover_epoch))) {
LOG_WARN("fail to get_new_switchover_epoch_", KR(ret), K(old_switchover_epoch), K(old_status),
K(new_status));
} else if (OB_FAIL(tenant_info.init(tenant_id, new_role, new_status, new_switchover_epoch,
sync_scn, replayable_scn, readable_scn,
recovery_until_scn, NOARCHIVE_MODE, sys_recovery_scn))) {
LOG_WARN("failed to init tenant_info", KR(ret), K(tenant_id), K(new_role),
K(new_status), K(new_switchover_epoch), K(sync_scn),
K(replayable_scn), K(readable_scn), K(recovery_until_scn),
K(sys_recovery_scn));
} else if (OB_UNLIKELY(OB_INVALID_VERSION == new_switchover_epoch)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("new_switchover_ts is invalid", KR(ret), K(new_switchover_epoch),
@ -656,20 +683,22 @@ int ObAllTenantInfoProxy::update_tenant_status(
} else if (OB_FAIL(sql.assign_fmt(
"update %s set tenant_role = '%s', switchover_status = '%s', "
"switchover_epoch = %ld, sync_scn = %lu, replayable_scn = %lu, "
"readable_scn = %lu, recovery_until_scn = %lu where tenant_id = %lu "
"readable_scn = %lu, recovery_until_scn = %lu ,sys_recovery_scn = %lu where tenant_id = %lu "
"and switchover_status = '%s' and switchover_epoch = %ld "
"and readable_scn <= replayable_scn and replayable_scn <= sync_scn and sync_scn <= recovery_until_scn "
"and sync_scn <= %lu and replayable_scn <= %lu and readable_scn <= %lu ",
"and sync_scn <= %lu and replayable_scn <= %lu and readable_scn <= %lu and sys_recovery_scn <= %lu",
OB_ALL_TENANT_INFO_TNAME, new_role.to_str(), new_status.to_str(),
new_switchover_epoch,
sync_scn.get_val_for_inner_table_field(),
replayable_scn.get_val_for_inner_table_field(),
readable_scn.get_val_for_inner_table_field(),
recovery_until_scn.get_val_for_inner_table_field(),
sys_recovery_scn.get_val_for_inner_table_field(),
tenant_id, old_status.to_str(), old_switchover_epoch,
sync_scn.get_val_for_inner_table_field(),
replayable_scn.get_val_for_inner_table_field(),
readable_scn.get_val_for_inner_table_field()))) {
readable_scn.get_val_for_inner_table_field(),
sys_recovery_scn.get_val_for_inner_table_field()))) {
LOG_WARN("failed to assign sql", KR(ret), K(tenant_id), K(sql));
} else if (OB_FAIL(trans.write(exec_tenant_id, sql.ptr(), affected_rows))) {
LOG_WARN("failed to execute sql", KR(ret), K(exec_tenant_id), K(sql));
@ -686,22 +715,6 @@ int ObAllTenantInfoProxy::update_tenant_status(
LOG_WARN("failed to update_recovery_until_scn", KR(ret), K(tenant_id), K(recovery_until_scn));
}
ObAllTenantInfo tenant_info;
int tmp_ret = OB_SUCCESS;
if (OB_SUCCESS != (tmp_ret = tenant_info.init(tenant_id,
new_role,
new_status,
new_switchover_epoch,
sync_scn,
replayable_scn,
readable_scn,
recovery_until_scn))) {
LOG_WARN("failed to init tenant_info", KR(ret), KR(tmp_ret), K(tenant_id), K(new_role),
K(new_status), K(new_switchover_epoch), K(sync_scn),
K(replayable_scn), K(readable_scn), K(recovery_until_scn));
ret = OB_SUCC(ret) ? tmp_ret : ret;
}
int64_t cost = ObTimeUtility::current_time() - begin_time;
ROOTSERVICE_EVENT_ADD("tenant_info", "update_tenant_role", K(ret), K(tenant_id),
K(tenant_info), K(old_status), K(old_switchover_epoch), K(cost));
@ -770,6 +783,71 @@ int ObAllTenantInfoProxy::update_tenant_log_mode(
return ret;
}
}
int ObAllTenantInfoProxy::update_tenant_sys_recovery_scn(
const uint64_t tenant_id,
const share::SCN &sys_recovery_scn,
bool update_sync_scn, ObISQLClient *proxy)
{
int ret = OB_SUCCESS;
const uint64_t exec_tenant_id = gen_meta_tenant_id(tenant_id);
if (OB_UNLIKELY(!is_user_tenant(tenant_id)
|| !sys_recovery_scn.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(sys_recovery_scn));
} else {
TAKE_IN_TRANS(update_tenant_sys_recovery_scn, proxy, exec_tenant_id,
tenant_id, sys_recovery_scn, update_sync_scn);
}
return ret;
}
int ObAllTenantInfoProxy::update_tenant_sys_recovery_scn_in_trans(
const uint64_t tenant_id,
const share::SCN &sys_recovery_scn,
bool update_sync_scn, common::ObMySQLTransaction &trans)
{
int ret = OB_SUCCESS;
const uint64_t exec_tenant_id = gen_meta_tenant_id(tenant_id);
ObSqlString sql;
int64_t affected_rows = 0;
ObTimeoutCtx ctx;
ObAllTenantInfo tenant_info;
if (OB_UNLIKELY(!is_user_tenant(tenant_id)
|| !sys_recovery_scn.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(sys_recovery_scn));
} else if (OB_FAIL(rootserver::ObRootUtils::get_rs_default_timeout_ctx(ctx))) {
LOG_WARN("fail to get timeout ctx", KR(ret), K(ctx));
} else if (OB_FAIL(load_tenant_info(tenant_id, &trans, true, tenant_info))) {
LOG_WARN("failed to load tenant info", KR(ret), K(tenant_id));
} else if (!tenant_info.get_switchover_status().can_report_recovery_status()
|| sys_recovery_scn <= tenant_info.get_sys_recovery_scn()
|| sys_recovery_scn > tenant_info.get_recovery_until_scn()
|| sys_recovery_scn < tenant_info.get_sync_scn()) {
ret = OB_NEED_RETRY;
LOG_WARN("can not report recovery status", KR(ret),
K(tenant_info), K(sys_recovery_scn));
} else if (OB_FAIL(sql.assign_fmt(
"update %s set sys_recovery_scn = %lu ",
OB_ALL_TENANT_INFO_TNAME, sys_recovery_scn.get_val_for_inner_table_field()))) {
LOG_WARN("failed to assign sql", KR(ret), K(tenant_id), K(sys_recovery_scn), K(sql));
} else if (update_sync_scn && OB_FAIL(sql.append_fmt(", sync_scn = %lu ",
sys_recovery_scn.get_val_for_inner_table_field()))) {
LOG_WARN("failed to append sql", KR(ret), K(sys_recovery_scn), K(sql));
} else if (OB_FAIL(trans.write(exec_tenant_id, sql.ptr(), affected_rows))) {
LOG_WARN("failed to execute sql", KR(ret), K(exec_tenant_id), K(sql));
} else if (!is_single_row(affected_rows)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("expect updating one row", KR(ret), K(affected_rows), K(sql));
}
return ret;
LOG_TRACE("update sys_recovery_scn", KR(ret), K(tenant_id), K(sys_recovery_scn),
K(update_sync_scn));
}
} // namespace share
}

42
src/share/ob_tenant_info_proxy.h
View File

@ -43,9 +43,12 @@ bool is_valid_tenant_scn(
const share::SCN &sync_scn,
const share::SCN &replayable_scn,
const share::SCN &standby_scn,
const share::SCN &recovery_until_scn);
const share::SCN &recovery_until_scn,
const share::SCN &sys_recovery_scn);
SCN gen_new_sync_scn(const share::SCN &cur_sync_scn, const share::SCN &desired_sync_scn, const share::SCN &cur_recovery_until_scn);
SCN gen_new_sync_scn(const share::SCN &cur_sync_scn, const share::SCN &desired_sync_scn,
const share::SCN &cur_recovery_until_scn,
const share::SCN &sys_recovery_scn);
struct ObAllTenantInfo
{
@ -64,6 +67,7 @@ public:
* @param[in] standby_scn
* @param[in] recovery_until_scn
* @param[in] log_mode
* @param[in] sys_recovery_scn
*/
int init(const uint64_t tenant_id,
const ObTenantRole &tenant_role,
@ -73,7 +77,8 @@ public:
const SCN &replayable_scn = SCN::base_scn(),
const SCN &standby_scn = SCN::base_scn(),
const SCN &recovery_until_scn = SCN::base_scn(),
const ObArchiveMode &log_mode = NOARCHIVE_MODE);
const ObArchiveMode &log_mode = NOARCHIVE_MODE,
const SCN &sys_recovery_scn = SCN::base_scn());
ObAllTenantInfo &operator=(const ObAllTenantInfo &other);
void assign(const ObAllTenantInfo &other);
void reset();
@ -90,12 +95,7 @@ public:
* Because STS will be changed when switchover to standby.
*/
bool is_sts_ready() const { return !(tenant_role_.is_primary()
|| tenant_is_switchover_to_standby()); }
// ************* Functions that describe what tenant is doing *********************
// tenant is in switchover from primary to standby
bool tenant_is_switchover_to_standby() const { return is_prepare_switching_to_standby_status()
|| is_switching_to_standby_status(); }
|| is_switching_to_standby_status());}
// ObTenantSwitchoverStatus related function
#define IS_TENANT_STATUS(STATUS) \
@ -106,13 +106,12 @@ IS_TENANT_STATUS(switching_to_primary)
IS_TENANT_STATUS(prepare_flashback_for_failover_to_primary)
IS_TENANT_STATUS(flashback)
IS_TENANT_STATUS(switching_to_standby)
IS_TENANT_STATUS(prepare_switching_to_standby)
IS_TENANT_STATUS(prepare_flashback_for_switch_to_primary)
#undef IS_TENANT_STATUS
TO_STRING_KV(K_(tenant_id), K_(tenant_role), K_(switchover_status),
K_(switchover_epoch), K_(sync_scn), K_(replayable_scn),
K_(standby_scn), K_(recovery_until_scn), K_(log_mode));
K_(standby_scn), K_(recovery_until_scn), K_(log_mode), K_(sys_recovery_scn));
// Getter&Setter
const ObTenantRole &get_tenant_role() const { return tenant_role_; }
@ -132,6 +131,7 @@ public:\
Property_declare_var(share::SCN, standby_scn)
Property_declare_var(share::SCN, recovery_until_scn)
Property_declare_var(ObArchiveMode, log_mode)
Property_declare_var(share::SCN, sys_recovery_scn)
#undef Property_declare_var
private:
ObTenantRole tenant_role_;
@ -169,14 +169,12 @@ public:
* @description: update tenant recovery status
* @param[in] tenant_id
* @param[in] proxy
* @param[in] status: the target status while update recovery status
* @param[in] sync_scn : sync point
* @param[in] replay_scn : max replay point
* @param[in] reabable_scn : standby readable scn
*/
static int update_tenant_recovery_status(const uint64_t tenant_id,
ObMySQLProxy *proxy,
ObTenantSwitchoverStatus status,
const SCN &sync_scn,
const SCN &replay_scn,
const SCN &reabable_scn);
@ -219,6 +217,7 @@ public:
const share::SCN &replayable_scn,
const share::SCN &readable_scn,
const share::SCN &recovery_until_scn,
const share::SCN &sys_recovery_scn,
const int64_t old_switchover_epoch);
/**
@ -282,6 +281,23 @@ public:
const ObArchiveMode &old_log_mode,
const ObArchiveMode &new_log_mode);
/**
* @description: update tenant sys ls log sys recovery scn
* @param[in] tenant_id
* @param[in] trans
* @param[in] sys_recovery_scn new sys recovery scn
* @param[in] update_sync_scn : When iterating to the LS operations, these operations can be executed provided that
* the sync_scn of other ls have reached the LS operation's point,
* so when updating the sys_recovery_scn, can the sync_scn be updated at the same time.
* return :
* OB_SUCCESS update tenant sys recovery scn successfully
* OB_NEED_RETRY sys recovery scn backoff, need retry
*/
DEFINE_IN_TRANS_FUC1(update_tenant_sys_recovery_scn,
const uint64_t tenant_id,
const share::SCN &sys_recovery_scn,
bool update_sync_scn);
};
}
}

1
src/share/ob_tenant_switchover_status.cpp
View File

@ -27,7 +27,6 @@ static const char* TENANT_SWITCHOVER_ARRAY[] =
"SWITCHING TO PRIMARY",
"PREPARE FLASHBACK",
"FLASHBACK",
"PREPARE SWITCHING TO STANDBY",
"SWITCHING TO STANDBY",
"PREPARE SWITCHING TO PRIMARY",
};

16
src/share/ob_tenant_switchover_status.h
View File

@ -33,10 +33,9 @@ public:
SWITCHING_TO_PRIMARY_STATUS = 2,
PREPARE_FLASHBACK_FOR_FAILOVER_TO_PRIMARY_STATUS = 3,
FLASHBACK_STATUS = 4,
PREPARE_SWITCHING_TO_STANDBY_STATUS = 5,
SWITCHING_TO_STANDBY_STATUS = 6,
PREPARE_FLASHBACK_FOR_SWITCH_TO_PRIMARY_STATUS = 7,
MAX_STATUS = 8
SWITCHING_TO_STANDBY_STATUS = 5,
PREPARE_FLASHBACK_FOR_SWITCH_TO_PRIMARY_STATUS = 6,
MAX_STATUS = 7
};
public:
ObTenantSwitchoverStatus() : value_(INVALID_STATUS) {}
@ -49,6 +48,13 @@ public:
bool is_valid() const { return INVALID_STATUS != value_; }
ObTenantSwitchoverStatus::Status value() const { return value_; }
const char* to_str() const;
//In the flashback status, the log needs to be truncated,
//and sync_scn will fall back, so at this stage,
//__all_ls_recovery_stat/__all_tenant_info will not be reported
bool can_report_recovery_status() const
{
return FLASHBACK_STATUS != value_;
}
// compare operator
bool operator == (const ObTenantSwitchoverStatus &other) const { return value_ == other.value_; }
@ -69,7 +75,6 @@ IS_TENANT_STATUS(NORMAL_STATUS, normal)
IS_TENANT_STATUS(SWITCHING_TO_PRIMARY_STATUS, switching_to_primary)
IS_TENANT_STATUS(PREPARE_FLASHBACK_FOR_FAILOVER_TO_PRIMARY_STATUS, prepare_flashback_for_failover_to_primary)
IS_TENANT_STATUS(FLASHBACK_STATUS, flashback)
IS_TENANT_STATUS(PREPARE_SWITCHING_TO_STANDBY_STATUS, prepare_switching_to_standby)
IS_TENANT_STATUS(SWITCHING_TO_STANDBY_STATUS, switching_to_standby)
IS_TENANT_STATUS(PREPARE_FLASHBACK_FOR_SWITCH_TO_PRIMARY_STATUS, prepare_flashback_for_switch_to_primary)
#undef IS_TENANT_STATUS
@ -84,7 +89,6 @@ static const ObTenantSwitchoverStatus NORMAL_SWITCHOVER_STATUS(ObTenantSwitchove
static const ObTenantSwitchoverStatus PREPARE_FLASHBACK_FOR_FAILOVER_TO_PRIMARY_SWITCHOVER_STATUS(ObTenantSwitchoverStatus::PREPARE_FLASHBACK_FOR_FAILOVER_TO_PRIMARY_STATUS);
static const ObTenantSwitchoverStatus FLASHBACK_SWITCHOVER_STATUS(ObTenantSwitchoverStatus::FLASHBACK_STATUS);
static const ObTenantSwitchoverStatus SWITCHING_TO_PRIMARY_SWITCHOVER_STATUS(ObTenantSwitchoverStatus::SWITCHING_TO_PRIMARY_STATUS);
static const ObTenantSwitchoverStatus PREP_SWITCHING_TO_STANDBY_SWITCHOVER_STATUS(ObTenantSwitchoverStatus::PREPARE_SWITCHING_TO_STANDBY_STATUS);
static const ObTenantSwitchoverStatus SWITCHING_TO_STANDBY_SWITCHOVER_STATUS(ObTenantSwitchoverStatus::SWITCHING_TO_STANDBY_STATUS);
static const ObTenantSwitchoverStatus PREPARE_FLASHBACK_FOR_SWITCH_TO_PRIMARY_SWITCHOVER_STATUS(ObTenantSwitchoverStatus::PREPARE_FLASHBACK_FOR_SWITCH_TO_PRIMARY_STATUS);

106
src/share/ob_upgrade_utils.cpp
View File

@ -21,6 +21,10 @@
#include "rootserver/ob_root_service.h"
#include "sql/resolver/expr/ob_raw_expr_util.h"
#include "share/ob_rpc_struct.h"
#include "share/ob_tenant_info_proxy.h"//AllTenantInfo
#include "share/ls/ob_ls_i_life_manager.h"//start_transaction
#include "rootserver/ob_ls_service_helper.h"
#include "ob_upgrade_utils.h"
namespace oceanbase
{
@ -791,6 +795,8 @@ int ObUpgradeFor4100Processor::post_upgrade()
LOG_WARN("fail to check inner stat", KR(ret));
} else if (OB_FAIL(recompile_all_views_and_synonyms(tenant_id))) {
LOG_WARN("fail to init rewrite rule version", K(ret), K(tenant_id));
} else if (OB_FAIL(init_tenant_sys_recovery_scn(tenant_id))) {
LOG_WARN("failed to init tenant sys recovery scn", KR(ret), K(tenant_id));
}
return ret;
}
@ -924,6 +930,106 @@ int ObUpgradeFor4100Processor::recompile_all_views_and_synonyms(const uint64_t t
return ret;
}
int ObUpgradeFor4100Processor::init_tenant_sys_recovery_scn(const uint64_t tenant_id)
{
int ret = OB_SUCCESS;
int64_t start = ObTimeUtility::current_time();
if (!is_user_tenant(tenant_id)) {
//no need to update meta tenant or sys tenant sys recovery scn
} else {
//check sys recovery scn, make status to equal and set sys_recovery_scn
const uint64_t exec_tenant_id = gen_meta_tenant_id(tenant_id);
ObAllTenantInfo tenant_info;
SCN sync_scn;
ObTenantSchema tenant_schema;
if (OB_FAIL(rootserver::ObTenantThreadHelper::get_tenant_schema(tenant_id, tenant_schema))) {
LOG_WARN("failed to get tenant schema", KR(ret), K(tenant_id));
}
START_TRANSACTION(sql_proxy_, exec_tenant_id);
rootserver::ObTenantLSInfo tenant_stat(sql_proxy_, &tenant_schema,
tenant_id, GCTX.srv_rpc_proxy_,
GCTX.lst_operator_);
if (FAILEDx(ObAllTenantInfoProxy::load_tenant_info(tenant_id, &trans,
true, tenant_info))) {
LOG_WARN("failed to load tenant info", KR(ret), K(tenant_id));
} else if (!tenant_info.get_sys_recovery_scn().is_min()) {
LOG_INFO("tenant sys recovery scn already update, no need to init", K(tenant_info));
} else if (OB_FAIL(tenant_stat.revision_to_equal_status(trans))) {
LOG_WARN("failed to make revision status", KR(ret), K(tenant_id));
} else if (OB_FAIL(get_sys_ls_max_sync_scn_(tenant_id, sync_scn))) {
LOG_WARN("failed to get sys ls max sync scn", KR(ret), K(tenant_id));
} else if (OB_FAIL(ObAllTenantInfoProxy::update_tenant_sys_recovery_scn_in_trans(
tenant_id, sync_scn, false, trans))) {
LOG_WARN("failed to update tenant sys recovery scn", KR(ret), K(tenant_id),
K(sync_scn), K(tenant_info));
} else {
LOG_INFO("init tenant sys recovery scn success", K(tenant_id), K(sync_scn));
}
END_TRANSACTION(trans);
}
LOG_INFO("add tenant sys recovery scn finish", K(ret), K(tenant_id_), "cost", ObTimeUtility::current_time() - start);
return ret;
}
int ObUpgradeFor4100Processor::get_sys_ls_max_sync_scn_(const uint64_t tenant_id, SCN &sync_scn)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(GCTX.sql_proxy_) || OB_ISNULL(GCTX.srv_rpc_proxy_) ||
OB_ISNULL(GCTX.location_service_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("error unexpcted", KR(ret), KP(GCTX.sql_proxy_), KP(GCTX.srv_rpc_proxy_),
KP(GCTX.location_service_));
} else {
ObTimeoutCtx ctx;
ObAddr leader;
obrpc::ObGetLSSyncScnArg arg;
ObGetLSSyncScnRes res;
int tmp_ret = OB_SUCCESS;
rootserver::ObGetLSSyncScnProxy proxy(
*GCTX.srv_rpc_proxy_, &obrpc::ObSrvRpcProxy::get_ls_sync_scn);
const int64_t timeout = GCONF.internal_sql_execute_timeout;
if (FAILEDx(arg.init(tenant_id, SYS_LS, false))) {
LOG_WARN("failed to init arg", KR(ret), K(tenant_id));
} else if (OB_FAIL(ObShareUtil::set_default_timeout_ctx(ctx, timeout))) {
LOG_WARN("fail to set timeout ctx", KR(ret), K(timeout));
}
while (OB_SUCC(ret)) {
proxy.reuse();
if (ctx.is_timeouted()) {
ret = OB_TIMEOUT;
LOG_WARN("already timeout, failed to get sys ls sync scn", KR(ret), K(ctx), K(tenant_id));
} else {
if (OB_FAIL(GCTX.location_service_->get_leader(
GCONF.cluster_id, tenant_id, SYS_LS, true, leader))) {
LOG_WARN("failed to get leader", KR(ret), K(tenant_id));
} else if (OB_FAIL(proxy.call(leader, ctx.get_timeout(), tenant_id, arg))) {
LOG_WARN("failed to get ls sync scn", KR(ret), K(arg), K(ctx),
K(leader));
}
if (OB_TMP_FAIL(proxy.wait())) {
ret = OB_SUCC(ret) ? tmp_ret : ret;
LOG_WARN("failed to wait all", KR(ret), KR(tmp_ret));
} else if (OB_FAIL(ret)) {
} else if (1 != proxy.get_results().count() ||
OB_ISNULL(proxy.get_results().at(0))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to get rpc result", KR(ret), "proxy result", proxy.get_results());
} else {
sync_scn = proxy.get_results().at(0)->get_cur_sync_scn();
break;
}
if (OB_FAIL(ret)) {
//ignore error of each rpc until timeout
LOG_WARN("failed to get sys ls sync scn, sleep and try again", KR(ret));
ret = OB_SUCCESS;
usleep(100 * 1000L);
}
}
}
}
return ret;
}
/* =========== 4100 upgrade processor end ============= */
/* =========== special upgrade processor end ============= */
} // end share

8
src/share/ob_upgrade_utils.h
View File

@ -22,9 +22,13 @@
namespace oceanbase
{
namespace common
{
class ObTimeoutCtx;
}
namespace share
{
class SCN;
static const int64_t UPGRADE_JOB_TYPE_COUNT = 1;
static const rootserver::ObRsJobType upgrade_job_type_array[UPGRADE_JOB_TYPE_COUNT] = {
rootserver::JOB_TYPE_INVALID,
@ -186,6 +190,8 @@ private:
int post_upgrade_for_srs();
int init_rewrite_rule_version(const uint64_t tenant_id);
static int recompile_all_views_and_synonyms(const uint64_t tenant_id);
int init_tenant_sys_recovery_scn(const uint64_t tenant_id);
int get_sys_ls_max_sync_scn_(const uint64_t tenant_id, share::SCN &sys_scn);
};
/* =========== special upgrade processor end ============= */

8
src/share/rc/ob_tenant_base.h
View File

@ -124,9 +124,11 @@ namespace rootserver
{
class ObPrimaryMajorFreezeService;
class ObRestoreMajorFreezeService;
class ObTenantRecoveryReportor;
class ObLSRecoveryReportor;
class ObTenantInfoLoader;
class ObTenantInfoReportor;
class ObPrimaryLSService;
class ObCommonLSService;
class ObRestoreService;
class ObRecoveryLSService;
class ObArbitrationService;
@ -198,8 +200,10 @@ using ObPartTransCtxObjPool = common::ObServerObjectPool<transaction::ObPartTran
observer::ObTenantMetaChecker*, \
observer::QueueThread *, \
storage::ObStorageHAHandlerService*, \
rootserver::ObTenantRecoveryReportor*, \
rootserver::ObLSRecoveryReportor*, \
rootserver::ObTenantInfoLoader*, \
rootserver::ObTenantInfoReportor*, \
rootserver::ObCommonLSService*, \
rootserver::ObPrimaryLSService*, \
rootserver::ObRecoveryLSService*, \
rootserver::ObRestoreService*, \

37
src/storage/ls/ob_ls.cpp
View File

@ -38,6 +38,8 @@
#include "storage/tx/ob_standby_timestamp_service.h"
#include "rootserver/freeze/ob_major_freeze_service.h"
#include "rootserver/ob_primary_ls_service.h"
#include "rootserver/ob_common_ls_service.h"
#include "rootserver/ob_tenant_info_report.h"
#include "rootserver/ob_recovery_ls_service.h"
#include "rootserver/restore/ob_restore_scheduler.h"
#include "sql/das/ob_das_id_service.h"
@ -197,18 +199,27 @@ int ObLS::init(const share::ObLSID &ls_id,
REGISTER_TO_LOGSERVICE(logservice::GAIS_LOG_BASE_TYPE, MTL(share::ObGlobalAutoIncService *));
MTL(share::ObGlobalAutoIncService *)->set_cache_ls(this);
}
if (OB_SUCC(ret) && ls_id.is_sys_ls()) {
//meta tenant need create thread for ls manager
if (OB_SUCC(ret) && ls_id.is_sys_ls() && is_user_tenant(tenant_id)) {
REGISTER_TO_LOGSERVICE(logservice::PRIMARY_LS_SERVICE_LOG_BASE_TYPE, MTL(rootserver::ObPrimaryLSService *));
LOG_INFO("primary ls manager registre to logservice success");
LOG_INFO("primary ls manager register to logservice success");
}
if (OB_SUCC(ret) && ls_id.is_sys_ls() && !is_user_tenant(tenant_id)) {
REGISTER_TO_LOGSERVICE(logservice::COMMON_LS_SERVICE_LOG_BASE_TYPE, MTL(rootserver::ObCommonLSService *));
LOG_INFO("common ls manager register to logservice success");
}
if (OB_SUCC(ret) && ls_id.is_sys_ls() && is_meta_tenant(tenant_id)) {
//sys no need to report
REGISTER_TO_LOGSERVICE(logservice::TENANT_INFO_REPORTOR_LOG_BASE_TYPE, MTL(rootserver::ObTenantInfoReportor *));
LOG_INFO("tenant info report register to logservice success");
}
if (OB_SUCC(ret) && is_user_tenant(tenant_id) && ls_id.is_sys_ls()) {
// only user tenant need dump datadict
REGISTER_TO_LOGSERVICE(logservice::DATA_DICT_LOG_BASE_TYPE, MTL(datadict::ObDataDictService *));
//only user table need recovery
REGISTER_TO_RESTORESERVICE(logservice::RECOVERY_LS_SERVICE_LOG_BASE_TYPE, MTL(rootserver::ObRecoveryLSService *));
LOG_INFO("recovery ls manager registre to restoreservice success");
LOG_INFO("recovery ls manager register to restoreservice success");
}
if (OB_SUCC(ret) && !is_user_tenant(tenant_id) && ls_id.is_sys_ls()) {
@ -632,20 +643,24 @@ void ObLS::destroy()
UNREGISTER_FROM_LOGSERVICE(logservice::GAIS_LOG_BASE_TYPE, MTL(share::ObGlobalAutoIncService *));
MTL(share::ObGlobalAutoIncService *)->set_cache_ls(nullptr);
}
if (OB_SUCC(ret) && ls_meta_.ls_id_.is_sys_ls()) {
if (ls_meta_.ls_id_.is_sys_ls() && is_user_tenant(MTL_ID())) {
rootserver::ObPrimaryLSService* ls_service = MTL(rootserver::ObPrimaryLSService*);
UNREGISTER_FROM_LOGSERVICE(logservice::PRIMARY_LS_SERVICE_LOG_BASE_TYPE, ls_service);
}
if (OB_SUCC(ret) && is_user_tenant(MTL_ID()) && ls_meta_.ls_id_.is_sys_ls()) {
if (ls_meta_.ls_id_.is_sys_ls() && !is_user_tenant(MTL_ID())) {
rootserver::ObCommonLSService *ls_service = MTL(rootserver::ObCommonLSService*);
UNREGISTER_FROM_LOGSERVICE(logservice::COMMON_LS_SERVICE_LOG_BASE_TYPE, ls_service);
rootserver::ObRestoreService * restore_service = MTL(rootserver::ObRestoreService*);
UNREGISTER_FROM_LOGSERVICE(logservice::RESTORE_SERVICE_LOG_BASE_TYPE, restore_service);
}
if (ls_meta_.ls_id_.is_sys_ls() && is_meta_tenant(MTL_ID())) {
UNREGISTER_FROM_LOGSERVICE(logservice::TENANT_INFO_REPORTOR_LOG_BASE_TYPE, MTL(rootserver::ObTenantInfoReportor*));
}
if (is_user_tenant(MTL_ID()) && ls_meta_.ls_id_.is_sys_ls()) {
UNREGISTER_FROM_LOGSERVICE(logservice::DATA_DICT_LOG_BASE_TYPE, MTL(datadict::ObDataDictService *));
rootserver::ObRecoveryLSService* ls_service = MTL(rootserver::ObRecoveryLSService*);
UNREGISTER_FROM_RESTORESERVICE(logservice::RECOVERY_LS_SERVICE_LOG_BASE_TYPE, ls_service);
}
if (OB_SUCC(ret) && !is_user_tenant(MTL_ID()) && ls_meta_.ls_id_.is_sys_ls()) {
rootserver::ObRestoreService * restore_service = MTL(rootserver::ObRestoreService*);
UNREGISTER_FROM_LOGSERVICE(logservice::RESTORE_SERVICE_LOG_BASE_TYPE, restore_service);
}
tx_table_.destroy();
lock_table_.destroy();
ls_tablet_svr_.destroy();

1
src/storage/tx/ob_trans_service_v4.cpp
View File

@ -39,7 +39,6 @@
#include "storage/tx_storage/ob_ls_handle.h"
#include "storage/ls/ob_ls.h"
#include "ob_xa_service.h"
#include "rootserver/ob_tenant_recovery_reportor.h"
/* interface(s) */
namespace oceanbase {

58
src/storage/tx_storage/ob_checkpoint_service.cpp
View File

@ -25,7 +25,11 @@
#include "logservice/palf/palf_env.h"
#include "logservice/palf/lsn.h"
#include "logservice/archiveservice/ob_archive_service.h"
#include "logservice/ob_log_handler.h" //ObLogHandler
#include "storage/tx_storage/ob_ls_handle.h"
#include "rootserver/ob_tenant_info_loader.h"//get_tenant_info
#include "share/ob_cluster_version.h"//get_tenant_data_version
#include "share/ob_tenant_info_proxy.h"//ObAllTenantInfo
namespace oceanbase
{
@ -177,13 +181,29 @@ void ObCheckPointService::ObCheckpointTask::runTimerTask()
K(archive_lsn), K(checkpoint_lsn), KPC(ls));
checkpoint_lsn = archive_lsn;
}
if (OB_FAIL(ls->get_log_handler()->advance_base_lsn(checkpoint_lsn))) {
ARCHIVE_LOG(WARN, "advance base lsn failed", K(ret), K(checkpoint_lsn));
} else {
FLOG_INFO("[CHECKPOINT] advance palf base lsn successfully",
K(checkpoint_lsn), K(ls->get_ls_id()));
}
}
if (OB_SUCC(ret) && ls->get_ls_id().is_sys_ls()) {
//The check_point point of the sys ls refers to the sys_recovery_scn of __all_tenant_info
const uint64_t tenant_id = ls->get_tenant_id();
palf::LSN sys_recovery_lsn;
if (!is_user_tenant(tenant_id)) {
//nothing todo
} else if (OB_FAIL(get_sys_ls_recovery_lsn_(ls->get_log_handler(), sys_recovery_lsn))) {
LOG_WARN("failed to get sys ls recovery lsn", KR(ret));
} else if (sys_recovery_lsn < checkpoint_lsn) {
LOG_TRACE(
"sys ls recovery lsn is small than checkpoint_lsn, set base_lsn with sys recovery lsn",
K(sys_recovery_lsn), K(checkpoint_lsn));
checkpoint_lsn = sys_recovery_lsn;
} // end for user tenant
}
if (FAILEDx(ls->get_log_handler()->advance_base_lsn(checkpoint_lsn))) {
ARCHIVE_LOG(WARN, "advance base lsn failed", K(ret), K(checkpoint_lsn));
} else {
FLOG_INFO("[CHECKPOINT] advance palf base lsn successfully",
K(checkpoint_lsn), K(ls->get_ls_id()));
}
}
if (ret == OB_ITER_END) {
ret = OB_SUCCESS;
@ -196,6 +216,32 @@ void ObCheckPointService::ObCheckpointTask::runTimerTask()
}
}
int ObCheckPointService::ObCheckpointTask::get_sys_ls_recovery_lsn_(
logservice::ObLogHandler *log_handler, palf::LSN &recovery_lsn)
{
int ret = OB_SUCCESS;
share::ObAllTenantInfo tenant_info;
SCN recovery_scn;
recovery_lsn.reset();
if (OB_FAIL(MTL(rootserver::ObTenantInfoLoader *)->get_tenant_info(tenant_info))) {
LOG_WARN("failed to get tenant info", KR(ret));
} else if (tenant_info.get_sys_recovery_scn().is_min()) {
// min is during upgrade, use sync scn instead
recovery_scn = tenant_info.get_sync_scn();
} else {
recovery_scn = tenant_info.get_sys_recovery_scn();
}
if (OB_FAIL(ret)) {
} else if (OB_ISNULL(log_handler)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("log handle is null", KR(ret), K(tenant_info));
} else if (OB_FAIL(log_handler->locate_by_scn_coarsely(recovery_scn,
recovery_lsn))) {
LOG_WARN("failed to locate scn", KR(ret), K(tenant_info));
}
return ret;
}
bool ObCheckPointService::get_disk_usage_threshold_(int64_t &threshold)
{
int ret = OB_SUCCESS;

9
src/storage/tx_storage/ob_checkpoint_service.h
View File

@ -19,6 +19,13 @@
namespace oceanbase
{
namespace logservice
{
class ObLogHandler;
}
namespace palf {
struct LSN;
}
namespace storage
{
namespace checkpoint
@ -75,6 +82,8 @@ private:
virtual ~ObCheckpointTask() {}
virtual void runTimerTask();
private:
int get_sys_ls_recovery_lsn_(logservice::ObLogHandler *log_handler, palf::LSN &recovery_lsn);
};
class ObTraversalFlushTask : public common::ObTimerTask

2
tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp
View File

@ -2152,7 +2152,7 @@ int ObAdminDumpBackupDataExecutor::dump_ls_attr_info_(const share::ObLSAttr &ls_
PrintHelper::print_dump_line("ls_group_id", ls_attr.get_ls_group_id());
PrintHelper::print_dump_line("flag", ls_attr.get_ls_flag());
PrintHelper::print_dump_line("status", ls_attr.get_ls_status());
PrintHelper::print_dump_line("operation_type", ls_attr.get_ls_operatin_type());
PrintHelper::print_dump_line("operation_type", ls_attr.get_ls_operation_type());
return ret;
}

30
unittest/rootserver/test_primary_ls_service.cpp
View File

@ -14,7 +14,7 @@
#include <gtest/gtest.h>
#include <gmock/gmock.h>
#define private public
#include "rootserver/ob_primary_ls_service.h"
#include "rootserver/ob_common_ls_service.h"
namespace oceanbase {
using namespace common;
using namespace share;
@ -57,12 +57,12 @@ TEST_F(TestPrimaryLSService, zone_balance)
ASSERT_EQ(ret, OB_SUCCESS);
ret = primary_zone_array.push_back(z1);
ASSERT_EQ(ret, OB_SUCCESS);
ret = ObTenantLSInfo::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone);
ret = ObCommonLSService::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone);
ASSERT_EQ(ret, OB_SUCCESS);
LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone));
ASSERT_EQ(z1, ls_primary_zone.at(0));
ASSERT_EQ(1, count_group_by_zone.at(0));
ret = ObTenantLSInfo::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone);
ret = ObCommonLSService::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone);
ASSERT_EQ(ret, OB_SUCCESS);
ASSERT_EQ(z1, ls_primary_zone.at(0));
ASSERT_EQ(1, count_group_by_zone.at(0));
@ -74,14 +74,14 @@ TEST_F(TestPrimaryLSService, zone_balance)
ASSERT_EQ(ret, OB_SUCCESS);
ret = primary_zone_array.push_back(z2);
ASSERT_EQ(ret, OB_SUCCESS);
ret = ObTenantLSInfo::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone);
ret = ObCommonLSService::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone);
LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone));
ASSERT_EQ(ret, OB_SUCCESS);
ASSERT_EQ(z2, ls_primary_zone.at(0));
ASSERT_EQ(z2, ls_primary_zone.at(1));
ASSERT_EQ(0, count_group_by_zone.at(0));
ASSERT_EQ(2, count_group_by_zone.at(1));
ret = ObTenantLSInfo::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone);
ret = ObCommonLSService::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone);
LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone));
ASSERT_EQ(ret, OB_SUCCESS);
ASSERT_EQ(z1, ls_primary_zone.at(0));
@ -97,14 +97,14 @@ TEST_F(TestPrimaryLSService, zone_balance)
ASSERT_EQ(ret, OB_SUCCESS);
ret = primary_zone_array.push_back(z3);
ASSERT_EQ(ret, OB_SUCCESS);
ret = ObTenantLSInfo::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone);
ret = ObCommonLSService::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone);
LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone));
ASSERT_EQ(ret, OB_SUCCESS);
ASSERT_EQ(z1, ls_primary_zone.at(0));
ASSERT_EQ(z3, ls_primary_zone.at(1));
ASSERT_EQ(1, count_group_by_zone.at(0));
ASSERT_EQ(1, count_group_by_zone.at(1));
ret = ObTenantLSInfo::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone);
ret = ObCommonLSService::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone);
LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone));
ASSERT_EQ(ret, OB_SUCCESS);
ASSERT_EQ(z1, ls_primary_zone.at(0));
@ -117,7 +117,7 @@ TEST_F(TestPrimaryLSService, zone_balance)
count_group_by_zone.reset();
ret = primary_zone_array.push_back(z2);
ASSERT_EQ(ret, OB_SUCCESS);
ret = ObTenantLSInfo::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone);
ret = ObCommonLSService::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone);
LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone));
ASSERT_EQ(ret, OB_SUCCESS);
ASSERT_EQ(z2, ls_primary_zone.at(0));
@ -125,7 +125,7 @@ TEST_F(TestPrimaryLSService, zone_balance)
ASSERT_EQ(0, count_group_by_zone.at(0));
ASSERT_EQ(0, count_group_by_zone.at(1));
ASSERT_EQ(2, count_group_by_zone.at(2));
ret = ObTenantLSInfo::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone);
ret = ObCommonLSService::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone);
LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone));
ASSERT_EQ(ret, OB_SUCCESS);
ASSERT_EQ(z1, ls_primary_zone.at(0));
@ -145,7 +145,7 @@ TEST_F(TestPrimaryLSService, zone_balance)
ASSERT_EQ(ret, OB_SUCCESS);
ret = primary_zone_infos.push_back(info);
ASSERT_EQ(ret, OB_SUCCESS);
ret = ObTenantLSInfo::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone);
ret = ObCommonLSService::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone);
LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone));
ASSERT_EQ(ret, OB_SUCCESS);
ASSERT_EQ(z2, ls_primary_zone.at(0));
@ -155,7 +155,7 @@ TEST_F(TestPrimaryLSService, zone_balance)
ASSERT_EQ(1, count_group_by_zone.at(0));
ASSERT_EQ(0, count_group_by_zone.at(1));
ASSERT_EQ(3, count_group_by_zone.at(2));
ret = ObTenantLSInfo::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone);
ret = ObCommonLSService::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone);
LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone));
ASSERT_EQ(ret, OB_SUCCESS);
ASSERT_EQ(z3, ls_primary_zone.at(0));
@ -184,7 +184,7 @@ TEST_F(TestPrimaryLSService, zone_balance)
ASSERT_EQ(ret, OB_SUCCESS);
ret = primary_zone_infos.push_back(info);
ASSERT_EQ(ret, OB_SUCCESS);
ret = ObTenantLSInfo::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone);
ret = ObCommonLSService::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone);
LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone));
ASSERT_EQ(ret, OB_SUCCESS);
ASSERT_EQ(z1, ls_primary_zone.at(0));
@ -195,7 +195,7 @@ TEST_F(TestPrimaryLSService, zone_balance)
ASSERT_EQ(3, count_group_by_zone.at(0));
ASSERT_EQ(1, count_group_by_zone.at(1));
ASSERT_EQ(1, count_group_by_zone.at(2));
ret = ObTenantLSInfo::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone);
ret = ObCommonLSService::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone);
LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone));
ASSERT_EQ(z3, ls_primary_zone.at(0));
ASSERT_EQ(z1, ls_primary_zone.at(1));
@ -232,7 +232,7 @@ TEST_F(TestPrimaryLSService, zone_balance)
ASSERT_EQ(ret, OB_SUCCESS);
ret = primary_zone_infos.push_back(info);
ASSERT_EQ(ret, OB_SUCCESS);
ret = ObTenantLSInfo::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone);
ret = ObCommonLSService::set_ls_to_primary_zone(primary_zone_array, primary_zone_infos, ls_primary_zone, count_group_by_zone);
LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone));
ASSERT_EQ(ret, OB_SUCCESS);
ASSERT_EQ(z1, ls_primary_zone.at(0));
@ -246,7 +246,7 @@ TEST_F(TestPrimaryLSService, zone_balance)
ASSERT_EQ(4, count_group_by_zone.at(0));
ASSERT_EQ(0, count_group_by_zone.at(1));
ASSERT_EQ(4, count_group_by_zone.at(2));
ret = ObTenantLSInfo::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone);
ret = ObCommonLSService::balance_ls_primary_zone(primary_zone_array, ls_primary_zone, count_group_by_zone);
LOG_INFO("set ls to primary zone", K(ls_primary_zone), K(count_group_by_zone));
ASSERT_EQ(z3, ls_primary_zone.at(0));
ASSERT_EQ(z1, ls_primary_zone.at(1));