626 lines
29 KiB
C++
626 lines
29 KiB
C++
/**
|
|
* Copyright (c) 2021 OceanBase
|
|
* OceanBase CE is licensed under Mulan PubL v2.
|
|
* You can use this software according to the terms and conditions of the Mulan PubL v2.
|
|
* You may obtain a copy of Mulan PubL v2 at:
|
|
* http://license.coscl.org.cn/MulanPubL-2.0
|
|
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
|
|
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
|
|
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
|
|
* See the Mulan PubL v2 for more details.
|
|
*/
|
|
|
|
#define USING_LOG_PREFIX STANDBY
|
|
|
|
#include "ob_primary_standby_service.h" // ObPrimaryStandbyService
|
|
#include "lib/oblog/ob_log_module.h" // LOG_*
|
|
#include "lib/utility/ob_print_utils.h" // TO_STRING_KV
|
|
#include "rootserver/ob_cluster_event.h" // CLUSTER_EVENT_ADD_CONTROL
|
|
#include "rootserver/ob_rs_event_history_table_operator.h" // ROOTSERVICE_EVENT_ADD
|
|
#include "rootserver/ob_tenant_role_transition_service.h" // ObTenantRoleTransitionService
|
|
#include "rootserver/ob_primary_ls_service.h"//ObTenantLSInfo
|
|
#include "share/restore/ob_log_restore_source_mgr.h" // ObLogRestoreSourceMgr
|
|
#include "share/ls/ob_ls_recovery_stat_operator.h"// ObLSRecoveryStatOperator
|
|
#include "share/ls/ob_ls_life_manager.h" //ObLSLifeAgentManager
|
|
#include "share/ls/ob_ls_operator.h" //ObLSAttr
|
|
#include "storage/tx/ob_timestamp_service.h" // ObTimestampService
|
|
#include "share/ob_standby_upgrade.h" // ObStandbyUpgrade
|
|
#include "observer/ob_inner_sql_connection.h"//ObInnerSQLConnection
|
|
#include "storage/tx/ob_trans_service.h" //ObTransService
|
|
|
|
namespace oceanbase
|
|
{
|
|
using namespace oceanbase;
|
|
using namespace common;
|
|
using namespace obrpc;
|
|
using namespace share;
|
|
using namespace rootserver;
|
|
|
|
namespace standby
|
|
{
|
|
|
|
int ObPrimaryStandbyService::init(
|
|
ObMySQLProxy *sql_proxy,
|
|
share::schema::ObMultiVersionSchemaService *schema_service)
|
|
{
|
|
int ret = OB_SUCCESS;
|
|
if (OB_ISNULL(sql_proxy)
|
|
|| OB_ISNULL(schema_service)) {
|
|
ret = OB_INVALID_ARGUMENT;
|
|
LOG_WARN("invalid argument", KR(ret), KP(sql_proxy), KP(schema_service));
|
|
} else {
|
|
sql_proxy_ = sql_proxy;
|
|
schema_service_ = schema_service;
|
|
inited_ = true;
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
void ObPrimaryStandbyService::destroy()
|
|
{
|
|
if (OB_UNLIKELY(!inited_)) {
|
|
LOG_INFO("ObPrimaryStandbyService has been destroyed", K_(inited));
|
|
} else {
|
|
LOG_INFO("ObPrimaryStandbyService begin to destroy", K_(inited));
|
|
sql_proxy_ = NULL;
|
|
schema_service_ = NULL;
|
|
inited_ = false;
|
|
LOG_INFO("ObPrimaryStandbyService destroyed", K_(inited));
|
|
}
|
|
}
|
|
|
|
int ObPrimaryStandbyService::check_inner_stat_()
|
|
{
|
|
int ret = OB_SUCCESS;
|
|
if (OB_UNLIKELY(!inited_)) {
|
|
ret = OB_NOT_INIT;
|
|
LOG_WARN("not init", KR(ret));
|
|
} else if (OB_ISNULL(sql_proxy_) || OB_ISNULL(schema_service_)) {
|
|
ret = OB_ERR_UNEXPECTED;
|
|
LOG_WARN("Member variables is NULL", KR(ret), KP(sql_proxy_), KP(schema_service_));
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
int ObPrimaryStandbyService::switch_tenant(const obrpc::ObSwitchTenantArg &arg)
|
|
{
|
|
int ret = OB_SUCCESS;
|
|
int64_t begin_time = ObTimeUtility::current_time();
|
|
uint64_t switch_tenant_id = OB_INVALID_ID;
|
|
ObSchemaGetterGuard schema_guard;
|
|
const char *alter_cluster_event = arg.get_alter_type_str();
|
|
const ObSimpleTenantSchema *tenant_schema = nullptr;
|
|
CLUSTER_EVENT_ADD_CONTROL_START(ret, alter_cluster_event, "stmt_str", arg.get_stmt_str());
|
|
if (OB_FAIL(check_inner_stat_())) {
|
|
LOG_WARN("inner stat error", KR(ret), K_(inited));
|
|
} else if (!arg.is_valid()) {
|
|
ret = OB_INVALID_ARGUMENT;
|
|
LOG_WARN("invalid arg", K(arg), KR(ret));
|
|
} else if (OB_FAIL(get_target_tenant_id(arg.get_tenant_name(), arg.get_exec_tenant_id(), switch_tenant_id))) {
|
|
LOG_WARN("failed to get_target_tenant_id", KR(ret), K(switch_tenant_id), K(arg));
|
|
} else if (OB_FAIL(schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID, schema_guard))) {
|
|
LOG_WARN("failed to get schema guard", KR(ret));
|
|
} else if (OB_FAIL(schema_guard.get_tenant_info(switch_tenant_id, tenant_schema))) {
|
|
LOG_WARN("failed to get tenant info", KR(ret), K(switch_tenant_id));
|
|
} else if (OB_ISNULL(tenant_schema)) {
|
|
ret = OB_ERR_UNEXPECTED;
|
|
LOG_WARN("tenant_schema is null", KR(ret), K(switch_tenant_id), K(arg));
|
|
} else if (tenant_schema->is_normal()) {
|
|
switch (arg.get_op_type()) {
|
|
case ObSwitchTenantArg::SWITCH_TO_PRIMARY :
|
|
if (OB_FAIL(switch_to_primary(switch_tenant_id, arg.get_op_type()))) {
|
|
LOG_WARN("failed to switch_to_primary", KR(ret), K(switch_tenant_id), K(arg), KPC(tenant_schema));
|
|
}
|
|
break;
|
|
case ObSwitchTenantArg::SWITCH_TO_STANDBY :
|
|
if (OB_FAIL(switch_to_standby(switch_tenant_id, arg.get_op_type()))) {
|
|
LOG_WARN("failed to switch_to_standby", KR(ret), K(switch_tenant_id), K(arg), KPC(tenant_schema));
|
|
}
|
|
break;
|
|
case ObSwitchTenantArg::FAILOVER_TO_PRIMARY :
|
|
if (OB_FAIL(failover_to_primary(switch_tenant_id, arg.get_op_type()))) {
|
|
LOG_WARN("failed to failover_to_primary", KR(ret), K(switch_tenant_id), K(arg), KPC(tenant_schema));
|
|
}
|
|
break;
|
|
default :
|
|
ret = OB_INVALID_ARGUMENT;
|
|
LOG_WARN("unkown op_type", K(arg));
|
|
}
|
|
} else {
|
|
ret = OB_OP_NOT_ALLOW;
|
|
LOG_WARN("tenant status is not normal, switch tenant is not allowed", KR(ret), K(switch_tenant_id), K(arg), KPC(tenant_schema));
|
|
LOG_USER_ERROR(OB_OP_NOT_ALLOW, "tenant status is not normal, switch tenant is");
|
|
}
|
|
|
|
int64_t cost = ObTimeUtility::current_time() - begin_time;
|
|
CLUSTER_EVENT_ADD_CONTROL_FINISH(ret, alter_cluster_event,
|
|
K(cost),
|
|
"stmt_str", arg.get_stmt_str());
|
|
|
|
return ret;
|
|
}
|
|
|
|
int ObPrimaryStandbyService::failover_to_primary(const uint64_t tenant_id,
|
|
const obrpc::ObSwitchTenantArg::OpType &switch_optype)
|
|
{
|
|
int ret = OB_SUCCESS;
|
|
ObAllTenantInfo tenant_info;
|
|
ObSchemaGetterGuard schema_guard;
|
|
const ObSimpleTenantSchema *tenant_schema = nullptr;
|
|
if (OB_FAIL(check_inner_stat_())) {
|
|
LOG_WARN("inner stat error", KR(ret), K_(inited));
|
|
} else if (OB_ISNULL(GCTX.srv_rpc_proxy_) || OB_ISNULL(schema_service_)) {
|
|
ret = OB_ERR_UNEXPECTED;
|
|
LOG_WARN("pointer is null", KR(ret), KP(GCTX.srv_rpc_proxy_), KP(schema_service_));
|
|
} else if (OB_UNLIKELY(obrpc::ObSwitchTenantArg::OpType::INVALID == switch_optype)) {
|
|
ret = OB_INVALID_ARGUMENT;
|
|
LOG_WARN("invalid switch_optype", KR(ret), K(switch_optype));
|
|
} else if (!is_user_tenant(tenant_id)) {
|
|
ret = OB_INVALID_ARGUMENT;
|
|
LOG_WARN("only support switch user tenant", KR(ret), K(tenant_id));
|
|
LOG_USER_ERROR(OB_INVALID_ARGUMENT, "tenant id, only support operating user tenant");
|
|
} else if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(tenant_id, sql_proxy_,
|
|
false, tenant_info))) {
|
|
LOG_WARN("failed to load tenant info", KR(ret), K(tenant_id));
|
|
} else if (tenant_info.is_primary() && tenant_info.is_normal_status()) {
|
|
LOG_INFO("already is primary tenant, no need switch", K(tenant_info));
|
|
} else if (OB_FAIL(schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID, schema_guard))) {
|
|
LOG_WARN("failed to get schema guard", KR(ret), K(tenant_id));
|
|
} else if (OB_FAIL(schema_guard.get_tenant_info(tenant_id, tenant_schema))) {
|
|
LOG_WARN("failed to get tenant info", KR(ret), K(tenant_id));
|
|
} else if (OB_ISNULL(tenant_schema)) {
|
|
ret = OB_ERR_UNEXPECTED;
|
|
LOG_WARN("tenant_schema is null", KR(ret), K(tenant_id));
|
|
} else if (tenant_schema->is_normal()) {
|
|
ObTenantRoleTransitionService role_transition_service(tenant_id, sql_proxy_, GCTX.srv_rpc_proxy_, switch_optype);
|
|
if (OB_FAIL(role_transition_service.failover_to_primary())) {
|
|
LOG_WARN("failed to failover to primary", KR(ret), K(tenant_id));
|
|
}
|
|
} else {
|
|
ret = OB_OP_NOT_ALLOW;
|
|
LOG_WARN("tenant status is not normal, failover is not allowed", KR(ret), K(tenant_id), KPC(tenant_schema));
|
|
LOG_USER_ERROR(OB_OP_NOT_ALLOW, "tenant status is not normal, failover is");
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
int ObPrimaryStandbyService::get_target_tenant_id(const ObString &tenant_name,
|
|
const uint64_t exec_tenant_id,
|
|
uint64_t &switch_tenant_id)
|
|
{
|
|
int ret = OB_SUCCESS;
|
|
switch_tenant_id = OB_INVALID_ID;
|
|
if (OB_INVALID_TENANT_ID == exec_tenant_id) {
|
|
ret = OB_INVALID_ARGUMENT;
|
|
LOG_WARN("invalid arg", K(exec_tenant_id), KR(ret));
|
|
} else if (tenant_name.empty()) {
|
|
if (!is_user_tenant(exec_tenant_id)) {
|
|
ret = OB_NOT_SUPPORTED;
|
|
LOG_WARN("can't operate tenant without tenant name using SYS/meta tenant session", KR(ret), K(tenant_name), K(exec_tenant_id));
|
|
LOG_USER_ERROR(OB_NOT_SUPPORTED, "omitting tenant name is ");
|
|
} else {
|
|
switch_tenant_id = exec_tenant_id;
|
|
}
|
|
} else {
|
|
// tenant_name not empty
|
|
if (OB_SYS_TENANT_ID != exec_tenant_id) {
|
|
ret = OB_INVALID_ARGUMENT;
|
|
LOG_WARN("can't specify tenant name using user tenant session", KR(ret), K(tenant_name), K(exec_tenant_id));
|
|
LOG_USER_ERROR(OB_INVALID_ARGUMENT, "tenant name, please don't specify tenant name");
|
|
} else {
|
|
if (OB_ISNULL(schema_service_)) {
|
|
ret = OB_ERR_UNEXPECTED;
|
|
LOG_ERROR("invalid schema service", KR(ret), KP(schema_service_));
|
|
} else {
|
|
share::schema::ObSchemaGetterGuard guard;
|
|
if (OB_FAIL(schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID, guard))) {
|
|
LOG_WARN("get_schema_guard failed", KR(ret));
|
|
} else if (OB_FAIL(guard.get_tenant_id(tenant_name, switch_tenant_id))) {
|
|
LOG_WARN("get_tenant_id failed", KR(ret), K(tenant_name), K(exec_tenant_id));
|
|
} else if (!is_user_tenant(switch_tenant_id)) {
|
|
ret = OB_INVALID_ARGUMENT;
|
|
LOG_WARN("only support switch user tenant", KR(ret), K(tenant_name), K(exec_tenant_id), K(switch_tenant_id));
|
|
LOG_USER_ERROR(OB_INVALID_ARGUMENT, "tenant name, only support operating user tenant");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
int ObPrimaryStandbyService::recover_tenant(const obrpc::ObRecoverTenantArg &arg)
|
|
{
|
|
int ret = OB_SUCCESS;
|
|
int64_t begin_time = ObTimeUtility::current_time();
|
|
uint64_t tenant_id = OB_INVALID_ID;
|
|
const char *alter_cluster_event = "recover_tenant";
|
|
CLUSTER_EVENT_ADD_CONTROL_START(ret, alter_cluster_event, "stmt_str", arg.get_stmt_str());
|
|
if (OB_FAIL(check_inner_stat_())) {
|
|
LOG_WARN("inner stat error", KR(ret), K_(inited));
|
|
} else if (!arg.is_valid()) {
|
|
ret = OB_INVALID_ARGUMENT;
|
|
LOG_WARN("invalid arg", K(arg), KR(ret));
|
|
} else if (OB_FAIL(get_target_tenant_id(arg.get_tenant_name(), arg.get_exec_tenant_id(), tenant_id))) {
|
|
LOG_WARN("failed to get_target_tenant_id", KR(ret), K(tenant_id), K(arg));
|
|
} else if (OB_FAIL(do_recover_tenant(arg, tenant_id))) {
|
|
LOG_WARN("failed to do_recover_tenant", KR(ret), K(tenant_id), K(arg));
|
|
}
|
|
|
|
int64_t cost = ObTimeUtility::current_time() - begin_time;
|
|
CLUSTER_EVENT_ADD_CONTROL_FINISH(ret, alter_cluster_event,
|
|
K(cost),
|
|
"stmt_str", arg.get_stmt_str());
|
|
|
|
return ret;
|
|
}
|
|
|
|
int ObPrimaryStandbyService::do_recover_tenant(const obrpc::ObRecoverTenantArg &arg, const uint64_t tenant_id)
|
|
{
|
|
int ret = OB_SUCCESS;
|
|
ObAllTenantInfo tenant_info;
|
|
ObSchemaGetterGuard schema_guard;
|
|
const uint64_t exec_tenant_id = gen_meta_tenant_id(tenant_id);
|
|
common::ObMySQLTransaction trans;
|
|
const ObSimpleTenantSchema *tenant_schema = nullptr;
|
|
ObLSRecoveryStatOperator ls_recovery_operator;
|
|
ObLSRecoveryStat sys_ls_recovery;
|
|
if (OB_FAIL(check_inner_stat_())) {
|
|
LOG_WARN("inner stat error", KR(ret), K_(inited));
|
|
} else if (!arg.is_valid()) {
|
|
ret = OB_INVALID_ARGUMENT;
|
|
LOG_WARN("invalid arg", K(arg), KR(ret));
|
|
} else if (OB_ISNULL(GCTX.srv_rpc_proxy_) || OB_ISNULL(schema_service_) || OB_ISNULL(sql_proxy_)) {
|
|
ret = OB_ERR_UNEXPECTED;
|
|
LOG_WARN("pointer is null", KR(ret), KP(GCTX.srv_rpc_proxy_), KP(schema_service_), KP(sql_proxy_));
|
|
} else if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id)) {
|
|
ret = OB_INVALID_ARGUMENT;
|
|
LOG_WARN("invalid argument", KR(ret), K(tenant_id));
|
|
} else if (OB_FAIL(schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID, schema_guard))) {
|
|
LOG_WARN("failed to get schema guard", KR(ret), K(tenant_id));
|
|
} else if (OB_FAIL(schema_guard.get_tenant_info(tenant_id, tenant_schema))) {
|
|
LOG_WARN("failed to get tenant info", KR(ret), K(tenant_id));
|
|
} else if (OB_ISNULL(tenant_schema)) {
|
|
ret = OB_ERR_UNEXPECTED;
|
|
LOG_WARN("tenant_schema is null", KR(ret), K(tenant_id), K(arg));
|
|
} else if (OB_FAIL(trans.start(sql_proxy_, exec_tenant_id))) {
|
|
LOG_WARN("failed to start trans", KR(ret), K(exec_tenant_id), K(tenant_id));
|
|
} else if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(tenant_id, &trans, true, tenant_info))) {
|
|
LOG_WARN("failed to load all tenant info", KR(ret), K(tenant_id));
|
|
} else if (!tenant_info.is_standby()) {
|
|
ret = OB_OP_NOT_ALLOW;
|
|
LOG_WARN("tenant role is not STANDBY", K(tenant_info));
|
|
LOG_USER_ERROR(OB_OP_NOT_ALLOW, "tenant role is not STANDBY, recover is");
|
|
} else if (!tenant_info.is_normal_status()) {
|
|
ret = OB_OP_NOT_ALLOW;
|
|
LOG_WARN("tenant switchover_status is not NORMAL", K(tenant_info));
|
|
LOG_USER_ERROR(OB_OP_NOT_ALLOW, "tenant switchover_status is not NORMAL, recover is");
|
|
} else if (OB_FAIL(ls_recovery_operator.get_ls_recovery_stat(tenant_id, share::SYS_LS,
|
|
true /*for_update*/, sys_ls_recovery, trans))) {
|
|
LOG_WARN("failed to get ls recovery stat", KR(ret), K(tenant_id));
|
|
} else if (obrpc::ObRecoverTenantArg::RecoverType::UNTIL == arg.get_type()
|
|
&& (arg.get_recovery_until_scn() < tenant_info.get_sync_scn()
|
|
|| arg.get_recovery_until_scn() < sys_ls_recovery.get_sync_scn())) {
|
|
ret = OB_OP_NOT_ALLOW;
|
|
LOG_WARN("recover before tenant sync_scn or SYS LS sync_scn is not allow", KR(ret), K(tenant_info),
|
|
K(tenant_id), K(arg), K(sys_ls_recovery));
|
|
LOG_USER_ERROR(OB_OP_NOT_ALLOW, "recover before tenant sync_scn or SYS LS sync_scn is");
|
|
} else if (tenant_schema->is_normal()) {
|
|
ObLogRestoreSourceMgr restore_source_mgr;
|
|
const SCN &recovery_until_scn = obrpc::ObRecoverTenantArg::RecoverType::UNTIL == arg.get_type() ?
|
|
arg.get_recovery_until_scn() : SCN::max(tenant_info.get_sync_scn(), sys_ls_recovery.get_sync_scn());
|
|
if (tenant_info.get_recovery_until_scn() == recovery_until_scn) {
|
|
LOG_WARN("recovery_until_scn is same with original", KR(ret), K(tenant_info), K(tenant_id), K(arg));
|
|
} else if (OB_FAIL(restore_source_mgr.init(tenant_id, &trans))) {
|
|
LOG_WARN("failed to init restore_source_mgr", KR(ret), K(tenant_id), K(arg));
|
|
} else if (OB_FAIL(restore_source_mgr.update_recovery_until_scn(recovery_until_scn))) {
|
|
LOG_WARN("failed to update_recovery_until_scn", KR(ret), K(tenant_id), K(arg));
|
|
} else if (OB_FAIL(ObAllTenantInfoProxy::update_tenant_recovery_until_scn(
|
|
tenant_id, trans, tenant_info.get_switchover_epoch(), recovery_until_scn))) {
|
|
LOG_WARN("failed to update_tenant_recovery_until_scn", KR(ret), K(tenant_id), K(arg));
|
|
}
|
|
} else {
|
|
ret = OB_OP_NOT_ALLOW;
|
|
LOG_WARN("tenant status is not normal, recover is not allowed", KR(ret), K(tenant_id), K(arg), KPC(tenant_schema));
|
|
LOG_USER_ERROR(OB_OP_NOT_ALLOW, "tenant status is not normal, recover is");
|
|
}
|
|
|
|
if (trans.is_started()) {
|
|
int tmp_ret = OB_SUCCESS;
|
|
if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) {
|
|
LOG_WARN("failed to commit trans", KR(ret), KR(tmp_ret));
|
|
ret = OB_SUCC(ret) ? tmp_ret : ret;
|
|
}
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
int ObPrimaryStandbyService::switch_to_primary(
|
|
const uint64_t tenant_id,
|
|
const obrpc::ObSwitchTenantArg::OpType &switch_optype)
|
|
{
|
|
int ret = OB_SUCCESS;
|
|
int64_t begin_time = ObTimeUtility::current_time();
|
|
ObAllTenantInfo tenant_info;
|
|
if (OB_FAIL(check_inner_stat_())) {
|
|
LOG_WARN("inner stat error", KR(ret), K_(inited));
|
|
} else if (OB_ISNULL(GCTX.srv_rpc_proxy_) || OB_ISNULL(sql_proxy_)) {
|
|
ret = OB_ERR_UNEXPECTED;
|
|
LOG_WARN("pointer is null", KR(ret), KP(GCTX.srv_rpc_proxy_), KP(sql_proxy_));
|
|
} else if (OB_UNLIKELY(obrpc::ObSwitchTenantArg::OpType::INVALID == switch_optype)) {
|
|
ret = OB_INVALID_ARGUMENT;
|
|
LOG_WARN("invalid switch_optype", KR(ret), K(switch_optype));
|
|
} else if (!is_user_tenant(tenant_id)) {
|
|
ret = OB_INVALID_ARGUMENT;
|
|
LOG_WARN("only support switch user tenant", KR(ret), K(tenant_id));
|
|
LOG_USER_ERROR(OB_INVALID_ARGUMENT, "tenant id, only support operating user tenant");
|
|
} else {
|
|
ObTenantRoleTransitionService role_transition_service(tenant_id, sql_proxy_, GCTX.srv_rpc_proxy_, switch_optype);
|
|
(void)role_transition_service.set_switchover_epoch(tenant_info.get_switchover_epoch());
|
|
if (OB_FAIL(role_transition_service.failover_to_primary())) {
|
|
LOG_WARN("failed to failover to primary", KR(ret), K(tenant_id));
|
|
}
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
int ObPrimaryStandbyService::switch_to_standby(
|
|
const uint64_t tenant_id,
|
|
const obrpc::ObSwitchTenantArg::OpType &switch_optype)
|
|
{
|
|
int ret = OB_SUCCESS;
|
|
ObAllTenantInfo tenant_info;
|
|
|
|
if (OB_FAIL(check_inner_stat_())) {
|
|
LOG_WARN("inner stat error", KR(ret), K_(inited));
|
|
} else if (OB_ISNULL(GCTX.srv_rpc_proxy_)) {
|
|
ret = OB_ERR_UNEXPECTED;
|
|
LOG_WARN("pointer is null", KR(ret), KP(GCTX.srv_rpc_proxy_));
|
|
} else if (OB_UNLIKELY(obrpc::ObSwitchTenantArg::OpType::INVALID == switch_optype)) {
|
|
ret = OB_INVALID_ARGUMENT;
|
|
LOG_WARN("invalid switch_optype", KR(ret), K(switch_optype));
|
|
} else if (!is_user_tenant(tenant_id)) {
|
|
ret = OB_INVALID_ARGUMENT;
|
|
LOG_WARN("only support switch user tenant", KR(ret), K(tenant_id));
|
|
LOG_USER_ERROR(OB_INVALID_ARGUMENT, "tenant id, only support operating user tenant");
|
|
} else if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(tenant_id, sql_proxy_, false, tenant_info))) {
|
|
LOG_WARN("failed to load tenant info", KR(ret), K(tenant_id));
|
|
} else if (tenant_info.is_standby() && tenant_info.is_normal_status()) {
|
|
LOG_INFO("already is standby tenant, no need switch", K(tenant_id), K(tenant_info));
|
|
} else {
|
|
switch(tenant_info.get_switchover_status().value()) {
|
|
case share::ObTenantSwitchoverStatus::NORMAL_STATUS: {
|
|
if (OB_FAIL(ret)) {
|
|
} else if (!tenant_info.is_primary()) {
|
|
ret = OB_OP_NOT_ALLOW;
|
|
LOG_WARN("unexpected tenant role", KR(ret), K(tenant_info));
|
|
LOG_USER_ERROR(OB_OP_NOT_ALLOW, "tenant role is not PRIMARY, switchover to standby is");
|
|
} else if (OB_FAIL(update_tenant_status_before_sw_to_standby_(
|
|
tenant_info.get_switchover_status(),
|
|
tenant_info.get_tenant_role(),
|
|
tenant_info.get_switchover_epoch(),
|
|
tenant_id,
|
|
tenant_info))) {
|
|
LOG_WARN("failed to update_tenant_status_before_sw_to_standby_", KR(ret), K(tenant_info),
|
|
K(tenant_id));
|
|
}
|
|
}
|
|
case share::ObTenantSwitchoverStatus::PREPARE_SWITCHING_TO_STANDBY_STATUS: {
|
|
if (OB_FAIL(ret)) {
|
|
} else if (OB_FAIL(switch_to_standby_prepare_ls_status_(tenant_id,
|
|
tenant_info.get_switchover_status(),
|
|
tenant_info.get_switchover_epoch(),
|
|
tenant_info))) {
|
|
LOG_WARN("failed to switch_to_standby_prepare_ls_status_", KR(ret), K(tenant_id), K(tenant_info));
|
|
}
|
|
}
|
|
case share::ObTenantSwitchoverStatus::SWITCHING_TO_STANDBY_STATUS: {
|
|
if (OB_FAIL(ret)) {
|
|
} else {
|
|
ObTenantRoleTransitionService role_transition_service(tenant_id, sql_proxy_, GCTX.srv_rpc_proxy_, switch_optype);
|
|
|
|
(void)role_transition_service.set_switchover_epoch(tenant_info.get_switchover_epoch());
|
|
if (OB_FAIL(role_transition_service.do_switch_access_mode_to_raw_rw(tenant_info))) {
|
|
LOG_WARN("failed to do_switch_access_mode", KR(ret), K(tenant_id), K(tenant_info));
|
|
} else if (OB_FAIL(role_transition_service.switchover_update_tenant_status(tenant_id,
|
|
false /* switch_to_standby */,
|
|
share::STANDBY_TENANT_ROLE,
|
|
tenant_info.get_switchover_status(),
|
|
share::NORMAL_SWITCHOVER_STATUS,
|
|
tenant_info.get_switchover_epoch(),
|
|
tenant_info))) {
|
|
LOG_WARN("fail to switchover_update_tenant_status", KR(ret), K(tenant_id), K(tenant_info));
|
|
} else {
|
|
(void)role_transition_service.broadcast_tenant_info(
|
|
ObTenantRoleTransitionConstants::SWITCH_TO_STANDBY_LOG_MOD_STR);
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
default: {
|
|
ret = OB_OP_NOT_ALLOW;
|
|
LOG_WARN("switchover status not match", KR(ret), K(tenant_info), K(tenant_id));
|
|
LOG_USER_ERROR(OB_OP_NOT_ALLOW, "switchover status not match, switchover to standby");
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
int ObPrimaryStandbyService::update_tenant_status_before_sw_to_standby_(
|
|
const ObTenantSwitchoverStatus cur_switchover_status,
|
|
const ObTenantRole cur_tenant_role,
|
|
const int64_t cur_switchover_epoch,
|
|
const uint64_t tenant_id,
|
|
ObAllTenantInfo &new_tenant_info)
|
|
{
|
|
int ret = OB_SUCCESS;
|
|
ObMySQLTransaction trans;
|
|
ObAllTenantInfo tenant_info;
|
|
int64_t new_switchover_ts = common::OB_INVALID_TIMESTAMP;
|
|
|
|
if (OB_UNLIKELY(!cur_switchover_status.is_valid()
|
|
|| !cur_tenant_role.is_valid()
|
|
|| !is_user_tenant(tenant_id))) {
|
|
ret = OB_INVALID_ARGUMENT;
|
|
LOG_WARN("invalid arg", KR(ret), K(tenant_id), K(cur_switchover_status), K(cur_tenant_role));
|
|
} else if (OB_FAIL(check_inner_stat_())) {
|
|
LOG_WARN("inner stat error", KR(ret), K_(inited));
|
|
} else {
|
|
const uint64_t exec_tenant_id = gen_meta_tenant_id(tenant_id);
|
|
if (OB_FAIL(trans.start(sql_proxy_, exec_tenant_id))) {
|
|
LOG_WARN("fail to start trans", KR(ret), K(tenant_id));
|
|
} else if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(
|
|
tenant_id, &trans, true, tenant_info))) {
|
|
LOG_WARN("failed to load tenant info", KR(ret), K(tenant_id));
|
|
} else if (OB_UNLIKELY(!tenant_info.get_recovery_until_scn().is_max())) {
|
|
ret = OB_OP_NOT_ALLOW;
|
|
LOG_WARN("recovery_until_scn has been changed ", KR(ret), K(tenant_id), K(tenant_info));
|
|
LOG_USER_ERROR(OB_OP_NOT_ALLOW, "recovery_until_scn has been changed, switchover to standby");
|
|
} else if (cur_switchover_status != tenant_info.get_switchover_status()) {
|
|
ret = OB_NEED_RETRY;
|
|
LOG_WARN("tenant not expect switchover status", KR(ret), K(tenant_info), K(cur_switchover_status));
|
|
} else if (cur_tenant_role != tenant_info.get_tenant_role()) {
|
|
ret = OB_NEED_RETRY;
|
|
LOG_WARN("tenant not expect tenant role", KR(ret), K(tenant_info), K(cur_tenant_role));
|
|
} else if (cur_switchover_epoch != tenant_info.get_switchover_epoch()) {
|
|
ret = OB_NEED_RETRY;
|
|
LOG_WARN("tenant not expect switchover epoch", KR(ret), K(tenant_info), K(cur_switchover_epoch));
|
|
} else if (OB_FAIL(ObAllTenantInfoProxy::update_tenant_role(
|
|
tenant_id, &trans, cur_switchover_epoch,
|
|
PRIMARY_TENANT_ROLE, cur_switchover_status,
|
|
share::PREP_SWITCHING_TO_STANDBY_SWITCHOVER_STATUS, new_switchover_ts))) {
|
|
LOG_WARN("failed to update tenant role", KR(ret), K(tenant_id), K(cur_switchover_epoch), K(tenant_info));
|
|
} else if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(
|
|
tenant_id, &trans, true, new_tenant_info))) {
|
|
LOG_WARN("failed to load tenant info", KR(ret), K(tenant_id));
|
|
}
|
|
}
|
|
|
|
if (trans.is_started()) {
|
|
int temp_ret = OB_SUCCESS;
|
|
if (OB_SUCCESS != (temp_ret = trans.end(OB_SUCC(ret)))) {
|
|
LOG_WARN("trans end failed", "is_commit", OB_SUCCESS == ret, KR(temp_ret));
|
|
ret = OB_SUCC(ret) ? temp_ret : ret;
|
|
}
|
|
}
|
|
|
|
CLUSTER_EVENT_ADD_LOG(ret, "update tenant before switchover to standby",
|
|
"tenant id", tenant_id,
|
|
"old switchover#", cur_switchover_epoch,
|
|
"new switchover#", tenant_info.get_switchover_epoch(),
|
|
K(cur_switchover_status), K(cur_tenant_role));
|
|
return ret;
|
|
}
|
|
|
|
int ObPrimaryStandbyService::switch_to_standby_prepare_ls_status_(
|
|
const uint64_t tenant_id,
|
|
const ObTenantSwitchoverStatus &status,
|
|
const int64_t switchover_epoch,
|
|
ObAllTenantInfo &new_tenant_info)
|
|
{
|
|
int ret = OB_SUCCESS;
|
|
ObMySQLTransaction trans;
|
|
ObLSAttr sys_ls_attr;
|
|
share::ObLSAttrOperator ls_operator(tenant_id, sql_proxy_);
|
|
share::schema::ObSchemaGetterGuard schema_guard;
|
|
const share::schema::ObTenantSchema *tenant_schema = NULL;
|
|
int64_t new_switchover_epoch = OB_INVALID_VERSION;
|
|
|
|
if (!is_user_tenant(tenant_id)) {
|
|
ret = OB_INVALID_ARGUMENT;
|
|
LOG_WARN("invalid argument", KR(ret), K(tenant_id));
|
|
} else if (OB_ISNULL(GCTX.schema_service_)) {
|
|
ret = OB_ERR_UNEXPECTED;
|
|
LOG_WARN("schema_service_ is NULL", KR(ret));
|
|
} else if (OB_FAIL(check_inner_stat_())) {
|
|
LOG_WARN("inner stat error", KR(ret), K_(inited));
|
|
} else if (OB_UNLIKELY(!status.is_prepare_switching_to_standby_status())) {
|
|
ret = OB_OP_NOT_ALLOW;
|
|
LOG_WARN("switchover status not match, switchover to standby not allow", KR(ret), K(status));
|
|
LOG_USER_ERROR(OB_OP_NOT_ALLOW, "switchover status not match, switchover to standby");
|
|
} else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID, schema_guard))) {
|
|
LOG_WARN("fail to get schema guard", KR(ret));
|
|
} else if (OB_FAIL(schema_guard.get_tenant_info(tenant_id, tenant_schema))) {
|
|
LOG_WARN("failed to get tenant ids", KR(ret), K(tenant_id));
|
|
} else if (OB_ISNULL(tenant_schema)) {
|
|
ret = OB_TENANT_NOT_EXIST;
|
|
LOG_WARN("tenant not exist", KR(ret), K(tenant_id));
|
|
} else {
|
|
ObTenantLSInfo tenant_stat(GCTX.sql_proxy_, tenant_schema, tenant_id,
|
|
GCTX.srv_rpc_proxy_, GCTX.lst_operator_);
|
|
/* lock SYS_LS to get accurate LS list, then fix ls status to make ls status consistency
|
|
between __all_ls&__all_ls_status.
|
|
Refer to ls operator, insert/update/delete of ls table are executed in the SYS_LS lock
|
|
and normal switchover status */
|
|
if (OB_FAIL(tenant_stat.process_ls_status_missmatch(true/* lock_sys_ls */,
|
|
share::PREP_SWITCHING_TO_STANDBY_SWITCHOVER_STATUS))) {
|
|
LOG_WARN("failed to process_ls_status_missmatch", KR(ret));
|
|
} else if (OB_FAIL(ObAllTenantInfoProxy::update_tenant_role(
|
|
tenant_id, sql_proxy_, switchover_epoch,
|
|
share::STANDBY_TENANT_ROLE, status,
|
|
share::SWITCHING_TO_STANDBY_SWITCHOVER_STATUS, new_switchover_epoch))) {
|
|
LOG_WARN("failed to update tenant role", KR(ret), K(tenant_id), K(switchover_epoch));
|
|
} else if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(
|
|
tenant_id, sql_proxy_, false, new_tenant_info))) {
|
|
LOG_WARN("failed to load tenant info", KR(ret), K(tenant_id));
|
|
} else if (OB_UNLIKELY(new_tenant_info.get_switchover_epoch() != new_switchover_epoch)) {
|
|
ret = OB_NEED_RETRY;
|
|
LOG_WARN("switchover is concurrency", KR(ret), K(switchover_epoch), K(new_tenant_info));
|
|
}
|
|
|
|
DEBUG_SYNC(SWITCHING_TO_STANDBY);
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
int ObPrimaryStandbyService::write_upgrade_barrier_log(
|
|
ObMySQLTransaction &trans,
|
|
const uint64_t tenant_id,
|
|
const uint64_t data_version)
|
|
{
|
|
int ret = OB_SUCCESS;
|
|
ObStandbyUpgrade primary_data_version(data_version);
|
|
observer::ObInnerSQLConnection *inner_conn = static_cast<observer::ObInnerSQLConnection *>(trans.get_connection());
|
|
if (OB_FAIL(check_inner_stat_())) {
|
|
LOG_WARN("inner stat error", KR(ret), K_(inited));
|
|
} else if (OB_ISNULL(inner_conn)) {
|
|
ret = OB_ERR_UNEXPECTED;
|
|
LOG_WARN("connection or trans service is null", KR(ret), KP(inner_conn));
|
|
} else if (!is_user_tenant(tenant_id)) {
|
|
ret = OB_INVALID_ARGUMENT;
|
|
LOG_WARN("not user tenant_id", KR(ret), K(tenant_id));
|
|
} else if (!ObClusterVersion::check_version_valid_(data_version)) {
|
|
ret = OB_INVALID_ARGUMENT;
|
|
LOG_WARN("invalid data_version", KR(ret), K(data_version));
|
|
} else {
|
|
const int64_t length = primary_data_version.get_serialize_size();
|
|
char *buf = NULL;
|
|
int64_t pos = 0;
|
|
ObArenaAllocator allocator("StandbyUpgrade");
|
|
if (OB_ISNULL(buf = static_cast<char *>(allocator.alloc(length)))) {
|
|
ret = OB_ALLOCATE_MEMORY_FAILED;
|
|
LOG_WARN("fail to alloc buf", KR(ret), K(length));
|
|
} else if (OB_FAIL(primary_data_version.serialize(buf, length, pos))) {
|
|
LOG_WARN("failed to serialize", KR(ret), K(primary_data_version), K(length), K(pos));
|
|
} else if (OB_UNLIKELY(pos > length)) {
|
|
ret = OB_SIZE_OVERFLOW;
|
|
LOG_WARN("serialize error", KR(ret), K(pos), K(length), K(primary_data_version));
|
|
} else if (OB_FAIL(inner_conn->register_multi_data_source(
|
|
tenant_id, SYS_LS, transaction::ObTxDataSourceType::STANDBY_UPGRADE,
|
|
buf, length))) {
|
|
LOG_WARN("failed to register tx data", KR(ret), K(tenant_id));
|
|
}
|
|
LOG_INFO("write_upgrade_barrier_log finished", KR(ret), K(tenant_id), K(primary_data_version), K(length), KPHEX(buf, length));
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
}
|
|
}
|