hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity
Files
9c1c501715f2bc11cf488eafa62697ba8758cf88
oceanbase/src/rootserver/ob_server_zone_op_service.cpp
linqiucen 9c1c501715 all_server_list improvement
2023-05-24 06:41:30 +00:00

757 lines
31 KiB
C++
Raw Blame History

/**
* Copyright (c) 2022 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX RS
#include "ob_server_zone_op_service.h"
#include "share/ob_zone_table_operation.h"
#include "share/ob_service_epoch_proxy.h"
#include "share/ob_max_id_fetcher.h"
#include "lib/mysqlclient/ob_mysql_transaction.h" // ObMySQLTransaction
#include "lib/utility/ob_tracepoint.h" // ERRSIM
#include "rootserver/ob_root_service.h" // callback
#include "share/ob_all_server_tracer.h"
#include "rootserver/ob_server_manager.h"
namespace oceanbase
{
using namespace common;
using namespace share;
using namespace obrpc;
namespace rootserver
{
ObServerZoneOpService::ObServerZoneOpService()
: is_inited_(false),
server_change_callback_(NULL),
rpc_proxy_(NULL),
sql_proxy_(NULL),
lst_operator_(NULL),
unit_manager_(NULL)
{
}
ObServerZoneOpService::~ObServerZoneOpService()
{
}
int ObServerZoneOpService::init(
ObIServerChangeCallback &server_change_callback,
ObSrvRpcProxy &rpc_proxy,
ObLSTableOperator &lst_operator,
ObUnitManager &unit_manager,
ObMySQLProxy &sql_proxy
)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(is_inited_)) {
ret = OB_INIT_TWICE;
LOG_WARN("server zone operation service has been inited already", KR(ret), K(is_inited_));
} else if (OB_FAIL(st_operator_.init(&sql_proxy))) {
LOG_WARN("fail to init server table operator", KR(ret));
} else {
server_change_callback_ = &server_change_callback;
rpc_proxy_ = &rpc_proxy;
sql_proxy_ = &sql_proxy;
lst_operator_ = &lst_operator;
unit_manager_ = &unit_manager;
is_inited_ = true;
}
return ret;
}
int ObServerZoneOpService::add_servers(const ObIArray<ObAddr> &servers, const ObZone &zone, bool is_bootstrap)
{
int ret = OB_SUCCESS;
uint64_t sys_tenant_data_version = 0;
ObCheckServerForAddingServerArg rpc_arg;
ObCheckServerForAddingServerResult rpc_result;
ObZone picked_zone;
ObTimeoutCtx ctx;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", KR(ret), K(is_inited_));
} else if (OB_FAIL(GET_MIN_DATA_VERSION(OB_SYS_TENANT_ID, sys_tenant_data_version))) {
LOG_WARN("fail to get sys tenant's min data version", KR(ret));
} else if (OB_ISNULL(rpc_proxy_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("rpc_proxy_ is null", KR(ret), KP(rpc_proxy_));
} else if (OB_FAIL(rootserver::ObRootUtils::get_rs_default_timeout_ctx(ctx))) {
LOG_WARN("fail to get timeout ctx", KR(ret), K(ctx));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < servers.count(); ++i) {
const ObAddr &addr = servers.at(i);
int64_t timeout = ctx.get_timeout();
uint64_t server_id = OB_INVALID_ID;
const int64_t ERR_MSG_BUF_LEN = OB_MAX_SERVER_ADDR_SIZE + 100;
char non_empty_server_err_msg[ERR_MSG_BUF_LEN] = "";
int64_t pos = 0;
rpc_arg.reset();
if (OB_UNLIKELY(timeout <= 0)) {
ret = OB_TIMEOUT;
LOG_WARN("ctx time out", KR(ret), K(timeout));
} else if (OB_FAIL(databuff_printf(
non_empty_server_err_msg,
ERR_MSG_BUF_LEN,
pos,
"add non-empty server %s",
to_cstring(addr)))) {
LOG_WARN("fail to execute databuff_printf", KR(ret), K(addr));
} else if (OB_FAIL(fetch_new_server_id_(server_id))) {
// fetch a new server id and insert the server into __all_server table
LOG_WARN("fail to fetch new server id", KR(ret));
} else if (OB_UNLIKELY(!is_valid_server_id(server_id))) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("server id is invalid", KR(ret), K(server_id));
} else if (OB_FAIL(rpc_arg.init(
ObCheckServerForAddingServerArg::ADD_SERVER,
sys_tenant_data_version,
server_id))) {
LOG_WARN("fail to init rpc arg", KR(ret), K(sys_tenant_data_version), K(server_id));
} else if (OB_FAIL(rpc_proxy_->to(addr)
.timeout(timeout)
.check_server_for_adding_server(rpc_arg, rpc_result))) {
LOG_WARN("fail to check whether the server is empty", KR(ret), K(addr));
} else if (!rpc_result.get_is_server_empty()) {
ret = OB_OP_NOT_ALLOW;
LOG_WARN("adding non-empty server is not allowed", KR(ret));
LOG_USER_ERROR(OB_OP_NOT_ALLOW, non_empty_server_err_msg);
} else if (OB_FAIL(zone_checking_for_adding_server_(zone, rpc_result.get_zone(), picked_zone))) {
LOG_WARN("zone checking for adding server is failed", KR(ret), K(zone), K(rpc_result.get_zone()));
} else if (OB_FAIL(add_server_(
addr,
server_id,
picked_zone,
rpc_result.get_sql_port(),
rpc_result.get_build_version()))) {
LOG_WARN("add_server failed", KR(ret), K(addr), K(server_id), K(picked_zone), "sql_port",
rpc_result.get_sql_port(), "build_version", rpc_result.get_build_version());
} else {}
}
}
return ret;
}
int ObServerZoneOpService::delete_servers(
const ObIArray<ObAddr> &servers,
const ObZone &zone)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", KR(ret), K(is_inited_));
} else if (OB_ISNULL(GCTX.root_service_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("root_service_ is null", KR(ret), KP(GCTX.root_service_));
} else if (OB_UNLIKELY(servers.count() <= 0)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KR(ret), K(servers));
} else if (OB_FAIL(check_server_have_enough_resource_for_delete_server_(servers, zone))) {
LOG_WARN("not enough resource, cannot delete servers", KR(ret), K(servers), K(zone));
} else if (OB_FAIL(GCTX.root_service_->check_all_ls_has_leader("delete server"))) {
LOG_WARN("fail to check all ls has leader", KR(ret));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < servers.count(); ++i) {
if (OB_FAIL(delete_server_(servers.at(i), zone))) {
LOG_WARN("delete_server failed", "server", servers.at(i), "zone", zone, KR(ret));
}
}
}
return ret;
}
int ObServerZoneOpService::cancel_delete_servers(
const ObIArray<ObAddr> &servers,
const ObZone &zone)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", KR(ret), K(is_inited_));
} else if (OB_ISNULL(unit_manager_) || OB_ISNULL(sql_proxy_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unit_manager_ or sql_proxy_ or server_change_callback_ is null", KR(ret),
KP(unit_manager_), KP(sql_proxy_));
} else {
ObServerInfoInTable server_info_in_table;
for (int64_t i = 0; OB_SUCC(ret) && i < servers.count(); ++i) {
const ObAddr &server = servers.at(i);
const int64_t now = ObTimeUtility::current_time();
ObMySQLTransaction trans;
server_info_in_table.reset();
if (OB_FAIL(trans.start(sql_proxy_, OB_SYS_TENANT_ID))) {
LOG_WARN("fail to start trans", KR(ret));
} else if (OB_FAIL(check_and_end_delete_server_(trans, server, zone, true /* is_cancel */, server_info_in_table))) {
LOG_WARN("fail to check and end delete server", KR(ret), K(server), K(zone));
} else if (OB_FAIL(ObServerTableOperator::update_status(
trans,
server,
ObServerStatus::OB_SERVER_DELETING,
server_info_in_table.is_alive() ? ObServerStatus::OB_SERVER_ACTIVE : ObServerStatus::OB_SERVER_INACTIVE))) {
LOG_WARN("fail to update status in __all_server table", KR(ret),
K(server), K(server_info_in_table));
} else if (OB_FAIL(unit_manager_->cancel_migrate_out_units(server))) {
LOG_WARN("unit_manager_ cancel_migrate_out_units failed", KR(ret), K(server));
}
(void) end_trans_and_on_server_change_(ret, trans, "cancel_delete_server", server, server_info_in_table.get_zone(), now);
}
}
return ret;
}
int ObServerZoneOpService::finish_delete_server(
const ObAddr &server,
const ObZone &zone)
{
int ret = OB_SUCCESS;
ObServerInfoInTable server_info_in_table;
const int64_t now = ObTimeUtility::current_time();
ObMySQLTransaction trans;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", KR(ret), K(is_inited_));
} else if (OB_ISNULL(sql_proxy_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("sql_proxy_ is null", KR(ret), KP(sql_proxy_));
} else if (OB_FAIL(trans.start(sql_proxy_, OB_SYS_TENANT_ID))) {
LOG_WARN("fail to start trans", KR(ret));
} else if (OB_FAIL(check_and_end_delete_server_(trans, server, zone, false /* is_cancel */, server_info_in_table))) {
LOG_WARN("fail to check and end delete server", KR(ret), K(server), K(zone));
} else if (OB_FAIL(ObServerManager::try_delete_server_working_dir(
server_info_in_table.get_zone(),
server,
server_info_in_table.get_server_id()))) {
LOG_WARN("fail to delete server working dir", KR(ret), K(server_info_in_table));
} else if (OB_FAIL(st_operator_.remove(server, trans))) {
LOG_WARN("fail to remove this server from __all_server table", KR(ret), K(server));
}
(void) end_trans_and_on_server_change_(ret, trans, "finish_delete_server", server, server_info_in_table.get_zone(), now);
return ret;
}
int ObServerZoneOpService::stop_servers(
const ObIArray<ObAddr> &servers,
const ObZone &zone,
const obrpc::ObAdminServerArg::AdminServerOp &op)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", KR(ret), K(is_inited_));
} else if (OB_FAIL(stop_server_precheck(servers, op))) {
LOG_WARN("fail to precheck stop server", KR(ret), K(servers), K(zone));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < servers.count(); i++) {
const ObAddr &server = servers.at(i);
if (OB_FAIL(start_or_stop_server_(server, zone, op))) {
LOG_WARN("fail to stop server", KR(ret), K(server), K(zone));
}
}
}
return ret;
}
int ObServerZoneOpService::start_servers(
const ObIArray<ObAddr> &servers,
const ObZone &zone)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", KR(ret), K(is_inited_));
} else if (OB_UNLIKELY(servers.count() <= 0)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("servers' count is zero", KR(ret), K(servers));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < servers.count(); ++i) {
const ObAddr &server = servers.at(i);
if (OB_FAIL(start_or_stop_server_(server, zone, ObAdminServerArg::START))) {
LOG_WARN("fail to start server", KR(ret), K(server), K(zone));
}
}
}
return ret;
}
int ObServerZoneOpService::stop_server_precheck(
const ObIArray<ObAddr> &servers,
const obrpc::ObAdminServerArg::AdminServerOp &op)
{
int ret = OB_SUCCESS;
ObZone zone;
bool is_same_zone = false;
bool is_all_stopped = false;
ObArray<ObServerInfoInTable> all_servers_info_in_table;
ObServerInfoInTable server_info;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", KR(ret), K(is_inited_));
} else if (OB_UNLIKELY(servers.count() <= 0)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("servers' count is zero", KR(ret), K(servers));
} else if (OB_ISNULL(GCTX.root_service_) || OB_ISNULL(sql_proxy_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("GCTX.root_service_ or sql_proxy_ is null", KR(ret), KP(GCTX.root_service_), KP(sql_proxy_));
} else if (OB_FAIL(ObServerTableOperator::get(*sql_proxy_, all_servers_info_in_table))) {
LOG_WARN("fail to read __all_server table", KR(ret), KP(sql_proxy_));
} else if (OB_FAIL(check_zone_and_server_(
all_servers_info_in_table,
servers,
is_same_zone,
is_all_stopped))) {
LOG_WARN("fail to check zone and server", KR(ret), K(all_servers_info_in_table), K(servers));
} else if (is_all_stopped) {
//nothing todo
} else if (!is_same_zone) {
ret = OB_STOP_SERVER_IN_MULTIPLE_ZONES;
LOG_WARN("can not stop servers in multiple zones", KR(ret), K(server_info), K(servers));
} else if (OB_FAIL((ObRootUtils::find_server_info(all_servers_info_in_table, servers.at(0), server_info)))) {
LOG_WARN("fail to find server info", KR(ret), K(all_servers_info_in_table), K(servers.at(0)));
} else {
const ObZone &zone = server_info.get_zone();
if (ObAdminServerArg::ISOLATE == op) {
//"Isolate server" does not need to check the total number and status of replicas; it cannot be restarted later;
if (OB_FAIL(GCTX.root_service_->check_can_stop(zone, servers, false /*is_stop_zone*/))) {
LOG_WARN("fail to check can stop", KR(ret), K(zone), K(servers), K(op));
if (OB_OP_NOT_ALLOW == ret) {
LOG_USER_ERROR(OB_OP_NOT_ALLOW, "Stop all servers in primary region is");
}
}
} else {
if (ObRootUtils::have_other_stop_task(zone)) {
ret = OB_STOP_SERVER_IN_MULTIPLE_ZONES;
LOG_WARN("can not stop servers in multiple zones", KR(ret), K(zone), K(servers), K(op));
LOG_USER_ERROR(OB_STOP_SERVER_IN_MULTIPLE_ZONES,
"cannot stop server or stop zone in multiple zones");
} else if (OB_FAIL(GCTX.root_service_->check_majority_and_log_in_sync(
servers,
ObAdminServerArg::FORCE_STOP == op,/*skip_log_sync_check*/
"stop server"))) {
LOG_WARN("fail to check majority and log in-sync", KR(ret), K(zone), K(servers), K(op));
}
}
}
return ret;
}
int ObServerZoneOpService::zone_checking_for_adding_server_(
const ObZone &command_zone,
const ObZone &rpc_zone,
ObZone &picked_zone)
{
int ret = OB_SUCCESS;
// command_zone: the zone specified in the system command ADD SERVER
// rpc_zone: the zone specified in the server's local config and send to rs via rpc
// picked_zone: the zone we will use in add_server
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", KR(ret), K(is_inited_));
} else if (OB_UNLIKELY(rpc_zone.is_empty())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("rpc_zone cannot be empty. It implies that server's local config zone is empty.",
KR(ret), K(rpc_zone));
} else if (!command_zone.is_empty() && command_zone != rpc_zone) {
ret = OB_SERVER_ZONE_NOT_MATCH;
LOG_WARN("the zone specified in the server's local config is not the same as"
" the zone specified in the command", KR(ret), K(command_zone), K(rpc_zone));
} else if (OB_FAIL(picked_zone.assign(rpc_zone))) {
LOG_WARN("fail to assign picked_zone", KR(ret), K(rpc_zone));
} else {}
return ret;
}
int ObServerZoneOpService::add_server_(
const ObAddr &server,
const uint64_t server_id,
const ObZone &zone,
const int64_t sql_port,
const ObServerInfoInTable::ObBuildVersion &build_version)
{
int ret = OB_SUCCESS;
bool is_active = false;
const int64_t now = ObTimeUtility::current_time();
ObServerInfoInTable server_info_in_table;
ObMySQLTransaction trans;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", KR(ret), K(is_inited_));
} else if (OB_UNLIKELY(!server.is_valid()
|| !is_valid_server_id(server_id)
|| zone.is_empty()
|| sql_port <= 0
|| build_version.is_empty())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KR(ret), K(server), K(server_id), K(zone), K(sql_port), K(build_version));
} else if (OB_ISNULL(sql_proxy_) || OB_ISNULL(server_change_callback_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("sql_proxy_ or server_change_callback_ is null", KR(ret),
KP(sql_proxy_), KP(server_change_callback_));
} else if (OB_FAIL(trans.start(sql_proxy_, OB_SYS_TENANT_ID))) {
LOG_WARN("fail to start trans", KR(ret));
} else if (OB_FAIL(check_and_update_service_epoch_(trans))) {
LOG_WARN("fail to check and update service epoch", KR(ret));
} else if (OB_FAIL(ObZoneTableOperation::check_zone_active(trans, zone, is_active))){
// we do not need to lock the zone info in __all_zone table
// all server/zone operations are mutually exclusive since we locked the service epoch
LOG_WARN("fail to check whether the zone is active", KR(ret), K(zone));
} else if (OB_UNLIKELY(!is_active)) {
ret = OB_ZONE_NOT_ACTIVE;
LOG_WARN("the zone is not active", KR(ret), K(zone), K(is_active));
} else if (OB_FAIL(ObServerTableOperator::get(trans, server, server_info_in_table))) {
if (OB_SERVER_NOT_IN_WHITE_LIST == ret) {
ret = OB_SUCCESS;
} else {
LOG_WARN("fail to get server_info in table", KR(ret), K(server));
}
} else {
ret = OB_ENTRY_EXIST;
LOG_WARN("server exists", KR(ret), K(server_info_in_table));
}
if (FAILEDx(server_info_in_table.init(
server,
server_id,
zone,
sql_port,
false, /* with_rootserver */
ObServerStatus::OB_SERVER_ACTIVE,
build_version,
0, /* stop_time */
0, /* start_service_time */
0 /* last_offline_time */))) {
LOG_WARN("fail to init server info in table", KR(ret), K(server), K(server_id), K(zone),
K(sql_port), K(build_version), K(now));
} else if (OB_FAIL(ObServerTableOperator::insert(trans, server_info_in_table))) {
LOG_WARN("fail to insert server info into __all_server table", KR(ret), K(server_info_in_table));
}
(void) end_trans_and_on_server_change_(ret, trans, "add_server", server, zone, now);
return ret;
}
int ObServerZoneOpService::delete_server_(
const common::ObAddr &server,
const ObZone &zone)
{
int ret = OB_SUCCESS;
ObServerInfoInTable server_info_in_table;
const int64_t now = ObTimeUtility::current_time();
char ip[OB_MAX_SERVER_ADDR_SIZE] = "";
ObMySQLTransaction trans;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", KR(ret), K(is_inited_));
} else if (OB_UNLIKELY(!server.is_valid() || !server.ip_to_string(ip, sizeof(ip)))) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KR(ret), K(server));
} else if (OB_ISNULL(sql_proxy_) || OB_ISNULL(server_change_callback_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("sql_proxy_ or server_change_callback_ is null", KR(ret),
KP(sql_proxy_), KP(server_change_callback_));
} else if (OB_FAIL(trans.start(sql_proxy_, OB_SYS_TENANT_ID))) {
LOG_WARN("fail to start trans", KR(ret));
} else if (OB_FAIL(check_and_update_service_epoch_(trans))) {
LOG_WARN("fail to check and update service epoch", KR(ret));
} else if (OB_FAIL(ObServerTableOperator::get(trans, server, server_info_in_table))) {
LOG_WARN("fail to get server_info in table", KR(ret), K(server));
} else if (!zone.is_empty() && zone != server_info_in_table.get_zone()) {
ret = OB_SERVER_ZONE_NOT_MATCH;
LOG_WARN("zone not matches", KR(ret), K(server), K(zone), K(server_info_in_table));
} else if (OB_UNLIKELY(server_info_in_table.is_deleting())) {
ret = OB_SERVER_ALREADY_DELETED;
LOG_WARN("the server has been deleted", KR(ret), K(server_info_in_table));
} else {
int64_t job_id = RS_JOB_CREATE(DELETE_SERVER, trans, "svr_ip", ip, "svr_port", server.get_port());
if (job_id < 1) {
ret = OB_SQL_OPT_ERROR;
LOG_WARN("insert into all_rootservice_job failed ", K(ret));
} else if (OB_FAIL(ObServerTableOperator::update_status(
trans,
server,
server_info_in_table.get_status(),
ObServerStatus::OB_SERVER_DELETING))) {
LOG_WARN("fail to update status", KR(ret), K(server), K(server_info_in_table));
}
}
(void) end_trans_and_on_server_change_(ret, trans, "delete_server", server, server_info_in_table.get_zone(), now);
return ret;
}
int ObServerZoneOpService::check_and_end_delete_server_(
common::ObMySQLTransaction &trans,
const common::ObAddr &server,
const ObZone &zone,
const bool is_cancel,
share::ObServerInfoInTable &server_info)
{
int ret = OB_SUCCESS;
server_info.reset();
char ip[OB_MAX_SERVER_ADDR_SIZE] = "";
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", KR(ret), K(is_inited_));
} else if (OB_UNLIKELY(!server.is_valid() || !server.ip_to_string(ip, sizeof(ip)))) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KR(ret), K(server));
} else if (OB_FAIL(check_and_update_service_epoch_(trans))) {
LOG_WARN("fail to check and update service epoch", KR(ret));
} else if (OB_FAIL(ObServerTableOperator::get(trans, server, server_info))) {
LOG_WARN("fail to get server_info in table", KR(ret), K(server));
} else if (!zone.is_empty() && zone != server_info.get_zone()) {
ret = OB_SERVER_ZONE_NOT_MATCH;
LOG_WARN("zone not matches", KR(ret), K(server), K(zone), K(server_info));
} else if (OB_UNLIKELY(!server_info.is_deleting())) {
ret = OB_SERVER_NOT_DELETING;
LOG_ERROR("server is not in deleting status, cannot be removed from __all_server table",
KR(ret), K(server_info));
} else {
ObRsJobInfo job_info;
ret = RS_JOB_FIND(job_info, trans, "job_type", "DELETE_SERVER",
"job_status", "INPROGRESS",
"svr_ip", ip, "svr_port", server.get_port());
if (OB_SUCC(ret) && job_info.job_id_ > 0) {
int tmp_ret = is_cancel ? OB_CANCELED : OB_SUCCESS;
if (OB_FAIL(RS_JOB_COMPLETE(job_info.job_id_, tmp_ret, trans))) {
LOG_WARN("fail to all_rootservice_job" , KR(ret), K(server));
}
} else {
LOG_WARN("failed to find job", KR(ret), K(server));
}
}
return ret;
}
int ObServerZoneOpService::start_or_stop_server_(
const common::ObAddr &server,
const ObZone &zone,
const obrpc::ObAdminServerArg::AdminServerOp &op)
{
int ret = OB_SUCCESS;
const int64_t now = ObTimeUtility::current_time();
ObServerInfoInTable server_info;
ObMySQLTransaction trans;
bool is_start = (ObAdminServerArg::START == op);
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", KR(ret), K(is_inited_));
} else if (OB_UNLIKELY(!server.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KR(ret), K(server));
} else if (OB_ISNULL(sql_proxy_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("sql_proxy_ is null", KR(ret), KP(sql_proxy_));
} else if (OB_FAIL(trans.start(sql_proxy_, OB_SYS_TENANT_ID))) {
LOG_WARN("fail to start trans", KR(ret));
} else if (OB_FAIL(check_and_update_service_epoch_(trans))) {
LOG_WARN("fail to check and update service epoch", KR(ret));
} else if (OB_FAIL(ObServerTableOperator::get(trans, server, server_info))) {
LOG_WARN("fail to get server_info", KR(ret), K(server));
} else if (!zone.is_empty() && zone != server_info.get_zone()) {
ret = OB_SERVER_ZONE_NOT_MATCH;
LOG_WARN("zone not matches", KR(ret), K(server), K(zone), K(server_info));
} else if (ObAdminServerArg::STOP == op || ObAdminServerArg::FORCE_STOP == op) {
// check again, if there exists stopped servers in other zones
if (ObRootUtils::have_other_stop_task(server_info.get_zone())) {
ret = OB_STOP_SERVER_IN_MULTIPLE_ZONES;
LOG_WARN("can not stop servers in multiple zones", KR(ret), K(server_info.get_zone()));
LOG_USER_ERROR(OB_STOP_SERVER_IN_MULTIPLE_ZONES,
"cannot stop server or stop zone in multiple zones");
}
}
if (OB_SUCC(ret)) {
int64_t new_stop_time = is_start ? 0 : now;
int64_t old_stop_time = server_info.get_stop_time();
if ((is_start && 0 != old_stop_time) || (!is_start && 0 == old_stop_time)) {
if (OB_FAIL(ObServerTableOperator::update_stop_time(
trans,
server,
old_stop_time,
new_stop_time))) {
LOG_WARN("fail to update stop_time", KR(ret), K(server), K(old_stop_time), K(new_stop_time));
}
}
LOG_INFO("update stop time", KR(ret), K(server_info),
K(old_stop_time), K(new_stop_time), K(op), K(is_start));
}
const char *op_print_str = is_start ? "start_server" : "stop_server";
(void) end_trans_and_on_server_change_(ret, trans, op_print_str, server, server_info.get_zone(), now);
return ret;
}
int ObServerZoneOpService::construct_rs_list_arg(ObRsListArg &rs_list_arg)
{
int ret = OB_SUCCESS;
ObLSInfo ls_info;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", KR(ret), K(is_inited_));
} else if (OB_ISNULL(lst_operator_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("lst operator is null", KR(ret), KP(lst_operator_));
} else if (OB_FAIL(lst_operator_->get(
GCONF.cluster_id,
OB_SYS_TENANT_ID,
SYS_LS,
share::ObLSTable::DEFAULT_MODE,
ls_info))) {
LOG_WARN("fail to get ls info", KR(ret));
} else {
rs_list_arg.master_rs_ = GCONF.self_addr_;
FOREACH_CNT_X(replica, ls_info.get_replicas(), OB_SUCC(ret)) {
if (replica->get_server() == GCONF.self_addr_
|| (replica->is_in_service()
&& ObReplicaTypeCheck::is_paxos_replica_V2(replica->get_replica_type()))) {
if (OB_FAIL(rs_list_arg.rs_list_.push_back(replica->get_server()))) {
LOG_WARN("fail to push a server into rs list", KR(ret), K(replica->get_server()));
}
}
}
}
return ret;
}
int ObServerZoneOpService::check_and_update_service_epoch_(ObMySQLTransaction &trans)
{
int ret = OB_SUCCESS;
int64_t service_epoch_in_table = palf::INVALID_PROPOSAL_ID;
int64_t proposal_id = palf::INVALID_PROPOSAL_ID;
ObRole role;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", KR(ret), K(is_inited_));
} else if (OB_FAIL(ObRootUtils::get_proposal_id_from_sys_ls(proposal_id, role))) {
LOG_WARN("fail to get proposal id from sys ls", KR(ret));
} else if (ObRole::LEADER != role) {
ret = OB_NOT_MASTER;
LOG_WARN("not leader ls", KR(ret), K(proposal_id), K(service_epoch_in_table), K(role));
} else if (palf::INVALID_PROPOSAL_ID == proposal_id) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid proposal id", KR(ret), K(proposal_id));
} else if (OB_FAIL(ObServiceEpochProxy::check_and_update_service_epoch(
trans,
OB_SYS_TENANT_ID,
ObServiceEpochProxy::SERVER_ZONE_OP_SERVICE_EPOCH,
proposal_id))) {
LOG_WARN("fail to check and update server zone op service epoch", KR(ret), K(proposal_id));
} else {}
return ret;
}
int ObServerZoneOpService::fetch_new_server_id_(uint64_t &server_id)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", KR(ret), K(is_inited_));
} else if (OB_ISNULL(sql_proxy_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid sql proxy", KR(ret), KP(sql_proxy_));
} else {
uint64_t new_max_id = OB_INVALID_ID;
ObMaxIdFetcher id_fetcher(*sql_proxy_);
if (OB_FAIL(id_fetcher.fetch_new_max_id(
OB_SYS_TENANT_ID,
OB_MAX_USED_SERVER_ID_TYPE,
new_max_id))) {
LOG_WARN("fetch_new_max_id failed", KR(ret));
} else {
server_id = new_max_id;
}
}
return ret;
}
int ObServerZoneOpService::check_server_have_enough_resource_for_delete_server_(
const ObIArray<ObAddr> &servers,
const ObZone &zone)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", KR(ret), K(is_inited_));
} else if (OB_ISNULL(unit_manager_) || OB_ISNULL(sql_proxy_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unit_manager_ or sql_proxy_ is null", KR(ret), KP(unit_manager_), KP(sql_proxy_));
} else {
ObServerInfoInTable server_info;
FOREACH_CNT_X(server, servers, OB_SUCC(ret)) {
server_info.reset();
if (OB_FAIL(ObServerTableOperator::get(*sql_proxy_, *server, server_info))) {
LOG_WARN("fail to get server_info in table", KR(ret), KP(sql_proxy_), KPC(server));
} else if (!zone.is_empty() && server_info.get_zone() != zone) {
ret = OB_SERVER_ZONE_NOT_MATCH;
LOG_WARN("the arg zone is not the same as the server's zone in __all_server table", KR(ret),
K(zone), K(server_info));
} else if (OB_FAIL(unit_manager_->check_enough_resource_for_delete_server(
*server, server_info.get_zone()))) {
LOG_WARN("fail to check enouch resource", KR(ret), KPC(server), K(server_info));
}
}//end for each
}
return ret;
}
int ObServerZoneOpService::check_zone_and_server_(
const ObIArray<share::ObServerInfoInTable> &servers_info,
const ObIArray<ObAddr> &servers,
bool &is_same_zone,
bool &is_all_stopped)
{
int ret = OB_SUCCESS;
is_same_zone = true;
is_all_stopped = true;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", KR(ret), K(is_inited_));
} else {
ObServerInfoInTable server_info;
ObZone zone;
for (int64_t i = 0; i < servers.count() && OB_SUCC(ret) && (is_same_zone || is_all_stopped); i++) {
const ObAddr &server = servers.at(i);
server_info.reset();
if (OB_FAIL(ObRootUtils::find_server_info(servers_info, server, server_info))) {
LOG_WARN("fail to get server info", KR(ret), K(servers_info), K(server));
} else if (0 == i) {
if (OB_FAIL(zone.assign(server_info.get_zone()))) {
LOG_WARN("fail to assign zone", KR(ret), K(server_info.get_zone()));
}
} else if (zone != server_info.get_zone()) {
is_same_zone = false;
LOG_WARN("server zone not same", K(zone), K(server_info), K(servers));
}
if (OB_FAIL(ret)) {
} else if (!server_info.is_stopped()) {
is_all_stopped = false;
}
}
}
return ret;
}
ERRSIM_POINT_DEF(ALL_SERVER_LIST_ERROR);
void ObServerZoneOpService::end_trans_and_on_server_change_(
int &ret,
common::ObMySQLTransaction &trans,
const char *op_print_str,
const common::ObAddr &server,
const ObZone &zone,
const int64_t start_time)
{
int tmp_ret = OB_SUCCESS;
LOG_INFO("start execute end_trans_and_on_server_change_", KR(ret),
K(op_print_str), K(server), K(zone), K(start_time));
if (OB_UNLIKELY(!trans.is_started())) {
LOG_WARN("the transaction is not started");
} else {
if (OB_TMP_FAIL(trans.end(OB_SUCC(ret)))) {
LOG_WARN("fail to commit the transaction", KR(ret), KR(tmp_ret), K(server), K(zone));
ret = OB_SUCC(ret) ? tmp_ret : ret;
}
}
if (OB_TMP_FAIL(SVR_TRACER.refresh())) {
LOG_WARN("fail to refresh server tracer", KR(ret), KR(tmp_ret));
}
bool no_on_server_change = ALL_SERVER_LIST_ERROR ? true : false;
if (OB_ISNULL(server_change_callback_)) {
tmp_ret = OB_ERR_UNEXPECTED;
LOG_WARN("server_change_callback_ is null", KR(ret), KR(tmp_ret), KP(server_change_callback_));
ret = OB_SUCC(ret) ? tmp_ret : ret;
} else if (no_on_server_change) {
} else if (OB_TMP_FAIL(server_change_callback_->on_server_change())) {
LOG_WARN("fail to callback on server change", KR(ret), KR(tmp_ret));
}
int64_t time_cost = ::oceanbase::common::ObTimeUtility::current_time() - start_time;
FLOG_INFO(op_print_str, K(server), K(zone), "time cost", time_cost, KR(ret));
ROOTSERVICE_EVENT_ADD("server", op_print_str, K(server), K(ret));
}
}
}
Reference in New Issue View Git Blame Copy Permalink