hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[NetStandby] Add retry when query primary failed and modify log level for several specific error numbers

Browse Source
This commit is contained in:
obdev
2023-07-13 06:42:20 +00:00
committed by ob-robot
parent 67fbfa127f
commit 075115e5dc
3 changed files with 249 additions and 222 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
src/logservice/logfetcher/ob_log_ls_fetch_stream.cpp
View File

@ -32,6 +32,11 @@ namespace oceanbase
{ {
namespace logfetcher namespace logfetcher
{ {
#define IS_WARN_LOG_LEVEL(arg) \
if (OB_TIMEOUT == arg \
|| OB_TENANT_NOT_EXIST == arg \
|| OB_TENANT_NOT_IN_SERVER == arg \
|| OB_IN_STOP_STATE == arg) \
int64_t FetchStream::g_rpc_timeout = ObLogFetcherConfig::default_fetch_log_rpc_timeout_sec * _SEC_; int64_t FetchStream::g_rpc_timeout = ObLogFetcherConfig::default_fetch_log_rpc_timeout_sec * _SEC_;
int64_t FetchStream::g_dml_progress_limit = ObLogFetcherConfig::default_progress_limit_sec_for_dml * _SEC_; int64_t FetchStream::g_dml_progress_limit = ObLogFetcherConfig::default_progress_limit_sec_for_dml * _SEC_;
@ -1238,15 +1243,13 @@ int FetchStream::handle_fetch_log_error_(
need_kick_out = true; need_kick_out = true;
kick_out_reason = FETCH_LOG_FAIL_ON_RPC; kick_out_reason = FETCH_LOG_FAIL_ON_RPC;
if (OB_NOT_NULL(ls_fetch_ctx_)) { if (OB_NOT_NULL(ls_fetch_ctx_)) {
if (OB_IN_STOP_STATE != rcode.rcode_) { ls_fetch_ctx_->handle_error(ls_fetch_ctx_->get_tls_id().get_ls_id(),
ls_fetch_ctx_->handle_error(ls_fetch_ctx_->get_tls_id().get_ls_id(), IObLogErrHandler::ErrType::FETCH_LOG,
IObLogErrHandler::ErrType::FETCH_LOG, trace_id,
trace_id, ls_fetch_ctx_->get_next_lsn(),
ls_fetch_ctx_->get_next_lsn(), rcode.rcode_,
rcode.rcode_, "%s");
"%s"); IS_WARN_LOG_LEVEL(rcode.rcode_) {
}
if (OB_TIMEOUT == rcode.rcode_) {
LOG_WARN("fetch log fail on rpc, need_switch_server", K(svr_), K(rcode), "fetch_stream", this); LOG_WARN("fetch log fail on rpc, need_switch_server", K(svr_), K(rcode), "fetch_stream", this);
} else { } else {
LOG_ERROR("fetch log fail on rpc, need_switch_server", K(svr_), K(rcode), "fetch_stream", this); LOG_ERROR("fetch log fail on rpc, need_switch_server", K(svr_), K(rcode), "fetch_stream", this);
@ -1259,15 +1262,13 @@ int FetchStream::handle_fetch_log_error_(
need_kick_out = true; need_kick_out = true;
kick_out_reason = FETCH_LOG_FAIL_ON_SERVER; kick_out_reason = FETCH_LOG_FAIL_ON_SERVER;
if (OB_NOT_NULL(ls_fetch_ctx_)) { if (OB_NOT_NULL(ls_fetch_ctx_)) {
if (OB_IN_STOP_STATE != resp.get_err()) { ls_fetch_ctx_->handle_error(ls_fetch_ctx_->get_tls_id().get_ls_id(),
ls_fetch_ctx_->handle_error(ls_fetch_ctx_->get_tls_id().get_ls_id(), IObLogErrHandler::ErrType::FETCH_LOG,
IObLogErrHandler::ErrType::FETCH_LOG, trace_id,
trace_id, ls_fetch_ctx_->get_next_lsn(),
ls_fetch_ctx_->get_next_lsn(), resp.get_err(),
resp.get_err(), "%s");
"%s"); IS_WARN_LOG_LEVEL(resp.get_err()) {
}
if (OB_TIMEOUT == resp.get_err()) {
LOG_WARN("fetch log fail on server, need_switch_server", "fetch_stream", this, K(svr_), LOG_WARN("fetch log fail on server, need_switch_server", "fetch_stream", this, K(svr_),
"svr_err", resp.get_err(), "svr_debug_err", resp.get_debug_err(), "svr_err", resp.get_err(), "svr_debug_err", resp.get_debug_err(),
K(rcode), K(resp)); K(rcode), K(resp));

7
src/logservice/restoreservice/ob_log_restore_handler.cpp
View File

@ -513,8 +513,11 @@ void ObLogRestoreHandler::mark_error(share::ObTaskId &trace_id,
context_.error_context_.ret_code_ = ret_code; context_.error_context_.ret_code_ = ret_code;
context_.error_context_.trace_id_.set(trace_id); context_.error_context_.trace_id_.set(trace_id);
context_.error_context_.err_lsn_ = lsn; context_.error_context_.err_lsn_ = lsn;
if (OB_TIMEOUT == ret_code && ObLogRestoreErrorContext::ErrorType::FETCH_LOG == error_type) { if ((OB_TIMEOUT == ret_code && ObLogRestoreErrorContext::ErrorType::FETCH_LOG == error_type)
CLOG_LOG(WARN, "fetch log timeout in restore", KPC(parent_), KPC(this)); || (OB_TENANT_NOT_EXIST == ret_code && ObLogRestoreErrorContext::ErrorType::FETCH_LOG == error_type)
|| (OB_TENANT_NOT_IN_SERVER == ret_code && ObLogRestoreErrorContext::ErrorType::FETCH_LOG == error_type)
|| (OB_IN_STOP_STATE == ret_code && ObLogRestoreErrorContext::ErrorType::FETCH_LOG == error_type)) {
CLOG_LOG(WARN, "fetch log failed in restore", KPC(parent_), KPC(this));
} else if(OB_SUCCESS != ret_code) { } else if(OB_SUCCESS != ret_code) {
CLOG_LOG(ERROR, "fatal error occur in restore", KPC(parent_), KPC(this)); CLOG_LOG(ERROR, "fatal error occur in restore", KPC(parent_), KPC(this));
} }

427
src/share/ob_log_restore_proxy.cpp
View File

@ -41,22 +41,30 @@ namespace share
LOG_USER_ERROR(OB_INVALID_ARGUMENT, "get primary " args ", privilege is insufficient"); \ LOG_USER_ERROR(OB_INVALID_ARGUMENT, "get primary " args ", privilege is insufficient"); \
break; \ break; \
case -ER_ACCESS_DENIED_ERROR: \ case -ER_ACCESS_DENIED_ERROR: \
LOG_USER_ERROR(OB_INVALID_ARGUMENT, "get primary " args ", please check the user and password"); \ LOG_USER_ERROR(OB_INVALID_ARGUMENT, "get primary " args ", please check the user and password"); \
break; \ break; \
case -ER_DBACCESS_DENIED_ERROR: \ case -ER_DBACCESS_DENIED_ERROR: \
case OB_ERR_NO_LOGIN_PRIVILEGE: \ case OB_ERR_NO_LOGIN_PRIVILEGE: \
LOG_USER_ERROR(OB_INVALID_ARGUMENT, "get primary " args ", please check the privileges"); \ LOG_USER_ERROR(OB_INVALID_ARGUMENT, "get primary " args ", please check the privileges"); \
break; \ break; \
case OB_ERR_NULL_VALUE: \ case OB_ERR_NULL_VALUE: \
LOG_USER_ERROR(OB_INVALID_ARGUMENT, "get primary " args ", primary may not be ready"); \ case OB_ERR_WAIT_REMOTE_SCHEMA_REFRESH: \
LOG_USER_ERROR(OB_INVALID_ARGUMENT, "get primary " args ", query primary failed"); \
break; \ break; \
case -ER_CONNECT_FAILED: \ case -ER_CONNECT_FAILED: \
LOG_USER_ERROR(OB_INVALID_ARGUMENT, "get primary " args ", please check the network"); \ LOG_USER_ERROR(OB_INVALID_ARGUMENT, "get primary " args ", please check the network"); \
break; \ break; \
default: \ default: \
LOG_USER_ERROR(OB_INVALID_ARGUMENT, "get primary " args); \ LOG_USER_ERROR(OB_INVALID_ARGUMENT, "get primary " args); \
} \ } \
#define RESTORE_RETRY(arg) \
int64_t retry_time = 0; \
do { \
arg \
} while (OB_FAIL(ret) && retry_time++ < server_prover_.get_server_count() - 1);
ObLogRestoreMySQLProvider::ObLogRestoreMySQLProvider() : server_list_() {} ObLogRestoreMySQLProvider::ObLogRestoreMySQLProvider() : server_list_() {}
ObLogRestoreMySQLProvider::~ObLogRestoreMySQLProvider() ObLogRestoreMySQLProvider::~ObLogRestoreMySQLProvider()
@ -288,29 +296,10 @@ int ObLogRestoreProxyUtil::try_init(const uint64_t tenant_id,
const char *ORACLE_DB = "SYS"; const char *ORACLE_DB = "SYS";
if (OB_SUCC(init(tenant_id, server_list, user_name, user_password, MYSQL_DB))) { if (OB_SUCC(init(tenant_id, server_list, user_name, user_password, MYSQL_DB))) {
SMART_VAR(ObMySQLProxy::MySQLResult, result) { RESTORE_RETRY(
ObSqlString sql;
if (OB_FAIL(sql.assign_fmt("SELECT 1"))) {
LOG_WARN("fail to generate sql");
} else if (OB_FAIL(sql_proxy_.read(result, sql.ptr()))) {
LOG_WARN("fail to execute sql", K(sql));
} else if (OB_ISNULL(result.get_result())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("query result is null", K(tenant_id), K(sql));
} else if (OB_FAIL(result.get_result()->next())) {
LOG_WARN("get result next failed", K(sql));
} else {
LOG_INFO("proxy connect to primary oceanabse db success");
}
}
}
if (OB_FAIL(ret)) {
LOG_WARN("proxy connect to primary oceanbase db failed, then try connect to sys db");
(void)destroy();
if (OB_SUCC(init(tenant_id, server_list, user_name, user_password, ORACLE_DB))) {
SMART_VAR(ObMySQLProxy::MySQLResult, result) { SMART_VAR(ObMySQLProxy::MySQLResult, result) {
ObSqlString sql; ObSqlString sql;
if (OB_FAIL(sql.assign_fmt("SELECT 1 FROM DUAL"))) { if (OB_FAIL(sql.assign_fmt("SELECT 1"))) {
LOG_WARN("fail to generate sql"); LOG_WARN("fail to generate sql");
} else if (OB_FAIL(sql_proxy_.read(result, sql.ptr()))) { } else if (OB_FAIL(sql_proxy_.read(result, sql.ptr()))) {
LOG_WARN("fail to execute sql", K(sql)); LOG_WARN("fail to execute sql", K(sql));
@ -320,9 +309,32 @@ int ObLogRestoreProxyUtil::try_init(const uint64_t tenant_id,
} else if (OB_FAIL(result.get_result()->next())) { } else if (OB_FAIL(result.get_result()->next())) {
LOG_WARN("get result next failed", K(sql)); LOG_WARN("get result next failed", K(sql));
} else { } else {
LOG_INFO("proxy connect to sys db success"); LOG_INFO("proxy connect to primary oceanabse db success");
} }
} }
)
}
if (OB_FAIL(ret)) {
LOG_WARN("proxy connect to primary oceanbase db failed, then try connect to sys db");
(void)destroy();
if (OB_SUCC(init(tenant_id, server_list, user_name, user_password, ORACLE_DB))) {
RESTORE_RETRY(
SMART_VAR(ObMySQLProxy::MySQLResult, result) {
ObSqlString sql;
if (OB_FAIL(sql.assign_fmt("SELECT 1 FROM DUAL"))) {
LOG_WARN("fail to generate sql");
} else if (OB_FAIL(sql_proxy_.read(result, sql.ptr()))) {
LOG_WARN("fail to execute sql", K(sql));
} else if (OB_ISNULL(result.get_result())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("query result is null", K(tenant_id), K(sql));
} else if (OB_FAIL(result.get_result()->next())) {
LOG_WARN("get result next failed", K(sql));
} else {
LOG_INFO("proxy connect to sys db success");
}
}
)
} }
} }
if (OB_FAIL(ret)) { if (OB_FAIL(ret)) {
@ -336,26 +348,28 @@ int ObLogRestoreProxyUtil::try_init(const uint64_t tenant_id,
int ObLogRestoreProxyUtil::get_tenant_id(char *tenant_name, uint64_t &tenant_id) int ObLogRestoreProxyUtil::get_tenant_id(char *tenant_name, uint64_t &tenant_id)
{ {
int ret = OB_SUCCESS; int ret = OB_SUCCESS;
SMART_VAR(ObMySQLProxy::MySQLResult, result) { RESTORE_RETRY(
ObSqlString sql; SMART_VAR(ObMySQLProxy::MySQLResult, result) {
if (OB_FAIL(sql.assign_fmt("SELECT TENANT_ID FROM %s WHERE TENANT_NAME='%s'", ObSqlString sql;
OB_DBA_OB_TENANTS_TNAME, tenant_name))) { if (OB_FAIL(sql.assign_fmt("SELECT TENANT_ID FROM %s WHERE TENANT_NAME='%s'",
LOG_WARN("fail to generate sql"); OB_DBA_OB_TENANTS_TNAME, tenant_name))) {
} else if (OB_FAIL(sql_proxy_.read(result, sql.ptr()))) { LOG_WARN("fail to generate sql");
LOG_WARN("fail to execute sql", K(sql), K(connection_.get_db_name())); } else if (OB_FAIL(sql_proxy_.read(result, sql.ptr()))) {
} else if (OB_ISNULL(result.get_result())) { LOG_WARN("fail to execute sql", K(sql), K(connection_.get_db_name()));
ret = OB_ERR_UNEXPECTED; } else if (OB_ISNULL(result.get_result())) {
LOG_WARN("query result is null", K(tenant_name), K(sql)); ret = OB_ERR_UNEXPECTED;
} else if (OB_FAIL(result.get_result()->next())) { LOG_WARN("query result is null", K(tenant_name), K(sql));
LOG_WARN("get result next failed", K(tenant_name), K(sql)); } else if (OB_FAIL(result.get_result()->next())) {
} else { LOG_WARN("get result next failed", K(tenant_name), K(sql));
EXTRACT_UINT_FIELD_MYSQL(*result.get_result(), "TENANT_ID", tenant_id, uint64_t); } else {
EXTRACT_UINT_FIELD_MYSQL(*result.get_result(), "TENANT_ID", tenant_id, uint64_t);
if (OB_FAIL(ret)) { if (OB_FAIL(ret)) {
LOG_WARN("failed to get tenant id result", K(tenant_name), K(sql)); LOG_WARN("failed to get tenant id result", K(tenant_name), K(sql));
}
} }
} }
} )
if (OB_FAIL(ret)) { if (OB_FAIL(ret)) {
LOG_WARN("failed to get tenant id result"); LOG_WARN("failed to get tenant id result");
RESTORE_PROXY_USER_ERROR("tenant id"); RESTORE_PROXY_USER_ERROR("tenant id");
@ -366,25 +380,27 @@ int ObLogRestoreProxyUtil::get_tenant_id(char *tenant_name, uint64_t &tenant_id)
int ObLogRestoreProxyUtil::get_cluster_id(uint64_t tenant_id, int64_t &cluster_id) int ObLogRestoreProxyUtil::get_cluster_id(uint64_t tenant_id, int64_t &cluster_id)
{ {
int ret = OB_SUCCESS; int ret = OB_SUCCESS;
SMART_VAR(ObMySQLProxy::MySQLResult, result) { RESTORE_RETRY(
ObSqlString sql; SMART_VAR(ObMySQLProxy::MySQLResult, result) {
if (OB_FAIL(sql.assign_fmt("SELECT VALUE FROM %s WHERE NAME='cluster_id'", OB_GV_OB_PARAMETERS_TNAME))) { ObSqlString sql;
LOG_WARN("fail to generate sql"); if (OB_FAIL(sql.assign_fmt("SELECT VALUE FROM %s WHERE NAME='cluster_id'", OB_GV_OB_PARAMETERS_TNAME))) {
} else if (OB_FAIL(sql_proxy_.read(result, sql.ptr()))) { LOG_WARN("fail to generate sql");
LOG_WARN("fail to execute sql", K(sql)); } else if (OB_FAIL(sql_proxy_.read(result, sql.ptr()))) {
} else if (OB_ISNULL(result.get_result())) { LOG_WARN("fail to execute sql", K(sql));
ret = OB_ERR_UNEXPECTED; } else if (OB_ISNULL(result.get_result())) {
LOG_WARN("query result is null", K(sql)); ret = OB_ERR_UNEXPECTED;
} else if (OB_FAIL(result.get_result()->next())) { LOG_WARN("query result is null", K(sql));
LOG_WARN("get result next failed", K(tenant_id), K(sql)); } else if (OB_FAIL(result.get_result()->next())) {
} else { LOG_WARN("get result next failed", K(sql));
EXTRACT_INT_FIELD_MYSQL(*result.get_result(), "VALUE", cluster_id, int64_t); } else {
EXTRACT_INT_FIELD_MYSQL(*result.get_result(), "VALUE", cluster_id, int64_t);
if (OB_FAIL(ret)) { if (OB_FAIL(ret)) {
LOG_WARN("fail to get cluster id result", K(cluster_id), K(sql)); LOG_WARN("fail to get cluster id result", K(cluster_id), K(sql));
}
} }
} }
} )
if (OB_FAIL(ret)) { if (OB_FAIL(ret)) {
LOG_WARN("fail to get cluster id result"); LOG_WARN("fail to get cluster id result");
RESTORE_PROXY_USER_ERROR("cluster id"); RESTORE_PROXY_USER_ERROR("cluster id");
@ -397,39 +413,41 @@ int ObLogRestoreProxyUtil::get_compatibility_mode(const uint64_t tenant_id, ObCo
int ret = OB_SUCCESS; int ret = OB_SUCCESS;
const char *MYSQL_STR = "MYSQL"; const char *MYSQL_STR = "MYSQL";
const char *ORACLE_STR = "ORACLE"; const char *ORACLE_STR = "ORACLE";
SMART_VAR(ObMySQLProxy::MySQLResult, result) { RESTORE_RETRY(
ObSqlString sql; SMART_VAR(ObMySQLProxy::MySQLResult, result) {
if (OB_FAIL(sql.assign_fmt("SELECT COMPATIBILITY_MODE FROM %s WHERE TENANT_ID=%ld", ObSqlString sql;
OB_DBA_OB_TENANTS_TNAME, tenant_id))) { if (OB_FAIL(sql.assign_fmt("SELECT COMPATIBILITY_MODE FROM %s WHERE TENANT_ID=%ld",
LOG_WARN("fail to generate sql", K(tenant_id)); OB_DBA_OB_TENANTS_TNAME, tenant_id))) {
} else if (OB_FAIL(sql_proxy_.read(result, sql.ptr()))) { LOG_WARN("fail to generate sql", K(tenant_id));
LOG_WARN("read value from DBA_OB_TENANTS failed", K(tenant_id), K(sql.ptr())); } else if (OB_FAIL(sql_proxy_.read(result, sql.ptr()))) {
} else if (OB_ISNULL(result.get_result())) { LOG_WARN("read value from DBA_OB_TENANTS failed", K(tenant_id), K(sql.ptr()));
ret = OB_ERR_UNEXPECTED; } else if (OB_ISNULL(result.get_result())) {
LOG_WARN("query result is null", K(tenant_id), K(sql));
} else if (OB_FAIL(result.get_result()->next())) {
LOG_WARN("get result next failed", K(tenant_id), K(sql));
} else {
ObString tmp_compat_mode;
char compact[OB_MAX_COMPAT_MODE_STR_LEN + 1] = { 0 };
EXTRACT_VARCHAR_FIELD_MYSQL(*result.get_result(), "COMPATIBILITY_MODE", tmp_compat_mode);
if (OB_FAIL(ret)) {
LOG_WARN("fail to get compact mode result", K(tenant_id), K(sql.ptr()));
RESTORE_PROXY_USER_ERROR("compatibility mode");
} else if (OB_FAIL(databuff_printf(compact, sizeof(compact), "%.*s",
static_cast<int>(tmp_compat_mode.length()), tmp_compat_mode.ptr()))) {
LOG_WARN("fail to print compact_mode", K(tmp_compat_mode.ptr()));
} else if (0 == STRCASECMP(compact, ORACLE_STR)) {
compat_mode = ObCompatibilityMode::ORACLE_MODE;
} else if (0 == STRCASECMP(compact, MYSQL_STR)) {
compat_mode = ObCompatibilityMode::MYSQL_MODE;
} else {
ret = OB_ERR_UNEXPECTED; ret = OB_ERR_UNEXPECTED;
LOG_WARN("compatibility mode is not expected"); LOG_WARN("query result is null", K(tenant_id), K(sql));
} else if (OB_FAIL(result.get_result()->next())) {
LOG_WARN("get result next failed", K(sql));
} else {
ObString tmp_compat_mode;
char compact[OB_MAX_COMPAT_MODE_STR_LEN + 1] = { 0 };
EXTRACT_VARCHAR_FIELD_MYSQL(*result.get_result(), "COMPATIBILITY_MODE", tmp_compat_mode);
if (OB_FAIL(ret)) {
LOG_WARN("fail to get compact mode result", K(tenant_id), K(sql.ptr()));
RESTORE_PROXY_USER_ERROR("compatibility mode");
} else if (OB_FAIL(databuff_printf(compact, sizeof(compact), "%.*s",
static_cast<int>(tmp_compat_mode.length()), tmp_compat_mode.ptr()))) {
LOG_WARN("fail to print compact_mode", K(tmp_compat_mode.ptr()));
} else if (0 == STRCASECMP(compact, ORACLE_STR)) {
compat_mode = ObCompatibilityMode::ORACLE_MODE;
} else if (0 == STRCASECMP(compact, MYSQL_STR)) {
compat_mode = ObCompatibilityMode::MYSQL_MODE;
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("compatibility mode is not expected");
}
} }
} }
} )
return ret; return ret;
} }
@ -441,39 +459,38 @@ int ObLogRestoreProxyUtil::check_begin_lsn(const uint64_t tenant_id)
ret = OB_INVALID_ARGUMENT; ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KR(ret), K(tenant_id)); LOG_WARN("invalid argument", KR(ret), K(tenant_id));
} else { } else {
SMART_VAR(ObMySQLProxy::MySQLResult, result) { RESTORE_RETRY(
ObSqlString sql; SMART_VAR(ObMySQLProxy::MySQLResult, result) {
if (OB_FAIL(sql.assign_fmt("SELECT COUNT(*) AS CNT FROM %s OB_LS LEFT JOIN" ObSqlString sql;
"(SELECT TENANT_ID, LS_ID, BEGIN_LSN FROM %s WHERE ROLE= 'LEADER') LOG_STAT " if (OB_FAIL(sql.assign_fmt("SELECT COUNT(*) AS CNT FROM %s OB_LS LEFT JOIN"
"ON OB_LS.LS_ID = LOG_STAT.LS_ID " "(SELECT TENANT_ID, LS_ID, BEGIN_LSN FROM %s WHERE ROLE= 'LEADER') LOG_STAT "
"WHERE LOG_STAT.TENANT_ID = %lu AND (BEGIN_LSN IS NULL OR BEGIN_LSN != 0)" "ON OB_LS.LS_ID = LOG_STAT.LS_ID "
"AND OB_LS.STATUS NOT IN ('TENANT_DROPPING', 'CREATE_ABORT', 'PRE_TENANT_DROPPING')", "WHERE LOG_STAT.TENANT_ID = %lu AND (BEGIN_LSN IS NULL OR BEGIN_LSN != 0)"
OB_DBA_OB_LS_TNAME, OB_GV_OB_LOG_STAT_ORA_TNAME, tenant_id))) { "AND OB_LS.STATUS NOT IN ('TENANT_DROPPING', 'CREATE_ABORT', 'PRE_TENANT_DROPPING')",
LOG_WARN("fail to generate sql", KR(ret), K(tenant_id)); OB_DBA_OB_LS_TNAME, OB_GV_OB_LOG_STAT_ORA_TNAME, tenant_id))) {
} else if (OB_FAIL(sql_proxy_.read(result, sql.ptr()))) { LOG_WARN("fail to generate sql", KR(ret), K(tenant_id));
LOG_WARN("check_begin_lsn failed", KR(ret), K(tenant_id), K(sql)); } else if (OB_FAIL(sql_proxy_.read(result, sql.ptr()))) {
RESTORE_PROXY_USER_ERROR("tenant ls begin_lsn failed"); LOG_WARN("check_begin_lsn failed", KR(ret), K(tenant_id), K(sql));
ret = OB_INVALID_ARGUMENT; RESTORE_PROXY_USER_ERROR("tenant ls begin_lsn failed");
} else if (OB_ISNULL(result.get_result())) { ret = OB_INVALID_ARGUMENT;
ret = OB_ERR_UNEXPECTED; } else if (OB_ISNULL(result.get_result())) {
LOG_WARN("config result is null", KR(ret), K(tenant_id), K(sql)); ret = OB_ERR_UNEXPECTED;
} else if (OB_FAIL(result.get_result()->next())) { LOG_WARN("config result is null", KR(ret), K(tenant_id), K(sql));
LOG_WARN("get result next failed", KR(ret), K(tenant_id), K(sql)); } else if (OB_FAIL(result.get_result()->next())) {
} else if (OB_ISNULL(result.get_result())) { LOG_WARN("get result next failed", K(sql));
ret = OB_ERR_UNEXPECTED; } else {
LOG_WARN("result is null", KR(ret), K(tenant_id), K(sql)); uint64_t cnt = 0;
} else { EXTRACT_INT_FIELD_MYSQL(*result.get_result(), "CNT", cnt, uint64_t);
uint64_t cnt = 0; if (OB_FAIL(ret)) {
EXTRACT_INT_FIELD_MYSQL(*result.get_result(), "CNT", cnt, uint64_t); LOG_WARN("failed to get result", KR(ret), K(tenant_id), K(sql));
if (OB_FAIL(ret)) { } else if (cnt > 0) {
LOG_WARN("failed to get result", KR(ret), K(tenant_id), K(sql)); ret = OB_OP_NOT_ALLOW;
} else if (cnt > 0) { LOG_WARN("primary tenant LS log may be recycled, create standby tenant is not allow", KR(ret), K(tenant_id), K(sql));
ret = OB_OP_NOT_ALLOW; LOG_USER_ERROR(OB_OP_NOT_ALLOW, "primary tenant LS log may be recycled, create standby tenant is");
LOG_WARN("primary tenant LS log may be recycled, create standby tenant is not allow", KR(ret), K(tenant_id), K(sql)); }
LOG_USER_ERROR(OB_OP_NOT_ALLOW, "primary tenant LS log may be recycled, create standby tenant is");
} }
} }
} )
} }
return ret; return ret;
} }
@ -481,45 +498,47 @@ int ObLogRestoreProxyUtil::check_begin_lsn(const uint64_t tenant_id)
int ObLogRestoreProxyUtil::get_server_ip_list(const uint64_t tenant_id, common::ObArray<common::ObAddr> &addrs) int ObLogRestoreProxyUtil::get_server_ip_list(const uint64_t tenant_id, common::ObArray<common::ObAddr> &addrs)
{ {
int ret = OB_SUCCESS; int ret = OB_SUCCESS;
SMART_VAR(ObMySQLProxy::MySQLResult ,result) { RESTORE_RETRY(
ObSqlString sql; SMART_VAR(ObMySQLProxy::MySQLResult ,result) {
ObMySQLResult *res = NULL; ObSqlString sql;
if (OB_FAIL(sql.assign_fmt("SELECT SVR_IP, SQL_PORT FROM %s WHERE TENANT_ID=%ld", ObMySQLResult *res = NULL;
OB_DBA_OB_ACCESS_POINT_TNAME, tenant_id))) { if (OB_FAIL(sql.assign_fmt("SELECT SVR_IP, SQL_PORT FROM %s WHERE TENANT_ID=%ld",
LOG_WARN("fail to generate sql"); OB_DBA_OB_ACCESS_POINT_TNAME, tenant_id))) {
} else if (OB_FAIL(sql_proxy_.read(result, OB_INVALID_TENANT_ID, sql.ptr()))) { LOG_WARN("fail to generate sql");
LOG_WARN("read value from DBA_OB_ACCESS_POINT failed", K(tenant_id), K(sql)); } else if (OB_FAIL(sql_proxy_.read(result, OB_INVALID_TENANT_ID, sql.ptr()))) {
} else if (OB_ISNULL(res = result.get_result())) { LOG_WARN("read value from DBA_OB_ACCESS_POINT failed", K(tenant_id), K(sql));
ret = OB_ERR_UNEXPECTED; } else if (OB_ISNULL(res = result.get_result())) {
LOG_WARN("query result is null", K(tenant_id), K(sql)); ret = OB_ERR_UNEXPECTED;
} else { LOG_WARN("query result is null", K(tenant_id), K(sql));
while (OB_SUCC(ret) && OB_SUCC(res->next())) { } else {
ObString tmp_ip; while (OB_SUCC(ret) && OB_SUCC(res->next())) {
int32_t tmp_port; ObString tmp_ip;
ObAddr addr; int32_t tmp_port;
ObAddr addr;
EXTRACT_VARCHAR_FIELD_MYSQL(*res, "SVR_IP", tmp_ip); EXTRACT_VARCHAR_FIELD_MYSQL(*res, "SVR_IP", tmp_ip);
EXTRACT_INT_FIELD_MYSQL(*res, "SQL_PORT", tmp_port, int32_t); EXTRACT_INT_FIELD_MYSQL(*res, "SQL_PORT", tmp_port, int32_t);
if (OB_FAIL(ret)) { if (OB_FAIL(ret)) {
LOG_WARN("fail to get server ip and sql port", K(tmp_ip), K(tmp_port), K(tenant_id), K(sql)); LOG_WARN("fail to get server ip and sql port", K(tmp_ip), K(tmp_port), K(tenant_id), K(sql));
} else if (!addr.set_ip_addr(tmp_ip, tmp_port)) { } else if (!addr.set_ip_addr(tmp_ip, tmp_port)) {
ret = OB_ERR_UNEXPECTED; ret = OB_ERR_UNEXPECTED;
LOG_WARN("fail to set addr", K(tmp_ip), K(tmp_port), K(tenant_id), K(sql)); LOG_WARN("fail to set addr", K(tmp_ip), K(tmp_port), K(tenant_id), K(sql));
} else if (!addr.is_valid()) { } else if (!addr.is_valid()) {
ret = OB_ERR_UNEXPECTED; ret = OB_ERR_UNEXPECTED;
LOG_WARN("addr is invalid", K(addr), K(tenant_id), K(sql)); LOG_WARN("addr is invalid", K(addr), K(tenant_id), K(sql));
} else if (OB_FAIL(addrs.push_back(addr))) { } else if (OB_FAIL(addrs.push_back(addr))) {
LOG_WARN("fail to push back addr to addrs", K(addr), K(tenant_id), K(sql)); LOG_WARN("fail to push back addr to addrs", K(addr), K(tenant_id), K(sql));
}
}
if (OB_ITER_END != ret) {
SERVER_LOG(WARN, "failed to get ip list", K(tenant_id));
} else {
ret = OB_SUCCESS;
} }
} }
if (OB_ITER_END != ret) {
SERVER_LOG(WARN, "failed to get ip list", K(tenant_id));
} else {
ret = OB_SUCCESS;
}
} }
} )
return ret; return ret;
} }
@ -539,33 +558,35 @@ int ObLogRestoreProxyUtil::get_tenant_info(ObTenantRole &role, schema::ObTenantS
if (OB_UNLIKELY(!inited_)) { if (OB_UNLIKELY(!inited_)) {
ret = OB_NOT_INIT; ret = OB_NOT_INIT;
} else { } else {
SMART_VAR(common::ObMySQLProxy::MySQLResult, res) { RESTORE_RETRY(
common::sqlclient::ObMySQLResult *result = NULL; SMART_VAR(common::ObMySQLProxy::MySQLResult, res) {
common::ObSqlString sql; common::sqlclient::ObMySQLResult *result = NULL;
const char *GET_TENANT_INFO_SQL = "SELECT %s, %s FROM %s"; common::ObSqlString sql;
if (OB_FAIL(sql.append_fmt(GET_TENANT_INFO_SQL, TENANT_ROLE, TENANT_STATUS, OB_DBA_OB_TENANTS_TNAME))) { const char *GET_TENANT_INFO_SQL = "SELECT %s, %s FROM %s";
LOG_WARN("append_fmt failed"); if (OB_FAIL(sql.append_fmt(GET_TENANT_INFO_SQL, TENANT_ROLE, TENANT_STATUS, OB_DBA_OB_TENANTS_TNAME))) {
} else if (OB_FAIL(proxy->read(res, sql.ptr()))) { LOG_WARN("append_fmt failed");
LOG_WARN("excute sql failed", K(sql)); } else if (OB_FAIL(proxy->read(res, sql.ptr()))) {
} else if (OB_ISNULL(result = res.get_result())) { LOG_WARN("excute sql failed", K(sql));
ret = OB_ERR_UNEXPECTED; } else if (OB_ISNULL(result = res.get_result())) {
LOG_WARN("get result failed", K(sql)); ret = OB_ERR_UNEXPECTED;
} else if (OB_FAIL(result->next())) { LOG_WARN("get result failed", K(sql));
LOG_WARN("next failed"); } else if (OB_FAIL(result->next())) {
} else { LOG_WARN("next failed");
ObString status_str; } else {
ObString role_str; ObString status_str;
EXTRACT_VARCHAR_FIELD_MYSQL(*result, TENANT_ROLE, role_str); ObString role_str;
EXTRACT_VARCHAR_FIELD_MYSQL(*result, TENANT_STATUS, status_str); EXTRACT_VARCHAR_FIELD_MYSQL(*result, TENANT_ROLE, role_str);
if (OB_SUCC(ret)) { EXTRACT_VARCHAR_FIELD_MYSQL(*result, TENANT_STATUS, status_str);
if (OB_FAIL(schema::get_tenant_status(status_str, status))) { if (OB_SUCC(ret)) {
LOG_WARN("get tenant status failed"); if (OB_FAIL(schema::get_tenant_status(status_str, status))) {
} else { LOG_WARN("get tenant status failed");
role = ObTenantRole(role_str.ptr()); } else {
role = ObTenantRole(role_str.ptr());
}
} }
} }
} }
} )
} }
return ret; return ret;
} }
@ -583,34 +604,36 @@ int ObLogRestoreProxyUtil::get_max_log_info(const ObLSID &id, palf::AccessMode &
ret = OB_INVALID_ARGUMENT; ret = OB_INVALID_ARGUMENT;
LOG_WARN("invlaid argument", K(id)); LOG_WARN("invlaid argument", K(id));
} else { } else {
SMART_VAR(common::ObMySQLProxy::MySQLResult, res) { RESTORE_RETRY(
common::sqlclient::ObMySQLResult *result = NULL; SMART_VAR(common::ObMySQLProxy::MySQLResult, res) {
common::ObSqlString sql; common::sqlclient::ObMySQLResult *result = NULL;
const char *GET_MAX_LOG_INFO_SQL = "SELECT %s, %s, %s FROM %s WHERE %s=%ld and ROLE = 'LEADER'"; common::ObSqlString sql;
if (OB_FAIL(sql.append_fmt(GET_MAX_LOG_INFO_SQL, LS_ID, MAX_SCN, ACCESS_MODE, const char *GET_MAX_LOG_INFO_SQL = "SELECT %s, %s, %s FROM %s WHERE %s=%ld and ROLE = 'LEADER'";
OB_GV_OB_LOG_STAT_TNAME, LS_ID, id.id()))) { if (OB_FAIL(sql.append_fmt(GET_MAX_LOG_INFO_SQL, LS_ID, MAX_SCN, ACCESS_MODE,
LOG_WARN("append_fmt failed"); OB_GV_OB_LOG_STAT_TNAME, LS_ID, id.id()))) {
} else if (OB_FAIL(proxy->read(res, sql.ptr()))) { LOG_WARN("append_fmt failed");
LOG_WARN("excute sql failed", K(sql)); } else if (OB_FAIL(proxy->read(res, sql.ptr()))) {
} else if (OB_ISNULL(result = res.get_result())) { LOG_WARN("excute sql failed", K(sql));
ret = OB_ERR_UNEXPECTED; } else if (OB_ISNULL(result = res.get_result())) {
LOG_WARN("get result failed", K(sql)); ret = OB_ERR_UNEXPECTED;
} else if (OB_FAIL(result->next())) { LOG_WARN("get result failed", K(sql));
LOG_WARN("next failed"); } else if (OB_FAIL(result->next())) {
} else { LOG_WARN("next failed");
uint64_t max_scn = 0; } else {
ObString access_mode_str; uint64_t max_scn = 0;
EXTRACT_VARCHAR_FIELD_MYSQL(*result, ACCESS_MODE, access_mode_str); ObString access_mode_str;
EXTRACT_UINT_FIELD_MYSQL(*result, MAX_SCN, max_scn, uint64_t); EXTRACT_VARCHAR_FIELD_MYSQL(*result, ACCESS_MODE, access_mode_str);
if (OB_SUCC(ret) && OB_FAIL(palf::get_access_mode(access_mode_str, mode))) { EXTRACT_UINT_FIELD_MYSQL(*result, MAX_SCN, max_scn, uint64_t);
LOG_WARN("get_access_mode failed", K(id), K(access_mode_str)); if (OB_SUCC(ret) && OB_FAIL(palf::get_access_mode(access_mode_str, mode))) {
} LOG_WARN("get_access_mode failed", K(id), K(access_mode_str));
}
if (OB_SUCC(ret) && OB_FAIL(scn.convert_for_logservice(max_scn))) { if (OB_SUCC(ret) && OB_FAIL(scn.convert_for_logservice(max_scn))) {
LOG_WARN("convert_for_logservice failed", K(id), K(max_scn)); LOG_WARN("convert_for_logservice failed", K(id), K(max_scn));
}
} }
} }
} )
} }
return ret; return ret;
} }