From cdff7836ec6d9b6b44162dfb498e3db1f40fe752 Mon Sep 17 00:00:00 2001 From: obdev Date: Tue, 6 Feb 2024 20:31:13 +0000 Subject: [PATCH] [NetStandby] Optimize restore proxy --- src/share/ob_log_restore_proxy.cpp | 69 +++++++++++++++++++++++------- src/share/ob_log_restore_proxy.h | 6 ++- 2 files changed, 58 insertions(+), 17 deletions(-) diff --git a/src/share/ob_log_restore_proxy.cpp b/src/share/ob_log_restore_proxy.cpp index 3ec13cbb0c..294ab1d5b5 100644 --- a/src/share/ob_log_restore_proxy.cpp +++ b/src/share/ob_log_restore_proxy.cpp @@ -69,6 +69,9 @@ namespace share case OB_ERR_TENANT_IS_LOCKED: \ LOG_USER_ERROR(OB_INVALID_ARGUMENT, "get primary " args ", primary tenant is locked"); \ break; \ + case OB_CONNECT_ERROR: \ + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "get primary " args ", all servers are unreachable"); \ + break; \ default: \ LOG_USER_ERROR(OB_INVALID_ARGUMENT, "get primary " args); \ } \ @@ -324,13 +327,16 @@ int ObLogRestoreProxyUtil::try_init(const uint64_t tenant_id, int ret = OB_SUCCESS; const char *db_name = nullptr; ObLogRestoreMySQLProvider tmp_server_prover; + common::ObArray fixed_server_list; - if (OB_FAIL(tmp_server_prover.init(server_list))) { - LOG_WARN("server_prover_ init failed", K(tenant_id), K(server_list)); - } else if (OB_FAIL(detect_tenant_mode_(&tmp_server_prover, user_name, user_password))) { + if (OB_FAIL(fixed_server_list.assign(server_list))) { + LOG_WARN("fail to assign fixed server list", K(tenant_id), K(server_list)); + } else if (OB_FAIL(tmp_server_prover.init(server_list))) { + LOG_WARN("tmp_server_prover_ init failed", K(tenant_id), K(server_list)); + } else if (OB_FAIL(detect_tenant_mode_(&tmp_server_prover, user_name, user_password, &fixed_server_list))) { LOG_WARN("detect tenant mode failed", KR(ret), K_(user_name)); } else if (OB_FALSE_IT(db_name = is_oracle_mode_ ? OB_ORA_SYS_SCHEMA_NAME : OB_SYS_DATABASE_NAME)) { - } else if (OB_FAIL(init(tenant_id, server_list, user_name, user_password, db_name))) { + } else if (OB_FAIL(init(tenant_id, fixed_server_list, user_name, user_password, db_name))) { LOG_WARN("[RESTORE PROXY] fail to init restore proxy"); } @@ -726,10 +732,12 @@ void ObLogRestoreProxyUtil::destroy_tg_() int ObLogRestoreProxyUtil::detect_tenant_mode_(common::sqlclient::ObMySQLServerProvider *server_provider, const char *user_name, - const char *user_password) + const char *user_password, + common::ObIArray *fixed_server_list) { + LOG_INFO("start to detec tenant mode"); int ret = OB_SUCCESS; - + common::ObArray invalid_ip_array; if (OB_ISNULL(server_provider)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("[RESTORE PROXY] invalid server_provider", KR(ret)); @@ -740,8 +748,7 @@ int ObLogRestoreProxyUtil::detect_tenant_mode_(common::sqlclient::ObMySQLServerP LOG_WARN("[RESTORE PROXY] expect valid svr_cnt", K(svr_cnt)); } else { bool detect_succ = false; - - for (int idx = 0; OB_SUCC(ret) && ! detect_succ && idx < svr_cnt; idx++) { + for (int idx = 0; idx < svr_cnt; idx++) { MYSQL *mysql; common::ObAddr server; if (OB_FAIL(server_provider->get_server(idx, server))) { @@ -756,11 +763,17 @@ int ObLogRestoreProxyUtil::detect_tenant_mode_(common::sqlclient::ObMySQLServerP ret = OB_BUF_NOT_ENOUGH; LOG_WARN("fail to get host.", K(server), K(ret)); } else if (NULL == (mysql = mysql_init(NULL))) { - LOG_ERROR("mysql_init fail", KR(ret)); ret = OB_ERR_UNEXPECTED; + LOG_ERROR("mysql_init fail", KR(ret)); } else { + int64_t final_timeout = cal_timeout_(); + mysql_options(mysql, MYSQL_OPT_CONNECT_TIMEOUT, &final_timeout); if (mysql != mysql_real_connect(mysql, host, user_name, user_password, default_db_name, port, NULL, 0)) { - LOG_WARN("mysql connect failed", "mysql_error", mysql_error(mysql), K(host), K(port)); + int mysql_error_code = mysql_errno(mysql); // remove invalid ip if mysql error code is ER_CONNECT_FAILED + if ((ER_CONNECT_FAILED == mysql_error_code) && OB_FAIL(invalid_ip_array.push_back(idx))) { + LOG_WARN("[RESTORE PROXY] fail to push back invalid_ip_array", K(invalid_ip_array), K(idx)); + } + LOG_WARN("[RESTORE PROXY] mysql connect failed", "mysql_error", mysql_error(mysql), K(mysql_error_code), K(host), K(port)); } else { #ifdef OB_BUILD_ORACLE_PARSER is_oracle_mode_ = mysql->oracle_mode; @@ -768,24 +781,48 @@ int ObLogRestoreProxyUtil::detect_tenant_mode_(common::sqlclient::ObMySQLServerP is_oracle_mode_ = false; #endif detect_succ = true; - LOG_INFO("[RESTORE PROXY][DETECT_TENANT_MODE] detect tenant mode success", KR(ret)); + LOG_INFO("[RESTORE PROXY] detect tenant mode success", KR(ret), K(host), K(port)); + } + if (OB_NOT_NULL(mysql)) { + mysql_close(mysql); } } } - if (OB_SUCC(ret) && OB_NOT_NULL(mysql)) { - mysql_close(mysql); - mysql = NULL; - } + mysql = NULL; } if (OB_UNLIKELY(!detect_succ)) { ret = OB_CONNECT_ERROR; LOG_WARN("[RESTORE PROXY][DETECT_TENANT_MODE] connect to all server in tenant endpoint list failed", KR(ret), K(svr_cnt)); + } else if (!invalid_ip_array.empty()) { + for (int64_t idx = invalid_ip_array.count() - 1; idx >= 0; idx--) { + if (OB_FAIL(fixed_server_list->remove(invalid_ip_array.at(idx)))) { + LOG_WARN("[RESTORE PROXY] fail to remove from fixed_server_list", K(invalid_ip_array), K(*fixed_server_list), K(idx)); + } + } + LOG_INFO("[RESTORE PROXY] fixed server list", K_(tenant_id), K(*fixed_server_list)); + } + if (fixed_server_list->count() <= 0) { + ret = OB_CONNECT_ERROR; + LOG_WARN("[RESTORE PROXY] all servers are unreachable", K_(tenant_id), K(ret)); } } } - return ret; } +int64_t ObLogRestoreProxyUtil::cal_timeout_() { + const int64_t DEFAULT_MAX_TIMEOUT = 10; // default max time is 10s + int64_t final_timeout = DEFAULT_MAX_TIMEOUT; + int64_t abs_timeout_ts = THIS_WORKER.get_timeout_ts(); + int64_t curr_ts = ObTimeUtility::fast_current_time(); + int64_t abs_timeout = (abs_timeout_ts - curr_ts) /1000 /1000 /2; // half of the total timeout + if (abs_timeout > 0) { + final_timeout = abs_timeout < DEFAULT_MAX_TIMEOUT ? abs_timeout : DEFAULT_MAX_TIMEOUT; + } + LOG_INFO("[RESTORE PROXY] set time out,", + K(abs_timeout_ts), K(curr_ts), K(abs_timeout), K(final_timeout), K(DEFAULT_MAX_TIMEOUT)); + return final_timeout; +} + } // namespace share } // namespace oceanbase diff --git a/src/share/ob_log_restore_proxy.h b/src/share/ob_log_restore_proxy.h index 6e2f4ea841..12fb34fec4 100644 --- a/src/share/ob_log_restore_proxy.h +++ b/src/share/ob_log_restore_proxy.h @@ -154,7 +154,11 @@ private: // check if user or password changed bool is_user_changed_(const char *user_name, const char *user_password); void destroy_tg_(); - int detect_tenant_mode_(common::sqlclient::ObMySQLServerProvider *server_provider, const char *user_name, const char *user_password); + int detect_tenant_mode_(common::sqlclient::ObMySQLServerProvider *server_provider, + const char *user_name, + const char *user_password, + common::ObIArray *new_server_list); + int64_t cal_timeout_(); private: int construct_server_ip_list(const common::ObSqlString &sql, common::ObIArray &addrs); bool inited_;