diff --git a/deps/oblib/src/rpc/obmysql/ob_sql_nio.cpp b/deps/oblib/src/rpc/obmysql/ob_sql_nio.cpp index 055d24238..e1266af6f 100644 --- a/deps/oblib/src/rpc/obmysql/ob_sql_nio.cpp +++ b/deps/oblib/src/rpc/obmysql/ob_sql_nio.cpp @@ -581,7 +581,9 @@ private: class ObSqlNioImpl { public: - ObSqlNioImpl(ObISqlSockHandler& handler): handler_(handler), epfd_(-1), lfd_(-1) {} + ObSqlNioImpl(ObISqlSockHandler& handler): + handler_(handler), epfd_(-1), lfd_(-1), tcp_keepalive_enabled_(0), + tcp_keepidle_(0), tcp_keepintvl_(0), tcp_keepcnt_(0) {} ~ObSqlNioImpl() {} int init(int port) { int ret = OB_SUCCESS; @@ -613,6 +615,7 @@ public: handle_write_req_queue(); handle_close_req_queue(); handle_pending_destroy_list(); + update_tcp_keepalive_parameters(); print_session_info(); } void push_close_req(ObSqlSock* s) { @@ -645,7 +648,12 @@ public: } } } - + void update_tcp_keepalive_params(int keepalive_enabled, uint32_t tcp_keepidle, uint32_t tcp_keepintvl, uint32_t tcp_keepcnt) { + tcp_keepalive_enabled_ = keepalive_enabled; + tcp_keepidle_ = tcp_keepidle; + tcp_keepintvl_ = tcp_keepintvl; + tcp_keepcnt_ = tcp_keepcnt; + } private: void handle_epoll_event() { const int maxevents = 512; @@ -828,6 +836,29 @@ private: all_list_.del(&s->all_list_link_); } + void update_tcp_keepalive_parameters() { + if (TC_REACH_TIME_INTERVAL(5*1000*1000L)) { + if (1 == tcp_keepalive_enabled_) { + ObDLink* head = all_list_.head(); + ObLink* cur = head->next_; + while (cur != head) { + ObSqlSock* s = CONTAINER_OF(cur, ObSqlSock, all_list_link_); + cur = cur->next_; + int fd = s->get_fd(); + if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, (const void *)&tcp_keepalive_enabled_, sizeof(tcp_keepalive_enabled_)) < 0) { + LOG_WARN("setsockopt SO_KEEPALIVE failed", K(fd), KERRNOMSG(errno)); + } else if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE, (const void *)&tcp_keepidle_, sizeof(tcp_keepidle_)) < 0) { + LOG_WARN("setsockopt TCP_KEEPIDLE failed", K(fd), KERRNOMSG(errno)); + } else if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPINTVL, (const void *)&tcp_keepintvl_, sizeof(tcp_keepintvl_)) < 0) { + LOG_WARN("setsockopt TCP_KEEPINTVL failed", K(fd), KERRNOMSG(errno)); + } else if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPCNT, (const void *)&tcp_keepcnt_, sizeof(tcp_keepcnt_)) < 0) { + LOG_WARN("setsockopt TCP_KEEPCNT failed", K(fd), KERRNOMSG(errno)); + } + } + } + } + } + void print_session_info() { static const int64_t max_process_time = 1000L * 1000L * 20L; // 20s if (TC_REACH_TIME_INTERVAL(15*1000*1000L)) { @@ -857,6 +888,10 @@ private: ObSpScLinkQueue write_req_queue_; ObDList pending_destroy_list_; ObDList all_list_; + int tcp_keepalive_enabled_; + uint32_t tcp_keepidle_; + uint32_t tcp_keepintvl_; + uint32_t tcp_keepcnt_; }; int ObSqlNio::start(int port, ObISqlSockHandler* handler, int n_thread) @@ -989,5 +1024,18 @@ SSL* ObSqlNio::get_ssl_st(void* sess) ObSqlSock* sock = sess2sock(sess); return sock->get_ssl_st(); } + +void ObSqlNio::update_tcp_keepalive_params(int keepalive_enabled, uint32_t tcp_keepidle, uint32_t tcp_keepintvl, uint32_t tcp_keepcnt) +{ + int thread_count = get_thread_count(); + if (NULL != impl_) { + for (int index = 0; index < thread_count; index++) { + impl_[index].update_tcp_keepalive_params(keepalive_enabled, tcp_keepidle, tcp_keepintvl, tcp_keepcnt); + } + } else { + LOG_ERROR("sql nio impl_ is nullptr", KP(impl_)); + } +} + }; // end namespace obmysql }; // end namespace oceanbase diff --git a/deps/oblib/src/rpc/obmysql/ob_sql_nio.h b/deps/oblib/src/rpc/obmysql/ob_sql_nio.h index 607424004..0995439de 100644 --- a/deps/oblib/src/rpc/obmysql/ob_sql_nio.h +++ b/deps/oblib/src/rpc/obmysql/ob_sql_nio.h @@ -46,6 +46,7 @@ public: void shutdown(void* sess); int set_ssl_enabled(void* sess); SSL* get_ssl_st(void* sess); + void update_tcp_keepalive_params(int keepalive_enabled, uint32_t tcp_keepidle, uint32_t tcp_keepintvl, uint32_t tcp_keepcnt); private: void run(int64_t idx); private: diff --git a/deps/oblib/src/rpc/obmysql/ob_sql_nio_server.cpp b/deps/oblib/src/rpc/obmysql/ob_sql_nio_server.cpp index f2d32daea..f12bec69a 100644 --- a/deps/oblib/src/rpc/obmysql/ob_sql_nio_server.cpp +++ b/deps/oblib/src/rpc/obmysql/ob_sql_nio_server.cpp @@ -45,6 +45,11 @@ void ObSqlNioServer::destroy() nio_.destroy(); } +void ObSqlNioServer::update_tcp_keepalive_params(int keepalive_enabled, uint32_t tcp_keepidle, uint32_t tcp_keepintvl, uint32_t tcp_keepcnt) +{ + nio_.update_tcp_keepalive_params(keepalive_enabled, tcp_keepidle, tcp_keepintvl, tcp_keepcnt); +} + ObSqlNioServer* global_sql_nio_server = NULL; }; // end namespace obmysql }; // end namespace oceanbase diff --git a/deps/oblib/src/rpc/obmysql/ob_sql_nio_server.h b/deps/oblib/src/rpc/obmysql/ob_sql_nio_server.h index 3db54a977..679d893f3 100644 --- a/deps/oblib/src/rpc/obmysql/ob_sql_nio_server.h +++ b/deps/oblib/src/rpc/obmysql/ob_sql_nio_server.h @@ -33,6 +33,7 @@ public: void stop(); void wait(); void destroy(); + void update_tcp_keepalive_params(int keepalive_enabled, uint32_t tcp_keepidle, uint32_t tcp_keepintvl, uint32_t tcp_keepcnt); private: ObSqlSockProcessor thread_processor_; // for tenant worker ObSqlSockHandler io_handler_; // for io thread diff --git a/src/observer/ob_srv_network_frame.cpp b/src/observer/ob_srv_network_frame.cpp index f3089d44e..11cf83abf 100644 --- a/src/observer/ob_srv_network_frame.cpp +++ b/src/observer/ob_srv_network_frame.cpp @@ -30,6 +30,7 @@ using namespace oceanbase::rpc::frame; using namespace oceanbase::common; using namespace oceanbase::observer; using namespace oceanbase::share; +using namespace oceanbase::obmysql; ObSrvNetworkFrame::ObSrvNetworkFrame(ObGlobalContext &gctx) : gctx_(gctx), @@ -77,6 +78,19 @@ static int get_default_net_thread_count() return cnt; } +static int update_tcp_keepalive_parameters_for_sql_nio_server(int tcp_keepalive_enabled, int64_t tcp_keepidle, int64_t tcp_keepintvl, int64_t tcp_keepcnt) +{ + int ret = OB_SUCCESS; + if (enable_new_sql_nio()) { + if (NULL != global_sql_nio_server) { + tcp_keepidle = max(tcp_keepidle/1000000, 1); + tcp_keepintvl = max(tcp_keepintvl/1000000, 1); + global_sql_nio_server->update_tcp_keepalive_params(tcp_keepalive_enabled, tcp_keepidle, tcp_keepintvl, tcp_keepcnt); + } + } + return ret; +} + int ObSrvNetworkFrame::init() { int ret = OB_SUCCESS; @@ -185,6 +199,7 @@ int ObSrvNetworkFrame::start() return ret; } + int ObSrvNetworkFrame::reload_config() { int ret = common::OB_SUCCESS; @@ -217,6 +232,10 @@ int ObSrvNetworkFrame::reload_config() tcp_keepidle, tcp_keepintvl, tcp_keepcnt))) { LOG_WARN("Failed to set sql tcp keepalive parameters."); + } else if (OB_FAIL(update_tcp_keepalive_parameters_for_sql_nio_server(enable_tcp_keepalive, + tcp_keepidle, tcp_keepintvl, + tcp_keepcnt))) { + LOG_WARN("Failed to set sql tcp keepalive parameters for sql nio server", K(ret)); } return ret;