From e078f95a4298f01297c0988e8eba7927b8c97a09 Mon Sep 17 00:00:00 2001 From: Marko Date: Mon, 27 May 2019 18:22:06 +0300 Subject: [PATCH] MXS-1550 Add net_write_timeout paramater for service net_write_timeout is used to timeout writes take too long. --- include/maxscale/config.hh | 1 + include/maxscale/dcb.hh | 3 ++- include/maxscale/service.hh | 1 + server/core/config.cc | 7 ++++++ server/core/dcb.cc | 41 +++++++++++++++++++++++++++++------- server/core/routingworker.cc | 2 +- server/core/service.cc | 13 +++++++++++- 7 files changed, 57 insertions(+), 11 deletions(-) diff --git a/include/maxscale/config.hh b/include/maxscale/config.hh index 9892ffc84..b5a68759c 100644 --- a/include/maxscale/config.hh +++ b/include/maxscale/config.hh @@ -158,6 +158,7 @@ extern const char CN_MONITOR_INTERVAL[]; extern const char CN_MONITORS[]; extern const char CN_MS_TIMESTAMP[]; extern const char CN_NAME[]; +extern const char CN_NET_WRITE_TIMEOUT[]; extern const char CN_NON_BLOCKING_POLLS[]; extern const char CN_OPERATION[]; extern const char CN_OPTIONS[]; diff --git a/include/maxscale/dcb.hh b/include/maxscale/dcb.hh index af93e33e6..02f4e51b4 100644 --- a/include/maxscale/dcb.hh +++ b/include/maxscale/dcb.hh @@ -194,6 +194,7 @@ struct DCB : public MXB_POLL_DATA void* authenticator_data = nullptr;/**< The authenticator data for this DCB */ DCB_CALLBACK* callbacks = nullptr; /**< The list of callbacks for the DCB */ int64_t last_read = 0; /**< Last time the DCB received data */ + int64_t last_write = 0; /**< Last time the DCB sent data */ struct SERVER* server = nullptr; /**< The associated backend server */ SSL* ssl = nullptr; /**< SSL struct for connection */ bool ssl_read_want_read = false; @@ -285,7 +286,7 @@ int dcb_accept_SSL(DCB* dcb); int dcb_connect_SSL(DCB* dcb); int dcb_listen(DCB* listener, const char* config); void dcb_enable_session_timeouts(); -void dcb_process_idle_sessions(int thr); +void dcb_process_timeouts(int thr); /** * @brief Append a buffer the DCB's readqueue diff --git a/include/maxscale/service.hh b/include/maxscale/service.hh index 630f9efee..862cc8a32 100644 --- a/include/maxscale/service.hh +++ b/include/maxscale/service.hh @@ -126,6 +126,7 @@ public: * to escape at least the underscore * character. */ int64_t conn_idle_timeout; /**< Session timeout in seconds */ + int64_t net_write_timeout; /**< Write timeout in seconds */ char weightby[MAX_SERVICE_WEIGHTBY_LEN]; /**< Service weighting parameter name * */ bool retry_start; /**< If starting of the service should diff --git a/server/core/config.cc b/server/core/config.cc index 0c2d9d68c..918067034 100644 --- a/server/core/config.cc +++ b/server/core/config.cc @@ -103,6 +103,7 @@ const char CN_CLASSIFICATION[] = "classification"; const char CN_CLASSIFY[] = "classify"; const char CN_CLUSTER[] = "cluster"; const char CN_CONNECTION_TIMEOUT[] = "connection_timeout"; +const char CN_NET_WRITE_TIMEOUT[] = "net_write_timeout"; const char CN_DATA[] = "data"; const char CN_DEFAULT[] = "default"; const char CN_DESCRIPTION[] = "description"; @@ -339,6 +340,12 @@ const MXS_MODULE_PARAM config_service_params[] = "0", MXS_MODULE_OPT_DURATION_S }, + { + CN_NET_WRITE_TIMEOUT, + MXS_MODULE_PARAM_DURATION, + "0", + MXS_MODULE_OPT_DURATION_S + }, { CN_AUTH_ALL_SERVERS, MXS_MODULE_PARAM_BOOL, diff --git a/server/core/dcb.cc b/server/core/dcb.cc index 92ac5ce1a..9f7c16ef8 100644 --- a/server/core/dcb.cc +++ b/server/core/dcb.cc @@ -164,6 +164,7 @@ DCB::DCB(Role role, MXS_SESSION* session) , low_water(config_writeq_low_water()) , service(session->service) , last_read(mxs_clock()) + , last_write(mxs_clock()) , m_uid(this_unit.uid_generator.fetch_add(1, std::memory_order_relaxed)) { // TODO: Remove DCB::Role::INTERNAL to always have a valid listener @@ -347,6 +348,7 @@ DCB* dcb_connect(SERVER* srv, MXS_SESSION* session, const char* protocol) dcb->persistentstart = 0; dcb->was_persistent = true; dcb->last_read = mxs_clock(); + dcb->last_write = mxs_clock(); mxb::atomic::add(&server->stats.n_from_pool, 1, mxb::atomic::RELAXED); return dcb; } @@ -984,6 +986,10 @@ int dcb_drain_writeq(DCB* dcb) * so the remaining data is put back at the front of the write * queue. */ + if (written) + { + dcb->last_write = mxs_clock(); + } if (stop_writing) { dcb->writeq = dcb->writeq ? gwbuf_append(local_writeq, dcb->writeq) : local_writeq; @@ -2447,28 +2453,32 @@ void dcb_enable_session_timeouts() } /** - * Close sessions that have been idle for too long. + * Close sessions that have been idle or write to the socket has taken for too long. * - * If the time since a session last sent data is greater than the set value in the - * service, it is disconnected. The connection timeout is disabled by default. + * If the time since a session last sent data is greater than the set connection_timeout + * value in the service, it is disconnected. If the time since last write + * to the socket is greater net_write_timeout the session is also disconnected. + * The timeouts are disabled by default. */ -void dcb_process_idle_sessions(int thr) +void dcb_process_timeouts(int thr) { if (this_unit.check_timeouts && mxs_clock() >= this_thread.next_timeout_check) { - /** Because the resolution of the timeout is one second, we only need to - * check for it once per second. One heartbeat is 100 milliseconds. */ + /** Because the resolutions of the timeouts is one second, we only need to + * check them once per second. One heartbeat is 100 milliseconds. */ this_thread.next_timeout_check = mxs_clock() + 10; for (DCB* dcb = this_unit.all_dcbs[thr]; dcb; dcb = dcb->thread.next) { - if (dcb->role == DCB::Role::CLIENT) + if (dcb->role == DCB::Role::CLIENT && dcb->state == DCB_STATE_POLLING) { SERVICE* service = dcb->session->service; - if (service->conn_idle_timeout && dcb->state == DCB_STATE_POLLING) + if (service->conn_idle_timeout) { int64_t idle = mxs_clock() - dcb->last_read; + // Multiply by 10 to match conn_idle_timeout resolution + // to the 100 millisecond tics. int64_t timeout = service->conn_idle_timeout * 10; if (idle > timeout) @@ -2481,6 +2491,21 @@ void dcb_process_idle_sessions(int thr) poll_fake_hangup_event(dcb); } } + + if (service->net_write_timeout && dcb->writeqlen > 0) + { + int64_t idle = mxs_clock() - dcb->last_write; + // Multiply by 10 to match net_write_timeout resolution + // to the 100 millisecond tics. + if (idle > dcb->service->net_write_timeout * 10) + { + MXS_WARNING("network write timed out for '%s'@%s, ", + dcb->user ? dcb->user : "", + dcb->remote ? dcb->remote : ""); + dcb->session->close_reason = SESSION_CLOSE_TIMEOUT; + poll_fake_hangup_event(dcb); + } + } } } } diff --git a/server/core/routingworker.cc b/server/core/routingworker.cc index b57e4d560..67cac1364 100644 --- a/server/core/routingworker.cc +++ b/server/core/routingworker.cc @@ -625,7 +625,7 @@ RoutingWorker* RoutingWorker::create(int epoll_listener_fd) void RoutingWorker::epoll_tick() { - dcb_process_idle_sessions(m_id); + dcb_process_timeouts(m_id); delete_zombies(); diff --git a/server/core/service.cc b/server/core/service.cc index f5a86349d..6d1df2a85 100644 --- a/server/core/service.cc +++ b/server/core/service.cc @@ -106,7 +106,7 @@ Service* service_alloc(const char* name, const char* router, MXS_CONFIG_PARAMETE return NULL; } - if (service->conn_idle_timeout) + if (service->conn_idle_timeout || service->net_write_timeout) { dcb_enable_session_timeouts(); } @@ -220,6 +220,7 @@ Service::Service(const std::string& name, retry_start = params->get_bool(CN_RETRY_ON_FAILURE); enable_root = params->get_bool(CN_ENABLE_ROOT_USER); conn_idle_timeout = params->get_duration(CN_CONNECTION_TIMEOUT).count(); + net_write_timeout = params->get_duration(CN_NET_WRITE_TIMEOUT).count(); max_connections = params->get_integer(CN_MAX_CONNECTIONS); log_auth_warnings = params->get_bool(CN_LOG_AUTH_WARNINGS); strip_db_esc = params->get_bool(CN_STRIP_DB_ESC); @@ -1904,6 +1905,7 @@ bool Service::is_basic_parameter(const std::string& name) { CN_AUTH_ALL_SERVERS, CN_CONNECTION_TIMEOUT, + CN_NET_WRITE_TIMEOUT, CN_ENABLE_ROOT_USER, CN_LOCALHOST_MATCH_WILDCARD_HOST, CN_LOG_AUTH_WARNINGS, @@ -1957,6 +1959,15 @@ void Service::update_basic_parameter(const std::string& key, const std::string& mxb_assert(conn_idle_timeout >= 0); } + else if (key == CN_NET_WRITE_TIMEOUT) + { + if ((net_write_timeout = std::stoi(value))) + { + dcb_enable_session_timeouts(); + } + + mxb_assert(net_write_timeout >= 0); + } else if (key == CN_AUTH_ALL_SERVERS) { users_from_all = config_truth_value(value.c_str());