From 40d05e82789321dd0ed4ffad04ede4477fa80ee9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 19 Sep 2019 12:55:31 +0300 Subject: [PATCH 1/3] Report more details when the master is unavailable By dumping the connection state we'll know when the connection was opened and closed and the reason why it was closed. --- server/modules/routing/readwritesplit/rwsplit_route_stmt.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/modules/routing/readwritesplit/rwsplit_route_stmt.cc b/server/modules/routing/readwritesplit/rwsplit_route_stmt.cc index 7919d5a0a..50e4907ae 100644 --- a/server/modules/routing/readwritesplit/rwsplit_route_stmt.cc +++ b/server/modules/routing/readwritesplit/rwsplit_route_stmt.cc @@ -738,8 +738,8 @@ SRWBackend RWSplitSession::get_master_backend() else { MXS_ERROR("Cannot choose server '%s' as the master because it is not " - "in use and a new connection to it cannot be created.", - master->name()); + "in use and a new connection to it cannot be created. Connection status: %s", + master->name(), master->get_verbose_status().c_str()); } } From fd0c156655c6c2f3aaa61bf4d74c84c96d2664b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 19 Sep 2019 13:18:06 +0300 Subject: [PATCH 2/3] MXS-2564: Reconnect only when necessary By doing the reconnection only when a new query arrives, we prevent the excessive reconnecting that is done when a server's actual and monitored states are in conflict. --- .../routing/readwritesplit/rwsplitsession.cc | 35 ++++--------------- .../routing/readwritesplit/rwsplitsession.hh | 8 +++++ 2 files changed, 14 insertions(+), 29 deletions(-) diff --git a/server/modules/routing/readwritesplit/rwsplitsession.cc b/server/modules/routing/readwritesplit/rwsplitsession.cc index aad2e8a20..0cc70e97d 100644 --- a/server/modules/routing/readwritesplit/rwsplitsession.cc +++ b/server/modules/routing/readwritesplit/rwsplitsession.cc @@ -1175,39 +1175,16 @@ bool RWSplitSession::handle_error_new_connection(DCB* backend_dcb, GWBUF* errmsg route_stored_query(); } - bool succp = false; - /** - * Try to get replacement slave or at least the minimum - * number of slave connections for router session. - */ - if (m_recv_sescmd > 0 && m_config.disable_sescmd_history) - { - for (const auto& a : m_backends) - { - if (a->in_use()) - { - succp = true; - break; - } - } + bool ok = can_recover_servers() || have_open_connections(); - if (!succp) - { - MXS_ERROR("Unable to continue session as all connections have failed, " - "last server to fail was '%s'.", backend->name()); - } - } - else + if (!ok) { - succp = m_router->select_connect_backend_servers(ses, - m_backends, - m_current_master, - &m_sescmd_list, - &m_expected_responses, - connection_type::SLAVE); + MXS_ERROR("Unable to continue session as all connections have failed and " + "new connections cannot be created. Last server to fail was '%s'.", + backend->name()); } - return succp; + return ok; } /** diff --git a/server/modules/routing/readwritesplit/rwsplitsession.hh b/server/modules/routing/readwritesplit/rwsplitsession.hh index 9da56dcf2..ded4b20b5 100644 --- a/server/modules/routing/readwritesplit/rwsplitsession.hh +++ b/server/modules/routing/readwritesplit/rwsplitsession.hh @@ -293,6 +293,14 @@ private: return !m_config.disable_sescmd_history || m_recv_sescmd == 0; } + inline bool have_open_connections() const + { + return std::any_of( + m_backends.begin(), m_backends.end(), [](const mxs::SRWBackend& b) { + return b->in_use(); + }); + } + inline bool is_large_query(GWBUF* buf) { uint32_t buflen = gwbuf_length(buf); From f46f873dc122c9acd1ba8730f3863ebc595ed1e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 19 Sep 2019 13:22:58 +0300 Subject: [PATCH 3/3] Add verbose backend status helper This allows the same verbose information to be logged in the cases where it is of use. Mostly this information can be used to figure out why a certain session was closed. --- .../routing/readwritesplit/rwsplit_route_stmt.cc | 13 +++---------- .../routing/readwritesplit/rwsplitsession.cc | 1 + .../routing/readwritesplit/rwsplitsession.hh | 13 +++++++++++++ 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/server/modules/routing/readwritesplit/rwsplit_route_stmt.cc b/server/modules/routing/readwritesplit/rwsplit_route_stmt.cc index 50e4907ae..9af9b6c55 100644 --- a/server/modules/routing/readwritesplit/rwsplit_route_stmt.cc +++ b/server/modules/routing/readwritesplit/rwsplit_route_stmt.cc @@ -591,16 +591,9 @@ bool RWSplitSession::route_session_write(GWBUF* querybuf, uint8_t command, uint3 } else { - std::string status; - for (const auto& a : m_backends) - { - status += "\n"; - status += a->get_verbose_status(); - } - - MXS_ERROR("Could not route session command: %s. Connection information: %s", + MXS_ERROR("Could not route session command: %s. Connection status: %s", attempted_write ? "Write to all backends failed" : "All connections have failed", - status.c_str()); + get_verbose_status().c_str()); } return nsucc; @@ -739,7 +732,7 @@ SRWBackend RWSplitSession::get_master_backend() { MXS_ERROR("Cannot choose server '%s' as the master because it is not " "in use and a new connection to it cannot be created. Connection status: %s", - master->name(), master->get_verbose_status().c_str()); + master->name(), get_verbose_status().c_str()); } } diff --git a/server/modules/routing/readwritesplit/rwsplitsession.cc b/server/modules/routing/readwritesplit/rwsplitsession.cc index 0cc70e97d..09dae7656 100644 --- a/server/modules/routing/readwritesplit/rwsplitsession.cc +++ b/server/modules/routing/readwritesplit/rwsplitsession.cc @@ -1182,6 +1182,7 @@ bool RWSplitSession::handle_error_new_connection(DCB* backend_dcb, GWBUF* errmsg MXS_ERROR("Unable to continue session as all connections have failed and " "new connections cannot be created. Last server to fail was '%s'.", backend->name()); + MXS_INFO("Connection status: %s", get_verbose_status().c_str()); } return ok; diff --git a/server/modules/routing/readwritesplit/rwsplitsession.hh b/server/modules/routing/readwritesplit/rwsplitsession.hh index ded4b20b5..7d2a8a713 100644 --- a/server/modules/routing/readwritesplit/rwsplitsession.hh +++ b/server/modules/routing/readwritesplit/rwsplitsession.hh @@ -301,6 +301,19 @@ private: }); } + std::string get_verbose_status() + { + std::string status; + + for (const auto& a : m_backends) + { + status += "\n"; + status += a->get_verbose_status(); + } + + return status; + } + inline bool is_large_query(GWBUF* buf) { uint32_t buflen = gwbuf_length(buf);