Readwritesplit: Graceful maintenance mode

By allowing transactions to the master to end even if the server is in
maintenance mode makes it possible to terminate connections at a known
point. This helps prevent interrupted transactions which can help reduce
errors that are visible to the clients.

server/modules/routing/readwritesplit/readwritesplit.hh

@@ -386,6 +386,25 @@ static inline const char* failure_mode_to_str(enum failure_mode type)
 void closed_session_reply(GWBUF* querybuf);
 bool send_readonly_error(DCB* dcb);
 
+/**
+ * See if the current master is still a valid TARGET_MASTER candidate
+ *
+ * The master is valid if it's in `Master, Running` state or if it is still in but in maintenance mode while a
+ * transaction is open. If a transaction is open to a master in maintenance mode, the connection is closed on
+ * the next COMMIT or ROLLBACK.
+ *
+ * @see RWSplitSession::close_stale_connections()
+ */
+inline bool can_continue_using_master(const mxs::RWBackend* current_master)
+{
+    constexpr uint64_t bits = SERVER_MASTER | SERVER_RUNNING | SERVER_MAINT;
+    auto server = current_master->server();
+
+    return server->is_master() || (current_master->in_use()
+                                   && (server->status & bits) == bits
+                                   && session_trx_is_active(current_master->dcb()->session));
+}
+
 mxs::RWBackend* get_root_master(const mxs::PRWBackends& backends, mxs::RWBackend* current_master,
                                 const BackendSelectFunction& func);
 

server/modules/routing/readwritesplit/rwsplit_route_stmt.cc

@@ -638,7 +638,7 @@ RWBackend* RWSplitSession::get_master_backend()
     {
         if (master->in_use() || (m_config.master_reconnection && master->can_connect()))
         {
-            if (master->is_master())
+            if (can_continue_using_master(master))
             {
                 rval = master;
             }

server/modules/routing/readwritesplit/rwsplit_select_backends.cc

@@ -395,7 +395,7 @@ static void log_server_connections(select_criteria_t criteria, const PRWBackends
 RWBackend* get_root_master(const PRWBackends& backends, RWBackend* current_master,
                            const BackendSelectFunction& func)
 {
-    if (current_master && current_master->in_use() && current_master->is_master())
+    if (current_master && current_master->in_use() && can_continue_using_master(current_master))
     {
         return current_master;
     }

server/modules/routing/readwritesplit/rwsplitsession.cc

@@ -541,17 +541,26 @@ void RWSplitSession::close_stale_connections()
 {
     auto current_rank = get_current_rank();
 
-    for (auto& backend : m_backends)
+    for (auto& backend : m_raw_backends)
     {
         if (backend->in_use())
         {
             auto server = backend->server();
 
             if (!server->is_usable())
+            {
+                if (backend == m_current_master
+                    && can_continue_using_master(m_current_master)
+                    && !session_trx_is_ending(m_client->session))
+                {
+                    MXS_INFO("Keeping connection to '%s' open until transaction ends", backend->name());
+                }
+                else
                 {
                     MXS_INFO("Discarding connection to '%s': Server is in maintenance", backend->name());
                     backend->close();
                 }
+            }
             else if (server->rank() != current_rank)
             {
                 MXS_INFO("Discarding connection to '%s': Server has rank %ld and current rank is %ld",