MXS-359: Allow master changes mid-session

With the `allow_master_change` parameter enabled, sessions can start using
a different master node if one is available. This will not prevent
sessions from closing if a write query is received while no master
replacement is available.

Documentation/Routers/ReadWriteSplit.md

@@ -105,6 +105,24 @@ connections alive even if they are not used. This is a common problem if the
 backend servers have a low _wait_timeout_ value and the client connections live
 for a long time.
 
+### `allow_master_change`
+
+Allow the master server to change mid-session. This feature was introduced in
+MaxScale 2.3.0 and is disabled by default.
+
+When a readwritesplit session starts, it will pick a master server as the
+current master server of that session. By default, when this master server
+changes mid-session, the connection will be closed.
+
+If the `allow_master_change` parameter is enabled, the master server is allowed
+to change as long as the session meets the following criteria:
+
+* The session is already connected to the slave that was chosen to be the new master
+* No transaction is open
+* Autocommit is enabled
+* No `LOAD DATA LOCAL INFILE` is in progress
+* There are no queries being actively routed to the old master
+
 ## Router options
 
 **`router_options`** may include multiple **readwritesplit**-specific options.

server/modules/routing/readwritesplit/readwritesplit.cc

@@ -1507,6 +1507,7 @@ MXS_MODULE *MXS_CREATE_MODULE()
             {"connection_keepalive", MXS_MODULE_PARAM_COUNT, "0"},
             {"enable_causal_read", MXS_MODULE_PARAM_BOOL, "false"},
             {"causal_read_timeout", MXS_MODULE_PARAM_STRING, "0"},
+            {"allow_master_change", MXS_MODULE_PARAM_BOOL, "false"},
             {MXS_END_MODULE_PARAMS}
         }
     };

server/modules/routing/readwritesplit/readwritesplit.hh

@@ -174,7 +174,8 @@ struct Config
         rw_max_slave_conn_percent(0),
         max_slave_connections(0),
         enable_causal_read(config_get_bool(params, "enable_causal_read")),
-        causal_read_timeout(config_get_string(params, "causal_read_timeout"))
+        causal_read_timeout(config_get_string(params, "causal_read_timeout")),
+        allow_master_change(config_get_bool(params, "allow_master_change"))
     {
         if (enable_causal_read)
         {
@@ -201,7 +202,7 @@ struct Config
     int               max_slave_connections;     /**< Maximum number of slaves for each connection*/
     bool              enable_causal_read;        /**< Enable causual read */
     std::string       causal_read_timeout;       /**< Timetout, second parameter of function master_wait_gtid */
-
+    bool              allow_master_change;       /**< Allow changes in master server */
 };
 
 /**

server/modules/routing/readwritesplit/rwsplit_route_stmt.cc

@@ -180,6 +180,11 @@ route_target_t get_target_type(RWSplitSession *rses, GWBUF *buffer,
     return route_target;
 }
 
+static inline bool locked_to_master(RWSplitSession *rses)
+{
+    return rses->large_query || (rses->current_master && rses->target_node == rses->current_master);
+}
+
 /**
  * Routing function. Find out query type, backend type, and target DCB(s).
  * Then route query to found target(s).
@@ -197,8 +202,7 @@ bool route_single_stmt(RWSplit *inst, RWSplitSession *rses, GWBUF *querybuf, con
     uint8_t command = info.command;
     uint32_t qtype = info.type;
     route_target_t route_target = info.target;
-    bool not_locked_to_master = !rses->large_query &&
-                                (!rses->target_node || rses->target_node != rses->current_master);
+    bool not_locked_to_master = !locked_to_master(rses);
 
     if (not_locked_to_master && is_ps_command(command))
     {
@@ -1064,6 +1068,17 @@ static void log_master_routing_failure(RWSplitSession *rses, bool found,
                 rses->client_dcb->remote, errmsg);
 }
 
+bool should_replace_master(RWSplitSession *rses, SRWBackend& target)
+{
+    return rses->rses_config.allow_master_change &&
+        // We have a target server and it's not the current master
+        target && target != rses->current_master &&
+        // We are not inside a transaction (also checks for autocommit=1)
+        !session_trx_is_active(rses->client_dcb->session) &&
+        // We are not locked to the old master
+        !locked_to_master(rses);
+}
+
 /**
  * @brief Handle master is the target
  *
@@ -1081,6 +1096,13 @@ bool handle_master_is_target(RWSplit *inst, RWSplitSession *rses,
     SRWBackend target = get_target_backend(rses, BE_MASTER, NULL, MAX_RLAG_UNDEFINED);
     bool succp = true;
 
+    if (should_replace_master(rses, target))
+    {
+        MXS_INFO("Replacing old master '%s' with new master '%s'", rses->current_master ?
+                 rses->current_master->name() : "<no previous master>", target->name());
+        rses->current_master = target;
+    }
+
     if (target && target == rses->current_master)
     {
         atomic_add_uint64(&inst->stats().n_master, 1);