MXS-1507: Make transaction replay configurable

The transaction retrying behavior is now configurable and documented. The
`transaction_replay` parameter implicitly enables the required
functionality in the router that it needs.

Documentation/Routers/ReadWriteSplit.md

@@ -329,6 +329,36 @@ execution is an acceptable risk.
 The number of seconds to wait until an error is returned to the client when
 `delayed_retry` is enabled. The default value is 10 seconds.
 
+### `transaction_replay`
+
+Replay interrupted transactions. This parameter was added in MaxScale 2.3.0 and
+is disabled by default. Enabling this parameter implicitly enables both the
+`delayed_retry` and `master_reconnection` parameters.
+
+When the server where the transaction is in progress fails, readwritesplit can
+migrate the transaction to a replacement server. This can completely hide the
+failure of a master node without any visible effects to the client.
+
+If no replacement node becomes available before the timeout controlled by
+`delayed_retry_timeout` is exceeded, the client connection is closed.
+
+Not all transactions can be safely replayed. Only when the following criteria
+are met, the transaction can be safely replayed.
+
+* Transaction contains only data modification (`INSERT`, `UPDATE`, `DELETE`
+  etc.) or `SELECT ... FOR UPDATE` statements.
+
+* The replacement server where the transaction is applied returns results
+  identical to the original partial transaction.
+
+If the results from the replacement server are not identical when the
+transaction is replayed, the client connection is closed.
+
+Performing MVCC reads (`SELECT` queries without `FOR UPDATE` or `LOCK IN SHARE MODE`)
+with transaction replay is discouraged. If such statements are executed
+but the results of each reply are identical, the transaction is replayed but the results
+are not guaranteed to be consistent on the database level.
+
 ## Routing hints
 
 The readwritesplit router supports routing hints. For a detailed guide on hint

server/modules/routing/readwritesplit/readwritesplit.cc

@@ -290,6 +290,16 @@ RWSplit* RWSplit::create(SERVICE *service, char **options)
         config.max_sescmd_history = 0;
     }
 
+    if (config.transaction_replay)
+    {
+        /**
+         * Replaying transactions requires that we are able to do delayed query
+         * retries and reconnect to a master.
+         */
+        config.delayed_retry = true;
+        config.master_reconnection = true;
+    }
+
     return new (std::nothrow) RWSplit(service, config);
 }
 
@@ -496,6 +506,7 @@ MXS_MODULE *MXS_CREATE_MODULE()
             {"master_reconnection", MXS_MODULE_PARAM_BOOL, "false"},
             {"delayed_retry", MXS_MODULE_PARAM_BOOL, "false"},
             {"delayed_retry_timeout", MXS_MODULE_PARAM_COUNT, "10"},
+            {"transaction_replay", MXS_MODULE_PARAM_BOOL, "false"},
             {MXS_END_MODULE_PARAMS}
         }
     };

server/modules/routing/readwritesplit/readwritesplit.hh

@@ -163,7 +163,8 @@ struct Config
         causal_read_timeout(config_get_string(params, "causal_read_timeout")),
         master_reconnection(config_get_bool(params, "master_reconnection")),
         delayed_retry(config_get_bool(params, "delayed_retry")),
-        delayed_retry_timeout(config_get_integer(params, "delayed_retry_timeout"))
+        delayed_retry_timeout(config_get_integer(params, "delayed_retry_timeout")),
+        transaction_replay(config_get_bool(params, "transaction_replay"))
     {
         if (enable_causal_read)
         {
@@ -193,6 +194,7 @@ struct Config
     bool              master_reconnection;       /**< Allow changes in master server */
     bool              delayed_retry;             /**< Delay routing if no target found */
     uint64_t          delayed_retry_timeout;     /**< How long to delay until an error is returned */
+    bool              transaction_replay;        /**< Replay failed transactions */
 };
 
 /**

server/modules/routing/readwritesplit/rwsplitsession.cc

@@ -437,7 +437,7 @@ void RWSplitSession::clientReply(GWBUF *writebuf, DCB *backend_dcb)
         return;
     }
 
-    if (session_trx_is_active(m_client->session))
+    if (m_config.transaction_replay && session_trx_is_active(m_client->session))
     {
         m_trx.add_stmt(m_current_query.release());
         m_trx.add_result(writebuf);
@@ -464,13 +464,14 @@ void RWSplitSession::clientReply(GWBUF *writebuf, DCB *backend_dcb)
 
     if (m_is_replay_active)
     {
+        ss_dassert(m_config.transaction_replay);
         handle_trx_replay();
 
         // Ignore the response, the client doesn't need it
         gwbuf_free(writebuf);
         return;
     }
-    else if (session_trx_is_ending(m_client->session))
+    else if (m_config.transaction_replay && session_trx_is_ending(m_client->session))
     {
         m_trx.close();
     }
@@ -630,12 +631,19 @@ void RWSplitSession::handleError(GWBUF *errmsgbuf, DCB *problem_dcb,
                     can_continue = false;
                 }
                 else if (session_trx_is_active(session))
+                {
+                    if (m_config.transaction_replay)
                     {
                         // Stash any interrupted queries while we replay the transaction
                         m_interrupted_query.reset(m_current_query.release());
                         can_continue = true;
                         start_trx_replay();
                     }
+                    else
+                    {
+                        can_continue = false;
+                    }
+                }
 
                 *succp = can_continue;
                 backend->close();