MXS-2456: Test transaction replay cap

Added a test that makes sure the transaction replay cap is respected. Also
improved the logging to show how many transaction replay attemps have been
done and to log if a replay is not done due to too many attempts.

maxscale-system-test/CMakeLists.txt

@@ -318,6 +318,9 @@ add_test_executable(mxs1961_standalone_rejoin.cpp mxs1961_standalone_rejoin mxs1
 # https://jira.mariadb.org/browse/MXS-1493
 add_test_executable(verify_master_failure.cpp verify_master_failure verify_master_failure LABELS mysqlmon REPL_BACKEND)
 
+# MXS-2456: Cap transaction replay attempts
+add_test_executable(mxs2456_trx_replay_cap.cpp mxs2456_trx_replay_cap mxs2456_trx_replay_cap LABELS REPL_BACKEND)
+
 ############################################
 # END: Tests that require GTID             #
 ############################################

maxscale-system-test/cnf/maxscale.cnf.template.mxs2456_trx_replay_cap

@@ -0,0 +1,32 @@
+[maxscale]
+threads=###threads###
+log_info=1
+
+###server###
+
+[MariaDB-Monitor]
+type=monitor
+module=mysqlmon
+servers=###server_line###
+user=maxskysql
+password=skysql
+monitor_interval=1000
+auto_failover=true
+auto_rejoin=true
+failcount=1
+
+[RW-Split-Router]
+type=service
+router=readwritesplit
+servers=###server_line###
+user=maxskysql
+password=skysql
+delayed_retry_timeout=30
+transaction_replay=true
+transaction_replay_attempts=2
+
+[RW-Split-Listener]
+type=listener
+service=RW-Split-Router
+protocol=MySQLClient
+port=4006

maxscale-system-test/mxs2456_trx_replay_cap.cpp

@@ -0,0 +1,83 @@
+/**
+ * MXS-2456: Cap transaction replay attempts
+ * https://jira.mariadb.org/browse/MXS-2456
+ */
+
+#include "testconnections.h"
+
+#define EXPECT(a) test.expect(a, "%s", "Assertion failed: " #a)
+
+void test_replay_ok(TestConnections& test)
+{
+    Connection c = test.maxscales->rwsplit();
+    EXPECT(c.connect());
+    EXPECT(c.query("BEGIN"));
+    EXPECT(c.query("SELECT 1"));
+    EXPECT(c.query("SELECT SLEEP(15)"));
+
+    // Block the node where the transaction was started
+    test.repl->block_node(0);
+    test.maxscales->wait_for_monitor();
+    sleep(5);
+
+    // Then block the node where the transaction replay is attempted before the last statement finishes
+    test.repl->block_node(1);
+    test.maxscales->wait_for_monitor();
+    sleep(5);
+
+    // The next query should succeed as we do two replay attempts
+    test.expect(c.query("SELECT 2"), "Two transaction replays should work");
+
+    // Reset the replication
+    test.repl->unblock_node(1);
+    test.repl->unblock_node(0);
+    test.maxscales->wait_for_monitor();
+    test.check_maxctrl("call command mariadbmon reset-replication MariaDB-Monitor server1");
+    test.maxscales->wait_for_monitor();
+}
+
+void test_replay_failure(TestConnections& test)
+{
+    Connection c = test.maxscales->rwsplit();
+    c.connect();
+    c.query("BEGIN");
+    c.query("SELECT 1");
+    c.query("SELECT SLEEP(15)");
+
+    // Block the node where the transaction was started
+    test.repl->block_node(0);
+    test.maxscales->wait_for_monitor();
+    sleep(5);
+
+    // Then block the node where the first transaction replay is attempted
+    test.repl->block_node(1);
+    test.maxscales->wait_for_monitor();
+    sleep(5);
+
+    // Block the final node before the replay completes
+    test.repl->block_node(2);
+    test.maxscales->wait_for_monitor();
+    sleep(5);
+
+    // The next query should fail as we exceeded the cap of two replays
+    test.expect(!c.query("SELECT 2"), "Three transaction replays should NOT work");
+
+    // Reset the replication
+    test.repl->unblock_node(2);
+    test.repl->unblock_node(1);
+    test.repl->unblock_node(0);
+    test.maxscales->wait_for_monitor();
+    test.check_maxctrl("call command mariadbmon reset-replication MariaDB-Monitor server1");
+    test.maxscales->wait_for_monitor();
+}
+
+int main(int argc, char* argv[])
+{
+    Mariadb_nodes::require_gtid(true);
+    TestConnections test(argc, argv);
+
+    test_replay_ok(test);
+    test_replay_failure(test);
+
+    return test.global_result;
+}

server/modules/routing/readwritesplit/rwsplitsession.cc

@@ -885,6 +885,8 @@ bool RWSplitSession::start_trx_replay()
 
     if (m_config.transaction_replay && m_can_replay_trx && m_num_trx_replays < m_config.trx_max_attempts)
     {
+        ++m_num_trx_replays;
+
         if (!m_is_replay_active)
         {
             // This is the first time we're retrying this transaction, store it and the interrupted query
@@ -910,7 +912,7 @@ bool RWSplitSession::start_trx_replay()
             // Stash any interrupted queries while we replay the transaction
             m_interrupted_query.reset(m_current_query.release());
 
-            MXS_INFO("Starting transaction replay");
+            MXS_INFO("Starting transaction replay %ld", m_num_trx_replays);
             m_is_replay_active = true;
 
             /**
@@ -951,9 +953,14 @@ bool RWSplitSession::start_trx_replay()
                                "transaction had no statements and no query was interrupted");
         }
 
-        ++m_num_trx_replays;
         rval = true;
     }
+    else if (m_num_trx_replays >= m_config.trx_max_attempts)
+    {
+        mxb_assert(m_num_trx_replays == m_config.trx_max_attempts);
+        MXS_INFO("Transaction replay attempt cap of %ld exceeded, not attempting replay",
+                 m_config.trx_max_attempts);
+    }
 
     return rval;
 }