MXS-2609: Add master reconnection test case

The test case covers a few bugs that were fixed by the previous
commits. The first part of the test covers the case when master
reconnection fails while session command history is being executed. The
second part of the test makes sure exceeding the session command history
will prevent master reconnections from taking place.

maxscale-system-test/CMakeLists.txt

@@ -976,6 +976,9 @@ add_test_executable(mxs2521_double_exec.cpp mxs2521_double_exec mxs2521_double_e
 # MXS-2490: Direct execution doesn't work with MaxScale
 add_test_executable(mxs2490_ps_execute_direct.cpp mxs2490_ps_execute_direct replication LABELS REPL_BACKEND readwritesplit)
 
+# MXS-2609: Maxscale crash in RWSplitSession::retry_master_query()
+add_test_executable(mxs2609_history_replay.cpp mxs2609_history_replay mxs2609_history_replay LABELS readwritesplit REPL_BACKEND)
+
 # MXS-2621: Incorrect SQL if lower_case_table_names is used.
 add_test_executable(mxs2621_lower_case_tables.cpp mxs2621_lower_case_tables mxs2621_lower_case_tables LABELS REPL_BACKEND)
 

maxscale-system-test/cnf/maxscale.cnf.template.mxs2609_history_replay

@@ -0,0 +1,27 @@
+[maxscale]
+threads=###threads###
+
+###server###
+
+[MySQL-Monitor]
+type=monitor
+module=mariadbmon
+servers=###server_line###
+user=maxskysql
+password=skysql
+monitor_interval=1000
+
+[RW-Split-Router]
+type=service
+router=readwritesplit
+servers=###server_line###
+user=maxskysql
+password=skysql
+transaction_replay=true
+master_accept_reads=true
+
+[RW-Split-Listener]
+type=listener
+service=RW-Split-Router
+protocol=MySQLClient
+port=4006

maxscale-system-test/mxs2609_history_replay.cpp

@@ -0,0 +1,69 @@
+/**
+ * MXS-2609: Maxscale crash in RWSplitSession::retry_master_query()
+ *
+ * https://jira.mariadb.org/browse/MXS-2609
+ *
+ * This test attempts to reproduce the crash described in MXS-2609 which
+ * occurred during a retrying attempt of a session command that failed on
+ * the master.
+ */
+
+
+#include "testconnections.h"
+
+int main(int argc, char** argv)
+{
+    TestConnections test(argc, argv);
+
+    auto block = [&test](int n){
+                     test.repl->block_node(n);
+                     test.maxscales->wait_for_monitor();
+                     test.repl->unblock_node(n);
+                     test.maxscales->wait_for_monitor();
+                 };
+    auto conn = test.maxscales->rwsplit();
+
+    //
+    // Test 1: Master failure mid-reconnect should trigger query replay
+    //
+
+    test.expect(conn.connect(), "First connect should work: %s", conn.error());
+
+    // Queue up session commands so that the history replay takes some time
+    for (int i = 0; i < 10; i++)
+    {
+        conn.query("SET @a = (SLEEP 1)");
+    }
+
+    block(0);
+
+    test.set_timeout(90);
+
+    std::thread([&](){
+                    sleep(5);
+                    block(0);
+                }).detach();
+
+    test.expect(conn.query("SELECT @@last_insert_id"), "Query should work: %s", conn.error());
+
+    test.stop_timeout();
+    conn.disconnect();
+
+    //
+    // Test 2: Exceed history limit and trigger a master reconnection
+    //
+
+    test.maxctrl("alter service RW-Split-Router max_sescmd_history 2");
+    test.expect(conn.connect(), "Second should work: %s", conn.error());
+
+    for (int i = 0; i < 5; i++)
+    {
+        conn.query("SET @a = (SLEEP 1)");
+    }
+
+    block(0);
+
+    test.expect(!conn.query("SELECT @@last_insert_id"), "Query should fail");
+
+    return test.global_result;
+}