MXS-359: Add test with master_failure_mode=error_on_write

Testing of routing behavior with master_failure_mode=error_on_write and
allow_master_changes=true. By sending an error instead of closing the
connection when the master fails, the connection can resume execution if a
new master becomes available.

maxscale-system-test/CMakeLists.txt

@@ -417,6 +417,9 @@ add_test_executable(mxs359_master_switch.cpp mxs359_master_switch mxs359_master_
 # Test read-only connections with master replacement
 add_test_executable(mxs359_read_only.cpp mxs359_read_only mxs359_read_only LABELS REPL_BACKEND)
 
+# Test master_failure_mode=error_on_write and master replacement
+add_test_executable(mxs359_error_on_write.cpp mxs359_error_on_write mxs359_error_on_write LABELS REPL_BACKEND)
+
 # Binary protocol prepared statement tests
 add_test_executable(binary_ps.cpp binary_ps replication LABELS readwritesplit LIGHT REPL_BACKEND)
 add_test_executable(binary_ps_cursor.cpp binary_ps_cursor replication LABELS readwritesplit LIGHT REPL_BACKEND)

maxscale-system-test/cnf/maxscale.cnf.template.mxs359_error_on_write

@@ -0,0 +1,60 @@
+[maxscale]
+threads=###threads###
+
+[MySQL-Monitor]
+type=monitor
+module=mysqlmon
+###repl51###
+servers=server1,server2,server3,server4
+user=maxskysql
+passwd=skysql
+monitor_interval=1000
+
+[RW-Split-Router]
+type=service
+router=readwritesplit
+servers=server1,server2,server3,server4
+user=maxskysql
+passwd=skysql
+allow_master_change=true
+master_failure_mode=error_on_write
+
+[RW-Split-Listener]
+type=listener
+service=RW-Split-Router
+protocol=MySQLClient
+port=4006
+
+[CLI]
+type=service
+router=cli
+
+[CLI-Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=default
+
+[server1]
+type=server
+address=###node_server_IP_1###
+port=###node_server_port_1###
+protocol=MySQLBackend
+
+[server2]
+type=server
+address=###node_server_IP_2###
+port=###node_server_port_2###
+protocol=MySQLBackend
+
+[server3]
+type=server
+address=###node_server_IP_3###
+port=###node_server_port_3###
+protocol=MySQLBackend
+
+[server4]
+type=server
+address=###node_server_IP_4###
+port=###node_server_port_4###
+protocol=MySQLBackend

maxscale-system-test/mxs359_error_on_write.cpp

@@ -0,0 +1,146 @@
+/**
+ * MXS-359: Starting sessions without master
+ *
+ * https://jira.mariadb.org/browse/MXS-359
+ */
+#include "testconnections.h"
+#include <vector>
+#include <iostream>
+#include <functional>
+
+using std::cout;
+using std::endl;
+
+TestConnections* self;
+
+static void change_master(int next, int current)
+{
+    TestConnections& test = *self;
+    test.maxscales->ssh_node_f(0, true, "maxadmin shutdown monitor MySQL-Monitor");
+    test.repl->connect();
+    test.repl->change_master(next, current);
+    test.repl->close_connections();
+    test.maxscales->ssh_node_f(0, true, "maxadmin restart monitor MySQL-Monitor");
+}
+
+struct Query
+{
+    const char* query;
+    bool        should_work;
+};
+
+typedef std::vector<Query> Queries;
+
+typedef std::function<void()> Func;
+
+struct Step
+{
+    const char* description;
+    Func func;
+    Queries queries;
+};
+
+struct TestCase
+{
+    const char* description;
+    std::vector<Step> steps;
+};
+
+int main(int argc, char** argv)
+{
+    TestConnections test(argc, argv);
+    self = &test;
+
+    Queries rw_ok({{"INSERT INTO test.t1 VALUES (1)", true}, {"SELECT * FROM test.t1", true}});
+    Queries rw_err({{"INSERT INTO test.t1 VALUES (1)", false}, {"SELECT * FROM test.t1", true}});
+
+    Func block_master = [&test]()
+    {
+        test.repl->block_node(0);
+        sleep(10);
+    };
+
+    Func unblock_master = [&test]()
+    {
+        test.repl->unblock_node(0);
+        sleep(10);
+    };
+
+    Func master_change = [&test]()
+    {
+        change_master(1, 0);
+        sleep(10);
+    };
+
+    Func reset = [&test]()
+    {
+        test.repl->unblock_node(0);
+        change_master(0, 1);
+        sleep(10);
+    };
+
+    Func noop = []() {};
+
+    std::vector<TestCase> tests(
+    {
+        {
+            "Master failure and replacement",
+            {
+                {"Check that writes work at startup", noop, rw_ok},
+                {"Block master and check that writes fail", block_master, rw_err},
+                {"Change master and check that writes work", master_change, rw_ok},
+                {"Reset cluster", reset, {}}
+            }
+        },
+        {
+            "No master on startup",
+            {
+                {"Block master and check that writes fail", block_master, rw_err},
+                {"Unblock master and check that writes still fail", unblock_master, rw_err},
+                {"Change master and check that writes work", master_change, rw_ok},
+                {"Reset cluster", reset, {}}
+            }
+        }
+    });
+
+    // Create a table for testing
+    test.maxscales->connect();
+    test.try_query(test.maxscales->conn_rwsplit[0], "CREATE OR REPLACE TABLE test.t1(id INT)");
+    test.repl->sync_slaves();
+    test.maxscales->disconnect();
+
+    for (auto& i : tests)
+    {
+        test.tprintf("Running test: %s", i.description);
+        test.maxscales->connect();
+
+        for (auto t : i.steps)
+        {
+            cout << t.description << endl;
+            t.func();
+            for (auto q : t.queries)
+            {
+                int rc = execute_query_silent(test.maxscales->conn_rwsplit[0], q.query);
+                test.assert(q.should_work == (rc == 0), "Step '%s': Query '%s' should %s: %s",
+                            i.description, q.query, q.should_work ? "work" : "fail",
+                            mysql_error(test.maxscales->conn_rwsplit[0]));
+            }
+        }
+
+        if (test.global_result)
+        {
+            test.tprintf("Test '%s' failed", i.description);
+            break;
+        }
+    }
+
+    // Wait for the monitoring to stabilize before dropping the table
+    sleep(5);
+
+    test.maxscales->connect();
+    test.try_query(test.maxscales->conn_rwsplit[0], "DROP TABLE test.t1");
+    test.repl->fix_replication();
+    test.maxscales->disconnect();
+
+    return test.global_result;
+}