MXS-359: Test read-only mode and master replacement

Added test cases that verify that the functionality works as
expected. Also made Mariadb_nodes::change_master less verbose when one of
the nodes is down.

maxscale-system-test/CMakeLists.txt

@@ -414,6 +414,9 @@ add_test_executable(mxs314.cpp mxs314 galera LABELS MySQLProtocol LIGHT GALERA_B
 # Testing of the master replacement feature
 add_test_executable(mxs359_master_switch.cpp mxs359_master_switch mxs359_master_switch LABELS REPL_BACKEND)
 
+# Test read-only connections with master replacement
+add_test_executable(mxs359_read_only.cpp mxs359_read_only mxs359_read_only LABELS REPL_BACKEND)
+
 # Binary protocol prepared statement tests
 add_test_executable(binary_ps.cpp binary_ps replication LABELS readwritesplit LIGHT REPL_BACKEND)
 add_test_executable(binary_ps_cursor.cpp binary_ps_cursor replication LABELS readwritesplit LIGHT REPL_BACKEND)

maxscale-system-test/cnf/maxscale.cnf.template.mxs359_read_only

@@ -0,0 +1,60 @@
+[maxscale]
+threads=###threads###
+
+[MySQL-Monitor]
+type=monitor
+module=mysqlmon
+###repl51###
+servers=server1,server2,server3,server4
+user=maxskysql
+passwd=skysql
+monitor_interval=1000
+
+[RW-Split-Router]
+type=service
+router=readwritesplit
+servers=server1,server2,server3,server4
+user=maxskysql
+passwd=skysql
+allow_master_change=true
+master_failure_mode=fail_on_write
+
+[RW-Split-Listener]
+type=listener
+service=RW-Split-Router
+protocol=MySQLClient
+port=4006
+
+[CLI]
+type=service
+router=cli
+
+[CLI-Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=default
+
+[server1]
+type=server
+address=###node_server_IP_1###
+port=###node_server_port_1###
+protocol=MySQLBackend
+
+[server2]
+type=server
+address=###node_server_IP_2###
+port=###node_server_port_2###
+protocol=MySQLBackend
+
+[server3]
+type=server
+address=###node_server_IP_3###
+port=###node_server_port_3###
+protocol=MySQLBackend
+
+[server4]
+type=server
+address=###node_server_IP_4###
+port=###node_server_port_4###
+protocol=MySQLBackend

maxscale-system-test/mariadb_nodes.cpp

@@ -259,13 +259,19 @@ int Mariadb_nodes::change_master(int NewMaster, int OldMaster)
 {
     for (int i = 0; i < N; i++)
     {
-        execute_query(nodes[i], "STOP SLAVE");
+        if (mysql_ping(nodes[i]) == 0)
+        {
+            execute_query(nodes[i], "STOP SLAVE");
+        }
     }
 
     execute_query(nodes[NewMaster], "RESET SLAVE ALL");
     execute_query(nodes[NewMaster], create_repl_user);
-    execute_query(nodes[OldMaster], "RESET MASTER");
 
+    if (mysql_ping(nodes[OldMaster]) == 0)
+    {
+        execute_query(nodes[OldMaster], "RESET MASTER");
+    }
     char log_file[256];
     char log_pos[256];
     find_field(nodes[NewMaster], "show master status", "File", &log_file[0]);
@@ -273,7 +279,7 @@ int Mariadb_nodes::change_master(int NewMaster, int OldMaster)
 
     for (int i = 0; i < N; i++)
     {
-        if (i != NewMaster)
+        if (i != NewMaster && mysql_ping(nodes[i]) == 0)
         {
             char str[1024];
             sprintf(str, setup_slave, IP[NewMaster], log_file, log_pos, port[NewMaster]);

maxscale-system-test/mxs359_read_only.cpp

@@ -0,0 +1,139 @@
+/**
+ * MXS-359: Starting sessions without master
+ *
+ * https://jira.mariadb.org/browse/MXS-359
+ */
+#include "testconnections.h"
+#include <vector>
+#include <iostream>
+#include <sstream>
+
+using std::cout;
+using std::endl;
+
+struct TestCase
+{
+    const char* description;
+    void (*func)(TestConnections&, std::ostream&);
+};
+
+TestConnections* global_test;
+
+void change_master(int next, int current)
+{
+    TestConnections& test = *global_test;
+    test.maxscales->ssh_node_f(0, true, "maxadmin shutdown monitor MySQL-Monitor");
+    test.repl->connect();
+    test.repl->change_master(next, current);
+    test.repl->close_connections();
+    test.maxscales->ssh_node_f(0, true, "maxadmin restart monitor MySQL-Monitor");
+}
+
+void test_replaced_master(TestConnections& test, std::ostream& out)
+{
+    out << "Sanity check that reads and writes work" << endl;
+    test.maxscales->connect();
+    test.try_query(test.maxscales->conn_rwsplit[0], "INSERT INTO test.t1 VALUES (1)");
+    test.try_query(test.maxscales->conn_rwsplit[0], "SELECT * FROM test.t1");
+
+    test.repl->block_node(0);
+    sleep(10);
+
+    out << "Reads should still work even if no master is available" << endl;
+    test.try_query(test.maxscales->conn_rwsplit[0], "SELECT * FROM test.t1");
+
+    test.repl->unblock_node(0);
+    change_master(1, 0);
+    sleep(10);
+
+    out << "Reads and writes after master change should work" << endl;
+    test.try_query(test.maxscales->conn_rwsplit[0], "INSERT INTO test.t1 VALUES (2)");
+    test.try_query(test.maxscales->conn_rwsplit[0], "SELECT * FROM test.t1");
+
+    test.maxscales->disconnect();
+    change_master(0, 1);
+}
+
+void test_new_master(TestConnections& test, std::ostream& out)
+{
+    out << "Block the master before connecting" << endl;
+    test.repl->block_node(0);
+    sleep(10);
+
+    out << "Connect and check that read-only mode works" << endl;
+    test.maxscales->connect();
+    test.try_query(test.maxscales->conn_rwsplit[0], "SELECT * FROM test.t1");
+
+    change_master(1, 0);
+    sleep(10);
+
+    out << "Both reads and writes after master change should work" << endl;
+    test.try_query(test.maxscales->conn_rwsplit[0], "INSERT INTO test.t1 VALUES (2)");
+    test.try_query(test.maxscales->conn_rwsplit[0], "SELECT * FROM test.t1");
+
+    test.repl->unblock_node(0);
+    test.maxscales->disconnect();
+    change_master(0, 1);
+}
+
+void test_master_failure(TestConnections& test, std::ostream& out)
+{
+    out << "Sanity check that reads and writes work" << endl;
+    test.maxscales->connect();
+    test.try_query(test.maxscales->conn_rwsplit[0], "INSERT INTO test.t1 VALUES (1)");
+    test.try_query(test.maxscales->conn_rwsplit[0], "SELECT * FROM test.t1");
+
+    test.repl->block_node(0);
+    sleep(10);
+
+    out << "Reads should still work even if no master is available" << endl;
+    test.try_query(test.maxscales->conn_rwsplit[0], "SELECT * FROM test.t1");
+
+    out << "Writes should fail" << endl;
+    int rc = execute_query_silent(test.maxscales->conn_rwsplit[0], "INSERT INTO test.t1 VALUES (1)");
+    test.assert(rc != 0, "Write after master failure should fail");
+
+    test.repl->unblock_node(0);
+    test.maxscales->disconnect();
+}
+
+int main(int argc, char** argv)
+{
+    TestConnections test(argc, argv);
+    global_test = &test;
+
+    std::vector<TestCase> tests(
+    {
+        {"test_replaced_master", test_replaced_master},
+        {"test_new_master", test_new_master},
+        {"test_master_failure", test_master_failure}
+    });
+
+    // Create a table for testing
+    test.maxscales->connect();
+    test.try_query(test.maxscales->conn_rwsplit[0], "CREATE OR REPLACE TABLE test.t1(id INT)");
+    test.repl->sync_slaves();
+    test.maxscales->disconnect();
+
+    for (auto& i : tests)
+    {
+        std::stringstream out;
+        test.tprintf("Running test: %s", i.description);
+        i.func(test, out);
+        if (test.global_result)
+        {
+            test.tprintf("Test '%s' failed: %s", i.description, out.str().c_str());
+            break;
+        }
+    }
+
+    // Wait for the monitoring to stabilize before dropping the table
+    sleep(5);
+
+    test.maxscales->connect();
+    test.try_query(test.maxscales->conn_rwsplit[0], "DROP TABLE test.t1");
+    test.repl->fix_replication();
+    test.maxscales->disconnect();
+
+    return test.global_result;
+}