MXS-2900 Move MariaDB-Monitor tests

maxscale-system-test/mariadbmonitor/CMakeLists.txt

@@ -1,3 +1,101 @@
+# MySQL Monitor with Multi-master configurations
+add_test_executable_ex(NAME mysqlmon_multimaster SOURCE mysqlmon_multimaster.cpp
+        CONFIG mysqlmon_multimaster.cnf VMS repl_backend LABELS mysqlmon BREAKS_REPL)
+
+# MySQL Monitor with Multi-master configurations (assume_unique_hostnames=OFF)
+add_test_executable_ex(NAME mysqlmon_multimaster_serverid SOURCE mysqlmon_multimaster.cpp
+        CONFIG mysqlmon_multimaster_serverid.cnf VMS repl_backend LABELS mysqlmon BREAKS_REPL)
+
+# MySQL Monitor Failover Test
+add_test_executable_ex(NAME mysqlmon_detect_standalone_master SOURCE mysqlmon_detect_standalone_master.cpp
+        CONFIG mysqlmon_detect_standalone_master.cnf VMS repl_backend LABELS mysqlmon)
+
+# MySQL Monitor Failover (automatic) Test
+add_test_executable_ex(NAME mysqlmon_failover_auto SOURCE mysqlmon_failover_auto.cpp
+        CONFIG mysqlmon_failover_auto.cnf VMS repl_backend LABELS mysqlmon)
+
+# MySQL Monitor Failover (manual) Test
+add_test_executable_ex(NAME mysqlmon_failover_manual SOURCE mysqlmon_failover_manual.cpp
+        CONFIG mysqlmon_failover_manual.cnf VMS repl_backend LABELS mysqlmon)
+
+# MySQL Monitor manual failover with many valid candidates
+add_test_executable_ex(NAME mysqlmon_failover_manual2_4 SOURCE mysqlmon_failover_manual2.cpp
+        CONFIG mysqlmon_failover_manual2_4.cnf VMS repl_backend LABELS mysqlmon)
+add_test_executable_ex(NAME mysqlmon_failover_manual2_3 SOURCE mysqlmon_failover_manual2.cpp
+        CONFIG mysqlmon_failover_manual2_3.cnf VMS repl_backend LABELS mysqlmon)
+add_test_executable_ex(NAME mysqlmon_failover_manual2_2 SOURCE mysqlmon_failover_manual2.cpp
+        CONFIG mysqlmon_failover_manual2_2.cnf VMS repl_backend LABELS mysqlmon)
+
 # MySQL Monitor switchover
 add_test_executable_ex(NAME mysqlmon_switchover SOURCE mysqlmon_switchover.cpp CONFIG mysqlmon_switchover.cnf
-        VMS repl_backend LABELS switchover)
+        VMS repl_backend LABELS mysqlmon switchover)
+
+# MySQL Monitor switchover with bad master
+add_test_executable_ex(NAME mysqlmon_switchover_bad_master SOURCE mysqlmon_switchover_bad_master.cpp
+        CONFIG mysqlmon_switchover_bad_master.cnf VMS repl_backend LABELS mysqlmon switchover)
+
+# MySQL Monitor manual failover with no valid slaves, uses config of mysqlmon_failover_auto. Also MXS-2652.
+add_test_executable_ex(NAME mysqlmon_failover_no_slaves SOURCE mysqlmon_failover_no_slaves.cpp
+        CONFIG mysqlmon_failover_auto.cnf VMS repl_backend LABELS mysqlmon failover)
+
+# MySQL Monitor Rejoin (good) Test
+add_test_executable_ex(NAME mysqlmon_rejoin_good SOURCE mysqlmon_rejoin_good.cpp
+        CONFIG mysqlmon_rejoin_good.cnf VMS repl_backend LABELS mysqlmon)
+
+# MySQL Monitor Rejoin (bad) Test, use template for Rejoin (good)
+add_test_executable_ex(NAME mysqlmon_rejoin_bad SOURCE mysqlmon_rejoin_bad.cpp CONFIG mysqlmon_rejoin_good.cnf
+        VMS repl_backend LABELS mysqlmon)
+
+# MySQL Monitor Rejoin (bad2) Test, use template for Rejoin (good)
+add_test_executable_ex(NAME mysqlmon_rejoin_bad2 SOURCE mysqlmon_rejoin_bad2.cpp CONFIG mysqlmon_rejoin_good.cnf
+        VMS repl_backend LABELS mysqlmon)
+
+# MySQL Monitor Rejoin tests
+add_test_executable_ex(NAME mysqlmon_rejoin_manual SOURCE mysqlmon_rejoin_manual.cpp CONFIG mysqlmon_rejoin_manual.cnf
+        VMS repl_backend LABELS mysqlmon)
+add_test_executable_ex(NAME mysqlmon_rejoin_manual2 SOURCE mysqlmon_rejoin_manual2.cpp CONFIG mysqlmon_rejoin_manual.cnf
+        VMS repl_backend LABELS mysqlmon)
+
+# MySQL Monitor rolling master
+add_test_executable_ex(NAME mysqlmon_failover_rolling_master SOURCE mysqlmon_failover_rolling_master.cpp
+        CONFIG mysqlmon_failover_rolling_master.cnf VMS repl_backend LABELS mysqlmon)
+
+# MySQL Monitor rejoin old slave
+add_test_executable_ex(NAME mysqlmon_failover_rejoin_old_slave SOURCE mysqlmon_failover_rejoin_old_slave.cpp
+        CONFIG mysqlmon_failover_rejoin_old_slave.cnf VMS repl_backend LABELS mysqlmon)
+
+# MySQL Monitor rolling restart slaves
+add_test_executable_ex(NAME mysqlmon_failover_rolling_restart_slaves SOURCE mysqlmon_failover_rolling_restart_slaves.cpp
+        CONFIG mysqlmon_failover_rolling_restart_slaves.cnf VMS repl_backend LABELS mysqlmon)
+
+# MySQL Monitor failover stress
+add_test_executable_ex(NAME mysqlmon_failover_stress SOURCE mysqlmon_failover_stress.cpp
+        CONFIG mysqlmon_failover_stress.cnf VMS repl_backend LABELS mysqlmon)
+
+# MySQL Monitor switchover stress
+add_test_executable_ex(NAME mysqlmon_switchover_stress SOURCE mysqlmon_switchover_stress.cpp
+        CONFIG mysqlmon_switchover_stress.cnf VMS repl_backend LABELS mysqlmon)
+
+# Check monitoring and failover when ignore_external_masters is enabled
+add_test_executable_ex(NAME mysqlmon_external_master SOURCE mysqlmon_external_master.cpp
+        CONFIG mysqlmon_external_master.cnf VMS repl_backend LABELS mysqlmon)
+
+# Check failover, switchover and rejoin with scheduled server events, uses config of mysqlmon_rejoin_good
+add_test_executable_ex(NAME mysqlmon_fail_switch_events SOURCE mysqlmon_fail_switch_events.cpp
+        CONFIG mysqlmon_rejoin_good.cnf VMS repl_backend LABELS mysqlmon)
+
+# MySQL Monitor reset-replication test, use template for Rejoin (good)
+add_test_executable_ex(NAME mysqlmon_reset_replication SOURCE mysqlmon_reset_replication.cpp
+        CONFIG mysqlmon_rejoin_good.cnf VMS repl_backend LABELS mysqlmon)
+
+# MySQL Monitor low disk switchover
+add_test_executable_ex(NAME mysqlmon_switchover_auto SOURCE mysqlmon_switchover_auto.cpp
+        CONFIG mysqlmon_switchover_auto.cnf VMS repl_backend LABELS mysqlmon)
+
+# MySQL Monitor series of failovers and rejoins
+add_test_executable_ex(NAME mysqlmon_failover_readonly SOURCE mysqlmon_failover_readonly.cpp
+        CONFIG mysqlmon_failover_readonly.cnf VMS repl_backend LABELS mysqlmon)
+
+# MXS-1493: Use replication heartbeat in mysqlmon https://jira.mariadb.org/browse/MXS-1493
+add_test_executable_ex(NAME verify_master_failure SOURCE verify_master_failure.cpp
+        CONFIG verify_master_failure.cnf VMS repl_backend LABELS mysqlmon)

maxscale-system-test/mariadbmonitor/mysqlmon_detect_standalone_master.cnf

@@ -0,0 +1,91 @@
+[maxscale]
+threads=###threads###
+
+[MySQL Monitor]
+type=monitor
+module=mysqlmon
+servers= server1, server2, server3, server4
+user=maxskysql
+password= skysql
+monitor_interval=1000
+detect_standalone_master=true
+failcount=1
+allow_cluster_recovery=false
+backend_connect_timeout=10
+backend_read_timeout=10
+backend_write_timeout=10
+
+[RW Split Router]
+type=service
+router= readwritesplit
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[Read Connection Router Slave]
+type=service
+router=readconnroute
+router_options= slave
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[Read Connection Router Master]
+type=service
+router=readconnroute
+router_options=master
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[RW Split Listener]
+type=listener
+service=RW Split Router
+protocol=MySQLClient
+port=4006
+
+[Read Connection Listener Slave]
+type=listener
+service=Read Connection Router Slave
+protocol=MySQLClient
+port=4009
+
+[Read Connection Listener Master]
+type=listener
+service=Read Connection Router Master
+protocol=MySQLClient
+port=4008
+
+[CLI]
+type=service
+router=cli
+
+[CLI Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=default
+
+[server1]
+type=server
+address=###node_server_IP_1###
+port=###node_server_port_1###
+protocol=MySQLBackend
+
+[server2]
+type=server
+address=###node_server_IP_2###
+port=###node_server_port_2###
+protocol=MySQLBackend
+
+[server3]
+type=server
+address=###node_server_IP_3###
+port=###node_server_port_3###
+protocol=MySQLBackend
+
+[server4]
+type=server
+address=###node_server_IP_4###
+port=###node_server_port_4###
+protocol=MySQLBackend

maxscale-system-test/mariadbmonitor/mysqlmon_detect_standalone_master.cpp

@@ -0,0 +1,167 @@
+/**
+ * @file mysqlmon_detect_standalone_master.cpp MySQL Monitor Standalone Master Test
+ * - block all nodes, but one
+ * - wait for monitor (monitor_interval), monitor should select remaining node as master
+ * - check maxadmin output
+ * - check that queries work
+ * - unblock backend nodes
+ * - wait for monitor
+ * - check that monitor is still using the same node and that the old nodes are in maintenance mode
+ */
+
+#include <iostream>
+#include <sstream>
+#include "testconnections.h"
+#include "fail_switch_rejoin_common.cpp"
+
+using std::stringstream;
+using std::cout;
+using std::endl;
+
+void check_maxscale(TestConnections& test)
+{
+    test.tprintf("Connecting to Maxscale\n");
+    test.add_result(test.maxscales->connect_maxscale(0), "Can not connect to Maxscale\n");
+    test.tprintf("Trying simple query against all sevices\n");
+    test.tprintf("RWSplit \n");
+    test.try_query(test.maxscales->conn_rwsplit[0], (char*) "show databases;");
+    test.tprintf("ReadConn Master \n");
+    test.try_query(test.maxscales->conn_master[0], (char*) "show databases;");
+}
+
+void replicate_from(TestConnections& test, int server_ind, int target_ind)
+{
+    stringstream change_master;
+    change_master << "CHANGE MASTER TO MASTER_HOST = '" << test.repl->IP_private[target_ind]
+                  << "', MASTER_PORT = " << test.repl->port[target_ind] << ", MASTER_USE_GTID = current_pos, "
+                                                             "MASTER_USER='repl', MASTER_PASSWORD='repl';";
+    cout << "Server " << server_ind + 1 << " starting to replicate from server " << target_ind + 1 << endl;
+    if (test.verbose)
+    {
+        cout << "Query is '" << change_master.str() << "'" << endl;
+    }
+    test.try_query(test.repl->nodes[server_ind], "STOP SLAVE;");
+    test.try_query(test.repl->nodes[server_ind], "%s", change_master.str().c_str());
+    test.try_query(test.repl->nodes[server_ind], "START SLAVE;");
+}
+
+void restore_servers(TestConnections& test, bool events_added)
+{
+    test.repl->unblock_node(0);
+    test.repl->unblock_node(1);
+    test.repl->unblock_node(2);
+    int dummy;
+    char* o1 = test.maxscales->ssh_node_output(0, "maxadmin clear server server1 Maint", true, &dummy);
+    char* o2 = test.maxscales->ssh_node_output(0, "maxadmin clear server server2 Maint", true, &dummy);
+    char* o3 = test.maxscales->ssh_node_output(0, "maxadmin clear server server3 Maint", true, &dummy);
+    free(o1);
+    free(o2);
+    free(o3);
+    if (events_added)
+    {
+        // Events have been added to server4, so it must be the real new master. Then switchover to server1.
+        replicate_from(test, 0, 3);
+        replicate_from(test, 1, 3);
+        replicate_from(test, 2, 3);
+        test.maxscales->wait_for_monitor();
+        o1 = test.maxscales->ssh_node_output(0,
+                                             "maxadmin call command mariadbmon switchover MySQL-Monitor server1 server4",
+                                             true,
+                                             &dummy);
+        test.maxscales->wait_for_monitor();
+        int master_id = get_master_server_id(test);
+        test.expect(master_id == 1, "Switchover failed to set server1 as master.");
+    }
+    else
+    {
+        // No events added, it should be enough to start slave on server4
+        replicate_from(test, 3, 0);
+    }
+}
+
+int main(int argc, char* argv[])
+{
+    Mariadb_nodes::require_gtid(true);
+    TestConnections test(argc, argv);
+    test.maxscales->connect_maxscale(0);
+    test.repl->connect();
+    delete_slave_binlogs(test);
+    print_gtids(test);
+    test.tprintf(" Create the test table and insert some data ");
+    test.try_query(test.maxscales->conn_rwsplit[0], "CREATE OR REPLACE TABLE test.t1 (id int)");
+    test.try_query(test.maxscales->conn_rwsplit[0], "INSERT INTO test.t1 VALUES (1)");
+    test.repl->sync_slaves();
+
+    print_gtids(test);
+
+    test.maxscales->close_maxscale_connections(0);
+    if (test.global_result != 0)
+    {
+        return test.global_result;
+    }
+
+    test.tprintf(" Block all but one node, stop slave on server 4 ");
+    test.repl->block_node(0);
+    test.repl->block_node(1);
+    test.repl->block_node(2);
+
+    test.try_query(test.repl->nodes[3], "STOP SLAVE;RESET SLAVE ALL;");
+
+    test.tprintf(" Wait for the monitor to detect it ");
+    test.maxscales->wait_for_monitor(3);
+
+    test.tprintf(" Connect and insert should work ");
+    get_output(test);
+
+    int master_id = get_master_server_id(test);
+    test.expect(master_id == 4, "Server 4 should be master, but master is server %d.", master_id);
+
+    if (test.global_result != 0)
+    {
+        restore_servers(test, false);
+        return test.global_result;
+    }
+
+    test.maxscales->connect_maxscale(0);
+    test.try_query(test.maxscales->conn_rwsplit[0], "INSERT INTO test.t1 VALUES (1)");
+    test.maxscales->close_maxscale_connections(0);
+    test.repl->connect(3);
+    char result_tmp[bufsize];
+    if (find_field(test.repl->nodes[3], GTID_QUERY, GTID_FIELD, result_tmp) == 0)
+    {
+        test.tprintf("Node 3 gtid: %s", result_tmp);
+    }
+
+    test.tprintf("Unblock nodes ");
+    test.repl->unblock_node(0);
+    test.repl->unblock_node(1);
+    test.repl->unblock_node(2);
+
+    test.tprintf(" Wait for the monitor to detect it ");
+    test.maxscales->wait_for_monitor();
+
+    test.tprintf("Check that we are still using the last node to which we failed over "
+                 "to and that the old nodes are in maintenance mode");
+
+    test.maxscales->connect_maxscale(0);
+    get_output(test);
+
+    test.try_query(test.maxscales->conn_rwsplit[0], "INSERT INTO test.t1 VALUES (1)");
+    master_id = get_master_server_id(test);
+    test.tprintf("Master server id is %d", master_id);
+
+    test.repl->connect();
+    int real_id = test.repl->get_server_id(3);
+    test.expect(master_id == real_id, "@@server_id is different: %d != %d", master_id, real_id);
+    print_gtids(test);
+    test.maxscales->close_maxscale_connections(0);
+
+    test.tprintf("Check that MaxScale is running");
+    check_maxscale(test);
+    if (test.global_result == 0)
+    {
+        cout << "Test successful, restoring original state." << endl;
+        restore_servers(test, true);
+    }
+    return test.global_result;
+}

maxscale-system-test/mariadbmonitor/mysqlmon_external_master.cnf

@@ -0,0 +1,62 @@
+[maxscale]
+threads=###threads###
+
+[MySQL-Monitor]
+type=monitor
+module=mysqlmon
+servers=server1,server2,server3
+user=maxskysql
+password=skysql
+monitor_interval=1000
+detect_standalone_master=true
+failcount=1
+allow_cluster_recovery=true
+auto_failover=true
+auto_rejoin=true
+replication_user=repl
+replication_password=repl
+ignore_external_masters=true
+backend_connect_timeout=10
+backend_read_timeout=10
+backend_write_timeout=10
+
+[RW Split Router]
+type=service
+router=readwritesplit
+servers=server1,server2,server3
+user=maxskysql
+password=skysql
+
+[RW Split Listener]
+type=listener
+service=RW Split Router
+protocol=MySQLClient
+port=4006
+
+[CLI]
+type=service
+router=cli
+
+[CLI Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=default
+
+[server1]
+type=server
+address=###node_server_IP_1###
+port=###node_server_port_1###
+protocol=MySQLBackend
+
+[server2]
+type=server
+address=###node_server_IP_2###
+port=###node_server_port_2###
+protocol=MySQLBackend
+
+[server3]
+type=server
+address=###node_server_IP_3###
+port=###node_server_port_3###
+protocol=MySQLBackend

maxscale-system-test/mariadbmonitor/mysqlmon_external_master.cpp

@@ -0,0 +1,121 @@
+/**
+ * Test monitoring and failover with ignore_external_masters=true
+ */
+#include "testconnections.h"
+#include "fail_switch_rejoin_common.cpp"
+#include <atomic>
+#include <thread>
+
+const char DOWN[] = "Down";
+const char RUNNING[] = "Running";
+const char MASTER[] = "Master";
+const char SLAVE[] = "Slave";
+
+const StringSet master_running = {MASTER, RUNNING};
+const StringSet slave_running = {SLAVE, RUNNING};
+const StringSet running = {RUNNING};
+const StringSet down = {DOWN};
+
+static std::atomic<bool> is_running(true);
+
+void check_status(TestConnections& test, const char* server, const StringSet& expected, const char* message)
+{
+    StringSet state = test.get_server_status(server);
+    test.expect(state == expected, "%s: %s", message, dump_status(state, expected).c_str());
+}
+
+void writer_func(TestConnections* test)
+{
+    while (is_running)
+    {
+        MYSQL* conn = open_conn(test->maxscales->rwsplit_port[0], test->maxscales->IP[0],
+                                "test", "test", false);
+
+        for (int i = 0; i < 100; i++)
+        {
+            if (execute_query_silent(conn, "INSERT INTO test.t1 VALUES (SELECT SLEEP(0.5))"))
+            {
+                sleep(1);
+                break;
+            }
+        }
+        mysql_close(conn);
+    }
+}
+
+int main(int argc, char** argv)
+{
+    Mariadb_nodes::require_gtid(true);
+    TestConnections test(argc, argv);
+    test.repl->connect();
+    delete_slave_binlogs(test);
+
+    // Create a table and a user and start a thread that does writes
+    MYSQL* node0 = test.repl->nodes[0];
+    execute_query(node0, "CREATE OR REPLACE TABLE test.t1 (id INT)");
+    execute_query(node0, "DROP USER IF EXISTS 'test'@'%%'");
+    execute_query(node0, "CREATE USER 'test'@'%%' IDENTIFIED BY 'test'");
+    execute_query(node0, "GRANT INSERT, SELECT, UPDATE, DELETE ON *.* TO 'test'@'%%'");
+    test.repl->sync_slaves();
+    std::thread thr(writer_func, &test);
+
+    test.tprintf("Start by having the current master replicate from the external server.");
+    test.repl->replicate_from(0, 3);
+    test.maxscales->wait_for_monitor(1);
+    check_status(test, "server1", master_running, "server1 should be the master");
+    check_status(test, "server2", slave_running, "server2 should be a slave");
+    check_status(test, "server3", slave_running, "server3 should be a slave");
+
+    test.tprintf("Stop server1, expect server2 to be promoted as the master");
+    test.repl->stop_node(0);
+    test.maxscales->wait_for_monitor(2);
+
+    check_status(test, "server1", down, "server1 should be down");
+    check_status(test, "server2", master_running, "server2 should be the master");
+    check_status(test, "server3", slave_running, "server3 should be a slave");
+
+    test.tprintf("Configure master-master replication between server2 and the external server");
+    // Comment away next line since failover already created the external connection. Failover/switchover
+    // does not respect 'ignore_external_master' when copying slave connections. Whether it should do it
+    // is questionable.
+    // TODO: Think about what to do with this test and the setting in general.
+    //    test.repl->replicate_from(1, 3);
+    test.repl->replicate_from(3, 1);
+    test.maxscales->wait_for_monitor(1);
+    check_status(test, "server2", master_running, "server2 should still be the master");
+    check_status(test, "server3", slave_running, "server3 should be a slave");
+
+    test.tprintf("Start server1, expect it to rejoin the cluster");
+    // The rejoin should redirect the existing external master connection in server1.
+    test.repl->start_node(0);
+    test.maxscales->wait_for_monitor(2);
+    check_status(test, "server1", slave_running, "server1 should be a slave");
+    check_status(test, "server2", master_running, "server2 should still be the master");
+    check_status(test, "server3", slave_running, "server3 should be a slave");
+
+    test.tprintf("Stop server2, expect server1 to be promoted as the master");
+    test.repl->stop_node(1);
+    test.maxscales->wait_for_monitor(2);
+    test.repl->connect();
+    // Same as before.
+    // test.repl->replicate_from(0, 3);
+    test.repl->replicate_from(3, 0);
+
+    check_status(test, "server1", master_running, "server1 should be the master");
+    check_status(test, "server2", down, "server2 should be down");
+    check_status(test, "server3", slave_running, "server3 should be a slave");
+
+    test.tprintf("Start server2, expect it to rejoin the cluster");
+    test.repl->start_node(1);
+    test.maxscales->wait_for_monitor(2);
+    check_status(test, "server1", master_running, "server1 should still be the master");
+    check_status(test, "server2", slave_running, "server2 should be a slave");
+    check_status(test, "server3", slave_running, "server3 should be a slave");
+
+    // Cleanup
+    is_running = false;
+    thr.join();
+    execute_query(test.repl->nodes[0], "STOP SLAVE; RESET SLAVE ALL;");
+
+    return test.global_result;
+}

maxscale-system-test/mariadbmonitor/mysqlmon_fail_switch_events.cpp

@@ -0,0 +1,324 @@
+/*
+ * Copyright (c) 2018 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2024-03-10
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#include "testconnections.h"
+#include "failover_common.cpp"
+#include <string>
+
+using std::string;
+
+const char EVENT_NAME[] = "test_event";
+const char EVENT_SHCEDULER[] = "SET GLOBAL event_scheduler = %s;";
+const char USE_TEST[] = "USE test;";
+const char DELETE_EVENT[] = "DROP EVENT %s;";
+
+const char EV_STATE_ENABLED[] = "ENABLED";
+const char EV_STATE_DISABLED[] = "DISABLED";
+const char EV_STATE_SLAVE_DISABLED[] = "SLAVESIDE_DISABLED";
+
+const char WRONG_MASTER_FMT[] = "%s is not master as expected. Current master id: %i.";
+
+int read_incremented_field(TestConnections& test)
+{
+    int rval = -1;
+    MYSQL* conn = test.maxscales->open_rwsplit_connection();
+    char output[100];
+    if (find_field(conn, "SELECT * FROM test.t1;", "c1", output) == 0)
+    {
+        char* endptr = NULL;
+        auto colvalue = strtol(output, &endptr, 0);
+        if (endptr && *endptr == '\0')
+        {
+            rval = colvalue;
+        }
+        else
+        {
+            test.expect(false, "Could not read value from query result '%s'.", output);
+        }
+    }
+    else
+    {
+        test.expect(false, "Could not perform query: %s.", mysql_error(conn));
+    }
+    return rval;
+}
+
+bool field_is_incrementing(TestConnections& test)
+{
+    int old_val = read_incremented_field(test);
+    sleep(2);   // Should be enough to allow the event to run once.
+    // Check that the event is running and increasing the value
+    int new_val = read_incremented_field(test);
+    return new_val > old_val;
+}
+
+void create_event(TestConnections& test)
+{
+    // Create table, enable scheduler and add an event
+    test.tprintf("Creating table, inserting data and scheduling an event.");
+    test.maxscales->connect_maxscale(0);
+    MYSQL* conn = test.maxscales->conn_rwsplit[0];
+    const char create_event_query[] = "CREATE EVENT %s ON SCHEDULE EVERY 1 SECOND "
+                                      "DO UPDATE test.t1 SET c1 = c1 + 1;";
+
+    if ((test.try_query(conn, EVENT_SHCEDULER, "ON") == 0)
+        && (test.try_query(conn, "CREATE OR REPLACE TABLE test.t1(c1 INT);") == 0)
+        && (test.try_query(conn, USE_TEST) == 0)
+        && (test.try_query(conn, "INSERT INTO t1 VALUES (1);") == 0)
+        && (test.try_query(conn, create_event_query, EVENT_NAME) == 0))
+    {
+        test.repl->sync_slaves();
+        // Check that the event is running and increasing the value
+        test.expect(field_is_incrementing(test),
+                    "Value in column did not increment. Current value %i.", read_incremented_field(test));
+    }
+    print_gtids(test);
+}
+
+void delete_event(TestConnections& test)
+{
+    test.maxscales->connect_maxscale(0);
+    MYSQL* conn = test.maxscales->conn_rwsplit[0];
+
+    if ((test.try_query(conn, EVENT_SHCEDULER, "OFF") == 0)
+        && (test.try_query(conn, USE_TEST) == 0)
+        && (test.try_query(conn, DELETE_EVENT, EVENT_NAME) == 0))
+    {
+        test.repl->sync_slaves();
+        test.expect(!field_is_incrementing(test),
+                    "Value in column was incremented when it should not be. Current value %i.",
+                    read_incremented_field(test));
+    }
+}
+
+void try_delete_event(TestConnections& test)
+{
+    test.maxscales->connect_maxscale(0);
+    MYSQL* conn = test.maxscales->conn_rwsplit[0];
+
+    execute_query(conn, EVENT_SHCEDULER, "OFF");
+    execute_query(conn, USE_TEST);
+    execute_query(conn, DELETE_EVENT, EVENT_NAME);
+    test.repl->sync_slaves();
+}
+
+string string_set_to_string(const StringSet& set)
+{
+    string rval;
+    for (auto elem : set)
+    {
+        rval += elem + ", ";
+    }
+    return rval;
+}
+
+bool check_event_status(TestConnections& test, int node,
+                        const string& event_name, const string& expected_state)
+{
+    bool rval = false;
+    test.repl->connect();
+    string query = "SELECT * FROM information_schema.EVENTS WHERE EVENT_NAME = '" + event_name + "';";
+    char status[100];
+    if (find_field(test.repl->nodes[node], query.c_str(), "STATUS", status) != 0)
+    {
+        test.expect(false, "Could not query event status: %s", mysql_error(test.repl->nodes[0]));
+    }
+    else
+    {
+        if (expected_state != status)
+        {
+            test.expect(false, "Wrong event status, found %s when %s was expected.",
+                        status, expected_state.c_str());
+        }
+        else
+        {
+            rval = true;
+            cout << "Event '" << event_name << "' is '" << status << "' on node " << node <<
+                    " as it should.\n";
+        }
+    }
+    return rval;
+}
+
+void set_event_state(TestConnections& test, const string& event_name, const string& new_state)
+{
+    bool success = false;
+    test.maxscales->connect_maxscale(0);
+    MYSQL* conn = test.maxscales->conn_rwsplit[0];
+    const char query_fmt[] = "ALTER EVENT %s %s;";
+
+    if ((test.try_query(conn, USE_TEST) == 0)
+        && (test.try_query(conn, query_fmt, event_name.c_str(), new_state.c_str()) == 0))
+    {
+        success = true;
+    }
+    test.expect(success, "ALTER EVENT failed: %s", mysql_error(conn));
+    if (success)
+    {
+        cout << "Event '" << event_name << "' set to '" << new_state << "'.\n";
+    }
+}
+
+void switchover(TestConnections& test, const string& new_master)
+{
+    string switch_cmd = "call command mysqlmon switchover MySQL-Monitor " + new_master;
+    test.maxscales->execute_maxadmin_command_print(0, switch_cmd.c_str());
+    test.maxscales->wait_for_monitor(2);
+    // Check success.
+    auto new_master_status = test.get_server_status(new_master.c_str());
+    auto new_master_id = test.get_master_server_id();
+    string status_string;
+    for (auto elem : new_master_status)
+    {
+        status_string += elem + ", ";
+    }
+
+    bool success = (new_master_status.count("Master") == 1);
+    test.expect(success,
+                "%s is not master as expected. Status: %s. Current master id: %i",
+                new_master.c_str(), status_string.c_str(), new_master_id);
+
+    if (success)
+    {
+        cout << "Switchover success, " + new_master + " is new master.\n";
+    }
+}
+
+int main(int argc, char** argv)
+{
+    Mariadb_nodes::require_gtid(true);
+    TestConnections test(argc, argv);
+    test.repl->connect();
+    delete_slave_binlogs(test);
+
+    try_delete_event(test);
+    // Schedule a repeating event.
+    create_event(test);
+
+    int server1_ind = 0;
+    int server2_ind = 1;
+    int server1_id = test.repl->get_server_id(server1_ind);
+
+    const char* server_names[] = {"server1", "server2", "server3", "server4"};
+    auto server1_name = server_names[server1_ind];
+    auto server2_name = server_names[server2_ind];
+
+    int master_id_begin = test.get_master_server_id();
+
+    test.expect(master_id_begin == server1_id, WRONG_MASTER_FMT, server1_name, master_id_begin);
+
+    // If initialisation failed, fail the test immediately.
+    if (test.global_result != 0)
+    {
+        try_delete_event(test);
+        return test.global_result;
+    }
+
+    // Part 1: Do a failover
+    cout << "\nStep 1: Stop master and wait for failover. Check that another server is promoted.\n";
+    test.repl->stop_node(server1_ind);
+    test.maxscales->wait_for_monitor(3);
+    get_output(test);
+    int master_id_failover = test.get_master_server_id();
+    cout << "Master server id is " << master_id_failover << ".\n";
+    test.expect(master_id_failover > 0 && master_id_failover != master_id_begin,
+                "Master did not change or no master detected.");
+    // Check that events are still running.
+    test.expect(field_is_incrementing(test),
+                "Value in column did not increment. Current value %i.",
+                read_incremented_field(test));
+    // Again, stop on failure.
+    if (test.global_result != 0)
+    {
+        try_delete_event(test);
+        return test.global_result;
+    }
+
+    // Part 2: Start node 0, let it join the cluster and check that the event is properly disabled.
+    cout << "\nStep 2: Restart " << server1_name << ". It should join the cluster.\n";
+    test.repl->start_node(server1_ind);
+    test.maxscales->wait_for_monitor(4);
+    get_output(test);
+
+    auto states = test.get_server_status(server1_name);
+    if (states.count("Slave") < 1)
+    {
+        test.expect(false, "%s is not a slave as expected. Status: %s",
+                    server1_name, string_set_to_string(states).c_str());
+    }
+    else
+    {
+        // Old master joined as slave, check that event is disabled.
+        check_event_status(test, server1_ind, EVENT_NAME, EV_STATE_SLAVE_DISABLED);
+    }
+
+    if (test.global_result != 0)
+    {
+        try_delete_event(test);
+        return test.global_result;
+    }
+
+    // Part 3: Switchover back to server1 as master. The event will most likely not run because the old
+    // master doesn't have event scheduler on anymore.
+    cout << "\nStep 3: Switchover back to " << server1_name << ". Check that event is enabled. "
+            "Don't check that the event is running since the scheduler process is likely off.\n";
+    switchover(test, server1_name);
+    if (test.ok())
+    {
+        check_event_status(test, server1_ind, EVENT_NAME, EV_STATE_ENABLED);
+    }
+
+    // Part 4: Disable the event on master. The event should still be "SLAVESIDE_DISABLED" on slaves.
+    // Check that after switchover, the event is not enabled.
+    cout << "\nStep 4: Disable event on master, switchover to " << server2_name << ". "
+            "Check that event is still disabled.\n";
+    if (test.ok())
+    {
+        set_event_state(test, EVENT_NAME, "DISABLE");
+        test.maxscales->wait_for_monitor(); // Wait for the monitor to detect the change.
+        check_event_status(test, server1_ind, EVENT_NAME, EV_STATE_DISABLED);
+        check_event_status(test, server2_ind, EVENT_NAME, EV_STATE_SLAVE_DISABLED);
+
+        if (test.ok())
+        {
+            cout << "Event is disabled on master and slaveside-disabled on slave.\n";
+            switchover(test, server2_name);
+            if (test.ok())
+            {
+                // Event should not have been touched.
+                check_event_status(test, server2_ind, EVENT_NAME, EV_STATE_SLAVE_DISABLED);
+            }
+
+            // Switchover back.
+            switchover(test, server1_name);
+        }
+    }
+
+    if (test.ok())
+    {
+           // Check that all other nodes are slaves.
+        for (int i = 1; i < test.repl->N; i++)
+        {
+            string server_name = server_names[i];
+            auto states = test.maxscales->get_server_status(server_name.c_str());
+            test.expect(states.count("Slave") == 1, "%s is not a slave.", server_name.c_str());
+        }
+    }
+
+    try_delete_event(test);
+    if (test.global_result != 0)
+    {
+        test.repl->fix_replication();
+    }
+    return test.global_result;
+}

maxscale-system-test/mariadbmonitor/mysqlmon_failover_auto.cnf

@@ -0,0 +1,94 @@
+[maxscale]
+threads=###threads###
+
+[MySQL-Monitor]
+type=monitor
+module=mysqlmon
+servers= server1, server2, server3, server4
+user=maxskysql
+password= skysql
+monitor_interval=1000
+detect_standalone_master=true
+failcount=1
+allow_cluster_recovery=true
+auto_failover=true
+replication_user=repl
+replication_password=repl
+backend_connect_timeout=10
+backend_read_timeout=10
+backend_write_timeout=10
+
+[RW Split Router]
+type=service
+router= readwritesplit
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[Read Connection Router Slave]
+type=service
+router=readconnroute
+router_options= slave
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[Read Connection Router Master]
+type=service
+router=readconnroute
+router_options=master
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[RW Split Listener]
+type=listener
+service=RW Split Router
+protocol=MySQLClient
+port=4006
+
+[Read Connection Listener Slave]
+type=listener
+service=Read Connection Router Slave
+protocol=MySQLClient
+port=4009
+
+[Read Connection Listener Master]
+type=listener
+service=Read Connection Router Master
+protocol=MySQLClient
+port=4008
+
+[CLI]
+type=service
+router=cli
+
+[CLI Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=default
+
+[server1]
+type=server
+address=###node_server_IP_1###
+port=###node_server_port_1###
+protocol=MySQLBackend
+
+[server2]
+type=server
+address=###node_server_IP_2###
+port=###node_server_port_2###
+protocol=MySQLBackend
+
+[server3]
+type=server
+address=###node_server_IP_3###
+port=###node_server_port_3###
+protocol=MySQLBackend
+
+[server4]
+type=server
+address=###node_server_IP_4###
+port=###node_server_port_4###
+protocol=MySQLBackend

maxscale-system-test/mariadbmonitor/mysqlmon_failover_auto.cpp

@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2016 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2024-03-10
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#include "mariadb_nodes.h"
+#include "failover_common.cpp"
+
+int main(int argc, char** argv)
+{
+    Mariadb_nodes::require_gtid(true);
+    TestConnections test(argc, argv);
+    test.repl->connect();
+    delete_slave_binlogs(test);
+
+    test.maxscales->wait_for_monitor(2);
+    basic_test(test);
+    print_gtids(test);
+
+    // Part 1
+    int node0_id = prepare_test_1(test);
+    test.maxscales->wait_for_monitor(2);
+    check_test_1(test, node0_id);
+
+    if (test.global_result != 0)
+    {
+        return test.global_result;
+    }
+
+    // Part 2
+    prepare_test_2(test);
+    test.maxscales->wait_for_monitor(2);
+    check_test_2(test);
+
+    if (test.global_result != 0)
+    {
+        return test.global_result;
+    }
+
+    // Part 3
+    prepare_test_3(test);
+    test.maxscales->wait_for_monitor(2);
+    check_test_3(test);
+
+    return test.global_result;
+}

maxscale-system-test/mariadbmonitor/mysqlmon_failover_manual.cnf

@@ -0,0 +1,93 @@
+[maxscale]
+threads=###threads###
+
+[MySQL-Monitor]
+type=monitor
+module=mysqlmon
+servers= server1, server2, server3, server4
+user=maxskysql
+password= skysql
+monitor_interval=1000
+detect_standalone_master=true
+failcount=1
+allow_cluster_recovery=true
+replication_user=repl
+replication_password=repl
+backend_connect_timeout=10
+backend_read_timeout=10
+backend_write_timeout=10
+
+[RW Split Router]
+type=service
+router= readwritesplit
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[Read Connection Router Slave]
+type=service
+router=readconnroute
+router_options= slave
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[Read Connection Router Master]
+type=service
+router=readconnroute
+router_options=master
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[RW Split Listener]
+type=listener
+service=RW Split Router
+protocol=MySQLClient
+port=4006
+
+[Read Connection Listener Slave]
+type=listener
+service=Read Connection Router Slave
+protocol=MySQLClient
+port=4009
+
+[Read Connection Listener Master]
+type=listener
+service=Read Connection Router Master
+protocol=MySQLClient
+port=4008
+
+[CLI]
+type=service
+router=cli
+
+[CLI Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=default
+
+[server1]
+type=server
+address=###node_server_IP_1###
+port=###node_server_port_1###
+protocol=MySQLBackend
+
+[server2]
+type=server
+address=###node_server_IP_2###
+port=###node_server_port_2###
+protocol=MySQLBackend
+
+[server3]
+type=server
+address=###node_server_IP_3###
+port=###node_server_port_3###
+protocol=MySQLBackend
+
+[server4]
+type=server
+address=###node_server_IP_4###
+port=###node_server_port_4###
+protocol=MySQLBackend

maxscale-system-test/mariadbmonitor/mysqlmon_failover_manual.cpp

@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2016 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2024-03-10
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#include "testconnections.h"
+#include "failover_common.cpp"
+
+int main(int argc, char** argv)
+{
+    const char FAILOVER_CMD[] = "maxadmin call command mysqlmon failover MySQL-Monitor";
+    // interactive = strcmp(argv[argc - 1], "interactive") == 0;
+    Mariadb_nodes::require_gtid(true);
+    TestConnections test(argc, argv);
+    test.repl->connect();
+    delete_slave_binlogs(test);
+
+    basic_test(test);
+    print_gtids(test);
+
+    int node0_id = -1;
+    int ec = -1;
+
+    // Part 1
+    node0_id = prepare_test_1(test);
+
+    test.maxscales->ssh_node_output(0, FAILOVER_CMD, true, &ec);
+    test.maxscales->wait_for_monitor();
+
+    check_test_1(test, node0_id);
+    if (test.global_result != 0)
+    {
+        return test.global_result;
+    }
+
+    // Part 2
+    prepare_test_2(test);
+
+    test.maxscales->ssh_node_output(0, FAILOVER_CMD, true, &ec);
+    test.maxscales->wait_for_monitor();
+
+    check_test_2(test);
+    if (test.global_result != 0)
+    {
+        return test.global_result;
+    }
+
+    // Part 3
+    prepare_test_3(test);
+
+    test.maxscales->ssh_node_output(0, FAILOVER_CMD, true, &ec);
+    test.maxscales->wait_for_monitor();
+
+    check_test_3(test);
+    return test.global_result;
+}

maxscale-system-test/mariadbmonitor/mysqlmon_failover_manual2.cpp

@@ -0,0 +1,176 @@
+/*
+ * Copyright (c) 2016 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2024-03-10
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#include "testconnections.h"
+#include <exception>
+#include <iostream>
+#include <sstream>
+#include <string>
+
+using std::cerr;
+using std::cout;
+using std::flush;
+using std::endl;
+using std::string;
+using std::stringstream;
+
+namespace
+{
+
+namespace x
+{
+
+void connect_maxscale(TestConnections& test)
+{
+    if (test.maxscales->connect_maxscale(0) != 0)
+    {
+        ++test.global_result;
+        throw std::runtime_error("Could not connect to MaxScale.");
+    }
+}
+
+void try_query(TestConnections& test, const char* zQuery)
+{
+    if (test.try_query(test.maxscales->conn_rwsplit[0], "%s", zQuery) != 0)
+    {
+        string s("Could not execute query: ");
+        s += zQuery;
+
+        throw std::runtime_error(s);
+    }
+}
+
+void try_query(TestConnections& test, const std::string& query)
+{
+    try_query(test, query.c_str());
+}
+
+void stop_node(Mariadb_nodes& nodes, int node)
+{
+    if (nodes.stop_node(node) != 0)
+    {
+        throw std::runtime_error("Could not stop node.");
+    }
+}
+
+void fail_query(TestConnections& test)
+{
+    int rv = execute_query(test.maxscales->conn_rwsplit[0], "BEGIN");
+
+    if (rv == 0)
+    {
+        const char MESSAGE[] = "A query that was expected to fail, did not fail.";
+
+        test.add_result(false, "%s", MESSAGE);
+        throw std::runtime_error(MESSAGE);
+    }
+}
+}
+}
+
+namespace
+{
+
+void list_servers(TestConnections& test)
+{
+    test.maxscales->execute_maxadmin_command_print(0, (char*)"list servers");
+}
+
+void create_table(TestConnections& test)
+{
+    x::try_query(test, "DROP TABLE IF EXISTS test.t1");
+    x::try_query(test, "CREATE TABLE test.t1(id INT)");
+}
+
+void insert_data(TestConnections& test)
+{
+    x::try_query(test, "BEGIN");
+    for (int i = 0; i < 20; ++i)
+    {
+        stringstream ss;
+        ss << "INSERT INTO test.t1 VALUES (" << i << ")";
+        x::try_query(test, ss.str());
+    }
+    x::try_query(test, "COMMIT");
+}
+
+void run(TestConnections& test)
+{
+    cout << "\nConnecting to MaxScale." << endl;
+    x::connect_maxscale(test);
+
+    cout << "\nCreating table." << endl;
+    create_table(test);
+
+    cout << "\nInserting data." << endl;
+    insert_data(test);
+
+    list_servers(test);
+
+    cout << "\nSyncing slaves." << endl;
+    test.repl->sync_slaves();
+
+    cout << "\nStopping master." << endl;
+    x::stop_node(*test.repl, 0);
+
+    list_servers(test);
+
+    cout << "\nShould fail as master is no longer available, but trying to execute a query... " << endl;
+    x::fail_query(test);
+    cout << "Failed as expected." << endl;
+
+    list_servers(test);
+
+    cout << "\nPerforming failover... " << endl;
+    test.maxscales->wait_for_monitor();
+    test.maxscales->execute_maxadmin_command_print(0, (char*)"call command mysqlmon failover MySQL-Monitor");
+
+    list_servers(test);
+
+    cout << "\nShould still fail as there is not transparent master failover, "
+         << "but trying to execute a query... " << endl;
+    x::fail_query(test);
+    cout << "Failed as expected." << endl;
+
+    cout << "\nClosing connection to MaxScale." << endl;
+    test.maxscales->close_maxscale_connections(0);
+    test.maxscales->wait_for_monitor();
+
+    cout << "\nConnecting to MaxScale." << endl;
+    x::connect_maxscale(test);
+
+    list_servers(test);
+
+    cout << "Trying to insert data... " << flush;
+    insert_data(test);
+    cout << "succeeded." << endl;
+    x::try_query(test, "DROP TABLE test.t1");
+}
+}
+
+int main(int argc, char** argv)
+{
+    Mariadb_nodes::require_gtid(true);
+    TestConnections test(argc, argv);
+
+    try
+    {
+        run(test);
+    }
+    catch (const std::exception& x)
+    {
+        cerr << "error: Execution was terminated due to an exception: " << x.what() << endl;
+    }
+
+    return test.global_result;
+}

maxscale-system-test/mariadbmonitor/mysqlmon_failover_manual2_2.cnf

@@ -0,0 +1,81 @@
+[maxscale]
+threads=###threads###
+
+[MySQL-Monitor]
+type=monitor
+module=mysqlmon
+servers= server1, server2
+user=maxskysql
+password= skysql
+monitor_interval=1000
+allow_cluster_recovery=true
+detect_standalone_master=true
+auto_failover=false
+replication_user=repl
+replication_password=repl
+backend_connect_timeout=10
+backend_read_timeout=10
+backend_write_timeout=10
+
+[RW-Split-Router]
+type=service
+router= readwritesplit
+servers=server1, server2
+user=maxskysql
+password=skysql
+
+[Read-Connection-Router-Slave]
+type=service
+router=readconnroute
+router_options= slave
+servers=server1, server2
+user=maxskysql
+password=skysql
+
+[Read-Connection-Router-Master]
+type=service
+router=readconnroute
+router_options=master
+servers=server1, server2
+user=maxskysql
+password=skysql
+
+[RW-Split-Listener]
+type=listener
+service=RW-Split-Router
+protocol=MySQLClient
+port=4006
+
+[Read-Connection-Listener-Slave]
+type=listener
+service=Read-Connection-Router-Slave
+protocol=MySQLClient
+port=4009
+
+[Read-Connection-Listener-Master]
+type=listener
+service=Read-Connection-Router-Master
+protocol=MySQLClient
+port=4008
+
+[CLI]
+type=service
+router=cli
+
+[CLI Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=default
+
+[server1]
+type=server
+address=###node_server_IP_1###
+port=###node_server_port_1###
+protocol=MySQLBackend
+
+[server2]
+type=server
+address=###node_server_IP_2###
+port=###node_server_port_2###
+protocol=MySQLBackend

maxscale-system-test/mariadbmonitor/mysqlmon_failover_manual2_3.cnf

@@ -0,0 +1,87 @@
+[maxscale]
+threads=###threads###
+
+[MySQL Monitor]
+type=monitor
+module=mysqlmon
+servers= server1, server2, server3
+user=maxskysql
+password= skysql
+monitor_interval=1000
+allow_cluster_recovery=true
+detect_standalone_master=true
+auto_failover=false
+replication_user=repl
+replication_password=repl
+backend_connect_timeout=10
+backend_read_timeout=10
+backend_write_timeout=10
+
+[RW Split Router]
+type=service
+router= readwritesplit
+servers=server1, server2, server3
+user=maxskysql
+password=skysql
+
+[Read-Connection-Router-Slave]
+type=service
+router=readconnroute
+router_options= slave
+servers=server1, server2, server3
+user=maxskysql
+password=skysql
+
+[Read-Connection-Router-Master]
+type=service
+router=readconnroute
+router_options=master
+servers=server1, server2, server3
+user=maxskysql
+password=skysql
+
+[RW-Split-Listener]
+type=listener
+service=RW Split Router
+protocol=MySQLClient
+port=4006
+
+[Read-Connection-Listener-Slave]
+type=listener
+service=Read-Connection-Router-Slave
+protocol=MySQLClient
+port=4009
+
+[Read Connection Listener Master]
+type=listener
+service=Read-Connection-Router-Master
+protocol=MySQLClient
+port=4008
+
+[CLI]
+type=service
+router=cli
+
+[CLI Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=default
+
+[server1]
+type=server
+address=###node_server_IP_1###
+port=###node_server_port_1###
+protocol=MySQLBackend
+
+[server2]
+type=server
+address=###node_server_IP_2###
+port=###node_server_port_2###
+protocol=MySQLBackend
+
+[server3]
+type=server
+address=###node_server_IP_3###
+port=###node_server_port_3###
+protocol=MySQLBackend

maxscale-system-test/mariadbmonitor/mysqlmon_failover_manual2_4.cnf

@@ -0,0 +1,93 @@
+[maxscale]
+threads=###threads###
+
+[MySQL-Monitor]
+type=monitor
+module=mysqlmon
+servers= server1, server2, server3, server4
+user=maxskysql
+password= skysql
+monitor_interval=1000
+allow_cluster_recovery=true
+detect_standalone_master=true
+auto_failover=false
+replication_user=repl
+replication_password=repl
+backend_connect_timeout=10
+backend_read_timeout=10
+backend_write_timeout=10
+
+[RW-Split-Router]
+type=service
+router= readwritesplit
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[Read-Connection-Router-Slave]
+type=service
+router=readconnroute
+router_options= slave
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[Read-Connection-Router-Master]
+type=service
+router=readconnroute
+router_options=master
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[RW-Split-Listener]
+type=listener
+service=RW-Split-Router
+protocol=MySQLClient
+port=4006
+
+[Read-Connection-Listener-Slave]
+type=listener
+service=Read-Connection-Router-Slave
+protocol=MySQLClient
+port=4009
+
+[Read-Connection-Listener-Master]
+type=listener
+service=Read-Connection-Router-Master
+protocol=MySQLClient
+port=4008
+
+[CLI]
+type=service
+router=cli
+
+[CLI Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=default
+
+[server1]
+type=server
+address=###node_server_IP_1###
+port=###node_server_port_1###
+protocol=MySQLBackend
+
+[server2]
+type=server
+address=###node_server_IP_2###
+port=###node_server_port_2###
+protocol=MySQLBackend
+
+[server3]
+type=server
+address=###node_server_IP_3###
+port=###node_server_port_3###
+protocol=MySQLBackend
+
+[server4]
+type=server
+address=###node_server_IP_4###
+port=###node_server_port_4###
+protocol=MySQLBackend

maxscale-system-test/mariadbmonitor/mysqlmon_failover_no_slaves.cpp

@@ -0,0 +1,135 @@
+/*
+ * Copyright (c) 2016 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2024-03-10
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+// MXS-2652: https://jira.mariadb.org/browse/MXS-2652
+
+#include "testconnections.h"
+#include "fail_switch_rejoin_common.cpp"
+using std::string;
+
+namespace
+{
+void expect_maintenance(TestConnections& test, std::string server_name, bool value);
+void expect_running(TestConnections& test, std::string server_name, bool value);
+
+const string running = "Running";
+const string down = "Down";
+const string maint = "Maintenance";
+}
+
+int main(int argc, char** argv)
+{
+    interactive = strcmp(argv[argc - 1], "interactive") == 0;
+    Mariadb_nodes::require_gtid(true);
+    TestConnections test(argc, argv);
+
+    delete_slave_binlogs(test);
+    basic_test(test);
+    MYSQL* conn = test.maxscales->open_rwsplit_connection(0);
+    if (!generate_traffic_and_check(test, conn, 5))
+    {
+        return test.global_result;
+    }
+
+    // Make all three slaves ineligible for promotion in different ways.
+    test.repl->connect();
+    MYSQL** nodes = test.repl->nodes;
+    // Slave 1. Just stop slave.
+    test.try_query(nodes[1], "STOP SLAVE;");
+    // Slave 2. Disable binlog.
+    test.repl->stop_node(2);
+    test.repl->stash_server_settings(2);
+    test.repl->disable_server_setting(2, "log-bin");
+    test.repl->start_node(2, (char*) "");
+    test.maxscales->wait_for_monitor();
+
+    // Slave 3. Set node to maintenance, then shut it down. Simultaneously check issue
+    // MXS-2652: Maintenance flag should persist when server goes down & comes back up.
+    int server_ind = 3;
+    int server_num = server_ind + 1;
+    string server_name = "server" + std::to_string(server_num);
+    expect_maintenance(test, server_name, false);
+
+    if (test.ok())
+    {
+        test.maxscales->ssh_node_f(0, true, "maxadmin set server %s maintenance", server_name.c_str());
+        test.maxscales->wait_for_monitor();
+        expect_running(test, server_name, true);
+        expect_maintenance(test, server_name, true);
+
+        test.repl->stop_node(server_ind);
+        test.maxscales->wait_for_monitor();
+        expect_running(test, server_name, false);
+        expect_maintenance(test, server_name, true);
+
+        test.repl->start_node(server_ind);
+        test.maxscales->wait_for_monitor();
+        expect_running(test, server_name, true);
+        expect_maintenance(test, server_name, true);
+    }
+
+    get_output(test);
+
+    test.tprintf(LINE);
+    test.tprintf("Stopping master. Failover should not happen.");
+    test.repl->block_node(0);
+    test.maxscales->wait_for_monitor();
+    get_output(test);
+    int master_id = get_master_server_id(test);
+    test.expect(master_id == -1, "Master was promoted even when no slave was eligible.");
+
+    test.repl->unblock_node(0);
+
+    // Restore normal settings.
+    test.try_query(nodes[1], "START SLAVE;");
+    test.repl->stop_node(2);
+    test.repl->restore_server_settings(2);
+    test.repl->start_node(2, (char*) "");
+    test.maxscales->ssh_node_f(0, true, "maxadmin clear server %s maintenance", server_name.c_str());
+
+    test.repl->fix_replication();
+    return test.global_result;
+}
+
+namespace
+{
+void expect_running(TestConnections& test, std::string server_name, bool value)
+{
+    auto states = test.get_server_status(server_name.c_str());
+    if (value)
+    {
+        test.expect(states.count(running) == 1, "'%s' is not running when it should be.",
+                    server_name.c_str());
+    }
+    else
+    {
+        test.expect(states.count(down) == 1, "'%s' is not down when it should be.",
+                    server_name.c_str());
+    }
+}
+
+void expect_maintenance(TestConnections& test, std::string server_name, bool value)
+{
+    auto states = test.get_server_status(server_name.c_str());
+    if (value)
+    {
+        test.expect(states.count(maint) == 1, "'%s' is not in maintenance when it should be.",
+                    server_name.c_str());
+    }
+    else
+    {
+        test.expect(states.count(maint) == 0, "'%s' is in maintenance when it should not be.",
+                    server_name.c_str());
+    }
+}
+}

maxscale-system-test/mariadbmonitor/mysqlmon_failover_readonly.cnf

@@ -0,0 +1,62 @@
+[maxscale]
+threads=###threads###
+
+[MariaDB-Monitor]
+type=monitor
+module=mysqlmon
+servers=server1, server2, server3
+user=maxskysql
+password=skysql
+monitor_interval=2000
+auto_failover=true
+auto_rejoin=true
+enforce_read_only_slaves=1
+replication_user=repl
+replication_password=repl
+backend_connect_timeout=5
+backend_read_timeout=5
+backend_write_timeout=5
+failcount=1
+
+[RW-Split-Router]
+type=service
+router= readwritesplit
+servers=server1, server2, server3
+user=maxskysql
+password=skysql
+transaction_replay=1
+delayed_retry_timeout=30
+
+[RW-Split-Listener]
+type=listener
+service=RW-Split-Router
+protocol=MySQLClient
+port=4006
+
+[CLI]
+type=service
+router=cli
+
+[CLI Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=default
+
+[server1]
+type=server
+address=###node_server_IP_1###
+port=###node_server_port_1###
+protocol=MySQLBackend
+
+[server2]
+type=server
+address=###node_server_IP_2###
+port=###node_server_port_2###
+protocol=MySQLBackend
+
+[server3]
+type=server
+address=###node_server_IP_3###
+port=###node_server_port_3###
+protocol=MySQLBackend

maxscale-system-test/mariadbmonitor/mysqlmon_failover_readonly.cpp

@@ -0,0 +1,225 @@
+/*
+ * Copyright (c) 2016 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2024-03-10
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#include "testconnections.h"
+#include "fail_switch_rejoin_common.cpp"
+#include <iostream>
+#include <string>
+#include <vector>
+
+using std::string;
+using std::cout;
+
+int main(int argc, char** argv)
+{
+    Mariadb_nodes::require_gtid(true);
+    TestConnections test(argc, argv);
+    // Delete binlogs to sync gtid:s
+    delete_slave_binlogs(test);
+    // Test uses 2 slaves, stop the last one to prevent it from replicating anything.
+    test.repl->stop_node(3);
+    // Set up test table
+    basic_test(test);
+    // Advance gtid:s a bit to so gtid variables are updated.
+    MYSQL* maxconn = test.maxscales->open_rwsplit_connection(0);
+    generate_traffic_and_check(test, maxconn, 1);
+    test.repl->sync_slaves(0);
+    get_output(test);
+    print_gtids(test);
+
+    auto expect_server_status = [&test](const string& server_name, const string& status) {
+            bool found = (test.maxscales->get_server_status(server_name.c_str()).count(status) == 1);
+            test.expect(found, "%s was not %s as was expected.", server_name.c_str(), status.c_str());
+        };
+
+    string server_names[] = {"server1", "server2", "server3"};
+    auto expect_server_status_multi =
+        [&test, &expect_server_status, &server_names](std::vector<string> expected) {
+            int expected_size = expected.size();
+            test.expect(expected_size <= test.repl->N && expected_size <= 3, "Too many expected values.");
+            int tests = (expected_size < test.repl->N) ? expected_size : test.repl->N;
+            for (int node = 0; node < tests; node++)
+            {
+                expect_server_status(server_names[node], expected[node]);
+            }
+        };
+
+    auto expect_read_only = [&test](int node, bool expected) {
+            test.expect(test.repl->connect(node) == 0, "Connection to node %i failed.", node);
+            char result[2];
+            const char query[] = "SELECT @@read_only;";
+            if (find_field(test.repl->nodes[node], query, "@@read_only", result) == 0)
+            {
+                char expected_result = expected ? '1' : '0';
+                test.expect(result[0] == expected_result, "read_only on node %i was %c when %c was expected.",
+                            node, result[0], expected_result);
+            }
+            else
+            {
+                test.expect(false, "Query '%s' failed on node %i.", query, node);
+            }
+        };
+
+    auto expect_read_only_multi = [&test, &expect_read_only](std::vector<bool> expected) {
+            int expected_size = expected.size();
+            test.expect(expected_size <= test.repl->N, "Too many expected values.");
+            int tests = (expected_size < test.repl->N) ? expected_size : test.repl->N;
+            for (int node = 0; node < tests; node++)
+            {
+                expect_read_only(node, expected[node]);
+            }
+        };
+
+    auto mon_wait = [&test](int ticks) {
+            test.maxscales->wait_for_monitor(ticks);
+        };
+
+    auto crash_node = [&test](int node) {
+            test.repl->ssh_node(node, "kill -s 11 `pidof mysqld`", true);
+            test.repl->stop_node(node); // To prevent autostart.
+        };
+
+    string master = "Master";
+    string slave = "Slave";
+    string down = "Down";
+
+    cout << "Step 1: All should be cool.\n";
+    get_output(test);
+    expect_server_status_multi({master, slave, slave});
+    expect_read_only_multi({false, true, true});
+
+    if (test.ok())
+    {
+        cout << "Step 2: Crash slave 2.\n";
+        crash_node(2);
+        mon_wait(1);
+        get_output(test);
+        expect_server_status_multi({master, slave, down});
+        expect_read_only_multi({false, true});
+        generate_traffic_and_check(test, maxconn, 2);
+
+        cout << "Step 2.1: Slave 2 comes back up, check that read_only is set.\n";
+        test.repl->start_node(2, "");
+        mon_wait(2);
+        get_output(test);
+        expect_server_status_multi({master, slave, slave});
+        expect_read_only_multi({false, true, true});
+        generate_traffic_and_check(test, maxconn, 3);
+
+        cout << "Step 3: Slave 1 crashes.\n";
+        crash_node(1);
+        mon_wait(1);
+        get_output(test);
+        expect_server_status_multi({master, down, slave});
+        expect_read_only(2, true);
+        generate_traffic_and_check(test, maxconn, 4);
+
+        cout << "Step 4: Slave 2 goes down again, this time normally.\n";
+        test.repl->stop_node(2);
+        mon_wait(1);
+        get_output(test);
+        expect_server_status_multi({master, down, down});
+        generate_traffic_and_check(test, maxconn, 5);
+
+        cout << "Step 4.1: Slave 1 comes back up, check that read_only is set.\n";
+        test.repl->start_node(1, "");
+        mon_wait(2);
+        get_output(test);
+        expect_server_status_multi({master, slave, down});
+        expect_read_only_multi({false, true});
+        generate_traffic_and_check(test, maxconn, 6);
+
+        cout << "Step 4.2: Slave 2 is back up, all should be well.\n";
+        test.repl->start_node(2, "");
+        mon_wait(2);
+        get_output(test);
+        expect_server_status_multi({master, slave, slave});
+        expect_read_only_multi({false, true, true});
+        generate_traffic_and_check(test, maxconn, 5);
+    }
+    mysql_close(maxconn);
+
+    // Intermission, quit if a test step failed.
+    if (test.ok())
+    {
+        // Some of the following tests depend on manipulating backends during the same monitor tick or
+        // between ticks. Slow down the monitor to make this more likely. Not fool-proof in the slightest.
+        int rval = test.maxscales->execute_maxadmin_command(0, "alter monitor MariaDB-Monitor "
+                                                               "monitor_interval=4000");
+        test.expect(rval == 0, "MaxAdmin command failed.");
+    }
+
+    if (test.ok())
+    {
+        cout << "Step 5: Master crashes but comes back during the next loop,"
+                " slave 1 should be promoted, old master rejoined.\n";
+        crash_node(0);
+        mon_wait(1);    // The timing is probably a bit iffy here.
+        expect_server_status(server_names[0], down);
+        get_output(test);
+        test.repl->start_node(0, "");
+        mon_wait(2);
+        get_output(test);
+        // Slave 2 could be promoted as well, but in this case there is no reason to choose it.
+        expect_server_status_multi({slave, master, slave});
+        expect_read_only_multi({true, false, true});
+        maxconn = test.maxscales->open_rwsplit_connection(0);
+        generate_traffic_and_check(test, maxconn, 4);
+
+        cout << "Step 6: Servers 1 & 3 go down. Server 2 should remain as master.\n";
+        test.repl->stop_node(0);
+        test.repl->stop_node(2);
+        mon_wait(1);
+        get_output(test);
+        expect_server_status_multi({down, master, down});
+        generate_traffic_and_check(test, maxconn, 3);
+
+        cout << "Step 6.1: Servers 1 & 3 come back. Check that read_only is set.\n";
+        test.repl->start_node(2, "");
+        test.repl->start_node(0, "");
+        mon_wait(2);
+        get_output(test);
+        expect_server_status_multi({slave, master, slave});
+        expect_read_only_multi({true, false, true});
+        generate_traffic_and_check(test, maxconn, 2);
+
+        cout << "Step 7: Servers 1 & 2 go down. Check that 3 is promoted.\n";
+        mysql_close(maxconn);
+        test.repl->stop_node(0);
+        test.repl->stop_node(1);
+        mon_wait(2);
+        get_output(test);
+        expect_server_status_multi({down, down, master});
+        maxconn = test.maxscales->open_rwsplit_connection(0);
+        generate_traffic_and_check(test, maxconn, 1);
+        mysql_close(maxconn);
+    }
+
+    
+    // Start the servers, in case they weren't on already.
+    for (int i = 0; i < 3; i++)
+    {
+        test.repl->start_node(i);
+    }
+    sleep(1);
+    test.repl->connect();
+    // Delete the test table from all databases, reset replication.
+    const char drop_query[] = "DROP TABLE IF EXISTS test.t1;";
+    for (int i = 0; i < 3; i++)
+    {
+        test.try_query(test.repl->nodes[i], drop_query);
+    }
+    test.maxscales->execute_maxadmin_command(0, "call command mariadbmon reset-replication "
+                                                "MariaDB-Monitor server1");
+    return test.global_result;
+}

maxscale-system-test/mariadbmonitor/mysqlmon_failover_rejoin_old_slave.cnf

@@ -0,0 +1,95 @@
+[maxscale]
+threads=###threads###
+
+[MySQL-Monitor]
+type=monitor
+module=mysqlmon
+servers= server1, server2, server3, server4
+user=maxskysql
+password= skysql
+monitor_interval=1000
+allow_cluster_recovery=true
+detect_standalone_master=true
+auto_failover=true
+auto_rejoin=true
+replication_user=repl
+replication_password=repl
+backend_connect_timeout=10
+backend_read_timeout=10
+backend_write_timeout=10
+failcount=1
+
+[RW-Split-Router]
+type=service
+router= readwritesplit
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[Read-Connection-Router-Slave]
+type=service
+router=readconnroute
+router_options= slave
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[Read-Connection-Router-Master]
+type=service
+router=readconnroute
+router_options=master
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[RW-Split-Listener]
+type=listener
+service=RW-Split-Router
+protocol=MySQLClient
+port=4006
+
+[Read-Connection-Listener-Slave]
+type=listener
+service=Read-Connection-Router-Slave
+protocol=MySQLClient
+port=4009
+
+[Read-Connection-Listener-Master]
+type=listener
+service=Read-Connection-Router-Master
+protocol=MySQLClient
+port=4008
+
+[CLI]
+type=service
+router=cli
+
+[CLI Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=default
+
+[server1]
+type=server
+address=###node_server_IP_1###
+port=###node_server_port_1###
+protocol=MySQLBackend
+
+[server2]
+type=server
+address=###node_server_IP_2###
+port=###node_server_port_2###
+protocol=MySQLBackend
+
+[server3]
+type=server
+address=###node_server_IP_3###
+port=###node_server_port_3###
+protocol=MySQLBackend
+
+[server4]
+type=server
+address=###node_server_IP_4###
+port=###node_server_port_4###
+protocol=MySQLBackend

maxscale-system-test/mariadbmonitor/mysqlmon_failover_rejoin_old_slave.cpp

@@ -0,0 +1,187 @@
+/*
+ * Copyright (c) 2016 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2024-03-10
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#include <iostream>
+#include <iterator>
+#include <string>
+#include <sstream>
+#include "testconnections.h"
+
+using std::cerr;
+using std::cout;
+using std::endl;
+using std::flush;
+using std::string;
+using std::stringstream;
+
+namespace
+{
+
+void create_table(TestConnections& test)
+{
+    MYSQL* pConn = test.maxscales->conn_rwsplit[0];
+
+    test.try_query(pConn, "DROP TABLE IF EXISTS test.t1");
+    test.try_query(pConn, "CREATE TABLE test.t1(id INT)");
+}
+
+static int i_start = 0;
+static int n_rows = 20;
+static int i_end = 0;
+
+void insert_data(TestConnections& test)
+{
+    MYSQL* pConn = test.maxscales->conn_rwsplit[0];
+
+    test.try_query(pConn, "BEGIN");
+
+    i_end = i_start + n_rows;
+
+    for (int i = i_start; i < i_end; ++i)
+    {
+        stringstream ss;
+        ss << "INSERT INTO test.t1 VALUES (" << i << ")";
+        test.try_query(pConn, "%s", ss.str().c_str());
+    }
+
+    test.try_query(pConn, "COMMIT");
+
+    i_start = i_end;
+}
+
+void expect(TestConnections& test, const char* zServer, const StringSet& expected)
+{
+    StringSet found = test.get_server_status(zServer);
+
+    std::ostream_iterator<string> oi(cout, ", ");
+
+    cout << zServer
+         << ", expected states: ";
+    std::copy(expected.begin(), expected.end(), oi);
+    cout << endl;
+
+    cout << zServer
+         << ", found states   : ";
+    std::copy(found.begin(), found.end(), oi);
+    cout << endl;
+
+    if (found != expected)
+    {
+        cout << "ERROR, found states are not the same as the expected ones." << endl;
+        ++test.global_result;
+    }
+
+    cout << endl;
+}
+
+void expect(TestConnections& test, const char* zServer, const char* zState)
+{
+    StringSet s;
+    s.insert(zState);
+
+    expect(test, zServer, s);
+}
+
+void expect(TestConnections& test, const char* zServer, const char* zState1, const char* zState2)
+{
+    StringSet s;
+    s.insert(zState1);
+    s.insert(zState2);
+
+    expect(test, zServer, s);
+}
+
+void run(TestConnections& test)
+{
+    test.maxscales->wait_for_monitor();
+
+    int N = test.repl->N;
+    cout << "Nodes: " << N << endl;
+
+    expect(test, "server1", "Master", "Running");
+    expect(test, "server2", "Slave", "Running");
+    expect(test, "server3", "Slave", "Running");
+    expect(test, "server4", "Slave", "Running");
+
+    cout << "\nConnecting to MaxScale." << endl;
+    test.maxscales->connect_maxscale(0);
+
+    cout << "\nCreating table." << endl;
+    create_table(test);
+
+    cout << "\nInserting data." << endl;
+    insert_data(test);
+
+    cout << "\nSyncing slaves." << endl;
+    test.repl->sync_slaves();
+
+    cout << "\nStopping slave " << N - 1 << endl;
+    test.repl->stop_node(N - 1);
+
+    test.maxscales->wait_for_monitor();
+
+    // server4 was stopped, so we expect the state of it to be /Down/,
+    // and the states of the other ones not to have changed.
+    expect(test, "server1", "Master", "Running");
+    expect(test, "server2", "Slave", "Running");
+    expect(test, "server3", "Slave", "Running");
+    expect(test, "server4", "Down");
+
+    cout << "\nClosing connection to MaxScale." << endl;
+    test.maxscales->close_maxscale_connections(0);
+
+    cout << "\nConnecting to MaxScale." << endl;
+    test.maxscales->connect_maxscale(0);
+
+    cout << "\nInserting data." << endl;
+    insert_data(test);
+
+    cout << "\nSyncing slaves." << endl;
+    test.repl->sync_slaves();
+
+    cout << "\nStopping master." << endl;
+    test.repl->stop_node(0);
+
+    test.maxscales->wait_for_monitor(2);
+
+    // server1 (previous master) was taken down, so its state should be /Down/.
+    // server2 should have been made into master, and server4 should still be down.
+    expect(test, "server1", "Down");
+    expect(test, "server2", "Master", "Running");
+    expect(test, "server3", "Slave", "Running");
+    expect(test, "server4", "Down");
+
+    cout << "\nBringing up slave " << N - 1 << endl;
+    test.repl->start_node(N - 1, (char*)"");
+
+    test.maxscales->wait_for_monitor(2);
+
+    // server1 should still be down, server2 still master, and server3 still
+    // a slave. server4 was brought up, so it should have been rejoined and
+    // turned into a slave.
+    expect(test, "server1", "Down");
+    expect(test, "server2", "Master", "Running");
+    expect(test, "server3", "Slave", "Running");
+    expect(test, "server4", "Slave", "Running");
+}
+}
+
+int main(int argc, char* argv[])
+{
+    Mariadb_nodes::require_gtid(true);
+    TestConnections test(argc, argv);
+
+    run(test);
+
+    return test.global_result;
+}

maxscale-system-test/mariadbmonitor/mysqlmon_failover_rolling_master.cnf

@@ -0,0 +1,93 @@
+[maxscale]
+threads=###threads###
+
+[MySQL-Monitor]
+type=monitor
+module=mysqlmon
+servers= server1, server2, server3, server4
+user=maxskysql
+password= skysql
+monitor_interval=1000
+allow_cluster_recovery=true
+detect_standalone_master=true
+auto_failover=true
+replication_user=repl
+replication_password=repl
+backend_connect_timeout=10
+backend_read_timeout=10
+backend_write_timeout=10
+
+[RW-Split-Router]
+type=service
+router= readwritesplit
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[Read-Connection-Router-Slave]
+type=service
+router=readconnroute
+router_options= slave
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[Read-Connection-Router-Master]
+type=service
+router=readconnroute
+router_options=master
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[RW-Split-Listener]
+type=listener
+service=RW-Split-Router
+protocol=MySQLClient
+port=4006
+
+[Read-Connection-Listener-Slave]
+type=listener
+service=Read-Connection-Router-Slave
+protocol=MySQLClient
+port=4009
+
+[Read-Connection-Listener-Master]
+type=listener
+service=Read-Connection-Router-Master
+protocol=MySQLClient
+port=4008
+
+[CLI]
+type=service
+router=cli
+
+[CLI Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=default
+
+[server1]
+type=server
+address=###node_server_IP_1###
+port=###node_server_port_1###
+protocol=MySQLBackend
+
+[server2]
+type=server
+address=###node_server_IP_2###
+port=###node_server_port_2###
+protocol=MySQLBackend
+
+[server3]
+type=server
+address=###node_server_IP_3###
+port=###node_server_port_3###
+protocol=MySQLBackend
+
+[server4]
+type=server
+address=###node_server_IP_4###
+port=###node_server_port_4###
+protocol=MySQLBackend

maxscale-system-test/mariadbmonitor/mysqlmon_failover_rolling_master.cpp

@@ -0,0 +1,272 @@
+/*
+ * Copyright (c) 2016 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2024-03-10
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#include <iostream>
+#include <string>
+#include <sstream>
+#include "testconnections.h"
+
+using std::cerr;
+using std::cout;
+using std::endl;
+using std::flush;
+using std::string;
+using std::stringstream;
+
+namespace
+{
+
+void sleep(int s)
+{
+    cout << "Sleeping " << s << " times 1 second" << flush;
+    do
+    {
+        ::sleep(1);
+        cout << "." << flush;
+        --s;
+    }
+    while (s > 0);
+
+    cout << endl;
+}
+
+int get_server_id(Maxscales& maxscales)
+{
+    MYSQL* conn = maxscales.open_rwsplit_connection(0);
+    int id = -1;
+    char str[1024];
+
+    if (find_field(conn, "SELECT @@server_id, @@last_insert_id;", "@@server_id", str) == 0)
+    {
+        id = atoi(str);
+    }
+
+    mysql_close(conn);
+
+    if (id == -1)
+    {
+        throw std::runtime_error("Could not get server id.");
+    }
+
+    return id;
+}
+}
+
+namespace
+{
+
+class XTestConnections : private TestConnections
+{
+public:
+    using TestConnections::add_result;
+    using TestConnections::global_result;
+    using TestConnections::maxscales;
+    using TestConnections::repl;
+
+    XTestConnections(int argc, char* argv[])
+        : TestConnections(argc, argv)
+    {
+    }
+
+    TestConnections& nothrow()
+    {
+        return *this;
+    }
+
+    void connect_maxscale(int m = 0)
+    {
+        if (maxscales->connect_maxscale(m) != 0)
+        {
+            ++global_result;
+            throw std::runtime_error("Could not connect to MaxScale.");
+        }
+    }
+
+    void try_query(MYSQL* conn, const char* format, ...)
+    {
+        va_list valist;
+
+        va_start(valist, format);
+        int message_len = vsnprintf(NULL, 0, format, valist);
+        va_end(valist);
+
+        char sql[message_len + 1];
+
+        va_start(valist, format);
+        vsnprintf(sql, sizeof(sql), format, valist);
+        va_end(valist);
+
+        int res = execute_query_silent(conn, sql, false);
+        add_result(res,
+                   "Query '%.*s%s' failed!\n",
+                   message_len < 100 ? message_len : 100,
+                   sql,
+                   message_len < 100 ? "" : "...");
+
+        if (res != 0)
+        {
+            string s("Could not execute query: ");
+            s += sql;
+
+            if (s.length() > 80)
+            {
+                s = s.substr(0, 77);
+                s += "...";
+            }
+
+            throw std::runtime_error(s.c_str());
+        }
+    }
+};
+
+void list_servers(XTestConnections& test)
+{
+    cout << endl;
+    test.maxscales->execute_maxadmin_command_print(0, (char*)"list servers");
+}
+}
+
+namespace
+{
+
+void create_table(XTestConnections& test)
+{
+    test.try_query(test.maxscales->conn_rwsplit[0], "DROP TABLE IF EXISTS test.t1");
+    test.try_query(test.maxscales->conn_rwsplit[0], "CREATE TABLE test.t1(id INT)");
+}
+
+static int i_start = 0;
+static int n_rows = 20;
+static int i_end = 0;
+
+void insert_data(XTestConnections& test)
+{
+    test.try_query(test.maxscales->conn_rwsplit[0], "BEGIN");
+
+    i_end = i_start + n_rows;
+
+    for (int i = i_start; i < i_end; ++i)
+    {
+        stringstream ss;
+        ss << "INSERT INTO test.t1 VALUES (" << i << ")";
+        test.try_query(test.maxscales->conn_rwsplit[0], ss.str().c_str());
+    }
+    test.try_query(test.maxscales->conn_rwsplit[0], "COMMIT");
+
+    i_start = i_end;
+}
+
+void check(XTestConnections& test)
+{
+    MYSQL* pConn = test.maxscales->open_rwsplit_connection(0);
+    const char* zQuery = "SELECT * FROM test.t1";
+
+    test.try_query(pConn, "BEGIN");
+    mysql_query(pConn, zQuery);
+
+    MYSQL_RES* pRes = mysql_store_result(pConn);
+    test.add_result(pRes == NULL, "Query should return a result set.");
+
+
+    if (!pRes)
+    {
+        mysql_close(pConn);
+        throw std::runtime_error("Query did not return a result set.");
+    }
+
+    std::string values;
+    int num_rows = mysql_num_rows(pRes);
+    test.add_result(num_rows != i_end,
+                    "Query returned %d rows when %d rows were expected",
+                    num_rows,
+                    i_end);
+    test.nothrow().try_query(pConn, "COMMIT");
+    mysql_close(pConn);
+}
+
+void stop_node(XTestConnections& test, int index)
+{
+    if (test.repl->stop_node(index) != 0)
+    {
+        throw std::runtime_error("Could not stop node.");
+    }
+
+    list_servers(test);
+}
+
+void run(XTestConnections& test)
+{
+    test.maxscales->wait_for_monitor();
+
+    int N = test.repl->N;
+    cout << "Nodes: " << N << endl;
+
+    cout << "\nConnecting to MaxScale." << endl;
+    test.connect_maxscale();
+
+    cout << "\nCreating table." << endl;
+    create_table(test);
+
+    list_servers(test);
+
+    for (int i = 0; i < N - 1; ++i)
+    {
+        cout << "Round: " << i << "\n"
+             << "--------" << endl;
+        cout << "\nInserting data." << endl;
+        insert_data(test);
+
+        cout << "\nSyncing slaves." << endl;
+        test.repl->sync_slaves();
+
+        int master_id = get_server_id(*test.maxscales);
+        int master_index = master_id - 1;
+
+        cout << "\nStopping master." << endl;
+        stop_node(test, master_index);
+
+        cout << "\nClosing connection to MaxScale." << endl;
+        test.maxscales->close_maxscale_connections(0);
+
+        test.maxscales->wait_for_monitor();
+
+        list_servers(test);
+
+        master_id = get_server_id(*test.maxscales);
+        cout << "\nNew master is: " << master_id << endl;
+
+        cout << "\nConnecting to MaxScale." << endl;
+        test.connect_maxscale();
+
+        cout << "\nChecking result." << endl;
+        check(test);
+    }
+}
+}
+
+int main(int argc, char* argv[])
+{
+    Mariadb_nodes::require_gtid(true);
+    XTestConnections test(argc, argv);
+
+    try
+    {
+        run(test);
+    }
+    catch (const std::exception& x)
+    {
+        cerr << "error: Execution was terminated due to an exception: " << x.what() << endl;
+    }
+
+    return test.global_result;
+}

maxscale-system-test/mariadbmonitor/mysqlmon_failover_rolling_restart_slaves.cnf

@@ -0,0 +1,94 @@
+[maxscale]
+threads=###threads###
+
+[MySQL-Monitor]
+type=monitor
+module=mysqlmon
+servers= server1, server2, server3, server4
+user=maxskysql
+password= skysql
+monitor_interval=1000
+allow_cluster_recovery=true
+detect_standalone_master=true
+auto_failover=true
+auto_rejoin=true
+replication_user=repl
+replication_password=repl
+backend_connect_timeout=10
+backend_read_timeout=10
+backend_write_timeout=10
+
+[RW-Split-Router]
+type=service
+router= readwritesplit
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[Read-Connection-Router-Slave]
+type=service
+router=readconnroute
+router_options= slave
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[Read-Connection-Router-Master]
+type=service
+router=readconnroute
+router_options=master
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[RW-Split-Listener]
+type=listener
+service=RW-Split-Router
+protocol=MySQLClient
+port=4006
+
+[Read-Connection-Listener-Slave]
+type=listener
+service=Read-Connection-Router-Slave
+protocol=MySQLClient
+port=4009
+
+[Read-Connection-Listener-Master]
+type=listener
+service=Read-Connection-Router-Master
+protocol=MySQLClient
+port=4008
+
+[CLI]
+type=service
+router=cli
+
+[CLI Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=default
+
+[server1]
+type=server
+address=###node_server_IP_1###
+port=###node_server_port_1###
+protocol=MySQLBackend
+
+[server2]
+type=server
+address=###node_server_IP_2###
+port=###node_server_port_2###
+protocol=MySQLBackend
+
+[server3]
+type=server
+address=###node_server_IP_3###
+port=###node_server_port_3###
+protocol=MySQLBackend
+
+[server4]
+type=server
+address=###node_server_IP_4###
+port=###node_server_port_4###
+protocol=MySQLBackend

maxscale-system-test/mariadbmonitor/mysqlmon_failover_rolling_restart_slaves.cpp

@@ -0,0 +1,207 @@
+/*
+ * Copyright (c) 2016 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2024-03-10
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#include <iostream>
+#include <iterator>
+#include <string>
+#include <sstream>
+#include "testconnections.h"
+
+using std::cerr;
+using std::cout;
+using std::endl;
+using std::flush;
+using std::string;
+using std::stringstream;
+
+namespace
+{
+
+void sleep(int s)
+{
+    cout << "Sleeping " << s << " times 1 second" << flush;
+    do
+    {
+        ::sleep(1);
+        cout << "." << flush;
+        --s;
+    }
+    while (s > 0);
+
+    cout << endl;
+}
+}
+
+namespace
+{
+
+void create_table(TestConnections& test)
+{
+    MYSQL* pConn = test.maxscales->conn_rwsplit[0];
+
+    test.try_query(pConn, "DROP TABLE IF EXISTS test.t1");
+    test.try_query(pConn, "CREATE TABLE test.t1(id INT)");
+}
+
+static int i_start = 0;
+static int n_rows = 20;
+static int i_end = 0;
+
+void insert_data(TestConnections& test)
+{
+    MYSQL* pConn = test.maxscales->conn_rwsplit[0];
+
+    test.try_query(pConn, "BEGIN");
+
+    i_end = i_start + n_rows;
+
+    for (int i = i_start; i < i_end; ++i)
+    {
+        stringstream ss;
+        ss << "INSERT INTO test.t1 VALUES (" << i << ")";
+        test.try_query(pConn, "%s", ss.str().c_str());
+    }
+
+    test.try_query(pConn, "COMMIT");
+
+    i_start = i_end;
+}
+
+void expect(TestConnections& test, const char* zServer, const StringSet& expected)
+{
+    StringSet found = test.get_server_status(zServer);
+
+    std::ostream_iterator<string> oi(cout, ", ");
+
+    cout << zServer
+         << ", expected states: ";
+    std::copy(expected.begin(), expected.end(), oi);
+    cout << endl;
+
+    cout << zServer
+         << ", found states   : ";
+    std::copy(found.begin(), found.end(), oi);
+    cout << endl;
+
+    if (found != expected)
+    {
+        cout << "ERROR, found states are not the same as the expected ones." << endl;
+        ++test.global_result;
+    }
+
+    cout << endl;
+}
+
+void expect(TestConnections& test, const char* zServer, const char* zState)
+{
+    StringSet s;
+    s.insert(zState);
+
+    expect(test, zServer, s);
+}
+
+void expect(TestConnections& test, const string& server, const char* zState)
+{
+    expect(test, server.c_str(), zState);
+}
+
+void expect(TestConnections& test, const char* zServer, const char* zState1, const char* zState2)
+{
+    StringSet s;
+    s.insert(zState1);
+    s.insert(zState2);
+
+    expect(test, zServer, s);
+}
+
+void expect(TestConnections& test, const string& server, const char* zState1, const char* zState2)
+{
+    expect(test, server.c_str(), zState1, zState2);
+}
+
+string server_name(int i)
+{
+    stringstream ss;
+    ss << "server" << (i + 1);
+    return ss.str();
+}
+
+void check_server_status(TestConnections& test, int N, int down = -1)
+{
+    expect(test, "server1", "Master", "Running");
+
+    for (int i = 1; i < N; ++i)
+    {
+        string slave = server_name(i);
+        if (i == down)
+        {
+            expect(test, slave, "Down");
+        }
+        else
+        {
+            expect(test, slave, "Slave", "Running");
+        }
+    }
+}
+
+void run(TestConnections& test)
+{
+    test.maxscales->wait_for_monitor();
+
+    int N = test.repl->N;
+    cout << "Nodes: " << N << endl;
+
+    check_server_status(test, N);
+
+    cout << "\nConnecting to MaxScale." << endl;
+    test.maxscales->connect_maxscale(0);
+
+    cout << "\nCreating table." << endl;
+    create_table(test);
+
+    cout << "\nInserting data." << endl;
+    insert_data(test);
+
+    cout << "\nSyncing slaves." << endl;
+    test.repl->sync_slaves();
+
+    for (int i = 1; i < N; ++i)
+    {
+        string slave = server_name(i);
+
+        cout << "\nStopping slave " << slave << endl;
+        test.repl->stop_node(i);
+
+        test.maxscales->wait_for_monitor();
+
+        check_server_status(test, N, i);
+
+        cout << "\nStarting slave " << slave << endl;
+        test.repl->start_node(i, (char*)"");
+
+        test.maxscales->wait_for_monitor();
+
+        check_server_status(test, N);
+    }
+}
+}
+
+int main(int argc, char* argv[])
+{
+    Mariadb_nodes::require_gtid(true);
+    TestConnections test(argc, argv);
+
+    run(test);
+
+    return test.global_result;
+}

maxscale-system-test/mariadbmonitor/mysqlmon_failover_stress.cnf

@@ -0,0 +1,95 @@
+[maxscale]
+threads=###threads###
+
+[MySQL-Monitor]
+type=monitor
+module=mysqlmon
+servers= server1, server2, server3, server4
+user=maxskysql
+password= skysql
+monitor_interval=1000
+allow_cluster_recovery=true
+detect_standalone_master=true
+auto_failover=true
+auto_rejoin=true
+replication_user=repl
+replication_password=repl
+backend_connect_timeout=10
+backend_read_timeout=10
+backend_write_timeout=10
+failcount=1
+
+[RW-Split-Router]
+type=service
+router= readwritesplit
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[Read-Connection-Router-Slave]
+type=service
+router=readconnroute
+router_options= slave
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[Read-Connection-Router-Master]
+type=service
+router=readconnroute
+router_options=master
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[RW-Split-Listener]
+type=listener
+service=RW-Split-Router
+protocol=MySQLClient
+port=4006
+
+[Read-Connection-Listener-Slave]
+type=listener
+service=Read-Connection-Router-Slave
+protocol=MySQLClient
+port=4009
+
+[Read-Connection-Listener-Master]
+type=listener
+service=Read-Connection-Router-Master
+protocol=MySQLClient
+port=4008
+
+[CLI]
+type=service
+router=cli
+
+[CLI Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=default
+
+[server1]
+type=server
+address=###node_server_IP_1###
+port=###node_server_port_1###
+protocol=MySQLBackend
+
+[server2]
+type=server
+address=###node_server_IP_2###
+port=###node_server_port_2###
+protocol=MySQLBackend
+
+[server3]
+type=server
+address=###node_server_IP_3###
+port=###node_server_port_3###
+protocol=MySQLBackend
+
+[server4]
+type=server
+address=###node_server_IP_4###
+port=###node_server_port_4###
+protocol=MySQLBackend

maxscale-system-test/mariadbmonitor/mysqlmon_failover_stress.cpp

@@ -0,0 +1,595 @@
+/*
+ * Copyright (c) 2016 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2024-03-10
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#include <iostream>
+#include <iterator>
+#include <string>
+#include <sstream>
+#include <thread>
+#include <vector>
+#include <random>
+#include "testconnections.h"
+#include "fail_switch_rejoin_common.cpp"
+
+using namespace std;
+
+// How often the monitor checks the server state.
+// NOTE: Ensure this is identical with the value in the configuration file.
+const time_t MONITOR_INTERVAL = 1;
+
+// After how many seconds should the failover/rejoin operation surely have
+// been performed. Not very critical.
+const time_t FAILOVER_DURATION = 5;
+
+// The test now runs only two failovers. Change for a longer time limit later.
+// TODO: add semisync to remove this limitation.
+
+#define CMESSAGE(msg) \
+    do { \
+        stringstream ss; \
+        ss << "client(" << m_id << ") : " << msg << "\n"; \
+        cout << ss.str() << flush; \
+    } while (false)
+
+#if !defined (NDEBUG)
+
+#define ss_dassert(x) do {if (!(x)) {fprintf(stderr, "Assertion failed: %s\n", #x); abort();}} while (false)
+#define ss_debug(x)   x
+
+#else
+
+#define ss_dassert(s)
+#define ss_debug(x)
+
+#endif
+
+
+namespace
+{
+
+class Client
+{
+public:
+    enum
+    {
+        DEFAULT_N_CLIENTS = 4,
+        DEFAULT_N_ROWS    = 100
+    };
+
+    static void init(TestConnections& test, size_t nClients, size_t nRows)
+    {
+        s_nClients = nClients;
+        s_nRows = nRows;
+
+        if (create_tables(test))
+        {
+            if (insert_data(test))
+            {
+                cout << "\nSyncing slaves." << endl;
+                test.repl->sync_slaves();
+            }
+        }
+    }
+
+    static void start(bool verbose,
+                      const char* zHost,
+                      int port,
+                      const char* zUser,
+                      const char* zPassword)
+    {
+        for (size_t i = 0; i < s_nClients; ++i)
+        {
+            s_threads.push_back(std::thread(&Client::thread_main,
+                                            i,
+                                            verbose,
+                                            zHost,
+                                            port,
+                                            zUser,
+                                            zPassword));
+        }
+    }
+
+    static void stop()
+    {
+        s_shutdown = true;
+
+        for (size_t i = 0; i < s_nClients; ++i)
+        {
+            s_threads[i].join();
+        }
+    }
+
+private:
+    Client(int id, bool verbose)
+        : m_id(id)
+        , m_verbose(verbose)
+        , m_value(1)
+        , m_rand_dist(0.0, 1.0)
+    {
+    }
+
+    enum action_t
+    {
+        ACTION_SELECT,
+        ACTION_UPDATE
+    };
+
+    action_t action() const
+    {
+        double d = random_decimal_fraction();
+
+        // 20% updates
+        // 80% selects
+        if (d <= 0.2)
+        {
+            return ACTION_UPDATE;
+        }
+        else
+        {
+            return ACTION_SELECT;
+        }
+    }
+
+    bool run(MYSQL* pConn)
+    {
+        bool rv = false;
+
+        switch (action())
+        {
+        case ACTION_SELECT:
+            rv = run_select(pConn);
+            break;
+
+        case ACTION_UPDATE:
+            rv = run_update(pConn);
+            break;
+
+        default:
+            ss_dassert(!true);
+        }
+
+        return rv;
+    }
+
+    bool run_select(MYSQL* pConn)
+    {
+        bool rv = true;
+
+        string stmt("SELECT * FROM test.t");
+        stmt += std::to_string(m_id);
+        stmt += " WHERE id=";
+        stmt += std::to_string(get_random_id());
+
+        if (mysql_query(pConn, stmt.c_str()) == 0)
+        {
+            flush_response(pConn);
+        }
+        else
+        {
+            if (m_verbose)
+            {
+                CMESSAGE("\"" << stmt << "\" failed: " << mysql_error(pConn));
+            }
+            rv = false;
+        }
+
+        return rv;
+    }
+
+    bool run_update(MYSQL* pConn)
+    {
+        bool rv = true;
+
+        string stmt("UPDATE test.t");
+        stmt += std::to_string(m_id);
+        stmt += " SET id=";
+        stmt += std::to_string(m_value);
+        stmt += " WHERE id=";
+        stmt += std::to_string(get_random_id());
+        m_value = (m_value + 1) % s_nRows;
+
+        if (mysql_query(pConn, stmt.c_str()) == 0)
+        {
+            flush_response(pConn);
+        }
+        else
+        {
+            if (m_verbose)
+            {
+                CMESSAGE("\"" << stmt << "\" failed: " << mysql_error(pConn));
+            }
+            rv = false;
+        }
+
+        return rv;
+    }
+
+    static void flush_response(MYSQL* pConn)
+    {
+        do
+        {
+            MYSQL_RES* pRes = mysql_store_result(pConn);
+            mysql_free_result(pRes);
+        }
+        while (mysql_next_result(pConn) == 0);
+    }
+
+    int get_random_id() const
+    {
+        int id = s_nRows * random_decimal_fraction();
+
+        ss_dassert(id >= 0);
+        ss_dassert(id <= (int)s_nRows);
+
+        return id;
+    }
+
+    double random_decimal_fraction() const
+    {
+        return m_rand_dist(m_rand_gen);
+    }
+
+    void run(const char* zHost, int port, const char* zUser, const char* zPassword)
+    {
+        do
+        {
+            MYSQL* pMysql = mysql_init(NULL);
+
+            if (pMysql)
+            {
+                unsigned int timeout = 5;
+                mysql_options(pMysql, MYSQL_OPT_CONNECT_TIMEOUT, &timeout);
+                mysql_options(pMysql, MYSQL_OPT_READ_TIMEOUT, &timeout);
+                mysql_options(pMysql, MYSQL_OPT_WRITE_TIMEOUT, &timeout);
+
+                if (m_verbose)
+                {
+                    CMESSAGE("Connecting");
+                }
+
+                if (mysql_real_connect(pMysql, zHost, zUser, zPassword, "test", port, NULL, 0))
+                {
+                    if (m_verbose)
+                    {
+                        CMESSAGE("Connected.");
+                    }
+
+                    while (!s_shutdown && run(pMysql))
+                    {
+                    }
+                }
+                else
+                {
+                    if (m_verbose)
+                    {
+                        CMESSAGE("mysql_real_connect() failed: " << mysql_error(pMysql));
+                    }
+                }
+
+                if (m_verbose)
+                {
+                    CMESSAGE("Closing");
+                }
+                mysql_close(pMysql);
+            }
+            else
+            {
+                CMESSAGE("mysql_init() failed.");
+            }
+
+            // To prevent some backend from becoming overwhelmed.
+            sleep(1);
+        }
+        while (!s_shutdown);
+    }
+
+    static void thread_main(int i,
+                            bool verbose,
+                            const char* zHost,
+                            int port,
+                            const char* zUser,
+                            const char* zPassword)
+    {
+        if (mysql_thread_init() == 0)
+        {
+            Client client(i, verbose);
+
+            client.run(zHost, port, zUser, zPassword);
+
+            mysql_thread_end();
+        }
+        else
+        {
+            int m_id = i;
+            CMESSAGE("mysql_thread_init() failed.");
+        }
+    }
+
+    static bool create_tables(TestConnections& test)
+    {
+        cout << "\nCreating tables." << endl;
+
+        MYSQL* pConn = test.maxscales->conn_rwsplit[0];
+
+        string drop_head("DROP TABLE IF EXISTS test.t");
+        string create_head("CREATE TABLE test.t");
+        string create_tail(" (id INT)");
+
+        for (size_t i = 0; i < s_nClients; ++i)
+        {
+            string drop = drop_head + std::to_string(i);
+            test.try_query(pConn, "%s", drop.c_str());
+
+            string create = create_head + std::to_string(i) + create_tail;
+            test.try_query(pConn, "%s", create.c_str());
+        }
+
+        return test.ok();
+    }
+
+    static bool insert_data(TestConnections& test)
+    {
+        cout << "\nInserting data." << endl;
+
+        MYSQL* pConn = test.maxscales->conn_rwsplit[0];
+
+        for (size_t i = 0; i < s_nClients; ++i)
+        {
+            string insert("insert into test.t");
+            insert += std::to_string(i);
+            insert += " values ";
+
+            for (size_t j = 0; j < s_nRows; ++j)
+            {
+                insert += "(";
+                insert += std::to_string(j);
+                insert += ")";
+
+                if (j < s_nRows - 1)
+                {
+                    insert += ", ";
+                }
+            }
+
+            test.try_query(pConn, "%s", insert.c_str());
+        }
+
+        return test.ok();
+    }
+
+private:
+    enum
+    {
+        INITSTATE_SIZE = 32
+    };
+
+    size_t                                         m_id;
+    bool                                           m_verbose;
+    size_t                                         m_value;
+    mutable std::mt19937                           m_rand_gen;
+    mutable std::uniform_real_distribution<double> m_rand_dist;
+
+    static size_t s_nClients;
+    static size_t s_nRows;
+    static bool   s_shutdown;
+
+    static std::vector<std::thread> s_threads;
+};
+
+size_t Client::s_nClients;
+size_t Client::s_nRows;
+bool Client::s_shutdown;
+std::vector<std::thread> Client::s_threads;
+}
+
+namespace
+{
+
+void list_servers(TestConnections& test)
+{
+    test.maxscales->execute_maxadmin_command_print(0, (char*)"list servers");
+}
+
+void sleep(int s)
+{
+    cout << "Sleeping " << s << " times 1 second" << flush;
+    do
+    {
+        ::sleep(1);
+        cout << "." << flush;
+        --s;
+    }
+    while (s > 0);
+
+    cout << endl;
+}
+
+bool check_server_status(TestConnections& test, int id)
+{
+    bool is_master = false;
+
+    Mariadb_nodes* pRepl = test.repl;
+
+    string server = string("server") + std::to_string(id);
+
+    StringSet statuses = test.get_server_status(server.c_str());
+    std::ostream_iterator<string> oi(cout, " ");
+
+    cout << server << ": ";
+    std::copy(statuses.begin(), statuses.end(), oi);
+
+    cout << " => ";
+
+    if (statuses.count("Master"))
+    {
+        is_master = true;
+        cout << "OK";
+    }
+    else if (statuses.count("Slave"))
+    {
+        cout << "OK";
+    }
+    else if (statuses.count("Running"))
+    {
+        MYSQL* pConn = pRepl->nodes[id - 1];
+
+        char result[1024];
+        if (find_field(pConn, "SHOW SLAVE STATUS", "Last_IO_Error", result) == 0)
+        {
+            const char needle[] =
+                ", which is not in the master's binlog. "
+                "Since the master's binlog contains GTIDs with higher sequence numbers, "
+                "it probably means that the slave has diverged due to executing extra "
+                "erroneous transactions";
+
+            if (strstr(result, needle))
+            {
+                // A rejoin was attempted, but it failed because the node (old master)
+                // had events that were not present in the new master. That is, a rejoin
+                // is not possible in principle without corrective action.
+                cout << "OK (could not be joined due to GTID issue)";
+            }
+            else
+            {
+                cout << result;
+                test.expect(false, "Merely 'Running' node did not error in expected way.");
+            }
+        }
+        else
+        {
+            test.expect(false, "Could not execute \"SHOW SLAVE STATUS\"");
+        }
+    }
+    else
+    {
+        test.expect(false, "Unexpected server state for %s.", server.c_str());
+    }
+
+    cout << endl;
+
+    return is_master;
+}
+
+void check_server_statuses(TestConnections& test)
+{
+    int masters = 0;
+
+    masters += check_server_status(test, 1);
+    masters += check_server_status(test, 2);
+    masters += check_server_status(test, 3);
+    masters += check_server_status(test, 4);
+
+    if (masters == 0)
+    {
+        test.global_result = 0;
+        test.tprintf("No master, checking that autofail has been turned off.");
+        test.log_includes(0, "disabling automatic failover");
+    }
+    else if (masters != 1)
+    {
+        test.expect(!true, "Unexpected number of masters: %d", masters);
+    }
+}
+
+bool is_valid_server_id(TestConnections& test, int id)
+{
+    std::set<int> ids;
+    test.repl->connect();
+
+    for (int i = 0; i < test.repl->N; i++)
+    {
+        ids.insert(test.repl->get_server_id(i));
+    }
+
+    test.repl->disconnect();
+    return ids.count(id);
+}
+
+void run(TestConnections& test)
+{
+    cout << "\nConnecting to MaxScale." << endl;
+    test.maxscales->connect_maxscale();
+
+    Client::init(test, Client::DEFAULT_N_CLIENTS, Client::DEFAULT_N_ROWS);
+
+    if (test.ok())
+    {
+        const char* zHost = test.maxscales->IP[0];
+        int port = test.maxscales->rwsplit_port[0];
+        const char* zUser = test.maxscales->user_name;
+        const char* zPassword = test.maxscales->password;
+
+        cout << "Connecting to " << zHost << ":" << port << " as " << zUser << ":" << zPassword << endl;
+        cout << "Starting clients." << endl;
+        Client::start(test.verbose, zHost, port, zUser, zPassword);
+
+        list_servers(test);
+
+        for (int i = 0; i < 2; i++)
+        {
+            test.set_timeout(20);
+            test.maxscales->wait_for_monitor();
+
+            int master_id = get_master_server_id(test);
+
+            if (is_valid_server_id(test, master_id))
+            {
+                test.set_timeout(20);
+                cout << "\nStopping node: " << master_id << endl;
+                test.repl->stop_node(master_id - 1);
+
+                test.maxscales->wait_for_monitor();
+                list_servers(test);
+
+                test.maxscales->wait_for_monitor();
+                list_servers(test);
+
+                test.set_timeout(20);
+                test.maxscales->wait_for_monitor();
+                cout << "\nStarting node: " << master_id << endl;
+                test.repl->start_node(master_id - 1);
+
+                test.maxscales->wait_for_monitor();
+                list_servers(test);
+
+                test.maxscales->wait_for_monitor();
+                list_servers(test);
+            }
+            else
+            {
+                test.expect(false, "Unexpected master id: %d", master_id);
+            }
+        }
+
+        test.maxscales->wait_for_monitor();
+
+        cout << "\nStopping clients.\n" << flush;
+        Client::stop();
+
+        test.repl->close_connections();
+        test.repl->connect();
+
+        check_server_statuses(test);
+    }
+}
+}
+
+int main(int argc, char* argv[])
+{
+    Mariadb_nodes::require_gtid(true);
+    TestConnections test(argc, argv);
+
+    run(test);
+
+    return test.global_result;
+}

maxscale-system-test/mariadbmonitor/mysqlmon_multimaster.cnf

@@ -0,0 +1,89 @@
+[maxscale]
+threads=###threads###
+log_warning=1
+
+[MySQL Monitor]
+type=monitor
+module=mysqlmon
+servers= server1, server2, server3, server4
+user=maxskysql
+password= skysql
+detect_stale_master=0
+monitor_interval=1000
+
+[RW Split Router]
+type=service
+router= readwritesplit
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+slave_selection_criteria=LEAST_ROUTER_CONNECTIONS
+max_slave_replication_lag=1
+
+[Read Connection Router Slave]
+type=service
+router=readconnroute
+router_options= slave
+servers=server1,server2
+user=maxskysql
+password=skysql
+
+[Read Connection Router Master]
+type=service
+router=readconnroute
+router_options=master
+servers=server1,server2
+user=maxskysql
+password=skysql
+
+[RW Split Listener]
+type=listener
+service=RW Split Router
+protocol=MySQLClient
+port=4006
+
+[Read Connection Listener Slave]
+type=listener
+service=Read Connection Router Slave
+protocol=MySQLClient
+port=4009
+
+[Read Connection Listener Master]
+type=listener
+service=Read Connection Router Master
+protocol=MySQLClient
+port=4008
+
+[CLI]
+type=service
+router=cli
+
+[CLI Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=default
+
+[server1]
+type=server
+address=###node_server_IP_1###
+port=###node_server_port_1###
+protocol=MySQLBackend
+
+[server2]
+type=server
+address=###node_server_IP_2###
+port=###node_server_port_2###
+protocol=MySQLBackend
+
+[server3]
+type=server
+address=###node_server_IP_3###
+port=###node_server_port_3###
+protocol=MySQLBackend
+
+[server4]
+type=server
+address=###node_server_IP_4###
+port=###node_server_port_4###
+protocol=MySQLBackend

maxscale-system-test/mariadbmonitor/mysqlmon_multimaster.cpp

@@ -0,0 +1,370 @@
+/**
+ * @file mysqlmon_multimaster.cpp MySQL Monitor Multi-master Test
+ * - Configure all servers into a multi-master ring with one slave
+ * - check status using Maxadmin 'show servers' and 'show monitor "MySQL Monitor"'
+ * - Set nodes 0 and 1 into read-only mode
+ * - repeat status check
+ * - Configure nodes 1 and 2 (server2 and server3) into a master-master pair, make node 0 a slave of node 1
+ * and node 3 a slave of node 2
+ * - repeat status check
+ * - Set node 1 into read-only mode
+ * - repeat status check
+ * - Create two distinct groups (server1 and server2 are masters for eache others and same for server3 and
+ * server4)
+ * - repeat status check
+ * - Set nodes 1 and 3 (server2 and server4) into read-only mode
+ *
+ * Addition: add delays to some slave connections and check that the monitor correctly detects the delay
+ */
+
+
+#include <iostream>
+#include "testconnections.h"
+#include "maxadmin_operations.h"
+#include "sql_t1.h"
+#include <jansson.h>
+#include <string>
+
+using std::cout;
+using std::string;
+
+void check_status(TestConnections& test, const char* server, const char* status)
+{
+    char cmd[256];
+    char maxadmin_result[1024];
+
+    sprintf(cmd, "show server %s", server);
+    test.maxscales->get_maxadmin_param(0, cmd, (char*)"Status:", maxadmin_result);
+
+    if (maxadmin_result == NULL)
+    {
+        test.add_result(1, "maxadmin execution error\n");
+        return;
+    }
+
+    if (strstr(maxadmin_result, status) == NULL)
+    {
+        test.add_result(1, "Test failed, server '%s' status is '%s', expected '%s'\n",
+                        server, maxadmin_result, status);
+    }
+}
+
+json_t* get_json_data(TestConnections& test, const char* query)
+{
+    json_t* rval = NULL;
+    int exit_code = 1;
+    char* output = test.maxscales->ssh_node_output(0, query, true, &exit_code);
+    if (output == NULL)
+    {
+        test.add_result(1, "Query '%s' execution error, no output.", query);
+    }
+    else
+    {
+        json_error_t error;
+        rval = json_loads(output, 0, &error);
+        free(output);
+        if (rval == NULL)
+        {
+            test.add_result(1, "JSON decode error: %s\n", error.text);
+        }
+    }
+    return rval;
+}
+
+json_t* traverse_json(TestConnections& test, json_t* object, const std::vector<string>& keys)
+{
+    test.expect(object, "JSON object is NULL\n");
+
+    json_t* current_object = object;
+    for (auto iter = keys.begin(); iter != keys.end() && current_object; ++iter)
+    {
+        current_object = json_object_get(current_object, (*iter).c_str());
+        test.expect(current_object, "Key %s was not found in json data.\n", (*iter).c_str());
+    }
+    return current_object;
+}
+
+json_t* find_array_elem_json(TestConnections& test, json_t* object,
+                             std::string key, std::string expected_val)
+{
+    bool is_array = json_is_array(object);
+    test.expect(is_array, "JSON object is not an array\n");
+    json_t* found_elem = NULL;
+
+    if (is_array)
+    {
+        size_t arr_size = json_array_size(object);
+        for (size_t i = 0; i < arr_size; i++)
+        {
+            json_t* arr_elem = json_array_get(object, i);
+            json_t* elem_val = json_object_get(arr_elem, key.c_str());
+            bool is_string = json_is_string(elem_val);
+            test.expect(is_string, "Key %s was not found in json data or the data is not string.\n", key.c_str());
+            if (is_string)
+            {
+                std::string elem_field = json_string_value(elem_val);
+                if (elem_field == expected_val)
+                {
+                    found_elem = arr_elem;
+                    break;
+                }
+            }
+        }
+
+        test.expect(found_elem, "Array element with %s->%s was not found in json array\n",
+                    key.c_str(), expected_val.c_str());
+    }
+    return found_elem;
+}
+
+void check_group(TestConnections& test, const char* server, int expected_group)
+{
+    json_t* monitor_data = get_json_data(test, "maxctrl api get monitors/MySQL-Monitor");
+    if (monitor_data == NULL)
+    {
+        return;
+    }
+
+    monitor_data = traverse_json(test, monitor_data,
+                                 {"data", "attributes", "monitor_diagnostics", "server_info"});
+    if (monitor_data)
+    {
+        auto server_data = find_array_elem_json(test, monitor_data, "name", server);
+        if (server_data)
+        {
+            int found_group = json_integer_value(json_object_get(server_data, "master_group"));
+            test.expect(found_group == expected_group, "Server '%s', expected group '%d', not '%d'",
+                        server, expected_group, found_group);
+        }
+    }
+}
+
+void check_rlag(TestConnections& test, const char* server, int min_rlag, int max_rlag)
+{
+    json_t* object_data = get_json_data(test, "maxctrl api get servers");
+    if (object_data)
+    {
+        auto servers_data = traverse_json(test, object_data, {"data"});
+        auto server_data = find_array_elem_json(test, servers_data, "id", server);
+        auto rlag_object = traverse_json(test, server_data, {"attributes", "replication_lag"});
+        if (rlag_object)
+        {
+            int found_rlag = json_integer_value(rlag_object);
+            if (found_rlag >= min_rlag && found_rlag <= max_rlag)
+            {
+                cout << "Replication lag of " << server << " is " << found_rlag << " seconds.\n";
+            }
+            else
+            {
+                test.expect(false,
+                            "Replication lag of %s is out of bounds: value: %i min: %i max: %i\n",
+                            server, found_rlag, min_rlag, max_rlag);
+            }
+        }
+    }
+}
+
+void change_master(TestConnections& test ,int slave, int master, const string& conn_name = "",
+                   int replication_delay = 0)
+{
+    const char query[] = "CHANGE MASTER '%s' TO master_host='%s', master_port=%d, "
+                         "master_log_file='mar-bin.000001', master_log_pos=4, "
+                         "master_user='repl', master_password='repl', master_delay=%d; "
+                         "START SLAVE '%s';";
+    test.try_query(test.repl->nodes[slave], query, conn_name.c_str(),
+           test.repl->IP_private[master], test.repl->port[master],
+                   replication_delay, conn_name.c_str());
+}
+
+int main(int argc, char* argv[])
+{
+    const char mm_master_states[] = "Master, Running";
+    const char mm_slave_states[] = "Relay Master, Slave, Running";
+    const char slave_states[] = "Slave, Running";
+    const char running_state[] = "Running";
+    const char reset_query[] = "STOP SLAVE; RESET SLAVE ALL; RESET MASTER; SET GLOBAL read_only='OFF'";
+    const char readonly_on_query[] = "SET GLOBAL read_only='ON'";
+
+    TestConnections::require_repl_version("10.2.3"); // Delayed replication needs this.
+    TestConnections test(argc, argv);
+
+    test.tprintf("Test 1 - Configure all servers into a multi-master ring with one slave");
+    int max_rlag = 100;
+    test.set_timeout(120);
+    test.repl->execute_query_all_nodes(reset_query);
+    test.repl->connect();
+    change_master(test, 0, 1);
+    change_master(test, 1, 2);
+    change_master(test, 2, 0);
+    change_master(test, 3, 2, "", max_rlag);
+
+    test.maxscales->wait_for_monitor(2);
+    auto maxconn = test.maxscales->open_rwsplit_connection();
+    test.try_query(maxconn, "FLUSH TABLES;");
+    test.maxscales->wait_for_monitor(1);
+
+    check_status(test, "server1", mm_master_states);
+    check_status(test, "server2", mm_slave_states);
+    check_status(test, "server3", mm_slave_states);
+    check_status(test, "server4", slave_states);
+    check_group(test, "server1", 1);
+    check_group(test, "server2", 1);
+    check_group(test, "server3", 1);
+    check_group(test, "server4", 0);
+    check_rlag(test, "server4", 1, max_rlag);
+    // Need to send a read query so that rwsplit detects replication lag.
+    test.try_query(maxconn, "SHOW DATABASES;");
+    mysql_close(maxconn);
+    test.log_includes(0, "is excluded from query routing.");
+
+    test.tprintf("Test 2 - Set nodes 0 and 1 into read-only mode");
+
+    test.set_timeout(120);
+    execute_query(test.repl->nodes[0], readonly_on_query);
+    execute_query(test.repl->nodes[1], readonly_on_query);
+    test.maxscales->wait_for_monitor(1);
+
+    check_status(test, "server1", mm_slave_states);
+    check_status(test, "server2", mm_slave_states);
+    check_status(test, "server3", mm_master_states);
+    check_status(test, "server4", slave_states);
+    check_group(test, "server1", 1);
+    check_group(test, "server2", 1);
+    check_group(test, "server3", 1);
+    check_group(test, "server4", 0);
+    check_rlag(test, "server4", 1, max_rlag);
+
+    test.tprintf("Test 3 - Configure nodes 1 and 2 into a master-master pair, make node 0 "
+                 "a slave of node 1 and node 3 a slave of node 2");
+
+    test.set_timeout(120);
+    test.repl->execute_query_all_nodes(reset_query);
+    test.repl->connect();
+    change_master(test, 0, 1);
+    change_master(test, 1, 2);
+    change_master(test, 2, 1, "", max_rlag);
+    change_master(test, 3, 2);
+
+    test.maxscales->wait_for_monitor(1);
+    maxconn = test.maxscales->open_rwsplit_connection();
+    test.try_query(maxconn, "FLUSH TABLES;");
+    mysql_close(maxconn);
+    test.maxscales->wait_for_monitor(1);
+
+    check_status(test, "server1", slave_states);
+    check_status(test, "server2", mm_master_states);
+    check_status(test, "server3", mm_slave_states);
+    check_status(test, "server4", slave_states);
+    check_group(test, "server1", 0);
+    check_group(test, "server2", 1);
+    check_group(test, "server3", 1);
+    check_group(test, "server4", 0);
+    check_rlag(test, "server3", 1, max_rlag);
+
+    test.tprintf("Test 4 - Set node 1 into read-only mode");
+
+    test.set_timeout(120);
+    execute_query(test.repl->nodes[1], readonly_on_query);
+    test.maxscales->wait_for_monitor(1);
+
+    check_status(test, "server1", slave_states);
+    check_status(test, "server2", mm_slave_states);
+    check_status(test, "server3", mm_master_states);
+    check_status(test, "server4", slave_states);
+    check_group(test, "server1", 0);
+    check_group(test, "server2", 1);
+    check_group(test, "server3", 1);
+    check_group(test, "server4", 0);
+
+    test.tprintf("Test 5 - Create two distinct groups");
+
+    test.set_timeout(120);
+    test.repl->execute_query_all_nodes(reset_query);
+    test.repl->connect();
+    change_master(test, 0, 1);
+    change_master(test, 1, 0);
+    change_master(test, 2, 3);
+    change_master(test, 3, 2);
+
+    test.maxscales->wait_for_monitor(1);
+
+    // Even though the servers are in two distinct groups, only one of them
+    // contains a master and a slave. Only one master may exist in a cluster
+    // at once, since by definition this is the server to which routers may
+    // direct writes.
+    check_status(test, "server1", mm_master_states);
+    check_status(test, "server2", mm_slave_states);
+    check_status(test, "server3", running_state);
+    check_status(test, "server4", running_state);
+    check_group(test, "server1", 1);
+    check_group(test, "server2", 1);
+    check_group(test, "server3", 2);
+    check_group(test, "server4", 2);
+
+    test.tprintf("Test 6 - Set nodes 1 and 3 into read-only mode");
+
+    test.set_timeout(120);
+    execute_query(test.repl->nodes[1], readonly_on_query);
+    execute_query(test.repl->nodes[3], readonly_on_query);
+
+    test.maxscales->wait_for_monitor(1);
+
+    check_status(test, "server1", mm_master_states);
+    check_status(test, "server2", mm_slave_states);
+    check_status(test, "server3", running_state);
+    check_status(test, "server4", running_state);
+    check_group(test, "server1", 1);
+    check_group(test, "server2", 1);
+    check_group(test, "server3", 2);
+    check_group(test, "server4", 2);
+
+    test.tprintf("Test 7 - Diamond topology with delay");
+
+    test.repl->execute_query_all_nodes(reset_query);
+    test.repl->connect();
+    change_master(test, 0, 1, "a", max_rlag);
+    change_master(test, 0, 2, "b", max_rlag);
+    change_master(test, 1, 3);
+    change_master(test, 2, 3);
+
+    test.maxscales->wait_for_monitor(1);
+    maxconn = test.maxscales->open_rwsplit_connection();
+    test.try_query(maxconn, "FLUSH TABLES;");
+    test.maxscales->wait_for_monitor(1);
+
+    check_status(test, "server1", slave_states);
+    check_status(test, "server2", mm_slave_states);
+    check_status(test, "server3", mm_slave_states);
+    check_status(test, "server4", mm_master_states);
+    check_group(test, "server1", 0);
+    check_group(test, "server2", 0);
+    check_group(test, "server3", 0);
+    check_group(test, "server4", 0);
+    check_rlag(test, "server1", 1, max_rlag);
+
+    test.tprintf("Test 8 - Diamond topology with no delay");
+
+    const char remove_delay[] = "STOP SLAVE '%s'; CHANGE MASTER '%s' TO master_delay=0; START SLAVE '%s';";
+    test.try_query(test.repl->nodes[0], remove_delay, "a", "a", "a");
+    test.maxscales->wait_for_monitor(1);
+
+    check_status(test, "server1", slave_states);
+    check_rlag(test, "server1", 0, 0);
+    // Rwsplit should detects that replication lag is 0.
+    test.try_query(maxconn, "SHOW DATABASES;");
+    mysql_close(maxconn);
+    test.log_includes(0, "is returned to query routing.");
+
+    // Test over, reset topology.
+    const char reset_with_name[] = "STOP SLAVE '%s'; RESET SLAVE '%s' ALL;";
+    test.try_query(test.repl->nodes[0], reset_with_name, "a", "a");
+    test.try_query(test.repl->nodes[0], reset_with_name, "b", "b");
+
+    test.repl->execute_query_all_nodes(reset_query);
+    test.repl->connect();
+    change_master(test, 1, 0);
+    change_master(test, 2, 0);
+    change_master(test, 3, 0);
+
+    return test.global_result;
+}

maxscale-system-test/mariadbmonitor/mysqlmon_multimaster_serverid.cnf

@@ -0,0 +1,90 @@
+[maxscale]
+threads=###threads###
+log_warning=1
+
+[MySQL Monitor]
+type=monitor
+module=mysqlmon
+servers= server1, server2, server3, server4
+user=maxskysql
+password= skysql
+detect_stale_master=0
+monitor_interval=1000
+assume_unique_hostnames=false
+
+[RW Split Router]
+type=service
+router= readwritesplit
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+slave_selection_criteria=LEAST_ROUTER_CONNECTIONS
+max_slave_replication_lag=1
+
+[Read Connection Router Slave]
+type=service
+router=readconnroute
+router_options= slave
+servers=server1,server2
+user=maxskysql
+password=skysql
+
+[Read Connection Router Master]
+type=service
+router=readconnroute
+router_options=master
+servers=server1,server2
+user=maxskysql
+password=skysql
+
+[RW Split Listener]
+type=listener
+service=RW Split Router
+protocol=MySQLClient
+port=4006
+
+[Read Connection Listener Slave]
+type=listener
+service=Read Connection Router Slave
+protocol=MySQLClient
+port=4009
+
+[Read Connection Listener Master]
+type=listener
+service=Read Connection Router Master
+protocol=MySQLClient
+port=4008
+
+[CLI]
+type=service
+router=cli
+
+[CLI Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=default
+
+[server1]
+type=server
+address=###node_server_IP_1###
+port=###node_server_port_1###
+protocol=MySQLBackend
+
+[server2]
+type=server
+address=###node_server_IP_2###
+port=###node_server_port_2###
+protocol=MySQLBackend
+
+[server3]
+type=server
+address=###node_server_IP_3###
+port=###node_server_port_3###
+protocol=MySQLBackend
+
+[server4]
+type=server
+address=###node_server_IP_4###
+port=###node_server_port_4###
+protocol=MySQLBackend

maxscale-system-test/mariadbmonitor/mysqlmon_rejoin_bad.cpp

@@ -0,0 +1,144 @@
+/*
+ * Copyright (c) 2016 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2024-03-10
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#include <iostream>
+#include <vector>
+
+#include "testconnections.h"
+#include "fail_switch_rejoin_common.cpp"
+
+using std::string;
+using std::cout;
+
+int main(int argc, char** argv)
+{
+    interactive = strcmp(argv[argc - 1], "interactive") == 0;
+    Mariadb_nodes::require_gtid(true);
+    TestConnections test(argc, argv);
+    test.repl->connect();
+    // Delete binlogs to sync gtid:s
+    delete_slave_binlogs(test);
+    // Set up test table
+    basic_test(test);
+    // Advance gtid:s a bit to so gtid variables are updated.
+    MYSQL* maxconn = test.maxscales->open_rwsplit_connection(0);
+    generate_traffic_and_check(test, maxconn, 10);
+    test.repl->sync_slaves(0);
+
+    test.tprintf(LINE);
+    print_gtids(test);
+    test.tprintf(LINE);
+    char result_tmp[bufsize];
+    string gtid_begin;
+    if (find_field(maxconn, GTID_QUERY, GTID_FIELD, result_tmp) == 0)
+    {
+        gtid_begin = result_tmp;
+    }
+    mysql_close(maxconn);
+
+    test.tprintf("Stopping MaxScale...");
+    // Mess with the slaves to fix situation such that only one slave can be rejoined. Stop maxscale.
+    if (test.stop_maxscale(0) != 0)
+    {
+        test.expect(false, "Could not stop MaxScale.");
+        return test.global_result;
+    }
+
+    // Leave first of three slaves connected so it's clear which one is the master server.
+    const char STOP_SLAVE[] = "STOP SLAVE;";
+    const char RESET_SLAVE[] = "RESET SLAVE ALL;";
+    const char READ_ONLY_OFF[] = "SET GLOBAL read_only=0;";
+
+    const int FIRST_MOD_NODE = 2;   // Modify nodes 2 & 3
+    const int NODE_COUNT = test.repl->N;
+    MYSQL** nodes = test.repl->nodes;
+    for (int i = FIRST_MOD_NODE; i < NODE_COUNT; i++)
+    {
+        if ((mysql_query(nodes[i], STOP_SLAVE) != 0) || (mysql_query(nodes[i], RESET_SLAVE) != 0)
+            || (mysql_query(nodes[i], READ_ONLY_OFF) != 0))
+        {
+            test.expect(false, "Could not stop slave connections and/or disable read_only for node %d.", i);
+        }
+    }
+
+    if (test.ok())
+    {
+        // Add more events to node3.
+        string gtid_node2, gtid_node3;
+        test.tprintf("Sending more inserts to server 4.");
+        generate_traffic_and_check(test, nodes[3], 10);
+        // Save gtids
+        if (find_field(nodes[2], GTID_QUERY, GTID_FIELD, result_tmp) == 0)
+        {
+            gtid_node2 = result_tmp;
+        }
+        if (find_field(nodes[3], GTID_QUERY, GTID_FIELD, result_tmp) == 0)
+        {
+            gtid_node3 = result_tmp;
+        }
+        print_gtids(test);
+        bool gtids_ok = (gtid_begin == gtid_node2 && gtid_node2 < gtid_node3);
+        test.expect(gtids_ok, "Gtid:s have not advanced correctly.");
+    }
+
+
+    test.tprintf("Restarting MaxScale. Server 4 should not rejoin the cluster.");
+    test.tprintf(LINE);
+    if (test.start_maxscale(0) != 0)
+    {
+        test.expect(false, "Could not start MaxScale.");
+        return test.global_result;
+    }
+    test.maxscales->wait_for_monitor(2);
+    get_output(test);
+
+    if (test.ok())
+    {
+        StringSet node2_states = test.get_server_status("server3");
+        StringSet node3_states = test.get_server_status("server4");
+        bool states_n2_ok = (node2_states.find("Slave") != node2_states.end());
+        bool states_n3_ok = (node3_states.find("Slave") == node3_states.end());
+        test.expect(states_n2_ok, "Node 2 has not rejoined when it should have.");
+        test.expect(states_n3_ok, "Node 3 rejoined when it shouldn't have.");
+    }
+
+    if (test.ok())
+    {
+        // Finally, fix replication by telling the current master to replicate from server4
+        test.tprintf(
+            "Setting server 1 to replicate from server 4. Auto-rejoin should redirect servers 2 and 3.");
+        const char CHANGE_CMD_FMT[] = "CHANGE MASTER TO MASTER_HOST = '%s', MASTER_PORT = %d, "
+                                      "MASTER_USE_GTID = current_pos, MASTER_USER='repl', MASTER_PASSWORD = 'repl';";
+        char change_cmd[256];
+        snprintf(change_cmd, sizeof(change_cmd), CHANGE_CMD_FMT, test.repl->IP_private[3], test.repl->port[3]);
+        test.try_query(nodes[0], "%s", change_cmd);
+        test.try_query(nodes[0], "START SLAVE;");
+        test.maxscales->wait_for_monitor(2);
+        get_output(test);
+        int master_id = get_master_server_id(test);
+        test.expect(master_id == 4, "Server 4 should be the cluster master.");
+        StringSet node0_states = test.get_server_status("server1");
+        bool states_n0_ok = (node0_states.find("Slave") != node0_states.end()
+                             && node0_states.find("Relay Master") == node0_states.end());
+        test.expect(states_n0_ok, "Server 1 is not a slave when it should be.");
+    }
+
+    cout << "Reseting cluster...\n";
+    int ec;
+    string reset_cmd = "maxadmin call command mysqlmon reset-replication MySQL-Monitor server1";
+    test.maxscales->ssh_node_output(0, reset_cmd.c_str(), true, &ec);
+    test.maxscales->wait_for_monitor(1);
+    test.expect(get_master_server_id(test) == 1, "server1 is not the master when it should. "
+                                                 "reset-replication must have failed.");
+    return test.global_result;
+}

maxscale-system-test/mariadbmonitor/mysqlmon_rejoin_bad2.cpp

@@ -0,0 +1,154 @@
+/*
+ * Copyright (c) 2016 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2024-03-10
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#include <vector>
+#include <iostream>
+#include <iterator>
+
+#include "testconnections.h"
+#include "fail_switch_rejoin_common.cpp"
+
+using std::string;
+using std::cout;
+using std::endl;
+
+static void expect(TestConnections& test, const char* zServer, const StringSet& expected)
+{
+    StringSet found = test.get_server_status(zServer);
+
+    std::ostream_iterator<string> oi(cout, ", ");
+
+    cout << zServer << ", expected states: ";
+    std::copy(expected.begin(), expected.end(), oi);
+    cout << endl;
+
+    cout << zServer << ", found states   : ";
+    std::copy(found.begin(), found.end(), oi);
+    cout << endl;
+
+    if (found != expected)
+    {
+        cout << "ERROR, found states are not the same as the expected ones." << endl;
+        ++test.global_result;
+    }
+
+    cout << endl;
+}
+
+static void expect(TestConnections& test, const char* zServer, const char* zState)
+{
+    StringSet s;
+    s.insert(zState);
+    expect(test, zServer, s);
+}
+
+static void expect(TestConnections& test, const char* zServer, const char* zState1, const char* zState2)
+{
+    StringSet s;
+    s.insert(zState1);
+    s.insert(zState2);
+    expect(test, zServer, s);
+}
+
+int main(int argc, char** argv)
+{
+    interactive = strcmp(argv[argc - 1], "interactive") == 0;
+    Mariadb_nodes::require_gtid(true);
+    TestConnections test(argc, argv);
+    test.repl->connect();
+
+    // Delete binlogs to sync gtid:s
+    delete_slave_binlogs(test);
+    // Set up test table
+    basic_test(test);
+
+    MYSQL* maxconn = test.maxscales->open_rwsplit_connection(0);
+    // Advance gtid:s a bit to so gtid variables are updated.
+    generate_traffic_and_check(test, maxconn, 5);
+    test.repl->sync_slaves(0);
+    get_output(test);
+
+    print_gtids(test);
+    get_input();
+    mysql_close(maxconn);
+
+    // Stop master, wait for failover
+    cout << "Stopping master, should auto-failover." << endl;
+    int master_id_old = get_master_server_id(test);
+    test.repl->stop_node(0);
+    test.maxscales->wait_for_monitor(2);
+    get_output(test);
+    int master_id_new = get_master_server_id(test);
+    cout << "Master server id is " << master_id_new << endl;
+    test.expect(master_id_new > 0 && master_id_new != master_id_old,
+                "Failover did not promote a new master.");
+    if (test.global_result != 0)
+    {
+        return test.global_result;
+    }
+
+    cout << "Stopping MaxScale for a moment.\n";
+    // Stop maxscale to prevent an unintended rejoin.
+    if (test.stop_maxscale(0))
+    {
+        test.expect(false, "Could not stop MaxScale.");
+        return test.global_result;
+    }
+
+    // Restart old master. Then add some events to it.
+    test.repl->start_node(0, (char*)"");
+    test.repl->connect();
+    cout << "Adding more events to node 0.\n";
+    generate_traffic_and_check(test, test.repl->nodes[0], 5);
+    print_gtids(test);
+
+    cout << "Starting MaxScale, node 0 should not be able to join because it has extra events.\n";
+    // Restart maxscale. Should not rejoin old master.
+    if (test.start_maxscale(0))
+    {
+        test.expect(false, "Could not start MaxScale.");
+        return test.global_result;
+    }
+    test.maxscales->wait_for_monitor(2);
+    get_output(test);
+
+    expect(test, "server1", "Running");
+    if (test.global_result != 0)
+    {
+        cout << "Old master is a member of the cluster when it should not be. \n";
+        return test.global_result;
+    }
+
+    // Set current master to replicate from the old master. The new master should lose its Master status
+    // and auto_rejoin will redirect servers 3 and 4 so that all replicate from server 1.
+    cout << "Setting server " << master_id_new << " to replicate from server 1. Server " << master_id_new
+         << " should lose its master status and other servers should be redirected to server 1.\n";
+    const char CHANGE_CMD_FMT[] = "CHANGE MASTER TO MASTER_HOST = '%s', MASTER_PORT = %d, "
+                                  "MASTER_USE_GTID = current_pos, "
+                                  "MASTER_USER='repl', MASTER_PASSWORD = 'repl';";
+    char cmd[256];
+    int ind = master_id_new - 1;
+    snprintf(cmd, sizeof(cmd), CHANGE_CMD_FMT, test.repl->IP_private[0], test.repl->port[0]);
+    MYSQL** nodes = test.repl->nodes;
+    mysql_query(nodes[ind], cmd);
+    mysql_query(nodes[ind], "START SLAVE;");
+    test.maxscales->wait_for_monitor(2);
+    get_output(test);
+
+    expect(test, "server1", "Master", "Running");
+    expect(test, "server2", "Slave", "Running");
+    expect(test, "server3", "Slave", "Running");
+    expect(test, "server4", "Slave", "Running");
+    test.repl->fix_replication();
+    return test.global_result;
+}

maxscale-system-test/mariadbmonitor/mysqlmon_rejoin_good.cnf

@@ -0,0 +1,95 @@
+[maxscale]
+threads=###threads###
+
+[MySQL-Monitor]
+type=monitor
+module=mysqlmon
+servers= server1, server2, server3, server4
+user=maxskysql
+password= skysql
+monitor_interval=1000
+detect_standalone_master=true
+failcount=1
+allow_cluster_recovery=true
+replication_user=repl
+replication_password=repl
+backend_connect_timeout=10
+backend_read_timeout=10
+backend_write_timeout=10
+auto_failover=true
+auto_rejoin=true
+
+[RW Split Router]
+type=service
+router= readwritesplit
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[Read Connection Router Slave]
+type=service
+router=readconnroute
+router_options= slave
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[Read Connection Router Master]
+type=service
+router=readconnroute
+router_options=master
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[RW Split Listener]
+type=listener
+service=RW Split Router
+protocol=MySQLClient
+port=4006
+
+[Read Connection Listener Slave]
+type=listener
+service=Read Connection Router Slave
+protocol=MySQLClient
+port=4009
+
+[Read Connection Listener Master]
+type=listener
+service=Read Connection Router Master
+protocol=MySQLClient
+port=4008
+
+[CLI]
+type=service
+router=cli
+
+[CLI Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=default
+
+[server1]
+type=server
+address=###node_server_IP_1###
+port=###node_server_port_1###
+protocol=MySQLBackend
+
+[server2]
+type=server
+address=###node_server_IP_2###
+port=###node_server_port_2###
+protocol=MySQLBackend
+
+[server3]
+type=server
+address=###node_server_IP_3###
+port=###node_server_port_3###
+protocol=MySQLBackend
+
+[server4]
+type=server
+address=###node_server_IP_4###
+port=###node_server_port_4###
+protocol=MySQLBackend

maxscale-system-test/mariadbmonitor/mysqlmon_rejoin_good.cpp

@@ -0,0 +1,100 @@
+/*
+ * Copyright (c) 2016 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2024-03-10
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#include "fail_switch_rejoin_common.cpp"
+
+using std::string;
+
+int main(int argc, char** argv)
+{
+    interactive = strcmp(argv[argc - 1], "interactive") == 0;
+    Mariadb_nodes::require_gtid(true);
+    TestConnections test(argc, argv);
+    MYSQL* maxconn = test.maxscales->open_rwsplit_connection(0);
+    // Set up test table
+    basic_test(test);
+    // Delete binlogs to sync gtid:s
+    delete_slave_binlogs(test);
+    char result_tmp[bufsize];
+    // Advance gtid:s a bit to so gtid variables are updated.
+    generate_traffic_and_check(test, maxconn, 10);
+    test.maxscales->wait_for_monitor();
+    test.tprintf(LINE);
+    print_gtids(test);
+    get_input();
+
+    test.tprintf("Stopping master and waiting for failover. Check that another server is promoted.");
+    test.tprintf(LINE);
+    const int old_master_id = get_master_server_id(test);   // Read master id now before shutdown.
+    const int master_index = test.repl->master;
+    test.repl->stop_node(master_index);
+    test.maxscales->wait_for_monitor();
+    // Recreate maxscale session
+    mysql_close(maxconn);
+    maxconn = test.maxscales->open_rwsplit_connection(0);
+    get_output(test);
+    int master_id = get_master_server_id(test);
+    test.tprintf(LINE);
+    test.tprintf(PRINT_ID, master_id);
+    const bool failover_ok = (master_id > 0 && master_id != old_master_id);
+    test.expect(failover_ok, "Master did not change or no master detected.");
+    string gtid_final;
+    if (failover_ok)
+    {
+        test.tprintf("Sending more inserts.");
+        generate_traffic_and_check(test, maxconn, 5);
+        test.maxscales->wait_for_monitor();
+        if (find_field(maxconn, GTID_QUERY, GTID_FIELD, result_tmp) == 0)
+        {
+            gtid_final = result_tmp;
+        }
+        print_gtids(test);
+        test.tprintf("Bringing old master back online. It should rejoin the cluster and catch up in events.");
+        test.tprintf(LINE);
+
+        test.repl->start_node(master_index, (char*) "");
+        test.maxscales->wait_for_monitor();
+        get_output(test);
+
+        test.repl->connect();
+        test.maxscales->wait_for_monitor();
+        string gtid_old_master;
+        if (find_field(test.repl->nodes[master_index], GTID_QUERY, GTID_FIELD, result_tmp) == 0)
+        {
+            gtid_old_master = result_tmp;
+        }
+        test.tprintf(LINE);
+        print_gtids(test);
+        test.tprintf(LINE);
+        test.expect(gtid_final == gtid_old_master, "Old master did not successfully rejoin the cluster.");
+        // Switch master back to server1 so last check is faster
+        int ec;
+        test.maxscales->ssh_node_output(0,
+                                        "maxadmin call command mysqlmon switchover "
+                                        "MySQL-Monitor server1 server2",
+                                        true,
+                                        &ec);
+        test.maxscales->wait_for_monitor();     // Wait for monitor to update status
+        get_output(test);
+        master_id = get_master_server_id(test);
+        test.expect(master_id == old_master_id, "Switchover back to server1 failed.");
+    }
+    else
+    {
+        test.repl->start_node(master_index, (char*) "");
+        test.maxscales->wait_for_monitor();
+    }
+
+    test.repl->fix_replication();
+    return test.global_result;
+}

maxscale-system-test/mariadbmonitor/mysqlmon_rejoin_manual.cnf

@@ -0,0 +1,94 @@
+[maxscale]
+threads=###threads###
+
+[MySQL-Monitor]
+type=monitor
+module=mysqlmon
+servers= server1, server2, server3, server4
+user=maxskysql
+password= skysql
+monitor_interval=1000
+detect_standalone_master=true
+failcount=1
+allow_cluster_recovery=true
+replication_user=repl
+replication_password=repl
+backend_connect_timeout=10
+backend_read_timeout=10
+backend_write_timeout=10
+auto_failover=true
+
+[RW Split Router]
+type=service
+router= readwritesplit
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[Read Connection Router Slave]
+type=service
+router=readconnroute
+router_options= slave
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[Read Connection Router Master]
+type=service
+router=readconnroute
+router_options=master
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[RW Split Listener]
+type=listener
+service=RW Split Router
+protocol=MySQLClient
+port=4006
+
+[Read Connection Listener Slave]
+type=listener
+service=Read Connection Router Slave
+protocol=MySQLClient
+port=4009
+
+[Read Connection Listener Master]
+type=listener
+service=Read Connection Router Master
+protocol=MySQLClient
+port=4008
+
+[CLI]
+type=service
+router=cli
+
+[CLI Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=default
+
+[server1]
+type=server
+address=###node_server_IP_1###
+port=###node_server_port_1###
+protocol=MySQLBackend
+
+[server2]
+type=server
+address=###node_server_IP_2###
+port=###node_server_port_2###
+protocol=MySQLBackend
+
+[server3]
+type=server
+address=###node_server_IP_3###
+port=###node_server_port_3###
+protocol=MySQLBackend
+
+[server4]
+type=server
+address=###node_server_IP_4###
+port=###node_server_port_4###
+protocol=MySQLBackend

maxscale-system-test/mariadbmonitor/mysqlmon_rejoin_manual.cpp

@@ -0,0 +1,146 @@
+/*
+ * Copyright (c) 2016 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2024-03-10
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#include "fail_switch_rejoin_common.cpp"
+#include <iostream>
+
+using std::string;
+using std::cout;
+using std::endl;
+
+int main(int argc, char** argv)
+{
+    interactive = strcmp(argv[argc - 1], "interactive") == 0;
+    Mariadb_nodes::require_gtid(true);
+    TestConnections test(argc, argv);
+    int ec;
+    MYSQL* maxconn = test.maxscales->open_rwsplit_connection(0);
+    // Set up test table
+    basic_test(test);
+    // Delete binlogs to sync gtid:s
+    delete_slave_binlogs(test);
+    char result_tmp[bufsize];
+    // Advance gtid:s a bit to so gtid variables are updated.
+    generate_traffic_and_check(test, maxconn, 10);
+    mysql_close(maxconn);
+    test.tprintf(LINE);
+    print_gtids(test);
+    get_input();
+
+    cout << "Stopping master and waiting for failover. Check that another server is promoted." << endl;
+    const int old_master_id = get_master_server_id(test);   // Read master id now before shutdown.
+    const int master_index = test.repl->master;
+    test.repl->stop_node(master_index);
+
+    // Wait until failover is performed
+    test.maxscales->wait_for_monitor(2);
+    get_output(test);
+
+    int master_id = get_master_server_id(test);
+    cout << "Master server id is " << master_id << endl;
+    const bool failover_ok = (master_id > 0 && master_id != old_master_id);
+    test.expect(failover_ok, "Master did not change or no master detected.");
+
+    if (test.ok())
+    {
+        // Recreate maxscale session
+        maxconn = test.maxscales->open_rwsplit_connection(0);
+        cout << "Sending more inserts." << endl;
+        generate_traffic_and_check(test, maxconn, 5);
+        print_gtids(test);
+        cout << "Bringing old master back online..." << endl;
+        test.repl->start_node(master_index, (char*) "");
+        test.maxscales->wait_for_monitor(2);
+        get_output(test);
+        test.tprintf("and manually rejoining it to cluster.");
+        const char REJOIN_CMD[] = "maxadmin call command mariadbmon rejoin MySQL-Monitor server1";
+        test.maxscales->ssh_node_output(0, REJOIN_CMD, true, &ec);
+        test.maxscales->wait_for_monitor(2);
+        get_output(test);
+        test.repl->connect();
+
+        string gtid_old_master;
+        test.repl->connect();
+        if (find_field(test.repl->nodes[master_index], GTID_QUERY, GTID_FIELD, result_tmp) == 0)
+        {
+            gtid_old_master = result_tmp;
+        }
+        string gtid_final;
+        if (find_field(maxconn, GTID_QUERY, GTID_FIELD, result_tmp) == 0)
+        {
+            gtid_final = result_tmp;
+        }
+
+        cout << LINE << "\n";
+        print_gtids(test);
+        cout << LINE << "\n";
+        test.expect(gtid_final == gtid_old_master,
+                    "Old master did not successfully rejoin the cluster (%s != %s).",
+                    gtid_final.c_str(), gtid_old_master.c_str());
+        // Switch master back to server1 so last check is faster
+        test.maxscales->ssh_node_output(0,
+                "maxadmin call command mysqlmon switchover MySQL-Monitor server1 server2", true, &ec);
+        test.maxscales->wait_for_monitor(2);
+        get_output(test);
+        master_id = get_master_server_id(test);
+        test.expect(master_id == old_master_id, "Switchover back to server1 failed.");
+
+        // STOP and RESET SLAVE on a server, then remove binlogs. Check that a server with empty binlogs
+        // can be rejoined.
+        if (test.ok())
+        {
+            cout << "Removing slave connection and deleting binlogs on server3 to get empty gtid.\n";
+            int slave_to_reset = 2;
+            test.repl->connect();
+            auto conn = test.repl->nodes[slave_to_reset];
+            string sstatus_query = "SHOW ALL SLAVES STATUS;";
+            test.try_query(conn,
+                           "STOP SLAVE; RESET SLAVE ALL; RESET MASTER; SET GLOBAL gtid_slave_pos='';");
+            test.maxscales->wait_for_monitor();
+            get_output(test);
+            auto row = get_row(conn, sstatus_query.c_str());
+            test.expect(row.empty(), "server3 is still replicating.");
+            row = get_row(conn, "SELECT @@gtid_current_pos;");
+            test.expect(row.empty() || row[0].empty(),
+                        "server3 gtid is not empty as it should (%s).", row[0].c_str());
+            cout << "Rejoining server3.\n";
+            test.maxscales->ssh_node_output(0,
+                    "maxadmin call command mysqlmon rejoin MySQL-Monitor server3", true, &ec);
+            test.maxscales->wait_for_monitor(2);
+            get_output(test);
+            test.repl->connect();
+            char result[100];
+            test.repl->connect();
+            if (find_field(conn, sstatus_query.c_str(), "Master_Host", result) == 0)
+            {
+                test.expect(strcmp(result, test.repl->IP_private[0]) == 0,
+                        "server3 did not rejoin the cluster (%s != %s).", result, test.repl->IP_private[0]);
+            }
+            else
+            {
+                test.expect(false, "Could not query slave status.");
+            }
+            if (test.ok())
+            {
+                cout << "server3 joined succesfully, test complete.\n";
+            }
+        }
+    }
+    else
+    {
+        test.repl->start_node(master_index, (char*) "");
+        test.maxscales->wait_for_monitor();
+    }
+
+    return test.global_result;
+}

maxscale-system-test/mariadbmonitor/mysqlmon_rejoin_manual2.cpp

@@ -0,0 +1,145 @@
+/*
+ * Copyright (c) 2016 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2024-03-10
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#include <vector>
+
+#include "testconnections.h"
+#include "fail_switch_rejoin_common.cpp"
+#include <iostream>
+
+using std::string;
+using std::cout;
+using std::endl;
+
+int main(int argc, char** argv)
+{
+    char result_tmp[bufsize];
+    interactive = strcmp(argv[argc - 1], "interactive") == 0;
+    Mariadb_nodes::require_gtid(true);
+    TestConnections test(argc, argv);
+    MYSQL* maxconn = test.maxscales->open_rwsplit_connection(0);
+
+    // Set up test table
+    basic_test(test);
+    // Delete binlogs to sync gtid:s
+    delete_slave_binlogs(test);
+    // Advance gtid:s a bit to so gtid variables are updated.
+    generate_traffic_and_check(test, maxconn, 10);
+    test.repl->sync_slaves(0);
+
+    cout << LINE << "\n";
+    print_gtids(test);
+    cout << LINE << "\n";
+    string gtid_begin;
+    if (find_field(maxconn, GTID_QUERY, GTID_FIELD, result_tmp) == 0)
+    {
+        gtid_begin = result_tmp;
+    }
+    mysql_close(maxconn);
+
+    // Leave first of three slaves connected so it's clear which one is the master server.
+    const char STOP_SLAVE[] = "STOP SLAVE;";
+    const char RESET_SLAVE[] = "RESET SLAVE ALL;";
+    const char READ_ONLY_OFF[] = "SET GLOBAL read_only=0;";
+    test.repl->connect();
+    const int FIRST_MOD_NODE = 2;   // Modify nodes 2 & 3
+    const int NODE_COUNT = test.repl->N;
+    MYSQL** nodes = test.repl->nodes;
+
+    for (int i = FIRST_MOD_NODE; i < NODE_COUNT; i++)
+    {
+        if (mysql_query(nodes[i], STOP_SLAVE) != 0
+            || mysql_query(nodes[i], RESET_SLAVE) != 0
+            || mysql_query(nodes[i], READ_ONLY_OFF) != 0)
+        {
+            test.expect(false, "Could not stop slave connections and/or disable read_only for node %d.", i);
+            return test.global_result;
+        }
+    }
+
+    // Add more events to node3.
+    string gtid_node2, gtid_node3;
+    cout << "Sending more inserts to server 4.\n";
+    generate_traffic_and_check(test, nodes[3], 10);
+    // Save gtids
+    if (find_field(nodes[2], GTID_QUERY, GTID_FIELD, result_tmp) == 0)
+    {
+        gtid_node2 = result_tmp;
+    }
+    if (find_field(nodes[3], GTID_QUERY, GTID_FIELD, result_tmp) == 0)
+    {
+        gtid_node3 = result_tmp;
+    }
+    print_gtids(test);
+    bool gtids_ok = (gtid_begin == gtid_node2 && gtid_node2 < gtid_node3);
+    test.expect(gtids_ok, "Gtid:s have not advanced correctly.");
+    if (!gtids_ok)
+    {
+        return test.global_result;
+    }
+    cout << "Sending rejoin commands for servers 3 & 4. Server 4 should not rejoin the cluster.\n";
+    const string REJOIN_CMD = "maxadmin call command mariadbmon rejoin MySQL-Monitor";
+    int ec;
+    string rejoin_s3 = REJOIN_CMD + " server3";
+    string rejoin_s4 = REJOIN_CMD + " server4";
+    test.maxscales->ssh_node_output(0, rejoin_s3.c_str(), true, &ec);
+    test.maxscales->ssh_node_output(0, rejoin_s4.c_str(), true, &ec);
+    test.maxscales->wait_for_monitor();
+    get_output(test);
+
+    StringSet node2_states = test.get_server_status("server3");
+    StringSet node3_states = test.get_server_status("server4");
+    bool states_n2_ok = (node2_states.find("Slave") != node2_states.end());
+    bool states_n3_ok = (node3_states.find("Slave") == node3_states.end());
+    test.expect(states_n2_ok, "Node 2 has not rejoined when it should have.");
+    test.expect(states_n3_ok, "Node 3 rejoined when it shouldn't have.");
+    if (!states_n2_ok || !states_n3_ok)
+    {
+        return test.global_result;
+    }
+    // Finally, fix replication by telling the current master to replicate from server4
+    test.tprintf("Setting server 1 to replicate from server 4. Manually rejoin servers 2 and 3.");
+    const char CHANGE_CMD_FMT[] = "CHANGE MASTER TO MASTER_HOST = '%s', MASTER_PORT = %d, "
+                                  "MASTER_USE_GTID = current_pos, MASTER_USER='repl', MASTER_PASSWORD = 'repl';";
+    char cmd[256];
+    snprintf(cmd, sizeof(cmd), CHANGE_CMD_FMT, test.repl->IP_private[3], test.repl->port[3]);
+    mysql_query(nodes[0], cmd);
+    mysql_query(nodes[0], "START SLAVE;");
+    test.maxscales->wait_for_monitor();
+    string rejoin_s2 = REJOIN_CMD + " server2";
+    test.maxscales->ssh_node_output(0, rejoin_s2.c_str(), true, &ec);
+    test.maxscales->ssh_node_output(0, rejoin_s3.c_str(), true, &ec);
+    test.maxscales->wait_for_monitor();
+    get_output(test);
+    int master_id = get_master_server_id(test);
+    test.expect(master_id == 4, "Server 4 should be the cluster master.");
+    StringSet node0_states = test.get_server_status("server1");
+    bool states_n0_ok = (node0_states.find("Slave") != node0_states.end()
+                         && node0_states.find("Relay Master") == node0_states.end());
+    test.expect(states_n0_ok, "Server 1 is not a slave when it should be.");
+    if (states_n0_ok)
+    {
+        int ec;
+        test.maxscales->ssh_node_output(0,
+                                        "maxadmin call command mysqlmon switchover MySQL-Monitor server1 server4",
+                                        true,
+                                        &ec);
+        test.maxscales->wait_for_monitor();
+        master_id = get_master_server_id(test);
+        test.expect(master_id == 1, "Server 1 should be the cluster master.");
+        get_output(test);
+    }
+
+    test.repl->fix_replication();
+    return test.global_result;
+}

maxscale-system-test/mariadbmonitor/mysqlmon_reset_replication.cpp

@@ -0,0 +1,149 @@
+/*
+ * Copyright (c) 2016 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2024-03-10
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#include "testconnections.h"
+#include "fail_switch_rejoin_common.cpp"
+#include <iostream>
+#include <string>
+
+using std::string;
+using std::cout;
+
+int main(int argc, char** argv)
+{
+    Mariadb_nodes::require_gtid(true);
+    TestConnections test(argc, argv);
+
+    auto expect_server_status = [&test](const string& server_name, const string& status) {
+            bool found = (test.maxscales->get_server_status(server_name.c_str()).count(status) == 1);
+            test.expect(found, "%s was not %s as was expected.", server_name.c_str(), status.c_str());
+        };
+
+    auto expect_not_server_status = [&test](const string& server_name, const string& status) {
+            bool not_found = (test.maxscales->get_server_status(server_name.c_str()).count(status) == 0);
+            test.expect(not_found, "%s was %s contrary to expectation.", server_name.c_str(), status.c_str());
+        };
+
+    auto read_sum = [&test](int server_ind) -> int {
+            int sum = -1;
+            const char query[] = "SELECT SUM(c1) FROM test.t1;";
+            const char field[] = "SUM(c1)";
+            char value[100];
+            if ((find_field(test.repl->nodes[server_ind], query, field, value) == 0) && strlen(value) > 0)
+            {
+                sum = atoi(value);
+            }
+            return sum;
+        };
+
+    const char insert_query[] = "INSERT INTO test.t1 VALUES (%i);";
+    const char drop_query[] = "DROP TABLE test.t1;";
+    const char strict_mode[] = "SET GLOBAL gtid_strict_mode=%i;";
+
+    string server_names[] = {"server1", "server2", "server3", "server4"};
+    string master = "Master";
+    string slave = "Slave";
+
+    // Set up test table
+    MYSQL* maxconn = test.maxscales->open_rwsplit_connection(0);
+    test.tprintf("Creating table and inserting data.");
+    test.try_query(maxconn, "CREATE OR REPLACE TABLE test.t1(c1 INT)");
+    int insert_val = 1;
+    test.try_query(maxconn, insert_query, insert_val++);
+    cout << "Setting gitd_strict_mode to ON.\n";
+    test.try_query(maxconn, strict_mode, 1);
+    test.repl->sync_slaves();
+    mysql_close(maxconn);
+
+    get_output(test);
+    print_gtids(test);
+    expect_server_status(server_names[0], master);
+    expect_server_status(server_names[1], slave);
+    expect_server_status(server_names[2], slave);
+    expect_server_status(server_names[3], slave);
+
+    // Stop MaxScale and mess with the nodes.
+    cout << "Inserting events directly to nodes while MaxScale is stopped.\n";
+    test.maxscales->stop_maxscale();
+    test.repl->connect();
+    // Modify the databases of backends identically. This will unsync gtid:s but not the actual data.
+    for (; insert_val <= 9; insert_val++)
+    {
+        // When inserting data, start from the slaves so replication breaks immediately.
+        test.try_query(test.repl->nodes[1], insert_query, insert_val);
+        test.try_query(test.repl->nodes[2], insert_query, insert_val);
+        test.try_query(test.repl->nodes[3], insert_query, insert_val);
+        test.try_query(test.repl->nodes[0], insert_query, insert_val);
+    }
+    // Restart MaxScale, there should be no slaves. Master is still ok.
+    test.maxscales->start_maxscale();
+    test.maxscales->wait_for_monitor(2);
+    cout << "Restarted MaxScale.\n";
+    print_gtids(test);
+    get_output(test);
+
+    expect_server_status(server_names[0], master);
+    expect_not_server_status(server_names[1], slave);
+    expect_not_server_status(server_names[2], slave);
+    expect_not_server_status(server_names[3], slave);
+
+    if (test.global_result == 0)
+    {
+        // Use the reset-replication command to magically fix the situation.
+        cout << "Running reset-replication to fix the situation.\n";
+        test.maxscales->execute_maxadmin_command(0, "call command mariadbmon reset-replication "
+                                                    "MySQL-Monitor server2");
+        test.maxscales->wait_for_monitor(1);
+        // Add another event to force gtid forward.
+        maxconn = test.maxscales->open_rwsplit_connection(0);
+        test.try_query(maxconn, "FLUSH TABLES;");
+        test.try_query(maxconn, insert_query, insert_val);
+        mysql_close(maxconn);
+
+        test.maxscales->wait_for_monitor(1);
+        get_output(test);
+        expect_server_status(server_names[0], slave);
+        expect_server_status(server_names[1], master);
+        expect_server_status(server_names[2], slave);
+        expect_server_status(server_names[3], slave);
+        // Check that the values on the databases are identical by summing the values.
+        int expected_sum = 55;      // 11 * 5
+        for (int i = 0; i < test.repl->N; i++)
+        {
+            int sum = read_sum(i);
+            test.expect(sum == expected_sum,
+                        "The values in server%i are wrong, sum is %i when %i was expected.",
+                        i + 1, sum, expected_sum);
+        }
+
+        // Finally, switchover back and erase table
+        cout << "Running switchover.\n";
+        test.maxscales->execute_maxadmin_command(0, "call command mariadbmon switchover MySQL-Monitor");
+        test.maxscales->wait_for_monitor(1);
+        get_output(test);
+        expect_server_status(server_names[0], master);
+        expect_server_status(server_names[1], slave);
+        expect_server_status(server_names[2], slave);
+        expect_server_status(server_names[3], slave);
+    }
+
+    if (test.global_result != 0)
+    {
+        test.repl->fix_replication();
+    }
+    maxconn = test.maxscales->open_rwsplit_connection(0);
+    test.try_query(maxconn, strict_mode, 0);
+    test.try_query(maxconn, drop_query);
+    mysql_close(maxconn);
+    return test.global_result;
+}

maxscale-system-test/mariadbmonitor/mysqlmon_switchover_auto.cnf

@@ -0,0 +1,104 @@
+[maxscale]
+threads=###threads###
+
+[MySQL-Monitor]
+type=monitor
+module=mysqlmon
+servers= server1, server2, server3, server4
+user=maxskysql
+password= skysql
+monitor_interval=1000
+failcount=1
+replication_user=repl
+replication_password=repl
+backend_connect_timeout=10
+backend_read_timeout=10
+backend_write_timeout=10
+auto_failover=true
+auto_rejoin=true
+switchover_on_low_disk_space=true
+disk_space_check_interval=2000
+disk_space_threshold=/:90
+
+[RW Split Router]
+type=service
+router= readwritesplit
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[Read Connection Router Slave]
+type=service
+router=readconnroute
+router_options= slave
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[Read Connection Router Master]
+type=service
+router=readconnroute
+router_options=master
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[RW Split Listener]
+type=listener
+service=RW Split Router
+protocol=MySQLClient
+port=4006
+
+[Read Connection Listener Slave]
+type=listener
+service=Read Connection Router Slave
+protocol=MySQLClient
+port=4009
+
+[Read Connection Listener Master]
+type=listener
+service=Read Connection Router Master
+protocol=MySQLClient
+port=4008
+
+[CLI]
+type=service
+router=cli
+
+[CLI Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=default
+
+[server1]
+type=server
+address=###node_server_IP_1###
+port=###node_server_port_1###
+protocol=MySQLBackend
+# This server uses monitor disk space threshold
+
+[server2]
+type=server
+address=###node_server_IP_2###
+port=###node_server_port_2###
+protocol=MySQLBackend
+# This server is always out of disk space
+disk_space_threshold=/:1
+
+[server3]
+type=server
+address=###node_server_IP_3###
+port=###node_server_port_3###
+protocol=MySQLBackend
+# This server is never out of disk space
+disk_space_threshold=/:99
+
+[server4]
+type=server
+address=###node_server_IP_4###
+port=###node_server_port_4###
+protocol=MySQLBackend
+# This server is never out of disk space
+disk_space_threshold=/:99
+

maxscale-system-test/mariadbmonitor/mysqlmon_switchover_auto.cpp

@@ -0,0 +1,163 @@
+/*
+ * Copyright (c) 2016 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2024-03-10
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#include "testconnections.h"
+#include "fail_switch_rejoin_common.cpp"
+#include <iostream>
+#include <string>
+
+using std::string;
+using std::cout;
+
+int main(int argc, char** argv)
+{
+    // Only in very recent server versions have the disks-plugin
+    TestConnections::require_repl_version("10.3.6");
+    Mariadb_nodes::require_gtid(true);
+    TestConnections::skip_maxscale_start(true);
+    TestConnections test(argc, argv);
+
+    test.repl->connect();
+    delete_slave_binlogs(test);
+
+    // Enable the disks-plugin on all servers. Has to be done before MaxScale is on to prevent disk space
+    // monitoring from disabling itself.
+    bool disks_plugin_loaded = false;
+    const char strict_mode[] = "SET GLOBAL gtid_strict_mode=%i;";
+    test.repl->connect();
+    for (int i = 0; i < test.repl->N; i++)
+    {
+        MYSQL* conn = test.repl->nodes[i];
+        test.try_query(conn, "INSTALL SONAME 'disks';");
+        test.try_query(conn, strict_mode, 1);
+    }
+
+    if (test.ok())
+    {
+        cout << "Disks-plugin installed and gtid_strict_mode enabled on all servers. "
+                "Starting MaxScale.\n";
+        test.start_maxscale();
+        test.maxscales->wait_for_monitor(1);
+        disks_plugin_loaded = true;
+    }
+    else
+    {
+        cout << "Test preparations failed.\n";
+    }
+
+    auto set_to_string = [](const StringSet& str_set) -> string {
+            string rval;
+            for (const string& elem : str_set)
+            {
+                rval += elem + ",";
+            }
+            return rval;
+        };
+
+    auto expect_server_status = [&test, &set_to_string](const string& server_name, const string& status) {
+            auto status_set = test.maxscales->get_server_status(server_name.c_str());
+            string status_str = set_to_string(status_set);
+            bool found = (status_set.count(status) == 1);
+            test.expect(found, "%s was not %s as was expected. Status: %s.",
+                        server_name.c_str(), status.c_str(), status_str.c_str());
+        };
+
+    string server_names[] = {"server1", "server2", "server3", "server4"};
+    string master = "Master";
+    string slave = "Slave";
+    string maint = "Maintenance";
+    const char insert_query[] = "INSERT INTO test.t1 VALUES (%i);";
+    int insert_val = 1;
+
+    if (test.ok())
+    {
+        // Set up test table to ensure queries are going through.
+        test.tprintf("Creating table and inserting data.");
+        auto maxconn = test.maxscales->open_rwsplit_connection(0);
+        test.try_query(maxconn, "CREATE OR REPLACE TABLE test.t1(c1 INT)");
+        test.try_query(maxconn, insert_query, insert_val++);
+        mysql_close(maxconn);
+
+        get_output(test);
+        print_gtids(test);
+
+        expect_server_status(server_names[0], master);
+        expect_server_status(server_names[1], maint);   // Always out of disk space
+        expect_server_status(server_names[2], slave);
+        expect_server_status(server_names[3], slave);
+    }
+
+    if (test.ok())
+    {
+        // If ok so far, change the disk space threshold to something really small to force a switchover.
+        cout << "Changing disk space threshold for the monitor, should cause a switchover.\n";
+        test.maxscales->execute_maxadmin_command(0, "alter monitor MySQL-Monitor disk_space_threshold=/:1");
+        sleep(2);   // The disk space is checked depending on wall clock time.
+        test.maxscales->wait_for_monitor(2);
+
+        // server2 was in maintenance before the switchover, so it was ignored. This means that it is
+        // still replicating from server1. server1 was redirected to the new master. Although server1
+        // is low on disk space, it is not set to maintenance since it is a relay.
+        expect_server_status(server_names[0], slave);
+        expect_server_status(server_names[1], maint);
+        expect_server_status(server_names[2], master);
+        expect_server_status(server_names[3], slave);
+
+        // Check that writes are working.
+        auto maxconn = test.maxscales->open_rwsplit_connection(0);
+        test.try_query(maxconn, insert_query, insert_val);
+        mysql_close(maxconn);
+
+        get_output(test);
+        print_gtids(test);
+
+        cout << "Changing disk space threshold for the monitor, should prevent low disk switchovers.\n";
+        test.maxscales->execute_maxadmin_command(0, "alter monitor MySQL-Monitor "
+                                                    "disk_space_threshold=/:100");
+        sleep(2);   // To update disk space status
+        test.maxscales->wait_for_monitor(1);
+        get_output(test);
+    }
+
+    // Use the reset-replication command to fix the situation.
+    cout << "Running reset-replication to fix the situation.\n";
+    test.maxscales->execute_maxadmin_command(0, "call command mariadbmon reset-replication "
+                                                "MySQL-Monitor server1");
+    sleep(2);
+    test.maxscales->wait_for_monitor(2);
+    get_output(test);
+    // Check that no auto switchover has happened.
+    expect_server_status(server_names[0], master);
+    expect_server_status(server_names[1], maint);
+    expect_server_status(server_names[2], slave);
+    expect_server_status(server_names[3], slave);
+
+    const char drop_query[] = "DROP TABLE test.t1;";
+    auto maxconn = test.maxscales->open_rwsplit_connection(0);
+    test.try_query(maxconn, drop_query);
+    mysql_close(maxconn);
+
+    if (disks_plugin_loaded)
+    {
+        // Enable the disks-plugin on all servers.
+        for (int i = 0; i < test.repl->N; i++)
+        {
+            MYSQL* conn = test.repl->nodes[i];
+            test.try_query(conn, "UNINSTALL SONAME 'disks';");
+            test.try_query(conn, strict_mode, 0);
+        }
+    }
+
+    test.repl->disconnect();
+    return test.global_result;
+}

maxscale-system-test/mariadbmonitor/mysqlmon_switchover_bad_master.cnf

@@ -0,0 +1,95 @@
+[maxscale]
+threads=###threads###
+
+[MySQL-Monitor]
+type=monitor
+module=mysqlmon
+servers= server1, server2, server3, server4
+user=maxskysql
+password= skysql
+monitor_interval=1000
+allow_cluster_recovery=true
+detect_standalone_master=true
+auto_failover=true
+auto_rejoin=false
+replication_user=repl
+replication_password=repl
+backend_connect_timeout=10
+backend_read_timeout=10
+backend_write_timeout=10
+failcount=1
+
+[RW-Split-Router]
+type=service
+router= readwritesplit
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[Read-Connection-Router-Slave]
+type=service
+router=readconnroute
+router_options= slave
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[Read-Connection-Router-Master]
+type=service
+router=readconnroute
+router_options=master
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[RW-Split-Listener]
+type=listener
+service=RW-Split-Router
+protocol=MySQLClient
+port=4006
+
+[Read-Connection-Listener-Slave]
+type=listener
+service=Read-Connection-Router-Slave
+protocol=MySQLClient
+port=4009
+
+[Read-Connection-Listener-Master]
+type=listener
+service=Read-Connection-Router-Master
+protocol=MySQLClient
+port=4008
+
+[CLI]
+type=service
+router=cli
+
+[CLI Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=default
+
+[server1]
+type=server
+address=###node_server_IP_1###
+port=###node_server_port_1###
+protocol=MySQLBackend
+
+[server2]
+type=server
+address=###node_server_IP_2###
+port=###node_server_port_2###
+protocol=MySQLBackend
+
+[server3]
+type=server
+address=###node_server_IP_3###
+port=###node_server_port_3###
+protocol=MySQLBackend
+
+[server4]
+type=server
+address=###node_server_IP_4###
+port=###node_server_port_4###
+protocol=MySQLBackend

maxscale-system-test/mariadbmonitor/mysqlmon_switchover_bad_master.cpp

@@ -0,0 +1,218 @@
+/*
+ * Copyright (c) 2016 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2024-03-10
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#include <iostream>
+#include <iterator>
+#include <string>
+#include <sstream>
+#include "testconnections.h"
+
+using std::cerr;
+using std::cout;
+using std::endl;
+using std::flush;
+using std::string;
+using std::stringstream;
+
+namespace
+{
+
+void sleep(int s)
+{
+    cout << "Sleeping " << s << " times 1 second" << flush;
+    do
+    {
+        ::sleep(1);
+        cout << "." << flush;
+        --s;
+    }
+    while (s > 0);
+
+    cout << endl;
+}
+}
+
+namespace
+{
+
+void create_table(TestConnections& test)
+{
+    MYSQL* pConn = test.maxscales->conn_rwsplit[0];
+
+    test.try_query(pConn, "DROP TABLE IF EXISTS test.t1");
+    test.try_query(pConn, "CREATE TABLE test.t1(id INT)");
+}
+
+static int i_start = 0;
+static int n_rows = 20;
+static int i_end = 0;
+
+void insert_data(TestConnections& test)
+{
+    MYSQL* pConn = test.maxscales->conn_rwsplit[0];
+
+    test.try_query(pConn, "BEGIN");
+
+    i_end = i_start + n_rows;
+
+    for (int i = i_start; i < i_end; ++i)
+    {
+        stringstream ss;
+        ss << "INSERT INTO test.t1 VALUES (" << i << ")";
+        test.try_query(pConn, "%s", ss.str().c_str());
+    }
+
+    test.try_query(pConn, "COMMIT");
+
+    i_start = i_end;
+}
+
+void expect(TestConnections& test, const char* zServer, const StringSet& expected)
+{
+    StringSet found = test.get_server_status(zServer);
+
+    std::ostream_iterator<string> oi(cout, ", ");
+
+    cout << zServer
+         << ", expected states: ";
+    std::copy(expected.begin(), expected.end(), oi);
+    cout << endl;
+
+    cout << zServer
+         << ", found states   : ";
+    std::copy(found.begin(), found.end(), oi);
+    cout << endl;
+
+    if (found != expected)
+    {
+        cout << "ERROR, found states are not the same as the expected ones." << endl;
+        ++test.global_result;
+    }
+
+    cout << endl;
+}
+
+void expect(TestConnections& test, const char* zServer, const char* zState)
+{
+    StringSet s;
+    s.insert(zState);
+
+    expect(test, zServer, s);
+}
+
+void expect(TestConnections& test, const char* zServer, const char* zState1, const char* zState2)
+{
+    StringSet s;
+    s.insert(zState1);
+    s.insert(zState2);
+
+    expect(test, zServer, s);
+}
+
+void run(TestConnections& test)
+{
+    test.maxscales->wait_for_monitor();
+
+    int N = test.repl->N;
+    cout << "Nodes: " << N << endl;
+
+    expect(test, "server1", "Master", "Running");
+    expect(test, "server2", "Slave", "Running");
+    expect(test, "server3", "Slave", "Running");
+    expect(test, "server4", "Slave", "Running");
+
+    cout << "\nConnecting to MaxScale." << endl;
+    test.maxscales->connect_maxscale(0);
+
+    cout << "\nCreating table." << endl;
+    create_table(test);
+
+    cout << "\nInserting data." << endl;
+    insert_data(test);
+
+    cout << "\nSyncing slaves." << endl;
+    test.repl->sync_slaves();
+
+    cout << "\nStopping slave " << N - 1 << endl;
+    test.repl->stop_node(N - 1);
+
+    test.maxscales->wait_for_monitor();
+
+    // server4 was stopped, so we expect the state of it to be /Down/,
+    // and the states of the other ones not to have changed.
+    expect(test, "server1", "Master", "Running");
+    expect(test, "server2", "Slave", "Running");
+    expect(test, "server3", "Slave", "Running");
+    expect(test, "server4", "Down");
+
+    cout << "\nClosing connection to MaxScale." << endl;
+    test.maxscales->close_maxscale_connections(0);
+
+    cout << "\nConnecting to MaxScale." << endl;
+    test.maxscales->connect_maxscale(0);
+
+    cout << "\nInserting data." << endl;
+    insert_data(test);
+
+    cout << "\nSyncing slaves." << endl;
+    test.repl->sync_slaves();
+
+    cout << "\nStopping master." << endl;
+    test.repl->stop_node(0);
+
+    test.maxscales->wait_for_monitor(3);
+
+    // server1 (previous master) was taken down, so its state should be /Down/.
+    // server2 should have been made into master, and server4 should still be down.
+    expect(test, "server1", "Down");
+    expect(test, "server2", "Master", "Running");
+    expect(test, "server3", "Slave", "Running");
+    expect(test, "server4", "Down");
+
+    cout << "\nBringing up slave " << N - 1 << endl;
+    test.repl->start_node(N - 1, (char*)"");
+
+    test.maxscales->wait_for_monitor();
+
+    // server1 should still be down, server2 still master, and server3 still
+    // a slave. server4 was brought up, but as auto_rejoin is false, it should
+    // be Running, but not Slave.
+    // turned into a slave.
+    expect(test, "server1", "Down");
+    expect(test, "server2", "Master", "Running");
+    expect(test, "server3", "Slave", "Running");
+    expect(test, "server4", "Running");
+
+    cout << "\nTrying to do manual switchover to server4" << endl;
+    const char* zCommand = "call command mysqlmon switchover MySQL-Monitor server4 server2";
+    test.maxscales->execute_maxadmin_command_print(0, (char*)zCommand);
+
+    test.maxscales->wait_for_monitor();
+
+    // The state should not change, as server4 is not good enough as master.
+    expect(test, "server1", "Down");
+    expect(test, "server2", "Master", "Running");
+    expect(test, "server3", "Slave", "Running");
+    expect(test, "server4", "Running");
+}
+}
+
+int main(int argc, char* argv[])
+{
+    Mariadb_nodes::require_gtid(true);
+    TestConnections test(argc, argv);
+
+    run(test);
+
+    return test.global_result;
+}

maxscale-system-test/mariadbmonitor/mysqlmon_switchover_stress.cnf

@@ -0,0 +1,94 @@
+[maxscale]
+threads=###threads###
+
+[MySQL-Monitor]
+type=monitor
+module=mysqlmon
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+monitor_interval=1000
+allow_cluster_recovery=true
+detect_standalone_master=true
+auto_failover=false
+auto_rejoin=false
+replication_user=repl
+replication_password=repl
+backend_connect_timeout=10
+backend_read_timeout=10
+backend_write_timeout=10
+
+[RW-Split-Router]
+type=service
+router= readwritesplit
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[Read-Connection-Router-Slave]
+type=service
+router=readconnroute
+router_options= slave
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[Read-Connection-Router-Master]
+type=service
+router=readconnroute
+router_options=master
+servers=server1, server2, server3, server4
+user=maxskysql
+password=skysql
+
+[RW-Split-Listener]
+type=listener
+service=RW-Split-Router
+protocol=MySQLClient
+port=4006
+
+[Read-Connection-Listener-Slave]
+type=listener
+service=Read-Connection-Router-Slave
+protocol=MySQLClient
+port=4009
+
+[Read-Connection-Listener-Master]
+type=listener
+service=Read-Connection-Router-Master
+protocol=MySQLClient
+port=4008
+
+[CLI]
+type=service
+router=cli
+
+[CLI Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=default
+
+[server1]
+type=server
+address=###node_server_IP_1###
+port=###node_server_port_1###
+protocol=MySQLBackend
+
+[server2]
+type=server
+address=###node_server_IP_2###
+port=###node_server_port_2###
+protocol=MySQLBackend
+
+[server3]
+type=server
+address=###node_server_IP_3###
+port=###node_server_port_3###
+protocol=MySQLBackend
+
+[server4]
+type=server
+address=###node_server_IP_4###
+port=###node_server_port_4###
+protocol=MySQLBackend

maxscale-system-test/mariadbmonitor/mysqlmon_switchover_stress.cpp

@@ -0,0 +1,649 @@
+/*
+ * Copyright (c) 2016 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2024-03-10
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#include <iostream>
+#include <iterator>
+#include <string>
+#include <sstream>
+#include <thread>
+#include <vector>
+#include <random>
+#include "testconnections.h"
+#include "fail_switch_rejoin_common.cpp"
+
+using namespace std;
+
+// How often the monitor checks the server state.
+// NOTE: Ensure this is identical with the value in the configuration file.
+const time_t MONITOR_INTERVAL = 1;
+
+// After how many seconds should the switchover operation surely have
+// been performed. Not very critical.
+const time_t SWITCHOVER_DURATION = 5;
+
+// How long should we keep in running.
+const time_t TEST_DURATION = 90;
+
+const char* CLIENT_USER = "mysqlmon_switchover_stress";
+const char* CLIENT_PASSWORD = "mysqlmon_switchover_stress";
+
+#define CMESSAGE(msg) \
+    do { \
+        stringstream ss; \
+        ss << "client(" << m_id << ") : " << msg << "\n"; \
+        cout << ss.str() << flush; \
+    } while (false)
+
+#if !defined (NDEBUG)
+
+#define ss_dassert(x) do {if (!(x)) {fprintf(stderr, "Assertion failed: %s\n", #x); abort();}} while (false)
+#define ss_debug(x)   x
+
+#else
+
+#define ss_dassert(s)
+#define ss_debug(x)
+
+#endif
+
+
+namespace
+{
+
+class Client
+{
+public:
+    enum
+    {
+        DEFAULT_N_CLIENTS = 4,
+        DEFAULT_N_ROWS    = 100
+    };
+
+    static void init(TestConnections& test, size_t nClients, size_t nRows)
+    {
+        s_nClients = nClients;
+        s_nRows = nRows;
+
+        if (create_tables(test))
+        {
+            if (insert_data(test))
+            {
+                cout << "\nSyncing slaves." << endl;
+                test.repl->sync_slaves();
+            }
+        }
+    }
+
+    static void start(bool verbose,
+                      const char* zHost,
+                      int port,
+                      const char* zUser,
+                      const char* zPassword)
+    {
+        for (size_t i = 0; i < s_nClients; ++i)
+        {
+            s_threads.push_back(std::thread(&Client::thread_main,
+                                            i,
+                                            verbose,
+                                            zHost,
+                                            port,
+                                            zUser,
+                                            zPassword));
+        }
+    }
+
+    static void stop()
+    {
+        s_shutdown = true;
+
+        for (size_t i = 0; i < s_nClients; ++i)
+        {
+            s_threads[i].join();
+        }
+    }
+
+private:
+    Client(int id, bool verbose)
+        : m_id(id)
+        , m_verbose(verbose)
+        , m_value(1)
+        , m_rand_dist(0.0, 1.0)
+    {
+    }
+
+    enum action_t
+    {
+        ACTION_SELECT,
+        ACTION_UPDATE
+    };
+
+    action_t action() const
+    {
+        double d = random_decimal_fraction();
+
+        // 20% updates
+        // 80% selects
+        if (d <= 0.2)
+        {
+            return ACTION_UPDATE;
+        }
+        else
+        {
+            return ACTION_SELECT;
+        }
+    }
+
+    bool run(MYSQL* pConn)
+    {
+        bool rv = false;
+
+        switch (action())
+        {
+        case ACTION_SELECT:
+            rv = run_select(pConn);
+            break;
+
+        case ACTION_UPDATE:
+            rv = run_update(pConn);
+            break;
+
+        default:
+            ss_dassert(!true);
+        }
+
+        return rv;
+    }
+
+    bool run_select(MYSQL* pConn)
+    {
+        bool rv = true;
+
+        string stmt("SELECT * FROM test.t");
+        stmt += std::to_string(m_id);
+        stmt += " WHERE id=";
+        stmt += std::to_string(get_random_id());
+
+        if (mysql_query(pConn, stmt.c_str()) == 0)
+        {
+            flush_response(pConn);
+        }
+        else
+        {
+            if (m_verbose)
+            {
+                CMESSAGE("\"" << stmt << "\" failed: " << mysql_error(pConn));
+            }
+            rv = false;
+        }
+
+        return rv;
+    }
+
+    bool run_update(MYSQL* pConn)
+    {
+        bool rv = true;
+
+        string stmt("UPDATE test.t");
+        stmt += std::to_string(m_id);
+        stmt += " SET id=";
+        stmt += std::to_string(m_value);
+        stmt += " WHERE id=";
+        stmt += std::to_string(get_random_id());
+        m_value = (m_value + 1) % s_nRows;
+
+        if (mysql_query(pConn, stmt.c_str()) == 0)
+        {
+            flush_response(pConn);
+        }
+        else
+        {
+            if (m_verbose)
+            {
+                CMESSAGE("\"" << stmt << "\" failed: " << mysql_error(pConn));
+            }
+            rv = false;
+        }
+
+        return rv;
+    }
+
+    static void flush_response(MYSQL* pConn)
+    {
+        do
+        {
+            MYSQL_RES* pRes = mysql_store_result(pConn);
+            mysql_free_result(pRes);
+        }
+        while (mysql_next_result(pConn) == 0);
+    }
+
+    int get_random_id() const
+    {
+        int id = s_nRows * random_decimal_fraction();
+
+        ss_dassert(id >= 0);
+        ss_dassert(id <= (int)s_nRows);
+
+        return id;
+    }
+
+    double random_decimal_fraction() const
+    {
+        return m_rand_dist(m_rand_gen);
+    }
+
+    void run(const char* zHost, int port, const char* zUser, const char* zPassword)
+    {
+        do
+        {
+            MYSQL* pMysql = mysql_init(NULL);
+
+            if (pMysql)
+            {
+                unsigned int timeout = 5;
+                mysql_options(pMysql, MYSQL_OPT_CONNECT_TIMEOUT, &timeout);
+                mysql_options(pMysql, MYSQL_OPT_READ_TIMEOUT, &timeout);
+                mysql_options(pMysql, MYSQL_OPT_WRITE_TIMEOUT, &timeout);
+
+                if (m_verbose)
+                {
+                    CMESSAGE("Connecting");
+                }
+
+                if (mysql_real_connect(pMysql, zHost, zUser, zPassword, "test", port, NULL, 0))
+                {
+                    if (m_verbose)
+                    {
+                        CMESSAGE("Connected.");
+                    }
+
+                    while (!s_shutdown && run(pMysql))
+                    {
+                    }
+                }
+                else
+                {
+                    if (m_verbose)
+                    {
+                        CMESSAGE("mysql_real_connect() failed: " << mysql_error(pMysql));
+                    }
+                }
+
+                if (m_verbose)
+                {
+                    CMESSAGE("Closing");
+                }
+                mysql_close(pMysql);
+            }
+            else
+            {
+                CMESSAGE("mysql_init() failed.");
+            }
+
+            // To prevent some backend from becoming overwhelmed.
+            sleep(1);
+        }
+        while (!s_shutdown);
+    }
+
+    static void thread_main(int i,
+                            bool verbose,
+                            const char* zHost,
+                            int port,
+                            const char* zUser,
+                            const char* zPassword)
+    {
+        if (mysql_thread_init() == 0)
+        {
+            Client client(i, verbose);
+
+            client.run(zHost, port, zUser, zPassword);
+
+            mysql_thread_end();
+        }
+        else
+        {
+            int m_id = i;
+            CMESSAGE("mysql_thread_init() failed.");
+        }
+    }
+
+    static bool create_tables(TestConnections& test)
+    {
+        cout << "\nCreating tables." << endl;
+
+        MYSQL* pConn = test.maxscales->conn_rwsplit[0];
+
+        string drop_head("DROP TABLE IF EXISTS test.t");
+        string create_head("CREATE TABLE test.t");
+        string create_tail(" (id INT)");
+
+        for (size_t i = 0; i < s_nClients; ++i)
+        {
+            string drop = drop_head + std::to_string(i);
+            test.try_query(pConn, "%s", drop.c_str());
+
+            string create = create_head + std::to_string(i) + create_tail;
+            test.try_query(pConn, "%s", create.c_str());
+        }
+
+        return test.ok();
+    }
+
+    static bool insert_data(TestConnections& test)
+    {
+        cout << "\nInserting data." << endl;
+
+        MYSQL* pConn = test.maxscales->conn_rwsplit[0];
+
+        for (size_t i = 0; i < s_nClients; ++i)
+        {
+            string insert("insert into test.t");
+            insert += std::to_string(i);
+            insert += " values ";
+
+            for (size_t j = 0; j < s_nRows; ++j)
+            {
+                insert += "(";
+                insert += std::to_string(j);
+                insert += ")";
+
+                if (j < s_nRows - 1)
+                {
+                    insert += ", ";
+                }
+            }
+
+            test.try_query(pConn, "%s", insert.c_str());
+        }
+
+        return test.ok();
+    }
+
+private:
+    enum
+    {
+        INITSTATE_SIZE = 32
+    };
+
+    size_t                                         m_id;
+    bool                                           m_verbose;
+    size_t                                         m_value;
+    mutable std::mt19937                           m_rand_gen;
+    mutable std::uniform_real_distribution<double> m_rand_dist;
+
+    static size_t s_nClients;
+    static size_t s_nRows;
+    static bool   s_shutdown;
+
+    static std::vector<std::thread> s_threads;
+};
+
+size_t Client::s_nClients;
+size_t Client::s_nRows;
+bool Client::s_shutdown;
+std::vector<std::thread> Client::s_threads;
+}
+
+namespace
+{
+
+void list_servers(TestConnections& test)
+{
+    test.maxscales->execute_maxadmin_command_print(0, (char*)"list servers");
+}
+
+void sleep(int s)
+{
+    cout << "Sleeping " << s << " times 1 second" << flush;
+    do
+    {
+        ::sleep(1);
+        cout << "." << flush;
+        --s;
+    }
+    while (s > 0);
+
+    cout << endl;
+}
+
+bool check_server_status(TestConnections& test, int id)
+{
+    bool is_master = false;
+
+    Mariadb_nodes* pRepl = test.repl;
+
+    string server = string("server") + std::to_string(id);
+
+    StringSet statuses = test.get_server_status(server.c_str());
+    std::ostream_iterator<string> oi(cout, " ");
+
+    cout << server << ": ";
+    std::copy(statuses.begin(), statuses.end(), oi);
+
+    cout << " => ";
+
+    if (statuses.count("Master"))
+    {
+        is_master = true;
+        cout << "OK" << endl;
+    }
+    else if (statuses.count("Slave"))
+    {
+        cout << "OK" << endl;
+    }
+    else if (statuses.count("Running"))
+    {
+        MYSQL* pConn = pRepl->nodes[id - 1];
+
+        char result[1024];
+        if (find_field(pConn, "SHOW SLAVE STATUS", "Last_IO_Error", result) == 0)
+        {
+            cout << result << endl;
+            test.expect(false, "Server is neither slave, nor master.");
+        }
+        else
+        {
+            cout << "?" << endl;
+            test.expect(false, "Could not execute \"SHOW SLAVE STATUS\"");
+        }
+    }
+    else
+    {
+        cout << "?" << endl;
+        test.expect(false, "Unexpected server state for %s.", server.c_str());
+    }
+
+    return is_master;
+}
+
+void check_server_statuses(TestConnections& test)
+{
+    int masters = 0;
+
+    masters += check_server_status(test, 1);
+    masters += check_server_status(test, 2);
+    masters += check_server_status(test, 3);
+    masters += check_server_status(test, 4);
+
+    test.expect(masters == 1, "Unpexpected number of masters: %d", masters);
+}
+
+int get_next_master_id(TestConnections& test, int current_id)
+{
+    int next_id = current_id;
+
+    do
+    {
+        next_id = (next_id + 1) % 5;
+        if (next_id == 0)
+        {
+            next_id = 1;
+        }
+        ss_dassert(next_id >= 1);
+        ss_dassert(next_id <= 4);
+        string server("server");
+        server += std::to_string(next_id);
+        StringSet states = test.get_server_status(server.c_str());
+        if (states.count("Slave") != 0)
+        {
+            break;
+        }
+    }
+    while (next_id != current_id);
+
+    return next_id != current_id ? next_id : -1;
+}
+
+void create_client_user(TestConnections& test)
+{
+    string stmt;
+
+    // Drop user
+    stmt = "DROP USER IF EXISTS ";
+    stmt += "'";
+    stmt += CLIENT_USER;
+    stmt += "'@'%%'";
+    test.try_query(test.maxscales->conn_rwsplit[0], "%s", stmt.c_str());
+
+    // Create user
+    stmt = "CREATE USER ";
+    stmt += "'";
+    stmt += CLIENT_USER;
+    stmt += "'@'%%'";
+    stmt += " IDENTIFIED BY ";
+    stmt += "'";
+    stmt += CLIENT_PASSWORD;
+    stmt += "'";
+    test.try_query(test.maxscales->conn_rwsplit[0], "%s", stmt.c_str());
+
+    // Grant access
+    stmt = "GRANT SELECT, INSERT, UPDATE ON *.* TO ";
+    stmt += "'";
+    stmt += CLIENT_USER;
+    stmt += "'@'%%'";
+    test.try_query(test.maxscales->conn_rwsplit[0], "%s", stmt.c_str());
+
+    test.try_query(test.maxscales->conn_rwsplit[0], "FLUSH PRIVILEGES");
+}
+
+void switchover(TestConnections& test, int next_master_id, int current_master_id)
+{
+    cout << "\nTrying to do manual switchover from server" << current_master_id
+         << " to server" << next_master_id << endl;
+
+    string command("call command mysqlmon switchover MySQL-Monitor ");
+    command += "server";
+    command += std::to_string(next_master_id);
+    command += " ";
+    command += "server";
+    command += std::to_string(current_master_id);
+
+    cout << "\nCommand: " << command << endl;
+
+    test.maxscales->execute_maxadmin_command_print(0, (char*)command.c_str());
+
+    sleep(1);
+    list_servers(test);
+}
+
+void run(TestConnections& test)
+{
+    cout << "\nConnecting to MaxScale." << endl;
+    test.maxscales->connect_maxscale();
+
+    create_client_user(test);
+
+    Client::init(test, Client::DEFAULT_N_CLIENTS, Client::DEFAULT_N_ROWS);
+
+    if (test.ok())
+    {
+        const char* zHost = test.maxscales->IP[0];
+        int port = test.maxscales->rwsplit_port[0];
+        const char* zUser = CLIENT_USER;
+        const char* zPassword = CLIENT_PASSWORD;
+
+        cout << "Connecting to " << zHost << ":" << port << " as " << zUser << ":" << zPassword << endl;
+        cout << "Starting clients." << endl;
+        Client::start(test.verbose, zHost, port, zUser, zPassword);
+
+        time_t start = time(NULL);
+
+        list_servers(test);
+
+        int current_master_id = 1;
+
+        while ((test.global_result == 0) && (time(NULL) - start < TEST_DURATION))
+        {
+            sleep(SWITCHOVER_DURATION);
+
+            int next_master_id = get_next_master_id(test, current_master_id);
+
+            if (next_master_id != -1)
+            {
+                switchover(test, next_master_id, current_master_id);
+                current_master_id = next_master_id;
+
+                sleep(SWITCHOVER_DURATION);
+
+                int master_id = get_master_server_id(test);
+
+                if (master_id < 0)
+                {
+                    test.expect(false, "No master available after switchover.");
+                }
+                else if (master_id != current_master_id)
+                {
+                    test.expect(false,
+                                "Master should have been server%d, but it was server%d.",
+                                current_master_id,
+                                master_id);
+                }
+            }
+            else
+            {
+                test.expect(false,
+                            "Could not find any slave to switch to.");
+            }
+        }
+
+        cout << "\nStopping clients.\n" << flush;
+        Client::stop();
+
+        // Ensure master is at server1. Shortens startup time for next test.
+        if (current_master_id != 1)
+        {
+            switchover(test, 1, current_master_id);
+        }
+
+        test.repl->close_connections();
+        test.repl->connect();
+
+        check_server_statuses(test);
+    }
+}
+}
+
+int main(int argc, char* argv[])
+{
+    Mariadb_nodes::require_gtid(true);
+    TestConnections test(argc, argv);
+
+    run(test);
+
+    test.repl->connect();
+    execute_query(test.repl->nodes[0], "DROP TABLE test.t");
+    test.repl->disconnect();
+
+    return test.global_result;
+}

maxscale-system-test/mariadbmonitor/verify_master_failure.cnf

@@ -0,0 +1,59 @@
+[maxscale]
+threads=###threads###
+log_info=1
+
+[MySQL Monitor]
+type=monitor
+module=mysqlmon
+servers=server1,server2,server3,server4
+user=maxskysql
+password=skysql
+monitor_interval=1000
+auto_failover=true
+
+[RW Split Router]
+type=service
+router=readwritesplit
+servers=server1,server2,server3,server4
+user=maxskysql
+password=skysql
+
+[RW Split Listener]
+type=listener
+service=RW Split Router
+protocol=MySQLClient
+port=4006
+
+[CLI]
+type=service
+router=cli
+
+[CLI Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=default
+
+[server1]
+type=server
+address=###node_server_IP_1###
+port=###node_server_port_1###
+protocol=MySQLBackend
+
+[server2]
+type=server
+address=###node_server_IP_2###
+port=###node_server_port_2###
+protocol=MySQLBackend
+
+[server3]
+type=server
+address=###node_server_IP_3###
+port=###node_server_port_3###
+protocol=MySQLBackend
+
+[server4]
+type=server
+address=###node_server_IP_4###
+port=###node_server_port_4###
+protocol=MySQLBackend

maxscale-system-test/mariadbmonitor/verify_master_failure.cpp

@@ -0,0 +1,27 @@
+/**
+ * MXS-1493: https://jira.mariadb.org/browse/MXS-1493
+ *
+ * Testing of master failure verification
+ */
+
+#include "testconnections.h"
+
+int main(int argc, char* argv[])
+{
+    Mariadb_nodes::require_gtid(true);
+    TestConnections test(argc, argv);
+
+    test.tprintf("Blocking master and checking that master failure is delayed at least once.");
+    test.repl->block_node(0);
+    sleep(5);
+    test.log_includes(0, "Delaying failover");
+
+    test.tprintf("Waiting to see if failover is performed.");
+    sleep(10);
+
+    test.log_includes(0, "Performing.*failover");
+
+    // TODO: Extend the test
+
+    return test.global_result;
+}