MXS-1561 Switchover to bad master

auto_failover=true
auto_rejoin=false

This test tests the following:

- Regular master-slave setup
- Create a table, insert some data
- Sync all slaves
- Stop a slave
- Insert some more data
- Sync remaining slaves
- Stop the master
- Expect the failover mechanism to pick a new master (server2)
- Bring up the slave
- Perform a switchover from server2 to server4
- Should fail

Currently it does fail, but only due to a timeout.

    [mysqlmon] MASTER_GTID_WAIT() timed out on slave 'server4'.

There should be some check that would ensure that the failure happens
faster than that.

maxscale-system-test/.gitignore

@@ -105,6 +105,7 @@ max_connections
 maxscale_process_user
 mm
 mm_mysqlmon
+mysqlmon_failover_bad_master
 mysqlmon_failover_manual
 mysqlmon_failover_manual2_4
 mysqlmon_failover_manual2_3

maxscale-system-test/CMakeLists.txt

@@ -263,6 +263,9 @@ add_test_executable(mysqlmon_failover_manual2.cpp mysqlmon_failover_manual2_4 my
 add_test_executable(mysqlmon_failover_manual2.cpp mysqlmon_failover_manual2_3 mysqlmon_failover_manual2_3 LABELS mysqlmon REPL_BACKEND)
 add_test_executable(mysqlmon_failover_manual2.cpp mysqlmon_failover_manual2_2 mysqlmon_failover_manual2_2 LABELS mysqlmon REPL_BACKEND)
 
+# MySQL Monitor manual failover with bad master
+add_test_executable(mysqlmon_failover_bad_master.cpp mysqlmon_failover_bad_master mysqlmon_failover_bad_master LABELS mysqlmon REPL_BACKEND)
+
 # MySQL Monitor Rejoin Test
 add_test_executable(mysqlmon_rejoin_good.cpp mysqlmon_rejoin_good mysqlmon_rejoin_good LABELS mysqlmon REPL_BACKEND)
 

maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_failover_bad_master

@@ -0,0 +1,92 @@
+[maxscale]
+threads=###threads###
+
+[MySQL-Monitor]
+type=monitor
+module=mysqlmon
+servers= server1, server2, server3, server4
+user=maxskysql
+passwd= skysql
+monitor_interval=1000
+allow_cluster_recovery=true
+detect_standalone_master=true
+auto_failover=true
+auto_rejoin=false
+replication_user=repl
+replication_password=repl
+backend_connect_timeout=1
+
+[RW-Split-Router]
+type=service
+router= readwritesplit
+servers=server1, server2, server3, server4
+user=maxskysql
+passwd=skysql
+
+[Read-Connection-Router-Slave]
+type=service
+router=readconnroute
+router_options= slave
+servers=server1, server2, server3, server4
+user=maxskysql
+passwd=skysql
+
+[Read-Connection-Router-Master]
+type=service
+router=readconnroute
+router_options=master
+servers=server1, server2, server3, server4
+user=maxskysql
+passwd=skysql
+
+[RW-Split-Listener]
+type=listener
+service=RW-Split-Router
+protocol=MySQLClient
+port=4006
+
+[Read-Connection-Listener-Slave]
+type=listener
+service=Read-Connection-Router-Slave
+protocol=MySQLClient
+port=4009
+
+[Read-Connection-Listener-Master]
+type=listener
+service=Read-Connection-Router-Master
+protocol=MySQLClient
+port=4008
+
+[CLI]
+type=service
+router=cli
+
+[CLI Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=default
+
+[server1]
+type=server
+address=###node_server_IP_1###
+port=###node_server_port_1###
+protocol=MySQLBackend
+
+[server2]
+type=server
+address=###node_server_IP_2###
+port=###node_server_port_2###
+protocol=MySQLBackend
+
+[server3]
+type=server
+address=###node_server_IP_3###
+port=###node_server_port_3###
+protocol=MySQLBackend
+
+[server4]
+type=server
+address=###node_server_IP_4###
+port=###node_server_port_4###
+protocol=MySQLBackend

maxscale-system-test/mysqlmon_failover_bad_master.cpp

@@ -0,0 +1,220 @@
+/*
+ * Copyright (c) 2016 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2020-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#include <iostream>
+#include <iterator>
+#include <string>
+#include <sstream>
+#include "testconnections.h"
+
+using std::cerr;
+using std::cout;
+using std::endl;
+using std::flush;
+using std::string;
+using std::stringstream;
+
+namespace
+{
+
+void sleep(int s)
+{
+    cout << "Sleeping " << s << " times 1 second" << flush;
+    do
+    {
+        ::sleep(1);
+        cout << "." << flush;
+        --s;
+    }
+    while (s > 0);
+
+    cout << endl;
+}
+
+}
+
+namespace
+{
+
+void create_table(TestConnections& test)
+{
+    MYSQL* pConn = test.maxscales->conn_rwsplit[0];
+
+    test.try_query(pConn, "DROP TABLE IF EXISTS test.t1");
+    test.try_query(pConn, "CREATE TABLE test.t1(id INT)");
+}
+
+static int i_start = 0;
+static int n_rows = 20;
+static int i_end = 0;
+
+void insert_data(TestConnections& test)
+{
+    MYSQL* pConn = test.maxscales->conn_rwsplit[0];
+
+    test.try_query(pConn, "BEGIN");
+
+    i_end = i_start + n_rows;
+
+    for (int i = i_start; i < i_end; ++i)
+    {
+        stringstream ss;
+        ss << "INSERT INTO test.t1 VALUES (" << i << ")";
+        test.try_query(pConn, ss.str().c_str());
+    }
+
+    test.try_query(pConn, "COMMIT");
+
+    i_start = i_end;
+}
+
+void expect(TestConnections& test, const char* zServer, const StringSet& expected)
+{
+    StringSet found = test.get_server_status(zServer);
+
+    std::ostream_iterator<string> oi(cout, ", ");
+
+    cout << zServer
+         << ", expected states: ";
+    std::copy(expected.begin(), expected.end(), oi);
+    cout << endl;
+
+    cout << zServer
+         << ", found states   : ";
+    std::copy(found.begin(), found.end(), oi);
+    cout << endl;
+
+    if (found != expected)
+    {
+        cout << "ERROR, found states are not the same as the expected ones." << endl;
+        ++test.global_result;
+    }
+
+    cout << endl;
+}
+
+void expect(TestConnections& test, const char* zServer, const char* zState)
+{
+    StringSet s;
+    s.insert(zState);
+
+    expect(test, zServer, s);
+}
+
+void expect(TestConnections& test, const char* zServer, const char* zState1, const char* zState2)
+{
+    StringSet s;
+    s.insert(zState1);
+    s.insert(zState2);
+
+    expect(test, zServer, s);
+}
+
+void run(TestConnections& test)
+{
+    sleep(5);
+
+    int N = test.repl->N;
+    cout << "Nodes: " << N << endl;
+
+    expect(test, "server1", "Master", "Running");
+    expect(test, "server2", "Slave", "Running");
+    expect(test, "server3", "Slave", "Running");
+    expect(test, "server4", "Slave", "Running");
+
+    cout << "\nConnecting to MaxScale." << endl;
+    test.maxscales->connect_maxscale(0);
+
+    cout << "\nCreating table." << endl;
+    create_table(test);
+
+    cout << "\nInserting data." << endl;
+    insert_data(test);
+
+    cout << "\nSyncing slaves." << endl;
+    test.repl->sync_slaves();
+
+    cout << "\nStopping slave " << N - 1 << endl;
+    test.repl->stop_node(N - 1);
+
+    sleep(8);
+
+    // server4 was stopped, so we expect the state of it to be /Down/,
+    // and the states of the other ones not to have changed.
+    expect(test, "server1", "Master", "Running");
+    expect(test, "server2", "Slave", "Running");
+    expect(test, "server3", "Slave", "Running");
+    expect(test, "server4", "Down");
+
+    cout << "\nClosing connection to MaxScale." << endl;
+    test.maxscales->close_maxscale_connections(0);
+
+    cout << "\nConnecting to MaxScale." << endl;
+    test.maxscales->connect_maxscale(0);
+
+    cout << "\nInserting data." << endl;
+    insert_data(test);
+
+    cout << "\nSyncing slaves." << endl;
+    test.repl->sync_slaves();
+
+    cout << "\nStopping master." << endl;
+    test.repl->stop_node(0);
+
+    sleep(8);
+
+    // server1 (previous master) was taken down, so its state should be /Down/.
+    // server2 should have been made into master, and server4 should still be down.
+    expect(test, "server1", "Down");
+    expect(test, "server2", "Master", "Running");
+    expect(test, "server3", "Slave", "Running");
+    expect(test, "server4", "Down");
+
+    cout << "\nBringing up slave " << N - 1 << endl;
+    test.repl->start_node(N - 1, (char*)"");
+
+    sleep(8);
+
+    // server1 should still be down, server2 still master, and server3 still
+    // a slave. server4 was brought up, but as auto_rejoin is false, it should
+    // be Running, but not Slave.
+    // turned into a slave.
+    expect(test, "server1", "Down");
+    expect(test, "server2", "Master", "Running");
+    expect(test, "server3", "Slave", "Running");
+    expect(test, "server4", "Running");
+
+    cout << "\nTrying to do manual switchover to server4" << endl;
+    test.maxscales->execute_maxadmin_command_print(0, (char*)"call command mysqlmon switchover MySQL-Monitor server4 server2");
+
+    sleep(8);
+
+    // The state should not change, as server4 is not good enough as master.
+    expect(test, "server1", "Down");
+    expect(test, "server2", "Master", "Running");
+    expect(test, "server3", "Slave", "Running");
+    expect(test, "server4", "Running");
+
+}
+
+}
+
+int main(int argc, char* argv[])
+{
+    TestConnections test(argc, argv);
+
+    run(test);
+
+    return test.global_result;
+}
+