From c4ba3ba4ca9c45f53bec286a7d099bc0602f9bf7 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Mon, 18 Dec 2017 15:41:21 +0200 Subject: [PATCH] MXS-1561 Switchover to bad master auto_failover=true auto_rejoin=false This test tests the following: - Regular master-slave setup - Create a table, insert some data - Sync all slaves - Stop a slave - Insert some more data - Sync remaining slaves - Stop the master - Expect the failover mechanism to pick a new master (server2) - Bring up the slave - Perform a switchover from server2 to server4 - Should fail Currently it does fail, but only due to a timeout. [mysqlmon] MASTER_GTID_WAIT() timed out on slave 'server4'. There should be some check that would ensure that the failure happens faster than that. --- maxscale-system-test/.gitignore | 1 + maxscale-system-test/CMakeLists.txt | 3 + ....cnf.template.mysqlmon_failover_bad_master | 92 ++++++++ .../mysqlmon_failover_bad_master.cpp | 220 ++++++++++++++++++ 4 files changed, 316 insertions(+) create mode 100644 maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_failover_bad_master create mode 100644 maxscale-system-test/mysqlmon_failover_bad_master.cpp diff --git a/maxscale-system-test/.gitignore b/maxscale-system-test/.gitignore index c396856e5..17c1ca033 100644 --- a/maxscale-system-test/.gitignore +++ b/maxscale-system-test/.gitignore @@ -105,6 +105,7 @@ max_connections maxscale_process_user mm mm_mysqlmon +mysqlmon_failover_bad_master mysqlmon_failover_manual mysqlmon_failover_manual2_4 mysqlmon_failover_manual2_3 diff --git a/maxscale-system-test/CMakeLists.txt b/maxscale-system-test/CMakeLists.txt index b29c7d091..6101e179b 100644 --- a/maxscale-system-test/CMakeLists.txt +++ b/maxscale-system-test/CMakeLists.txt @@ -263,6 +263,9 @@ add_test_executable(mysqlmon_failover_manual2.cpp mysqlmon_failover_manual2_4 my add_test_executable(mysqlmon_failover_manual2.cpp mysqlmon_failover_manual2_3 mysqlmon_failover_manual2_3 LABELS mysqlmon REPL_BACKEND) add_test_executable(mysqlmon_failover_manual2.cpp mysqlmon_failover_manual2_2 mysqlmon_failover_manual2_2 LABELS mysqlmon REPL_BACKEND) +# MySQL Monitor manual failover with bad master +add_test_executable(mysqlmon_failover_bad_master.cpp mysqlmon_failover_bad_master mysqlmon_failover_bad_master LABELS mysqlmon REPL_BACKEND) + # MySQL Monitor Rejoin Test add_test_executable(mysqlmon_rejoin_good.cpp mysqlmon_rejoin_good mysqlmon_rejoin_good LABELS mysqlmon REPL_BACKEND) diff --git a/maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_failover_bad_master b/maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_failover_bad_master new file mode 100644 index 000000000..7b0cc6c30 --- /dev/null +++ b/maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_failover_bad_master @@ -0,0 +1,92 @@ +[maxscale] +threads=###threads### + +[MySQL-Monitor] +type=monitor +module=mysqlmon +servers= server1, server2, server3, server4 +user=maxskysql +passwd= skysql +monitor_interval=1000 +allow_cluster_recovery=true +detect_standalone_master=true +auto_failover=true +auto_rejoin=false +replication_user=repl +replication_password=repl +backend_connect_timeout=1 + +[RW-Split-Router] +type=service +router= readwritesplit +servers=server1, server2, server3, server4 +user=maxskysql +passwd=skysql + +[Read-Connection-Router-Slave] +type=service +router=readconnroute +router_options= slave +servers=server1, server2, server3, server4 +user=maxskysql +passwd=skysql + +[Read-Connection-Router-Master] +type=service +router=readconnroute +router_options=master +servers=server1, server2, server3, server4 +user=maxskysql +passwd=skysql + +[RW-Split-Listener] +type=listener +service=RW-Split-Router +protocol=MySQLClient +port=4006 + +[Read-Connection-Listener-Slave] +type=listener +service=Read-Connection-Router-Slave +protocol=MySQLClient +port=4009 + +[Read-Connection-Listener-Master] +type=listener +service=Read-Connection-Router-Master +protocol=MySQLClient +port=4008 + +[CLI] +type=service +router=cli + +[CLI Listener] +type=listener +service=CLI +protocol=maxscaled +socket=default + +[server1] +type=server +address=###node_server_IP_1### +port=###node_server_port_1### +protocol=MySQLBackend + +[server2] +type=server +address=###node_server_IP_2### +port=###node_server_port_2### +protocol=MySQLBackend + +[server3] +type=server +address=###node_server_IP_3### +port=###node_server_port_3### +protocol=MySQLBackend + +[server4] +type=server +address=###node_server_IP_4### +port=###node_server_port_4### +protocol=MySQLBackend diff --git a/maxscale-system-test/mysqlmon_failover_bad_master.cpp b/maxscale-system-test/mysqlmon_failover_bad_master.cpp new file mode 100644 index 000000000..80d472163 --- /dev/null +++ b/maxscale-system-test/mysqlmon_failover_bad_master.cpp @@ -0,0 +1,220 @@ +/* + * Copyright (c) 2016 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl11. + * + * Change Date: 2020-01-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ + +#include +#include +#include +#include +#include "testconnections.h" + +using std::cerr; +using std::cout; +using std::endl; +using std::flush; +using std::string; +using std::stringstream; + +namespace +{ + +void sleep(int s) +{ + cout << "Sleeping " << s << " times 1 second" << flush; + do + { + ::sleep(1); + cout << "." << flush; + --s; + } + while (s > 0); + + cout << endl; +} + +} + +namespace +{ + +void create_table(TestConnections& test) +{ + MYSQL* pConn = test.maxscales->conn_rwsplit[0]; + + test.try_query(pConn, "DROP TABLE IF EXISTS test.t1"); + test.try_query(pConn, "CREATE TABLE test.t1(id INT)"); +} + +static int i_start = 0; +static int n_rows = 20; +static int i_end = 0; + +void insert_data(TestConnections& test) +{ + MYSQL* pConn = test.maxscales->conn_rwsplit[0]; + + test.try_query(pConn, "BEGIN"); + + i_end = i_start + n_rows; + + for (int i = i_start; i < i_end; ++i) + { + stringstream ss; + ss << "INSERT INTO test.t1 VALUES (" << i << ")"; + test.try_query(pConn, ss.str().c_str()); + } + + test.try_query(pConn, "COMMIT"); + + i_start = i_end; +} + +void expect(TestConnections& test, const char* zServer, const StringSet& expected) +{ + StringSet found = test.get_server_status(zServer); + + std::ostream_iterator oi(cout, ", "); + + cout << zServer + << ", expected states: "; + std::copy(expected.begin(), expected.end(), oi); + cout << endl; + + cout << zServer + << ", found states : "; + std::copy(found.begin(), found.end(), oi); + cout << endl; + + if (found != expected) + { + cout << "ERROR, found states are not the same as the expected ones." << endl; + ++test.global_result; + } + + cout << endl; +} + +void expect(TestConnections& test, const char* zServer, const char* zState) +{ + StringSet s; + s.insert(zState); + + expect(test, zServer, s); +} + +void expect(TestConnections& test, const char* zServer, const char* zState1, const char* zState2) +{ + StringSet s; + s.insert(zState1); + s.insert(zState2); + + expect(test, zServer, s); +} + +void run(TestConnections& test) +{ + sleep(5); + + int N = test.repl->N; + cout << "Nodes: " << N << endl; + + expect(test, "server1", "Master", "Running"); + expect(test, "server2", "Slave", "Running"); + expect(test, "server3", "Slave", "Running"); + expect(test, "server4", "Slave", "Running"); + + cout << "\nConnecting to MaxScale." << endl; + test.maxscales->connect_maxscale(0); + + cout << "\nCreating table." << endl; + create_table(test); + + cout << "\nInserting data." << endl; + insert_data(test); + + cout << "\nSyncing slaves." << endl; + test.repl->sync_slaves(); + + cout << "\nStopping slave " << N - 1 << endl; + test.repl->stop_node(N - 1); + + sleep(8); + + // server4 was stopped, so we expect the state of it to be /Down/, + // and the states of the other ones not to have changed. + expect(test, "server1", "Master", "Running"); + expect(test, "server2", "Slave", "Running"); + expect(test, "server3", "Slave", "Running"); + expect(test, "server4", "Down"); + + cout << "\nClosing connection to MaxScale." << endl; + test.maxscales->close_maxscale_connections(0); + + cout << "\nConnecting to MaxScale." << endl; + test.maxscales->connect_maxscale(0); + + cout << "\nInserting data." << endl; + insert_data(test); + + cout << "\nSyncing slaves." << endl; + test.repl->sync_slaves(); + + cout << "\nStopping master." << endl; + test.repl->stop_node(0); + + sleep(8); + + // server1 (previous master) was taken down, so its state should be /Down/. + // server2 should have been made into master, and server4 should still be down. + expect(test, "server1", "Down"); + expect(test, "server2", "Master", "Running"); + expect(test, "server3", "Slave", "Running"); + expect(test, "server4", "Down"); + + cout << "\nBringing up slave " << N - 1 << endl; + test.repl->start_node(N - 1, (char*)""); + + sleep(8); + + // server1 should still be down, server2 still master, and server3 still + // a slave. server4 was brought up, but as auto_rejoin is false, it should + // be Running, but not Slave. + // turned into a slave. + expect(test, "server1", "Down"); + expect(test, "server2", "Master", "Running"); + expect(test, "server3", "Slave", "Running"); + expect(test, "server4", "Running"); + + cout << "\nTrying to do manual switchover to server4" << endl; + test.maxscales->execute_maxadmin_command_print(0, (char*)"call command mysqlmon switchover MySQL-Monitor server4 server2"); + + sleep(8); + + // The state should not change, as server4 is not good enough as master. + expect(test, "server1", "Down"); + expect(test, "server2", "Master", "Running"); + expect(test, "server3", "Slave", "Running"); + expect(test, "server4", "Running"); + +} + +} + +int main(int argc, char* argv[]) +{ + TestConnections test(argc, argv); + + run(test); + + return test.global_result; +} +