diff --git a/maxscale-system-test/.gitignore b/maxscale-system-test/.gitignore index ce111daa4..5d831b984 100644 --- a/maxscale-system-test/.gitignore +++ b/maxscale-system-test/.gitignore @@ -183,6 +183,8 @@ mysqlmon_failover_manual mysqlmon_rejoin_good mysqlmon_rejoin_bad mysqlmon_rejoin_bad2 +mysqlmon_rejoin_manual +mysqlmon_rejoin_manual2 namedserverfilter no_password non_native_setup diff --git a/maxscale-system-test/CMakeLists.txt b/maxscale-system-test/CMakeLists.txt index 775f75e15..f497a20fc 100644 --- a/maxscale-system-test/CMakeLists.txt +++ b/maxscale-system-test/CMakeLists.txt @@ -281,6 +281,10 @@ add_test_executable(mysqlmon_rejoin_bad.cpp mysqlmon_rejoin_bad mysqlmon_rejoin_ # MySQL Monitor Rejoin (bad2) Test, use template for Rejoin (good) add_test_executable(mysqlmon_rejoin_bad2.cpp mysqlmon_rejoin_bad2 mysqlmon_rejoin_good LABELS mysqlmon REPL_BACKEND) +# MySQL Monitor Rejoin tests +add_test_executable(mysqlmon_rejoin_manual.cpp mysqlmon_rejoin_manual mysqlmon_rejoin_manual LABELS mysqlmon REPL_BACKEND) +add_test_executable(mysqlmon_rejoin_manual2.cpp mysqlmon_rejoin_manual2 mysqlmon_rejoin_manual LABELS mysqlmon REPL_BACKEND) + # MySQL Monitor rolling master add_test_executable(mysqlmon_failover_rolling_master.cpp mysqlmon_failover_rolling_master mysqlmon_failover_rolling_master LABELS mysqlmon REPL_BACKEND) diff --git a/maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_rejoin_manual b/maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_rejoin_manual new file mode 100644 index 000000000..dc66d89ce --- /dev/null +++ b/maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_rejoin_manual @@ -0,0 +1,94 @@ +[maxscale] +threads=###threads### + +[MySQL-Monitor] +type=monitor +module=mysqlmon +servers= server1, server2, server3, server4 +user=maxskysql +passwd= skysql +monitor_interval=1000 +detect_standalone_master=true +failcount=1 +allow_cluster_recovery=true +replication_user=repl +replication_password=repl +backend_connect_timeout=3 +backend_read_timeout=3 +backend_write_timeout=3 +auto_failover=true + +[RW Split Router] +type=service +router= readwritesplit +servers=server1, server2, server3, server4 +user=maxskysql +passwd=skysql + +[Read Connection Router Slave] +type=service +router=readconnroute +router_options= slave +servers=server1, server2, server3, server4 +user=maxskysql +passwd=skysql + +[Read Connection Router Master] +type=service +router=readconnroute +router_options=master +servers=server1, server2, server3, server4 +user=maxskysql +passwd=skysql + +[RW Split Listener] +type=listener +service=RW Split Router +protocol=MySQLClient +port=4006 + +[Read Connection Listener Slave] +type=listener +service=Read Connection Router Slave +protocol=MySQLClient +port=4009 + +[Read Connection Listener Master] +type=listener +service=Read Connection Router Master +protocol=MySQLClient +port=4008 + +[CLI] +type=service +router=cli + +[CLI Listener] +type=listener +service=CLI +protocol=maxscaled +socket=default + +[server1] +type=server +address=###node_server_IP_1### +port=###node_server_port_1### +protocol=MySQLBackend + +[server2] +type=server +address=###node_server_IP_2### +port=###node_server_port_2### +protocol=MySQLBackend + +[server3] +type=server +address=###node_server_IP_3### +port=###node_server_port_3### +protocol=MySQLBackend + +[server4] +type=server +address=###node_server_IP_4### +port=###node_server_port_4### +protocol=MySQLBackend diff --git a/maxscale-system-test/mysqlmon_rejoin_manual.cpp b/maxscale-system-test/mysqlmon_rejoin_manual.cpp new file mode 100644 index 000000000..51739b34e --- /dev/null +++ b/maxscale-system-test/mysqlmon_rejoin_manual.cpp @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2016 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl11. + * + * Change Date: 2020-01-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ + +#include "fail_switch_rejoin_common.cpp" +#include + +using std::string; +using std::cout; +using std::endl; + +int main(int argc, char** argv) +{ + interactive = strcmp(argv[argc - 1], "interactive") == 0; + Mariadb_nodes::require_gtid(true); + TestConnections test(argc, argv); + int ec; + MYSQL* maxconn = test.maxscales->open_rwsplit_connection(0); + // Set up test table + basic_test(test); + // Delete binlogs to sync gtid:s + delete_slave_binlogs(test); + char result_tmp[bufsize]; + // Advance gtid:s a bit to so gtid variables are updated. + generate_traffic_and_check(test, maxconn, 10); + sleep(1); + test.tprintf(LINE); + print_gtids(test); + get_input(); + + cout << "Stopping master and waiting for failover. Check that another server is promoted." << endl; + const int old_master_id = get_master_server_id(test); // Read master id now before shutdown. + const int master_index = test.repl->master; + test.repl->stop_node(master_index); + sleep(10); + // Recreate maxscale session + mysql_close(maxconn); + maxconn = test.maxscales->open_rwsplit_connection(0); + get_output(test); + int master_id = get_master_server_id(test); + cout << "Master server id is " << master_id << endl; + const bool failover_ok = (master_id > 0 && master_id != old_master_id); + test.assert(failover_ok, "Master did not change or no master detected."); + string gtid_final; + if (failover_ok) + { + cout << "Sending more inserts." << endl; + generate_traffic_and_check(test, maxconn, 5); + sleep(1); + if (find_field(maxconn, GTID_QUERY, GTID_FIELD, result_tmp) == 0) + { + gtid_final = result_tmp; + } + print_gtids(test); + cout << "Bringing old master back online..." << endl; + test.repl->start_node(master_index, (char*) ""); + sleep(5); + test.repl->connect(); + get_output(test); + test.tprintf("and manually rejoining it to cluster."); + const char REJOIN_CMD[] = "maxadmin call command mariadbmon rejoin MySQL-Monitor server1"; + test.maxscales->ssh_node_output(0, REJOIN_CMD , true, &ec); + sleep(2); + get_output(test); + + string gtid_old_master; + if (find_field(test.repl->nodes[master_index], GTID_QUERY, GTID_FIELD, result_tmp) == 0) + { + gtid_old_master = result_tmp; + } + cout << LINE << "\n"; + print_gtids(test); + cout << LINE << "\n"; + test.assert(gtid_final == gtid_old_master, "Old master did not successfully rejoin the cluster."); + // Switch master back to server1 so last check is faster + int ec; + test.maxscales->ssh_node_output(0, "maxadmin call command mysqlmon switchover " + "MySQL-Monitor server1 server2" , true, &ec); + sleep(5); // Wait for monitor to update status + get_output(test); + master_id = get_master_server_id(test); + test.assert(master_id == old_master_id, "Switchover back to server1 failed."); + } + else + { + test.repl->start_node(master_index, (char*) ""); + sleep(10); + } + + test.repl->fix_replication(); + return test.global_result; +} diff --git a/maxscale-system-test/mysqlmon_rejoin_manual2.cpp b/maxscale-system-test/mysqlmon_rejoin_manual2.cpp new file mode 100644 index 000000000..451023bcd --- /dev/null +++ b/maxscale-system-test/mysqlmon_rejoin_manual2.cpp @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2016 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl11. + * + * Change Date: 2020-01-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ + +#include + +#include "testconnections.h" +#include "fail_switch_rejoin_common.cpp" +#include + +using std::string; +using std::cout; +using std::endl; + +int main(int argc, char** argv) +{ + char result_tmp[bufsize]; + interactive = strcmp(argv[argc - 1], "interactive") == 0; + Mariadb_nodes::require_gtid(true); + TestConnections test(argc, argv); + MYSQL* maxconn = test.maxscales->open_rwsplit_connection(0); + + // Set up test table + basic_test(test); + // Delete binlogs to sync gtid:s + delete_slave_binlogs(test); + // Advance gtid:s a bit to so gtid variables are updated. + generate_traffic_and_check(test, maxconn, 10); + test.repl->sync_slaves(0); + + cout << LINE << "\n"; + print_gtids(test); + cout << LINE << "\n"; + string gtid_begin; + if (find_field(maxconn, GTID_QUERY, GTID_FIELD, result_tmp) == 0) + { + gtid_begin = result_tmp; + } + mysql_close(maxconn); + + // Leave first of three slaves connected so it's clear which one is the master server. + const char STOP_SLAVE[] = "STOP SLAVE;"; + const char RESET_SLAVE[] = "RESET SLAVE ALL;"; + const char READ_ONLY_OFF[] = "SET GLOBAL read_only=0;"; + test.repl->connect(); + const int FIRST_MOD_NODE = 2; // Modify nodes 2 & 3 + const int NODE_COUNT = test.repl->N; + MYSQL** nodes = test.repl->nodes; + + for (int i = FIRST_MOD_NODE; i < NODE_COUNT; i++) + { + if (mysql_query(nodes[i], STOP_SLAVE) != 0 || + mysql_query(nodes[i], RESET_SLAVE) != 0 || + mysql_query(nodes[i], READ_ONLY_OFF) != 0) + { + test.assert(false, "Could not stop slave connections and/or disable read_only for node %d.", i); + return test.global_result; + } + } + + // Add more events to node3. + string gtid_node2, gtid_node3; + cout << "Sending more inserts to server 4.\n"; + generate_traffic_and_check(test, nodes[3], 10); + // Save gtids + if (find_field(nodes[2], GTID_QUERY, GTID_FIELD, result_tmp) == 0) + { + gtid_node2 = result_tmp; + } + if (find_field(nodes[3], GTID_QUERY, GTID_FIELD, result_tmp) == 0) + { + gtid_node3 = result_tmp; + } + print_gtids(test); + bool gtids_ok = (gtid_begin == gtid_node2 && gtid_node2 < gtid_node3); + test.assert(gtids_ok, "Gtid:s have not advanced correctly."); + if (!gtids_ok) + { + return test.global_result; + } + cout << "Sending rejoin commands for servers 3 & 4. Server 4 should not rejoin the cluster.\n"; + const string REJOIN_CMD = "maxadmin call command mariadbmon rejoin MySQL-Monitor"; + int ec; + string rejoin_s3 = REJOIN_CMD + " server3"; + string rejoin_s4 = REJOIN_CMD + " server4"; + test.maxscales->ssh_node_output(0, rejoin_s3.c_str() , true, &ec); + test.maxscales->ssh_node_output(0, rejoin_s4.c_str() , true, &ec); + sleep(5); + get_output(test); + + StringSet node2_states = test.get_server_status("server3"); + StringSet node3_states = test.get_server_status("server4"); + bool states_n2_ok = (node2_states.find("Slave") != node2_states.end()); + bool states_n3_ok = (node3_states.find("Slave") == node3_states.end()); + test.assert(states_n2_ok, "Node 2 has not rejoined when it should have."); + test.assert(states_n3_ok, "Node 3 rejoined when it shouldn't have."); + if (!states_n2_ok || !states_n3_ok) + { + return test.global_result; + } + // Finally, fix replication by telling the current master to replicate from server4 + test.tprintf("Setting server 1 to replicate from server 4. Manually rejoin servers 2 and 3."); + const char CHANGE_CMD_FMT[] = "CHANGE MASTER TO MASTER_HOST = '%s', MASTER_PORT = %d, " + "MASTER_USE_GTID = current_pos, MASTER_USER='repl', MASTER_PASSWORD = 'repl';"; + char cmd[256]; + snprintf(cmd, sizeof(cmd), CHANGE_CMD_FMT, test.repl->IP[3], test.repl->port[3]); + mysql_query(nodes[0], cmd); + mysql_query(nodes[0], "START SLAVE;"); + sleep(5); + string rejoin_s2 = REJOIN_CMD + " server2"; + test.maxscales->ssh_node_output(0, rejoin_s2.c_str() , true, &ec); + test.maxscales->ssh_node_output(0, rejoin_s3.c_str() , true, &ec); + sleep(2); + get_output(test); + int master_id = get_master_server_id(test); + test.assert(master_id == 4, "Server 4 should be the cluster master."); + StringSet node0_states = test.get_server_status("server1"); + bool states_n0_ok = (node0_states.find("Slave") != node0_states.end() && + node0_states.find("Relay Master") == node0_states.end()); + test.assert(states_n0_ok, "Server 1 is not a slave when it should be."); + if (states_n0_ok) + { + int ec; + test.maxscales->ssh_node_output(0, + "maxadmin call command mysqlmon switchover MySQL-Monitor server1 server4" , true, &ec); + sleep(1); + master_id = get_master_server_id(test); + test.assert(master_id == 1, "Server 1 should be the cluster master."); + get_output(test); + } + + test.repl->fix_replication(); + return test.global_result; +}