From 1d91895f417401806a6f19b1c1ffce759a982abd Mon Sep 17 00:00:00 2001 From: Esa Korhonen Date: Tue, 27 Nov 2018 12:19:15 +0200 Subject: [PATCH] Fix switchover_auto and rejoin_bad tests Increased some timeouts which were a bit too short in some cases. Cleaned up the tests too. --- maxscale-system-test/mysqlmon_rejoin_bad.cpp | 131 +++++++++--------- .../mysqlmon_switchover_auto.cpp | 11 +- 2 files changed, 71 insertions(+), 71 deletions(-) diff --git a/maxscale-system-test/mysqlmon_rejoin_bad.cpp b/maxscale-system-test/mysqlmon_rejoin_bad.cpp index 046fa1f30..b2cd715c3 100644 --- a/maxscale-system-test/mysqlmon_rejoin_bad.cpp +++ b/maxscale-system-test/mysqlmon_rejoin_bad.cpp @@ -11,41 +11,44 @@ * Public License. */ +#include #include #include "testconnections.h" #include "fail_switch_rejoin_common.cpp" using std::string; +using std::cout; int main(int argc, char** argv) { - char result_tmp[bufsize]; interactive = strcmp(argv[argc - 1], "interactive") == 0; Mariadb_nodes::require_gtid(true); TestConnections test(argc, argv); - MYSQL* maxconn = test.maxscales->open_rwsplit_connection(0); - - // Set up test table - basic_test(test); + test.repl->connect(); // Delete binlogs to sync gtid:s delete_slave_binlogs(test); + // Set up test table + basic_test(test); // Advance gtid:s a bit to so gtid variables are updated. + MYSQL* maxconn = test.maxscales->open_rwsplit_connection(0); generate_traffic_and_check(test, maxconn, 10); test.repl->sync_slaves(0); test.tprintf(LINE); print_gtids(test); test.tprintf(LINE); + char result_tmp[bufsize]; string gtid_begin; if (find_field(maxconn, GTID_QUERY, GTID_FIELD, result_tmp) == 0) { gtid_begin = result_tmp; } mysql_close(maxconn); + test.tprintf("Stopping MaxScale..."); // Mess with the slaves to fix situation such that only one slave can be rejoined. Stop maxscale. - if (test.stop_maxscale(0)) + if (test.stop_maxscale(0) != 0) { test.expect(false, "Could not stop MaxScale."); return test.global_result; @@ -55,91 +58,87 @@ int main(int argc, char** argv) const char STOP_SLAVE[] = "STOP SLAVE;"; const char RESET_SLAVE[] = "RESET SLAVE ALL;"; const char READ_ONLY_OFF[] = "SET GLOBAL read_only=0;"; - test.repl->connect(); + const int FIRST_MOD_NODE = 2; // Modify nodes 2 & 3 const int NODE_COUNT = test.repl->N; MYSQL** nodes = test.repl->nodes; - for (int i = FIRST_MOD_NODE; i < NODE_COUNT; i++) { - if (mysql_query(nodes[i], STOP_SLAVE) != 0 - || mysql_query(nodes[i], RESET_SLAVE) != 0 - || mysql_query(nodes[i], READ_ONLY_OFF) != 0) + if ((mysql_query(nodes[i], STOP_SLAVE) != 0) || (mysql_query(nodes[i], RESET_SLAVE) != 0) + || (mysql_query(nodes[i], READ_ONLY_OFF) != 0)) { test.expect(false, "Could not stop slave connections and/or disable read_only for node %d.", i); - return test.global_result; } } - // Add more events to node3. - string gtid_node2, gtid_node3; - test.tprintf("Sending more inserts to server 4."); - generate_traffic_and_check(test, nodes[3], 10); - // Save gtids - if (find_field(nodes[2], GTID_QUERY, GTID_FIELD, result_tmp) == 0) + if (test.ok()) { - gtid_node2 = result_tmp; - } - if (find_field(nodes[3], GTID_QUERY, GTID_FIELD, result_tmp) == 0) - { - gtid_node3 = result_tmp; - } - print_gtids(test); - bool gtids_ok = (gtid_begin == gtid_node2 && gtid_node2 < gtid_node3); - test.expect(gtids_ok, "Gtid:s have not advanced correctly."); - if (!gtids_ok) - { - return test.global_result; + // Add more events to node3. + string gtid_node2, gtid_node3; + test.tprintf("Sending more inserts to server 4."); + generate_traffic_and_check(test, nodes[3], 10); + // Save gtids + if (find_field(nodes[2], GTID_QUERY, GTID_FIELD, result_tmp) == 0) + { + gtid_node2 = result_tmp; + } + if (find_field(nodes[3], GTID_QUERY, GTID_FIELD, result_tmp) == 0) + { + gtid_node3 = result_tmp; + } + print_gtids(test); + bool gtids_ok = (gtid_begin == gtid_node2 && gtid_node2 < gtid_node3); + test.expect(gtids_ok, "Gtid:s have not advanced correctly."); } + + test.tprintf("Restarting MaxScale. Server 4 should not rejoin the cluster."); test.tprintf(LINE); - if (test.start_maxscale(0)) + if (test.start_maxscale(0) != 0) { test.expect(false, "Could not start MaxScale."); return test.global_result; } - test.maxscales->wait_for_monitor(); + test.maxscales->wait_for_monitor(2); get_output(test); - StringSet node2_states = test.get_server_status("server3"); - StringSet node3_states = test.get_server_status("server4"); - bool states_n2_ok = (node2_states.find("Slave") != node2_states.end()); - bool states_n3_ok = (node3_states.find("Slave") == node3_states.end()); - test.expect(states_n2_ok, "Node 2 has not rejoined when it should have."); - test.expect(states_n3_ok, "Node 3 rejoined when it shouldn't have."); - if (!states_n2_ok || !states_n3_ok) + if (test.ok()) { - return test.global_result; + StringSet node2_states = test.get_server_status("server3"); + StringSet node3_states = test.get_server_status("server4"); + bool states_n2_ok = (node2_states.find("Slave") != node2_states.end()); + bool states_n3_ok = (node3_states.find("Slave") == node3_states.end()); + test.expect(states_n2_ok, "Node 2 has not rejoined when it should have."); + test.expect(states_n3_ok, "Node 3 rejoined when it shouldn't have."); } - // Finally, fix replication by telling the current master to replicate from server4 - test.tprintf("Setting server 1 to replicate from server 4. Auto-rejoin should redirect servers 2 and 3."); - const char CHANGE_CMD_FMT[] = "CHANGE MASTER TO MASTER_HOST = '%s', MASTER_PORT = %d, " - "MASTER_USE_GTID = current_pos, MASTER_USER='repl', MASTER_PASSWORD = 'repl';"; - char cmd[256]; - snprintf(cmd, sizeof(cmd), CHANGE_CMD_FMT, test.repl->IP[3], test.repl->port[3]); - mysql_query(nodes[0], cmd); - mysql_query(nodes[0], "START SLAVE;"); - test.maxscales->wait_for_monitor(); - get_output(test); - int master_id = get_master_server_id(test); - test.expect(master_id == 4, "Server 4 should be the cluster master."); - StringSet node0_states = test.get_server_status("server1"); - bool states_n0_ok = (node0_states.find("Slave") != node0_states.end() - && node0_states.find("Relay Master") == node0_states.end()); - test.expect(states_n0_ok, "Server 1 is not a slave when it should be."); - if (states_n0_ok) + + if (test.ok()) { - int ec; - test.maxscales->ssh_node_output(0, - "maxadmin call command mysqlmon switchover MySQL-Monitor server1 server4", - true, - &ec); - test.maxscales->wait_for_monitor(); - master_id = get_master_server_id(test); - test.expect(master_id == 1, "Server 1 should be the cluster master."); + // Finally, fix replication by telling the current master to replicate from server4 + test.tprintf( + "Setting server 1 to replicate from server 4. Auto-rejoin should redirect servers 2 and 3."); + const char CHANGE_CMD_FMT[] = "CHANGE MASTER TO MASTER_HOST = '%s', MASTER_PORT = %d, " + "MASTER_USE_GTID = current_pos, MASTER_USER='repl', MASTER_PASSWORD = 'repl';"; + char change_cmd[256]; + snprintf(change_cmd, sizeof(change_cmd), CHANGE_CMD_FMT, test.repl->IP[3], test.repl->port[3]); + test.try_query(nodes[0], "%s", change_cmd); + test.try_query(nodes[0], "START SLAVE;"); + test.maxscales->wait_for_monitor(2); get_output(test); + int master_id = get_master_server_id(test); + test.expect(master_id == 4, "Server 4 should be the cluster master."); + StringSet node0_states = test.get_server_status("server1"); + bool states_n0_ok = (node0_states.find("Slave") != node0_states.end() + && node0_states.find("Relay Master") == node0_states.end()); + test.expect(states_n0_ok, "Server 1 is not a slave when it should be."); } - test.repl->fix_replication(); + cout << "Reseting cluster...\n"; + int ec; + string reset_cmd = "maxadmin call command mysqlmon reset-replication MySQL-Monitor server1"; + test.maxscales->ssh_node_output(0, reset_cmd.c_str(), true, &ec); + test.maxscales->wait_for_monitor(1); + test.expect(get_master_server_id(test) == 1, "server1 is not the master when it should. " + "reset-replication must have failed."); return test.global_result; } diff --git a/maxscale-system-test/mysqlmon_switchover_auto.cpp b/maxscale-system-test/mysqlmon_switchover_auto.cpp index 4426ec7bb..ecaacbbcb 100644 --- a/maxscale-system-test/mysqlmon_switchover_auto.cpp +++ b/maxscale-system-test/mysqlmon_switchover_auto.cpp @@ -47,6 +47,7 @@ int main(int argc, char** argv) cout << "Disks-plugin installed and gtid_strict_mode enabled on all servers. " "Starting MaxScale.\n"; test.start_maxscale(); + test.maxscales->wait_for_monitor(1); disks_plugin_loaded = true; } else @@ -61,7 +62,7 @@ int main(int argc, char** argv) rval += elem + ","; } return rval; - }; + }; auto expect_server_status = [&test, &set_to_string](const string& server_name, const string& status) { auto status_set = test.maxscales->get_server_status(server_name.c_str()); @@ -91,7 +92,7 @@ int main(int argc, char** argv) print_gtids(test); expect_server_status(server_names[0], master); - expect_server_status(server_names[1], maint); // Always out of disk space + expect_server_status(server_names[1], maint); // Always out of disk space expect_server_status(server_names[2], slave); expect_server_status(server_names[3], slave); } @@ -101,7 +102,7 @@ int main(int argc, char** argv) // If ok so far, change the disk space threshold to something really small to force a switchover. cout << "Changing disk space threshold for the monitor, should cause a switchover.\n"; test.maxscales->execute_maxadmin_command(0, "alter monitor MySQL-Monitor disk_space_threshold=/:1"); - sleep(2); // The disk space is checked depending on wall clock time. + sleep(2); // The disk space is checked depending on wall clock time. test.maxscales->wait_for_monitor(2); // server2 was in maintenance before the switchover, so it was ignored. This means that it is @@ -122,8 +123,8 @@ int main(int argc, char** argv) cout << "Changing disk space threshold for the monitor, should prevent low disk switchovers.\n"; test.maxscales->execute_maxadmin_command(0, "alter monitor MySQL-Monitor " - "disk_space_threshold=/:100"); - sleep(2); // To update disk space status + "disk_space_threshold=/:100"); + sleep(2); // To update disk space status test.maxscales->wait_for_monitor(1); get_output(test); }