From d0feff5eb34ec148994f1c3acb567edd967bcc9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Tue, 5 Jun 2018 22:06:33 +0300 Subject: [PATCH] Wait for a number of monitor intervals in tests The tests can now wait for a number of monitor intervals. This removes the need to have hard-coded sleeps in the code and makes monitor tests more robust under heavier load. --- ...emplate.mysqlmon_failover_rejoin_old_slave | 1 + ...cale.cnf.template.mysqlmon_failover_stress | 1 + ...nf.template.mysqlmon_switchover_bad_master | 5 +- maxscale-system-test/failover_common.cpp | 12 ++-- maxscale-system-test/failover_mysqlmon.cpp | 4 +- maxscale-system-test/maxscales.cpp | 55 +++++++++++++++++++ maxscale-system-test/maxscales.h | 10 ++++ .../mysqlmon_detect_standalone_master.cpp | 8 +-- .../mysqlmon_external_master.cpp | 12 ++-- .../mysqlmon_failover_auto.cpp | 8 +-- .../mysqlmon_failover_manual.cpp | 10 +--- .../mysqlmon_failover_no_slaves.cpp | 4 +- .../mysqlmon_failover_rejoin_old_slave.cpp | 8 +-- .../mysqlmon_failover_rolling_master.cpp | 4 +- ...sqlmon_failover_rolling_restart_slaves.cpp | 6 +- .../mysqlmon_failover_stress.cpp | 14 ++--- maxscale-system-test/mysqlmon_rejoin_bad.cpp | 6 +- maxscale-system-test/mysqlmon_rejoin_bad2.cpp | 8 +-- maxscale-system-test/mysqlmon_rejoin_good.cpp | 14 ++--- .../mysqlmon_rejoin_manual.cpp | 24 ++++---- .../mysqlmon_rejoin_manual2.cpp | 8 +-- maxscale-system-test/mysqlmon_switchover.cpp | 6 +- .../mysqlmon_switchover_bad_master.cpp | 10 ++-- 23 files changed, 152 insertions(+), 86 deletions(-) diff --git a/maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_failover_rejoin_old_slave b/maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_failover_rejoin_old_slave index 109fd45b7..e124174f5 100644 --- a/maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_failover_rejoin_old_slave +++ b/maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_failover_rejoin_old_slave @@ -17,6 +17,7 @@ replication_password=repl backend_connect_timeout=10 backend_read_timeout=10 backend_write_timeout=10 +failcount=1 [RW-Split-Router] type=service diff --git a/maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_failover_stress b/maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_failover_stress index 109fd45b7..e124174f5 100644 --- a/maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_failover_stress +++ b/maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_failover_stress @@ -17,6 +17,7 @@ replication_password=repl backend_connect_timeout=10 backend_read_timeout=10 backend_write_timeout=10 +failcount=1 [RW-Split-Router] type=service diff --git a/maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_switchover_bad_master b/maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_switchover_bad_master index 7b0cc6c30..5e609ff33 100644 --- a/maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_switchover_bad_master +++ b/maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_switchover_bad_master @@ -14,7 +14,10 @@ auto_failover=true auto_rejoin=false replication_user=repl replication_password=repl -backend_connect_timeout=1 +backend_connect_timeout=10 +backend_read_timeout=10 +backend_write_timeout=10 +failcount=1 [RW-Split-Router] type=service diff --git a/maxscale-system-test/failover_common.cpp b/maxscale-system-test/failover_common.cpp index dc034a2df..9377e117a 100644 --- a/maxscale-system-test/failover_common.cpp +++ b/maxscale-system-test/failover_common.cpp @@ -24,13 +24,13 @@ void reset_replication(TestConnections& test) { int ind = master_id - 1; replicate_from(test, 0, ind); - sleep(3); + test.maxscales->wait_for_monitor(); get_output(test); int ec; stringstream switchover; switchover << "maxadmin call command mysqlmon switchover MySQL-Monitor server1 server" << master_id; test.maxscales->ssh_node_output(0, switchover.str().c_str() , true, &ec); - sleep(3); + test.maxscales->wait_for_monitor(); master_id = get_master_server_id(test); cout << "Master server id is now back to " << master_id << endl; test.assert(master_id == 1, "Switchover back to server1 failed"); @@ -80,11 +80,9 @@ void prepare_test_2(TestConnections& test) cout << LINE << endl; test.repl->connect(); check(test); - sleep(1); print_gtids(test); test.try_query(test.repl->nodes[1], "STOP SLAVE;"); test.try_query(test.repl->nodes[1], "RESET SLAVE ALL;"); - sleep(1); get_output(test); if (test.global_result == 0) { @@ -108,7 +106,7 @@ void check_test_2(TestConnections& test) // Reset state replicate_from(test, 1, master_id - 1); - sleep(3); + test.maxscales->wait_for_monitor(); get_output(test); StringSet node_states = test.get_server_status("server2"); test.assert(node_states.find("Slave") != node_states.end(), "Server 2 is not replicating."); @@ -145,7 +143,7 @@ void prepare_test_3(TestConnections& test) test.repl->start_node(2, (char *) ""); test.repl->start_node(3, (char *) ""); test.maxscales->start_maxscale(0); - sleep(2); + test.maxscales->wait_for_monitor(); test.repl->connect(); test.tprintf("Settings changed."); @@ -182,7 +180,6 @@ void check_test_3(TestConnections& test) test.repl->stop_node(1); test.repl->stop_node(2); test.repl->stop_node(3); - sleep(4); test.repl->restore_server_settings(1); test.repl->restore_server_settings(2); @@ -191,6 +188,5 @@ void check_test_3(TestConnections& test) test.repl->start_node(1, (char *) ""); test.repl->start_node(2, (char *) ""); test.repl->start_node(3, (char *) ""); - sleep(2); test.maxscales->start_maxscale(0); } diff --git a/maxscale-system-test/failover_mysqlmon.cpp b/maxscale-system-test/failover_mysqlmon.cpp index c872ab534..7de277715 100644 --- a/maxscale-system-test/failover_mysqlmon.cpp +++ b/maxscale-system-test/failover_mysqlmon.cpp @@ -33,7 +33,7 @@ int main(int argc, char *argv[]) execute_query(test->repl->nodes[3], "STOP SLAVE;RESET SLAVE ALL;"); test->tprintf("Wait for the monitor to detect it "); - sleep(15); + test.maxscales->wait_for_monitor(); test->tprintf("Connect and insert should work "); char *output = test->ssh_maxscale_output(true, "maxadmin list servers"); @@ -49,7 +49,7 @@ int main(int argc, char *argv[]) test->repl->unblock_node(2); test->tprintf("Wait for the monitor to detect it "); - sleep(15); + test.maxscales->wait_for_monitor(); test->tprintf("Check that we are still using the last node to which we failed over " "to and that the old nodes are in maintenance mode"); diff --git a/maxscale-system-test/maxscales.cpp b/maxscale-system-test/maxscales.cpp index 461b59c36..475e6800f 100644 --- a/maxscale-system-test/maxscales.cpp +++ b/maxscale-system-test/maxscales.cpp @@ -1,4 +1,6 @@ #include "maxscales.h" +#include +#include Maxscales::Maxscales(const char *pref, const char *test_cwd, bool verbose) { @@ -431,3 +433,56 @@ int Maxscales::port(enum service type, int m) const } return -1; } + +void Maxscales::wait_for_monitor(int intervals, int m) +{ + //Helper for getting number of monitor ticks + auto get_ticks = [&](std::string name) + { + int rc; + char* ticks = ssh_node_output_f(m, false, &rc, "maxctrl api get monitors/%s data.attributes.ticks", name.c_str()); + char* ptr; + int rval = strtol(ticks, &ptr, 10); + + if (ptr == ticks || (*ptr != '\0' && !isspace(*ptr))) + { + printf("ERROR, invalid monitor tick value: %s\n", ticks); + rval = -1; + } + + free(ticks); + return rval; + }; + + int rc = 0; + + // Get a list of monitor names that are running + char* monitors = ssh_node_output_f(m, false, &rc, "maxctrl --tsv list monitors|grep Running|cut -f 1"); + std::istringstream is; + is.str(monitors); + free(monitors); + std::string name; + std::unordered_map ticks; + + // For each monitor, store the current monitor tick + while (std::getline(is, name)) + { + ticks[name] = get_ticks(name); + } + + for (auto a: ticks) + { + // Wait a maximum of 60 seconds for a single monitor interval + for (int i = 0; i < 60; i++) + { + int start = a.second; + int end = get_ticks(a.first); + + if (start == -1 || end == -1 || end - start >= intervals) + { + break; + } + sleep(1); + } + } +} diff --git a/maxscale-system-test/maxscales.h b/maxscale-system-test/maxscales.h index 3dff1ddb8..5d9a9518f 100644 --- a/maxscale-system-test/maxscales.h +++ b/maxscale-system-test/maxscales.h @@ -262,6 +262,16 @@ public: */ StringSet get_server_status(const char* name, int m = 0); + /** + * Wait until the monitors have performed at least one monitoring operation + * + * The function waits until all monitors have performed at least one monitoring cycle. + * + * @param intervals The number of monitor intervals to wait + * @param m Number of Maxscale node + */ + void wait_for_monitor(int intervals = 1, int m = 0); + }; #endif // MAXSCALES_H diff --git a/maxscale-system-test/mysqlmon_detect_standalone_master.cpp b/maxscale-system-test/mysqlmon_detect_standalone_master.cpp index 1bc8c479a..a7927b2f1 100644 --- a/maxscale-system-test/mysqlmon_detect_standalone_master.cpp +++ b/maxscale-system-test/mysqlmon_detect_standalone_master.cpp @@ -63,10 +63,10 @@ void restore_servers(TestConnections& test, bool events_added) replicate_from(test, 0, 3); replicate_from(test, 1, 3); replicate_from(test, 2, 3); - sleep(10); + test.maxscales->wait_for_monitor(); o1 = test.maxscales->ssh_node_output(0, "maxadmin call command mariadbmon switchover MySQL-Monitor server1 server4", true, &dummy); - sleep(10); + test.maxscales->wait_for_monitor(); int master_id = get_master_server_id(test); test.assert(master_id == 1, "Switchover failed to set server1 as master."); } @@ -106,7 +106,7 @@ int main(int argc, char *argv[]) test.try_query(test.repl->nodes[3], "STOP SLAVE;RESET SLAVE ALL;"); test.tprintf(" Wait for the monitor to detect it "); - sleep(10); + test.maxscales->wait_for_monitor(3); test.tprintf(" Connect and insert should work "); get_output(test); @@ -136,7 +136,7 @@ int main(int argc, char *argv[]) test.repl->unblock_node(2); test.tprintf(" Wait for the monitor to detect it "); - sleep(10); + test.maxscales->wait_for_monitor(); test.tprintf("Check that we are still using the last node to which we failed over " "to and that the old nodes are in maintenance mode"); diff --git a/maxscale-system-test/mysqlmon_external_master.cpp b/maxscale-system-test/mysqlmon_external_master.cpp index 8b6f73aac..7a9073c54 100644 --- a/maxscale-system-test/mysqlmon_external_master.cpp +++ b/maxscale-system-test/mysqlmon_external_master.cpp @@ -60,14 +60,14 @@ int main(int argc, char** argv) test.tprintf("Start by having the current master replicate from the external server"); test.repl->connect(); test.repl->replicate_from(0, 3); - sleep(10); + test.maxscales->wait_for_monitor(); check_status(test, "server1", master_running, "server1 should be the master"); check_status(test, "server2", slave_running, "server2 should be a slave"); check_status(test, "server3", slave_running, "server3 should be a slave"); test.tprintf("Stop server1, expect server2 to be promoted as the master"); test.repl->stop_node(0); - sleep(10); + test.maxscales->wait_for_monitor(); check_status(test, "server1", down, "server1 should be down"); check_status(test, "server2", master_running, "server2 should be the master"); @@ -76,13 +76,13 @@ int main(int argc, char** argv) test.tprintf("Configure master-master replication between server2 and the external server"); test.repl->replicate_from(1, 3); test.repl->replicate_from(3, 1); - sleep(10); + test.maxscales->wait_for_monitor(); check_status(test, "server2", master_running, "server2 should still be the master"); check_status(test, "server3", slave_running, "server3 should be a slave"); test.tprintf("Start server1, expect it to rejoin the cluster"); test.repl->start_node(0); - sleep(10); + test.maxscales->wait_for_monitor(); check_status(test, "server1", slave_running, "server1 should be a slave"); check_status(test, "server2", master_running, "server2 should still be the master"); check_status(test, "server3", slave_running, "server3 should be a slave"); @@ -92,7 +92,7 @@ int main(int argc, char** argv) test.repl->connect(); test.repl->replicate_from(0, 3); test.repl->replicate_from(3, 0); - sleep(10); + test.maxscales->wait_for_monitor(); check_status(test, "server1", master_running, "server1 should be the master"); check_status(test, "server2", down, "server2 should be down"); @@ -100,7 +100,7 @@ int main(int argc, char** argv) test.tprintf("Start server2, expect it to rejoin the cluster"); test.repl->start_node(1); - sleep(10); + test.maxscales->wait_for_monitor(); check_status(test, "server1", master_running, "server1 should still be the master"); check_status(test, "server2", slave_running, "server2 should be a slave"); check_status(test, "server3", slave_running, "server3 should be a slave"); diff --git a/maxscale-system-test/mysqlmon_failover_auto.cpp b/maxscale-system-test/mysqlmon_failover_auto.cpp index 4925b4e35..6767b6e7e 100644 --- a/maxscale-system-test/mysqlmon_failover_auto.cpp +++ b/maxscale-system-test/mysqlmon_failover_auto.cpp @@ -21,13 +21,13 @@ int main(int argc, char** argv) test.repl->connect(); delete_slave_binlogs(test); - sleep(2); + test.maxscales->wait_for_monitor(); basic_test(test); print_gtids(test); // Part 1 int node0_id = prepare_test_1(test); - sleep(10); + test.maxscales->wait_for_monitor(); check_test_1(test, node0_id); if (test.global_result != 0) @@ -37,7 +37,7 @@ int main(int argc, char** argv) // Part 2 prepare_test_2(test); - sleep(10); + test.maxscales->wait_for_monitor(); check_test_2(test); if (test.global_result != 0) @@ -47,7 +47,7 @@ int main(int argc, char** argv) // Part 3 prepare_test_3(test); - sleep(10); + test.maxscales->wait_for_monitor(); check_test_3(test); return test.global_result; diff --git a/maxscale-system-test/mysqlmon_failover_manual.cpp b/maxscale-system-test/mysqlmon_failover_manual.cpp index 6c4957da6..6f9f5675b 100644 --- a/maxscale-system-test/mysqlmon_failover_manual.cpp +++ b/maxscale-system-test/mysqlmon_failover_manual.cpp @@ -23,7 +23,6 @@ int main(int argc, char** argv) test.repl->connect(); delete_slave_binlogs(test); - sleep(2); basic_test(test); print_gtids(test); @@ -32,10 +31,9 @@ int main(int argc, char** argv) // Part 1 node0_id = prepare_test_1(test); - sleep(3); test.maxscales->ssh_node_output(0, FAILOVER_CMD , true, &ec); - sleep(10); + test.maxscales->wait_for_monitor(); check_test_1(test, node0_id); if (test.global_result != 0) @@ -45,10 +43,9 @@ int main(int argc, char** argv) // Part 2 prepare_test_2(test); - sleep(3); test.maxscales->ssh_node_output(0, FAILOVER_CMD, true, &ec); - sleep(10); + test.maxscales->wait_for_monitor(); check_test_2(test); if (test.global_result != 0) @@ -58,10 +55,9 @@ int main(int argc, char** argv) // Part 3 prepare_test_3(test); - sleep(3); test.maxscales->ssh_node_output(0, FAILOVER_CMD, true, &ec); - sleep(10); + test.maxscales->wait_for_monitor(); check_test_3(test); return test.global_result; diff --git a/maxscale-system-test/mysqlmon_failover_no_slaves.cpp b/maxscale-system-test/mysqlmon_failover_no_slaves.cpp index 2f6d7e25d..edc5e4c24 100644 --- a/maxscale-system-test/mysqlmon_failover_no_slaves.cpp +++ b/maxscale-system-test/mysqlmon_failover_no_slaves.cpp @@ -44,13 +44,13 @@ int main(int argc, char** argv) test.try_query(nodes[3], CHANGE_CMD); test.try_query(nodes[3], "START SLAVE;"); - sleep(10); + test.maxscales->wait_for_monitor(); get_output(test); test.tprintf(LINE); test.tprintf("Stopping master. Failover should not happen."); test.repl->block_node(0); - sleep(10); + test.maxscales->wait_for_monitor(); get_output(test); int master_id = get_master_server_id(test); test.assert(master_id == -1, "Master was promoted even when no slave was eligible."); diff --git a/maxscale-system-test/mysqlmon_failover_rejoin_old_slave.cpp b/maxscale-system-test/mysqlmon_failover_rejoin_old_slave.cpp index bc55dd194..b41b47e51 100644 --- a/maxscale-system-test/mysqlmon_failover_rejoin_old_slave.cpp +++ b/maxscale-system-test/mysqlmon_failover_rejoin_old_slave.cpp @@ -122,7 +122,7 @@ void expect(TestConnections& test, const char* zServer, const char* zState1, con void run(TestConnections& test) { - sleep(10); + test.maxscales->wait_for_monitor(); int N = test.repl->N; cout << "Nodes: " << N << endl; @@ -147,7 +147,7 @@ void run(TestConnections& test) cout << "\nStopping slave " << N - 1 << endl; test.repl->stop_node(N - 1); - sleep(10); + test.maxscales->wait_for_monitor(); // server4 was stopped, so we expect the state of it to be /Down/, // and the states of the other ones not to have changed. @@ -171,7 +171,7 @@ void run(TestConnections& test) cout << "\nStopping master." << endl; test.repl->stop_node(0); - sleep(10); + test.maxscales->wait_for_monitor(); // server1 (previous master) was taken down, so its state should be /Down/. // server2 should have been made into master, and server4 should still be down. @@ -183,7 +183,7 @@ void run(TestConnections& test) cout << "\nBringing up slave " << N - 1 << endl; test.repl->start_node(N - 1, (char*)""); - sleep(10); + test.maxscales->wait_for_monitor(); // server1 should still be down, server2 still master, and server3 still // a slave. server4 was brought up, so it should have been rejoined and diff --git a/maxscale-system-test/mysqlmon_failover_rolling_master.cpp b/maxscale-system-test/mysqlmon_failover_rolling_master.cpp index 46418ad74..3ee8160c7 100644 --- a/maxscale-system-test/mysqlmon_failover_rolling_master.cpp +++ b/maxscale-system-test/mysqlmon_failover_rolling_master.cpp @@ -204,7 +204,7 @@ void stop_node(XTestConnections& test, int index) void run(XTestConnections& test) { - sleep(10); + test.maxscales->wait_for_monitor(); int N = test.repl->N; cout << "Nodes: " << N << endl; @@ -236,7 +236,7 @@ void run(XTestConnections& test) cout << "\nClosing connection to MaxScale." << endl; test.maxscales->close_maxscale_connections(0); - sleep(10); + test.maxscales->wait_for_monitor(); list_servers(test); diff --git a/maxscale-system-test/mysqlmon_failover_rolling_restart_slaves.cpp b/maxscale-system-test/mysqlmon_failover_rolling_restart_slaves.cpp index c7a9433d9..363e913dd 100644 --- a/maxscale-system-test/mysqlmon_failover_rolling_restart_slaves.cpp +++ b/maxscale-system-test/mysqlmon_failover_rolling_restart_slaves.cpp @@ -157,7 +157,7 @@ void check_server_status(TestConnections& test, int N, int down = -1) void run(TestConnections& test) { - sleep(10); + test.maxscales->wait_for_monitor(); int N = test.repl->N; cout << "Nodes: " << N << endl; @@ -183,14 +183,14 @@ void run(TestConnections& test) cout << "\nStopping slave " << slave << endl; test.repl->stop_node(i); - sleep(10); + test.maxscales->wait_for_monitor(); check_server_status(test, N, i); cout << "\nStarting slave " << slave << endl; test.repl->start_node(i, (char*)""); - sleep(10); + test.maxscales->wait_for_monitor(); check_server_status(test, N); } diff --git a/maxscale-system-test/mysqlmon_failover_stress.cpp b/maxscale-system-test/mysqlmon_failover_stress.cpp index bac0f9ff3..894049bab 100755 --- a/maxscale-system-test/mysqlmon_failover_stress.cpp +++ b/maxscale-system-test/mysqlmon_failover_stress.cpp @@ -525,7 +525,7 @@ void run(TestConnections& test) while (time(NULL) - start < TEST_DURATION) { - sleep(FAILOVER_DURATION); + test.maxscales->wait_for_monitor(); int master_id = get_master_server_id(test); @@ -534,20 +534,20 @@ void run(TestConnections& test) cout << "\nStopping node: " << master_id << endl; test.repl->stop_node(master_id - 1); - sleep(2 * MONITOR_INTERVAL); + test.maxscales->wait_for_monitor(); list_servers(test); - sleep(FAILOVER_DURATION); + test.maxscales->wait_for_monitor(); list_servers(test); - sleep(FAILOVER_DURATION); + test.maxscales->wait_for_monitor(); cout << "\nStarting node: " << master_id << endl; test.repl->start_node(master_id - 1); - sleep(2 * MONITOR_INTERVAL); + test.maxscales->wait_for_monitor(); list_servers(test); - sleep(FAILOVER_DURATION); + test.maxscales->wait_for_monitor(); list_servers(test); } else @@ -556,7 +556,7 @@ void run(TestConnections& test) } } - sleep(FAILOVER_DURATION); + test.maxscales->wait_for_monitor(); cout << "\nStopping clients.\n" << flush; Client::stop(); diff --git a/maxscale-system-test/mysqlmon_rejoin_bad.cpp b/maxscale-system-test/mysqlmon_rejoin_bad.cpp index 35ff51c2f..aabab85d9 100644 --- a/maxscale-system-test/mysqlmon_rejoin_bad.cpp +++ b/maxscale-system-test/mysqlmon_rejoin_bad.cpp @@ -98,7 +98,7 @@ int main(int argc, char** argv) test.assert(false, "Could not start MaxScale."); return test.global_result; } - sleep(10); + test.maxscales->wait_for_monitor(); get_output(test); StringSet node2_states = test.get_server_status("server3"); @@ -119,7 +119,7 @@ int main(int argc, char** argv) snprintf(cmd, sizeof(cmd), CHANGE_CMD_FMT, test.repl->IP[3], test.repl->port[3]); mysql_query(nodes[0], cmd); mysql_query(nodes[0], "START SLAVE;"); - sleep(10); + test.maxscales->wait_for_monitor(); get_output(test); int master_id = get_master_server_id(test); test.assert(master_id == 4, "Server 4 should be the cluster master."); @@ -132,7 +132,7 @@ int main(int argc, char** argv) int ec; test.maxscales->ssh_node_output(0, "maxadmin call command mysqlmon switchover MySQL-Monitor server1 server4" , true, &ec); - sleep(10); + test.maxscales->wait_for_monitor(); master_id = get_master_server_id(test); test.assert(master_id == 1, "Server 1 should be the cluster master."); get_output(test); diff --git a/maxscale-system-test/mysqlmon_rejoin_bad2.cpp b/maxscale-system-test/mysqlmon_rejoin_bad2.cpp index 42e1798b0..0a48aaadb 100644 --- a/maxscale-system-test/mysqlmon_rejoin_bad2.cpp +++ b/maxscale-system-test/mysqlmon_rejoin_bad2.cpp @@ -88,7 +88,7 @@ int main(int argc, char** argv) cout << "Stopping master, should auto-failover." << endl; int master_id_old = get_master_server_id(test); test.repl->stop_node(0); - sleep(10); + test.maxscales->wait_for_monitor(); get_output(test); int master_id_new = get_master_server_id(test); cout << "Master server id is " << master_id_new << endl; @@ -107,7 +107,7 @@ int main(int argc, char** argv) } // Restart old master. Then add some events to it. test.repl->start_node(0, (char*)""); - sleep(3); + test.maxscales->wait_for_monitor(); test.repl->connect(); cout << "Adding more events to node 0. It should not join the cluster." << endl; generate_traffic_and_check(test, test.repl->nodes[0], 5); @@ -118,7 +118,7 @@ int main(int argc, char** argv) test.assert(false, "Could not start MaxScale."); return test.global_result; } - sleep(10); + test.maxscales->wait_for_monitor(); get_output(test); expect(test, "server1", "Running"); @@ -139,7 +139,7 @@ int main(int argc, char** argv) MYSQL** nodes = test.repl->nodes; mysql_query(nodes[ind], cmd); mysql_query(nodes[ind], "START SLAVE;"); - sleep(10); + test.maxscales->wait_for_monitor(); get_output(test); expect(test, "server1", "Running"); diff --git a/maxscale-system-test/mysqlmon_rejoin_good.cpp b/maxscale-system-test/mysqlmon_rejoin_good.cpp index a1a8aea4e..e51495312 100644 --- a/maxscale-system-test/mysqlmon_rejoin_good.cpp +++ b/maxscale-system-test/mysqlmon_rejoin_good.cpp @@ -28,7 +28,7 @@ int main(int argc, char** argv) char result_tmp[bufsize]; // Advance gtid:s a bit to so gtid variables are updated. generate_traffic_and_check(test, maxconn, 10); - sleep(1); + test.maxscales->wait_for_monitor(); test.tprintf(LINE); print_gtids(test); get_input(); @@ -38,7 +38,7 @@ int main(int argc, char** argv) const int old_master_id = get_master_server_id(test); // Read master id now before shutdown. const int master_index = test.repl->master; test.repl->stop_node(master_index); - sleep(10); + test.maxscales->wait_for_monitor(); // Recreate maxscale session mysql_close(maxconn); maxconn = test.maxscales->open_rwsplit_connection(0); @@ -53,7 +53,7 @@ int main(int argc, char** argv) { test.tprintf("Sending more inserts."); generate_traffic_and_check(test, maxconn, 5); - sleep(1); + test.maxscales->wait_for_monitor(); if (find_field(maxconn, GTID_QUERY, GTID_FIELD, result_tmp) == 0) { gtid_final = result_tmp; @@ -63,11 +63,11 @@ int main(int argc, char** argv) test.tprintf(LINE); test.repl->start_node(master_index, (char*) ""); - sleep(10); + test.maxscales->wait_for_monitor(); get_output(test); test.repl->connect(); - sleep(1); + test.maxscales->wait_for_monitor(); string gtid_old_master; if (find_field(test.repl->nodes[master_index], GTID_QUERY, GTID_FIELD, result_tmp) == 0) { @@ -81,7 +81,7 @@ int main(int argc, char** argv) int ec; test.maxscales->ssh_node_output(0, "maxadmin call command mysqlmon switchover " "MySQL-Monitor server1 server2" , true, &ec); - sleep(10); // Wait for monitor to update status + test.maxscales->wait_for_monitor(); // Wait for monitor to update status get_output(test); master_id = get_master_server_id(test); test.assert(master_id == old_master_id, "Switchover back to server1 failed."); @@ -89,7 +89,7 @@ int main(int argc, char** argv) else { test.repl->start_node(master_index, (char*) ""); - sleep(10); + test.maxscales->wait_for_monitor(); } test.repl->fix_replication(); diff --git a/maxscale-system-test/mysqlmon_rejoin_manual.cpp b/maxscale-system-test/mysqlmon_rejoin_manual.cpp index f0a2c6aac..226df7d10 100644 --- a/maxscale-system-test/mysqlmon_rejoin_manual.cpp +++ b/maxscale-system-test/mysqlmon_rejoin_manual.cpp @@ -40,7 +40,10 @@ int main(int argc, char** argv) const int old_master_id = get_master_server_id(test); // Read master id now before shutdown. const int master_index = test.repl->master; test.repl->stop_node(master_index); - sleep(10); + + // Wait until failover is performed + test.maxscales->wait_for_monitor(3); + // Recreate maxscale session mysql_close(maxconn); maxconn = test.maxscales->open_rwsplit_connection(0); @@ -54,20 +57,16 @@ int main(int argc, char** argv) { cout << "Sending more inserts." << endl; generate_traffic_and_check(test, maxconn, 5); - if (find_field(maxconn, GTID_QUERY, GTID_FIELD, result_tmp) == 0) - { - gtid_final = result_tmp; - } print_gtids(test); cout << "Bringing old master back online..." << endl; test.repl->start_node(master_index, (char*) ""); - sleep(10); + test.maxscales->wait_for_monitor(); test.repl->connect(); get_output(test); test.tprintf("and manually rejoining it to cluster."); const char REJOIN_CMD[] = "maxadmin call command mariadbmon rejoin MySQL-Monitor server1"; test.maxscales->ssh_node_output(0, REJOIN_CMD , true, &ec); - sleep(10); + test.maxscales->wait_for_monitor(); get_output(test); string gtid_old_master; @@ -75,15 +74,20 @@ int main(int argc, char** argv) { gtid_old_master = result_tmp; } + if (find_field(maxconn, GTID_QUERY, GTID_FIELD, result_tmp) == 0) + { + gtid_final = result_tmp; + } + cout << LINE << "\n"; print_gtids(test); cout << LINE << "\n"; - test.assert(gtid_final == gtid_old_master, "Old master did not successfully rejoin the cluster."); + test.assert(gtid_final == gtid_old_master, "Old master did not successfully rejoin the cluster (%s != %s).", gtid_final.c_str(), gtid_old_master.c_str()); // Switch master back to server1 so last check is faster int ec; test.maxscales->ssh_node_output(0, "maxadmin call command mysqlmon switchover " "MySQL-Monitor server1 server2" , true, &ec); - sleep(10); // Wait for monitor to update status + test.maxscales->wait_for_monitor(); // Wait for monitor to update status get_output(test); master_id = get_master_server_id(test); test.assert(master_id == old_master_id, "Switchover back to server1 failed."); @@ -91,7 +95,7 @@ int main(int argc, char** argv) else { test.repl->start_node(master_index, (char*) ""); - sleep(10); + test.maxscales->wait_for_monitor(); } test.repl->fix_replication(); diff --git a/maxscale-system-test/mysqlmon_rejoin_manual2.cpp b/maxscale-system-test/mysqlmon_rejoin_manual2.cpp index fa3d052bc..73835e023 100644 --- a/maxscale-system-test/mysqlmon_rejoin_manual2.cpp +++ b/maxscale-system-test/mysqlmon_rejoin_manual2.cpp @@ -94,7 +94,7 @@ int main(int argc, char** argv) string rejoin_s4 = REJOIN_CMD + " server4"; test.maxscales->ssh_node_output(0, rejoin_s3.c_str() , true, &ec); test.maxscales->ssh_node_output(0, rejoin_s4.c_str() , true, &ec); - sleep(10); + test.maxscales->wait_for_monitor(); get_output(test); StringSet node2_states = test.get_server_status("server3"); @@ -115,11 +115,11 @@ int main(int argc, char** argv) snprintf(cmd, sizeof(cmd), CHANGE_CMD_FMT, test.repl->IP[3], test.repl->port[3]); mysql_query(nodes[0], cmd); mysql_query(nodes[0], "START SLAVE;"); - sleep(10); + test.maxscales->wait_for_monitor(); string rejoin_s2 = REJOIN_CMD + " server2"; test.maxscales->ssh_node_output(0, rejoin_s2.c_str() , true, &ec); test.maxscales->ssh_node_output(0, rejoin_s3.c_str() , true, &ec); - sleep(10); + test.maxscales->wait_for_monitor(); get_output(test); int master_id = get_master_server_id(test); test.assert(master_id == 4, "Server 4 should be the cluster master."); @@ -132,7 +132,7 @@ int main(int argc, char** argv) int ec; test.maxscales->ssh_node_output(0, "maxadmin call command mysqlmon switchover MySQL-Monitor server1 server4" , true, &ec); - sleep(10); + test.maxscales->wait_for_monitor(); master_id = get_master_server_id(test); test.assert(master_id == 1, "Server 1 should be the cluster master."); get_output(test); diff --git a/maxscale-system-test/mysqlmon_switchover.cpp b/maxscale-system-test/mysqlmon_switchover.cpp index b74ba34c1..e776854b2 100644 --- a/maxscale-system-test/mysqlmon_switchover.cpp +++ b/maxscale-system-test/mysqlmon_switchover.cpp @@ -122,7 +122,7 @@ void expect(TestConnections& test, const char* zServer, const char* zState1, con void run(TestConnections& test) { - sleep(10); + test.maxscales->wait_for_monitor(); int N = test.repl->N; cout << "Nodes: " << N << endl; @@ -149,7 +149,7 @@ void run(TestConnections& test) zCommand = "call command mysqlmon switchover MySQL-Monitor server2 server1"; test.maxscales->execute_maxadmin_command_print(0, (char*)zCommand); - sleep(10); + test.maxscales->wait_for_monitor(); expect(test, "server1", "Slave", "Running"); expect(test, "server2", "Master", "Running"); @@ -162,7 +162,7 @@ void run(TestConnections& test) zCommand = "call command mysqlmon switchover MySQL-Monitor server1 server2"; test.maxscales->execute_maxadmin_command_print(0, (char*)zCommand); - sleep(10); + test.maxscales->wait_for_monitor(); expect(test, "server1", "Master", "Running"); expect(test, "server2", "Slave", "Running"); diff --git a/maxscale-system-test/mysqlmon_switchover_bad_master.cpp b/maxscale-system-test/mysqlmon_switchover_bad_master.cpp index 4cd6ea411..5ceb82029 100644 --- a/maxscale-system-test/mysqlmon_switchover_bad_master.cpp +++ b/maxscale-system-test/mysqlmon_switchover_bad_master.cpp @@ -122,7 +122,7 @@ void expect(TestConnections& test, const char* zServer, const char* zState1, con void run(TestConnections& test) { - sleep(10); + test.maxscales->wait_for_monitor(); int N = test.repl->N; cout << "Nodes: " << N << endl; @@ -147,7 +147,7 @@ void run(TestConnections& test) cout << "\nStopping slave " << N - 1 << endl; test.repl->stop_node(N - 1); - sleep(10); + test.maxscales->wait_for_monitor(); // server4 was stopped, so we expect the state of it to be /Down/, // and the states of the other ones not to have changed. @@ -171,7 +171,7 @@ void run(TestConnections& test) cout << "\nStopping master." << endl; test.repl->stop_node(0); - sleep(10); + test.maxscales->wait_for_monitor(3); // server1 (previous master) was taken down, so its state should be /Down/. // server2 should have been made into master, and server4 should still be down. @@ -183,7 +183,7 @@ void run(TestConnections& test) cout << "\nBringing up slave " << N - 1 << endl; test.repl->start_node(N - 1, (char*)""); - sleep(10); + test.maxscales->wait_for_monitor(); // server1 should still be down, server2 still master, and server3 still // a slave. server4 was brought up, but as auto_rejoin is false, it should @@ -198,7 +198,7 @@ void run(TestConnections& test) const char* zCommand = "call command mysqlmon switchover MySQL-Monitor server4 server2"; test.maxscales->execute_maxadmin_command_print(0, (char*)zCommand); - sleep(10); + test.maxscales->wait_for_monitor(); // The state should not change, as server4 is not good enough as master. expect(test, "server1", "Down");