diff --git a/maxscale-system-test/.gitignore b/maxscale-system-test/.gitignore index 51da6f12a..d85dc5dd8 100644 --- a/maxscale-system-test/.gitignore +++ b/maxscale-system-test/.gitignore @@ -89,7 +89,6 @@ delete_rds different_size_binlog different_size_rwsplit encrypted_passwords -failover_mysqlmon false_monitor_state_change fwf fwf2 @@ -114,6 +113,7 @@ max_connections maxscale_process_user mm mm_mysqlmon +mysqlmon_detect_standalone_master mysqlmon_failover_no_slaves mysqlmon_failover_manual mysqlmon_failover_manual2_4 diff --git a/maxscale-system-test/CMakeLists.txt b/maxscale-system-test/CMakeLists.txt index 01ddb4f32..0d6dcd31f 100644 --- a/maxscale-system-test/CMakeLists.txt +++ b/maxscale-system-test/CMakeLists.txt @@ -250,7 +250,7 @@ add_test_executable(different_size_rwsplit.cpp different_size_rwsplit replicatio add_test_executable(encrypted_passwords.cpp encrypted_passwords replication LABELS maxscale LIGHT REPL_BACKEND) # MySQL Monitor Failover Test -add_test_executable(failover_mysqlmon.cpp failover_mysqlmon failover_mysqlmon LABELS mysqlmon REPL_BACKEND) +add_test_executable(mysqlmon_detect_standalone_master.cpp mysqlmon_detect_standalone_master mysqlmon_detect_standalone_master LABELS mysqlmon REPL_BACKEND) # MySQL Monitor Failover (automatic) Test add_test_executable(mysqlmon_failover_auto.cpp mysqlmon_failover_auto mysqlmon_failover_auto LABELS mysqlmon REPL_BACKEND) diff --git a/maxscale-system-test/cnf/maxscale.cnf.template.failover_mysqlmon b/maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_detect_standalone_master similarity index 95% rename from maxscale-system-test/cnf/maxscale.cnf.template.failover_mysqlmon rename to maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_detect_standalone_master index 09cc8a5ba..5c4a9145a 100644 --- a/maxscale-system-test/cnf/maxscale.cnf.template.failover_mysqlmon +++ b/maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_detect_standalone_master @@ -11,6 +11,9 @@ monitor_interval=1000 detect_standalone_master=true failcount=2 allow_cluster_recovery=false +backend_connect_timeout=2 +backend_read_timeout=3 +backend_write_timeout=3 [RW Split Router] type=service diff --git a/maxscale-system-test/failover_mysqlmon.cpp b/maxscale-system-test/failover_mysqlmon.cpp deleted file mode 100644 index e663dfaf9..000000000 --- a/maxscale-system-test/failover_mysqlmon.cpp +++ /dev/null @@ -1,72 +0,0 @@ -/** - * @file failover_mysqlmon.cpp MySQL Monitor Failover Test - * - block all nodes, but one - * - wait for minitor (monitor_interval) - * - check maxadmin output - * - check that queries work - * - unblock backend nodes - * - wait for monitor - * - check that we are still using the last node to which we failed over to and that the old nodes are in maintenance mode - */ - - -#include -#include "testconnections.h" - -int main(int argc, char *argv[]) -{ - int exit_code; - TestConnections * test = new TestConnections(argc, argv); - - test->tprintf(" Create the test table and insert some data "); - test->maxscales->connect_maxscale(0); - test->try_query(test->maxscales->conn_rwsplit[0], "CREATE OR REPLACE TABLE test.t1 (id int)"); - test->try_query(test->maxscales->conn_rwsplit[0], "INSERT INTO test.t1 VALUES (1)"); - test->maxscales->close_maxscale_connections(0); - - test->tprintf(" Block all but one node "); - test->repl->block_node(0); - test->repl->block_node(1); - test->repl->block_node(2); - test->repl->connect(); - execute_query(test->repl->nodes[3], "STOP SLAVE;RESET SLAVE ALL;"); - - test->tprintf(" Wait for the monitor to detect it "); - sleep(15); - - test->tprintf(" Connect and insert should work "); - char *output = test->maxscales->ssh_node_output(0, "maxadmin list servers", true, &exit_code); - test->tprintf("%s", output); - free(output); - test->maxscales->connect_maxscale(0); - test->try_query(test->maxscales->conn_rwsplit[0], "INSERT INTO test.t1 VALUES (1)"); - test->maxscales->close_maxscale_connections(0); - - test->tprintf(" Unblock nodes "); - test->repl->unblock_node(0); - test->repl->unblock_node(1); - test->repl->unblock_node(2); - - test->tprintf(" Wait for the monitor to detect it "); - sleep(15); - - test->tprintf("Check that we are still using the last node to which we failed over " - "to and that the old nodes are in maintenance mode"); - - test->maxscales->connect_maxscale(0); - test->try_query(test->maxscales->conn_rwsplit[0], "INSERT INTO test.t1 VALUES (1)"); - char maxscale_id[256], real_id[256]; - find_field(test->maxscales->conn_rwsplit[0], "SELECT @@server_id", "@@server_id", maxscale_id); - test->repl->connect(); - find_field(test->repl->nodes[3], "SELECT @@server_id", "@@server_id", real_id); - test->add_result(strcmp(maxscale_id, real_id) != 0, - "@@server_id is different: %s != %s", maxscale_id, real_id); - test->maxscales->close_maxscale_connections(0); - - test->tprintf(" Check that MaxScale is running "); - test->check_maxscale_alive(0); - - int rval = test->global_result; - delete test; - return rval; -} diff --git a/maxscale-system-test/mysqlmon_detect_standalone_master.cpp b/maxscale-system-test/mysqlmon_detect_standalone_master.cpp new file mode 100644 index 000000000..7d8f8d80f --- /dev/null +++ b/maxscale-system-test/mysqlmon_detect_standalone_master.cpp @@ -0,0 +1,165 @@ +/** + * @file mysqlmon_detect_standalone_master.cpp MySQL Monitor Standalone Master Test + * - block all nodes, but one + * - wait for monitor (monitor_interval), monitor should select remaining node as master + * - check maxadmin output + * - check that queries work + * - unblock backend nodes + * - wait for monitor + * - check that monitor is still using the same node and that the old nodes are in maintenance mode + */ + +#include +#include +#include "testconnections.h" +#include "fail_switch_rejoin_common.cpp" + +using std::stringstream; +using std::cout; +using std::endl; + +void check_maxscale(TestConnections& test) +{ + test.tprintf("Connecting to Maxscale\n"); + test.add_result(test.maxscales->connect_maxscale(0), "Can not connect to Maxscale\n"); + test.tprintf("Trying simple query against all sevices\n"); + test.tprintf("RWSplit \n"); + test.try_query(test.maxscales->conn_rwsplit[0], (char *) "show databases;"); + test.tprintf("ReadConn Master \n"); + test.try_query(test.maxscales->conn_master[0], (char *) "show databases;"); +} + +void replicate_from(TestConnections& test, int server_ind, int target_ind) +{ + stringstream change_master; + change_master << "CHANGE MASTER TO MASTER_HOST = '" << test.repl->IP[target_ind] + << "', MASTER_PORT = " << test.repl->port[target_ind] << ", MASTER_USE_GTID = current_pos, " + "MASTER_USER='repl', MASTER_PASSWORD='repl';"; + cout << "Server " << server_ind + 1 << " starting to replicate from server " << target_ind + 1 << endl; + if (test.verbose) + { + cout << "Query is '" << change_master.str() << "'" << endl; + } + test.try_query(test.repl->nodes[server_ind], "STOP SLAVE;"); + test.try_query(test.repl->nodes[server_ind], change_master.str().c_str()); + test.try_query(test.repl->nodes[server_ind], "START SLAVE;"); +} + +void restore_servers(TestConnections& test, bool events_added) +{ + test.repl->unblock_node(0); + test.repl->unblock_node(1); + test.repl->unblock_node(2); + int dummy; + char *o1 = test.maxscales->ssh_node_output(0, "maxadmin clear server server1 Maint", true, &dummy); + char *o2 = test.maxscales->ssh_node_output(0, "maxadmin clear server server2 Maint", true, &dummy); + char *o3 = test.maxscales->ssh_node_output(0, "maxadmin clear server server3 Maint", true, &dummy); + free(o1); + free(o2); + free(o3); + if (events_added) + { + // Events have been added to server4, so it must be the real new master. Then switchover to server1. + replicate_from(test, 0, 3); + replicate_from(test, 1, 3); + replicate_from(test, 2, 3); + sleep(3); + o1 = test.maxscales->ssh_node_output(0, + "maxadmin call command mariadbmon switchover MySQL-Monitor server1 server4", true, &dummy); + sleep(2); + int master_id = get_master_server_id(test); + test.assert(master_id == 1, "Switchover failed to set server1 as master."); + } + else + { + // No events added, it should be enough to start slave on server4 + replicate_from(test, 3, 0); + } +} + +int main(int argc, char *argv[]) +{ + Mariadb_nodes::require_gtid(true); + TestConnections test(argc, argv); + test.maxscales->connect_maxscale(0); + test.repl->connect(); + delete_slave_binlogs(test); + print_gtids(test); + test.tprintf(" Create the test table and insert some data "); + test.try_query(test.maxscales->conn_rwsplit[0], "CREATE OR REPLACE TABLE test.t1 (id int)"); + test.try_query(test.maxscales->conn_rwsplit[0], "INSERT INTO test.t1 VALUES (1)"); + sleep(2); + print_gtids(test); + + test.maxscales->close_maxscale_connections(0); + if (test.global_result != 0) + { + return test.global_result; + } + + test.tprintf(" Block all but one node, stop slave on server 4 "); + test.repl->block_node(0); + test.repl->block_node(1); + test.repl->block_node(2); + + test.try_query(test.repl->nodes[3], "STOP SLAVE;RESET SLAVE ALL;"); + + test.tprintf(" Wait for the monitor to detect it "); + sleep(8); + + test.tprintf(" Connect and insert should work "); + get_output(test); + + int master_id = get_master_server_id(test); + test.assert(master_id == 4, "Server 4 should be master, but master is server %d.", master_id); + + if (test.global_result != 0) + { + restore_servers(test, false); + return test.global_result; + } + + test.maxscales->connect_maxscale(0); + test.try_query(test.maxscales->conn_rwsplit[0], "INSERT INTO test.t1 VALUES (1)"); + test.maxscales->close_maxscale_connections(0); + sleep(1); + test.repl->connect(3); + char result_tmp[bufsize]; + if (find_field(test.repl->nodes[3], GTID_QUERY, GTID_FIELD, result_tmp) == 0) + { + test.tprintf("Node 3 gtid: %s", result_tmp); + } + + test.tprintf("Unblock nodes "); + test.repl->unblock_node(0); + test.repl->unblock_node(1); + test.repl->unblock_node(2); + + test.tprintf(" Wait for the monitor to detect it "); + sleep(8); + + test.tprintf("Check that we are still using the last node to which we failed over " + "to and that the old nodes are in maintenance mode"); + + test.maxscales->connect_maxscale(0); + get_output(test); + + test.try_query(test.maxscales->conn_rwsplit[0], "INSERT INTO test.t1 VALUES (1)"); + master_id = get_master_server_id(test); + test.tprintf("Master server id is %d", master_id); + + test.repl->connect(); + int real_id = test.repl->get_server_id(3); + test.assert(master_id == real_id, "@@server_id is different: %d != %d", master_id, real_id); + print_gtids(test); + test.maxscales->close_maxscale_connections(0); + + test.tprintf("Check that MaxScale is running"); + check_maxscale(test); + if (test.global_result == 0) + { + cout << "Test successful, restoring original state." << endl; + restore_servers(test, true); + } + return test.global_result; +}