diff --git a/maxscale-system-test/CMakeLists.txt b/maxscale-system-test/CMakeLists.txt index 20abc5853..6e6903b84 100644 --- a/maxscale-system-test/CMakeLists.txt +++ b/maxscale-system-test/CMakeLists.txt @@ -308,6 +308,9 @@ add_test_executable(mysqlmon_failover_stress.cpp mysqlmon_failover_stress mysqlm # MySQL Monitor switchover stress add_test_executable(mysqlmon_switchover_stress.cpp mysqlmon_switchover_stress mysqlmon_switchover_stress LABELS mysqlmon REPL_BACKEND) +# Check monitoring and failover when ignore_external_masters is enabled +add_test_executable(mysqlmon_external_master.cpp mysqlmon_external_master mysqlmon_external_master LABELS mysqlmon REPL_BACKEND) + # Test monitor state change events when manually clearing server bits add_test_executable(false_monitor_state_change.cpp false_monitor_state_change replication LABELS mysqlmon REPL_BACKEND) diff --git a/maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_external_master b/maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_external_master new file mode 100644 index 000000000..3fe144c9a --- /dev/null +++ b/maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_external_master @@ -0,0 +1,59 @@ +[maxscale] +threads=###threads### + +[MySQL-Monitor] +type=monitor +module=mysqlmon +servers=server1,server2,server3 +user=maxskysql +passwd=skysql +monitor_interval=1000 +detect_standalone_master=true +failcount=1 +allow_cluster_recovery=true +auto_failover=true +auto_rejoin=true +replication_user=repl +replication_password=repl +ignore_external_masters=true + +[RW Split Router] +type=service +router=readwritesplit +servers=server1,server2,server3 +user=maxskysql +passwd=skysql + +[RW Split Listener] +type=listener +service=RW Split Router +protocol=MySQLClient +port=4006 + +[CLI] +type=service +router=cli + +[CLI Listener] +type=listener +service=CLI +protocol=maxscaled +socket=default + +[server1] +type=server +address=###node_server_IP_1### +port=###node_server_port_1### +protocol=MySQLBackend + +[server2] +type=server +address=###node_server_IP_2### +port=###node_server_port_2### +protocol=MySQLBackend + +[server3] +type=server +address=###node_server_IP_3### +port=###node_server_port_3### +protocol=MySQLBackend diff --git a/maxscale-system-test/failover_common.cpp b/maxscale-system-test/failover_common.cpp index f24887702..dc034a2df 100644 --- a/maxscale-system-test/failover_common.cpp +++ b/maxscale-system-test/failover_common.cpp @@ -8,18 +8,7 @@ using std::endl; void replicate_from(TestConnections& test, int server_ind, int target_ind) { - stringstream change_master; - change_master << "CHANGE MASTER TO MASTER_HOST = '" << test.repl->IP[target_ind] - << "', MASTER_PORT = " << test.repl->port[target_ind] << ", MASTER_USE_GTID = current_pos, " - "MASTER_USER='repl', MASTER_PASSWORD='repl';"; - cout << "Server " << server_ind + 1 << " starting to replicate from server " << target_ind + 1 << endl; - if (test.verbose) - { - cout << "Query is '" << change_master.str() << "'" << endl; - } - execute_query(test.repl->nodes[server_ind], "STOP SLAVE;"); - execute_query(test.repl->nodes[server_ind], change_master.str().c_str()); - execute_query(test.repl->nodes[server_ind], "START SLAVE;"); + test.repl->replicate_from(server_ind, target_ind); } void reset_replication(TestConnections& test) diff --git a/maxscale-system-test/mariadb_nodes.cpp b/maxscale-system-test/mariadb_nodes.cpp index 1d8eff683..d6fa78e2c 100644 --- a/maxscale-system-test/mariadb_nodes.cpp +++ b/maxscale-system-test/mariadb_nodes.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include namespace @@ -1457,3 +1458,21 @@ int Mariadb_nodes::prepare_servers() } return rval; } + +void Mariadb_nodes::replicate_from(int slave, int master, const char* type) +{ + std::stringstream change_master; + change_master << "CHANGE MASTER TO MASTER_HOST = '" << IP[master] + << "', MASTER_PORT = " << port[master] << ", MASTER_USE_GTID = " << type << ", " + "MASTER_USER='repl', MASTER_PASSWORD='repl';"; + + if (verbose) + { + std::cout << "Server " << slave + 1 << " starting to replicate from server " << master + 1 << std::endl; + std::cout << "Query is '" << change_master.str() << "'" << std::endl; + } + + execute_query(nodes[slave], "STOP SLAVE;"); + execute_query(nodes[slave], change_master.str().c_str()); + execute_query(nodes[slave], "START SLAVE;"); +} diff --git a/maxscale-system-test/mariadb_nodes.h b/maxscale-system-test/mariadb_nodes.h index fb0517ae9..5e2995cf7 100644 --- a/maxscale-system-test/mariadb_nodes.h +++ b/maxscale-system-test/mariadb_nodes.h @@ -430,6 +430,18 @@ public: /** Whether to require GTID based replication, defaults to false */ static void require_gtid(bool value); + /** + * Configure a server as a slave of another server + * + * The servers are configured with GTID replicating using the configured + * GTID position, either slave_pos or current_pos. + * + * @param slave The node index to assign as slave + * @param master The node index of the master + * @param type Replication type + */ + void replicate_from(int slave, int master, const char* type = "current_pos"); + private: bool check_master_node(MYSQL *conn); diff --git a/maxscale-system-test/mxs1509.cpp b/maxscale-system-test/mxs1509.cpp index a6d89892e..6d09cf4eb 100644 --- a/maxscale-system-test/mxs1509.cpp +++ b/maxscale-system-test/mxs1509.cpp @@ -25,28 +25,6 @@ void change_master(TestConnections& test, int slave, int master, const char* nam source.c_str(), test.repl->IP[master], test.repl->user_name, test.repl->password, source.c_str()); } -std::string dump_status(const StringSet& current, const StringSet& expected) -{ - std::stringstream ss; - ss << "Current status: ("; - - for (const auto& a: current) - { - ss << a << ","; - } - - ss << ") Expected status: ("; - - for (const auto& a: expected) - { - ss << a << ","; - } - - ss << ")"; - - return ss.str(); -} - void check_status(TestConnections& test, const StringSet& expected_master, const StringSet& expected_slave) { sleep(2); diff --git a/maxscale-system-test/mysqlmon_external_master.cpp b/maxscale-system-test/mysqlmon_external_master.cpp new file mode 100644 index 000000000..bcc93adef --- /dev/null +++ b/maxscale-system-test/mysqlmon_external_master.cpp @@ -0,0 +1,114 @@ +/** + * Test monitoring and failover with ignore_external_masters=true + */ +#include "testconnections.h" + +#include + +#define DOWN "Down" +#define RUNNING "Running" +#define MASTER "Master" +#define SLAVE "Slave" + +const StringSet master_running = {MASTER, RUNNING}; +const StringSet slave_running = {SLAVE, RUNNING}; +const StringSet running = {RUNNING}; +const StringSet down = {DOWN}; + +void check_status(TestConnections& test, const char* server, const StringSet& expected, const char* message) +{ + StringSet state = test.get_server_status(server); + test.assert(state == expected, "%s: %s", message, dump_status(state, expected).c_str()); +} + +static bool is_running = true; + +void writer_func(TestConnections* test) +{ + while (is_running) + { + MYSQL* conn = open_conn(test->maxscales->rwsplit_port[0], test->maxscales->IP[0], + "test", "test", false); + + for (int i = 0; i < 100; i++) + { + if (execute_query_silent(conn, "INSERT INTO test.t1 VALUES (SELECT SLEEP(0.5))")) + { + sleep(1); + break; + } + } + + mysql_close(conn); + } +} + +int main(int argc, char** argv) +{ + Mariadb_nodes::require_gtid(true); + TestConnections test(argc, argv); + + // Create a table and a user and start a thread that does writes + test.repl->connect(); + execute_query(test.repl->nodes[0], "CREATE OR REPLACE TABLE test.t1 (id INT)"); + execute_query(test.repl->nodes[0], "DROP USER IF EXISTS 'test'@'%%'"); + execute_query(test.repl->nodes[0], "CREATE USER 'test'@'%%' IDENTIFIED BY 'test'"); + execute_query(test.repl->nodes[0], "GRANT INSERT, SELECT, UPDATE, DELETE ON *.* TO 'test'@'%%'"); + test.repl->sync_slaves(); + std::thread thr(writer_func, &test); + + test.tprintf("Start by having the current master replicate from the external server"); + test.repl->connect(); + test.repl->replicate_from(0, 3); + sleep(5); + check_status(test, "server1", master_running, "server1 should be the master"); + check_status(test, "server2", slave_running, "server2 should be a slave"); + check_status(test, "server3", slave_running, "server3 should be a slave"); + + test.tprintf("Stop server1, expect server2 to be promoted as the master"); + test.repl->stop_node(0); + sleep(10); + + check_status(test, "server1", down, "server1 should be down"); + check_status(test, "server2", master_running, "server2 should be the master"); + check_status(test, "server3", slave_running, "server3 should be a slave"); + + test.tprintf("Configure master-master replication between server2 and the external server"); + test.repl->replicate_from(1, 3); + test.repl->replicate_from(3, 1); + sleep(10); + check_status(test, "server2", master_running, "server2 should still be the master"); + check_status(test, "server3", slave_running, "server3 should be a slave"); + + test.tprintf("Start server1, expect it to rejoin the cluster"); + test.repl->start_node(0); + sleep(10); + check_status(test, "server1", slave_running, "server1 should be a slave"); + check_status(test, "server2", master_running, "server2 should still be the master"); + check_status(test, "server3", slave_running, "server3 should be a slave"); + + test.tprintf("Stop server2, expect server1 to be promoted as the master"); + test.repl->stop_node(1); + test.repl->connect(); + test.repl->replicate_from(0, 3); + test.repl->replicate_from(3, 0); + sleep(10); + + check_status(test, "server1", master_running, "server1 should be the master"); + check_status(test, "server2", down, "server2 should be down"); + check_status(test, "server3", slave_running, "server3 should be a slave"); + + test.tprintf("Start server2, expect it to rejoin the cluster"); + test.repl->start_node(1); + sleep(10); + check_status(test, "server1", master_running, "server1 should still be the master"); + check_status(test, "server2", slave_running, "server2 should be a slave"); + check_status(test, "server3", slave_running, "server3 should be a slave"); + + // Cleanup + is_running = false; + thr.join(); + execute_query(test.repl->nodes[0], "STOP SLAVE; RESET SLAVE ALL;"); + + return test.global_result; +} diff --git a/maxscale-system-test/testconnections.cpp b/maxscale-system-test/testconnections.cpp index 9e9f8c529..9828afa7b 100644 --- a/maxscale-system-test/testconnections.cpp +++ b/maxscale-system-test/testconnections.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include "mariadb_func.h" #include "maxadmin_operations.h" @@ -1884,3 +1885,25 @@ bool TestConnections::test_bad_config(int m, const char *config) return maxscales->ssh_node(m, "cp maxscale.cnf /etc/maxscale.cnf; service maxscale stop; " "maxscale -U maxscale -lstdout &> /dev/null && sleep 1 && pkill -9 maxscale", false) == 0; } + +std::string dump_status(const StringSet& current, const StringSet& expected) +{ + std::stringstream ss; + ss << "Current status: ("; + + for (const auto& a: current) + { + ss << a << ","; + } + + ss << ") Expected status: ("; + + for (const auto& a: expected) + { + ss << a << ","; + } + + ss << ")"; + + return ss.str(); +} diff --git a/maxscale-system-test/testconnections.h b/maxscale-system-test/testconnections.h index fdd195f8a..18aa92413 100644 --- a/maxscale-system-test/testconnections.h +++ b/maxscale-system-test/testconnections.h @@ -531,4 +531,14 @@ void * timeout_thread(void *ptr ); */ void * log_copy_thread(void *ptr ); +/** +* Dump two server status sets as strings +* +* @param current The current status +* @param expected The expected status +* +* @return String form comparison of status sets +*/ +std::string dump_status(const StringSet& current, const StringSet& expected); + #endif // TESTCONNECTIONS_H