diff --git a/maxscale-system-test/CMakeLists.txt b/maxscale-system-test/CMakeLists.txt index 10ff8816c..a1fc28aee 100644 --- a/maxscale-system-test/CMakeLists.txt +++ b/maxscale-system-test/CMakeLists.txt @@ -31,7 +31,8 @@ add_library(testcore SHARED testconnections.cpp nodes.cpp mariadb_nodes.cpp maxs labels_table.cpp envv.cpp clustrix_nodes.cpp maxrest.cc # Include the CDC connector in the core library ${CMAKE_SOURCE_DIR}/connectors/cdc-connector/cdc_connector.cpp) -target_link_libraries(testcore ${MARIADB_CONNECTOR_LIBRARIES} ${JANSSON_LIBRARIES} z m pthread ssl dl rt crypto crypt maxbase) +target_link_libraries(testcore + ${MARIADB_CONNECTOR_LIBRARIES} ${JANSSON_LIBRARIES} maxbase maxsql z m pthread ssl dl rt crypto crypt) install(TARGETS testcore DESTINATION system-test) add_dependencies(testcore connector-c jansson maxbase) @@ -290,6 +291,9 @@ add_test_executable(mysqlmon_failover_readonly.cpp mysqlmon_failover_readonly my # MariaDB-Monitor enforce_simple_topology add_test_executable(mysqlmon_enforce_simple.cpp mysqlmon_enforce_simple mysqlmon_enforce_simple LABELS mysqlmon REPL_BACKEND) +# MariaDB-Monitor multimaster failover/switchover +add_test_executable(mysqlmon_fail_switch_multi.cpp mysqlmon_fail_switch_multi mysqlmon_fail_switch_multi LABELS mysqlmon REPL_BACKEND) + # MXS-1506: Delayed query retry # https://jira.mariadb.org/browse/MXS-1506 add_test_executable(mxs1506_delayed_retry.cpp mxs1506_delayed_retry mxs1506_delayed_retry LABELS readwritesplit REPL_BACKEND) diff --git a/maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_fail_switch_multi b/maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_fail_switch_multi new file mode 100644 index 000000000..f155fef84 --- /dev/null +++ b/maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_fail_switch_multi @@ -0,0 +1,55 @@ +[maxscale] +threads=###threads### + +[MariaDB-Monitor] +type=monitor +module=mysqlmon +servers= server1, server2 +user=maxskysql +password= skysql +monitor_interval=1000 +detect_standalone_master=true +auto_failover=true +auto_rejoin=true +replication_user=repl +replication_password=repl +backend_connect_timeout=10 +backend_read_timeout=10 +backend_write_timeout=10 +failcount=1 + +[RW-Split-Router] +type=service +router= readwritesplit +servers=server1, server2 +user=maxskysql +password=skysql + +[RW-Split-Listener] +type=listener +service=RW-Split-Router +protocol=MySQLClient +port=4006 + +[CLI] +type=service +router=cli + +[CLI-Listener] +type=listener +service=CLI +protocol=maxscaled +socket=default + +[server1] +type=server +address=###node_server_IP_1### +port=###node_server_port_1### +protocol=MySQLBackend + +[server2] +type=server +address=###node_server_IP_2### +port=###node_server_port_2### +protocol=MySQLBackend + diff --git a/maxscale-system-test/mysqlmon_fail_switch_multi.cpp b/maxscale-system-test/mysqlmon_fail_switch_multi.cpp new file mode 100644 index 000000000..ac171c0a3 --- /dev/null +++ b/maxscale-system-test/mysqlmon_fail_switch_multi.cpp @@ -0,0 +1,197 @@ +/* + * Copyright (c) 2019 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl11. + * + * Change Date: 2026-01-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ + +#include "testconnections.h" +#include "fail_switch_rejoin_common.cpp" +#include +#include +#include +#include + +// Test failover/switchover with multiple masters. + +using std::string; +using std::cout; +using mxq::QueryResult; + +void change_master(TestConnections& test ,int slave, int master, const string& conn_name = "", + int replication_delay = 0); + +void reset_master(TestConnections& test ,int slave, const string& conn_name = ""); +void expect_replicating_from(TestConnections& test, int node, int master); + +string server_names[] = {"server1", "server2", "server3", "server4"}; + +int main(int argc, char** argv) +{ + Mariadb_nodes::require_gtid(true); + TestConnections test(argc, argv); + //delete_slave_binlogs(test); + // Set up test table + basic_test(test); + // Advance gtid:s a bit to so gtid variables are updated. + MYSQL* maxconn = test.maxscales->open_rwsplit_connection(0); + generate_traffic_and_check(test, maxconn, 1); + test.repl->sync_slaves(0); + get_output(test); + print_gtids(test); + int ec = -1; + auto expect_server_status = [&test](const string& server_name, const string& status) { + bool found = (test.maxscales->get_server_status(server_name.c_str()).count(status) == 1); + test.expect(found, "%s was not %s as was expected.", server_name.c_str(), status.c_str()); + }; + + auto expect_server_status_multi = + [&test, &expect_server_status](std::vector expected) { + int expected_size = expected.size(); + test.expect(expected_size <= test.repl->N && expected_size <= 3, "Too many expected values."); + int tests = (expected_size < test.repl->N) ? expected_size : test.repl->N; + for (int node = 0; node < tests; node++) + { + expect_server_status(server_names[node], expected[node]); + } + }; + + auto mon_wait = [&test](int ticks) { + test.maxscales->wait_for_monitor(ticks); + }; + + string master = "Master"; + string slave = "Slave"; + string down = "Down"; + string relay = "Relay Master"; + string running = "Running"; + string ext_master = "Slave of External Server"; + + const string secondary_slave_conn = "b"; + // Only monitoring two servers for now. Stop replication to non-monitored servers. + reset_master(test, 2); + reset_master(test, 3); + + cout << "Step 1: All should be cool.\n"; + get_output(test); + expect_server_status_multi({master, slave}); + + if (test.ok()) + { + cout << "Step 2: External replication to two servers\n"; + change_master(test, 0, 2); + change_master(test, 0, 3, secondary_slave_conn); + mon_wait(1); + get_output(test); + expect_server_status_multi({master, slave}); + expect_server_status(server_names[0], ext_master); + expect_replicating_from(test, 0, 2); + expect_replicating_from(test, 0, 3); + } + + if (test.ok()) + { + cout << "Step 3: Failover. Check that new master replicates from external servers.\n"; + test.repl->stop_node(0); + mon_wait(3); + get_output(test); + expect_server_status_multi({down, master}); + expect_server_status(server_names[1], ext_master); + expect_replicating_from(test, 1, 2); + expect_replicating_from(test, 1, 3); + } + + if (test.ok()) + { + cout << "Step 4: Bring up old master, allow it to rejoin, then switchover. " + "Check that new master replicates from external servers.\n"; + test.repl->start_node(0); + mon_wait(2); // Should not rejoin since has multiple slave connections. + expect_server_status_multi({running, master}); + test.repl->connect(); + reset_master(test, 0); + reset_master(test, 0, secondary_slave_conn); + mon_wait(2); + get_output(test); + expect_server_status_multi({slave, master}); + test.maxscales->ssh_node_output(0, "maxctrl call command mariadbmon switchover MariaDB-Monitor", + true, &ec); + mon_wait(2); + expect_replicating_from(test, 0, 2); + expect_replicating_from(test, 0, 3); + get_output(test); + + // TODO: Extend test later + } + + + // Cleanup + mysql_close(maxconn); + test.repl->connect(); + // Delete the test table from all databases, reset replication. + const char drop_query[] = "DROP TABLE IF EXISTS test.t1;"; + for (int i = 0; i < 3; i++) + { + test.try_query(test.repl->nodes[i], drop_query); + } + test.maxscales->ssh_node_output(0, "maxctrl call command mariadbmon reset-replication " + "MariaDB-Monitor server1", true, &ec); + return test.global_result; +} + +void change_master(TestConnections& test ,int slave, int master, const string& conn_name, + int replication_delay) +{ + const char query[] = "CHANGE MASTER '%s' TO master_host='%s', master_port=%d, " + "MASTER_USE_GTID = current_pos, " + "master_user='repl', master_password='repl', master_delay=%d; " + "START SLAVE '%s';"; + test.try_query(test.repl->nodes[slave], query, conn_name.c_str(), + test.repl->IP[master], test.repl->port[master], + replication_delay, conn_name.c_str()); +} + +void reset_master(TestConnections& test ,int slave, const string& conn_name) +{ + const char query[] = "STOP SLAVE '%s'; RESET SLAVE '%s' ALL;"; + test.try_query(test.repl->nodes[slave], query, conn_name.c_str(), conn_name.c_str()); +} + +void expect_replicating_from(TestConnections& test, int node, int master) +{ + bool found = false; + auto N = test.repl->N; + if (node < N && master < N) + { + auto conn = test.repl->nodes[node]; + if (mysql_query(conn, "SHOW ALL SLAVES STATUS;") == 0) + { + auto res = mysql_store_result(conn); + if (res) + { + QueryResult q_res(res); + auto search_host = test.repl->ip(master); + auto search_port = test.repl->port[master]; + while (q_res.next_row()) + { + auto host = q_res.get_string("Master_Host"); + auto port = q_res.get_int("Master_Port"); + if (host == search_host && port == search_port) + { + found = true; + break; + } + } + } + } + } + + test.expect(found, "Server %s is not replicating from %s.", + server_names[node].c_str(), server_names[master].c_str()); +}