From a6914ebefec4bc2cbe4c1db2ceaed1ba3542ac74 Mon Sep 17 00:00:00 2001 From: Esa Korhonen Date: Wed, 3 Jan 2018 12:38:03 +0200 Subject: [PATCH] MXS-1565: Test for invalid old master rejoin The test uses standard setup (1xMaster, 3xSlaves). 1. Shutdown master (server 1), check that autofailover promotes a new master. 2. Stop MaxScale. 3. Start server 1 and add some events to it so it can no longer rejoin cluster. 4. Start MaxScale, check that server 1 does not join. 5. Set current master to replicate from server 1, turning it to a relay master. 6. Check that server 1 is master, all others are slaves (due to auto-rejoin). --- maxscale-system-test/.gitignore | 1 + maxscale-system-test/CMakeLists.txt | 3 + maxscale-system-test/mysqlmon_rejoin_bad2.cpp | 151 ++++++++++++++++++ 3 files changed, 155 insertions(+) create mode 100644 maxscale-system-test/mysqlmon_rejoin_bad2.cpp diff --git a/maxscale-system-test/.gitignore b/maxscale-system-test/.gitignore index b33e105e0..51da6f12a 100644 --- a/maxscale-system-test/.gitignore +++ b/maxscale-system-test/.gitignore @@ -180,6 +180,7 @@ mysqlmon_failover_auto mysqlmon_failover_manual mysqlmon_rejoin_good mysqlmon_rejoin_bad +mysqlmon_rejoin_bad2 namedserverfilter no_password non_native_setup diff --git a/maxscale-system-test/CMakeLists.txt b/maxscale-system-test/CMakeLists.txt index 22283e486..01ddb4f32 100644 --- a/maxscale-system-test/CMakeLists.txt +++ b/maxscale-system-test/CMakeLists.txt @@ -278,6 +278,9 @@ add_test_executable(mysqlmon_rejoin_good.cpp mysqlmon_rejoin_good mysqlmon_rejoi # MySQL Monitor Rejoin (bad) Test, use template for Rejoin (good) add_test_executable(mysqlmon_rejoin_bad.cpp mysqlmon_rejoin_bad mysqlmon_rejoin_good LABELS mysqlmon REPL_BACKEND) +# MySQL Monitor Rejoin (bad2) Test, use template for Rejoin (good) +add_test_executable(mysqlmon_rejoin_bad2.cpp mysqlmon_rejoin_bad2 mysqlmon_rejoin_good LABELS mysqlmon REPL_BACKEND) + # MySQL Monitor rolling master add_test_executable(mysqlmon_failover_rolling_master.cpp mysqlmon_failover_rolling_master mysqlmon_failover_rolling_master LABELS mysqlmon REPL_BACKEND) diff --git a/maxscale-system-test/mysqlmon_rejoin_bad2.cpp b/maxscale-system-test/mysqlmon_rejoin_bad2.cpp new file mode 100644 index 000000000..27837696b --- /dev/null +++ b/maxscale-system-test/mysqlmon_rejoin_bad2.cpp @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2016 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl11. + * + * Change Date: 2020-01-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ + +#include +#include +#include + +#include "testconnections.h" +#include "fail_switch_rejoin_common.cpp" + +using std::string; +using std::cout; +using std::endl; + +static void expect(TestConnections& test, const char* zServer, const StringSet& expected) +{ + StringSet found = test.get_server_status(zServer); + + std::ostream_iterator oi(cout, ", "); + + cout << zServer + << ", expected states: "; + std::copy(expected.begin(), expected.end(), oi); + cout << endl; + + cout << zServer + << ", found states : "; + std::copy(found.begin(), found.end(), oi); + cout << endl; + + if (found != expected) + { + cout << "ERROR, found states are not the same as the expected ones." << endl; + ++test.global_result; + } + + cout << endl; +} + +static void expect(TestConnections& test, const char* zServer, const char* zState) +{ + StringSet s; + s.insert(zState); + + expect(test, zServer, s); +} + +static void expect(TestConnections& test, const char* zServer, const char* zState1, const char* zState2) +{ + StringSet s; + s.insert(zState1); + s.insert(zState2); + + expect(test, zServer, s); +} + +int main(int argc, char** argv) +{ + char result_tmp[bufsize]; + interactive = strcmp(argv[argc - 1], "interactive") == 0; + Mariadb_nodes::require_gtid(true); + TestConnections test(argc, argv); + MYSQL* maxconn = test.maxscales->open_rwsplit_connection(0); + + // Set up test table + basic_test(test); + // Delete binlogs to sync gtid:s + delete_slave_binlogs(test); + // Advance gtid:s a bit to so gtid variables are updated. + generate_traffic_and_check(test, maxconn, 5); + test.repl->sync_slaves(0); + get_output(test); + + print_gtids(test); + get_input(); + mysql_close(maxconn); + + // Stop master, wait for failover + cout << "Stopping master, should auto-failover." << endl; + int master_id_old = get_master_server_id(test); + test.repl->stop_node(0); + sleep(5); + get_output(test); + int master_id_new = get_master_server_id(test); + cout << "Master server id is " << master_id_new << endl; + test.assert(master_id_new > 0 && master_id_new != master_id_old, + "Failover did not promote a new master."); + if (test.global_result != 0) + { + return test.global_result; + } + + // Stop maxscale to prevent an unintended rejoin. + if (test.stop_maxscale(0)) + { + test.assert(false, "Could not stop MaxScale."); + return test.global_result; + } + // Restart old master. Then add some events to it. + test.repl->start_node(0, (char*)""); + sleep(3); + test.repl->connect(); + cout << "Adding more events to node 0. It should not join the cluster." << endl; + generate_traffic_and_check(test, test.repl->nodes[0], 5); + print_gtids(test); + // Restart maxscale. Should not rejoin old master. + if (test.start_maxscale(0)) + { + test.assert(false, "Could not start MaxScale."); + return test.global_result; + } + sleep(5); + get_output(test); + + expect(test, "server1", "Running"); + if (test.global_result != 0) + { + cout << "Old master is a member or the cluster when it should not be." << endl; + return test.global_result; + } + + // Manually set current master to replicate from the old master to quickly fix the cluster. + cout << "Setting server " << master_id_new << " to replicate from server 1. Auto-rejoin should redirect " + "servers so that in the end server 1 is master and all others are slaves." << endl; + const char CHANGE_CMD_FMT[] = "CHANGE MASTER TO MASTER_HOST = '%s', MASTER_PORT = %d, " + "MASTER_USE_GTID = current_pos, MASTER_USER='repl', MASTER_PASSWORD = 'repl';"; + char cmd[256]; + int ind = master_id_new - 1; + snprintf(cmd, sizeof(cmd), CHANGE_CMD_FMT, test.repl->IP[0], test.repl->port[0]); + MYSQL** nodes = test.repl->nodes; + mysql_query(nodes[ind], cmd); + mysql_query(nodes[ind], "START SLAVE;"); + sleep(5); + get_output(test); + + expect(test, "server1", "Master", "Running"); + expect(test, "server2", "Slave", "Running"); + expect(test, "server3", "Slave", "Running"); + expect(test, "server4", "Slave", "Running"); + return test.global_result; +}