hcq/MaxScale
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add manual rejoin tests

Browse Source
This commit is contained in:
Esa Korhonen
2018-01-23 14:09:29 +02:00
parent 9093f19c8b
commit d681d0f2fa
5 changed files with 344 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
maxscale-system-test/.gitignore vendored
View File

@ -183,6 +183,8 @@ mysqlmon_failover_manual
mysqlmon_rejoin_good mysqlmon_rejoin_good
mysqlmon_rejoin_bad mysqlmon_rejoin_bad
mysqlmon_rejoin_bad2 mysqlmon_rejoin_bad2
mysqlmon_rejoin_manual
mysqlmon_rejoin_manual2
namedserverfilter namedserverfilter
no_password no_password
non_native_setup non_native_setup

4
maxscale-system-test/CMakeLists.txt
View File

@ -281,6 +281,10 @@ add_test_executable(mysqlmon_rejoin_bad.cpp mysqlmon_rejoin_bad mysqlmon_rejoin_
# MySQL Monitor Rejoin (bad2) Test, use template for Rejoin (good) # MySQL Monitor Rejoin (bad2) Test, use template for Rejoin (good)
add_test_executable(mysqlmon_rejoin_bad2.cpp mysqlmon_rejoin_bad2 mysqlmon_rejoin_good LABELS mysqlmon REPL_BACKEND) add_test_executable(mysqlmon_rejoin_bad2.cpp mysqlmon_rejoin_bad2 mysqlmon_rejoin_good LABELS mysqlmon REPL_BACKEND)
# MySQL Monitor Rejoin tests
add_test_executable(mysqlmon_rejoin_manual.cpp mysqlmon_rejoin_manual mysqlmon_rejoin_manual LABELS mysqlmon REPL_BACKEND)
add_test_executable(mysqlmon_rejoin_manual2.cpp mysqlmon_rejoin_manual2 mysqlmon_rejoin_manual LABELS mysqlmon REPL_BACKEND)
# MySQL Monitor rolling master # MySQL Monitor rolling master
add_test_executable(mysqlmon_failover_rolling_master.cpp mysqlmon_failover_rolling_master mysqlmon_failover_rolling_master LABELS mysqlmon REPL_BACKEND) add_test_executable(mysqlmon_failover_rolling_master.cpp mysqlmon_failover_rolling_master mysqlmon_failover_rolling_master LABELS mysqlmon REPL_BACKEND)

94
maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_rejoin_manual Normal file
View File

@ -0,0 +1,94 @@
[maxscale]
threads=###threads###
[MySQL-Monitor]
type=monitor
module=mysqlmon
servers= server1, server2, server3, server4
user=maxskysql
passwd= skysql
monitor_interval=1000
detect_standalone_master=true
failcount=1
allow_cluster_recovery=true
replication_user=repl
replication_password=repl
backend_connect_timeout=3
backend_read_timeout=3
backend_write_timeout=3
auto_failover=true
[RW Split Router]
type=service
router= readwritesplit
servers=server1, server2, server3, server4
user=maxskysql
passwd=skysql
[Read Connection Router Slave]
type=service
router=readconnroute
router_options= slave
servers=server1, server2, server3, server4
user=maxskysql
passwd=skysql
[Read Connection Router Master]
type=service
router=readconnroute
router_options=master
servers=server1, server2, server3, server4
user=maxskysql
passwd=skysql
[RW Split Listener]
type=listener
service=RW Split Router
protocol=MySQLClient
port=4006
[Read Connection Listener Slave]
type=listener
service=Read Connection Router Slave
protocol=MySQLClient
port=4009
[Read Connection Listener Master]
type=listener
service=Read Connection Router Master
protocol=MySQLClient
port=4008
[CLI]
type=service
router=cli
[CLI Listener]
type=listener
service=CLI
protocol=maxscaled
socket=default
[server1]
type=server
address=###node_server_IP_1###
port=###node_server_port_1###
protocol=MySQLBackend
[server2]
type=server
address=###node_server_IP_2###
port=###node_server_port_2###
protocol=MySQLBackend
[server3]
type=server
address=###node_server_IP_3###
port=###node_server_port_3###
protocol=MySQLBackend
[server4]
type=server
address=###node_server_IP_4###
port=###node_server_port_4###
protocol=MySQLBackend

101
maxscale-system-test/mysqlmon_rejoin_manual.cpp Normal file
View File

@ -0,0 +1,101 @@
/*
* Copyright (c) 2016 MariaDB Corporation Ab
*
* Use of this software is governed by the Business Source License included
* in the LICENSE.TXT file and at www.mariadb.com/bsl11.
*
* Change Date: 2020-01-01
*
* On the date above, in accordance with the Business Source License, use
* of this software will be governed by version 2 or later of the General
* Public License.
*/
#include "fail_switch_rejoin_common.cpp"
#include <iostream>
using std::string;
using std::cout;
using std::endl;
int main(int argc, char** argv)
{
interactive = strcmp(argv[argc - 1], "interactive") == 0;
Mariadb_nodes::require_gtid(true);
TestConnections test(argc, argv);
int ec;
MYSQL* maxconn = test.maxscales->open_rwsplit_connection(0);
// Set up test table
basic_test(test);
// Delete binlogs to sync gtid:s
delete_slave_binlogs(test);
char result_tmp[bufsize];
// Advance gtid:s a bit to so gtid variables are updated.
generate_traffic_and_check(test, maxconn, 10);
sleep(1);
test.tprintf(LINE);
print_gtids(test);
get_input();
cout << "Stopping master and waiting for failover. Check that another server is promoted." << endl;
const int old_master_id = get_master_server_id(test); // Read master id now before shutdown.
const int master_index = test.repl->master;
test.repl->stop_node(master_index);
sleep(10);
// Recreate maxscale session
mysql_close(maxconn);
maxconn = test.maxscales->open_rwsplit_connection(0);
get_output(test);
int master_id = get_master_server_id(test);
cout << "Master server id is " << master_id << endl;
const bool failover_ok = (master_id > 0 && master_id != old_master_id);
test.assert(failover_ok, "Master did not change or no master detected.");
string gtid_final;
if (failover_ok)
{
cout << "Sending more inserts." << endl;
generate_traffic_and_check(test, maxconn, 5);
sleep(1);
if (find_field(maxconn, GTID_QUERY, GTID_FIELD, result_tmp) == 0)
{
gtid_final = result_tmp;
}
print_gtids(test);
cout << "Bringing old master back online..." << endl;
test.repl->start_node(master_index, (char*) "");
sleep(5);
test.repl->connect();
get_output(test);
test.tprintf("and manually rejoining it to cluster.");
const char REJOIN_CMD[] = "maxadmin call command mariadbmon rejoin MySQL-Monitor server1";
test.maxscales->ssh_node_output(0, REJOIN_CMD , true, &ec);
sleep(2);
get_output(test);
string gtid_old_master;
if (find_field(test.repl->nodes[master_index], GTID_QUERY, GTID_FIELD, result_tmp) == 0)
{
gtid_old_master = result_tmp;
}
cout << LINE << "\n";
print_gtids(test);
cout << LINE << "\n";
test.assert(gtid_final == gtid_old_master, "Old master did not successfully rejoin the cluster.");
// Switch master back to server1 so last check is faster
int ec;
test.maxscales->ssh_node_output(0, "maxadmin call command mysqlmon switchover "
"MySQL-Monitor server1 server2" , true, &ec);
sleep(5); // Wait for monitor to update status
get_output(test);
master_id = get_master_server_id(test);
test.assert(master_id == old_master_id, "Switchover back to server1 failed.");
}
else
{
test.repl->start_node(master_index, (char*) "");
sleep(10);
}
test.repl->fix_replication();
return test.global_result;
}

143
maxscale-system-test/mysqlmon_rejoin_manual2.cpp Normal file
View File

@ -0,0 +1,143 @@
/*
* Copyright (c) 2016 MariaDB Corporation Ab
*
* Use of this software is governed by the Business Source License included
* in the LICENSE.TXT file and at www.mariadb.com/bsl11.
*
* Change Date: 2020-01-01
*
* On the date above, in accordance with the Business Source License, use
* of this software will be governed by version 2 or later of the General
* Public License.
*/
#include <vector>
#include "testconnections.h"
#include "fail_switch_rejoin_common.cpp"
#include <iostream>
using std::string;
using std::cout;
using std::endl;
int main(int argc, char** argv)
{
char result_tmp[bufsize];
interactive = strcmp(argv[argc - 1], "interactive") == 0;
Mariadb_nodes::require_gtid(true);
TestConnections test(argc, argv);
MYSQL* maxconn = test.maxscales->open_rwsplit_connection(0);
// Set up test table
basic_test(test);
// Delete binlogs to sync gtid:s
delete_slave_binlogs(test);
// Advance gtid:s a bit to so gtid variables are updated.
generate_traffic_and_check(test, maxconn, 10);
test.repl->sync_slaves(0);
cout << LINE << "\n";
print_gtids(test);
cout << LINE << "\n";
string gtid_begin;
if (find_field(maxconn, GTID_QUERY, GTID_FIELD, result_tmp) == 0)
{
gtid_begin = result_tmp;
}
mysql_close(maxconn);
// Leave first of three slaves connected so it's clear which one is the master server.
const char STOP_SLAVE[] = "STOP SLAVE;";
const char RESET_SLAVE[] = "RESET SLAVE ALL;";
const char READ_ONLY_OFF[] = "SET GLOBAL read_only=0;";
test.repl->connect();
const int FIRST_MOD_NODE = 2; // Modify nodes 2 & 3
const int NODE_COUNT = test.repl->N;
MYSQL** nodes = test.repl->nodes;
for (int i = FIRST_MOD_NODE; i < NODE_COUNT; i++)
{
if (mysql_query(nodes[i], STOP_SLAVE) != 0 ||
mysql_query(nodes[i], RESET_SLAVE) != 0 ||
mysql_query(nodes[i], READ_ONLY_OFF) != 0)
{
test.assert(false, "Could not stop slave connections and/or disable read_only for node %d.", i);
return test.global_result;
}
}
// Add more events to node3.
string gtid_node2, gtid_node3;
cout << "Sending more inserts to server 4.\n";
generate_traffic_and_check(test, nodes[3], 10);
// Save gtids
if (find_field(nodes[2], GTID_QUERY, GTID_FIELD, result_tmp) == 0)
{
gtid_node2 = result_tmp;
}
if (find_field(nodes[3], GTID_QUERY, GTID_FIELD, result_tmp) == 0)
{
gtid_node3 = result_tmp;
}
print_gtids(test);
bool gtids_ok = (gtid_begin == gtid_node2 && gtid_node2 < gtid_node3);
test.assert(gtids_ok, "Gtid:s have not advanced correctly.");
if (!gtids_ok)
{
return test.global_result;
}
cout << "Sending rejoin commands for servers 3 & 4. Server 4 should not rejoin the cluster.\n";
const string REJOIN_CMD = "maxadmin call command mariadbmon rejoin MySQL-Monitor";
int ec;
string rejoin_s3 = REJOIN_CMD + " server3";
string rejoin_s4 = REJOIN_CMD + " server4";
test.maxscales->ssh_node_output(0, rejoin_s3.c_str() , true, &ec);
test.maxscales->ssh_node_output(0, rejoin_s4.c_str() , true, &ec);
sleep(5);
get_output(test);
StringSet node2_states = test.get_server_status("server3");
StringSet node3_states = test.get_server_status("server4");
bool states_n2_ok = (node2_states.find("Slave") != node2_states.end());
bool states_n3_ok = (node3_states.find("Slave") == node3_states.end());
test.assert(states_n2_ok, "Node 2 has not rejoined when it should have.");
test.assert(states_n3_ok, "Node 3 rejoined when it shouldn't have.");
if (!states_n2_ok || !states_n3_ok)
{
return test.global_result;
}
// Finally, fix replication by telling the current master to replicate from server4
test.tprintf("Setting server 1 to replicate from server 4. Manually rejoin servers 2 and 3.");
const char CHANGE_CMD_FMT[] = "CHANGE MASTER TO MASTER_HOST = '%s', MASTER_PORT = %d, "
"MASTER_USE_GTID = current_pos, MASTER_USER='repl', MASTER_PASSWORD = 'repl';";
char cmd[256];
snprintf(cmd, sizeof(cmd), CHANGE_CMD_FMT, test.repl->IP[3], test.repl->port[3]);
mysql_query(nodes[0], cmd);
mysql_query(nodes[0], "START SLAVE;");
sleep(5);
string rejoin_s2 = REJOIN_CMD + " server2";
test.maxscales->ssh_node_output(0, rejoin_s2.c_str() , true, &ec);
test.maxscales->ssh_node_output(0, rejoin_s3.c_str() , true, &ec);
sleep(2);
get_output(test);
int master_id = get_master_server_id(test);
test.assert(master_id == 4, "Server 4 should be the cluster master.");
StringSet node0_states = test.get_server_status("server1");
bool states_n0_ok = (node0_states.find("Slave") != node0_states.end() &&
node0_states.find("Relay Master") == node0_states.end());
test.assert(states_n0_ok, "Server 1 is not a slave when it should be.");
if (states_n0_ok)
{
int ec;
test.maxscales->ssh_node_output(0,
"maxadmin call command mysqlmon switchover MySQL-Monitor server1 server4" , true, &ec);
sleep(1);
master_id = get_master_server_id(test);
test.assert(master_id == 1, "Server 1 should be the cluster master.");
get_output(test);
}
test.repl->fix_replication();
return test.global_result;
}