hcq/MaxScale
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add failover and read_only test

Browse Source 
The test performs a series of node stops and starts and checks that server
states are correct and read_only is set on slaves.
This commit is contained in:
Esa Korhonen
2018-11-22 16:21:00 +02:00
parent 1df42326d5
commit 19ad915685
3 changed files with 290 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
maxscale-system-test/CMakeLists.txt
View File

@ -276,6 +276,9 @@ add_test_executable(mysqlmon_reset_replication.cpp mysqlmon_reset_replication my
# MySQL Monitor low disk switchover
add_test_executable(mysqlmon_switchover_auto.cpp mysqlmon_switchover_auto mysqlmon_switchover_auto LABELS mysqlmon REPL_BACKEND)
# MySQL Monitor series of failovers and rejoins
add_test_executable(mysqlmon_failover_readonly.cpp mysqlmon_failover_readonly mysqlmon_failover_readonly LABELS mysqlmon REPL_BACKEND)
# MXS-1506: Delayed query retry
# https://jira.mariadb.org/browse/MXS-1506
add_test_executable(mxs1506_delayed_retry.cpp mxs1506_delayed_retry mxs1506_delayed_retry LABELS readwritesplit REPL_BACKEND)

62
maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_failover_readonly Normal file
View File

@ -0,0 +1,62 @@
[maxscale]
threads=###threads###
[MariaDB-Monitor]
type=monitor
module=mysqlmon
servers=server1, server2, server3
user=maxskysql
password=skysql
monitor_interval=2000
auto_failover=true
auto_rejoin=true
enforce_read_only_slaves=1
replication_user=repl
replication_password=repl
backend_connect_timeout=5
backend_read_timeout=5
backend_write_timeout=5
failcount=1
[RW-Split-Router]
type=service
router= readwritesplit
servers=server1, server2, server3
user=maxskysql
password=skysql
transaction_replay=1
delayed_retry_timeout=30
[RW-Split-Listener]
type=listener
service=RW-Split-Router
protocol=MySQLClient
port=4006
[CLI]
type=service
router=cli
[CLI Listener]
type=listener
service=CLI
protocol=maxscaled
socket=default
[server1]
type=server
address=###node_server_IP_1###
port=###node_server_port_1###
protocol=MySQLBackend
[server2]
type=server
address=###node_server_IP_2###
port=###node_server_port_2###
protocol=MySQLBackend
[server3]
type=server
address=###node_server_IP_3###
port=###node_server_port_3###
protocol=MySQLBackend

225
maxscale-system-test/mysqlmon_failover_readonly.cpp Normal file
View File

@ -0,0 +1,225 @@
/*
* Copyright (c) 2016 MariaDB Corporation Ab
*
* Use of this software is governed by the Business Source License included
* in the LICENSE.TXT file and at www.mariadb.com/bsl11.
*
* Change Date: 2022-01-01
*
* On the date above, in accordance with the Business Source License, use
* of this software will be governed by version 2 or later of the General
* Public License.
*/
#include "testconnections.h"
#include "fail_switch_rejoin_common.cpp"
#include <iostream>
#include <string>
#include <vector>
using std::string;
using std::cout;
int main(int argc, char** argv)
{
Mariadb_nodes::require_gtid(true);
TestConnections test(argc, argv);
// Delete binlogs to sync gtid:s
delete_slave_binlogs(test);
// Test uses 2 slaves, stop the last one to prevent it from replicating anything.
test.repl->stop_node(3);
// Set up test table
basic_test(test);
// Advance gtid:s a bit to so gtid variables are updated.
MYSQL* maxconn = test.maxscales->open_rwsplit_connection(0);
generate_traffic_and_check(test, maxconn, 1);
test.repl->sync_slaves(0);
get_output(test);
print_gtids(test);
auto expect_server_status = [&test](const string& server_name, const string& status) {
bool found = (test.maxscales->get_server_status(server_name.c_str()).count(status) == 1);
test.expect(found, "%s was not %s as was expected.", server_name.c_str(), status.c_str());
};
string server_names[] = {"server1", "server2", "server3"};
auto expect_server_status_multi =
[&test, &expect_server_status, &server_names](std::vector<string> expected) {
int expected_size = expected.size();
test.expect(expected_size <= test.repl->N && expected_size <= 3, "Too many expected values.");
int tests = (expected_size < test.repl->N) ? expected_size : test.repl->N;
for (int node = 0; node < tests; node++)
{
expect_server_status(server_names[node], expected[node]);
}
};
auto expect_read_only = [&test](int node, bool expected) {
test.expect(test.repl->connect(node) == 0, "Connection to node %i failed.", node);
char result[2];
const char query[] = "SELECT @@read_only;";
if (find_field(test.repl->nodes[node], query, "@@read_only", result) == 0)
{
char expected_result = expected ? '1' : '0';
test.expect(result[0] == expected_result, "read_only on node %i was %c when %c was expected.",
node, result[0], expected_result);
}
else
{
test.expect(false, "Query '%s' failed on node %i.", query, node);
}
};
auto expect_read_only_multi = [&test, &expect_read_only](std::vector<bool> expected) {
int expected_size = expected.size();
test.expect(expected_size <= test.repl->N, "Too many expected values.");
int tests = (expected_size < test.repl->N) ? expected_size : test.repl->N;
for (int node = 0; node < tests; node++)
{
expect_read_only(node, expected[node]);
}
};
auto mon_wait = [&test](int ticks) {
test.maxscales->wait_for_monitor(ticks);
};
auto crash_node = [&test](int node) {
test.repl->ssh_node(node, "kill -s 11 `pidof mysqld`", true);
test.repl->stop_node(node); // To prevent autostart.
};
string master = "Master";
string slave = "Slave";
string down = "Down";
cout << "Step 1: All should be cool.\n";
get_output(test);
expect_server_status_multi({master, slave, slave});
expect_read_only_multi({false, true, true});
if (test.ok())
{
cout << "Step 2: Crash slave 2.\n";
crash_node(2);
mon_wait(1);
get_output(test);
expect_server_status_multi({master, slave, down});
expect_read_only_multi({false, true});
generate_traffic_and_check(test, maxconn, 2);
cout << "Step 2.1: Slave 2 comes back up, check that read_only is set.\n";
test.repl->start_node(2, "");
mon_wait(2);
get_output(test);
expect_server_status_multi({master, slave, slave});
expect_read_only_multi({false, true, true});
generate_traffic_and_check(test, maxconn, 3);
cout << "Step 3: Slave 1 crashes.\n";
crash_node(1);
mon_wait(1);
get_output(test);
expect_server_status_multi({master, down, slave});
expect_read_only(2, true);
generate_traffic_and_check(test, maxconn, 4);
cout << "Step 4: Slave 2 goes down again, this time normally.\n";
test.repl->stop_node(2);
mon_wait(1);
get_output(test);
expect_server_status_multi({master, down, down});
generate_traffic_and_check(test, maxconn, 5);
cout << "Step 4.1: Slave 1 comes back up, check that read_only is set.\n";
test.repl->start_node(1, "");
mon_wait(2);
get_output(test);
expect_server_status_multi({master, slave, down});
expect_read_only_multi({false, true});
generate_traffic_and_check(test, maxconn, 6);
cout << "Step 4.2: Slave 2 is back up, all should be well.\n";
test.repl->start_node(2, "");
mon_wait(2);
get_output(test);
expect_server_status_multi({master, slave, slave});
expect_read_only_multi({false, true, true});
generate_traffic_and_check(test, maxconn, 5);
}
mysql_close(maxconn);
// Intermission, quit if a test step failed.
if (test.ok())
{
// Some of the following tests depend on manipulating backends during the same monitor tick or
// between ticks. Slow down the monitor to make this more likely. Not fool-proof in the slightest.
int rval = test.maxscales->execute_maxadmin_command(0, "alter monitor MariaDB-Monitor "
"monitor_interval=4000");
test.expect(rval == 0, "MaxAdmin command failed.");
}
if (test.ok())
{
cout << "Step 5: Master crashes but comes back during the next loop,"
" slave 1 should be promoted, old master rejoined.\n";
crash_node(0);
mon_wait(1); // The timing is probably a bit iffy here.
expect_server_status(server_names[0], down);
get_output(test);
test.repl->start_node(0, "");
mon_wait(2);
get_output(test);
// Slave 2 could be promoted as well, but in this case there is no reason to choose it.
expect_server_status_multi({slave, master, slave});
expect_read_only_multi({true, false, true});
maxconn = test.maxscales->open_rwsplit_connection(0);
generate_traffic_and_check(test, maxconn, 4);
cout << "Step 6: Servers 1 & 3 go down. Server 2 should remain as master.\n";
test.repl->stop_node(0);
test.repl->stop_node(2);
mon_wait(1);
get_output(test);
expect_server_status_multi({down, master, down});
generate_traffic_and_check(test, maxconn, 3);
cout << "Step 6.1: Servers 1 & 3 come back. Check that read_only is set.\n";
test.repl->start_node(2, "");
test.repl->start_node(0, "");
mon_wait(2);
get_output(test);
expect_server_status_multi({slave, master, slave});
expect_read_only_multi({true, false, true});
generate_traffic_and_check(test, maxconn, 2);
cout << "Step 7: Servers 1 & 2 go down. Check that 3 is promoted.\n";
mysql_close(maxconn);
test.repl->stop_node(0);
test.repl->stop_node(1);
mon_wait(2);
get_output(test);
expect_server_status_multi({down, down, master});
maxconn = test.maxscales->open_rwsplit_connection(0);
generate_traffic_and_check(test, maxconn, 1);
mysql_close(maxconn);
}
// Start the servers, in case they weren't on already.
for (int i = 0; i < 3; i++)
{
test.repl->start_node(i);
}
sleep(1);
test.repl->connect();
// Delete the test table from all databases, reset replication.
const char drop_query[] = "DROP TABLE IF EXISTS test.t1;";
for (int i = 0; i < 3; i++)
{
test.try_query(test.repl->nodes[i], drop_query);
}
test.maxscales->execute_maxadmin_command(0, "call command mariadbmon reset-replication "
"MariaDB-Monitor server1");
return test.global_result;
}