MXS-1567 Rolling master promotion
- The test starts with the usual setup of 1 master and 3 slaves. - Then the master is taken down and it is checked that the failover mechanism promotes some slave to master. - This is continued until there is a single master left (no slaves).
This commit is contained in:
Johan Wikman
@ -258,7 +258,7 @@ add_test_executable(mysqlmon_failover_auto.cpp mysqlmon_failover_auto mysqlmon_f
|
||||
# MySQL Monitor Failover (manual) Test
|
||||
add_test_executable(mysqlmon_failover_manual.cpp mysqlmon_failover_manual mysqlmon_failover_manual LABELS mysqlmon REPL_BACKEND)
|
||||
|
||||
# MySQL Monitor manual failover with one candidate
|
||||
# MySQL Monitor manual failover with many valid candidates
|
||||
add_test_executable(mysqlmon_failover_manual2.cpp mysqlmon_failover_manual2_4 mysqlmon_failover_manual2_4 LABELS mysqlmon REPL_BACKEND)
|
||||
add_test_executable(mysqlmon_failover_manual2.cpp mysqlmon_failover_manual2_3 mysqlmon_failover_manual2_3 LABELS mysqlmon REPL_BACKEND)
|
||||
add_test_executable(mysqlmon_failover_manual2.cpp mysqlmon_failover_manual2_2 mysqlmon_failover_manual2_2 LABELS mysqlmon REPL_BACKEND)
|
||||
@ -266,6 +266,9 @@ add_test_executable(mysqlmon_failover_manual2.cpp mysqlmon_failover_manual2_2 my
|
||||
# MySQL Monitor Rejoin Test
|
||||
add_test_executable(mysqlmon_rejoin_good.cpp mysqlmon_rejoin_good mysqlmon_rejoin_good LABELS mysqlmon REPL_BACKEND)
|
||||
|
||||
# MySQL Monitor rolling master
|
||||
add_test_executable(mysqlmon_failover_rolling_master.cpp mysqlmon_failover_rolling_master mysqlmon_failover_rolling_master LABELS mysqlmon REPL_BACKEND)
|
||||
|
||||
# Test monitor state change events when manually clearing server bits
|
||||
add_test_executable(false_monitor_state_change.cpp false_monitor_state_change replication LABELS mysqlmon REPL_BACKEND)
|
||||
|
||||
|
||||
@ -0,0 +1,91 @@
|
||||
[maxscale]
|
||||
threads=###threads###
|
||||
|
||||
[MySQL-Monitor]
|
||||
type=monitor
|
||||
module=mysqlmon
|
||||
servers= server1, server2, server3, server4
|
||||
user=maxskysql
|
||||
passwd= skysql
|
||||
monitor_interval=1000
|
||||
allow_cluster_recovery=true
|
||||
detect_standalone_master=true
|
||||
auto_failover=true
|
||||
replication_user=repl
|
||||
replication_password=repl
|
||||
backend_connect_timeout=1
|
||||
|
||||
[RW-Split-Router]
|
||||
type=service
|
||||
router= readwritesplit
|
||||
servers=server1, server2, server3, server4
|
||||
user=maxskysql
|
||||
passwd=skysql
|
||||
|
||||
[Read-Connection-Router-Slave]
|
||||
type=service
|
||||
router=readconnroute
|
||||
router_options= slave
|
||||
servers=server1, server2, server3, server4
|
||||
user=maxskysql
|
||||
passwd=skysql
|
||||
|
||||
[Read-Connection-Router-Master]
|
||||
type=service
|
||||
router=readconnroute
|
||||
router_options=master
|
||||
servers=server1, server2, server3, server4
|
||||
user=maxskysql
|
||||
passwd=skysql
|
||||
|
||||
[RW-Split-Listener]
|
||||
type=listener
|
||||
service=RW-Split-Router
|
||||
protocol=MySQLClient
|
||||
port=4006
|
||||
|
||||
[Read-Connection-Listener-Slave]
|
||||
type=listener
|
||||
service=Read-Connection-Router-Slave
|
||||
protocol=MySQLClient
|
||||
port=4009
|
||||
|
||||
[Read-Connection-Listener-Master]
|
||||
type=listener
|
||||
service=Read-Connection-Router-Master
|
||||
protocol=MySQLClient
|
||||
port=4008
|
||||
|
||||
[CLI]
|
||||
type=service
|
||||
router=cli
|
||||
|
||||
[CLI Listener]
|
||||
type=listener
|
||||
service=CLI
|
||||
protocol=maxscaled
|
||||
socket=default
|
||||
|
||||
[server1]
|
||||
type=server
|
||||
address=###node_server_IP_1###
|
||||
port=###node_server_port_1###
|
||||
protocol=MySQLBackend
|
||||
|
||||
[server2]
|
||||
type=server
|
||||
address=###node_server_IP_2###
|
||||
port=###node_server_port_2###
|
||||
protocol=MySQLBackend
|
||||
|
||||
[server3]
|
||||
type=server
|
||||
address=###node_server_IP_3###
|
||||
port=###node_server_port_3###
|
||||
protocol=MySQLBackend
|
||||
|
||||
[server4]
|
||||
type=server
|
||||
address=###node_server_IP_4###
|
||||
port=###node_server_port_4###
|
||||
protocol=MySQLBackend
|
||||
271
maxscale-system-test/mysqlmon_failover_rolling_master.cpp
Normal file
271
maxscale-system-test/mysqlmon_failover_rolling_master.cpp
Normal file
@ -0,0 +1,271 @@
|
||||
/*
|
||||
* Copyright (c) 2016 MariaDB Corporation Ab
|
||||
*
|
||||
* Use of this software is governed by the Business Source License included
|
||||
* in the LICENSE.TXT file and at www.mariadb.com/bsl11.
|
||||
*
|
||||
* Change Date: 2020-01-01
|
||||
*
|
||||
* On the date above, in accordance with the Business Source License, use
|
||||
* of this software will be governed by version 2 or later of the General
|
||||
* Public License.
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
#include <sstream>
|
||||
#include "testconnections.h"
|
||||
|
||||
using std::cerr;
|
||||
using std::cout;
|
||||
using std::endl;
|
||||
using std::flush;
|
||||
using std::string;
|
||||
using std::stringstream;
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
void sleep(int s)
|
||||
{
|
||||
cout << "Sleeping " << s << " times 1 second" << flush;
|
||||
do
|
||||
{
|
||||
::sleep(1);
|
||||
cout << "." << flush;
|
||||
--s;
|
||||
}
|
||||
while (s > 0);
|
||||
|
||||
cout << endl;
|
||||
}
|
||||
|
||||
int get_server_id(Maxscales& maxscales)
|
||||
{
|
||||
MYSQL *conn = maxscales.open_rwsplit_connection(0);
|
||||
int id = -1;
|
||||
char str[1024];
|
||||
|
||||
if (find_field(conn, "SELECT @@server_id, @@last_insert_id;", "@@server_id", str) == 0)
|
||||
{
|
||||
id = atoi(str);
|
||||
}
|
||||
|
||||
mysql_close(conn);
|
||||
|
||||
if (id == -1)
|
||||
{
|
||||
throw std::runtime_error("Could not get server id.");
|
||||
}
|
||||
|
||||
return id;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
class XTestConnections : private TestConnections
|
||||
{
|
||||
public:
|
||||
using TestConnections::add_result;
|
||||
using TestConnections::global_result;
|
||||
using TestConnections::maxscales;
|
||||
using TestConnections::repl;
|
||||
|
||||
XTestConnections(int argc, char* argv[])
|
||||
: TestConnections(argc, argv)
|
||||
{
|
||||
}
|
||||
|
||||
TestConnections& nothrow()
|
||||
{
|
||||
return *this;
|
||||
}
|
||||
|
||||
void connect_maxscale(int m = 0)
|
||||
{
|
||||
if (maxscales->connect_maxscale(m) != 0)
|
||||
{
|
||||
++global_result;
|
||||
throw std::runtime_error("Could not connect to MaxScale.");
|
||||
}
|
||||
}
|
||||
|
||||
void try_query(MYSQL *conn, const char *format, ...)
|
||||
{
|
||||
va_list valist;
|
||||
|
||||
va_start(valist, format);
|
||||
int message_len = vsnprintf(NULL, 0, format, valist);
|
||||
va_end(valist);
|
||||
|
||||
char sql[message_len + 1];
|
||||
|
||||
va_start(valist, format);
|
||||
vsnprintf(sql, sizeof(sql), format, valist);
|
||||
va_end(valist);
|
||||
|
||||
int res = execute_query_silent(conn, sql, false);
|
||||
add_result(res, "Query '%.*s%s' failed!\n", message_len < 100 ? message_len : 100, sql,
|
||||
message_len < 100 ? "" : "...");
|
||||
|
||||
if (res != 0)
|
||||
{
|
||||
string s("Could not execute query: ");
|
||||
s += sql;
|
||||
|
||||
if (s.length() > 80)
|
||||
{
|
||||
s = s.substr(0, 77);
|
||||
s += "...";
|
||||
}
|
||||
|
||||
throw std::runtime_error(s.c_str());
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
void list_servers(XTestConnections& test)
|
||||
{
|
||||
cout << endl;
|
||||
test.maxscales->execute_maxadmin_command_print(0, (char*)"list servers");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
void create_table(XTestConnections& test)
|
||||
{
|
||||
test.try_query(test.maxscales->conn_rwsplit[0], "DROP TABLE IF EXISTS test.t1");
|
||||
test.try_query(test.maxscales->conn_rwsplit[0], "CREATE TABLE test.t1(id INT)");
|
||||
}
|
||||
|
||||
static int i_start = 0;
|
||||
static int n_rows = 20;
|
||||
static int i_end = 0;
|
||||
|
||||
void insert_data(XTestConnections& test)
|
||||
{
|
||||
test.try_query(test.maxscales->conn_rwsplit[0], "BEGIN");
|
||||
|
||||
i_end = i_start + n_rows;
|
||||
|
||||
for (int i = i_start; i < i_end; ++i)
|
||||
{
|
||||
stringstream ss;
|
||||
ss << "INSERT INTO test.t1 VALUES (" << i << ")";
|
||||
test.try_query(test.maxscales->conn_rwsplit[0], ss.str().c_str());
|
||||
}
|
||||
test.try_query(test.maxscales->conn_rwsplit[0], "COMMIT");
|
||||
|
||||
i_start = i_end;
|
||||
}
|
||||
|
||||
void check(XTestConnections& test)
|
||||
{
|
||||
MYSQL* pConn = test.maxscales->open_rwsplit_connection(0);
|
||||
const char* zQuery = "SELECT * FROM test.t1";
|
||||
|
||||
test.try_query(pConn, "BEGIN");
|
||||
mysql_query(pConn, zQuery);
|
||||
|
||||
MYSQL_RES* pRes = mysql_store_result(pConn);
|
||||
test.add_result(pRes == NULL, "Query should return a result set.");
|
||||
|
||||
|
||||
if (!pRes)
|
||||
{
|
||||
mysql_close(pConn);
|
||||
throw std::runtime_error("Query did not return a result set.");
|
||||
}
|
||||
|
||||
std::string values;
|
||||
MYSQL_ROW row;
|
||||
int num_rows = mysql_num_rows(pRes);
|
||||
test.add_result(num_rows != i_end, "Query returned %d rows when %d rows were expected",
|
||||
num_rows, i_end);
|
||||
test.nothrow().try_query(pConn, "COMMIT");
|
||||
mysql_close(pConn);
|
||||
}
|
||||
|
||||
void stop_node(XTestConnections& test, int index)
|
||||
{
|
||||
if (test.repl->stop_node(index) != 0)
|
||||
{
|
||||
throw std::runtime_error("Could not stop node.");
|
||||
}
|
||||
|
||||
list_servers(test);
|
||||
}
|
||||
|
||||
void run(XTestConnections& test)
|
||||
{
|
||||
sleep(5);
|
||||
|
||||
int N = test.repl->N;
|
||||
cout << "Nodes: " << N << endl;
|
||||
|
||||
cout << "\nConnecting to MaxScale." << endl;
|
||||
test.connect_maxscale();
|
||||
|
||||
cout << "\nCreating table." << endl;
|
||||
create_table(test);
|
||||
|
||||
list_servers(test);
|
||||
|
||||
for (int i = 0; i < N - 1; ++i)
|
||||
{
|
||||
cout << "Round: " << i << "\n"
|
||||
<< "--------" << endl;
|
||||
cout << "\nInserting data." << endl;
|
||||
insert_data(test);
|
||||
|
||||
cout << "\nSyncing slaves." << endl;
|
||||
test.repl->sync_slaves();
|
||||
|
||||
int master_id = get_server_id(*test.maxscales);
|
||||
int master_index = master_id - 1;
|
||||
|
||||
cout << "\nStopping master." << endl;
|
||||
stop_node(test, master_index);
|
||||
|
||||
cout << "\nClosing connection to MaxScale." << endl;
|
||||
test.maxscales->close_maxscale_connections(0);
|
||||
|
||||
sleep(5);
|
||||
|
||||
list_servers(test);
|
||||
|
||||
master_id = get_server_id(*test.maxscales);
|
||||
cout << "\nNew master is: " << master_id << endl;
|
||||
|
||||
cout << "\nConnecting to MaxScale." << endl;
|
||||
test.connect_maxscale();
|
||||
|
||||
cout << "\nChecking result." << endl;
|
||||
check(test);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
int main(int argc, char* argv[])
|
||||
{
|
||||
XTestConnections test(argc, argv);
|
||||
|
||||
try
|
||||
{
|
||||
run(test);
|
||||
}
|
||||
catch (const std::exception& x)
|
||||
{
|
||||
cerr << "error: Execution was terminated due to an exception: " << x.what() << endl;
|
||||
}
|
||||
|
||||
return test.global_result;
|
||||
}
|
||||
Reference in New Issue
Block a user