402 lines
14 KiB
C++
402 lines
14 KiB
C++
/**
|
|
* @file mysqlmon_multimaster.cpp MySQL Monitor Multi-master Test
|
|
* - Configure all servers into a multi-master ring with one slave
|
|
* - check status using Maxadmin 'show servers' and 'show monitor "MySQL Monitor"'
|
|
* - Set nodes 0 and 1 into read-only mode
|
|
* - repeat status check
|
|
* - Configure nodes 1 and 2 (server2 and server3) into a master-master pair, make node 0 a slave of node 1
|
|
* and node 3 a slave of node 2
|
|
* - repeat status check
|
|
* - Set node 1 into read-only mode
|
|
* - repeat status check
|
|
* - Create two distinct groups (server1 and server2 are masters for eache others and same for server3 and
|
|
* server4)
|
|
* - repeat status check
|
|
* - Set nodes 1 and 3 (server2 and server4) into read-only mode
|
|
*
|
|
* Addition: add delays to some slave connections and check that the monitor correctly detects the delay
|
|
*/
|
|
|
|
|
|
#include <iostream>
|
|
#include <string>
|
|
#include <set>
|
|
#include <maxtest/testconnections.hh>
|
|
#include <maxtest/maxadmin_operations.hh>
|
|
#include <maxtest/sql_t1.hh>
|
|
#include <jansson.h>
|
|
#include <maxbase/string.hh>
|
|
|
|
using std::cout;
|
|
using std::string;
|
|
using std::set;
|
|
|
|
void check_status(TestConnections& test, const string& server, const set<string>& expected_status)
|
|
{
|
|
string cmd = "show server " + server;
|
|
char buf[100];
|
|
|
|
test.maxscales->get_maxadmin_param(0, cmd.c_str(), "Status:", buf);
|
|
string maxadmin_result = buf;
|
|
|
|
if (maxadmin_result.empty())
|
|
{
|
|
test.expect(false, "maxadmin execution error\n");
|
|
}
|
|
else
|
|
{
|
|
// First tokenize by comma, then trim. Cannot tokenize by whitespace due to
|
|
// "Relay Master"-element.
|
|
auto tokens = mxb::strtok(maxadmin_result, ",");
|
|
auto trimmer = [](string& s) {
|
|
mxb::trim(s);
|
|
};
|
|
std::for_each(tokens.begin(), tokens.end(), trimmer);
|
|
set<string> tokenset(tokens.begin(), tokens.end());
|
|
|
|
if (tokenset != expected_status)
|
|
{
|
|
auto expected_joined = mxb::join(expected_status);
|
|
test.expect(false, "Wrong states for '%s'. Got '%s', expected '%s'.",
|
|
server.c_str(), maxadmin_result.c_str(), expected_joined.c_str());
|
|
}
|
|
}
|
|
}
|
|
|
|
json_t* get_json_data(TestConnections& test, const char* query)
|
|
{
|
|
json_t* rval = NULL;
|
|
int exit_code = 1;
|
|
char* output = test.maxscales->ssh_node_output(0, query, true, &exit_code);
|
|
if (output == NULL)
|
|
{
|
|
test.add_result(1, "Query '%s' execution error, no output.", query);
|
|
}
|
|
else
|
|
{
|
|
json_error_t error;
|
|
rval = json_loads(output, 0, &error);
|
|
free(output);
|
|
if (rval == NULL)
|
|
{
|
|
test.add_result(1, "JSON decode error: %s\n", error.text);
|
|
}
|
|
}
|
|
return rval;
|
|
}
|
|
|
|
json_t* traverse_json(TestConnections& test, json_t* object, const std::vector<string>& keys)
|
|
{
|
|
test.expect(object, "JSON object is NULL\n");
|
|
|
|
json_t* current_object = object;
|
|
for (auto iter = keys.begin(); iter != keys.end() && current_object; ++iter)
|
|
{
|
|
current_object = json_object_get(current_object, (*iter).c_str());
|
|
test.expect(current_object, "Key %s was not found in json data.\n", (*iter).c_str());
|
|
}
|
|
return current_object;
|
|
}
|
|
|
|
json_t* find_array_elem_json(TestConnections& test, json_t* object,
|
|
std::string key, std::string expected_val)
|
|
{
|
|
bool is_array = json_is_array(object);
|
|
test.expect(is_array, "JSON object is not an array\n");
|
|
json_t* found_elem = NULL;
|
|
|
|
if (is_array)
|
|
{
|
|
size_t arr_size = json_array_size(object);
|
|
for (size_t i = 0; i < arr_size; i++)
|
|
{
|
|
json_t* arr_elem = json_array_get(object, i);
|
|
json_t* elem_val = json_object_get(arr_elem, key.c_str());
|
|
bool is_string = json_is_string(elem_val);
|
|
test.expect(is_string,
|
|
"Key %s was not found in json data or the data is not string.\n",
|
|
key.c_str());
|
|
if (is_string)
|
|
{
|
|
std::string elem_field = json_string_value(elem_val);
|
|
if (elem_field == expected_val)
|
|
{
|
|
found_elem = arr_elem;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
test.expect(found_elem, "Array element with %s->%s was not found in json array\n",
|
|
key.c_str(), expected_val.c_str());
|
|
}
|
|
return found_elem;
|
|
}
|
|
|
|
void check_group(TestConnections& test, const char* server, int expected_group)
|
|
{
|
|
json_t* monitor_data = get_json_data(test, "maxctrl api get monitors/MySQL-Monitor");
|
|
if (monitor_data == NULL)
|
|
{
|
|
return;
|
|
}
|
|
|
|
monitor_data = traverse_json(test, monitor_data,
|
|
{"data", "attributes", "monitor_diagnostics", "server_info"});
|
|
if (monitor_data)
|
|
{
|
|
auto server_data = find_array_elem_json(test, monitor_data, "name", server);
|
|
if (server_data)
|
|
{
|
|
int found_group = json_integer_value(json_object_get(server_data, "master_group"));
|
|
test.expect(found_group == expected_group, "Server '%s', expected group '%d', not '%d'",
|
|
server, expected_group, found_group);
|
|
}
|
|
}
|
|
}
|
|
|
|
void check_rlag(TestConnections& test, const char* server, int min_rlag, int max_rlag)
|
|
{
|
|
json_t* object_data = get_json_data(test, "maxctrl api get servers");
|
|
if (object_data)
|
|
{
|
|
auto servers_data = traverse_json(test, object_data, {"data"});
|
|
auto server_data = find_array_elem_json(test, servers_data, "id", server);
|
|
auto rlag_object = traverse_json(test, server_data, {"attributes", "replication_lag"});
|
|
if (rlag_object)
|
|
{
|
|
int found_rlag = json_integer_value(rlag_object);
|
|
if (found_rlag >= min_rlag && found_rlag <= max_rlag)
|
|
{
|
|
cout << "Replication lag of " << server << " is " << found_rlag << " seconds.\n";
|
|
}
|
|
else
|
|
{
|
|
test.expect(false,
|
|
"Replication lag of %s is out of bounds: value: %i min: %i max: %i\n",
|
|
server, found_rlag, min_rlag, max_rlag);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void change_master(TestConnections& test, int slave, int master, const string& conn_name = "",
|
|
int replication_delay = 0)
|
|
{
|
|
const char query[] = "CHANGE MASTER '%s' TO master_host='%s', master_port=%d, "
|
|
"master_log_file='mar-bin.000001', master_log_pos=4, "
|
|
"master_user='repl', master_password='repl', master_delay=%d; "
|
|
"START SLAVE '%s';";
|
|
test.try_query(test.repl->nodes[slave], query, conn_name.c_str(),
|
|
test.repl->IP_private[master], test.repl->port[master],
|
|
replication_delay, conn_name.c_str());
|
|
}
|
|
|
|
int main(int argc, char* argv[])
|
|
{
|
|
string master = "Master";
|
|
string running = "Running";
|
|
string slave = "Slave";
|
|
|
|
set<string> mm_master_states = {master, running};
|
|
set<string> mm_slave_states = {"Relay Master", slave, running};
|
|
set<string> slave_states = {slave, running};
|
|
set<string> running_state = {running};
|
|
|
|
const char reset_query[] = "STOP SLAVE; RESET SLAVE ALL; RESET MASTER; SET GLOBAL read_only='OFF'";
|
|
const char readonly_on_query[] = "SET GLOBAL read_only='ON'";
|
|
|
|
TestConnections::require_repl_version("10.2.3"); // Delayed replication needs this.
|
|
TestConnections test(argc, argv);
|
|
|
|
test.tprintf("Test 1 - Configure all servers into a multi-master ring with one slave");
|
|
int max_rlag = 100;
|
|
test.set_timeout(120);
|
|
test.repl->execute_query_all_nodes(reset_query);
|
|
test.repl->connect();
|
|
change_master(test, 0, 1);
|
|
change_master(test, 1, 2);
|
|
change_master(test, 2, 0);
|
|
change_master(test, 3, 2, "", max_rlag);
|
|
|
|
test.maxscales->wait_for_monitor(2);
|
|
auto maxconn = test.maxscales->open_rwsplit_connection();
|
|
test.try_query(maxconn, "FLUSH TABLES;");
|
|
test.maxscales->wait_for_monitor(1);
|
|
|
|
check_status(test, "server1", mm_master_states);
|
|
check_status(test, "server2", mm_slave_states);
|
|
check_status(test, "server3", mm_slave_states);
|
|
check_status(test, "server4", slave_states);
|
|
check_group(test, "server1", 1);
|
|
check_group(test, "server2", 1);
|
|
check_group(test, "server3", 1);
|
|
check_group(test, "server4", 0);
|
|
check_rlag(test, "server4", 1, max_rlag);
|
|
// Need to send a read query so that rwsplit detects replication lag.
|
|
test.try_query(maxconn, "SHOW DATABASES;");
|
|
mysql_close(maxconn);
|
|
test.log_includes(0, "is excluded from query routing.");
|
|
|
|
test.tprintf("Test 2 - Set nodes 0 and 1 into read-only mode");
|
|
|
|
test.set_timeout(120);
|
|
execute_query(test.repl->nodes[0], readonly_on_query);
|
|
execute_query(test.repl->nodes[1], readonly_on_query);
|
|
test.maxscales->wait_for_monitor(1);
|
|
|
|
check_status(test, "server1", mm_slave_states);
|
|
check_status(test, "server2", mm_slave_states);
|
|
check_status(test, "server3", mm_master_states);
|
|
check_status(test, "server4", slave_states);
|
|
check_group(test, "server1", 1);
|
|
check_group(test, "server2", 1);
|
|
check_group(test, "server3", 1);
|
|
check_group(test, "server4", 0);
|
|
check_rlag(test, "server4", 1, max_rlag);
|
|
|
|
test.tprintf("Test 3 - Configure nodes 1 and 2 into a master-master pair, make node 0 "
|
|
"a slave of node 1 and node 3 a slave of node 2");
|
|
|
|
test.maxscales->stop_maxscale();
|
|
test.set_timeout(120);
|
|
test.repl->execute_query_all_nodes(reset_query);
|
|
test.repl->connect();
|
|
|
|
change_master(test, 0, 1);
|
|
change_master(test, 1, 2);
|
|
change_master(test, 2, 1, "", max_rlag);
|
|
change_master(test, 3, 2);
|
|
|
|
test.maxscales->start_maxscale();
|
|
sleep(2);
|
|
test.maxscales->wait_for_monitor(1);
|
|
|
|
maxconn = test.maxscales->open_rwsplit_connection();
|
|
test.try_query(maxconn, "FLUSH TABLES;");
|
|
mysql_close(maxconn);
|
|
test.maxscales->wait_for_monitor(1);
|
|
|
|
check_status(test, "server1", slave_states);
|
|
check_status(test, "server2", mm_master_states);
|
|
check_status(test, "server3", mm_slave_states);
|
|
check_status(test, "server4", slave_states);
|
|
check_group(test, "server1", 0);
|
|
check_group(test, "server2", 1);
|
|
check_group(test, "server3", 1);
|
|
check_group(test, "server4", 0);
|
|
check_rlag(test, "server3", 1, max_rlag);
|
|
|
|
test.tprintf("Test 4 - Set node 1 into read-only mode");
|
|
|
|
test.set_timeout(120);
|
|
execute_query(test.repl->nodes[1], readonly_on_query);
|
|
test.maxscales->wait_for_monitor(1);
|
|
|
|
check_status(test, "server1", slave_states);
|
|
check_status(test, "server2", mm_slave_states);
|
|
check_status(test, "server3", mm_master_states);
|
|
check_status(test, "server4", slave_states);
|
|
check_group(test, "server1", 0);
|
|
check_group(test, "server2", 1);
|
|
check_group(test, "server3", 1);
|
|
check_group(test, "server4", 0);
|
|
|
|
test.tprintf("Test 5 - Create two distinct groups");
|
|
|
|
test.maxscales->stop_maxscale();
|
|
test.set_timeout(120);
|
|
test.repl->execute_query_all_nodes(reset_query);
|
|
test.repl->connect();
|
|
|
|
change_master(test, 0, 1);
|
|
change_master(test, 1, 0);
|
|
change_master(test, 2, 3);
|
|
change_master(test, 3, 2);
|
|
|
|
test.maxscales->start_maxscale();
|
|
sleep(2);
|
|
test.maxscales->wait_for_monitor(1);
|
|
|
|
// Even though the servers are in two distinct groups, only one of them
|
|
// contains a master and a slave. Only one master may exist in a cluster
|
|
// at once, since by definition this is the server to which routers may
|
|
// direct writes.
|
|
check_status(test, "server1", mm_master_states);
|
|
check_status(test, "server2", mm_slave_states);
|
|
check_status(test, "server3", running_state);
|
|
check_status(test, "server4", running_state);
|
|
check_group(test, "server1", 1);
|
|
check_group(test, "server2", 1);
|
|
check_group(test, "server3", 2);
|
|
check_group(test, "server4", 2);
|
|
|
|
test.tprintf("Test 6 - Set nodes 1 and 3 into read-only mode");
|
|
|
|
test.set_timeout(120);
|
|
execute_query(test.repl->nodes[1], readonly_on_query);
|
|
execute_query(test.repl->nodes[3], readonly_on_query);
|
|
|
|
test.maxscales->wait_for_monitor(1);
|
|
|
|
check_status(test, "server1", mm_master_states);
|
|
check_status(test, "server2", mm_slave_states);
|
|
check_status(test, "server3", running_state);
|
|
check_status(test, "server4", running_state);
|
|
check_group(test, "server1", 1);
|
|
check_group(test, "server2", 1);
|
|
check_group(test, "server3", 2);
|
|
check_group(test, "server4", 2);
|
|
|
|
test.tprintf("Test 7 - Diamond topology with delay");
|
|
|
|
test.repl->execute_query_all_nodes(reset_query);
|
|
test.repl->connect();
|
|
change_master(test, 0, 1, "a", max_rlag);
|
|
change_master(test, 0, 2, "b", max_rlag);
|
|
change_master(test, 1, 3);
|
|
change_master(test, 2, 3);
|
|
|
|
test.maxscales->wait_for_monitor(1);
|
|
maxconn = test.maxscales->open_rwsplit_connection();
|
|
test.try_query(maxconn, "FLUSH TABLES;");
|
|
test.maxscales->wait_for_monitor(2);
|
|
test.try_query(maxconn, "SHOW DATABASES;");
|
|
|
|
check_status(test, "server1", slave_states);
|
|
check_status(test, "server2", mm_slave_states);
|
|
check_status(test, "server3", mm_slave_states);
|
|
check_status(test, "server4", mm_master_states);
|
|
check_group(test, "server1", 0);
|
|
check_group(test, "server2", 0);
|
|
check_group(test, "server3", 0);
|
|
check_group(test, "server4", 0);
|
|
check_rlag(test, "server1", 1, max_rlag);
|
|
|
|
test.tprintf("Test 8 - Diamond topology with no delay");
|
|
|
|
const char remove_delay[] = "STOP SLAVE '%s'; CHANGE MASTER '%s' TO master_delay=0; START SLAVE '%s';";
|
|
test.try_query(test.repl->nodes[0], remove_delay, "a", "a", "a");
|
|
test.maxscales->wait_for_monitor(2);
|
|
|
|
check_status(test, "server1", slave_states);
|
|
check_rlag(test, "server1", 0, 0);
|
|
// Rwsplit should detects that replication lag is 0.
|
|
test.try_query(maxconn, "SHOW DATABASES;");
|
|
mysql_close(maxconn);
|
|
test.log_includes(0, "is returned to query routing.");
|
|
|
|
// Test over, reset topology.
|
|
const char reset_with_name[] = "STOP SLAVE '%s'; RESET SLAVE '%s' ALL;";
|
|
test.try_query(test.repl->nodes[0], reset_with_name, "a", "a");
|
|
test.try_query(test.repl->nodes[0], reset_with_name, "b", "b");
|
|
|
|
test.repl->execute_query_all_nodes(reset_query);
|
|
test.repl->connect();
|
|
change_master(test, 1, 0);
|
|
change_master(test, 2, 0);
|
|
change_master(test, 3, 0);
|
|
|
|
return test.global_result;
|
|
}
|