MXS-1514: Split failover test in two files, add manual version
The same test now has two versions. In the automatic version failover begins automatically. In the manual version failover is started with maxadmin. The tests are otherwise identical.
This commit is contained in:
Esa Korhonen
222
maxscale-system-test/mysqlmon_failover_common.cpp
Normal file
222
maxscale-system-test/mysqlmon_failover_common.cpp
Normal file
@ -0,0 +1,222 @@
|
||||
#include "testconnections.h"
|
||||
|
||||
int inserts = 0;
|
||||
bool interactive = false;
|
||||
|
||||
void get_output(TestConnections& test)
|
||||
{
|
||||
int ec;
|
||||
test.tprintf("Maxadmin output:");
|
||||
char *output = test.maxscales->ssh_node_output(0, "maxadmin list servers", true, &ec);
|
||||
test.tprintf("%s", output);
|
||||
free(output);
|
||||
|
||||
test.tprintf("MaxScale output:");
|
||||
output = test.maxscales->ssh_node_output(0, "cat /var/log/maxscale/maxscale.log && "
|
||||
"sudo truncate -s 0 /var/log/maxscale/maxscale.log",
|
||||
true, &ec);
|
||||
test.tprintf("%s", output);
|
||||
free(output);
|
||||
}
|
||||
|
||||
void check(TestConnections& test)
|
||||
{
|
||||
MYSQL *conn = test.maxscales->open_rwsplit_connection(0);
|
||||
const char *query1 = "INSERT INTO test.t1 VALUES (%d)";
|
||||
const char *query2 = "SELECT * FROM test.t1";
|
||||
|
||||
test.try_query(conn, "BEGIN");
|
||||
test.tprintf(query1, inserts);
|
||||
test.try_query(conn, query1, inserts++);
|
||||
mysql_query(conn, query2);
|
||||
|
||||
MYSQL_RES *res = mysql_store_result(conn);
|
||||
test.add_result(res == NULL, "Query should return a result set");
|
||||
|
||||
if (res)
|
||||
{
|
||||
std::string values;
|
||||
MYSQL_ROW row;
|
||||
int num_rows = mysql_num_rows(res);
|
||||
test.add_result(num_rows != inserts, "Query returned %d rows when %d rows were expected",
|
||||
num_rows, inserts);
|
||||
const char *separator = "";
|
||||
|
||||
while ((row = mysql_fetch_row(res)))
|
||||
{
|
||||
values += separator;
|
||||
values += row[0];
|
||||
separator = ", ";
|
||||
}
|
||||
test.tprintf("%s: %s", query2, values.c_str());
|
||||
}
|
||||
test.try_query(conn, "COMMIT");
|
||||
mysql_close(conn);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get master server id (master decided by MaxScale)
|
||||
*
|
||||
* @param test Tester object
|
||||
* @return Master server id
|
||||
*/
|
||||
int get_server_id(TestConnections& test)
|
||||
{
|
||||
MYSQL *conn = test.maxscales->open_rwsplit_connection(0);
|
||||
int id = -1;
|
||||
char str[1024];
|
||||
|
||||
if (find_field(conn, "SELECT @@server_id, @@last_insert_id;", "@@server_id", str) == 0)
|
||||
{
|
||||
id = atoi(str);
|
||||
}
|
||||
|
||||
mysql_close(conn);
|
||||
return id;
|
||||
}
|
||||
|
||||
void get_input()
|
||||
{
|
||||
if (interactive)
|
||||
{
|
||||
printf("--- Press any key to confinue ---\n");
|
||||
getchar();
|
||||
}
|
||||
}
|
||||
|
||||
void fix_replication_create_table(TestConnections& test)
|
||||
{
|
||||
test.tprintf("Fix replication and recreate table.");
|
||||
test.maxscales->close_maxscale_connections(0);
|
||||
test.repl->fix_replication();
|
||||
test.maxscales->connect_maxscale(0);
|
||||
test.try_query(test.maxscales->conn_rwsplit[0], "CREATE OR REPLACE TABLE test.t1(id INT)");
|
||||
test.repl->sync_slaves();
|
||||
inserts = 0;
|
||||
|
||||
check(test);
|
||||
get_output(test);
|
||||
}
|
||||
|
||||
void delete_slave_binlogs(TestConnections& test)
|
||||
{
|
||||
const char RESET[] = "RESET MASTER;";
|
||||
execute_query(test.repl->nodes[1], RESET);
|
||||
execute_query(test.repl->nodes[2], RESET);
|
||||
execute_query(test.repl->nodes[3], RESET);
|
||||
}
|
||||
|
||||
const char LINE[] = "------------------------------------------";
|
||||
const char PRINT_ID[] = "Master server id is %d.";
|
||||
const char WRONG_SLAVE[] = "Wrong slave was promoted or promotion failed.";
|
||||
|
||||
void basic_test(TestConnections& test)
|
||||
{
|
||||
test.tprintf("Creating table and inserting data.");
|
||||
get_input();
|
||||
test.maxscales->connect_maxscale(0);
|
||||
test.try_query(test.maxscales->conn_rwsplit[0], "CREATE OR REPLACE TABLE test.t1(id INT)");
|
||||
test.repl->sync_slaves();
|
||||
|
||||
check(test);
|
||||
get_output(test);
|
||||
}
|
||||
int prepare_test_1(TestConnections& test)
|
||||
{
|
||||
delete_slave_binlogs(test);
|
||||
test.tprintf("Test 1: Stopping master and waiting for failover. Check that another server is promoted.\n"
|
||||
"%s", LINE);
|
||||
get_input();
|
||||
int node0_id = test.repl->get_server_id(0); // Read master id now before shutdown.
|
||||
test.repl->stop_node(0);
|
||||
return node0_id;
|
||||
}
|
||||
|
||||
void check_test_1(TestConnections& test, int node0_id)
|
||||
{
|
||||
check(test);
|
||||
get_output(test);
|
||||
int master_id = get_server_id(test);
|
||||
test.tprintf(PRINT_ID, master_id);
|
||||
test.add_result(master_id < 1 && master_id == node0_id, "Master did not change or no master detected.");
|
||||
fix_replication_create_table(test);
|
||||
test.repl->connect();
|
||||
}
|
||||
|
||||
void prepare_test_2(TestConnections& test)
|
||||
{
|
||||
delete_slave_binlogs(test);
|
||||
test.tprintf("Test 2: Disable replication on server 2 and kill master, check that server 3 or 4 is "
|
||||
"promoted.\n%s", LINE);
|
||||
get_input();
|
||||
execute_query(test.repl->nodes[1], "STOP SLAVE; RESET SLAVE ALL;");
|
||||
sleep(2);
|
||||
test.repl->stop_node(0);
|
||||
}
|
||||
|
||||
void check_test_2(TestConnections& test)
|
||||
{
|
||||
check(test);
|
||||
get_output(test);
|
||||
|
||||
int master_id = get_server_id(test);
|
||||
test.tprintf(PRINT_ID, master_id);
|
||||
test.add_result(master_id < 1 ||
|
||||
(master_id != test.repl->get_server_id(2) && master_id != test.repl->get_server_id(3)),
|
||||
WRONG_SLAVE);
|
||||
fix_replication_create_table(test);
|
||||
test.repl->connect();
|
||||
}
|
||||
void prepare_test_3(TestConnections& test)
|
||||
{
|
||||
delete_slave_binlogs(test);
|
||||
test.tprintf("Test3: Shutdown two slaves (servers 2 and 4). Disable log_bin on server 2, making it "
|
||||
"invalid for promotion. Enable log-slave-updates on servers 2 and 4. Check that server 4 is "
|
||||
"promoted on master failure.\n%s", LINE);
|
||||
get_input();
|
||||
|
||||
test.repl->stop_node(1);
|
||||
test.repl->stop_node(3);
|
||||
test.repl->stash_server_settings(1);
|
||||
test.repl->stash_server_settings(3);
|
||||
test.repl->disable_server_setting(1, "log-bin");
|
||||
const char* log_slave = "log_slave_updates=1";
|
||||
test.repl->add_server_setting(1, log_slave);
|
||||
test.repl->add_server_setting(3, log_slave);
|
||||
test.repl->start_node(1, (char *) "");
|
||||
test.repl->start_node(3, (char *) "");
|
||||
sleep(4);
|
||||
test.tprintf("Settings changed.");
|
||||
get_output(test);
|
||||
test.tprintf("Stopping master.");
|
||||
test.repl->stop_node(0);
|
||||
}
|
||||
|
||||
void check_test_3(TestConnections& test)
|
||||
{
|
||||
check(test);
|
||||
get_output(test);
|
||||
|
||||
int master_id = get_server_id(test);
|
||||
// Because servers have been restarted, redo connections.
|
||||
test.repl->connect();
|
||||
sleep(2);
|
||||
test.tprintf(PRINT_ID, master_id);
|
||||
test.add_result(master_id < 1 || master_id != test.repl->get_server_id(3), WRONG_SLAVE);
|
||||
// Restore server 2 and 4 settings. Because server 4 is now the master, shutting it down causes
|
||||
// another failover. Prevent this by stopping maxscale.
|
||||
test.tprintf("Restoring server settings.");
|
||||
test.maxscales->stop_maxscale(0);
|
||||
test.repl->stop_node(1);
|
||||
test.repl->stop_node(3);
|
||||
sleep(4);
|
||||
test.repl->restore_server_settings(1);
|
||||
test.repl->restore_server_settings(3);
|
||||
test.repl->start_node(0, (char *) "");
|
||||
test.repl->start_node(1, (char *) "");
|
||||
test.repl->start_node(3, (char *) "");
|
||||
sleep(4);
|
||||
test.maxscales->start_maxscale(0);
|
||||
sleep(2);
|
||||
get_output(test);
|
||||
}
|
||||
Reference in New Issue
Block a user