Fix keepalived_masterdown
Backends use gtid-replication and the test is faster.
This commit is contained in:
Esa Korhonen
@ -6,15 +6,144 @@
|
||||
#include <iostream>
|
||||
#include "testconnections.h"
|
||||
#include "keepalived_func.h"
|
||||
#include "failover_common.cpp"
|
||||
|
||||
bool check_maxscale_passive(TestConnections* Test, int node)
|
||||
static bool check_maxscale_passive(TestConnections& test, int node);
|
||||
static void expect_maxscale_active_passive(TestConnections& test, int active_node);
|
||||
const char ms_is_passive[] = "Maxscale %i is passive when active was expected.";
|
||||
const char ms_is_active[] = "Maxscale %i is active when passive was expected.";
|
||||
|
||||
int main(int argc, char* argv[])
|
||||
{
|
||||
bool passive;
|
||||
char* passive_str;
|
||||
int ec;
|
||||
Test->tprintf("Checking status of Maxscale %03d", node);
|
||||
passive_str = Test->maxscales->ssh_node_output(node, "maxctrl show maxscale | grep passive", false, &ec);
|
||||
Test->tprintf("maxctrl output string: %s\n", passive_str);
|
||||
const int failover_mon_ticks = 2;
|
||||
// Keepalived takes more time to switch primary MaxScale than a normal failover.
|
||||
// For now, assume that the time scales with monitor ticks.
|
||||
const int keepalived_switch_mon_ticks = 6;
|
||||
|
||||
TestConnections::multiple_maxscales(true);
|
||||
Mariadb_nodes::require_gtid(true);
|
||||
TestConnections test(argc, argv);
|
||||
test.repl->connect();
|
||||
delete_slave_binlogs(test);
|
||||
basic_test(test);
|
||||
print_gtids(test);
|
||||
|
||||
test.tprintf("Number of MaxScales: %d\n", test.maxscales->N);
|
||||
test.expect(test.maxscales->N == 2,
|
||||
"Two Maxscales are needed for this test, %i nodes was/were detected.", test.maxscales->N);
|
||||
if (test.global_result != 0)
|
||||
{
|
||||
return test.global_result;
|
||||
}
|
||||
|
||||
test.tprintf("Configuring 'keepalived'\n");
|
||||
// Get test client IP, replace last number in it with 253 and use it as Virtual IP
|
||||
configure_keepalived(&test, (char*) "masterdown");
|
||||
|
||||
print_version_string(&test);
|
||||
test.maxscales->wait_for_monitor(1, 0);
|
||||
test.maxscales->wait_for_monitor(1, 1);
|
||||
|
||||
// initial state: 000 expected to be active, 001 - passive
|
||||
int active_node = 0;
|
||||
expect_maxscale_active_passive(test, active_node);
|
||||
if (test.global_result != 0)
|
||||
{
|
||||
return test.global_result;
|
||||
}
|
||||
|
||||
// Test a normal failover.
|
||||
int first_master = test.repl->find_master();
|
||||
test.tprintf("Stop Master - node %d\n", first_master);
|
||||
test.repl->stop_node(first_master);
|
||||
test.maxscales->wait_for_monitor(failover_mon_ticks, active_node);
|
||||
|
||||
int second_master = test.repl->find_master();
|
||||
test.tprintf("new master is node %d\n", second_master);
|
||||
test.expect(first_master != second_master, "Master did not change, failover did not happen.");
|
||||
|
||||
char str[1024];
|
||||
sprintf(str, "Performing automatic failover to replace failed master 'server%d'", first_master + 1);
|
||||
test.tprintf("Checking Maxscale log on 000 for the failover message %s\n", str);
|
||||
test.check_log_err(0, str, true);
|
||||
sprintf(str, "Performing automatic failover to replace failed master");
|
||||
test.tprintf("Checking Maxscale log on 001 for the lack of failover message\n");
|
||||
test.check_log_err(1, str, false);
|
||||
if (test.global_result != 0)
|
||||
{
|
||||
return test.global_result;
|
||||
}
|
||||
|
||||
// Stop MaxScale 0. Keepalived should promote MaxScale 1 to primary.
|
||||
test.tprintf("Stop Maxscale 000\n");
|
||||
test.maxscales->stop_maxscale(0);
|
||||
active_node = 1;
|
||||
test.maxscales->wait_for_monitor(keepalived_switch_mon_ticks, active_node);
|
||||
test.expect(!check_maxscale_passive(test, active_node), ms_is_passive, active_node);
|
||||
if (test.global_result != 0)
|
||||
{
|
||||
return test.global_result;
|
||||
}
|
||||
|
||||
// Check that a failover still happens.
|
||||
test.tprintf("Stop new Master - node %d\n", second_master);
|
||||
test.repl->stop_node(second_master);
|
||||
test.maxscales->wait_for_monitor(failover_mon_ticks, active_node);
|
||||
|
||||
int third_master = test.repl->find_master();
|
||||
test.tprintf("new master (third one) is node %d\n", third_master);
|
||||
test.expect(third_master != second_master, "Master did not change, failover did not happen.");
|
||||
|
||||
sprintf(str, "Performing automatic failover to replace failed master 'server%d'", second_master + 1);
|
||||
test.tprintf("Checking Maxscale log on 001 for the failover message %s\n", str);
|
||||
test.check_log_err(1, str, true);
|
||||
|
||||
test.check_log_err(1, (char*) "Multiple failed master servers detected", false);
|
||||
test.check_log_err(1, (char*) "Failed to perform failover", false);
|
||||
test.check_log_err(1, (char*) "disabling automatic failover", false);
|
||||
if (test.global_result != 0)
|
||||
{
|
||||
return test.global_result;
|
||||
}
|
||||
|
||||
// Bring MaxScale 0 back up, check that it regains primary status.
|
||||
test.tprintf("Start Maxscale 000\n");
|
||||
test.maxscales->start_maxscale(0);
|
||||
active_node = 0;
|
||||
test.maxscales->wait_for_monitor(keepalived_switch_mon_ticks, active_node);
|
||||
expect_maxscale_active_passive(test, active_node);
|
||||
if (test.global_result != 0)
|
||||
{
|
||||
return test.global_result;
|
||||
}
|
||||
|
||||
sprintf(str, "Performing automatic failover to replace failed master 'server%d'", second_master + 1);
|
||||
test.tprintf("Checking Maxscale log on 001 for the failover message %s\n", str);
|
||||
test.check_log_err(1, str, true);
|
||||
test.tprintf("Checking Maxscale log on 000 for the lack of failover message %s\n", str);
|
||||
test.check_log_err(0, str, false);
|
||||
|
||||
test.check_log_err(1, (char*) "Multiple failed master servers detected", false);
|
||||
test.check_log_err(1, (char*) "Failed to perform failover", false);
|
||||
test.check_log_err(1, (char*) "disabling automatic failover", false);
|
||||
|
||||
test.check_log_err(0, (char*) "Multiple failed master servers detected", false);
|
||||
test.check_log_err(0, (char*) "Failed to perform failover", false);
|
||||
test.check_log_err(0, (char*) "disabling automatic failover", false);
|
||||
|
||||
stop_keepalived(&test);
|
||||
return test.global_result;
|
||||
}
|
||||
|
||||
bool check_maxscale_passive(TestConnections& test, int node)
|
||||
{
|
||||
bool passive = false;
|
||||
char* passive_str = NULL;
|
||||
int ec = -1;
|
||||
|
||||
test.tprintf("Checking status of Maxscale %03d", node);
|
||||
passive_str = test.maxscales->ssh_node_output(node, "maxctrl show maxscale | grep passive", false, &ec);
|
||||
test.tprintf("maxctrl output string: %s\n", passive_str);
|
||||
if (strstr(passive_str, "false") != NULL)
|
||||
{
|
||||
passive = false;
|
||||
@ -23,167 +152,20 @@ bool check_maxscale_passive(TestConnections* Test, int node)
|
||||
{
|
||||
if (strstr(passive_str, "true") == NULL)
|
||||
{
|
||||
Test->tprintf("Can't find 'true' or 'false' in the 'maxctrl' output\n");
|
||||
test.tprintf("Can't find 'true' or 'false' in the 'maxctrl' output\n");
|
||||
}
|
||||
passive = true;
|
||||
}
|
||||
free(passive_str);
|
||||
|
||||
Test->tprintf("Content of 'state.txt' file: %s\n",
|
||||
Test->maxscales->ssh_node_output(0, "cat /tmp/state.txt", false, &ec));
|
||||
test.tprintf("Content of 'state.txt' file: %s\n",
|
||||
test.maxscales->ssh_node_output(0, "cat /tmp/state.txt", false, &ec));
|
||||
return passive;
|
||||
}
|
||||
|
||||
int main(int argc, char* argv[])
|
||||
void expect_maxscale_active_passive(TestConnections& test, int active_node)
|
||||
{
|
||||
bool passive;
|
||||
char str[1024];
|
||||
|
||||
TestConnections::multiple_maxscales(true);
|
||||
TestConnections* Test = new TestConnections(argc, argv);
|
||||
// Test->set_timeout(10);
|
||||
|
||||
|
||||
|
||||
|
||||
Test->tprintf("Maxscale_N %d\n", Test->maxscales->N);
|
||||
if (Test->maxscales->N < 2)
|
||||
{
|
||||
Test->tprintf("At least 2 Maxscales are needed for this test. Exiting\n");
|
||||
exit(0);
|
||||
}
|
||||
|
||||
Test->tprintf("Starting replication with GTID\n");
|
||||
Test->repl->require_gtid(true);
|
||||
Test->repl->start_replication();
|
||||
|
||||
Test->tprintf("Configuring 'keepalived'\n");
|
||||
// Get test client IP, replace last number in it with 253 and use it as Virtual IP
|
||||
configure_keepalived(Test, (char*) "masterdown");
|
||||
|
||||
// Test->maxscales->ssh_node(1, (char *) "maxctrl alter maxscale passive true", false);
|
||||
|
||||
print_version_string(Test);
|
||||
|
||||
sleep(FAILOVER_WAIT_TIME);
|
||||
sleep(FAILOVER_WAIT_TIME);
|
||||
|
||||
// initial state: 000 expected to be active, 001 - passive
|
||||
passive = check_maxscale_passive(Test, 0);
|
||||
if (passive)
|
||||
{
|
||||
Test->add_result(1, "Maxscale 000 is in the passive mode\n");
|
||||
}
|
||||
passive = check_maxscale_passive(Test, 1);
|
||||
if (!passive)
|
||||
{
|
||||
Test->add_result(1, "Maxscale 001 is NOT in the passive mode\n");
|
||||
}
|
||||
|
||||
int first_master = Test->repl->find_master();
|
||||
|
||||
Test->tprintf("Stop Master - node %d\n", first_master);
|
||||
|
||||
Test->repl->stop_node(first_master);
|
||||
|
||||
sleep(FAILOVER_WAIT_TIME);
|
||||
|
||||
int second_master = Test->repl->find_master();
|
||||
|
||||
Test->tprintf("new master is node %d\n", second_master);
|
||||
|
||||
if (first_master == second_master)
|
||||
{
|
||||
Test->add_result(1, "Failover did not happen\n");
|
||||
}
|
||||
|
||||
sprintf(str, "Performing automatic failover to replace failed master 'server%d'", first_master + 1);
|
||||
Test->tprintf("Checking Maxscale log on 000 for the failover message %s\n", str);
|
||||
Test->check_log_err(0, str, true);
|
||||
sprintf(str, "Performing automatic failover to replace failed master");
|
||||
Test->tprintf("Checking Maxscale log on 001 for the lack of failover message\n");
|
||||
Test->check_log_err(1, str, false);
|
||||
|
||||
passive = check_maxscale_passive(Test, 0);
|
||||
if (passive)
|
||||
{
|
||||
Test->add_result(1, "Maxscale 000 is in the passive mode\n");
|
||||
}
|
||||
passive = check_maxscale_passive(Test, 1);
|
||||
if (!passive)
|
||||
{
|
||||
Test->add_result(1, "Maxscale 001 is NOT in the passive mode\n");
|
||||
}
|
||||
|
||||
Test->tprintf("Stop Maxscale 000\n");
|
||||
|
||||
Test->maxscales->stop_maxscale(0);
|
||||
|
||||
sleep(FAILOVER_WAIT_TIME);
|
||||
|
||||
passive = check_maxscale_passive(Test, 1);
|
||||
if (passive)
|
||||
{
|
||||
Test->add_result(1, "Maxscale 001 is in the passive mode\n");
|
||||
}
|
||||
|
||||
Test->tprintf("Stop new Master - node %d\n", second_master);
|
||||
Test->repl->stop_node(second_master);
|
||||
sleep(FAILOVER_WAIT_TIME);
|
||||
|
||||
int third_master = Test->repl->find_master();
|
||||
|
||||
Test->tprintf("new master (third one) is node %d\n", third_master);
|
||||
|
||||
if (second_master == third_master)
|
||||
{
|
||||
Test->add_result(1, "Failover did not happen\n");
|
||||
}
|
||||
sprintf(str, "Performing automatic failover to replace failed master 'server%d'", second_master + 1);
|
||||
Test->tprintf("Checking Maxscale log on 001 for the failover message %s\n", str);
|
||||
Test->check_log_err(1, str, true);
|
||||
|
||||
Test->check_log_err(1, (char*) "Multiple failed master servers detected", false);
|
||||
Test->check_log_err(1, (char*) "Failed to perform failover", false);
|
||||
Test->check_log_err(1, (char*) "disabling automatic failover", false);
|
||||
|
||||
Test->tprintf("Start Maxscale 000\n");
|
||||
|
||||
Test->maxscales->start_maxscale(0);
|
||||
|
||||
sleep(FAILOVER_WAIT_TIME);
|
||||
|
||||
passive = check_maxscale_passive(Test, 0);
|
||||
if (passive)
|
||||
{
|
||||
Test->add_result(1, "Maxscale 000 is in the passive mode\n");
|
||||
}
|
||||
passive = check_maxscale_passive(Test, 1);
|
||||
if (!passive)
|
||||
{
|
||||
Test->add_result(1, "Maxscale 001 is NOT in the passive mode\n");
|
||||
}
|
||||
|
||||
sprintf(str, "Performing automatic failover to replace failed master 'server%d'", second_master + 1);
|
||||
Test->tprintf("Checking Maxscale log on 001 for the failover message %s\n", str);
|
||||
Test->check_log_err(1, str, true);
|
||||
Test->tprintf("Checking Maxscale log on 000 for the lack of failover message %s\n", str);
|
||||
Test->check_log_err(0, str, false);
|
||||
|
||||
Test->check_log_err(1, (char*) "Multiple failed master servers detected", false);
|
||||
Test->check_log_err(1, (char*) "Failed to perform failover", false);
|
||||
Test->check_log_err(1, (char*) "disabling automatic failover", false);
|
||||
|
||||
Test->check_log_err(0, (char*) "Multiple failed master servers detected", false);
|
||||
Test->check_log_err(0, (char*) "Failed to perform failover", false);
|
||||
Test->check_log_err(0, (char*) "disabling automatic failover", false);
|
||||
|
||||
|
||||
// Test->repl->require_gtid(false);
|
||||
// Test->repl->start_replication();
|
||||
|
||||
stop_keepalived(Test);
|
||||
int rval = Test->global_result;
|
||||
delete Test;
|
||||
return rval;
|
||||
int passive_node = active_node == 0 ? 1 : 0;
|
||||
test.expect(!check_maxscale_passive(test, active_node), ms_is_passive, active_node);
|
||||
test.expect(check_maxscale_passive(test, passive_node), ms_is_active, passive_node);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user