hcq/MaxScale
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Mxs 1140 keepalived (#166)

Browse Source 
add keepalived test
This commit is contained in:
Timofey Turenko
2018-02-12 15:49:07 +02:00
committed by GitHub
parent ba821d0aff
commit f7857518f3
10 changed files with 436 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
maxscale-system-test/CMakeLists.txt
View File

@ -822,6 +822,9 @@ add_test_executable(setup_binlog_gtid.cpp setup_binlog_gtid setup_binlog_gtid LA
# TODO: make it working with zypper and apt, move part of KDC setup to MDBCI
add_test_executable(kerberos_setup.cpp kerberos_setup kerberos LABELS HEAVY gssapi REPL_BACKEND)
# Configures 'keepalived' on two Maxscale machines and tried failover
add_test_executable(keepalived.cpp keepalived keepalived LABELS REPL_BACKEND TWO_MAXSCALES)
# enable after fixing MXS-419
# add_test_executable(mxs419_lots_of_connections.cpp mxs419_lots_of_connections replication LABELS REPL_BACKEND)

93
maxscale-system-test/cnf/maxscale.cnf.template.keepalived.000 Executable file
View File

@ -0,0 +1,93 @@
[maxscale]
threads=###threads###
[MySQL Monitor]
type=monitor
module=mysqlmon
###repl51###
servers=server1,server2,server3,server4
user=maxskysql
passwd=skysql
monitor_interval=1000
detect_stale_master=false
detect_standalone_master=false
[RW Split Router]
type=service
router=readwritesplit
servers=server1,server2,server3,server4
user=maxskysql
passwd=skysql
router_options=slave_selection_criteria=LEAST_GLOBAL_CONNECTIONS
max_slave_connections=1
version_string=10.2-server1
[Read Connection Router Slave]
type=service
router=readconnroute
router_options=slave
servers=server1,server2,server3,server4
user=maxskysql
passwd=skysql
version_string=10.2-server1
[Read Connection Router Master]
type=service
router=readconnroute
router_options=master
servers=server1,server2,server3,server4
user=maxskysql
passwd=skysql
version_string=10.2-server1
[RW Split Listener]
type=listener
service=RW Split Router
protocol=MySQLClient
port=4006
[Read Connection Listener Slave]
type=listener
service=Read Connection Router Slave
protocol=MySQLClient
port=4009
[Read Connection Listener Master]
type=listener
service=Read Connection Router Master
protocol=MySQLClient
port=4008
[CLI]
type=service
router=cli
[CLI Listener]
type=listener
service=CLI
protocol=maxscaled
socket=default
[server1]
type=server
address=###node_server_IP_1###
port=###node_server_port_1###
protocol=MySQLBackend
[server2]
type=server
address=###node_server_IP_2###
port=###node_server_port_2###
protocol=MySQLBackend
[server3]
type=server
address=###node_server_IP_3###
port=###node_server_port_3###
protocol=MySQLBackend
[server4]
type=server
address=###node_server_IP_4###
port=###node_server_port_4###
protocol=MySQLBackend

93
maxscale-system-test/cnf/maxscale.cnf.template.keepalived.001 Executable file
View File

@ -0,0 +1,93 @@
[maxscale]
threads=###threads###
[MySQL Monitor]
type=monitor
module=mysqlmon
###repl51###
servers=server1,server2,server3,server4
user=maxskysql
passwd=skysql
monitor_interval=1000
detect_stale_master=false
detect_standalone_master=false
[RW Split Router]
type=service
router=readwritesplit
servers=server1,server2,server3,server4
user=maxskysql
passwd=skysql
router_options=slave_selection_criteria=LEAST_GLOBAL_CONNECTIONS
max_slave_connections=1
version_string=10.2-server2
[Read Connection Router Slave]
type=service
router=readconnroute
router_options=slave
servers=server1,server2,server3,server4
user=maxskysql
passwd=skysql
version_string=10.2-server2
[Read Connection Router Master]
type=service
router=readconnroute
router_options=master
servers=server1,server2,server3,server4
user=maxskysql
passwd=skysql
version_string=10.2-server2
[RW Split Listener]
type=listener
service=RW Split Router
protocol=MySQLClient
port=4006
[Read Connection Listener Slave]
type=listener
service=Read Connection Router Slave
protocol=MySQLClient
port=4009
[Read Connection Listener Master]
type=listener
service=Read Connection Router Master
protocol=MySQLClient
port=4008
[CLI]
type=service
router=cli
[CLI Listener]
type=listener
service=CLI
protocol=maxscaled
socket=default
[server1]
type=server
address=###node_server_IP_1###
port=###node_server_port_1###
protocol=MySQLBackend
[server2]
type=server
address=###node_server_IP_2###
port=###node_server_port_2###
protocol=MySQLBackend
[server3]
type=server
address=###node_server_IP_3###
port=###node_server_port_3###
protocol=MySQLBackend
[server4]
type=server
address=###node_server_IP_4###
port=###node_server_port_4###
protocol=MySQLBackend

146
maxscale-system-test/keepalived.cpp Normal file
View File

@ -0,0 +1,146 @@
/**
* @file keepalived.cpp keepalived Test of two Maxscale + keepalived failover
*
* - 'version_string' configured to be different for every Maxscale
* - configure keepalived for two nodes (uses xxx.xxx.xxx.253 as a virtual IP
* where xxx.xxx.xxx. - first 3 numbers from client IP)
* - suspend Maxscale 1
* - wait and check version_string from Maxscale on virtual IP, expect 10.2-server2
* - resume Maxscale 1, suspend Maxscale 2
* - wait and check version_string from Maxscale on virtual IP, expect 10.2-server1
* - resume Maxscale 2
* TODO: replace 'yum' call with executing Chef recipe
*/
#include <iostream>
#include "testconnections.h"
#define FAILOVER_WAIT_TIME 5
char virtual_ip[16];
char * print_version_string(TestConnections * Test)
{
MYSQL * keepalived_conn = open_conn(Test->maxscales->rwsplit_port[0], virtual_ip, Test->maxscales->user_name, Test->maxscales->password, Test->ssl);
const char * version_string;
mariadb_get_info(keepalived_conn, MARIADB_CONNECTION_SERVER_VERSION, (void *)&version_string);
Test->tprintf("%s\n", version_string);
mysql_close(keepalived_conn);
return((char*) version_string);
}
int main(int argc, char *argv[])
{
int i;
char * version;
TestConnections * Test = new TestConnections(argc, argv);
Test->set_timeout(10);
Test->tprintf("Maxscale_N %d\n", Test->maxscales->N);
if (Test->maxscales->N < 2)
{
Test->tprintf("At least 2 Maxscales are needed for this test. Exiting\n");
exit(0);
}
Test->check_maxscale_alive(0);
Test->check_maxscale_alive(1);
// Get test client IP, replace last number in it with 253 and use it as Virtual IP
char client_ip[24];
char * last_dot;
Test->get_client_ip(0, client_ip);
last_dot = client_ip;
Test->tprintf("My IP is %s\n", client_ip);
for (i = 0; i < 3; i++)
{
last_dot = strstr(last_dot, ".");
last_dot = &last_dot[1];
}
last_dot[0] = '\0';
Test->tprintf("First part of IP is %s\n", client_ip);
sprintf(virtual_ip, "%s253", client_ip);
for (i = 0; i < Test->maxscales->N; i++)
{
std::string src = std::string(test_dir) + "/keepalived_cnf/" + std::to_string(i + 1) + ".conf";
std::string cp_cmd = "cp " + std::string(Test->maxscales->access_homedir[i]) + std::to_string(i + 1) + ".conf " +
" /etc/keepalived/keepalived.conf";
Test->tprintf("%s\n", src.c_str());
Test->tprintf("%s\n", cp_cmd.c_str());
Test->maxscales->ssh_node(i, "yum install -y keepalived", true);
Test->maxscales->copy_to_node(i, src.c_str(), Test->maxscales->access_homedir[i]);
Test->maxscales->ssh_node(i, cp_cmd.c_str(), true);
Test->maxscales->ssh_node_f(i, true, "sed -i \"s/###virtual_ip###/%s/\" /etc/keepalived/keepalived.conf", virtual_ip);
std::string script_src = std::string(test_dir) + "/keepalived_cnf/is_maxscale_running.sh";
std::string script_cp_cmd = "cp " + std::string(Test->maxscales->access_homedir[i]) + "is_maxscale_running.sh /usr/bin/";
Test->maxscales->copy_to_node(i, script_src.c_str(), Test->maxscales->access_homedir[i]);
Test->maxscales->ssh_node(i, script_cp_cmd.c_str(), true);
Test->maxscales->ssh_node(i, "sudo service keepalived restart", true);
}
print_version_string(Test);
Test->tprintf("Suspend Maxscale 000 machine and waiting\n");
system(Test->maxscales->stop_vm_command[0]);
sleep(FAILOVER_WAIT_TIME);
version = print_version_string(Test);
if (strcmp(version, "10.2-server2") != 0)
{
Test->add_result(false, "Failover did not happen");
}
Test->tprintf("Resume Maxscale 000 machine and waiting\n");
system(Test->maxscales->start_vm_command[0]);
sleep(FAILOVER_WAIT_TIME);
print_version_string(Test);
Test->tprintf("Suspend Maxscale 001 machine and waiting\n");
system(Test->maxscales->stop_vm_command[1]);
sleep(FAILOVER_WAIT_TIME);
version = print_version_string(Test);
if (strcmp(version, "10.2-server1") != 0)
{
Test->add_result(false, "Failover did not happen");
}
print_version_string(Test);
Test->tprintf("Resume Maxscale 001 machine and waiting\n");
system(Test->maxscales->start_vm_command[1]);
sleep(FAILOVER_WAIT_TIME);
print_version_string(Test);
Test->tprintf("Stop Maxscale on 000 machine\n");
Test->stop_maxscale(0);
sleep(FAILOVER_WAIT_TIME);
version = print_version_string(Test);
if (strcmp(version, "10.2-server2") != 0)
{
Test->add_result(false, "Failover did not happen");
}
Test->tprintf("Start back Maxscale on 000 machine\n");
Test->start_maxscale(0);
sleep(FAILOVER_WAIT_TIME);
Test->tprintf("Stop Maxscale on 001 machine\n");
Test->stop_maxscale(1);
sleep(FAILOVER_WAIT_TIME);
version = print_version_string(Test);
if (strcmp(version, "10.2-server1") != 0)
{
Test->add_result(false, "Failover did not happen");
}
int rval = Test->global_result;
delete Test;
return rval;
}

24
maxscale-system-test/keepalived_cnf/is_maxscale_running.sh Executable file
View File

@ -0,0 +1,24 @@
#!/bin/bash
fileName="maxadmin_output.txt"
rm $fileName
timeout 2s maxadmin list servers > $fileName
to_result=$?
if [ $to_result -ge 1 ]
then
echo Timed out or error, timeout returned $to_result
exit 3
else
echo MaxAdmin success, rval is $to_result
echo Checking maxadmin output sanity
grep1=$(grep server1 $fileName)
grep2=$(grep server2 $fileName)
if [ "$grep1" ] && [ "$grep2" ]
then
echo All is fine
exit 0
else
echo Something is wrong
exit 3
fi
fi

7
maxscale-system-test/mdbci/set_env.sh
View File

@ -20,6 +20,7 @@ export maxscale_log_dir="/var/log/maxscale/"
# Number of nodes
export galera_N=`cat "$MDBCI_VM_PATH/$config_name"_network_config | grep galera | grep network | wc -l`
export node_N=`cat "$MDBCI_VM_PATH/$config_name"_network_config | grep node | grep network | wc -l`
export maxscale_N=`cat "$MDBCI_VM_PATH/$config_name"_network_config | grep maxscale | grep network | wc -l`
sed "s/^/export /g" "$MDBCI_VM_PATH/$config_name"_network_config > "$curr_dir"/"$config_name"_network_config_export
source "$curr_dir"/"$config_name"_network_config_export
@ -40,7 +41,7 @@ export maxscale_password="skysql"
export maxadmin_password="mariadb"
for prefix in "node" "galera"
for prefix in "node" "galera" "maxscale"
do
N_var="$prefix"_N
Nx=${!N_var}
@ -77,8 +78,8 @@ do
eval 'export $stop_cmd_var="$mysql_exe stop "'
fi
eval 'export "$prefix"_"$num"_start_vm_command="cd $mdbci_dir/$config_name;vagrant up node_$num --provider=$provider; cd $curr_dir"'
eval 'export "$prefix"_"$num"_kill_vm_command="cd $mdbci_dir/$config_name;vagrant halt node_$num --provider=$provider; cd $curr_dir"'
eval 'export "$prefix"_"$num"_start_vm_command="cd ${MDBCI_VM_PATH}/$config_name;vagrant resume ${prefix}_$num ; cd $curr_dir"'
eval 'export "$prefix"_"$num"_stop_vm_command="cd ${MDBCI_VM_PATH}/$config_name;vagrant suspend ${prefix}_$num ; cd $curr_dir"'
done
done

35
maxscale-system-test/nodes.cpp
View File

@ -197,7 +197,6 @@ int Nodes::ssh_node_f(int node, bool sudo, const char* format, ...)
va_start(valist, format);
vsnprintf(sys, message_len + 1, format, valist);
va_end(valist);
int result = ssh_node(node, sys, sudo);
free(sys);
return (result);
@ -428,6 +427,40 @@ int Nodes::read_basic_env()
{
sprintf(hostname[i], "%s", IP[i]);
}
sprintf(env_name, "%s_%03d_start_vm_command", prefix, i);
env = getenv(env_name);
if (env == NULL)
{
sprintf(env_name, "%s_start_vm_command", prefix);
env = getenv(env_name);
}
if (env != NULL)
{
sprintf(start_vm_command[i], "%s", env);
}
else
{
sprintf(start_vm_command[i], "exit 0");
}
sprintf(env_name, "%s_%03d_stop_vm_command", prefix, i);
env = getenv(env_name);
if (env == NULL)
{
sprintf(env_name, "%s_stop_vm_command", prefix);
env = getenv(env_name);
}
if (env != NULL)
{
sprintf(stop_vm_command[i], "%s", env);
}
else
{
sprintf(stop_vm_command[i], "exit 0");
}
}
}

10
maxscale-system-test/nodes.h
View File

@ -64,6 +64,16 @@ public:
char hostname[256][1024];
/**
* @brief stop_vm_command Command to suspend VM
*/
char stop_vm_command[256][1024];
/**
* @brief start_vm_command Command to resume VM
*/
char start_vm_command[256][1024];
/**
* @brief User name to access backend nodes
*/

27
maxscale-system-test/testconnections.cpp
View File

@ -6,6 +6,7 @@
#include <time.h>
#include <signal.h>
#include <execinfo.h>
#include <sys/stat.h>
#include <sstream>
#include "mariadb_func.h"
@ -281,7 +282,7 @@ TestConnections::TestConnections(int argc, char *argv[]):
if (maxscale_init)
{
init_maxscale(0);
init_maxscales();
}
if (backend_ssl)
@ -529,11 +530,23 @@ const char * get_template_name(char * test_name)
void TestConnections::process_template(int m, const char *template_name, const char *dest)
{
struct stat stb;
char str[4096];
char template_file[1024];
char extended_template_file[1024];
sprintf(template_file, "%s/cnf/maxscale.cnf.template.%s", test_dir, template_name);
sprintf(extended_template_file, "%s.%03d", template_file, m);
if (stat((char*)extended_template_file, &stb) == 0)
{
strcpy(template_file, extended_template_file);
}
tprintf("Template file is %s\n", template_file);
sprintf(str, "cp %s maxscale.cnf", template_file);
tprintf("Executing '%s' command\n", str);
if (system(str) != 0)
{
tprintf("Error copying maxscale.cnf template\n");
@ -595,12 +608,19 @@ void TestConnections::process_template(int m, const char *template_name, const c
maxscales->copy_to_node_legacy((char *) "maxscale.cnf", (char *) dest, m);
}
int TestConnections::init_maxscales()
{
for (int i = 0; i < maxscales->N; i++)
{
init_maxscale(i);
}
}
int TestConnections::init_maxscale(int m)
{
const char * template_name = get_template_name(test_name);
tprintf("Template is %s\n", template_name);
process_template(m, template_name, maxscales->access_homedir[m]);
process_template(m, template_name, maxscales->access_homedir[m]);
maxscales->ssh_node_f(m, true,
"cp maxscale.cnf %s;rm -rf %s/certs;mkdir -m a+wrx %s/certs;",
maxscales->maxscale_cnf[m],
@ -614,7 +634,6 @@ int TestConnections::init_maxscale(int m)
sprintf(str, "cp %s/ssl-cert/* .", test_dir);
system(str);
maxscales->ssh_node_f(m, true,
"chown maxscale:maxscale -R %s/certs;"
"chmod 664 %s/certs/*.pem;"

7
maxscale-system-test/testconnections.h
View File

@ -263,11 +263,16 @@ public:
/**
* @brief InitMaxscale Copies MaxSclae.cnf and start MaxScale
* @param m Number of Maxscale node
* @return 0 if case of success
*/
int init_maxscale(int m = 0);
/**
* @brief InitMaxscale Copies MaxSclae.cnf and start MaxScale on all Maxscale nodes
* @return 0 if case of success
*/
int init_maxscales();
/**
* @brief start_binlog configure first node as Master, Second as slave connected to Master and others as slave connected to MaxScale binlog router