diff --git a/maxscale-system-test/CMakeLists.txt b/maxscale-system-test/CMakeLists.txt index 748ebe651..5c481a7ce 100644 --- a/maxscale-system-test/CMakeLists.txt +++ b/maxscale-system-test/CMakeLists.txt @@ -822,6 +822,9 @@ add_test_executable(setup_binlog_gtid.cpp setup_binlog_gtid setup_binlog_gtid LA # TODO: make it working with zypper and apt, move part of KDC setup to MDBCI add_test_executable(kerberos_setup.cpp kerberos_setup kerberos LABELS HEAVY gssapi REPL_BACKEND) +# Configures 'keepalived' on two Maxscale machines and tried failover +add_test_executable(keepalived.cpp keepalived keepalived LABELS REPL_BACKEND TWO_MAXSCALES) + # enable after fixing MXS-419 # add_test_executable(mxs419_lots_of_connections.cpp mxs419_lots_of_connections replication LABELS REPL_BACKEND) diff --git a/maxscale-system-test/cnf/maxscale.cnf.template.keepalived.000 b/maxscale-system-test/cnf/maxscale.cnf.template.keepalived.000 new file mode 100755 index 000000000..7810d3583 --- /dev/null +++ b/maxscale-system-test/cnf/maxscale.cnf.template.keepalived.000 @@ -0,0 +1,93 @@ +[maxscale] +threads=###threads### + +[MySQL Monitor] +type=monitor +module=mysqlmon +###repl51### +servers=server1,server2,server3,server4 +user=maxskysql +passwd=skysql +monitor_interval=1000 +detect_stale_master=false +detect_standalone_master=false + +[RW Split Router] +type=service +router=readwritesplit +servers=server1,server2,server3,server4 +user=maxskysql +passwd=skysql +router_options=slave_selection_criteria=LEAST_GLOBAL_CONNECTIONS +max_slave_connections=1 +version_string=10.2-server1 + +[Read Connection Router Slave] +type=service +router=readconnroute +router_options=slave +servers=server1,server2,server3,server4 +user=maxskysql +passwd=skysql +version_string=10.2-server1 + +[Read Connection Router Master] +type=service +router=readconnroute +router_options=master +servers=server1,server2,server3,server4 +user=maxskysql +passwd=skysql +version_string=10.2-server1 + +[RW Split Listener] +type=listener +service=RW Split Router +protocol=MySQLClient +port=4006 + +[Read Connection Listener Slave] +type=listener +service=Read Connection Router Slave +protocol=MySQLClient +port=4009 + +[Read Connection Listener Master] +type=listener +service=Read Connection Router Master +protocol=MySQLClient +port=4008 + +[CLI] +type=service +router=cli + +[CLI Listener] +type=listener +service=CLI +protocol=maxscaled +socket=default + +[server1] +type=server +address=###node_server_IP_1### +port=###node_server_port_1### +protocol=MySQLBackend + +[server2] +type=server +address=###node_server_IP_2### +port=###node_server_port_2### +protocol=MySQLBackend + +[server3] +type=server +address=###node_server_IP_3### +port=###node_server_port_3### +protocol=MySQLBackend + +[server4] +type=server +address=###node_server_IP_4### +port=###node_server_port_4### +protocol=MySQLBackend diff --git a/maxscale-system-test/cnf/maxscale.cnf.template.keepalived.001 b/maxscale-system-test/cnf/maxscale.cnf.template.keepalived.001 new file mode 100755 index 000000000..c16a40c49 --- /dev/null +++ b/maxscale-system-test/cnf/maxscale.cnf.template.keepalived.001 @@ -0,0 +1,93 @@ +[maxscale] +threads=###threads### + +[MySQL Monitor] +type=monitor +module=mysqlmon +###repl51### +servers=server1,server2,server3,server4 +user=maxskysql +passwd=skysql +monitor_interval=1000 +detect_stale_master=false +detect_standalone_master=false + +[RW Split Router] +type=service +router=readwritesplit +servers=server1,server2,server3,server4 +user=maxskysql +passwd=skysql +router_options=slave_selection_criteria=LEAST_GLOBAL_CONNECTIONS +max_slave_connections=1 +version_string=10.2-server2 + +[Read Connection Router Slave] +type=service +router=readconnroute +router_options=slave +servers=server1,server2,server3,server4 +user=maxskysql +passwd=skysql +version_string=10.2-server2 + +[Read Connection Router Master] +type=service +router=readconnroute +router_options=master +servers=server1,server2,server3,server4 +user=maxskysql +passwd=skysql +version_string=10.2-server2 + +[RW Split Listener] +type=listener +service=RW Split Router +protocol=MySQLClient +port=4006 + +[Read Connection Listener Slave] +type=listener +service=Read Connection Router Slave +protocol=MySQLClient +port=4009 + +[Read Connection Listener Master] +type=listener +service=Read Connection Router Master +protocol=MySQLClient +port=4008 + +[CLI] +type=service +router=cli + +[CLI Listener] +type=listener +service=CLI +protocol=maxscaled +socket=default + +[server1] +type=server +address=###node_server_IP_1### +port=###node_server_port_1### +protocol=MySQLBackend + +[server2] +type=server +address=###node_server_IP_2### +port=###node_server_port_2### +protocol=MySQLBackend + +[server3] +type=server +address=###node_server_IP_3### +port=###node_server_port_3### +protocol=MySQLBackend + +[server4] +type=server +address=###node_server_IP_4### +port=###node_server_port_4### +protocol=MySQLBackend diff --git a/maxscale-system-test/keepalived.cpp b/maxscale-system-test/keepalived.cpp new file mode 100644 index 000000000..650a28039 --- /dev/null +++ b/maxscale-system-test/keepalived.cpp @@ -0,0 +1,146 @@ +/** + * @file keepalived.cpp keepalived Test of two Maxscale + keepalived failover + * + * - 'version_string' configured to be different for every Maxscale + * - configure keepalived for two nodes (uses xxx.xxx.xxx.253 as a virtual IP + * where xxx.xxx.xxx. - first 3 numbers from client IP) + * - suspend Maxscale 1 + * - wait and check version_string from Maxscale on virtual IP, expect 10.2-server2 + * - resume Maxscale 1, suspend Maxscale 2 + * - wait and check version_string from Maxscale on virtual IP, expect 10.2-server1 + * - resume Maxscale 2 + * TODO: replace 'yum' call with executing Chef recipe + */ + + +#include +#include "testconnections.h" + +#define FAILOVER_WAIT_TIME 5 + +char virtual_ip[16]; +char * print_version_string(TestConnections * Test) +{ + MYSQL * keepalived_conn = open_conn(Test->maxscales->rwsplit_port[0], virtual_ip, Test->maxscales->user_name, Test->maxscales->password, Test->ssl); + const char * version_string; + mariadb_get_info(keepalived_conn, MARIADB_CONNECTION_SERVER_VERSION, (void *)&version_string); + Test->tprintf("%s\n", version_string); + mysql_close(keepalived_conn); + return((char*) version_string); +} + +int main(int argc, char *argv[]) +{ + int i; + char * version; + + TestConnections * Test = new TestConnections(argc, argv); + Test->set_timeout(10); + + Test->tprintf("Maxscale_N %d\n", Test->maxscales->N); + if (Test->maxscales->N < 2) + { + Test->tprintf("At least 2 Maxscales are needed for this test. Exiting\n"); + exit(0); + } + + + Test->check_maxscale_alive(0); + Test->check_maxscale_alive(1); + + // Get test client IP, replace last number in it with 253 and use it as Virtual IP + char client_ip[24]; + char * last_dot; + Test->get_client_ip(0, client_ip); + last_dot = client_ip; + Test->tprintf("My IP is %s\n", client_ip); + for (i = 0; i < 3; i++) + { + last_dot = strstr(last_dot, "."); + last_dot = &last_dot[1]; + } + last_dot[0] = '\0'; + Test->tprintf("First part of IP is %s\n", client_ip); + + sprintf(virtual_ip, "%s253", client_ip); + + + for (i = 0; i < Test->maxscales->N; i++) + { + std::string src = std::string(test_dir) + "/keepalived_cnf/" + std::to_string(i + 1) + ".conf"; + std::string cp_cmd = "cp " + std::string(Test->maxscales->access_homedir[i]) + std::to_string(i + 1) + ".conf " + + " /etc/keepalived/keepalived.conf"; + Test->tprintf("%s\n", src.c_str()); + Test->tprintf("%s\n", cp_cmd.c_str()); + Test->maxscales->ssh_node(i, "yum install -y keepalived", true); + Test->maxscales->copy_to_node(i, src.c_str(), Test->maxscales->access_homedir[i]); + Test->maxscales->ssh_node(i, cp_cmd.c_str(), true); + Test->maxscales->ssh_node_f(i, true, "sed -i \"s/###virtual_ip###/%s/\" /etc/keepalived/keepalived.conf", virtual_ip); + std::string script_src = std::string(test_dir) + "/keepalived_cnf/is_maxscale_running.sh"; + std::string script_cp_cmd = "cp " + std::string(Test->maxscales->access_homedir[i]) + "is_maxscale_running.sh /usr/bin/"; + Test->maxscales->copy_to_node(i, script_src.c_str(), Test->maxscales->access_homedir[i]); + Test->maxscales->ssh_node(i, script_cp_cmd.c_str(), true); + Test->maxscales->ssh_node(i, "sudo service keepalived restart", true); + } + + print_version_string(Test); + + Test->tprintf("Suspend Maxscale 000 machine and waiting\n"); + system(Test->maxscales->stop_vm_command[0]); + sleep(FAILOVER_WAIT_TIME); + + version = print_version_string(Test); + if (strcmp(version, "10.2-server2") != 0) + { + Test->add_result(false, "Failover did not happen"); + } + + + Test->tprintf("Resume Maxscale 000 machine and waiting\n"); + system(Test->maxscales->start_vm_command[0]); + sleep(FAILOVER_WAIT_TIME); + print_version_string(Test); + + Test->tprintf("Suspend Maxscale 001 machine and waiting\n"); + system(Test->maxscales->stop_vm_command[1]); + sleep(FAILOVER_WAIT_TIME); + + version = print_version_string(Test); + if (strcmp(version, "10.2-server1") != 0) + { + Test->add_result(false, "Failover did not happen"); + } + + print_version_string(Test); + Test->tprintf("Resume Maxscale 001 machine and waiting\n"); + system(Test->maxscales->start_vm_command[1]); + sleep(FAILOVER_WAIT_TIME); + print_version_string(Test); + + Test->tprintf("Stop Maxscale on 000 machine\n"); + Test->stop_maxscale(0); + sleep(FAILOVER_WAIT_TIME); + version = print_version_string(Test); + if (strcmp(version, "10.2-server2") != 0) + { + Test->add_result(false, "Failover did not happen"); + } + + Test->tprintf("Start back Maxscale on 000 machine\n"); + Test->start_maxscale(0); + sleep(FAILOVER_WAIT_TIME); + + Test->tprintf("Stop Maxscale on 001 machine\n"); + Test->stop_maxscale(1); + sleep(FAILOVER_WAIT_TIME); + version = print_version_string(Test); + if (strcmp(version, "10.2-server1") != 0) + { + Test->add_result(false, "Failover did not happen"); + } + + int rval = Test->global_result; + delete Test; + return rval; +} + diff --git a/maxscale-system-test/keepalived_cnf/is_maxscale_running.sh b/maxscale-system-test/keepalived_cnf/is_maxscale_running.sh new file mode 100755 index 000000000..e0a894c89 --- /dev/null +++ b/maxscale-system-test/keepalived_cnf/is_maxscale_running.sh @@ -0,0 +1,24 @@ +#!/bin/bash +fileName="maxadmin_output.txt" +rm $fileName +timeout 2s maxadmin list servers > $fileName +to_result=$? +if [ $to_result -ge 1 ] +then + echo Timed out or error, timeout returned $to_result + exit 3 +else + echo MaxAdmin success, rval is $to_result + echo Checking maxadmin output sanity + grep1=$(grep server1 $fileName) + grep2=$(grep server2 $fileName) + + if [ "$grep1" ] && [ "$grep2" ] + then + echo All is fine + exit 0 + else + echo Something is wrong + exit 3 + fi +fi diff --git a/maxscale-system-test/mdbci/set_env.sh b/maxscale-system-test/mdbci/set_env.sh index 86459e8d1..8461744c3 100644 --- a/maxscale-system-test/mdbci/set_env.sh +++ b/maxscale-system-test/mdbci/set_env.sh @@ -20,6 +20,7 @@ export maxscale_log_dir="/var/log/maxscale/" # Number of nodes export galera_N=`cat "$MDBCI_VM_PATH/$config_name"_network_config | grep galera | grep network | wc -l` export node_N=`cat "$MDBCI_VM_PATH/$config_name"_network_config | grep node | grep network | wc -l` +export maxscale_N=`cat "$MDBCI_VM_PATH/$config_name"_network_config | grep maxscale | grep network | wc -l` sed "s/^/export /g" "$MDBCI_VM_PATH/$config_name"_network_config > "$curr_dir"/"$config_name"_network_config_export source "$curr_dir"/"$config_name"_network_config_export @@ -40,7 +41,7 @@ export maxscale_password="skysql" export maxadmin_password="mariadb" -for prefix in "node" "galera" +for prefix in "node" "galera" "maxscale" do N_var="$prefix"_N Nx=${!N_var} @@ -77,8 +78,8 @@ do eval 'export $stop_cmd_var="$mysql_exe stop "' fi - eval 'export "$prefix"_"$num"_start_vm_command="cd $mdbci_dir/$config_name;vagrant up node_$num --provider=$provider; cd $curr_dir"' - eval 'export "$prefix"_"$num"_kill_vm_command="cd $mdbci_dir/$config_name;vagrant halt node_$num --provider=$provider; cd $curr_dir"' + eval 'export "$prefix"_"$num"_start_vm_command="cd ${MDBCI_VM_PATH}/$config_name;vagrant resume ${prefix}_$num ; cd $curr_dir"' + eval 'export "$prefix"_"$num"_stop_vm_command="cd ${MDBCI_VM_PATH}/$config_name;vagrant suspend ${prefix}_$num ; cd $curr_dir"' done done diff --git a/maxscale-system-test/nodes.cpp b/maxscale-system-test/nodes.cpp index c85b19dab..125b53f16 100644 --- a/maxscale-system-test/nodes.cpp +++ b/maxscale-system-test/nodes.cpp @@ -197,7 +197,6 @@ int Nodes::ssh_node_f(int node, bool sudo, const char* format, ...) va_start(valist, format); vsnprintf(sys, message_len + 1, format, valist); va_end(valist); - int result = ssh_node(node, sys, sudo); free(sys); return (result); @@ -428,6 +427,40 @@ int Nodes::read_basic_env() { sprintf(hostname[i], "%s", IP[i]); } + + sprintf(env_name, "%s_%03d_start_vm_command", prefix, i); + env = getenv(env_name); + if (env == NULL) + { + sprintf(env_name, "%s_start_vm_command", prefix); + env = getenv(env_name); + } + + if (env != NULL) + { + sprintf(start_vm_command[i], "%s", env); + } + else + { + sprintf(start_vm_command[i], "exit 0"); + } + + sprintf(env_name, "%s_%03d_stop_vm_command", prefix, i); + env = getenv(env_name); + if (env == NULL) + { + sprintf(env_name, "%s_stop_vm_command", prefix); + env = getenv(env_name); + } + + if (env != NULL) + { + sprintf(stop_vm_command[i], "%s", env); + } + else + { + sprintf(stop_vm_command[i], "exit 0"); + } } } diff --git a/maxscale-system-test/nodes.h b/maxscale-system-test/nodes.h index b961e226f..376a03ae9 100644 --- a/maxscale-system-test/nodes.h +++ b/maxscale-system-test/nodes.h @@ -64,6 +64,16 @@ public: char hostname[256][1024]; + /** + * @brief stop_vm_command Command to suspend VM + */ + char stop_vm_command[256][1024]; + /** + + * @brief start_vm_command Command to resume VM + */ + char start_vm_command[256][1024]; + /** * @brief User name to access backend nodes */ diff --git a/maxscale-system-test/testconnections.cpp b/maxscale-system-test/testconnections.cpp index 9828afa7b..5c44be7df 100644 --- a/maxscale-system-test/testconnections.cpp +++ b/maxscale-system-test/testconnections.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include "mariadb_func.h" @@ -281,7 +282,7 @@ TestConnections::TestConnections(int argc, char *argv[]): if (maxscale_init) { - init_maxscale(0); + init_maxscales(); } if (backend_ssl) @@ -529,11 +530,23 @@ const char * get_template_name(char * test_name) void TestConnections::process_template(int m, const char *template_name, const char *dest) { + struct stat stb; char str[4096]; char template_file[1024]; + char extended_template_file[1024]; + sprintf(template_file, "%s/cnf/maxscale.cnf.template.%s", test_dir, template_name); + sprintf(extended_template_file, "%s.%03d", template_file, m); + + if (stat((char*)extended_template_file, &stb) == 0) + { + strcpy(template_file, extended_template_file); + } + tprintf("Template file is %s\n", template_file); + sprintf(str, "cp %s maxscale.cnf", template_file); + tprintf("Executing '%s' command\n", str); if (system(str) != 0) { tprintf("Error copying maxscale.cnf template\n"); @@ -595,12 +608,19 @@ void TestConnections::process_template(int m, const char *template_name, const c maxscales->copy_to_node_legacy((char *) "maxscale.cnf", (char *) dest, m); } +int TestConnections::init_maxscales() +{ + for (int i = 0; i < maxscales->N; i++) + { + init_maxscale(i); + } +} + int TestConnections::init_maxscale(int m) { const char * template_name = get_template_name(test_name); - tprintf("Template is %s\n", template_name); - process_template(m, template_name, maxscales->access_homedir[m]); + process_template(m, template_name, maxscales->access_homedir[m]); maxscales->ssh_node_f(m, true, "cp maxscale.cnf %s;rm -rf %s/certs;mkdir -m a+wrx %s/certs;", maxscales->maxscale_cnf[m], @@ -614,7 +634,6 @@ int TestConnections::init_maxscale(int m) sprintf(str, "cp %s/ssl-cert/* .", test_dir); system(str); - maxscales->ssh_node_f(m, true, "chown maxscale:maxscale -R %s/certs;" "chmod 664 %s/certs/*.pem;" diff --git a/maxscale-system-test/testconnections.h b/maxscale-system-test/testconnections.h index 18aa92413..824732aa8 100644 --- a/maxscale-system-test/testconnections.h +++ b/maxscale-system-test/testconnections.h @@ -263,11 +263,16 @@ public: /** * @brief InitMaxscale Copies MaxSclae.cnf and start MaxScale + * @param m Number of Maxscale node * @return 0 if case of success */ int init_maxscale(int m = 0); - + /** + * @brief InitMaxscale Copies MaxSclae.cnf and start MaxScale on all Maxscale nodes + * @return 0 if case of success + */ + int init_maxscales(); /** * @brief start_binlog configure first node as Master, Second as slave connected to Master and others as slave connected to MaxScale binlog router