Mxs 1665 keepalived masterdown test (#171)

* add keepalived_masterdown test

* add missing cnfs

maxscale-system-test/CMakeLists.txt

@@ -50,7 +50,7 @@ add_library(testcore SHARED testconnections.cpp nodes.cpp mariadb_nodes.cpp maxs
   mariadb_func.cpp get_com_select_insert.cpp maxadmin_operations.cpp big_transaction.cpp
   sql_t1.cpp test_binlog_fnc.cpp get_my_ip.cpp big_load.cpp  get_com_select_insert.cpp
   different_size.cpp fw_copy_rules maxinfo_func.cpp config_operations.cpp rds_vpc.cpp execute_cmd.cpp
-  blob_test.cpp
+  blob_test.cpp keepalived_func
   # Include the CDC connector in the core library
   ${CMAKE_SOURCE_DIR}/../connectors/cdc-connector/cdc_connector.cpp)
 target_link_libraries(testcore ${MYSQL_CLIENT} ${JANSSON_LIBRARIES} z nsl m pthread ssl dl rt crypto crypt)
@@ -837,6 +837,8 @@ add_test_executable(kerberos_setup.cpp kerberos_setup kerberos LABELS HEAVY gssa
 # Configures 'keepalived' on two Maxscale machines and tried failover
 add_test_executable(keepalived.cpp keepalived keepalived LABELS REPL_BACKEND TWO_MAXSCALES)
 
+# Configures 'keepalived' on two Maxscale machines and tried failover
+add_test_executable(keepalived_masterdown.cpp keepalived_masterdown keepalived_masterdown LABELS REPL_BACKEND TWO_MAXSCALES)
 
 # enable after fixing MXS-419
 # add_test_executable(mxs419_lots_of_connections.cpp mxs419_lots_of_connections replication LABELS REPL_BACKEND)

maxscale-system-test/cnf/maxscale.cnf.template.keepalived_masterdown.000

@@ -0,0 +1,94 @@
+[maxscale]
+threads=###threads###
+
+[MySQL Monitor]
+type=monitor
+module=mariadbmon
+###repl51###
+servers=server1,server2,server3,server4
+user=maxskysql
+passwd=skysql
+monitor_interval=1000
+detect_stale_master=false
+detect_standalone_master=false
+auto_failover=true
+
+[RW Split Router]
+type=service
+router=readwritesplit
+servers=server1,server2,server3,server4
+user=maxskysql
+passwd=skysql
+router_options=slave_selection_criteria=LEAST_GLOBAL_CONNECTIONS
+max_slave_connections=1
+version_string=10.2-server1
+
+[Read Connection Router Slave]
+type=service
+router=readconnroute
+router_options=slave
+servers=server1,server2,server3,server4
+user=maxskysql
+passwd=skysql
+version_string=10.2-server1
+
+[Read Connection Router Master]
+type=service
+router=readconnroute
+router_options=master
+servers=server1,server2,server3,server4
+user=maxskysql
+passwd=skysql
+version_string=10.2-server1
+
+[RW Split Listener]
+type=listener
+service=RW Split Router
+protocol=MySQLClient
+port=4006
+
+[Read Connection Listener Slave]
+type=listener
+service=Read Connection Router Slave
+protocol=MySQLClient
+port=4009
+
+[Read Connection Listener Master]
+type=listener
+service=Read Connection Router Master
+protocol=MySQLClient
+port=4008
+
+[CLI]
+type=service
+router=cli
+
+[CLI Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=default
+
+[server1]
+type=server
+address=###node_server_IP_1###
+port=###node_server_port_1###
+protocol=MySQLBackend
+
+[server2]
+type=server
+address=###node_server_IP_2###
+port=###node_server_port_2###
+protocol=MySQLBackend
+
+[server3]
+type=server
+address=###node_server_IP_3###
+port=###node_server_port_3###
+protocol=MySQLBackend
+
+[server4]
+type=server
+address=###node_server_IP_4###
+port=###node_server_port_4###
+protocol=MySQLBackend

maxscale-system-test/cnf/maxscale.cnf.template.keepalived_masterdown.001

@@ -0,0 +1,94 @@
+[maxscale]
+threads=###threads###
+
+[MySQL Monitor]
+type=monitor
+module=mariadbmon
+###repl51###
+servers=server1,server2,server3,server4
+user=maxskysql
+passwd=skysql
+monitor_interval=1000
+detect_stale_master=false
+detect_standalone_master=false
+auto_failover=true
+
+[RW Split Router]
+type=service
+router=readwritesplit
+servers=server1,server2,server3,server4
+user=maxskysql
+passwd=skysql
+router_options=slave_selection_criteria=LEAST_GLOBAL_CONNECTIONS
+max_slave_connections=1
+version_string=10.2-server2
+
+[Read Connection Router Slave]
+type=service
+router=readconnroute
+router_options=slave
+servers=server1,server2,server3,server4
+user=maxskysql
+passwd=skysql
+version_string=10.2-server2
+
+[Read Connection Router Master]
+type=service
+router=readconnroute
+router_options=master
+servers=server1,server2,server3,server4
+user=maxskysql
+passwd=skysql
+version_string=10.2-server2
+
+[RW Split Listener]
+type=listener
+service=RW Split Router
+protocol=MySQLClient
+port=4006
+
+[Read Connection Listener Slave]
+type=listener
+service=Read Connection Router Slave
+protocol=MySQLClient
+port=4009
+
+[Read Connection Listener Master]
+type=listener
+service=Read Connection Router Master
+protocol=MySQLClient
+port=4008
+
+[CLI]
+type=service
+router=cli
+
+[CLI Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=default
+
+[server1]
+type=server
+address=###node_server_IP_1###
+port=###node_server_port_1###
+protocol=MySQLBackend
+
+[server2]
+type=server
+address=###node_server_IP_2###
+port=###node_server_port_2###
+protocol=MySQLBackend
+
+[server3]
+type=server
+address=###node_server_IP_3###
+port=###node_server_port_3###
+protocol=MySQLBackend
+
+[server4]
+type=server
+address=###node_server_IP_4###
+port=###node_server_port_4###
+protocol=MySQLBackend

maxscale-system-test/keepalived.cpp

@@ -15,23 +15,11 @@
 
 #include <iostream>
 #include "testconnections.h"
-
-#define FAILOVER_WAIT_TIME 5
-
-char virtual_ip[16];
-char * print_version_string(TestConnections * Test)
-{
-    MYSQL * keepalived_conn = open_conn(Test->maxscales->rwsplit_port[0], virtual_ip, Test->maxscales->user_name, Test->maxscales->password, Test->ssl);
-    const char * version_string;
-    mariadb_get_info(keepalived_conn, MARIADB_CONNECTION_SERVER_VERSION, (void *)&version_string);
-    Test->tprintf("%s\n", version_string);
-    mysql_close(keepalived_conn);
-    return((char*) version_string);
-}
+#include "keepalived_func.h"
 
 int main(int argc, char *argv[])
 {
-    int i;
+
     char * version;
 
     TestConnections * Test = new TestConnections(argc, argv);
@@ -49,39 +37,7 @@ int main(int argc, char *argv[])
     Test->check_maxscale_alive(1);
 
     // Get test client IP, replace last number in it with 253 and use it as Virtual IP
-    char client_ip[24];
-    char * last_dot;
-    Test->get_client_ip(0, client_ip);
-    last_dot = client_ip;
-    Test->tprintf("My IP is %s\n", client_ip);
-    for (i = 0; i < 3; i++)
-    {
-        last_dot = strstr(last_dot, ".");
-        last_dot = &last_dot[1];
-    }
-    last_dot[0] = '\0';
-    Test->tprintf("First part of IP is %s\n", client_ip);
-
-    sprintf(virtual_ip, "%s253", client_ip);
-
-
-    for (i = 0; i < Test->maxscales->N; i++)
-    {
-        std::string src = std::string(test_dir) + "/keepalived_cnf/" + std::to_string(i + 1) + ".conf";
-        std::string cp_cmd = "cp " + std::string(Test->maxscales->access_homedir[i]) + std::to_string(i + 1) + ".conf " +
-                " /etc/keepalived/keepalived.conf";
-        Test->tprintf("%s\n", src.c_str());
-        Test->tprintf("%s\n", cp_cmd.c_str());
-        Test->maxscales->ssh_node(i, "yum install -y keepalived", true);
-        Test->maxscales->copy_to_node(i, src.c_str(), Test->maxscales->access_homedir[i]);
-        Test->maxscales->ssh_node(i, cp_cmd.c_str(), true);
-        Test->maxscales->ssh_node_f(i, true, "sed -i \"s/###virtual_ip###/%s/\" /etc/keepalived/keepalived.conf", virtual_ip);
-        std::string script_src = std::string(test_dir) + "/keepalived_cnf/is_maxscale_running.sh";
-        std::string script_cp_cmd = "cp " + std::string(Test->maxscales->access_homedir[i]) + "is_maxscale_running.sh /usr/bin/";
-        Test->maxscales->copy_to_node(i, script_src.c_str(), Test->maxscales->access_homedir[i]);
-        Test->maxscales->ssh_node(i, script_cp_cmd.c_str(), true);
-        Test->maxscales->ssh_node(i, "sudo service keepalived restart", true);
-    }
+    configure_keepalived(Test, (char *) "");
 
     print_version_string(Test);
 

maxscale-system-test/keepalived_cnf/1.conf

@@ -0,0 +1,24 @@
+vrrp_script chk_myscript {
+  script "/usr/bin/is_maxscale_running.sh"
+  interval 2 # check every 2 seconds
+  fall 2 # require 2 failures for KO
+  rise 2 # require 2 successes for OK
+}
+
+vrrp_instance VI_1 {
+  state MASTER
+  interface eth0
+  virtual_router_id 51
+  priority 150
+  advert_int 1
+  authentication {
+    auth_type PASS
+    auth_pass mypass
+  }
+  virtual_ipaddress {
+    ###virtual_ip###
+  }
+  track_script {
+    chk_myscript
+  }
+}

maxscale-system-test/keepalived_cnf/2.conf

@@ -0,0 +1,24 @@
+vrrp_script chk_myscript {
+  script "/usr/bin/is_maxscale_running.sh"
+  interval 2 # check every 2 seconds
+  fall 2 # require 2 failures for KO
+  rise 2 # require 2 successes for OK
+}
+
+vrrp_instance VI_1 {
+  state MASTER
+  interface eth0
+  virtual_router_id 51
+  priority 100
+  advert_int 1
+  authentication {
+    auth_type PASS
+    auth_pass mypass
+  }
+  virtual_ipaddress {
+    ###virtual_ip###
+  }
+  track_script {
+    chk_myscript
+  }
+}

maxscale-system-test/keepalived_cnf/masterdown1.conf

@@ -0,0 +1,25 @@
+vrrp_script chk_myscript {
+  script "/usr/bin/is_maxscale_running.sh"
+  interval 2 # check every 2 seconds
+  fall 2 # require 2 failures for KO
+  rise 2 # require 2 successes for OK
+}
+
+vrrp_instance VI_1 {
+  state MASTER
+  interface eth0
+  virtual_router_id 51
+  priority 150
+  advert_int 1
+  authentication {
+    auth_type PASS
+    auth_pass mypass
+  }
+  virtual_ipaddress {
+    ###virtual_ip###
+  }
+  track_script {
+    chk_myscript
+  }
+  notify /usr/bin/notify_script.sh
+}

maxscale-system-test/keepalived_cnf/masterdown2.conf

@@ -0,0 +1,25 @@
+vrrp_script chk_myscript {
+  script "/usr/bin/is_maxscale_running.sh"
+  interval 2 # check every 2 seconds
+  fall 2 # require 2 failures for KO
+  rise 2 # require 2 successes for OK
+}
+
+vrrp_instance VI_2 {
+  state MASTER
+  interface eth0
+  virtual_router_id 51
+  priority 100
+  advert_int 1
+  authentication {
+    auth_type PASS
+    auth_pass mypass
+  }
+  virtual_ipaddress {
+    ###virtual_ip###
+  }
+  track_script {
+    chk_myscript
+  }
+  notify /usr/bin/notify_script.sh
+}

maxscale-system-test/keepalived_cnf/notify_script.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+TYPE=$1
+NAME=$2
+STATE=$3
+
+OUTFILE=/tmp/state.txt
+touch $OUTFILE
+#OUTFILE=/home/vagrant/state.txt
+
+case $STATE in
+  "MASTER") echo "Setting this MaxScale node to active mode $TYPE $NAME $STATE" > $OUTFILE
+                  maxctrl alter maxscale passive false
+                  exit 0
+                  ;;
+  "BACKUP") echo "Setting this MaxScale node to passive mode $TYPE $NAME $STATE" > $OUTFILE
+                  maxctrl alter maxscale passive true
+                  exit 0
+                  ;;
+  "FAULT")  echo "MaxScale failed the status check. $TYPE $NAME $STATE" > $OUTFILE
+                  maxctrl alter maxscale passive true
+                  exit 0
+                  ;;
+        *)        echo "Unknown state $TYPE $NAME $STATE" > $OUTFILE
+                  exit 1
+                  ;;
+esac

maxscale-system-test/keepalived_func.cpp

@@ -0,0 +1,57 @@
+#include "keepalived_func.h"
+
+char * print_version_string(TestConnections * Test)
+{
+    MYSQL * keepalived_conn = open_conn(Test->maxscales->rwsplit_port[0], virtual_ip, Test->maxscales->user_name, Test->maxscales->password, Test->ssl);
+    const char * version_string;
+    mariadb_get_info(keepalived_conn, MARIADB_CONNECTION_SERVER_VERSION, (void *)&version_string);
+    Test->tprintf("%s\n", version_string);
+    mysql_close(keepalived_conn);
+    return((char*) version_string);
+}
+
+void configure_keepalived(TestConnections* Test, char * keepalived_file)
+{
+    int i;
+    char client_ip[24];
+    char * last_dot;
+    Test->get_client_ip(0, client_ip);
+    last_dot = client_ip;
+    Test->tprintf("My IP is %s\n", client_ip);
+    for (i = 0; i < 3; i++)
+    {
+        last_dot = strstr(last_dot, ".");
+        last_dot = &last_dot[1];
+    }
+    last_dot[0] = '\0';
+    Test->tprintf("First part of IP is %s\n", client_ip);
+
+    sprintf(virtual_ip, "%s253", client_ip);
+
+
+    for (i = 0; i < Test->maxscales->N; i++)
+    {
+        std::string src = std::string(test_dir)
+                + "/keepalived_cnf/"
+                + std::string(keepalived_file)
+                + std::to_string(i + 1)
+                + ".conf";
+        std::string cp_cmd = "cp "
+                + std::string(Test->maxscales->access_homedir[i])
+                + std::string(keepalived_file)
+                + std::to_string(i + 1) + ".conf "
+                + " /etc/keepalived/keepalived.conf";
+        Test->tprintf("%s\n", src.c_str());
+        Test->tprintf("%s\n", cp_cmd.c_str());
+        Test->maxscales->ssh_node(i, "yum install -y keepalived", true);
+        Test->maxscales->ssh_node(i, "service iptables stop", true);
+        Test->maxscales->copy_to_node(i, src.c_str(), Test->maxscales->access_homedir[i]);
+        Test->maxscales->ssh_node(i, cp_cmd.c_str(), true);
+        Test->maxscales->ssh_node_f(i, true, "sed -i \"s/###virtual_ip###/%s/\" /etc/keepalived/keepalived.conf", virtual_ip);
+        std::string script_src = std::string(test_dir) + "/keepalived_cnf/*.sh";
+        std::string script_cp_cmd = "cp " + std::string(Test->maxscales->access_homedir[i]) + "*.sh /usr/bin/";
+        Test->maxscales->copy_to_node(i, script_src.c_str(), Test->maxscales->access_homedir[i]);
+        Test->maxscales->ssh_node(i, script_cp_cmd.c_str(), true);
+        Test->maxscales->ssh_node(i, "sudo service keepalived restart", true);
+    }
+}

maxscale-system-test/keepalived_func.h

@@ -0,0 +1,12 @@
+#ifndef KEEPALIVED_FUNC_H
+#define KEEPALIVED_FUNC_H
+
+#include "testconnections.h"
+
+#define FAILOVER_WAIT_TIME 20
+
+char virtual_ip[16];
+char * print_version_string(TestConnections * Test);
+void configure_keepalived(TestConnections* Test, char *keepalived_file);
+
+#endif // KEEPALIVED_FUNC_H

maxscale-system-test/keepalived_masterdown.cpp

@@ -0,0 +1,185 @@
+/**
+ * @file keepalived:masterdown.cpp
+ */
+
+
+#include <iostream>
+#include "testconnections.h"
+#include "keepalived_func.h"
+
+bool check_maxscale_passive(TestConnections* Test, int node)
+{
+    bool passive;
+    char * passive_str;
+    int ec;
+    Test->tprintf("Checking status of Maxscale %03d", node);
+    passive_str = Test->maxscales->ssh_node_output(node, "maxctrl show maxscale | grep passive", false, &ec);
+    Test->tprintf("maxctrl output string: %s\n", passive_str);
+    if (strstr(passive_str, "false") != NULL)
+    {
+        passive = false;
+    }
+    else
+    {
+        if (strstr(passive_str, "true") == NULL)
+        {
+            Test->tprintf("Can't find 'true' or 'false' in the 'maxctrl' output\n");
+        }
+        passive = true;
+    }
+    free(passive_str);
+
+    Test->tprintf("Content of 'state.txt' file: %s\n", Test->maxscales->ssh_node_output(0, "cat /tmp/state.txt", false, &ec));
+    return passive;
+}
+
+int main(int argc, char *argv[])
+{
+    bool passive;
+    char str[1024];
+
+    TestConnections * Test = new TestConnections(argc, argv);
+    //Test->set_timeout(10);
+
+
+
+
+    Test->tprintf("Maxscale_N %d\n", Test->maxscales->N);
+    if (Test->maxscales->N < 2)
+    {
+        Test->tprintf("At least 2 Maxscales are needed for this test. Exiting\n");
+        exit(0);
+    }
+
+    Test->tprintf("Starting replication with GTID\n");
+    Test->repl->require_gtid(true);
+    Test->repl->start_replication();
+
+    Test->tprintf("Configuring 'keepalived'\n");
+    // Get test client IP, replace last number in it with 253 and use it as Virtual IP
+    configure_keepalived(Test, (char *) "masterdown");
+
+    //Test->maxscales->ssh_node(1, (char *) "maxctrl alter maxscale passive true", false);
+
+    print_version_string(Test);
+
+    sleep(FAILOVER_WAIT_TIME);sleep(FAILOVER_WAIT_TIME);
+
+    // initial state: 000 expected to be active, 001 - passive
+    passive = check_maxscale_passive(Test, 0);
+    if (passive)
+    {
+        Test->add_result(1, "Maxscale 000 is in the passive mode\n");
+    }
+    passive = check_maxscale_passive(Test, 1);
+    if (!passive)
+    {
+        Test->add_result(1, "Maxscale 001 is NOT in the passive mode\n");
+    }
+
+    int first_master = Test->repl->find_master();
+
+    Test->tprintf("Stop Master - node %d\n", first_master);
+
+    Test->repl->stop_node(first_master);
+
+    sleep(FAILOVER_WAIT_TIME);
+
+    int second_master= Test->repl->find_master();
+
+    Test->tprintf("new master is node %d\n", second_master);
+
+    if (first_master == second_master)
+    {
+        Test->add_result(1, "Failover did not happen\n");
+    }
+
+    sprintf(str, "Performing automatic failover to replace failed master 'server%d'", first_master + 1);
+    Test->tprintf("Checking Maxscale log on 000 for the failover message %s\n", str);
+    Test->check_log_err(0, str , true);
+    sprintf(str, "Performing automatic failover to replace failed master");
+    Test->tprintf("Checking Maxscale log on 001 for the lack of failover message\n");
+    Test->check_log_err(1, str , false);
+
+    passive = check_maxscale_passive(Test, 0);
+    if (passive)
+    {
+        Test->add_result(1, "Maxscale 000 is in the passive mode\n");
+    }
+    passive = check_maxscale_passive(Test, 1);
+    if (!passive)
+    {
+        Test->add_result(1, "Maxscale 001 is NOT in the passive mode\n");
+    }
+
+    Test->tprintf("Stop Maxscale 000\n");
+
+    Test->maxscales->stop_maxscale(0);
+
+    sleep(FAILOVER_WAIT_TIME);
+
+    passive = check_maxscale_passive(Test, 1);
+    if (passive)
+    {
+        Test->add_result(1, "Maxscale 001 is in the passive mode\n");
+    }
+
+    Test->tprintf("Stop new Master - node %d\n", second_master);
+    Test->repl->stop_node(second_master);
+    sleep(FAILOVER_WAIT_TIME);
+
+    int third_master= Test->repl->find_master();
+
+    Test->tprintf("new master (third one) is node %d\n", third_master);
+
+    if (second_master == third_master)
+    {
+        Test->add_result(1, "Failover did not happen\n");
+    }
+    sprintf(str, "Performing automatic failover to replace failed master 'server%d'", second_master + 1);
+    Test->tprintf("Checking Maxscale log on 001 for the failover message %s\n", str);
+    Test->check_log_err(1, str , true);
+
+    Test->check_log_err(1, (char *) "Multiple failed master servers detected" , false);
+    Test->check_log_err(1, (char *) "Failed to perform failover" , false);
+    Test->check_log_err(1, (char *) "disabling automatic failover" , false);
+
+    Test->tprintf("Start Maxscale 000\n");
+
+    Test->maxscales->start_maxscale(0);
+
+    sleep(FAILOVER_WAIT_TIME);
+
+    passive = check_maxscale_passive(Test, 0);
+    if (passive)
+    {
+        Test->add_result(1, "Maxscale 000 is in the passive mode\n");
+    }
+    passive = check_maxscale_passive(Test, 1);
+    if (!passive)
+    {
+        Test->add_result(1, "Maxscale 001 is NOT in the passive mode\n");
+    }
+
+    sprintf(str, "Performing automatic failover to replace failed master 'server%d'", second_master + 1);
+    Test->tprintf("Checking Maxscale log on 001 for the failover message %s\n", str);
+    Test->check_log_err(1, str , true);
+    Test->tprintf("Checking Maxscale log on 000 for the lack of failover message %s\n", str);
+    Test->check_log_err(0, str , false);
+
+    Test->check_log_err(1, (char *) "Multiple failed master servers detected" , false);
+    Test->check_log_err(1, (char *) "Failed to perform failover" , false);
+    Test->check_log_err(1, (char *) "disabling automatic failover" , false);
+
+    Test->check_log_err(0, (char *) "Multiple failed master servers detected" , false);
+    Test->check_log_err(0, (char *) "Failed to perform failover" , false);
+    Test->check_log_err(0, (char *) "disabling automatic failover" , false);
+
+
+//    Test->repl->require_gtid(false);
+//    Test->repl->start_replication();
+
+    int rval = Test->global_result;
+    delete Test;
+    return rval;
+}

maxscale-system-test/mariadb_func.cpp

@@ -476,7 +476,6 @@ int find_field(MYSQL* conn, const char* sql, const char* field_name, char* value
     unsigned int ret = 1;
     unsigned long long int filed_i = 0;
     unsigned long long int i = 0;
-
     if (conn != NULL )
     {
         if (mysql_query(conn, sql) != 0)
@@ -494,7 +493,6 @@ int find_field(MYSQL* conn, const char* sql, const char* field_name, char* value
             else
             {
                 num_fields = mysql_num_fields(res);
-
                 while ((field = mysql_fetch_field(res)) && ret != 0)
                 {
                     if (strstr(field->name, field_name) != NULL)
@@ -509,6 +507,11 @@ int find_field(MYSQL* conn, const char* sql, const char* field_name, char* value
                     row = mysql_fetch_row(res);
                     sprintf(value, "%s", row[filed_i]);
                 }
+                else
+                {
+                    sprintf(value, "%s", "");
+                    ret = 1;
+                }
             }
             mysql_free_result(res);
             do

maxscale-system-test/mariadb_nodes.cpp

@@ -699,7 +699,6 @@ static bool wrong_replication_type(MYSQL *conn)
         }
         sleep(1);
     }
-
     return rval;
 }
 

maxscale-system-test/mdbci/templates/twomaxscales.json.template

@@ -0,0 +1,76 @@
+{
+  "node_000" :
+  {
+        "hostname" : "node000",
+        "box" : "${backend_box}",
+        "memory_size" : "${vm_memory}",
+	"product" : {
+      		"name": "${product}",
+		"version": "${version}",
+                "cnf_template" : "server1.cnf",
+                "cnf_template_path": "${cnf_path}"
+	}
+
+  },
+
+  "node_001" :
+  {
+        "hostname" : "node001",
+        "box" : "${backend_box}",
+        "memory_size" : "${vm_memory}",
+        "product" : {
+                "name": "${product}",
+                "version": "${version}",
+                "cnf_template" : "server2.cnf",
+                "cnf_template_path": "${cnf_path}"
+        }
+  },
+
+  "node_002" :
+  {
+        "hostname" : "node002",
+        "box" : "${backend_box}",
+        "memory_size" : "${vm_memory}",
+        "product" : {
+                "name": "${product}",
+                "version": "${version}",
+                "cnf_template" : "server3.cnf",
+                "cnf_template_path": "${cnf_path}"
+        }
+  },
+
+  "node_003" :
+  {
+        "hostname" : "node003",
+        "box" : "${backend_box}",
+        "memory_size" : "${vm_memory}",
+        "product" : {
+                "name": "${product}",
+                "version": "${version}",
+                "cnf_template" : "server4.cnf",
+                "cnf_template_path": "${cnf_path}"
+        }
+  },
+
+  "maxscale_000" :
+  {
+        "hostname" : "maxscale",
+        "box" : "${box}",
+        "memory_size" : "${vm_memory}",
+        "product" : {
+                "name": "maxscale"
+        }
+
+  },
+
+  "maxscale_001" :
+  {
+        "hostname" : "maxscale",
+        "box" : "${box}",
+        "memory_size" : "${vm_memory}",
+        "product" : {
+                "name": "maxscale"
+        }
+
+  }
+}

maxscale-system-test/testconnections.cpp

@@ -636,19 +636,22 @@ int TestConnections::init_maxscale(int m)
     maxscales->copy_to_node_legacy(str, dtr, m);
     sprintf(str, "cp %s/ssl-cert/* .", test_dir);
     system(str);
-
     maxscales->ssh_node_f(m, true,
                           "chown maxscale:maxscale -R %s/certs;"
                           "chmod 664 %s/certs/*.pem;"
-                          " chmod a+x %s;"
+                          "chmod a+x %s;"
                           "%s"
                           "iptables -I INPUT -p tcp --dport 4001 -j ACCEPT;"
-                          "rm -f %s/maxscale.log %s/maxscale1.log;"
+                          "rm -f %s/maxscale.log;"
+                          "rm -f %s/maxscale1.log;"
                           "rm -rf /tmp/core* /dev/shm/* /var/lib/maxscale/maxscale.cnf.d/ /var/lib/maxscale/*;"
                           "%s",
-                          maxscales->access_homedir[m], maxscales->access_homedir[m], maxscales->access_homedir[m],
+                          maxscales->access_homedir[m],
+                          maxscales->access_homedir[m],
+                          maxscales->access_homedir[m],
                           maxscale::start ? "killall -9 maxscale;" : "",
-                          maxscales->maxscale_log_dir[m], maxscales->maxscale_log_dir[m],
+                          maxscales->maxscale_log_dir[m],
+                          maxscales->maxscale_log_dir[m],
                           maxscale::start ? "service maxscale restart" : "");
 
     fflush(stdout);