2.1 fix restore (#151)

* refactor test backend fixing

* return comatibility with 5.5 for backend restore

* remove backend configuration scripts

maxscale-system-test/check_backend.cpp

@@ -18,6 +18,9 @@ int main(int argc, char *argv[])
     Test->connect_maxscale();
     Test->tprintf("Testing connections\n");
     Test->add_result(Test->test_maxscale_connections(true, true, true), "Can't connect to backend\n");
+
+    if ((Test->galera != NULL) && (Test->galera->N != 0))
+    {
         Test->tprintf("Connecting to Maxscale router with Galera backend\n");
         MYSQL * g_conn = open_conn(4016 , Test->maxscale_IP, Test->maxscale_user, Test->maxscale_password, Test->ssl);
         if (g_conn != NULL )
@@ -26,6 +29,11 @@ int main(int argc, char *argv[])
             Test->add_result(Test->try_query(g_conn, (char *) "SELECT 1"),
                              (char *) "Error executing query against RWSplit Galera\n");
         }
+    }
+    else
+    {
+        Test->tprintf("Galera is not in use\n");
+    }
     Test->tprintf("Closing connections\n");
     Test->close_maxscale_connections();
     Test->check_maxscale_alive();
@@ -43,7 +51,6 @@ int main(int argc, char *argv[])
         Test->tprintf("Snapshots are not in use\n");
     }
 
-
     int rval = Test->global_result;
     delete Test;
     return rval;

maxscale-system-test/mariadb_nodes.cpp

@@ -43,12 +43,8 @@ Mariadb_nodes::~Mariadb_nodes()
     }
 }
 
-int Mariadb_nodes::connect()
+int Mariadb_nodes::connect(int i)
 {
-    int res = 0;
-
-    for (int i = 0; i < N; i++)
-    {
     if (nodes[i] == NULL || mysql_ping(nodes[i]) != 0)
     {
         if (nodes[i])
@@ -60,8 +56,21 @@ int Mariadb_nodes::connect()
 
     if ((nodes[i] != NULL) && (mysql_errno(nodes[i]) != 0))
     {
-            res++;
+        return 1;
     }
+    else
+    {
+        return 0;
+    }
+}
+
+int Mariadb_nodes::connect()
+{
+    int res = 0;
+
+    for (int i = 0; i < N; i++)
+    {
+        res += connect(i);
     }
 
     return res;
@@ -230,7 +239,7 @@ int Mariadb_nodes::read_env()
             }
             else
             {
-                sprintf(cleanup_db_command[i], " ");
+                sprintf(cleanup_db_command[i], "rm -rf /var/lib/mysql/*; killall -9 mysqld");
             }
 
             sprintf(env_name, "%s_%03d_whoami", prefix, i);
@@ -425,10 +434,35 @@ int Mariadb_nodes::start_replication()
     // Start all nodes
     for (int i = 0; i < N; i++)
     {
+        if (start_node(i, ""))
+        {
+            printf("Start of node %d failed, trying to cleanup and re-initialize node\n", i);
+            cleanup_db_node(i);
+            prepare_server(i);
             local_result += start_node(i, "");
+        }
+
+        printf("trying to get version\n");
+        if (connect(i))
+        {
+            printf("Connect attempt to node %d failed\n", i);
+        }
+        get_version(i);
+        close_connections();
+        printf("Node %d: Version is %s\n", i, version_major[i]);
+        if (strcmp(version_major[i], "5.5") == 0)
+        {
+            ssh_node(i, true,
+                     "mysql --force -u root %s -e \"STOP SLAVE; RESET SLAVE; RESET MASTER; SET GLOBAL read_only=OFF;\"",
+                     socket_cmd[i]);
+        }
+        else
+        {
             ssh_node(i, true,
                      "mysql --force -u root %s -e \"STOP SLAVE; STOP ALL SLAVES; RESET SLAVE; RESET SLAVE ALL; RESET MASTER; SET GLOBAL read_only=OFF;\"",
                      socket_cmd[i]);
+        }
+
         ssh_node(i, true, "sudo rm -f /etc/my.cnf.d/kerb.cnf");
         ssh_node(i, true,
                  "for i in `mysql -ss --force -u root %s -e \"SHOW DATABASES\"|grep -iv 'mysql\\|information_schema\\|performance_schema'`; "
@@ -443,20 +477,33 @@ int Mariadb_nodes::start_replication()
              user_name, password, access_homedir[0], socket_cmd[0]);
 
     // Create a database dump from the master and distribute it to the slaves
+    if (version_major[0][0] == '5')
+    {
+        printf("Version 5 on master detected, do not use --gtid flag for mysqldump\n");
+        ssh_node(0, true, "mysql --force -u root %s -e \"CREATE DATABASE test\"; "
+                 "mysqldump --all-databases --add-drop-database --flush-privileges --master-data=1 %s > /tmp/master_backup.sql",
+                 socket_cmd[0], socket_cmd[0]);
+    }
+    else
+    {
         ssh_node(0, true, "mysql --force -u root %s -e \"CREATE DATABASE test\"; "
                  "mysqldump --all-databases --add-drop-database --flush-privileges --master-data=1 --gtid %s > /tmp/master_backup.sql",
                  socket_cmd[0], socket_cmd[0]);
+    }
     sprintf(str, "%s/master_backup.sql", test_dir);
     copy_from_node("/tmp/master_backup.sql", str, 0);
 
     for (int i = 1; i < N; i++)
     {
         // Reset all nodes by first loading the dump and then starting the replication
-        printf("Starting node %d\n", i);
+        printf("Setting node %d\n", i);
         fflush(stdout);
         copy_to_node(str, "/tmp/master_backup.sql", i);
+        ssh_node(i, true, "mysql --force -u root %s -e \"STOP SLAVE;\"",
+                 socket_cmd[i]);
         ssh_node(i, true, "mysql --force -u root %s < /tmp/master_backup.sql",
                  socket_cmd[i]);
+        printf("change master to...\n");
         ssh_node(i, true, "mysql --force -u root %s -e \"CHANGE MASTER TO MASTER_HOST=\\\"%s\\\", MASTER_PORT=%d, "
                  "MASTER_USER=\\\"repl\\\", MASTER_PASSWORD=\\\"repl\\\";"
                  "START SLAVE;\"", socket_cmd[i], IP_private[0], port[0]);
@@ -467,7 +514,6 @@ int Mariadb_nodes::start_replication()
 
 int Galera_nodes::start_galera()
 {
-    char sys1[4096];
     char str[1024];
     int i;
     int local_result = 0;
@@ -481,7 +527,13 @@ int Galera_nodes::start_galera()
     ssh_node(0, false, "echo [mysqld] > cluster_address.cnf");
     ssh_node(0, false, "echo wsrep_cluster_address=gcomm:// >>  cluster_address.cnf");
     ssh_node(0, true, "cp cluster_address.cnf /etc/my.cnf.d/");
+
+    if (start_node(0, " --wsrep-cluster-address=gcomm://"))
+    {
+        cleanup_db_node(i);
+        prepare_server(i);
         local_result += start_node(0, " --wsrep-cluster-address=gcomm://");
+    }
 
     sprintf(str, "%s/create_user_galera.sh", test_dir);
     copy_to_node(str, "~/", 0);
@@ -655,11 +707,19 @@ bool Mariadb_nodes::check_master_node(MYSQL *conn)
     return rval;
 }
 
+/**
+ * @brief bad_slave_thread_status Check if filed in the slave status outpur is not 'yes'
+ * @param conn MYSQL struct (connection have to be open)
+ * @param field Filed to check
+ * @param node Node index
+ * @return false if requested filed is 'Yes'
+ */
 static bool bad_slave_thread_status(MYSQL *conn, const char *field, int node)
 {
     char str[1024] = "";
     bool rval = false;
 
+    // Doing 3 attempts to check status
     for (int i = 0; i < 2; i++)
     {
         if (find_field(conn, "SHOW SLAVE STATUS;", field, str) != 0)
@@ -670,9 +730,10 @@ static bool bad_slave_thread_status(MYSQL *conn, const char *field, int node)
         }
         else if (strcmp(str, "Yes") == 0 || strcmp(str, "No") == 0)
         {
+            printf("Node %d: filed %s is %s\n", node, field, str);
             break;
         }
-
+        printf("Node %d: filed %s is %s\n", node, field, str);
         /** Any other state is transient and we should try again */
         sleep(1);
     }
@@ -683,9 +744,16 @@ static bool bad_slave_thread_status(MYSQL *conn, const char *field, int node)
         fflush(stdout);
         rval = true;
     }
+
     return rval;
 }
 
+/**
+ * @brief multi_source_replication Check if slave is connected to more then one master
+ * @param conn MYSQL struct (have to be open)
+ * @param node Node index
+ * @return false if multisource replication is not detected
+ */
 static bool multi_source_replication(MYSQL *conn, int node)
 {
     bool rval = true;
@@ -704,6 +772,12 @@ static bool multi_source_replication(MYSQL *conn, int node)
             fflush(stdout);
         }
     }
+    else
+    {
+        printf("Node %d does not support SHOW ALL SLAVE STATUS, ignoring multi source replication check\n", node);
+        fflush(stdout);
+        rval = false;
+    }
 
     return rval;
 }
@@ -712,7 +786,6 @@ int Mariadb_nodes::check_replication()
 {
     int master = 0;
     int res = 0;
-    char str[1024];
 
     if (verbose)
     {
@@ -734,6 +807,10 @@ int Mariadb_nodes::check_replication()
             if (!check_master_node(nodes[i]))
             {
                 res = 1;
+                if (verbose)
+                {
+                    printf("Master node check failed for node %d\n", i);
+                }
             }
         }
         else if (bad_slave_thread_status(nodes[i], "Slave_IO_Running", i) ||
@@ -741,14 +818,24 @@ int Mariadb_nodes::check_replication()
                  multi_source_replication(nodes[i], i))
         {
             res = 1;
+            if (verbose)
+            {
+                printf("Slave %d check failed\n", i);
             }
         }
+    }
+
+    if (verbose)
+    {
+        printf("Replication check for %s gave code %d\n", prefix, res);
+    }
 
     return res;
 }
 
 bool Mariadb_nodes::fix_replication()
 {
+    verbose = true;
     if (check_replication())
     {
         unblock_all_nodes();
@@ -760,7 +847,8 @@ bool Mariadb_nodes::fix_replication()
         }
 
         int attempts = 2;
-        int attempts_with_cleanup = 2;
+        int attempts_with_cleanup = 1;
+        int attempts_with_revert = 1;
 
         while (check_replication() && attempts > 0)
         {
@@ -771,6 +859,7 @@ bool Mariadb_nodes::fix_replication()
 
             start_replication();
             close_connections();
+            check_replication();
 
             attempts--;
 
@@ -781,11 +870,21 @@ bool Mariadb_nodes::fix_replication()
                     printf("****** BACKEND IS STILL BROKEN! Trying to cleanup all nodes *****\n");
                     stop_nodes();
                     cleanup_db_nodes();
+                    prepare_servers();
                     attempts_with_cleanup--;
                     attempts = 2;
-                    sleep(30);
+                    sleep(10);
                     start_replication();
-                    sleep(30);
+                    sleep(10);
+                }
+                else
+                {
+                    if (attempts_with_revert > 0)
+                    {
+                        printf("****** BACKEND IS STILL BROKEN! Trying to revert all nodes from snapshot *****\n");
+                        revert_nodes_snapshot();
+                        attempts_with_cleanup = 1;
+                        attempts = 2;
                     }
                     else
                     {
@@ -794,12 +893,28 @@ bool Mariadb_nodes::fix_replication()
                     }
                 }
             }
+        }
         flush_hosts();
     }
 
     return true;
 }
 
+bool Mariadb_nodes::revert_nodes_snapshot()
+{
+    char str[1024];
+    bool rval = true;
+    for (int i = 0; i < N; i++)
+    {
+        sprintf(str, "%s clean --node-name %s_%03d", revert_snapshot_command, prefix, i);
+        if (system(str))
+        {
+            rval = false;
+        }
+    }
+    return rval;
+}
+
 int Galera_nodes::check_galera()
 {
     int res1 = 0;
@@ -946,6 +1061,7 @@ char * Mariadb_nodes::ssh_node_output(int node, const char *ssh, bool sudo, int
 {
     char sys[strlen(ssh) + 1024];
     generate_ssh_cmd(sys, node, ssh, sudo);
+    //printf("%s\n", sys);
     FILE *output = popen(sys, "r");
     if (output == NULL)
     {
@@ -1006,6 +1122,7 @@ int Mariadb_nodes::ssh_node(int node, bool sudo, const char *format, ...)
                 sshkey[node], access_user[node], IP[node]);
     }
     int rc = 1;
+    //printf("%s *** %s \n", cmd, sys);
     FILE *in = popen(cmd, "w");
 
     if (in)
@@ -1100,17 +1217,26 @@ int Mariadb_nodes::execute_query_all_nodes(const char* sql)
     return local_result;
 }
 
-int Mariadb_nodes::get_versions()
+int Mariadb_nodes::get_version(int i)
 {
-    int local_result = 0;
     char * str;
-    v51 = false;
+    int ec;
+    int local_result = 0;
+    if (find_field(nodes[i], "SELECT @@version", "@@version", version[i]))
+    {
+        printf("Failed to get version: %s, trying ssh node and use MariaDB client\n", mysql_error(nodes[i]));
+        str = ssh_node_output(i, "mysql --batch --silent  -e \"select @@version\"", true, &ec);
+        if (ec)
+        {
+            local_result++;
+            printf("Failed to get version, node %d is broken\n", i);
+        }
+        else
+        {
+            strcpy(version[i], str);
+            free(str);
+        }
 
-    for (int i = 0; i < N; i++)
-    {
-        if ((local_result += find_field(nodes[i], "SELECT @@version", "@@version", version[i])))
-        {
-            printf("Failed to get version: %s\n", mysql_error(nodes[i]));
     }
     strcpy(version_number[i], version[i]);
     str = strchr(version_number[i], '-');
@@ -1132,6 +1258,18 @@ int Mariadb_nodes::get_versions()
     {
         printf("Node %s%d: %s\t %s \t %s\n", prefix, i, version[i], version_number[i], version_major[i]);
     }
+    return local_result;
+}
+
+int Mariadb_nodes::get_versions()
+{
+    int local_result = 0;
+
+    v51 = false;
+
+    for (int i = 0; i < N; i++)
+    {
+        local_result += get_version(i);
     }
 
     for (int i = 0; i < N; i++)
@@ -1389,3 +1527,105 @@ void Mariadb_nodes::close_active_connections()
         }
     }
 }
+
+/**
+ * @brief extract_version_from_string Tries to find MariaDB server version number in the output of 'mysqld --version'
+ * Function does not allocate any memory
+ * @param version String returned by 'mysqld --version'
+ * @return pointer to the string with version number
+ */
+char * extract_version_from_string(char * version)
+{
+    int pos1 = 0;
+    int pos2 = 0;
+    int l = strlen(version);
+    while ((! isdigit(version[pos1])) && (pos1 < l))
+    {
+        pos1++;
+    }
+    pos2 = pos1;
+    while (((isdigit(version[pos2]) || version[pos2] == '.')) && (pos2 < l))
+    {
+        pos2++;
+    }
+    version[pos2] = '\0';
+    return &version[pos1];
+}
+
+int Mariadb_nodes::prepare_server(int i)
+{
+    int ec;
+
+    char * version;
+    char * version_digits;
+    char * tmp_pass;
+    char str1[1024];
+    char str2[1024];
+
+    ssh_node(i, true, stop_db_command[i]);
+    sleep(5);
+    ssh_node(i, true, "sed -i \"s/bind-address/#bind-address/g\" /etc/mysql/my.cnf.d/*.cnf");
+    ssh_node(i, true, "ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/usr.sbin.mysqld; sudo service apparmor restart");
+    version = ssh_node_output(i, "/usr/sbin/mysqld --version", false, &ec);
+    if (ec == 0)
+    {
+        version_digits = extract_version_from_string(version);
+        printf("Detected server version on node %d is %s\n", i, version_digits);
+
+        if (memcmp(version_digits, "5.", 2) == 0)
+        {
+            ssh_node(i, true, "sed -i \"s/binlog_row_image=full//\" /etc/my.cnf.d/*.cnf");
+        }
+        if (memcmp(version_digits, "5.7", 3) == 0)
+        {
+            // Disable 'validate_password' plugin, searach for random temporal
+            // password in the log and reseting passord to empty string
+            ssh_node(i, true, "/usr/sbin/mysqld --initialize; sudo chown -R mysql:mysql /var/lib/mysql");
+            ssh_node(i, true, start_db_command[i]);
+            tmp_pass = ssh_node_output(i, "cat /var/log/mysqld.log | grep \"temporary password\" | sed -n -e 's/^.*: //p'", true, &ec);
+            ssh_node(i, true, "mysqladmin -uroot -p'%s' password '%s'", tmp_pass, tmp_pass);
+            ssh_node(i, false, "echo \"UNINSTALL PLUGIN validate_password\" | sudo mysql -uroot -p'%s'", tmp_pass);
+            ssh_node(i, true, stop_db_command[i]);
+            ssh_node(i, true, start_db_command[i]);
+            ssh_node(i, true, "mysqladmin -uroot -p'%s' password ''", tmp_pass);
+        }
+        else
+        {
+            printf("Executing mysql_install_db on node %d\n", i);
+            ssh_node(i, true, "mysql_install_db; sudo chown -R mysql:mysql /var/lib/mysql");
+            printf("Starting server on node %d\n", i);
+            if (ssh_node(i, true, start_db_command[i]))
+            {
+                printf("Server start on node %d failed\n", i);
+            }
+        }
+        sleep(15);
+        sprintf(str1, "%s/mdbci/backend/create_*_user.sql", test_dir);
+        sprintf(str2, "%s/", access_homedir[i]);
+        copy_to_node(str1, str2, i);
+        sprintf(str1, "mysql < %s/create_repl_user.sql", access_homedir[i]);
+        ssh_node(i, true, str1);
+        sprintf(str1, "mysql < %s/create_skysql_user.sql", access_homedir[i]);
+        ssh_node(i, true, str1);
+
+        free(version);
+        return 0;
+    }
+    else
+    {
+        return 1;
+    }
+}
+
+int Mariadb_nodes::prepare_servers()
+{
+    int rval = 0;
+    for (int i; i < N; i++)
+    {
+        if (prepare_server(i))
+        {
+            rval = 1;
+        }
+    }
+    return rval;
+}

maxscale-system-test/mariadb_nodes.h

@@ -183,6 +183,18 @@ public:
     * @brief  Open connctions to all backend nodes (to 'test' DB)
     * @return 0 in case of success
     */
+
+    /**
+     * @brief make_snapshot_command Command line to create a snapshot of all VMs
+     */
+    char * take_snapshot_command;
+
+    /**
+     * @brief revert_snapshot_command Command line to revert a snapshot of all VMs
+     */
+    char * revert_snapshot_command;
+
+    int connect(int i);
     int connect();
 
     /**
@@ -374,7 +386,14 @@ public:
     int execute_query_all_nodes(const char* sql);
 
     /**
-     * @brief execute 'SELECT @@version' against all nodes and store result in 'version' fied
+     * @brief execute 'SELECT @@version' against one node and store result in 'version' field
+     * @param i Node index
+     * @return 0 in case of success
+     */
+    int get_version(int i);
+
+    /**
+     * @brief execute 'SELECT @@version' against all nodes and store result in 'version' field
      * @return 0 in case of success
      */
     int get_versions();
@@ -442,6 +461,21 @@ public:
      */
     bool fix_replication();
 
+    /**
+     * @brief revert_nodes_snapshot Execute MDBCI snapshot revert command for all nodes
+     * @return true in case of success
+     */
+    bool revert_nodes_snapshot();
+
+    /**
+     * @brief prepare_server Initialize MariaDB setup (run mysql_install_db) and create test users
+     * Tries to detect Mysql 5.7 installation and disable 'validate_password' pluging
+     * @param i Node index
+     * @return 0 in case of success
+     */
+    virtual int prepare_server(int i);
+    int prepare_servers();
+
 private:
 
     int check_node_ssh(int node);
@@ -468,6 +502,13 @@ public:
     {
         return check_galera();
     }
+
+    //int prepare_galera_server(int i);
+
+    //virtual int prepare_server(int i)
+    //{
+    //    return prepare_galera_server(i);
+    //}
 };
 
 #endif // MARIADB_NODES_H

maxscale-system-test/mdbci/backend/create_skysql_user.sql

@@ -1,15 +1,15 @@
-create user skysql@'%' identified by 'skysql';
+#create user skysql@'%' identified by 'skysql';
-create user skysql@'localhost' identified by 'skysql';
+#create user skysql@'localhost' identified by 'skysql';
 GRANT ALL PRIVILEGES ON *.* TO skysql@'%' WITH GRANT OPTION; 
 GRANT ALL PRIVILEGES ON *.* TO skysql@'localhost' WITH GRANT OPTION;
 
-create user maxuser@'%' identified by 'maxpwd';
+#create user maxuser@'%' identified by 'maxpwd';
-create user maxuser@'localhost' identified by 'maxpwd';
+#create user maxuser@'localhost' identified by 'maxpwd';
 GRANT ALL PRIVILEGES ON *.* TO maxuser@'%' WITH GRANT OPTION;
 GRANT ALL PRIVILEGES ON *.* TO maxuser@'localhost' WITH GRANT OPTION;
 
-create user maxskysql@'%' identified by 'skysql';
+#create user maxskysql@'%' identified by 'skysql';
-create user maxskysql@'localhost' identified by 'skysql';
+#create user maxskysql@'localhost' identified by 'skysql';
 GRANT ALL PRIVILEGES ON *.* TO maxskysql@'%' WITH GRANT OPTION;
 GRANT ALL PRIVILEGES ON *.* TO maxskysql@'localhost' WITH GRANT OPTION;
 

maxscale-system-test/mdbci/backend/galera/setup_galera.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-N=$galera_N
-
-x=`expr $N - 1`
-for i in $(seq 0 $x)
-do
-	num=`printf "%03d" $i`
-	sshkey_var=galera_"$num"_keyfile
-	user_var=galera_"$num"_whoami
-	IP_var=galera_"$num"_network
-
-	sshkey=${!sshkey_var}
-	user=${!user_var}
-	IP=${!IP_var}
-
-    ssh -i $sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $user@$IP "sudo mysql_install_db; sudo chown -R mysql:mysql /var/lib/mysql"
-done

maxscale-system-test/mdbci/backend/setup_repl.sh

@@ -1,61 +0,0 @@
-#!/bin/bash
-
-set -x
-
-x=`expr $node_N - 1`
-for i in $(seq 0 $x)
-do
-        num=`printf "%03d" $i`
-        sshkey_var=node_"$num"_keyfile
-        user_var=node_"$num"_whoami
-        IP_var=node_"$num"_network
-	start_cmd_var=node_"$num"_start_db_command
-	stop_cmd_var=node_"$num"_stop_db_command
-
-        sshkey=${!sshkey_var}
-        user=${!user_var}
-        IP=${!IP_var}
-	start_cmd=${!start_cmd_var}
-	stop_cmd=${!stop_cmd_var}
-
-	ssh -i $sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $user@$IP "sudo  $stop_cmd" 
-	sleep 5
-	ssh -i $sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $user@$IP 'sudo sed -i "s/bind-address/#bind-address/g" /etc/mysql/my.cnf'
-	ssh -i $sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $user@$IP 'sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/usr.sbin.mysqld; sudo service apparmor restart'
-
-	mysql_version=`ssh -i $sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $user@$IP  'mysql --version'`
-        echo $mysql_version | grep "5\."
-        if [ $? == 0 ] ; then
-		ssh -i $sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $user@$IP "sudo sed -i \"s/binlog_row_image=full//\" /etc/my.cnf.d/*.cnf"
-        fi
-
-	echo $mysql_version | grep "5\.7"
-	if [ $? == 0 ] ; then
-#		ssh -i $sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $user@$IP "sudo sed -i \"s/## x001/validate-password=OFF/\" /etc/my.cnf.d/*.cnf"
-                ssh -i $sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $user@$IP "sudo mysqld --initialize; sudo chown -R mysql:mysql /var/lib/mysql"
-                ssh -i $sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $user@$IP "sudo $start_cmd" 
-
-		mysql_root_password=`ssh -i $sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $user@$IP "sudo cat /var/log/mysqld.log | grep \"temporary password\" | sed -n -e 's/^.*: //p'"`
-
-		ssh -i $sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $user@$IP "sudo mysqladmin -uroot -p'$mysql_root_password' password '$mysql_root_password'"
-		ssh -i $sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $user@$IP "echo \"UNINSTALL PLUGIN validate_password\" | sudo mysql -uroot -p'$mysql_root_password' "
-
-
-		ssh -i $sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $user@$IP "sudo $stop_cmd"
-#		ssh -i $sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $user@$IP "sudo sed -i \"s/## x001/validate-password=OFF/\" /etc/my.cnf.d/*.cnf"
-
-		ssh -i $sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $user@$IP "sudo $start_cmd"
-
-#		mysql_root_password=`ssh -i $sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $user@$IP "sudo cat /var/log/mysqld.log | grep \"temporary password\" | sed -n -e 's/^.*: //p'"`
-		ssh -i $sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $user@$IP "echo \"show plugins\" | sudo mysql -uroot -p'$mysql_root_password' " 
-		ssh -i $sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $user@$IP "sudo mysqladmin -uroot -p'$mysql_root_password' password ''" 
-#		ssh -i $sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $user@$IP "sudo $start_cmd"
-	else
-		ssh -i $sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $user@$IP "sudo mysql_install_db; sudo chown -R mysql:mysql /var/lib/mysql"
-		ssh -i $sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $user@$IP "sudo $start_cmd" 
-	fi
-	sleep 15
-        scp -i $sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ${script_dir}/create_*_user.sql $user@$IP://home/$user/
-        ssh -i $sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $user@$IP "sudo mysql < /home/$user/create_repl_user.sql"
-        ssh -i $sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $user@$IP "sudo mysql < /home/$user/create_skysql_user.sql"
-done

maxscale-system-test/mdbci/configure_backend.sh

@@ -1,9 +0,0 @@
-. ${script_dir}/set_env.sh $name
-
-${script_dir}/backend/setup_repl.sh
-${script_dir}/backend/galera/setup_galera.sh
-
-${script_dir}/configure_core.sh
-
-rm ~/vagrant_lock
-

maxscale-system-test/mdbci/create_config.sh

@@ -52,7 +52,7 @@ if [ $? != 0 ]; then
 	exit 1
 fi
 
-cp ~/build-scripts/team_keys .
+#cp ~/build-scripts/team_keys .
 ${mdbci_dir}/mdbci public_keys --key ${team_keys} $name
 
 rm ~/vagrant_lock

maxscale-system-test/mdbci/run_test.sh

@@ -58,17 +58,15 @@ export name=`echo $name | sed "s/?//g"`
 
 . ${script_dir}/configure_log_dir.sh
 
-cd ${script_dir}/..
-
-cmake . -DBUILDNAME=$name -DCMAKE_BUILD_TYPE=Debug
-make
-
 ${script_dir}/create_config.sh
 res=$?
 
 if [ $res == 0 ] ; then
-    . ${script_dir}/configure_backend.sh
+#    . ${script_dir}/configure_backend.sh
-    ${mdbci_dir}/mdbci snapshot take --path-to-nodes $name --snapshot-name clean
+    . ${script_dir}/set_env.sh $name
+    cd ${script_dir}/..
+    cmake . -DBUILDNAME=$name -DCMAKE_BUILD_TYPE=Debug
+    make
 
     if [ ! -z "${named_test}" ] ; then
         ./${named_test}
@@ -82,6 +80,7 @@ if [ $res == 0 ] ; then
             rm ~/vagrant_lock
             exit 1
         fi
+        ${mdbci_dir}/mdbci snapshot take --path-to-nodes $name --snapshot-name clean
         ctest -VV -D Nightly ${test_set}
     fi
 

maxscale-system-test/mdbci/templates/nogalera.json.template

@@ -0,0 +1,65 @@
+{
+  "node_000" :
+  {
+        "hostname" : "node000",
+        "box" : "${backend_box}",
+        "memory_size" : "${vm_memory}",
+	"product" : {
+      		"name": "${product}",
+		"version": "${version}",
+                "cnf_template" : "server1.cnf",
+                "cnf_template_path": "${cnf_path}"
+	}
+
+  },
+
+  "node_001" :
+  {
+        "hostname" : "node001",
+        "box" : "${backend_box}",
+        "memory_size" : "${vm_memory}",
+        "product" : {
+                "name": "${product}",
+                "version": "${version}",
+                "cnf_template" : "server2.cnf",
+                "cnf_template_path": "${cnf_path}"
+        }
+  },
+
+  "node_002" :
+  {
+        "hostname" : "node002",
+        "box" : "${backend_box}",
+        "memory_size" : "${vm_memory}",
+        "product" : {
+                "name": "${product}",
+                "version": "${version}",
+                "cnf_template" : "server3.cnf",
+                "cnf_template_path": "${cnf_path}"
+        }
+  },
+
+  "node_003" :
+  {
+        "hostname" : "node003",
+        "box" : "${backend_box}",
+        "memory_size" : "${vm_memory}",
+        "product" : {
+                "name": "${product}",
+                "version": "${version}",
+                "cnf_template" : "server4.cnf",
+                "cnf_template_path": "${cnf_path}"
+        }
+  },
+
+  "maxscale" :
+  {
+        "hostname" : "maxscale",
+        "box" : "${box}",
+        "memory_size" : "${vm_memory}",
+        "product" : {
+                "name": "maxscale"
+        }
+
+  }
+}

maxscale-system-test/testconnections.cpp

@@ -197,13 +197,17 @@ TestConnections::TestConnections(int argc, char *argv[]):
         galera = new Galera_nodes("galera", test_dir, verbose);
         //galera->use_ipv6 = use_ipv6;
         galera->use_ipv6 = false;
+        galera->take_snapshot_command = take_snapshot_command;
+        galera->revert_snapshot_command = revert_snapshot_command;
     }
     else
     {
-        galera = repl;
+        galera = NULL;
     }
 
     repl->use_ipv6 = use_ipv6;
+    repl->take_snapshot_command = take_snapshot_command;
+    repl->revert_snapshot_command = revert_snapshot_command;
 
 
     if (maxscale::required_repl_version.length())
@@ -641,8 +645,14 @@ void TestConnections::process_template(const char *template_name, const char *de
     mdn[0] = repl;
     mdn[1] = galera;
     int i, j;
+    int mdn_n = 1;
 
-    for (j = 0; j < 2; j++)
+    if (galera == NULL)
+    {
+        mdn_n =1;
+    }
+
+    for (j = 0; j < mdn_n; j++)
     {
         for (i = 0; i < mdn[j]->N; i++)
         {
@@ -783,6 +793,8 @@ int TestConnections::copy_mariadb_logs(Mariadb_nodes * repl, const char* prefix)
     int exit_code;
     char str[4096];
 
+    if (repl == NULL) return local_result;
+
     sprintf(str, "mkdir -p LOGS/%s", test_name);
     system(str);
     for (i = 0; i < repl->N; i++)