Add virtual functions check_replication() and start_replication() for Clustrix

Implementations of check_replication() and start_replication() for Clustrix allows to use fix_replication()
also for Clustrix nodes without checking it.
Also several attempts to check nodes after restart are added - to wait for nodes if they are not running
right after server daemon restart

maxscale-system-test/clustrix_nodes.cpp

@@ -3,7 +3,7 @@
 #include <sstream>
 #include "clustrix_nodes.h"
 
-int Clustrix_nodes::install_clustrix(int m)
+int Clustrix_nodes::prepare_server(int m)
 {
     int ec;
     char* clustrix_rpm = ssh_node_output(m, "rpm -qa | grep clustrix-clxnode", true, &ec);
@@ -15,17 +15,22 @@ int Clustrix_nodes::install_clustrix(int m)
         printf("%s\n", ssh_node_output(m, INSTALL_CLUSTRIX, false, &ec));
         create_users(m);
     }
+    else
+    {
+        printf("%s\n", ssh_node_output(m, "systemctl restart clustrix", true, &ec));
+    }
+
     return 0;
 }
 
-int Clustrix_nodes::start_cluster()
+int Clustrix_nodes::start_replication()
 {
     for (int i = 0; i < N; i++)
     {
-        install_clustrix(i);
+        prepare_server(i);
     }
     std::string lic_filename = std::string(getenv("HOME"))
-        + std::string("/.config/mdbci/clustrix_license");
+            + std::string("/.config/mdbci/clustrix_license");
     std::ifstream lic_file;
     lic_file.open(lic_filename.c_str());
     std::stringstream strStream;
@@ -36,13 +41,13 @@ int Clustrix_nodes::start_cluster()
     execute_query_all_nodes(clustrix_license.c_str());
 
     std::string cluster_setup_sql = std::string("ALTER CLUSTER ADD '")
-        + std::string(IP_private[1])
-        + std::string("'");
+            + std::string(IP_private[1])
+            + std::string("'");
     for (int i = 2; i < N; i++)
     {
         cluster_setup_sql += std::string(",'")
-            + std::string(IP_private[i])
-            + std::string("'");
+                + std::string(IP_private[i])
+                + std::string("'");
     }
     connect();
     execute_query(nodes[0], "%s", cluster_setup_sql.c_str());
@@ -56,13 +61,29 @@ std::string Clustrix_nodes::cnf_servers()
     for (int i = 0; i < N; i++)
     {
         s += std::string("\\n[")
-            + cnf_server_name
-            + std::to_string(i + 1)
-            + std::string("]\\ntype=server\\naddress=")
-            + std::string(IP_private[i])
-            + std::string("\\nport=")
-            + std::to_string(port[i])
-            + std::string("\\nprotocol=MySQLBackend\\n");
+                + cnf_server_name
+                + std::to_string(i + 1)
+                + std::string("]\\ntype=server\\naddress=")
+                + std::string(IP_private[i])
+                + std::string("\\nport=")
+                + std::to_string(port[i])
+                + std::string("\\nprotocol=MySQLBackend\\n");
     }
     return s;
 }
+
+int Clustrix_nodes::check_replication()
+{
+    int res = 0;
+    connect();
+    for (int i = 0; i < N; i++)
+    {
+        if (execute_query_count_rows(nodes[i], "select * from system.nodeinfo") != N)
+        {
+            res = 1;
+        }
+    }
+    close_connections();
+
+    return res;
+}