Add virtual functions check_replication() and start_replication() for Clustrix

Implementations of check_replication() and start_replication() for Clustrix allows to use fix_replication()
also for Clustrix nodes without checking it.
Also several attempts to check nodes after restart are added - to wait for nodes if they are not running
right after server daemon restart

maxscale-system-test/clustrix_nodes.cpp

@@ -3,7 +3,7 @@
 #include <sstream>
 #include "clustrix_nodes.h"
 
-int Clustrix_nodes::install_clustrix(int m)
+int Clustrix_nodes::prepare_server(int m)
 {
     int ec;
     char* clustrix_rpm = ssh_node_output(m, "rpm -qa | grep clustrix-clxnode", true, &ec);
@@ -15,17 +15,22 @@ int Clustrix_nodes::install_clustrix(int m)
         printf("%s\n", ssh_node_output(m, INSTALL_CLUSTRIX, false, &ec));
         create_users(m);
     }
+    else
+    {
+        printf("%s\n", ssh_node_output(m, "systemctl restart clustrix", true, &ec));
+    }
+
     return 0;
 }
 
-int Clustrix_nodes::start_cluster()
+int Clustrix_nodes::start_replication()
 {
     for (int i = 0; i < N; i++)
     {
-        install_clustrix(i);
+        prepare_server(i);
     }
     std::string lic_filename = std::string(getenv("HOME"))
-        + std::string("/.config/mdbci/clustrix_license");
+            + std::string("/.config/mdbci/clustrix_license");
     std::ifstream lic_file;
     lic_file.open(lic_filename.c_str());
     std::stringstream strStream;
@@ -36,13 +41,13 @@ int Clustrix_nodes::start_cluster()
     execute_query_all_nodes(clustrix_license.c_str());
 
     std::string cluster_setup_sql = std::string("ALTER CLUSTER ADD '")
-        + std::string(IP_private[1])
-        + std::string("'");
+            + std::string(IP_private[1])
+            + std::string("'");
     for (int i = 2; i < N; i++)
     {
         cluster_setup_sql += std::string(",'")
-            + std::string(IP_private[i])
-            + std::string("'");
+                + std::string(IP_private[i])
+                + std::string("'");
     }
     connect();
     execute_query(nodes[0], "%s", cluster_setup_sql.c_str());
@@ -56,13 +61,29 @@ std::string Clustrix_nodes::cnf_servers()
     for (int i = 0; i < N; i++)
     {
         s += std::string("\\n[")
-            + cnf_server_name
-            + std::to_string(i + 1)
-            + std::string("]\\ntype=server\\naddress=")
-            + std::string(IP_private[i])
-            + std::string("\\nport=")
-            + std::to_string(port[i])
-            + std::string("\\nprotocol=MySQLBackend\\n");
+                + cnf_server_name
+                + std::to_string(i + 1)
+                + std::string("]\\ntype=server\\naddress=")
+                + std::string(IP_private[i])
+                + std::string("\\nport=")
+                + std::to_string(port[i])
+                + std::string("\\nprotocol=MySQLBackend\\n");
     }
     return s;
 }
+
+int Clustrix_nodes::check_replication()
+{
+    int res = 0;
+    connect();
+    for (int i = 0; i < N; i++)
+    {
+        if (execute_query_count_rows(nodes[i], "select * from system.nodeinfo") != N)
+        {
+            res = 1;
+        }
+    }
+    close_connections();
+
+    return res;
+}

maxscale-system-test/clustrix_nodes.h

@@ -30,22 +30,28 @@ public:
     {
     }
 
-    /**
-     * @brief install_clustrix
-     * @param m node index
-     * @return 0 in case of success
-     */
-    int install_clustrix(int m);
-
     /**
      * @brief start_cluster Intstalls Clustrix on all nodes, configure license, form cluster
      * @return 0 in case of success
      */
-    int start_cluster();
+    int start_replication();
 
     /**
      * @brief cnf_servers Generate Clustrix servers description for maxscale.cnf
      * @return text for maxscale.cnf
      */
     std::string cnf_servers();
+
+    /**
+     * @brief check_replication Checks if Clustrix Cluster is up and running
+     * @return 0 if Clustrix Cluster is ok
+     */
+    int check_replication();
+
+    /**
+     * @brief install_clustrix
+     * @param m node index
+     * @return 0 in case of success
+     */
+    int prepare_server(int i);
 };

maxscale-system-test/mariadb_nodes.cpp

@@ -751,6 +751,7 @@ int Mariadb_nodes::check_replication()
 bool Mariadb_nodes::fix_replication()
 {
     bool rval = true;
+    int attempts = 25;
 
     if (check_replication())
     {
@@ -764,6 +765,12 @@ bool Mariadb_nodes::fix_replication()
             cout << "Starting replication" << endl;
             start_replication();
 
+            while (check_replication() && (attempts > 0))
+            {
+                cout << "Replication is still broken, waiting" << endl;
+                sleep(10);
+                attempts--;
+            }
             if (check_replication() == 0)
             {
                 cout << "Replication is fixed" << endl;

maxscale-system-test/testconnections.cpp

@@ -402,7 +402,7 @@ TestConnections::TestConnections(int argc, char* argv[])
         clustrix->use_ipv6 = false;
         clustrix->take_snapshot_command = take_snapshot_command;
         clustrix->revert_snapshot_command = revert_snapshot_command;
-        clustrix->start_cluster();
+        clustrix->fix_replication();
     }
     else
     {