MXS-1509: Initial implementation of test case

Added some code to detect server states in a consistent manner and created
the test. The multi-source replication appeared to cause problems for the
test system which needs to be resolved.

Added --force flags to most direct `mysql` calls to prevent errors from
stopping the processing of remaining commands.

maxscale-system-test/CMakeLists.txt

@@ -510,6 +510,10 @@ add_test_executable(mxs1468.cpp mxs1468 mxs1468 LABELS REPL_BACKEND)
  # https://jira.mariadb.org/browse/MXS-1476
 add_test_executable(mxs1476.cpp mxs1476 mxs1476 LABELS GALERA_BACKEND)
 
+ # MXS-1509: Show correct server state for multisource replication
+ # https://jira.mariadb.org/browse/MXS-1509
+add_test_executable(mxs1509.cpp mxs1509 mxs1509 LABELS REPL_BACKEND)
+
 # 'namedserverfilter' test
 add_test_executable(namedserverfilter.cpp namedserverfilter namedserverfilter LABELS namedserverfilter LIGHT REPL_BACKEND)
 

maxscale-system-test/mariadb_nodes.cpp

@@ -375,9 +375,8 @@ int Mariadb_nodes::stop_nodes()
         printf("Stopping node %d\n", i);
         fflush(stdout);
         local_result += execute_query(nodes[i], "stop slave;");
-        fflush(stdout);
         local_result += stop_node(i);
-        fflush(stdout);
+        local_result += ssh_node(i, true, "rm -f /var/lib/mysql/*master*.info");
     }
     return local_result;
 }
@@ -428,12 +427,12 @@ int Mariadb_nodes::start_replication()
     {
         local_result += start_node(i, "");
         ssh_node(i, true,
-                 "mysql -u root %s -e \"STOP SLAVE; RESET SLAVE; RESET SLAVE ALL; RESET MASTER; SET GLOBAL read_only=OFF;\"",
+                 "mysql --force -u root %s -e \"STOP SLAVE; STOP ALL SLAVES; RESET SLAVE; RESET SLAVE ALL; RESET MASTER; SET GLOBAL read_only=OFF;\"",
                  socket_cmd[i]);
         ssh_node(i, true, "sudo rm -f /etc/my.cnf.d/kerb.cnf");
         ssh_node(i, true,
-                 "for i in `mysql  -ss -u root %s -e \"SHOW DATABASES\"|grep -iv 'mysql\\|information_schema\\|performance_schema'`; "
-                 "do mysql -u root %s -e \"DROP DATABASE $i\";"
+                 "for i in `mysql -ss --force -u root %s -e \"SHOW DATABASES\"|grep -iv 'mysql\\|information_schema\\|performance_schema'`; "
+                 "do mysql --force -u root %s -e \"DROP DATABASE $i\";"
                  "done", socket_cmd[i], socket_cmd[i]);
     }
 
@@ -444,7 +443,7 @@ int Mariadb_nodes::start_replication()
              user_name, password, access_homedir[0], socket_cmd[0]);
 
     // Create a database dump from the master and distribute it to the slaves
-    ssh_node(0, true, "mysql -u root %s -e \"CREATE DATABASE test\"; "
+    ssh_node(0, true, "mysql --force -u root %s -e \"CREATE DATABASE test\"; "
              "mysqldump --all-databases --add-drop-database --flush-privileges --master-data=1 --gtid %s > /tmp/master_backup.sql",
              socket_cmd[0], socket_cmd[0]);
     sprintf(str, "%s/master_backup.sql", test_dir);
@@ -456,9 +455,9 @@ int Mariadb_nodes::start_replication()
         printf("Starting node %d\n", i);
         fflush(stdout);
         copy_to_node(str, "/tmp/master_backup.sql", i);
-        ssh_node(i, true, "mysql -u root %s < /tmp/master_backup.sql",
+        ssh_node(i, true, "mysql --force -u root %s < /tmp/master_backup.sql",
                  socket_cmd[i]);
-        ssh_node(i, true, "mysql -u root %s -e \"CHANGE MASTER TO MASTER_HOST=\\\"%s\\\", MASTER_PORT=%d, "
+        ssh_node(i, true, "mysql --force -u root %s -e \"CHANGE MASTER TO MASTER_HOST=\\\"%s\\\", MASTER_PORT=%d, "
                  "MASTER_USER=\\\"repl\\\", MASTER_PASSWORD=\\\"repl\\\";"
                  "START SLAVE;\"", socket_cmd[i], IP_private[0], port[0]);
     }
@@ -565,8 +564,11 @@ int Mariadb_nodes::unblock_all_nodes()
 int Mariadb_nodes::check_node_ssh(int node)
 {
     int res = 0;
+    if (verbose)
+    {
         printf("Checking node %d\n", node);
         fflush(stdout);
+    }
 
     if (ssh_node(0, false, "ls > /dev/null") != 0)
     {
@@ -574,7 +576,7 @@ int Mariadb_nodes::check_node_ssh(int node)
         fflush(stdout);
         res = 1;
     }
-    else
+    else if (verbose)
     {
         printf("Node %d is OK\n", node);
         fflush(stdout);
@@ -684,6 +686,28 @@ static bool bad_slave_thread_status(MYSQL *conn, const char *field, int node)
     return rval;
 }
 
+static bool multi_source_replication(MYSQL *conn, int node)
+{
+    bool rval = true;
+    MYSQL_RES *res;
+
+    if (mysql_query(conn, "SHOW ALL SLAVES STATUS") == 0 &&
+        (res = mysql_store_result(conn)))
+    {
+        if (mysql_num_rows(res) == 1)
+        {
+            rval = false;
+        }
+        else
+        {
+            printf("Node %d: More than one configured slave\n", node);
+            fflush(stdout);
+        }
+    }
+
+    return rval;
+}
+
 int Mariadb_nodes::check_replication()
 {
     int master = 0;
@@ -713,7 +737,8 @@ int Mariadb_nodes::check_replication()
             }
         }
         else if (bad_slave_thread_status(nodes[i], "Slave_IO_Running", i) ||
-                 bad_slave_thread_status(nodes[i], "Slave_SQL_Running", i))
+                 bad_slave_thread_status(nodes[i], "Slave_SQL_Running", i) ||
+                 multi_source_replication(nodes[i], i))
         {
             res = 1;
         }

maxscale-system-test/mxs1509.cpp

@@ -0,0 +1,95 @@
+/**
+ * MXS-1509: Show correct server state for multisource replication
+ *
+ * https://jira.mariadb.org/browse/MXS-1509
+ */
+
+#include "testconnections.h"
+#include <sstream>
+
+void change_master(TestConnections& test, int slave, int master, const char* name = NULL)
+{
+    std::string source;
+
+    if (name)
+    {
+        source = "'";
+        source += name;
+        source += "'";
+    }
+
+    execute_query(test.repl->nodes[slave], "STOP ALL SLAVES;CHANGE MASTER %s TO master_host='%s', master_port=3306, "
+                  "master_user='%s', master_password='%s', master_use_gtid=slave_pos;START ALL SLAVES",
+                  source.c_str(), test.repl->IP[master], test.repl->user_name, test.repl->password, source.c_str());
+}
+
+const char* dump_status(const StringSet& current, const StringSet& expected)
+{
+    std::stringstream ss;
+    ss << "Current status: (";
+
+    for (const auto& a: current)
+    {
+        ss << a << ",";
+    }
+
+    ss << ") Expected status: (";
+
+    for (const auto& a: expected)
+    {
+        ss << a << ",";
+    }
+
+    ss << ")";
+
+    static std::string res = ss.str();
+    return res.c_str();
+}
+
+void check_status(TestConnections& test, const StringSet& expected_master, const StringSet& expected_slave)
+{
+    sleep(2);
+    StringSet master = test.get_server_status("server1");
+    StringSet slave = test.get_server_status("server2");
+    test.add_result(master != expected_master, "Master status is not what was expected: %s",
+                    dump_status(master, expected_master));
+    test.add_result(slave != expected_slave, "Slave status is not what was expected: %s",
+                    dump_status(slave, expected_slave));
+}
+
+int main(int argc, char** argv)
+{
+    TestConnections test(argc, argv);
+
+    // Stop replication on nodes three and four
+    test.repl->connect();
+    execute_query(test.repl->nodes[2], "STOP ALL SLAVES; RESET SLAVE ALL;");
+    execute_query(test.repl->nodes[3], "STOP ALL SLAVES; RESET SLAVE ALL;");
+
+    // Point the master to an external server
+    change_master(test, 1, 0);
+    change_master(test, 0, 2);
+    check_status(test, {"Master", "Running"}, {"Slave", "Running"});
+
+    // Resetting the slave on master should have no effect
+    execute_query(test.repl->nodes[0], "STOP ALL SLAVES; RESET SLAVE ALL;");
+    check_status(test, {"Master", "Running"}, {"Slave", "Running"});
+
+    // Configure multi-source replication, check that master status is as expected
+    change_master(test, 0, 2, "extra-slave");
+    change_master(test, 1, 2, "extra-slave");
+    check_status(test, {"Master", "Running"}, {"Slave", "Running", "Slave of External Server"});
+
+    // Stopping multi-source replication on slave should remove the Slave of External Server status
+    execute_query(test.repl->nodes[1], "STOP SLAVE 'extra-slave'; RESET SLAVE 'extra-slave';");
+    check_status(test, {"Master", "Running"}, {"Slave", "Running"});
+
+    // Doing the same on the master should have no effect
+    execute_query(test.repl->nodes[0], "STOP ALL SLAVES; RESET SLAVE ALL;");
+    check_status(test, {"Master", "Running"}, {"Slave", "Running"});
+
+    // Cleanup
+    test.repl->execute_query_all_nodes( "STOP ALL SLAVES; RESET SLAVE ALL;");
+    test.repl->fix_replication();
+    return test.global_result;
+}

maxscale-system-test/testconnections.cpp

@@ -1918,6 +1918,46 @@ int TestConnections::find_master_maxadmin(Mariadb_nodes * nodes)
     return master;
 }
 
+StringSet TestConnections::get_server_status(const char* name)
+{
+    std::set<std::string> rval;
+    char* res = ssh_maxscale_output(true, "maxadmin list servers|grep \'%s\'", name);
+    char* pipe = strrchr(res, '|');
+
+    if (res && pipe)
+    {
+        pipe++;
+        char* tok = strtok(pipe, ",");
+
+        while (tok)
+        {
+            char* p = tok;
+            char *end = strchr(tok, '\n');
+            if (!end)
+                end = strchr(tok, '\0');
+
+            // Trim leading whitespace
+            while (p < end && isspace(*p))
+            {
+                p++;
+            }
+
+            // Trim trailing whitespace
+            while (end > tok && isspace(*end))
+            {
+                *end-- = '\0';
+            }
+
+            rval.insert(p);
+            tok = strtok(NULL, ",\n");
+        }
+
+        free(res);
+    }
+
+    return rval;
+}
+
 int TestConnections::find_slave_maxadmin(Mariadb_nodes * nodes)
 {
     int slave = -1;

maxscale-system-test/testconnections.h

@@ -6,6 +6,10 @@
 #include <fcntl.h>
 #include <pthread.h>
 #include <sys/time.h>
+#include <set>
+#include <string>
+
+typedef std::set<std::string> StringSet;
 
 /**
  * @brief Class contains references to Master/Slave and Galera test setups
@@ -657,6 +661,15 @@ public:
     void check_current_operations(int value);
     void check_current_connections(int value);
 
+    /**
+     * @brief Get the set of labels that are assigned to server @c name
+     *
+     * @param name The name of the server that must be present in the output `maxadmin list servers`
+     *
+     * @return A set of string labels assigned to this server
+     */
+    StringSet get_server_status(const char* name);
+
     /**
      * @brief check_maxscale_processes Check if number of running Maxscale processes is equal to 'expected'
      * @param expected expected number of Maxscale processes