Add test case for ignore_external_masters + failover

The test checks that failover works even when the master of the monitored
cluster is a slave to an external masters. The test also verifies that the
servers do not get unexpected status labels.

maxscale-system-test/CMakeLists.txt

@@ -308,6 +308,9 @@ add_test_executable(mysqlmon_failover_stress.cpp mysqlmon_failover_stress mysqlm
 # MySQL Monitor switchover stress
 add_test_executable(mysqlmon_switchover_stress.cpp mysqlmon_switchover_stress mysqlmon_switchover_stress LABELS mysqlmon REPL_BACKEND)
 
+# Check monitoring and failover when ignore_external_masters is enabled
+add_test_executable(mysqlmon_external_master.cpp mysqlmon_external_master mysqlmon_external_master LABELS mysqlmon REPL_BACKEND)
+
 # Test monitor state change events when manually clearing server bits
 add_test_executable(false_monitor_state_change.cpp false_monitor_state_change replication LABELS mysqlmon REPL_BACKEND)
 

maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_external_master

@@ -0,0 +1,59 @@
+[maxscale]
+threads=###threads###
+
+[MySQL-Monitor]
+type=monitor
+module=mysqlmon
+servers=server1,server2,server3
+user=maxskysql
+passwd=skysql
+monitor_interval=1000
+detect_standalone_master=true
+failcount=1
+allow_cluster_recovery=true
+auto_failover=true
+auto_rejoin=true
+replication_user=repl
+replication_password=repl
+ignore_external_masters=true
+
+[RW Split Router]
+type=service
+router=readwritesplit
+servers=server1,server2,server3
+user=maxskysql
+passwd=skysql
+
+[RW Split Listener]
+type=listener
+service=RW Split Router
+protocol=MySQLClient
+port=4006
+
+[CLI]
+type=service
+router=cli
+
+[CLI Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=default
+
+[server1]
+type=server
+address=###node_server_IP_1###
+port=###node_server_port_1###
+protocol=MySQLBackend
+
+[server2]
+type=server
+address=###node_server_IP_2###
+port=###node_server_port_2###
+protocol=MySQLBackend
+
+[server3]
+type=server
+address=###node_server_IP_3###
+port=###node_server_port_3###
+protocol=MySQLBackend

maxscale-system-test/failover_common.cpp

@@ -8,18 +8,7 @@ using std::endl;
 
 void replicate_from(TestConnections& test, int server_ind, int target_ind)
 {
-    stringstream change_master;
+    test.repl->replicate_from(server_ind, target_ind);
-    change_master << "CHANGE MASTER TO MASTER_HOST = '" << test.repl->IP[target_ind]
-        << "', MASTER_PORT = " << test.repl->port[target_ind] << ", MASTER_USE_GTID = current_pos, "
-        "MASTER_USER='repl', MASTER_PASSWORD='repl';";
-    cout << "Server " << server_ind + 1 << " starting to replicate from server " << target_ind + 1 << endl;
-    if (test.verbose)
-    {
-       cout << "Query is '" << change_master.str() << "'" << endl;
-    }
-    execute_query(test.repl->nodes[server_ind], "STOP SLAVE;");
-    execute_query(test.repl->nodes[server_ind], change_master.str().c_str());
-    execute_query(test.repl->nodes[server_ind], "START SLAVE;");
 }
 
 void reset_replication(TestConnections& test)

maxscale-system-test/mariadb_nodes.cpp

@@ -15,6 +15,7 @@
 #include <climits>
 #include <string>
 #include <sstream>
+#include <iostream>
 #include <vector>
 
 namespace
@@ -1457,3 +1458,21 @@ int Mariadb_nodes::prepare_servers()
     }
     return rval;
 }
+
+void Mariadb_nodes::replicate_from(int slave, int master, const char* type)
+{
+    std::stringstream change_master;
+    change_master << "CHANGE MASTER TO MASTER_HOST = '" << IP[master]
+        << "', MASTER_PORT = " << port[master] << ", MASTER_USE_GTID = " << type << ", "
+        "MASTER_USER='repl', MASTER_PASSWORD='repl';";
+
+    if (verbose)
+    {
+        std::cout << "Server " << slave + 1 << " starting to replicate from server " << master + 1 << std::endl;
+       std::cout << "Query is '" << change_master.str() << "'" << std::endl;
+    }
+
+    execute_query(nodes[slave], "STOP SLAVE;");
+    execute_query(nodes[slave], change_master.str().c_str());
+    execute_query(nodes[slave], "START SLAVE;");
+}

maxscale-system-test/mariadb_nodes.h

@@ -430,6 +430,18 @@ public:
     /** Whether to require GTID based replication, defaults to false */
     static void require_gtid(bool value);
 
+    /**
+     * Configure a server as a slave of another server
+     *
+     * The servers are configured with GTID replicating using the configured
+     * GTID position, either slave_pos or current_pos.
+     *
+     * @param slave  The node index to assign as slave
+     * @param master The node index of the master
+     * @param type   Replication type
+     */
+    void replicate_from(int slave, int master, const char* type = "current_pos");
+
 private:
 
     bool check_master_node(MYSQL *conn);

maxscale-system-test/mxs1509.cpp

@@ -25,28 +25,6 @@ void change_master(TestConnections& test, int slave, int master, const char* nam
                   source.c_str(), test.repl->IP[master], test.repl->user_name, test.repl->password, source.c_str());
 }
 
-std::string dump_status(const StringSet& current, const StringSet& expected)
-{
-    std::stringstream ss;
-    ss << "Current status: (";
-
-    for (const auto& a: current)
-    {
-        ss << a << ",";
-    }
-
-    ss << ") Expected status: (";
-
-    for (const auto& a: expected)
-    {
-        ss << a << ",";
-    }
-
-    ss << ")";
-
-    return ss.str();
-}
-
 void check_status(TestConnections& test, const StringSet& expected_master, const StringSet& expected_slave)
 {
     sleep(2);

maxscale-system-test/mysqlmon_external_master.cpp

@@ -0,0 +1,114 @@
+/**
+ * Test monitoring and failover with ignore_external_masters=true
+ */
+#include "testconnections.h"
+
+#include <thread>
+
+#define DOWN                     "Down"
+#define RUNNING                  "Running"
+#define MASTER                   "Master"
+#define SLAVE                    "Slave"
+
+const StringSet master_running = {MASTER, RUNNING};
+const StringSet slave_running  = {SLAVE, RUNNING};
+const StringSet running        = {RUNNING};
+const StringSet down           = {DOWN};
+
+void check_status(TestConnections& test, const char* server, const StringSet& expected, const char* message)
+{
+    StringSet state = test.get_server_status(server);
+    test.assert(state == expected, "%s: %s", message, dump_status(state, expected).c_str());
+}
+
+static bool is_running = true;
+
+void writer_func(TestConnections* test)
+{
+    while (is_running)
+    {
+        MYSQL* conn = open_conn(test->maxscales->rwsplit_port[0], test->maxscales->IP[0],
+                                "test", "test", false);
+
+        for (int i = 0; i < 100; i++)
+        {
+            if (execute_query_silent(conn, "INSERT INTO test.t1 VALUES (SELECT SLEEP(0.5))"))
+            {
+                sleep(1);
+                break;
+            }
+        }
+
+        mysql_close(conn);
+    }
+}
+
+int main(int argc, char** argv)
+{
+    Mariadb_nodes::require_gtid(true);
+    TestConnections test(argc, argv);
+
+    // Create a table and a user and start a thread that does writes
+    test.repl->connect();
+    execute_query(test.repl->nodes[0], "CREATE OR REPLACE TABLE test.t1 (id INT)");
+    execute_query(test.repl->nodes[0], "DROP USER IF EXISTS 'test'@'%%'");
+    execute_query(test.repl->nodes[0], "CREATE USER 'test'@'%%' IDENTIFIED BY 'test'");
+    execute_query(test.repl->nodes[0], "GRANT INSERT, SELECT, UPDATE, DELETE ON *.* TO 'test'@'%%'");
+    test.repl->sync_slaves();
+    std::thread thr(writer_func, &test);
+
+    test.tprintf("Start by having the current master replicate from the external server");
+    test.repl->connect();
+    test.repl->replicate_from(0, 3);
+    sleep(5);
+    check_status(test, "server1", master_running, "server1 should be the master");
+    check_status(test, "server2", slave_running, "server2 should be a slave");
+    check_status(test, "server3", slave_running, "server3 should be a slave");
+
+    test.tprintf("Stop server1, expect server2 to be promoted as the master");
+    test.repl->stop_node(0);
+    sleep(10);
+
+    check_status(test, "server1", down, "server1 should be down");
+    check_status(test, "server2", master_running, "server2 should be the master");
+    check_status(test, "server3", slave_running, "server3 should be a slave");
+
+    test.tprintf("Configure master-master replication between server2 and the external server");
+    test.repl->replicate_from(1, 3);
+    test.repl->replicate_from(3, 1);
+    sleep(10);
+    check_status(test, "server2", master_running, "server2 should still be the master");
+    check_status(test, "server3", slave_running, "server3 should be a slave");
+
+    test.tprintf("Start server1, expect it to rejoin the cluster");
+    test.repl->start_node(0);
+    sleep(10);
+    check_status(test, "server1", slave_running, "server1 should be a slave");
+    check_status(test, "server2", master_running, "server2 should still be the master");
+    check_status(test, "server3", slave_running, "server3 should be a slave");
+
+    test.tprintf("Stop server2, expect server1 to be promoted as the master");
+    test.repl->stop_node(1);
+    test.repl->connect();
+    test.repl->replicate_from(0, 3);
+    test.repl->replicate_from(3, 0);
+    sleep(10);
+
+    check_status(test, "server1", master_running, "server1 should be the master");
+    check_status(test, "server2", down, "server2 should be down");
+    check_status(test, "server3", slave_running, "server3 should be a slave");
+
+    test.tprintf("Start server2, expect it to rejoin the cluster");
+    test.repl->start_node(1);
+    sleep(10);
+    check_status(test, "server1", master_running, "server1 should still be the master");
+    check_status(test, "server2", slave_running, "server2 should be a slave");
+    check_status(test, "server3", slave_running, "server3 should be a slave");
+
+    // Cleanup
+    is_running = false;
+    thr.join();
+    execute_query(test.repl->nodes[0], "STOP SLAVE; RESET SLAVE ALL;");
+
+    return test.global_result;
+}

maxscale-system-test/testconnections.cpp

@@ -6,6 +6,7 @@
 #include <time.h>
 #include <signal.h>
 #include <execinfo.h>
+#include <sstream>
 
 #include "mariadb_func.h"
 #include "maxadmin_operations.h"
@@ -1884,3 +1885,25 @@ bool TestConnections::test_bad_config(int m, const char *config)
     return maxscales->ssh_node(m, "cp maxscale.cnf /etc/maxscale.cnf; service maxscale stop; "
                                "maxscale -U maxscale -lstdout &> /dev/null && sleep 1 && pkill -9 maxscale", false) == 0;
 }
+
+std::string dump_status(const StringSet& current, const StringSet& expected)
+{
+    std::stringstream ss;
+    ss << "Current status: (";
+
+    for (const auto& a: current)
+    {
+        ss << a << ",";
+    }
+
+    ss << ") Expected status: (";
+
+    for (const auto& a: expected)
+    {
+        ss << a << ",";
+    }
+
+    ss << ")";
+
+    return ss.str();
+}

maxscale-system-test/testconnections.h

@@ -531,4 +531,14 @@ void * timeout_thread(void *ptr );
  */
 void * log_copy_thread(void *ptr );
 
+/**
+* Dump two server status sets as strings
+*
+* @param current  The current status
+* @param expected The expected status
+*
+* @return String form comparison of status sets
+*/
+std::string dump_status(const StringSet& current, const StringSet& expected);
+
 #endif // TESTCONNECTIONS_H