Add MariaDB-Monitor multimaster failover/switchover test

maxscale-system-test/CMakeLists.txt

@@ -31,7 +31,8 @@ add_library(testcore SHARED testconnections.cpp nodes.cpp mariadb_nodes.cpp maxs
   labels_table.cpp envv.cpp clustrix_nodes.cpp maxrest.cc
   # Include the CDC connector in the core library
   ${CMAKE_SOURCE_DIR}/connectors/cdc-connector/cdc_connector.cpp)
-target_link_libraries(testcore ${MARIADB_CONNECTOR_LIBRARIES} ${JANSSON_LIBRARIES} z m pthread ssl dl rt crypto crypt maxbase)
+target_link_libraries(testcore
+        ${MARIADB_CONNECTOR_LIBRARIES} ${JANSSON_LIBRARIES} maxbase maxsql z m pthread ssl dl rt crypto crypt)
 install(TARGETS testcore DESTINATION system-test)
 add_dependencies(testcore connector-c jansson maxbase)
 
@@ -290,6 +291,9 @@ add_test_executable(mysqlmon_failover_readonly.cpp mysqlmon_failover_readonly my
 # MariaDB-Monitor enforce_simple_topology
 add_test_executable(mysqlmon_enforce_simple.cpp mysqlmon_enforce_simple mysqlmon_enforce_simple LABELS mysqlmon REPL_BACKEND)
 
+# MariaDB-Monitor multimaster failover/switchover
+add_test_executable(mysqlmon_fail_switch_multi.cpp mysqlmon_fail_switch_multi mysqlmon_fail_switch_multi LABELS mysqlmon REPL_BACKEND)
+
 # MXS-1506: Delayed query retry
 # https://jira.mariadb.org/browse/MXS-1506
 add_test_executable(mxs1506_delayed_retry.cpp mxs1506_delayed_retry mxs1506_delayed_retry LABELS readwritesplit REPL_BACKEND)

maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_fail_switch_multi

@@ -0,0 +1,55 @@
+[maxscale]
+threads=###threads###
+
+[MariaDB-Monitor]
+type=monitor
+module=mysqlmon
+servers= server1, server2
+user=maxskysql
+password= skysql
+monitor_interval=1000
+detect_standalone_master=true
+auto_failover=true
+auto_rejoin=true
+replication_user=repl
+replication_password=repl
+backend_connect_timeout=10
+backend_read_timeout=10
+backend_write_timeout=10
+failcount=1
+
+[RW-Split-Router]
+type=service
+router= readwritesplit
+servers=server1, server2
+user=maxskysql
+password=skysql
+
+[RW-Split-Listener]
+type=listener
+service=RW-Split-Router
+protocol=MySQLClient
+port=4006
+
+[CLI]
+type=service
+router=cli
+
+[CLI-Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=default
+
+[server1]
+type=server
+address=###node_server_IP_1###
+port=###node_server_port_1###
+protocol=MySQLBackend
+
+[server2]
+type=server
+address=###node_server_IP_2###
+port=###node_server_port_2###
+protocol=MySQLBackend
+

maxscale-system-test/mysqlmon_fail_switch_multi.cpp

@@ -0,0 +1,197 @@
+/*
+ * Copyright (c) 2019 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2026-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#include "testconnections.h"
+#include "fail_switch_rejoin_common.cpp"
+#include <iostream>
+#include <string>
+#include <vector>
+#include <maxsql/queryresult.hh>
+
+// Test failover/switchover with multiple masters.
+
+using std::string;
+using std::cout;
+using mxq::QueryResult;
+
+void change_master(TestConnections& test ,int slave, int master, const string& conn_name = "",
+                   int replication_delay = 0);
+
+void reset_master(TestConnections& test ,int slave, const string& conn_name = "");
+void expect_replicating_from(TestConnections& test, int node, int master);
+
+string server_names[] = {"server1", "server2", "server3", "server4"};
+
+int main(int argc, char** argv)
+{
+    Mariadb_nodes::require_gtid(true);
+    TestConnections test(argc, argv);
+    //delete_slave_binlogs(test);
+    // Set up test table
+    basic_test(test);
+    // Advance gtid:s a bit to so gtid variables are updated.
+    MYSQL* maxconn = test.maxscales->open_rwsplit_connection(0);
+    generate_traffic_and_check(test, maxconn, 1);
+    test.repl->sync_slaves(0);
+    get_output(test);
+    print_gtids(test);
+    int ec = -1;
+    auto expect_server_status = [&test](const string& server_name, const string& status) {
+        bool found = (test.maxscales->get_server_status(server_name.c_str()).count(status) == 1);
+        test.expect(found, "%s was not %s as was expected.", server_name.c_str(), status.c_str());
+    };
+
+    auto expect_server_status_multi =
+            [&test, &expect_server_status](std::vector<string> expected) {
+                int expected_size = expected.size();
+                test.expect(expected_size <= test.repl->N && expected_size <= 3, "Too many expected values.");
+                int tests = (expected_size < test.repl->N) ? expected_size : test.repl->N;
+                for (int node = 0; node < tests; node++)
+                {
+                    expect_server_status(server_names[node], expected[node]);
+                }
+            };
+
+    auto mon_wait = [&test](int ticks) {
+        test.maxscales->wait_for_monitor(ticks);
+    };
+
+    string master = "Master";
+    string slave = "Slave";
+    string down = "Down";
+    string relay = "Relay Master";
+    string running = "Running";
+    string ext_master = "Slave of External Server";
+
+    const string secondary_slave_conn = "b";
+    // Only monitoring two servers for now. Stop replication to non-monitored servers.
+    reset_master(test, 2);
+    reset_master(test, 3);
+
+    cout << "Step 1: All should be cool.\n";
+    get_output(test);
+    expect_server_status_multi({master, slave});
+
+    if (test.ok())
+    {
+        cout << "Step 2: External replication to two servers\n";
+        change_master(test, 0, 2);
+        change_master(test, 0, 3, secondary_slave_conn);
+        mon_wait(1);
+        get_output(test);
+        expect_server_status_multi({master, slave});
+        expect_server_status(server_names[0], ext_master);
+        expect_replicating_from(test, 0, 2);
+        expect_replicating_from(test, 0, 3);
+    }
+
+    if (test.ok())
+    {
+        cout << "Step 3: Failover. Check that new master replicates from external servers.\n";
+        test.repl->stop_node(0);
+        mon_wait(3);
+	    get_output(test);
+        expect_server_status_multi({down, master});
+        expect_server_status(server_names[1], ext_master);
+        expect_replicating_from(test, 1, 2);
+        expect_replicating_from(test, 1, 3);
+    }
+
+    if (test.ok())
+    {
+        cout << "Step 4: Bring up old master, allow it to rejoin, then switchover. "
+                "Check that new master replicates from external servers.\n";
+        test.repl->start_node(0);
+        mon_wait(2); // Should not rejoin since has multiple slave connections.
+        expect_server_status_multi({running, master});
+	    test.repl->connect();
+        reset_master(test, 0);
+        reset_master(test, 0, secondary_slave_conn);
+        mon_wait(2);
+	    get_output(test);
+        expect_server_status_multi({slave, master});
+        test.maxscales->ssh_node_output(0, "maxctrl call command mariadbmon switchover MariaDB-Monitor",
+                                        true, &ec);
+        mon_wait(2);
+        expect_replicating_from(test, 0, 2);
+        expect_replicating_from(test, 0, 3);
+	    get_output(test);
+
+        // TODO: Extend test later
+    }
+
+
+    // Cleanup
+    mysql_close(maxconn);
+    test.repl->connect();
+    // Delete the test table from all databases, reset replication.
+    const char drop_query[] = "DROP TABLE IF EXISTS test.t1;";
+    for (int i = 0; i < 3; i++)
+    {
+        test.try_query(test.repl->nodes[i], drop_query);
+    }
+    test.maxscales->ssh_node_output(0, "maxctrl call command mariadbmon reset-replication "
+		                               "MariaDB-Monitor server1", true, &ec);
+    return test.global_result;
+}
+
+void change_master(TestConnections& test ,int slave, int master, const string& conn_name,
+                   int replication_delay)
+{
+    const char query[] = "CHANGE MASTER '%s' TO master_host='%s', master_port=%d, "
+                         "MASTER_USE_GTID = current_pos, "
+                         "master_user='repl', master_password='repl', master_delay=%d; "
+                         "START SLAVE '%s';";
+    test.try_query(test.repl->nodes[slave], query, conn_name.c_str(),
+                   test.repl->IP[master], test.repl->port[master],
+                   replication_delay, conn_name.c_str());
+}
+
+void reset_master(TestConnections& test ,int slave, const string& conn_name)
+{
+    const char query[] = "STOP SLAVE '%s'; RESET SLAVE '%s' ALL;";
+    test.try_query(test.repl->nodes[slave], query, conn_name.c_str(), conn_name.c_str());
+}
+
+void expect_replicating_from(TestConnections& test, int node, int master)
+{
+    bool found = false;
+    auto N = test.repl->N;
+    if (node < N && master < N)
+    {
+        auto conn = test.repl->nodes[node];
+        if (mysql_query(conn, "SHOW ALL SLAVES STATUS;") == 0)
+        {
+            auto res = mysql_store_result(conn);
+            if (res)
+            {
+                QueryResult q_res(res);
+		        auto search_host = test.repl->ip(master);
+		        auto search_port = test.repl->port[master];
+                while (q_res.next_row())
+                {
+                    auto host = q_res.get_string("Master_Host");
+		            auto port = q_res.get_int("Master_Port");
+                    if (host == search_host && port == search_port)
+                    {
+                        found = true;
+                        break;
+                    }
+                }
+            }
+        }
+    }
+
+    test.expect(found, "Server %s is not replicating from %s.",
+	            server_names[node].c_str(), server_names[master].c_str());
+}