MXS-1560 Manual failover with two good candidates

Not two specific good candidates, but all slaves are assumed
are equally good.

maxscale-system-test/.gitignore

@@ -105,6 +105,9 @@ maxscale_process_user
 mm
 mm_mysqlmon
 mysqlmon_failover_manual
+mysqlmon_failover_manual2_4
+mysqlmon_failover_manual2_3
+mysqlmon_failover_manual2_2
 mxs1045
 mxs1071_maxrows
 mxs1110_16mb

maxscale-system-test/CMakeLists.txt

@@ -259,7 +259,9 @@ add_test_executable(mysqlmon_failover_auto.cpp mysqlmon_failover_auto mysqlmon_f
 add_test_executable(mysqlmon_failover_manual.cpp mysqlmon_failover_manual mysqlmon_failover_manual LABELS mysqlmon REPL_BACKEND)
 
 # MySQL Monitor manual failover with one candidate
-add_test_executable(mysqlmon_failover_manual.cpp mysqlmon_failover_manual LABELS mysqlmon REPL_BACKEND)
+add_test_executable(mysqlmon_failover_manual2.cpp mysqlmon_failover_manual2_4 mysqlmon_failover_manual2_4 LABELS mysqlmon REPL_BACKEND)
+add_test_executable(mysqlmon_failover_manual2.cpp mysqlmon_failover_manual2_3 mysqlmon_failover_manual2_3 LABELS mysqlmon REPL_BACKEND)
+add_test_executable(mysqlmon_failover_manual2.cpp mysqlmon_failover_manual2_2 mysqlmon_failover_manual2_2 LABELS mysqlmon REPL_BACKEND)
 
 # Test monitor state change events when manually clearing server bits
 add_test_executable(false_monitor_state_change.cpp false_monitor_state_change replication LABELS mysqlmon REPL_BACKEND)

maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_failover_manual2_2

@@ -0,0 +1,78 @@
+[maxscale]
+threads=###threads###
+
+[MySQL-Monitor]
+type=monitor
+module=mysqlmon
+servers= server1, server2
+user=maxskysql
+passwd= skysql
+monitor_interval=1000
+allow_cluster_recovery=true
+auto_failover=false
+replication_user=repl
+replication_password=repl
+backend_connect_timeout=1
+
+[RW-Split-Router]
+type=service
+router= readwritesplit
+servers=server1, server2
+user=maxskysql
+passwd=skysql
+
+[Read-Connection-Router-Slave]
+type=service
+router=readconnroute
+router_options= slave
+servers=server1, server2
+user=maxskysql
+passwd=skysql
+
+[Read-Connection-Router-Master]
+type=service
+router=readconnroute
+router_options=master
+servers=server1, server2
+user=maxskysql
+passwd=skysql
+
+[RW-Split-Listener]
+type=listener
+service=RW-Split-Router
+protocol=MySQLClient
+port=4006
+
+[Read-Connection-Listener-Slave]
+type=listener
+service=Read-Connection-Router-Slave
+protocol=MySQLClient
+port=4009
+
+[Read-Connection-Listener-Master]
+type=listener
+service=Read-Connection-Router-Master
+protocol=MySQLClient
+port=4008
+
+[CLI]
+type=service
+router=cli
+
+[CLI Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=default
+
+[server1]
+type=server
+address=###node_server_IP_1###
+port=###node_server_port_1###
+protocol=MySQLBackend
+
+[server2]
+type=server
+address=###node_server_IP_2###
+port=###node_server_port_2###
+protocol=MySQLBackend

maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_failover_manual2_3

@@ -0,0 +1,84 @@
+[maxscale]
+threads=###threads###
+
+[MySQL Monitor]
+type=monitor
+module=mysqlmon
+servers= server1, server2, server3
+user=maxskysql
+passwd= skysql
+monitor_interval=1000
+allow_cluster_recovery=true
+auto_failover=false
+replication_user=repl
+replication_password=repl
+backend_connect_timeout=1
+
+[RW Split Router]
+type=service
+router= readwritesplit
+servers=server1, server2, server3
+user=maxskysql
+passwd=skysql
+
+[Read-Connection-Router-Slave]
+type=service
+router=readconnroute
+router_options= slave
+servers=server1, server2, server3
+user=maxskysql
+passwd=skysql
+
+[Read-Connection-Router-Master]
+type=service
+router=readconnroute
+router_options=master
+servers=server1, server2, server3
+user=maxskysql
+passwd=skysql
+
+[RW-Split-Listener]
+type=listener
+service=RW Split Router
+protocol=MySQLClient
+port=4006
+
+[Read-Connection-Listener-Slave]
+type=listener
+service=Read-Connection-Router-Slave
+protocol=MySQLClient
+port=4009
+
+[Read Connection Listener Master]
+type=listener
+service=Read-Connection-Router-Master
+protocol=MySQLClient
+port=4008
+
+[CLI]
+type=service
+router=cli
+
+[CLI Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=default
+
+[server1]
+type=server
+address=###node_server_IP_1###
+port=###node_server_port_1###
+protocol=MySQLBackend
+
+[server2]
+type=server
+address=###node_server_IP_2###
+port=###node_server_port_2###
+protocol=MySQLBackend
+
+[server3]
+type=server
+address=###node_server_IP_3###
+port=###node_server_port_3###
+protocol=MySQLBackend

maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_failover_manual2_4

@@ -0,0 +1,89 @@
+[maxscale]
+threads=###threads###
+
+[MySQL-Monitor]
+type=monitor
+module=mysqlmon
+servers= server1, server2, server3, server4
+user=maxskysql
+passwd= skysql
+monitor_interval=1000
+allow_cluster_recovery=true
+replication_user=repl
+replication_password=repl
+backend_connect_timeout=1
+
+[RW-Split-Router]
+type=service
+router= readwritesplit
+servers=server1, server2, server3, server4
+user=maxskysql
+passwd=skysql
+
+[Read-Connection-Router-Slave]
+type=service
+router=readconnroute
+router_options= slave
+servers=server1, server2, server3, server4
+user=maxskysql
+passwd=skysql
+
+[Read-Connection-Router-Master]
+type=service
+router=readconnroute
+router_options=master
+servers=server1, server2, server3, server4
+user=maxskysql
+passwd=skysql
+
+[RW-Split-Listener]
+type=listener
+service=RW-Split-Router
+protocol=MySQLClient
+port=4006
+
+[Read-Connection-Listener-Slave]
+type=listener
+service=Read-Connection-Router-Slave
+protocol=MySQLClient
+port=4009
+
+[Read-Connection-Listener-Master]
+type=listener
+service=Read-Connection-Router-Master
+protocol=MySQLClient
+port=4008
+
+[CLI]
+type=service
+router=cli
+
+[CLI Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=default
+
+[server1]
+type=server
+address=###node_server_IP_1###
+port=###node_server_port_1###
+protocol=MySQLBackend
+
+[server2]
+type=server
+address=###node_server_IP_2###
+port=###node_server_port_2###
+protocol=MySQLBackend
+
+[server3]
+type=server
+address=###node_server_IP_3###
+port=###node_server_port_3###
+protocol=MySQLBackend
+
+[server4]
+type=server
+address=###node_server_IP_4###
+port=###node_server_port_4###
+protocol=MySQLBackend

maxscale-system-test/mysqlmon_failover_manual2.cpp

@@ -0,0 +1,197 @@
+/*
+ * Copyright (c) 2016 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2020-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#include "testconnections.h"
+#include <exception>
+#include <iostream>
+#include <sstream>
+#include <string>
+
+using std::cerr;
+using std::cout;
+using std::flush;
+using std::endl;
+using std::string;
+using std::stringstream;
+
+namespace
+{
+
+void sleep(int s)
+{
+    cout << "Sleeping " << s << " seconds" << flush;
+    do
+    {
+        ::sleep(1);
+        cout << "." << flush;
+        --s;
+    }
+    while (s > 0);
+
+    cout << endl;
+}
+
+namespace x
+{
+
+void connect_maxscale(TestConnections& test)
+{
+    if (test.maxscales->connect_maxscale(0) != 0)
+    {
+        ++test.global_result;
+        throw std::runtime_error("Could not connect to MaxScale.");
+    }
+}
+
+void try_query(TestConnections& test, const char* zQuery)
+{
+    if (test.try_query(test.maxscales->conn_rwsplit[0], zQuery) != 0)
+    {
+        string s("Could not execute query: ");
+        s += zQuery;
+
+        throw std::runtime_error(s);
+    }
+}
+
+void try_query(TestConnections& test, const std::string& query)
+{
+    try_query(test, query.c_str());
+}
+
+void stop_node(Mariadb_nodes& nodes, int node)
+{
+    if (nodes.stop_node(node) != 0)
+    {
+        throw std::runtime_error("Could not stop node.");
+    }
+}
+
+void fail(void (*f)(TestConnections&), TestConnections& test)
+{
+    bool failed = false;
+    int global_result = test.global_result;
+
+    try
+    {
+        f(test);
+    }
+    catch (const std::exception& x)
+    {
+        test.global_result = global_result;
+        failed = true;
+    }
+
+    if (!failed)
+    {
+        throw std::runtime_error("Function did not fail as expected.");
+    }
+}
+
+}
+
+}
+
+namespace
+{
+
+void list_servers(TestConnections& test)
+{
+    test.maxscales->execute_maxadmin_command_print(0, (char*)"list servers");
+}
+
+void create_table(TestConnections& test)
+{
+    x::try_query(test, "DROP TABLE IF EXISTS test.t1");
+    x::try_query(test, "CREATE TABLE test.t1(id INT)");
+}
+
+void insert_data(TestConnections& test)
+{
+    x::try_query(test, "BEGIN");
+    for (int i = 0; i < 20; ++i)
+    {
+        stringstream ss;
+        ss << "INSERT INTO test.t1 VALUES (" << i << ")";
+        x::try_query(test, ss.str());
+    }
+    x::try_query(test, "COMMIT");
+}
+
+void run(TestConnections& test)
+{
+    sleep(5);
+
+    cout << "Connecting to MaxScale." << endl;
+    x::connect_maxscale(test);
+
+    cout << "Creating table." << endl;
+    create_table(test);
+
+    cout << "Inserting data." << endl;
+    insert_data(test);
+
+    list_servers(test);
+
+    cout << "Syncing slaves." << endl;
+    test.repl->sync_slaves();
+
+    cout << "Stopping master." << endl;
+    x::stop_node(*test.repl, 0);
+
+    list_servers(test);
+
+    cout << "Should fail as master is no longer available, but trying to insert data... " << endl;
+    x::fail(insert_data, test);
+    cout << "Failed as expected." << endl;
+
+    list_servers(test);
+
+    cout << "Performing failover... " << endl;
+    test.maxscales->execute_maxadmin_command_print(0, (char*)"call command mysqlmon failover MySQL-Monitor");
+
+    list_servers(test);
+
+    cout << "Should still fail as there is not transparent master failover, "
+         << "but trying to insert data... " << endl;
+    x::fail(insert_data, test);
+    cout << "Failed as expected." << endl;
+
+    cout << "Closing connection to MaxScale." << endl;
+    test.maxscales->close_maxscale_connections(0);
+
+    cout << "Connecting to MaxScale." << endl;
+    x::connect_maxscale(test);
+
+    cout << "Trying to insert data... " << flush;
+    insert_data(test);
+    cout << "succeeded." << endl;
+}
+
+}
+
+int main(int argc, char** argv)
+{
+    TestConnections test(argc, argv);
+
+    try
+    {
+        run(test);
+    }
+    catch (const std::exception& x)
+    {
+        cerr << "error: Execution was terminated due to an exception: " << x.what() << endl;
+    }
+
+    return test.global_result;
+}