MXS-1567 Rolling master promotion

- The test starts with the usual setup of 1 master and 3 slaves.
- Then the master is taken down and it is checked that the failover
  mechanism promotes some slave to master.
- This is continued until there is a single master left (no
  slaves).

maxscale-system-test/CMakeLists.txt

@@ -258,7 +258,7 @@ add_test_executable(mysqlmon_failover_auto.cpp mysqlmon_failover_auto mysqlmon_f
 # MySQL Monitor Failover (manual) Test
 add_test_executable(mysqlmon_failover_manual.cpp mysqlmon_failover_manual mysqlmon_failover_manual LABELS mysqlmon REPL_BACKEND)
 
-# MySQL Monitor manual failover with one candidate
+# MySQL Monitor manual failover with many valid candidates
 add_test_executable(mysqlmon_failover_manual2.cpp mysqlmon_failover_manual2_4 mysqlmon_failover_manual2_4 LABELS mysqlmon REPL_BACKEND)
 add_test_executable(mysqlmon_failover_manual2.cpp mysqlmon_failover_manual2_3 mysqlmon_failover_manual2_3 LABELS mysqlmon REPL_BACKEND)
 add_test_executable(mysqlmon_failover_manual2.cpp mysqlmon_failover_manual2_2 mysqlmon_failover_manual2_2 LABELS mysqlmon REPL_BACKEND)
@@ -266,6 +266,9 @@ add_test_executable(mysqlmon_failover_manual2.cpp mysqlmon_failover_manual2_2 my
 # MySQL Monitor Rejoin Test
 add_test_executable(mysqlmon_rejoin_good.cpp mysqlmon_rejoin_good mysqlmon_rejoin_good LABELS mysqlmon REPL_BACKEND)
 
+# MySQL Monitor rolling master
+add_test_executable(mysqlmon_failover_rolling_master.cpp mysqlmon_failover_rolling_master mysqlmon_failover_rolling_master LABELS mysqlmon REPL_BACKEND)
+
 # Test monitor state change events when manually clearing server bits
 add_test_executable(false_monitor_state_change.cpp false_monitor_state_change replication LABELS mysqlmon REPL_BACKEND)
 

maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_failover_rolling_master

@@ -0,0 +1,91 @@
+[maxscale]
+threads=###threads###
+
+[MySQL-Monitor]
+type=monitor
+module=mysqlmon
+servers= server1, server2, server3, server4
+user=maxskysql
+passwd= skysql
+monitor_interval=1000
+allow_cluster_recovery=true
+detect_standalone_master=true
+auto_failover=true
+replication_user=repl
+replication_password=repl
+backend_connect_timeout=1
+
+[RW-Split-Router]
+type=service
+router= readwritesplit
+servers=server1, server2, server3, server4
+user=maxskysql
+passwd=skysql
+
+[Read-Connection-Router-Slave]
+type=service
+router=readconnroute
+router_options= slave
+servers=server1, server2, server3, server4
+user=maxskysql
+passwd=skysql
+
+[Read-Connection-Router-Master]
+type=service
+router=readconnroute
+router_options=master
+servers=server1, server2, server3, server4
+user=maxskysql
+passwd=skysql
+
+[RW-Split-Listener]
+type=listener
+service=RW-Split-Router
+protocol=MySQLClient
+port=4006
+
+[Read-Connection-Listener-Slave]
+type=listener
+service=Read-Connection-Router-Slave
+protocol=MySQLClient
+port=4009
+
+[Read-Connection-Listener-Master]
+type=listener
+service=Read-Connection-Router-Master
+protocol=MySQLClient
+port=4008
+
+[CLI]
+type=service
+router=cli
+
+[CLI Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=default
+
+[server1]
+type=server
+address=###node_server_IP_1###
+port=###node_server_port_1###
+protocol=MySQLBackend
+
+[server2]
+type=server
+address=###node_server_IP_2###
+port=###node_server_port_2###
+protocol=MySQLBackend
+
+[server3]
+type=server
+address=###node_server_IP_3###
+port=###node_server_port_3###
+protocol=MySQLBackend
+
+[server4]
+type=server
+address=###node_server_IP_4###
+port=###node_server_port_4###
+protocol=MySQLBackend

maxscale-system-test/mysqlmon_failover_rolling_master.cpp

@@ -0,0 +1,271 @@
+/*
+ * Copyright (c) 2016 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2020-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#include <iostream>
+#include <string>
+#include <sstream>
+#include "testconnections.h"
+
+using std::cerr;
+using std::cout;
+using std::endl;
+using std::flush;
+using std::string;
+using std::stringstream;
+
+namespace
+{
+
+void sleep(int s)
+{
+    cout << "Sleeping " << s << " times 1 second" << flush;
+    do
+    {
+        ::sleep(1);
+        cout << "." << flush;
+        --s;
+    }
+    while (s > 0);
+
+    cout << endl;
+}
+
+int get_server_id(Maxscales& maxscales)
+{
+    MYSQL *conn = maxscales.open_rwsplit_connection(0);
+    int id = -1;
+    char str[1024];
+
+    if (find_field(conn, "SELECT @@server_id, @@last_insert_id;", "@@server_id", str) == 0)
+    {
+        id = atoi(str);
+    }
+
+    mysql_close(conn);
+
+    if (id == -1)
+    {
+        throw std::runtime_error("Could not get server id.");
+    }
+
+    return id;
+}
+
+
+}
+
+namespace
+{
+
+class XTestConnections : private TestConnections
+{
+public:
+    using TestConnections::add_result;
+    using TestConnections::global_result;
+    using TestConnections::maxscales;
+    using TestConnections::repl;
+
+    XTestConnections(int argc, char* argv[])
+        : TestConnections(argc, argv)
+    {
+    }
+
+    TestConnections& nothrow()
+    {
+        return *this;
+    }
+
+    void connect_maxscale(int m = 0)
+    {
+        if (maxscales->connect_maxscale(m) != 0)
+        {
+            ++global_result;
+            throw std::runtime_error("Could not connect to MaxScale.");
+        }
+    }
+
+    void try_query(MYSQL *conn, const char *format, ...)
+    {
+        va_list valist;
+
+        va_start(valist, format);
+        int message_len = vsnprintf(NULL, 0, format, valist);
+        va_end(valist);
+
+        char sql[message_len + 1];
+
+        va_start(valist, format);
+        vsnprintf(sql, sizeof(sql), format, valist);
+        va_end(valist);
+
+        int res = execute_query_silent(conn, sql, false);
+        add_result(res, "Query '%.*s%s' failed!\n", message_len < 100 ? message_len : 100, sql,
+                   message_len < 100 ? "" : "...");
+
+        if (res != 0)
+        {
+            string s("Could not execute query: ");
+            s += sql;
+
+            if (s.length() > 80)
+            {
+                s = s.substr(0, 77);
+                s += "...";
+            }
+
+            throw std::runtime_error(s.c_str());
+        }
+    }
+};
+
+void list_servers(XTestConnections& test)
+{
+    cout << endl;
+    test.maxscales->execute_maxadmin_command_print(0, (char*)"list servers");
+}
+
+}
+
+namespace
+{
+
+void create_table(XTestConnections& test)
+{
+    test.try_query(test.maxscales->conn_rwsplit[0], "DROP TABLE IF EXISTS test.t1");
+    test.try_query(test.maxscales->conn_rwsplit[0], "CREATE TABLE test.t1(id INT)");
+}
+
+static int i_start = 0;
+static int n_rows = 20;
+static int i_end = 0;
+
+void insert_data(XTestConnections& test)
+{
+    test.try_query(test.maxscales->conn_rwsplit[0], "BEGIN");
+
+    i_end = i_start + n_rows;
+
+    for (int i = i_start; i < i_end; ++i)
+    {
+        stringstream ss;
+        ss << "INSERT INTO test.t1 VALUES (" << i << ")";
+        test.try_query(test.maxscales->conn_rwsplit[0], ss.str().c_str());
+    }
+    test.try_query(test.maxscales->conn_rwsplit[0], "COMMIT");
+
+    i_start = i_end;
+}
+
+void check(XTestConnections& test)
+{
+    MYSQL* pConn = test.maxscales->open_rwsplit_connection(0);
+    const char* zQuery = "SELECT * FROM test.t1";
+
+    test.try_query(pConn, "BEGIN");
+    mysql_query(pConn, zQuery);
+
+    MYSQL_RES* pRes = mysql_store_result(pConn);
+    test.add_result(pRes == NULL, "Query should return a result set.");
+
+
+    if (!pRes)
+    {
+        mysql_close(pConn);
+        throw std::runtime_error("Query did not return a result set.");
+    }
+
+    std::string values;
+    MYSQL_ROW row;
+    int num_rows = mysql_num_rows(pRes);
+    test.add_result(num_rows != i_end, "Query returned %d rows when %d rows were expected",
+                    num_rows, i_end);
+    test.nothrow().try_query(pConn, "COMMIT");
+    mysql_close(pConn);
+}
+
+void stop_node(XTestConnections& test, int index)
+{
+    if (test.repl->stop_node(index) != 0)
+    {
+        throw std::runtime_error("Could not stop node.");
+    }
+
+    list_servers(test);
+}
+
+void run(XTestConnections& test)
+{
+    sleep(5);
+
+    int N = test.repl->N;
+    cout << "Nodes: " << N << endl;
+
+    cout << "\nConnecting to MaxScale." << endl;
+    test.connect_maxscale();
+
+    cout << "\nCreating table." << endl;
+    create_table(test);
+
+    list_servers(test);
+
+    for (int i = 0; i < N - 1; ++i)
+    {
+        cout << "Round: " << i << "\n"
+             << "--------" << endl;
+        cout << "\nInserting data." << endl;
+        insert_data(test);
+
+        cout << "\nSyncing slaves." << endl;
+        test.repl->sync_slaves();
+
+        int master_id = get_server_id(*test.maxscales);
+        int master_index = master_id - 1;
+
+        cout << "\nStopping master." << endl;
+        stop_node(test, master_index);
+
+        cout << "\nClosing connection to MaxScale." << endl;
+        test.maxscales->close_maxscale_connections(0);
+
+        sleep(5);
+
+        list_servers(test);
+
+        master_id = get_server_id(*test.maxscales);
+        cout << "\nNew master is: " << master_id << endl;
+
+        cout << "\nConnecting to MaxScale." << endl;
+        test.connect_maxscale();
+
+        cout << "\nChecking result." << endl;
+        check(test);
+    }
+}
+
+}
+
+int main(int argc, char* argv[])
+{
+    XTestConnections test(argc, argv);
+
+    try
+    {
+        run(test);
+    }
+    catch (const std::exception& x)
+    {
+        cerr << "error: Execution was terminated due to an exception: " << x.what() << endl;
+    }
+
+    return test.global_result;
+}