MXS-1564: Add rejoin test

The test combines auto-failover, rejoining a temporarily downed master
server and switchover.

maxscale-system-test/.gitignore

@@ -160,6 +160,7 @@ mxs951_utfmb4
 mxs957
 mysqlmon_failover_auto
 mysqlmon_failover_manual
+mysqlmon_rejoin_good
 namedserverfilter
 no_password
 non_native_setup

maxscale-system-test/CMakeLists.txt

@@ -263,6 +263,9 @@ add_test_executable(mysqlmon_failover_manual2.cpp mysqlmon_failover_manual2_4 my
 add_test_executable(mysqlmon_failover_manual2.cpp mysqlmon_failover_manual2_3 mysqlmon_failover_manual2_3 LABELS mysqlmon REPL_BACKEND)
 add_test_executable(mysqlmon_failover_manual2.cpp mysqlmon_failover_manual2_2 mysqlmon_failover_manual2_2 LABELS mysqlmon REPL_BACKEND)
 
+# MySQL Monitor Rejoin Test
+add_test_executable(mysqlmon_rejoin_good.cpp mysqlmon_rejoin_good mysqlmon_rejoin_good LABELS mysqlmon REPL_BACKEND)
+
 # Test monitor state change events when manually clearing server bits
 add_test_executable(false_monitor_state_change.cpp false_monitor_state_change replication LABELS mysqlmon REPL_BACKEND)
 

maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_rejoin_good

@@ -0,0 +1,95 @@
+[maxscale]
+threads=###threads###
+
+[MySQL-Monitor]
+type=monitor
+module=mysqlmon
+servers= server1, server2, server3, server4
+user=maxskysql
+passwd= skysql
+monitor_interval=1000
+detect_standalone_master=true
+failcount=1
+allow_cluster_recovery=true
+replication_user=repl
+replication_password=repl
+backend_connect_timeout=3
+backend_read_timeout=3
+backend_write_timeout=3
+auto_failover=true
+auto_rejoin=true
+
+[RW Split Router]
+type=service
+router= readwritesplit
+servers=server1, server2, server3, server4
+user=maxskysql
+passwd=skysql
+
+[Read Connection Router Slave]
+type=service
+router=readconnroute
+router_options= slave
+servers=server1, server2, server3, server4
+user=maxskysql
+passwd=skysql
+
+[Read Connection Router Master]
+type=service
+router=readconnroute
+router_options=master
+servers=server1, server2, server3, server4
+user=maxskysql
+passwd=skysql
+
+[RW Split Listener]
+type=listener
+service=RW Split Router
+protocol=MySQLClient
+port=4006
+
+[Read Connection Listener Slave]
+type=listener
+service=Read Connection Router Slave
+protocol=MySQLClient
+port=4009
+
+[Read Connection Listener Master]
+type=listener
+service=Read Connection Router Master
+protocol=MySQLClient
+port=4008
+
+[CLI]
+type=service
+router=cli
+
+[CLI Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=default
+
+[server1]
+type=server
+address=###node_server_IP_1###
+port=###node_server_port_1###
+protocol=MySQLBackend
+
+[server2]
+type=server
+address=###node_server_IP_2###
+port=###node_server_port_2###
+protocol=MySQLBackend
+
+[server3]
+type=server
+address=###node_server_IP_3###
+port=###node_server_port_3###
+protocol=MySQLBackend
+
+[server4]
+type=server
+address=###node_server_IP_4###
+port=###node_server_port_4###
+protocol=MySQLBackend

maxscale-system-test/mysqlmon_failover_common.cpp

@@ -60,7 +60,7 @@ void check(TestConnections& test)
  * @param test Tester object
  * @return Master server id
  */
-int get_server_id(TestConnections& test)
+int get_master_server_id(TestConnections& test)
 {
     MYSQL *conn = test.maxscales->open_rwsplit_connection(0);
     int id = -1;
@@ -136,7 +136,7 @@ void check_test_1(TestConnections& test, int node0_id)
 {
     check(test);
     get_output(test);
-    int master_id = get_server_id(test);
+    int master_id = get_master_server_id(test);
     test.tprintf(PRINT_ID, master_id);
     test.add_result(master_id < 1 && master_id == node0_id, "Master did not change or no master detected.");
     fix_replication_create_table(test);
@@ -159,7 +159,7 @@ void check_test_2(TestConnections& test)
     check(test);
     get_output(test);
 
-    int master_id = get_server_id(test);
+    int master_id = get_master_server_id(test);
     test.tprintf(PRINT_ID, master_id);
     test.add_result(master_id < 1 ||
                     (master_id != test.repl->get_server_id(2) && master_id != test.repl->get_server_id(3)),
@@ -197,7 +197,7 @@ void check_test_3(TestConnections& test)
     check(test);
     get_output(test);
 
-    int master_id = get_server_id(test);
+    int master_id = get_master_server_id(test);
     // Because servers have been restarted, redo connections.
     test.repl->connect();
     sleep(2);

maxscale-system-test/mysqlmon_rejoin_good.cpp

@@ -0,0 +1,174 @@
+/*
+ * Copyright (c) 2016 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2020-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#include <vector>
+
+#include "testconnections.h"
+#include "mysqlmon_failover_common.cpp"
+
+using std::string;
+typedef std::vector<string> StringVector;
+
+const char GTID_QUERY[] = "SELECT @@gtid_current_pos;";
+const char GTID_FIELD[] = "@@gtid_current_pos";
+const int bufsize = 512;
+/**
+ * Do inserts, check that results are as expected.
+ *
+ * @param test Test connections
+ * @paran insert_count
+ */
+void generate_traffic_and_check(TestConnections& test, int insert_count)
+{
+    MYSQL *conn = test.maxscales->open_rwsplit_connection(0);
+    const char INSERT[] = "INSERT INTO test.t1 VALUES (%d);";
+    const char SELECT[] = "SELECT * FROM test.t1 ORDER BY id ASC;";
+    for (int i = 0; i < insert_count; i++)
+    {
+        test.try_query(conn, INSERT, inserts++);
+        timespec time;
+        time.tv_sec = 0;
+        time.tv_nsec = 100000000;
+        nanosleep(&time, NULL);
+    }
+
+    mysql_query(conn, SELECT);
+    MYSQL_RES *res = mysql_store_result(conn);
+    test.assert(res != NULL, "Query did not return a result set");
+
+    if (res)
+    {
+        MYSQL_ROW row;
+        // Check all values, they should go from 0 to 'inserts'
+        int expected_val = 0;
+        while ((row = mysql_fetch_row(res)))
+        {
+            int value_read = strtol(row[0], NULL, 0);
+            if (value_read != expected_val)
+            {
+                test.assert(false, "Query returned %d when %d was expected", value_read, expected_val);
+                break;
+            }
+            expected_val++;
+        }
+        int num_rows = expected_val;
+        test.assert(num_rows == inserts, "Query returned %d rows when %d rows were expected",
+                    num_rows, inserts);
+        mysql_free_result(res);
+    }
+    mysql_close(conn);
+}
+
+void print_gtids(TestConnections& test)
+{
+    MYSQL* maxconn = test.maxscales->open_rwsplit_connection(0);
+    if (maxconn)
+    {
+        char result_tmp[bufsize];
+        if (find_field(maxconn, GTID_QUERY, GTID_FIELD, result_tmp) == 0)
+        {
+            test.tprintf("MaxScale gtid: %s", result_tmp);
+        }
+    }
+    mysql_close(maxconn);
+    test.repl->connect();
+    for (int i = 0; i < test.repl->N; i++)
+    {
+        char result_tmp[bufsize];
+        if (find_field(test.repl->nodes[i], GTID_QUERY, GTID_FIELD, result_tmp) == 0)
+        {
+            test.tprintf("Node %d gtid: %s", i, result_tmp);
+        }
+    }
+}
+
+int main(int argc, char** argv)
+{
+    interactive = strcmp(argv[argc - 1], "interactive") == 0;
+    TestConnections test(argc, argv);
+    MYSQL* maxconn = test.maxscales->open_rwsplit_connection(0);
+
+    // Set up test table
+    basic_test(test);
+    // Delete binlogs to sync gtid:s
+    delete_slave_binlogs(test);
+    char result_tmp[bufsize];
+    // Advance gtid:s a bit to so gtid variables are updated.
+    generate_traffic_and_check(test, 10);
+    sleep(1);
+    test.tprintf(LINE);
+    print_gtids(test);
+    get_input();
+
+    test.tprintf("Stopping master and waiting for failover. Check that another server is promoted.");
+    test.tprintf(LINE);
+    const int old_master_id = get_master_server_id(test); // Read master id now before shutdown.
+    const int master_index = test.repl->master;
+    test.repl->stop_node(master_index);
+    sleep(10);
+    // Recreate maxscale session
+    mysql_close(maxconn);
+    maxconn = test.maxscales->open_rwsplit_connection(0);
+    get_output(test);
+    int master_id = get_master_server_id(test);
+    test.tprintf(LINE);
+    test.tprintf(PRINT_ID, master_id);
+    const bool failover_ok = (master_id > 0 && master_id != old_master_id);
+    test.assert(failover_ok, "Master did not change or no master detected.");
+    string gtid_final;
+    if (failover_ok)
+    {
+        test.tprintf("Sending more inserts.");
+        generate_traffic_and_check(test, 5);
+        sleep(1);
+        if (find_field(maxconn, GTID_QUERY, GTID_FIELD, result_tmp) == 0)
+        {
+            gtid_final = result_tmp;
+        }
+        print_gtids(test);
+        test.tprintf("Bringing old master back online. It should rejoin the cluster and catch up in events.");
+        test.tprintf(LINE);
+
+        test.repl->start_node(master_index, (char*) "");
+        sleep(10);
+        get_output(test);
+
+        test.repl->connect();
+        sleep(1);
+        string gtid_old_master;
+        if (find_field(test.repl->nodes[master_index], GTID_QUERY, GTID_FIELD, result_tmp) == 0)
+        {
+            gtid_old_master = result_tmp;
+        }
+        test.tprintf(LINE);
+        print_gtids(test);
+        test.tprintf(LINE);
+        test.assert(gtid_final == gtid_old_master, "Old master did not successfully rejoin the cluster.");
+        // Switch master back to server1 so last check is faster
+        int ec;
+        test.maxscales->ssh_node_output(0, "maxadmin call command mysqlmon switchover "
+                                        "MySQL-Monitor server1 server2" , true, &ec);
+        sleep(5); // Wait for monitor to update status
+        get_output(test);
+        master_id = get_master_server_id(test);
+        test.assert(master_id == old_master_id, "Switchover back to server1 failed.");
+    }
+    else
+    {
+        test.repl->start_node(master_index, (char*) "");
+        sleep(10);
+    }
+
+    test.repl->fix_replication();
+    return test.global_result;
+}