MXS-1514: Split failover test in two files, add manual version

The same test now has two versions. In the automatic version failover
begins automatically. In the manual version failover is started with
maxadmin. The tests are otherwise identical.

maxscale-system-test/.gitignore

@@ -154,6 +154,8 @@ mxs922_scaling
 mxs922_server
 mxs951_utfmb4
 mxs957
+mysqlmon_failover_auto
+mysqlmon_failover_manual
 namedserverfilter
 no_password
 non_native_setup

maxscale-system-test/CMakeLists.txt

@@ -252,8 +252,11 @@ add_test_executable(encrypted_passwords.cpp encrypted_passwords replication LABE
 # MySQL Monitor Failover Test
 add_test_executable(failover_mysqlmon.cpp failover_mysqlmon failover_mysqlmon LABELS mysqlmon REPL_BACKEND)
 
-# MySQL Monitor Real Failover Test
-add_test_executable(failover_mysqlmon_mrm.cpp failover_mysqlmon_mrm failover_mysqlmon_mrm LABELS mysqlmon REPL_BACKEND)
+# MySQL Monitor Failover (automatic) Test
+add_test_executable(mysqlmon_failover_auto.cpp mysqlmon_failover_auto mysqlmon_failover_auto LABELS mysqlmon REPL_BACKEND)
+
+# MySQL Monitor Failover (manual) Test
+add_test_executable(mysqlmon_failover_manual.cpp mysqlmon_failover_manual mysqlmon_failover_manual LABELS mysqlmon REPL_BACKEND)
 
 # Test monitor state change events when manually clearing server bits
 add_test_executable(false_monitor_state_change.cpp false_monitor_state_change replication LABELS mysqlmon REPL_BACKEND)

maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_failover_auto

@@ -1,7 +1,7 @@
 [maxscale]
 threads=###threads###
 
-[MySQL Monitor]
+[MySQL-Monitor]
 type=monitor
 module=mysqlmon
 servers= server1, server2, server3, server4

maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_failover_manual

@@ -0,0 +1,91 @@
+[maxscale]
+threads=###threads###
+
+[MySQL-Monitor]
+type=monitor
+module=mysqlmon
+servers= server1, server2, server3, server4
+user=maxskysql
+passwd= skysql
+monitor_interval=1000
+detect_standalone_master=true
+failcount=1
+allow_cluster_recovery=true
+replication_user=repl
+replication_password=repl
+backend_connect_timeout=1
+
+[RW Split Router]
+type=service
+router= readwritesplit
+servers=server1, server2, server3, server4
+user=maxskysql
+passwd=skysql
+
+[Read Connection Router Slave]
+type=service
+router=readconnroute
+router_options= slave
+servers=server1, server2, server3, server4
+user=maxskysql
+passwd=skysql
+
+[Read Connection Router Master]
+type=service
+router=readconnroute
+router_options=master
+servers=server1, server2, server3, server4
+user=maxskysql
+passwd=skysql
+
+[RW Split Listener]
+type=listener
+service=RW Split Router
+protocol=MySQLClient
+port=4006
+
+[Read Connection Listener Slave]
+type=listener
+service=Read Connection Router Slave
+protocol=MySQLClient
+port=4009
+
+[Read Connection Listener Master]
+type=listener
+service=Read Connection Router Master
+protocol=MySQLClient
+port=4008
+
+[CLI]
+type=service
+router=cli
+
+[CLI Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=default
+
+[server1]
+type=server
+address=###node_server_IP_1###
+port=###node_server_port_1###
+protocol=MySQLBackend
+
+[server2]
+type=server
+address=###node_server_IP_2###
+port=###node_server_port_2###
+protocol=MySQLBackend
+
+[server3]
+type=server
+address=###node_server_IP_3###
+port=###node_server_port_3###
+protocol=MySQLBackend
+
+[server4]
+type=server
+address=###node_server_IP_4###
+port=###node_server_port_4###
+protocol=MySQLBackend

maxscale-system-test/mysqlmon_failover_auto.cpp

@@ -0,0 +1,34 @@
+/**
+ * Test auto_failover
+ */
+
+#include "testconnections.h"
+#include "mysqlmon_failover_common.cpp"
+
+int main(int argc, char** argv)
+{
+    interactive = strcmp(argv[argc - 1], "interactive") == 0;
+    TestConnections test(argc, argv);
+
+    // Wait a few seconds
+    sleep(5);
+    basic_test(test);
+
+    // Test 1
+    int node0_id = prepare_test_1(test);
+    sleep(10);
+    check_test_1(test, node0_id);
+
+    // Test 2
+    prepare_test_2(test);
+    sleep(10);
+    check_test_2(test);
+
+    // Test 3
+    prepare_test_3(test);
+    sleep(10);
+    check_test_3(test);
+
+    test.repl->fix_replication();
+    return test.global_result;
+}

maxscale-system-test/mysqlmon_failover_common.cpp

@@ -1,9 +1,8 @@
-/**
- * Test replication-manager
- */
-
 #include "testconnections.h"
 
+int inserts = 0;
+bool interactive = false;
+
 void get_output(TestConnections& test)
 {
     int ec;
@@ -20,8 +19,6 @@ void get_output(TestConnections& test)
     free(output);
 }
 
-static int inserts = 0;
-
 void check(TestConnections& test)
 {
     MYSQL *conn = test.maxscales->open_rwsplit_connection(0);
@@ -78,8 +75,6 @@ int get_server_id(TestConnections& test)
     return id;
 }
 
-static bool interactive = false;
-
 void get_input()
 {
     if (interactive)
@@ -111,19 +106,12 @@ void delete_slave_binlogs(TestConnections& test)
     execute_query(test.repl->nodes[3], RESET);
 }
 
-int main(int argc, char** argv)
+const char LINE[] = "------------------------------------------";
+const char PRINT_ID[] = "Master server id is %d.";
+const char WRONG_SLAVE[] = "Wrong slave was promoted or promotion failed.";
+
+void basic_test(TestConnections& test)
 {
-    const char* LINE = "------------------------------------------";
-    const char* PRINT_ID = "Master server id is %d.";
-    const char* WRONG_SLAVE = "Wrong slave was promoted or promotion failed.";
-
-    interactive = strcmp(argv[argc - 1], "interactive") == 0;
-    int master_id = -1;
-    TestConnections test(argc, argv);
-
-    // Wait a few seconds
-    sleep(5);
-
     test.tprintf("Creating table and inserting data.");
     get_input();
     test.maxscales->connect_maxscale(0);
@@ -132,26 +120,31 @@ int main(int argc, char** argv)
 
     check(test);
     get_output(test);
-
-    // Test 1
+}
+int prepare_test_1(TestConnections& test)
+{
     delete_slave_binlogs(test);
     test.tprintf("Test 1: Stopping master and waiting for failover. Check that another server is promoted.\n"
                  "%s", LINE);
     get_input();
     int node0_id = test.repl->get_server_id(0); // Read master id now before shutdown.
     test.repl->stop_node(0);
-    sleep(10);
+    return node0_id;
+}
 
+void check_test_1(TestConnections& test, int node0_id)
+{
     check(test);
     get_output(test);
-
-    master_id = get_server_id(test);
+    int master_id = get_server_id(test);
     test.tprintf(PRINT_ID, master_id);
     test.add_result(master_id < 1 && master_id == node0_id, "Master did not change or no master detected.");
     fix_replication_create_table(test);
     test.repl->connect();
+}
 
-    // Test 2
+void prepare_test_2(TestConnections& test)
+{
     delete_slave_binlogs(test);
     test.tprintf("Test 2: Disable replication on server 2 and kill master, check that server 3 or 4 is "
                  "promoted.\n%s", LINE);
@@ -159,21 +152,23 @@ int main(int argc, char** argv)
     execute_query(test.repl->nodes[1], "STOP SLAVE; RESET SLAVE ALL;");
     sleep(2);
     test.repl->stop_node(0);
-    sleep(10);
+}
 
+void check_test_2(TestConnections& test)
+{
     check(test);
     get_output(test);
 
-    master_id = get_server_id(test);
+    int master_id = get_server_id(test);
     test.tprintf(PRINT_ID, master_id);
     test.add_result(master_id < 1 ||
                     (master_id != test.repl->get_server_id(2) && master_id != test.repl->get_server_id(3)),
                     WRONG_SLAVE);
     fix_replication_create_table(test);
     test.repl->connect();
-
-
-    // Test 3
+}
+void prepare_test_3(TestConnections& test)
+{
     delete_slave_binlogs(test);
     test.tprintf("Test3: Shutdown two slaves (servers 2 and 4). Disable log_bin on server 2, making it "
                  "invalid for promotion. Enable log-slave-updates on servers 2 and 4. Check that server 4 is "
@@ -195,12 +190,14 @@ int main(int argc, char** argv)
     get_output(test);
     test.tprintf("Stopping master.");
     test.repl->stop_node(0);
-    sleep(10);
+}
 
+void check_test_3(TestConnections& test)
+{
     check(test);
     get_output(test);
 
-    master_id = get_server_id(test);
+    int master_id = get_server_id(test);
     // Because servers have been restarted, redo connections.
     test.repl->connect();
     sleep(2);
@@ -222,8 +219,4 @@ int main(int argc, char** argv)
     test.maxscales->start_maxscale(0);
     sleep(2);
     get_output(test);
-    get_input();
-
-    test.repl->fix_replication();
-    return test.global_result;
 }

maxscale-system-test/mysqlmon_failover_manual.cpp

@@ -0,0 +1,45 @@
+/**
+ * Test auto_failover
+ */
+
+#include "testconnections.h"
+#include "mysqlmon_failover_common.cpp"
+
+int main(int argc, char** argv)
+{
+    const char FAILOVER_CMD[] = "maxadmin call command mysqlmon failover MySQL-Monitor";
+    interactive = strcmp(argv[argc - 1], "interactive") == 0;
+    TestConnections test(argc, argv);
+    int ec;
+    // Wait a few seconds
+    sleep(5);
+    basic_test(test);
+
+    // Test 1
+    int node0_id = prepare_test_1(test);
+    sleep(3);
+    test.maxscales->ssh_node_output(0, FAILOVER_CMD , true, &ec);
+    sleep(10);
+
+    check_test_1(test, node0_id);
+
+    // Test 2
+    prepare_test_2(test);
+    sleep(3);
+    test.maxscales->ssh_node_output(0, FAILOVER_CMD, true, &ec);
+    sleep(10);
+
+    check_test_2(test);
+
+    // Test 3
+    prepare_test_3(test);
+    sleep(3);
+    test.maxscales->ssh_node_output(0, FAILOVER_CMD, true, &ec);
+    sleep(10);
+
+    check_test_3(test);
+
+    test.repl->fix_replication();
+    return test.global_result;
+}
+