MXS-1570: Test failover with no good candidate

- 1 master, 3 slaves
- "stop slave" on server 2
- "disable" log-bin on server 3
- set multi-source replication on server 4
- take down master
- no slave should be promoted

maxscale-system-test/.gitignore

@@ -106,6 +106,7 @@ maxscale_process_user
 mm
 mm_mysqlmon
 mysqlmon_failover_bad_master
+mysqlmon_failover_no_slaves
 mysqlmon_failover_manual
 mysqlmon_failover_manual2_4
 mysqlmon_failover_manual2_3

maxscale-system-test/CMakeLists.txt

@@ -266,6 +266,9 @@ add_test_executable(mysqlmon_failover_manual2.cpp mysqlmon_failover_manual2_2 my
 # MySQL Monitor manual failover with bad master
 add_test_executable(mysqlmon_failover_bad_master.cpp mysqlmon_failover_bad_master mysqlmon_failover_bad_master LABELS mysqlmon REPL_BACKEND)
 
+# MySQL Monitor manual failover with no valid slaves, uses config of mysqlmon_failover_auto
+add_test_executable(mysqlmon_failover_no_slaves.cpp mysqlmon_failover_no_slaves mysqlmon_failover_auto LABELS mysqlmon REPL_BACKEND)
+
 # MySQL Monitor Rejoin (good) Test
 add_test_executable(mysqlmon_rejoin_good.cpp mysqlmon_rejoin_good mysqlmon_rejoin_good LABELS mysqlmon REPL_BACKEND)
 

maxscale-system-test/fail_switch_rejoin_common.cpp

@@ -130,27 +130,31 @@ void basic_test(TestConnections& test)
  * Do inserts, check that results are as expected.
  *
  * @param test Test connections
- * @paran insert_count
+ * @param conn Which specific connection to use
+ * @param insert_count How many inserts should be done
+ * @return True, if successful
  */
-void generate_traffic_and_check(TestConnections& test, MYSQL* conn, int insert_count)
+bool generate_traffic_and_check(TestConnections& test, MYSQL* conn, int insert_count)
 {
     const char INSERT[] = "INSERT INTO test.t1 VALUES (%d);";
     const char SELECT[] = "SELECT * FROM test.t1 ORDER BY id ASC;";
+    timespec short_sleep;
+    short_sleep.tv_sec = 0;
+    short_sleep.tv_nsec = 100000000;
     for (int i = 0; i < insert_count; i++)
     {
         test.try_query(conn, INSERT, inserts++);
-        timespec time;
+        nanosleep(&short_sleep, NULL);
-        time.tv_sec = 0;
-        time.tv_nsec = 100000000;
-        nanosleep(&time, NULL);
     }
-
+    sleep(1);
+    bool rval = false;
     mysql_query(conn, SELECT);
     MYSQL_RES *res = mysql_store_result(conn);
     test.assert(res != NULL, "Query did not return a result set");
 
     if (res)
     {
+        rval = true;
         MYSQL_ROW row;
         // Check all values, they should go from 0 to 'inserts'
         int expected_val = 0;
@@ -160,6 +164,7 @@ void generate_traffic_and_check(TestConnections& test, MYSQL* conn, int insert_c
             if (value_read != expected_val)
             {
                 test.assert(false, "Query returned %d when %d was expected", value_read, expected_val);
+                rval = false;
                 break;
             }
             expected_val++;
@@ -167,9 +172,13 @@ void generate_traffic_and_check(TestConnections& test, MYSQL* conn, int insert_c
         int num_rows = expected_val;
         test.assert(num_rows == inserts, "Query returned %d rows when %d rows were expected",
                     num_rows, inserts);
+        if (num_rows != inserts)
+        {
+            rval = false;
+        }
         mysql_free_result(res);
     }
-  //  mysql_close(conn);
+    return rval;
 }
 
 void print_gtids(TestConnections& test)

maxscale-system-test/mysqlmon_failover_no_slaves.cpp

@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2016 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2020-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#include "testconnections.h"
+#include "fail_switch_rejoin_common.cpp"
+
+int main(int argc, char** argv)
+{
+    interactive = strcmp(argv[argc - 1], "interactive") == 0;
+    Mariadb_nodes::require_gtid(true);
+    TestConnections test(argc, argv);
+
+    delete_slave_binlogs(test);
+    basic_test(test);
+    MYSQL* conn = test.maxscales->open_rwsplit_connection(0);
+    if (!generate_traffic_and_check(test, conn, 5))
+    {
+        return test.global_result;
+    }
+
+    // Make all three slaves ineligible for promotion in different ways.
+    test.repl->connect();
+    MYSQL** nodes = test.repl->nodes;
+    // Slave 1. Just stop slave.
+    test.try_query(nodes[1], "STOP SLAVE;");
+    // Slave 2. Disable binlog.
+    test.repl->stop_node(2);
+    test.repl->stash_server_settings(2);
+    test.repl->disable_server_setting(2, "log-bin");
+    test.repl->start_node(2, (char *) "");
+    // Slave 3. Create a second slave connection to a non-existing server.
+    const char CHANGE_CMD[] = "CHANGE MASTER 'dummy' TO MASTER_HOST = 'imagination_host.img', "
+    "MASTER_PORT = 1234, MASTER_USE_GTID = current_pos, MASTER_USER='repl', MASTER_PASSWORD='repl';";
+    test.try_query(nodes[3], CHANGE_CMD);
+    test.try_query(nodes[3], "START SLAVE;");
+
+    sleep(4);
+    get_output(test);
+
+    test.tprintf(LINE);
+    test.tprintf("Stopping master. Failover should not happen.");
+    test.repl->block_node(0);
+    sleep(10);
+    get_output(test);
+    int master_id = get_master_server_id(test);
+    test.assert(master_id == -1, "Master was promoted even when no slave was eligible.");
+
+    test.repl->unblock_node(0);
+    sleep(1);
+
+    // Restore normal settings.
+    test.try_query(nodes[1], "START SLAVE;");
+    test.repl->stop_node(2);
+    test.repl->restore_server_settings(2);
+    test.repl->start_node(2, (char *) "");
+    test.try_query(nodes[3], "STOP SLAVE 'dummy';");
+    test.try_query(nodes[3], "RESET SLAVE 'dummy' ALL;");
+    test.repl->fix_replication();
+    return test.global_result;
+}