Merge branch '2.3' into 2.4

system-test/keepalived_masterdown.cpp

@@ -0,0 +1,174 @@
+/**
+ * @file keepalived:masterdown.cpp
+ */
+
+#include <iostream>
+#include <maxtest/failover_common.cpp>
+#include <maxtest/keepalived_func.h>
+#include <maxtest/testconnections.h>
+
+static bool check_maxscale_passive(TestConnections& test, int node);
+static void expect_maxscale_active_passive(TestConnections& test, int active_node);
+const char ms_is_passive[] = "Maxscale %i is passive when active was expected.";
+const char ms_is_active[] = "Maxscale %i is active when passive was expected.";
+
+int main(int argc, char* argv[])
+{
+    const int failover_mon_ticks = 2;
+    // Keepalived takes more time to switch primary MaxScale than a normal failover.
+    // For now, assume that the time scales with monitor ticks.
+    const int keepalived_switch_mon_ticks = 6;
+
+    TestConnections::multiple_maxscales(true);
+    Mariadb_nodes::require_gtid(true);
+    TestConnections test(argc, argv);
+
+    if (test.maxscales->N < 2)
+    {
+        test.tprintf("At least 2 Maxscales are needed for this test. Exiting");
+        exit(0);
+    }
+
+    test.on_destroy([&](){
+                        test.maxscales->ssh_node_f(0, true, "service keepalived stop");
+                        test.maxscales->ssh_node_f(1, true, "service keepalived stop");
+                    });
+
+    test.repl->connect();
+    delete_slave_binlogs(test);
+    basic_test(test);
+    print_gtids(test);
+
+    test.tprintf("Configuring 'keepalived'\n");
+    // Get test client IP, replace last number in it with 253 and use it as Virtual IP
+    configure_keepalived(&test, (char*) "masterdown");
+
+    print_version_string(&test);
+    test.maxscales->wait_for_monitor(1, 0);
+    test.maxscales->wait_for_monitor(1, 1);
+
+    // initial state: 000 expected to be active, 001 - passive
+    int active_node = 0;
+    expect_maxscale_active_passive(test, active_node);
+    if (test.global_result != 0)
+    {
+        return test.global_result;
+    }
+
+    // Test a normal failover.
+    int first_master = test.repl->find_master();
+    test.tprintf("Stop Master - node %d\n", first_master);
+    test.repl->stop_node(first_master);
+    test.maxscales->wait_for_monitor(failover_mon_ticks, active_node);
+
+    int second_master = test.repl->find_master();
+    test.tprintf("new master is node %d\n", second_master);
+    test.expect(first_master != second_master, "Master did not change, failover did not happen.");
+
+    char str[1024];
+    sprintf(str, "Performing automatic failover to replace failed master 'server%d'", first_master + 1);
+    test.tprintf("Checking Maxscale log on 000 for the failover message %s\n", str);
+    test.log_includes(0, str);
+    sprintf(str, "Performing automatic failover to replace failed master");
+    test.tprintf("Checking Maxscale log on 001 for the lack of failover message\n");
+    test.log_excludes(1, str);
+    if (test.global_result != 0)
+    {
+        return test.global_result;
+    }
+
+    // Stop MaxScale 0. Keepalived should promote MaxScale 1 to primary.
+    test.tprintf("Stop Maxscale 000\n");
+    test.maxscales->stop_maxscale(0);
+    active_node = 1;
+    test.maxscales->wait_for_monitor(keepalived_switch_mon_ticks, active_node);
+    test.expect(!check_maxscale_passive(test, active_node), ms_is_passive, active_node);
+    if (test.global_result != 0)
+    {
+        return test.global_result;
+    }
+
+    // Check that a failover still happens.
+    test.tprintf("Stop new Master - node %d\n", second_master);
+    test.repl->stop_node(second_master);
+    test.maxscales->wait_for_monitor(failover_mon_ticks, active_node);
+
+    int third_master = test.repl->find_master();
+    test.tprintf("new master (third one) is node %d\n", third_master);
+    test.expect(third_master != second_master, "Master did not change, failover did not happen.");
+
+    sprintf(str, "Performing automatic failover to replace failed master 'server%d'", second_master + 1);
+    test.tprintf("Checking Maxscale log on 001 for the failover message %s\n", str);
+    test.log_includes(1, str);
+
+    test.log_excludes(1, "Multiple failed master servers detected");
+    test.log_excludes(1, "Failed to perform failover");
+    test.log_excludes(1, "disabling automatic failover");
+    if (test.global_result != 0)
+    {
+        return test.global_result;
+    }
+
+    // Bring MaxScale 0 back up, check that it regains primary status.
+    test.tprintf("Start Maxscale 000\n");
+    test.maxscales->start_maxscale(0);
+    active_node = 0;
+    test.maxscales->wait_for_monitor(keepalived_switch_mon_ticks, active_node);
+    expect_maxscale_active_passive(test, active_node);
+    if (test.global_result != 0)
+    {
+        return test.global_result;
+    }
+
+    sprintf(str, "Performing automatic failover to replace failed master 'server%d'", second_master + 1);
+    test.tprintf("Checking Maxscale log on 001 for the failover message %s\n", str);
+    test.log_includes(1, str);
+    test.tprintf("Checking Maxscale log on 000 for the lack of failover message %s\n", str);
+    test.log_excludes(0, str);
+
+    test.log_excludes(1, "Multiple failed master servers detected");
+    test.log_excludes(1, "Failed to perform failover");
+    test.log_excludes(1, "disabling automatic failover");
+
+    test.log_excludes(0, "Multiple failed master servers detected");
+    test.log_excludes(0, "Failed to perform failover");
+    test.log_excludes(0, "disabling automatic failover");
+
+    stop_keepalived(&test);
+    return test.global_result;
+}
+
+bool check_maxscale_passive(TestConnections& test, int node)
+{
+    bool passive = false;
+    char* passive_str = NULL;
+    int ec = -1;
+
+    test.tprintf("Checking status of Maxscale %03d", node);
+    passive_str = test.maxscales->ssh_node_output(node, "maxctrl show maxscale | grep passive", false, &ec);
+    test.tprintf("maxctrl output string: %s\n", passive_str);
+    if (strstr(passive_str, "false") != NULL)
+    {
+        passive = false;
+    }
+    else
+    {
+        if (strstr(passive_str, "true") == NULL)
+        {
+            test.tprintf("Can't find 'true' or 'false' in the 'maxctrl' output\n");
+        }
+        passive = true;
+    }
+    free(passive_str);
+
+    test.tprintf("Content of 'state.txt' file: %s\n",
+                 test.maxscales->ssh_node_output(0, "cat /tmp/state.txt", false, &ec));
+    return passive;
+}
+
+void expect_maxscale_active_passive(TestConnections& test, int active_node)
+{
+    int passive_node = active_node == 0 ? 1 : 0;
+    test.expect(!check_maxscale_passive(test, active_node), ms_is_passive, active_node);
+    test.expect(check_maxscale_passive(test, passive_node), ms_is_active, passive_node);
+}