/* * Copyright (c) 2016 MariaDB Corporation Ab * * Use of this software is governed by the Business Source License included * in the LICENSE.TXT file and at www.mariadb.com/bsl11. * * Change Date: 2022-01-01 * * On the date above, in accordance with the Business Source License, use * of this software will be governed by version 2 or later of the General * Public License. */ #include "testconnections.h" #include "fail_switch_rejoin_common.cpp" #include #include #include using std::string; using std::cout; int main(int argc, char** argv) { Mariadb_nodes::require_gtid(true); TestConnections test(argc, argv); // Delete binlogs to sync gtid:s delete_slave_binlogs(test); // Test uses 2 slaves, stop the last one to prevent it from replicating anything. test.repl->stop_node(3); // Set up test table basic_test(test); // Advance gtid:s a bit to so gtid variables are updated. MYSQL* maxconn = test.maxscales->open_rwsplit_connection(0); generate_traffic_and_check(test, maxconn, 1); test.repl->sync_slaves(0); get_output(test); print_gtids(test); auto expect_server_status = [&test](const string& server_name, const string& status) { bool found = (test.maxscales->get_server_status(server_name.c_str()).count(status) == 1); test.expect(found, "%s was not %s as was expected.", server_name.c_str(), status.c_str()); }; string server_names[] = {"server1", "server2", "server3"}; auto expect_server_status_multi = [&test, &expect_server_status, &server_names](std::vector expected) { int expected_size = expected.size(); test.expect(expected_size <= test.repl->N && expected_size <= 3, "Too many expected values."); int tests = (expected_size < test.repl->N) ? expected_size : test.repl->N; for (int node = 0; node < tests; node++) { expect_server_status(server_names[node], expected[node]); } }; auto expect_read_only = [&test](int node, bool expected) { test.expect(test.repl->connect(node) == 0, "Connection to node %i failed.", node); char result[2]; const char query[] = "SELECT @@read_only;"; if (find_field(test.repl->nodes[node], query, "@@read_only", result) == 0) { char expected_result = expected ? '1' : '0'; test.expect(result[0] == expected_result, "read_only on node %i was %c when %c was expected.", node, result[0], expected_result); } else { test.expect(false, "Query '%s' failed on node %i.", query, node); } }; auto expect_read_only_multi = [&test, &expect_read_only](std::vector expected) { int expected_size = expected.size(); test.expect(expected_size <= test.repl->N, "Too many expected values."); int tests = (expected_size < test.repl->N) ? expected_size : test.repl->N; for (int node = 0; node < tests; node++) { expect_read_only(node, expected[node]); } }; auto mon_wait = [&test](int ticks) { test.maxscales->wait_for_monitor(ticks); }; auto crash_node = [&test](int node) { test.repl->ssh_node(node, "kill -s 11 `pidof mysqld`", true); test.repl->stop_node(node); // To prevent autostart. }; string master = "Master"; string slave = "Slave"; string down = "Down"; cout << "Step 1: All should be cool.\n"; get_output(test); expect_server_status_multi({master, slave, slave}); expect_read_only_multi({false, true, true}); if (test.ok()) { cout << "Step 2: Crash slave 2.\n"; crash_node(2); mon_wait(1); get_output(test); expect_server_status_multi({master, slave, down}); expect_read_only_multi({false, true}); generate_traffic_and_check(test, maxconn, 2); cout << "Step 2.1: Slave 2 comes back up, check that read_only is set.\n"; test.repl->start_node(2, ""); mon_wait(2); get_output(test); expect_server_status_multi({master, slave, slave}); expect_read_only_multi({false, true, true}); generate_traffic_and_check(test, maxconn, 3); cout << "Step 3: Slave 1 crashes.\n"; crash_node(1); mon_wait(1); get_output(test); expect_server_status_multi({master, down, slave}); expect_read_only(2, true); generate_traffic_and_check(test, maxconn, 4); cout << "Step 4: Slave 2 goes down again, this time normally.\n"; test.repl->stop_node(2); mon_wait(1); get_output(test); expect_server_status_multi({master, down, down}); generate_traffic_and_check(test, maxconn, 5); cout << "Step 4.1: Slave 1 comes back up, check that read_only is set.\n"; test.repl->start_node(1, ""); mon_wait(2); get_output(test); expect_server_status_multi({master, slave, down}); expect_read_only_multi({false, true}); generate_traffic_and_check(test, maxconn, 6); cout << "Step 4.2: Slave 2 is back up, all should be well.\n"; test.repl->start_node(2, ""); mon_wait(2); get_output(test); expect_server_status_multi({master, slave, slave}); expect_read_only_multi({false, true, true}); generate_traffic_and_check(test, maxconn, 5); } mysql_close(maxconn); // Intermission, quit if a test step failed. if (test.ok()) { // Some of the following tests depend on manipulating backends during the same monitor tick or // between ticks. Slow down the monitor to make this more likely. Not fool-proof in the slightest. int rval = test.maxscales->execute_maxadmin_command(0, "alter monitor MariaDB-Monitor " "monitor_interval=4000"); test.expect(rval == 0, "MaxAdmin command failed."); } if (test.ok()) { cout << "Step 5: Master crashes but comes back during the next loop," " slave 1 should be promoted, old master rejoined.\n"; crash_node(0); mon_wait(1); // The timing is probably a bit iffy here. expect_server_status(server_names[0], down); get_output(test); test.repl->start_node(0, ""); mon_wait(2); get_output(test); // Slave 2 could be promoted as well, but in this case there is no reason to choose it. expect_server_status_multi({slave, master, slave}); expect_read_only_multi({true, false, true}); maxconn = test.maxscales->open_rwsplit_connection(0); generate_traffic_and_check(test, maxconn, 4); cout << "Step 6: Servers 1 & 3 go down. Server 2 should remain as master.\n"; test.repl->stop_node(0); test.repl->stop_node(2); mon_wait(1); get_output(test); expect_server_status_multi({down, master, down}); generate_traffic_and_check(test, maxconn, 3); cout << "Step 6.1: Servers 1 & 3 come back. Check that read_only is set.\n"; test.repl->start_node(2, ""); test.repl->start_node(0, ""); mon_wait(2); get_output(test); expect_server_status_multi({slave, master, slave}); expect_read_only_multi({true, false, true}); generate_traffic_and_check(test, maxconn, 2); cout << "Step 7: Servers 1 & 2 go down. Check that 3 is promoted.\n"; mysql_close(maxconn); test.repl->stop_node(0); test.repl->stop_node(1); mon_wait(2); get_output(test); expect_server_status_multi({down, down, master}); maxconn = test.maxscales->open_rwsplit_connection(0); generate_traffic_and_check(test, maxconn, 1); mysql_close(maxconn); } // Start the servers, in case they weren't on already. for (int i = 0; i < 3; i++) { test.repl->start_node(i); } sleep(1); test.repl->connect(); // Delete the test table from all databases, reset replication. const char drop_query[] = "DROP TABLE IF EXISTS test.t1;"; for (int i = 0; i < 3; i++) { test.try_query(test.repl->nodes[i], drop_query); } test.maxscales->execute_maxadmin_command(0, "call command mariadbmon reset-replication " "MariaDB-Monitor server1"); return test.global_result; }