From 52b9cc59d5808b70e0716d20b75a252ef85d81fa Mon Sep 17 00:00:00 2001 From: Esa Korhonen Date: Tue, 13 Nov 2018 14:45:54 +0200 Subject: [PATCH] MXS-1775 Test automatic switchover Tests low disk space detection and automatic switchover. --- maxscale-system-test/CMakeLists.txt | 3 + ...cale.cnf.template.mysqlmon_switchover_auto | 104 +++++++++++ .../mysqlmon_switchover_auto.cpp | 162 ++++++++++++++++++ 3 files changed, 269 insertions(+) create mode 100644 maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_switchover_auto create mode 100644 maxscale-system-test/mysqlmon_switchover_auto.cpp diff --git a/maxscale-system-test/CMakeLists.txt b/maxscale-system-test/CMakeLists.txt index 6db5b1a1a..752a6ec68 100644 --- a/maxscale-system-test/CMakeLists.txt +++ b/maxscale-system-test/CMakeLists.txt @@ -266,6 +266,9 @@ add_test_executable(mysqlmon_fail_switch_events.cpp mysqlmon_fail_switch_events # MySQL Monitor reset-replication test, use template for Rejoin (good) add_test_executable(mysqlmon_reset_replication.cpp mysqlmon_reset_replication mysqlmon_rejoin_good LABELS mysqlmon REPL_BACKEND) +# MySQL Monitor low disk switchover +add_test_executable(mysqlmon_switchover_auto.cpp mysqlmon_switchover_auto mysqlmon_switchover_auto LABELS mysqlmon REPL_BACKEND) + # MXS-1506: Delayed query retry # https://jira.mariadb.org/browse/MXS-1506 add_test_executable(mxs1506_delayed_retry.cpp mxs1506_delayed_retry mxs1506_delayed_retry LABELS readwritesplit REPL_BACKEND) diff --git a/maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_switchover_auto b/maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_switchover_auto new file mode 100644 index 000000000..7b58abda2 --- /dev/null +++ b/maxscale-system-test/cnf/maxscale.cnf.template.mysqlmon_switchover_auto @@ -0,0 +1,104 @@ +[maxscale] +threads=###threads### + +[MySQL-Monitor] +type=monitor +module=mysqlmon +servers= server1, server2, server3, server4 +user=maxskysql +password= skysql +monitor_interval=1000 +failcount=1 +replication_user=repl +replication_password=repl +backend_connect_timeout=10 +backend_read_timeout=10 +backend_write_timeout=10 +auto_failover=true +auto_rejoin=true +switchover_on_low_disk_space=true +disk_space_check_interval=2000 +disk_space_threshold=/:90 + +[RW Split Router] +type=service +router= readwritesplit +servers=server1, server2, server3, server4 +user=maxskysql +password=skysql + +[Read Connection Router Slave] +type=service +router=readconnroute +router_options= slave +servers=server1, server2, server3, server4 +user=maxskysql +password=skysql + +[Read Connection Router Master] +type=service +router=readconnroute +router_options=master +servers=server1, server2, server3, server4 +user=maxskysql +password=skysql + +[RW Split Listener] +type=listener +service=RW Split Router +protocol=MySQLClient +port=4006 + +[Read Connection Listener Slave] +type=listener +service=Read Connection Router Slave +protocol=MySQLClient +port=4009 + +[Read Connection Listener Master] +type=listener +service=Read Connection Router Master +protocol=MySQLClient +port=4008 + +[CLI] +type=service +router=cli + +[CLI Listener] +type=listener +service=CLI +protocol=maxscaled +socket=default + +[server1] +type=server +address=###node_server_IP_1### +port=###node_server_port_1### +protocol=MySQLBackend +# This server uses monitor disk space threshold + +[server2] +type=server +address=###node_server_IP_2### +port=###node_server_port_2### +protocol=MySQLBackend +# This server is always out of disk space +disk_space_threshold=/:1 + +[server3] +type=server +address=###node_server_IP_3### +port=###node_server_port_3### +protocol=MySQLBackend +# This server is never out of disk space +disk_space_threshold=/:99 + +[server4] +type=server +address=###node_server_IP_4### +port=###node_server_port_4### +protocol=MySQLBackend +# This server is never out of disk space +disk_space_threshold=/:99 + diff --git a/maxscale-system-test/mysqlmon_switchover_auto.cpp b/maxscale-system-test/mysqlmon_switchover_auto.cpp new file mode 100644 index 000000000..4426ec7bb --- /dev/null +++ b/maxscale-system-test/mysqlmon_switchover_auto.cpp @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2016 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl11. + * + * Change Date: 2022-01-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ + +#include "testconnections.h" +#include "fail_switch_rejoin_common.cpp" +#include +#include + +using std::string; +using std::cout; + +int main(int argc, char** argv) +{ + // Only in very recent server versions have the disks-plugin + TestConnections::require_repl_version("10.3.6"); + Mariadb_nodes::require_gtid(true); + TestConnections::skip_maxscale_start(true); + TestConnections test(argc, argv); + + test.repl->connect(); + delete_slave_binlogs(test); + + // Enable the disks-plugin on all servers. Has to be done before MaxScale is on to prevent disk space + // monitoring from disabling itself. + bool disks_plugin_loaded = false; + const char strict_mode[] = "SET GLOBAL gtid_strict_mode=%i;"; + test.repl->connect(); + for (int i = 0; i < test.repl->N; i++) + { + MYSQL* conn = test.repl->nodes[i]; + test.try_query(conn, "INSTALL SONAME 'disks';"); + test.try_query(conn, strict_mode, 1); + } + + if (test.ok()) + { + cout << "Disks-plugin installed and gtid_strict_mode enabled on all servers. " + "Starting MaxScale.\n"; + test.start_maxscale(); + disks_plugin_loaded = true; + } + else + { + cout << "Test preparations failed.\n"; + } + + auto set_to_string = [](const StringSet& str_set) -> string { + string rval; + for (const string& elem : str_set) + { + rval += elem + ","; + } + return rval; + }; + + auto expect_server_status = [&test, &set_to_string](const string& server_name, const string& status) { + auto status_set = test.maxscales->get_server_status(server_name.c_str()); + string status_str = set_to_string(status_set); + bool found = (status_set.count(status) == 1); + test.expect(found, "%s was not %s as was expected. Status: %s.", + server_name.c_str(), status.c_str(), status_str.c_str()); + }; + + string server_names[] = {"server1", "server2", "server3", "server4"}; + string master = "Master"; + string slave = "Slave"; + string maint = "Maintenance"; + const char insert_query[] = "INSERT INTO test.t1 VALUES (%i);"; + int insert_val = 1; + + if (test.ok()) + { + // Set up test table to ensure queries are going through. + test.tprintf("Creating table and inserting data."); + auto maxconn = test.maxscales->open_rwsplit_connection(0); + test.try_query(maxconn, "CREATE OR REPLACE TABLE test.t1(c1 INT)"); + test.try_query(maxconn, insert_query, insert_val++); + mysql_close(maxconn); + + get_output(test); + print_gtids(test); + + expect_server_status(server_names[0], master); + expect_server_status(server_names[1], maint); // Always out of disk space + expect_server_status(server_names[2], slave); + expect_server_status(server_names[3], slave); + } + + if (test.ok()) + { + // If ok so far, change the disk space threshold to something really small to force a switchover. + cout << "Changing disk space threshold for the monitor, should cause a switchover.\n"; + test.maxscales->execute_maxadmin_command(0, "alter monitor MySQL-Monitor disk_space_threshold=/:1"); + sleep(2); // The disk space is checked depending on wall clock time. + test.maxscales->wait_for_monitor(2); + + // server2 was in maintenance before the switchover, so it was ignored. This means that it is + // still replicating from server1. server1 was redirected to the new master. Although server1 + // is low on disk space, it is not set to maintenance since it is a relay. + expect_server_status(server_names[0], slave); + expect_server_status(server_names[1], maint); + expect_server_status(server_names[2], master); + expect_server_status(server_names[3], slave); + + // Check that writes are working. + auto maxconn = test.maxscales->open_rwsplit_connection(0); + test.try_query(maxconn, insert_query, insert_val); + mysql_close(maxconn); + + get_output(test); + print_gtids(test); + + cout << "Changing disk space threshold for the monitor, should prevent low disk switchovers.\n"; + test.maxscales->execute_maxadmin_command(0, "alter monitor MySQL-Monitor " + "disk_space_threshold=/:100"); + sleep(2); // To update disk space status + test.maxscales->wait_for_monitor(1); + get_output(test); + } + + // Use the reset-replication command to fix the situation. + cout << "Running reset-replication to fix the situation.\n"; + test.maxscales->execute_maxadmin_command(0, "call command mariadbmon reset-replication " + "MySQL-Monitor server1"); + sleep(2); + test.maxscales->wait_for_monitor(2); + get_output(test); + // Check that no auto switchover has happened. + expect_server_status(server_names[0], master); + expect_server_status(server_names[1], maint); + expect_server_status(server_names[2], slave); + expect_server_status(server_names[3], slave); + + const char drop_query[] = "DROP TABLE test.t1;"; + auto maxconn = test.maxscales->open_rwsplit_connection(0); + test.try_query(maxconn, drop_query); + mysql_close(maxconn); + + if (disks_plugin_loaded) + { + // Enable the disks-plugin on all servers. + for (int i = 0; i < test.repl->N; i++) + { + MYSQL* conn = test.repl->nodes[i]; + test.try_query(conn, "UNINSTALL SONAME 'disks';"); + test.try_query(conn, strict_mode, 0); + } + } + + test.repl->disconnect(); + return test.global_result; +}