From 77f8a3b71b85d430ec13b538537a8f782309aa10 Mon Sep 17 00:00:00 2001 From: Niclas Antti Date: Tue, 13 Nov 2018 17:23:16 +0200 Subject: [PATCH] MXS-2057 System test --- etc/maxscale.service.in | 4 +- maxscale-system-test/CMakeLists.txt | 3 + ...cale.cnf.template.mxs2057_systemd_watchdog | 46 +++++++++ maxscale-system-test/infinite_loop.lua | 27 ++++++ .../mxs2057_systemd_watchdog.cpp | 94 +++++++++++++++++++ server/core/routingworker.cc | 2 +- 6 files changed, 172 insertions(+), 4 deletions(-) create mode 100644 maxscale-system-test/cnf/maxscale.cnf.template.mxs2057_systemd_watchdog create mode 100644 maxscale-system-test/infinite_loop.lua create mode 100644 maxscale-system-test/mxs2057_systemd_watchdog.cpp diff --git a/etc/maxscale.service.in b/etc/maxscale.service.in index 10ea0bc8a..4f335ed41 100644 --- a/etc/maxscale.service.in +++ b/etc/maxscale.service.in @@ -20,9 +20,7 @@ ExecStart=@CMAKE_INSTALL_PREFIX@/@MAXSCALE_BINDIR@/maxscale TimeoutStartSec=120 LimitNOFILE=65535 StartLimitBurst=0 - -# Uncomment to enable the systemd Watchdog -#WatchdogSec=30s +WatchdogSec=60s # Only relevant when MaxScale is linked with -fsanitize=address Environment=ASAN_OPTIONS=abort_on_error=1 diff --git a/maxscale-system-test/CMakeLists.txt b/maxscale-system-test/CMakeLists.txt index dd947a268..6db5b1a1a 100644 --- a/maxscale-system-test/CMakeLists.txt +++ b/maxscale-system-test/CMakeLists.txt @@ -956,6 +956,9 @@ add_test_executable(mxs1949_warn_user_injection.cpp mxs1949_warn_user_injection # MXS-2106: NULL values with avrorouter add_test_executable(mxs2106_avro_null.cpp mxs2106_avro_null avro LABELS avrorouter REPL_BACKEND) +# MXS-2057 systemd watchdog +add_test_executable(mxs2057_systemd_watchdog.cpp mxs2057_systemd_watchdog mxs2057_systemd_watchdog LABELS REPL_BACKEND) + ############################################ # END: binlogrouter and avrorouter tests # ############################################ diff --git a/maxscale-system-test/cnf/maxscale.cnf.template.mxs2057_systemd_watchdog b/maxscale-system-test/cnf/maxscale.cnf.template.mxs2057_systemd_watchdog new file mode 100644 index 000000000..2ef4a6f33 --- /dev/null +++ b/maxscale-system-test/cnf/maxscale.cnf.template.mxs2057_systemd_watchdog @@ -0,0 +1,46 @@ +[maxscale] +threads = ###threads### +log_debug=1 + +[Monitor] +type = monitor +module = mariadbmon +servers = server1 +user = maxskysql +password = skysql +monitor_interval = 1000 + +[RW-Split-Router] +type = service +router = readwritesplit +servers = server1 +user = maxskysql +password = skysql +filters = lua + +[RW-Split-Listener] +type = listener +service = RW-Split-Router +protocol = mariadbclient +port = 4006 + +[lua] +type = filter +module = luafilter +session_script=/home/vagrant/infinite_loop.lua + +[MaxAdmin-Service] +type = service +router = cli + +[MaxAdmin-Unix-Listener] +type = listener +service = MaxAdmin-Service +protocol = maxscaled +socket = default + +[server1] +type = server +address = ###node_server_IP_1### +port = ###node_server_port_1### +protocol = MariaDBBackend diff --git a/maxscale-system-test/infinite_loop.lua b/maxscale-system-test/infinite_loop.lua new file mode 100644 index 000000000..622009ba3 --- /dev/null +++ b/maxscale-system-test/infinite_loop.lua @@ -0,0 +1,27 @@ +function createInstance() +end + +function newSession(a, b) +end + +function closeSession() +end + +function routeQuery(sql) + print("LUA: routeQuery") + print(sql) + if (string.find(sql, "LUA_INFINITE_LOOP")) + then + while (true) + do + end + end +end + +function clientReply() + print("LUA: clientReply") +end + +function diagnostic() + return "Lua routeQuery will not return if sql = select LUA_INFINITE_LOOP" +end diff --git a/maxscale-system-test/mxs2057_systemd_watchdog.cpp b/maxscale-system-test/mxs2057_systemd_watchdog.cpp new file mode 100644 index 000000000..0d12de693 --- /dev/null +++ b/maxscale-system-test/mxs2057_systemd_watchdog.cpp @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2018 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl11. + * + * Change Date: 2022-01-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ + +#include "testconnections.h" +#include + +namespace +{ +// watchdog_interval 60 seconds, make sure it is the same in maxscale.service +const maxbase::Duration watchdog_interval {60.0}; + +// Return true if maxscale stays alive for the duration dur. +bool staying_alive(TestConnections& test, const maxbase::Duration& dur) +{ + bool alive = true; + maxbase::StopWatch sw_loop_start; + while (alive && sw_loop_start.split() < dur) + { + if (execute_query_silent(test.maxscales->conn_rwsplit[0], "select 1")) + { + alive = false; + break; + } + } + + return alive; +} + +// The bulk of the test. +void test_watchdog(TestConnections& test, int argc, char* argv[]) +{ + test.check_log_err(0, "The systemd watchdog is Enabled", true); + + // Wait for one watchdog interval, systemd should have been notified in that time. + bool maxscale_alive = staying_alive(test, watchdog_interval); + + test.check_log_err(0, "systemd watchdog keep-alive ping", true); + + test.set_timeout(2 * watchdog_interval.secs()); + + // Make one thread in maxscale hang + mysql_query(test.maxscales->conn_rwsplit[0], "select LUA_INFINITE_LOOP"); + + // maxscale should get killed by systemd in less than duration(interval - epsilon). + maxscale_alive = staying_alive(test, maxbase::Duration(1.2 * watchdog_interval.secs())); + + if (maxscale_alive) + { + test.add_result(true, "Although the systemd watchdog is enabled, " + "systemd did not terminate maxscale!"); + } + else + { + test.check_log_err(0, "received fatal signal 6", true); + if (test.global_result == 0) + { + test.tprintf("Maxscale was killed by systemd - ok"); + } + } +} +} + +int main(int argc, char* argv[]) +{ + TestConnections::skip_maxscale_start(true); + TestConnections test {argc, argv}; + + std::string lua_file("/infinite_loop.lua"); + std::string from = test_dir + lua_file; + std::string to = "/home/vagrant" + lua_file; + + test.maxscales->copy_to_node(0, from.c_str(), to.c_str()); + test.maxscales->start(); + sleep(2); + test.maxscales->wait_for_monitor(); + test.maxscales->connect_maxscale(0); + + if (!test.global_result) + { + test_watchdog(test, argc, argv); + } + + return test.global_result; +} diff --git a/server/core/routingworker.cc b/server/core/routingworker.cc index 6e2639cc4..23aaac277 100644 --- a/server/core/routingworker.cc +++ b/server/core/routingworker.cc @@ -988,7 +988,7 @@ void maxscale::RoutingWorker::set_watchdog_interval(uint64_t microseconds) // Do not call anything from here, assume nothing has been initialized (like logging). // The internal timeout is 2/3 of the systemd configured interval. - double seconds = 2.0 * microseconds / 3000000; + double seconds = 1.0 * microseconds / 2000000; s_watchdog_interval = maxbase::Duration(seconds); s_watchdog_next_check = maxbase::Clock::now();