MXS-2057 System test

etc/maxscale.service.in

@@ -20,9 +20,7 @@ ExecStart=@CMAKE_INSTALL_PREFIX@/@MAXSCALE_BINDIR@/maxscale
 TimeoutStartSec=120
 LimitNOFILE=65535
 StartLimitBurst=0
-
-# Uncomment to enable the systemd Watchdog
-#WatchdogSec=30s
+WatchdogSec=60s
 
 # Only relevant when MaxScale is linked with -fsanitize=address
 Environment=ASAN_OPTIONS=abort_on_error=1

maxscale-system-test/CMakeLists.txt

@@ -956,6 +956,9 @@ add_test_executable(mxs1949_warn_user_injection.cpp mxs1949_warn_user_injection
 # MXS-2106: NULL values with avrorouter
 add_test_executable(mxs2106_avro_null.cpp mxs2106_avro_null avro LABELS avrorouter REPL_BACKEND)
 
+# MXS-2057 systemd watchdog
+add_test_executable(mxs2057_systemd_watchdog.cpp mxs2057_systemd_watchdog mxs2057_systemd_watchdog LABELS REPL_BACKEND)
+
 ############################################
 # END: binlogrouter and avrorouter tests   #
 ############################################

maxscale-system-test/cnf/maxscale.cnf.template.mxs2057_systemd_watchdog

@@ -0,0 +1,46 @@
+[maxscale]
+threads = ###threads###
+log_debug=1
+
+[Monitor]
+type = monitor
+module = mariadbmon
+servers = server1
+user = maxskysql
+password = skysql
+monitor_interval = 1000
+
+[RW-Split-Router]
+type = service
+router = readwritesplit
+servers = server1
+user = maxskysql
+password = skysql
+filters = lua
+
+[RW-Split-Listener]
+type = listener
+service = RW-Split-Router
+protocol = mariadbclient
+port = 4006
+
+[lua]
+type = filter
+module = luafilter
+session_script=/home/vagrant/infinite_loop.lua
+
+[MaxAdmin-Service]
+type = service
+router = cli
+
+[MaxAdmin-Unix-Listener]
+type = listener
+service = MaxAdmin-Service
+protocol = maxscaled
+socket = default
+
+[server1]
+type = server
+address = ###node_server_IP_1###
+port = ###node_server_port_1###
+protocol = MariaDBBackend

maxscale-system-test/infinite_loop.lua

@@ -0,0 +1,27 @@
+function createInstance()
+end
+
+function newSession(a, b)
+end
+
+function closeSession()
+end
+
+function routeQuery(sql)
+    print("LUA: routeQuery")
+    print(sql)
+    if (string.find(sql, "LUA_INFINITE_LOOP"))
+    then
+        while (true)
+        do
+        end
+    end
+end
+
+function clientReply()
+    print("LUA: clientReply")
+end
+
+function diagnostic()
+    return "Lua routeQuery will not return if sql = select LUA_INFINITE_LOOP"
+end

maxscale-system-test/mxs2057_systemd_watchdog.cpp

@@ -0,0 +1,94 @@
+/*
+ * Copyright (c) 2018 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2022-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#include "testconnections.h"
+#include <maxbase/stopwatch.hh>
+
+namespace
+{
+// watchdog_interval 60 seconds, make sure it is the same in maxscale.service
+const maxbase::Duration watchdog_interval {60.0};
+
+// Return true if maxscale stays alive for the duration dur.
+bool staying_alive(TestConnections& test, const maxbase::Duration& dur)
+{
+    bool alive = true;
+    maxbase::StopWatch sw_loop_start;
+    while (alive && sw_loop_start.split() < dur)
+    {
+        if (execute_query_silent(test.maxscales->conn_rwsplit[0], "select 1"))
+        {
+            alive = false;
+            break;
+        }
+    }
+
+    return alive;
+}
+
+// The bulk of the test.
+void test_watchdog(TestConnections& test, int argc, char* argv[])
+{
+    test.check_log_err(0, "The systemd watchdog is Enabled", true);
+
+    // Wait for one watchdog interval, systemd should have been notified in that time.
+    bool maxscale_alive = staying_alive(test, watchdog_interval);
+
+    test.check_log_err(0, "systemd watchdog keep-alive ping", true);
+
+    test.set_timeout(2 * watchdog_interval.secs());
+
+    // Make one thread in maxscale hang
+    mysql_query(test.maxscales->conn_rwsplit[0], "select LUA_INFINITE_LOOP");
+
+    // maxscale should get killed by systemd in less than duration(interval - epsilon).
+    maxscale_alive = staying_alive(test, maxbase::Duration(1.2 * watchdog_interval.secs()));
+
+    if (maxscale_alive)
+    {
+        test.add_result(true, "Although the systemd watchdog is enabled, "
+                              "systemd did not terminate maxscale!");
+    }
+    else
+    {
+        test.check_log_err(0, "received fatal signal 6", true);
+        if (test.global_result == 0)
+        {
+            test.tprintf("Maxscale was killed by systemd - ok");
+        }
+    }
+}
+}
+
+int main(int argc, char* argv[])
+{
+    TestConnections::skip_maxscale_start(true);
+    TestConnections test {argc, argv};
+
+    std::string lua_file("/infinite_loop.lua");
+    std::string from = test_dir + lua_file;
+    std::string to = "/home/vagrant" + lua_file;
+
+    test.maxscales->copy_to_node(0, from.c_str(), to.c_str());
+    test.maxscales->start();
+    sleep(2);
+    test.maxscales->wait_for_monitor();
+    test.maxscales->connect_maxscale(0);
+
+    if (!test.global_result)
+    {
+        test_watchdog(test, argc, argv);
+    }
+
+    return test.global_result;
+}

server/core/routingworker.cc

@@ -988,7 +988,7 @@ void maxscale::RoutingWorker::set_watchdog_interval(uint64_t microseconds)
     // Do not call anything from here, assume nothing has been initialized (like logging).
 
     // The internal timeout is 2/3 of the systemd configured interval.
-    double seconds = 2.0 * microseconds / 3000000;
+    double seconds = 1.0 * microseconds / 2000000;
 
     s_watchdog_interval = maxbase::Duration(seconds);
     s_watchdog_next_check = maxbase::Clock::now();