MXS-1585: Add preliminary test case

The test case attempts to simulate the environment where the crash appears
to have happened. Local testing does not point out any problems.

maxscale-system-test/CMakeLists.txt

@@ -514,6 +514,10 @@ add_test_executable(mxs1476.cpp mxs1476 mxs1476 LABELS GALERA_BACKEND)
  # https://jira.mariadb.org/browse/MXS-1509
 add_test_executable(mxs1509.cpp mxs1509 mxs1509 LABELS REPL_BACKEND)
 
+ # MXS-1585: Crash in MaxScale 2.1.12
+ # https://jira.mariadb.org/browse/MXS-1585
+add_test_executable(mxs1585.cpp mxs1585 mxs1585 LABELS REPL_BACKEND)
+
 # 'namedserverfilter' test
 add_test_executable(namedserverfilter.cpp namedserverfilter namedserverfilter LABELS namedserverfilter LIGHT REPL_BACKEND)
 

maxscale-system-test/cnf/maxscale.cnf.template.mxs1585

@@ -0,0 +1,88 @@
+[maxscale]
+threads=###threads###
+
+[Galera Monitor]
+type=monitor
+module=galeramon
+servers=server1,server2,server3,server4
+user=maxskysql
+passwd=skysql
+monitor_interval=1000
+root_node_as_master=false
+
+[RW Split Router]
+type=service
+router=readwritesplit
+servers=server1,server2,server3,server4
+user=maxskysql
+passwd=skysql
+master_failure_mode=fail_on_write
+
+[Read Connection Router Slave]
+type=service
+router=readconnroute
+router_options=slave
+servers=server1,server2,server3,server4
+user=maxskysql
+passwd=skysql
+
+[Read Connection Router Master]
+type=service
+router=readconnroute
+router_options=master
+servers=server1,server2,server3,server4
+user=maxskysql
+passwd=skysql
+
+[RW Split Listener]
+type=listener
+service=RW Split Router
+protocol=MySQLClient
+port=4006
+
+[Read Connection Listener Slave]
+type=listener
+service=Read Connection Router Slave
+protocol=MySQLClient
+port=4009
+
+[Read Connection Listener Master]
+type=listener
+service=Read Connection Router Master
+protocol=MySQLClient
+port=4008
+
+[CLI]
+type=service
+router=cli
+
+[CLI Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=default
+
+[server1]
+type=server
+address=###galera_server_IP_1###
+port=###galera_server_port_1###
+protocol=MySQLBackend
+
+[server2]
+type=server
+address=###galera_server_IP_2###
+port=###galera_server_port_2###
+protocol=MySQLBackend
+
+[server3]
+type=server
+address=###galera_server_IP_3###
+port=###galera_server_port_3###
+protocol=MySQLBackend
+
+[server4]
+type=server
+address=###galera_server_IP_4###
+port=###galera_server_port_4###
+protocol=MySQLBackend
+

maxscale-system-test/mxs1585.cpp

@@ -0,0 +1,87 @@
+/**
+ * MXS-1585: https://jira.mariadb.org/browse/MXS-1585
+ *
+ * Check that MaxScale doesn't crash when the master is set into maintenance
+ * mode when master_failure_mode is fail_on_write.
+ */
+
+#include "testconnections.h"
+#include <vector>
+
+static bool running = true;
+
+void* query_thr(void* data)
+{
+    TestConnections* test = (TestConnections*)data;
+
+    while (running)
+    {
+        MYSQL* mysql = test->open_rwsplit_connection();
+
+        while (running)
+        {
+            if (mysql_query(mysql, "SET sql_log_bin = 0") ||
+                mysql_query(mysql, "INSERT INTO test.mxs1585 VALUES (1)") ||
+                mysql_query(mysql, "DELETE FROM test.mxs1585"))
+            {
+                break;
+            }
+        }
+
+        mysql_close(mysql);
+    }
+
+    return NULL;
+}
+
+int main(int argc, char** argv)
+{
+    TestConnections test(argc, argv);
+
+    test.connect_maxscale();
+    test.try_query(test.conn_rwsplit, "DROP TABLE IF EXISTS test.mxs1585");
+    test.try_query(test.conn_rwsplit, "CREATE TABLE test.mxs1585(id INT) ENGINE=MEMORY");
+    test.close_maxscale_connections();
+
+    std::vector<pthread_t> threads;
+    threads.resize(100);
+
+    for (auto& a: threads)
+    {
+        pthread_create(&a, NULL, query_thr, &test);
+    }
+
+    for (int i = 0; i < 15; i++)
+    {
+        for (int x = 1; x <= 4; x++)
+        {
+            test.ssh_maxscale(true, "maxadmin set server server%d maintenance", x);
+            sleep(1);
+            test.ssh_maxscale(true, "maxadmin clear server server%d maintenance", x);
+            sleep(2);
+
+            test.ssh_maxscale(true, "maxadmin remove server server%d \"RW Split Router\" \"Galera Monitor\"", x);
+            sleep(1);
+            test.ssh_maxscale(true, "maxadmin add server server%d \"RW Split Router\" \"Galera Monitor\"", x);
+            sleep(2);
+
+            test.galera->block_node(x - 1);
+            sleep(5);
+            test.galera->unblock_node(x - 1);
+            sleep(5);
+        }
+    }
+
+    running = false;
+
+    for (auto& a: threads)
+    {
+        pthread_join(a, NULL);
+    }
+
+    test.connect_maxscale();
+    test.try_query(test.conn_rwsplit, "DROP TABLE test.mxs1585");
+    test.check_maxscale_alive();
+
+    return test.global_result;
+}

server/modules/routing/readwritesplit/rwsplit_route_stmt.c

@@ -1090,6 +1090,8 @@ bool handle_slave_is_target(ROUTER_INSTANCE *inst, ROUTER_CLIENT_SES *rses,
 static void log_master_routing_failure(ROUTER_CLIENT_SES *rses, bool found,
                                        DCB *master_dcb, DCB *curr_master_dcb)
 {
+    ss_dassert(!master_dcb || master_dcb->dcb_role == DCB_ROLE_BACKEND_HANDLER);
+    ss_dassert(!curr_master_dcb || curr_master_dcb->dcb_role == DCB_ROLE_BACKEND_HANDLER);
     char errmsg[MAX_SERVER_NAME_LEN * 2 + 100]; // Extra space for error message
 
     if (!found)
@@ -1100,6 +1102,7 @@ static void log_master_routing_failure(ROUTER_CLIENT_SES *rses, bool found,
     {
         /** We found a master but it's not the same connection */
         ss_dassert(master_dcb != curr_master_dcb);
+        ss_dassert(master_dcb->server && curr_master_dcb->server);
         if (master_dcb->server != curr_master_dcb->server)
         {
             sprintf(errmsg, "Master server changed from '%s' to '%s'",