Add tests with big backend

Basic tests can be executed with 15 machines Master/slave backend. Tests have label BIG_REPL_BACKEND Default template modified to support big backend. Tests temporaraly labeled as UNSTABLE to prevent their execution nightly
For big test maxscale.cnf is automatically generated for any number of nodes

maxscale-system-test/CMakeLists.txt

@@ -1054,34 +1054,42 @@ add_test_executable(mxs2057_systemd_watchdog.cpp mxs2057_systemd_watchdog mxs205
 #add_test_executable_notest(long_test.cpp long_test_big replication LABELS readwritesplit REPL_BACKEND BIG_REPL_BACKEND UNSTABLE HEAVY)
 
 # Tries INSERTs with size close to 0x0ffffff * N
-add_test_derived(different_size_rwsplit_big different_size_rwsplit replication_big LABELS readwritesplit UNSTABLE HEAVY REPL_BACKEND BIG_REPL_BACKEND)
+add_test_derived(different_size_rwsplit_big different_size_rwsplit replication LABELS readwritesplit UNSTABLE HEAVY REPL_BACKEND BIG_REPL_BACKEND UNSTABLE)
 
 # Check how Maxscale works in case of one slave failure, only one slave is configured
-add_test_derived(slave_failover_big slave_failover replication.one_slave_big LABELS readwritesplit REPL_BACKEND BIG_REPL_BACKEND UNSTABLE HEAVY)
+add_test_derived(slave_failover_big slave_failover replication.one_slave LABELS readwritesplit REPL_BACKEND BIG_REPL_BACKEND UNSTABLE HEAVY)
 
 # Execute queries of different size, check data is the same when accessing via Maxscale and directly to backend
-add_test_derived(sql_queries_big sql_queries replication_big LABELS readwritesplit REPL_BACKEND BIG_REPL_BACKEND UNSTABLE HEAVY)
+add_test_derived(sql_queries_big sql_queries replication LABELS readwritesplit REPL_BACKEND BIG_REPL_BACKEND UNSTABLE HEAVY)
 
 # Check temporal tables commands functionality
-add_test_derived(temporal_tables_big temporal_tables replication_big LABELS readwritesplit REPL_BACKEND BIG_REPL_BACKEND UNSTABLE HEAVY)
+add_test_derived(temporal_tables_big temporal_tables replication LABELS readwritesplit REPL_BACKEND BIG_REPL_BACKEND UNSTABLE HEAVY)
 
 # Check if prepared statement works via Maxscale (via RWSplit)
-add_test_derived(prepared_statement_big prepared_statement replication_big LABELS readwritesplit REPL_BACKEND BIG_REPL_BACKEND UNSTABLE HEAVY)
+add_test_derived(prepared_statement_big prepared_statement replication LABELS readwritesplit REPL_BACKEND BIG_REPL_BACKEND UNSTABLE HEAVY)
 
 # Connect to ReadConn in master mode and check if there is only one backend connection to master
-add_test_derived(readconnrouter_master_big readconnrouter_master replication_big LABELS readconnroute REPL_BACKEND BIG_REPL_BACKEND UNSTABLE HEAVY)
+add_test_derived(readconnrouter_master_big readconnrouter_master replication LABELS readconnroute REPL_BACKEND BIG_REPL_BACKEND UNSTABLE HEAVY)
 
 # Creates 100 connections to ReadConn in slave mode and check if connections are distributed among all slaves
-add_test_derived(readconnrouter_slave_big readconnrouter_slave replication_big LABELS readconnroute REPL_BACKEND BIG_REPL_BACKEND UNSTABLE HEAVY)
+add_test_derived(readconnrouter_slave_big readconnrouter_slave replication LABELS readconnroute REPL_BACKEND BIG_REPL_BACKEND UNSTABLE HEAVY)
 
 # Checks changes of COM_SELECT and COM_INSERT after queris to check if RWSplit sends queries to master or to slave depending on if it is write or read only query
-add_test_derived(rw_select_insert_big rw_select_insert replication_big LABELS readwritesplit REPL_BACKEND BIG_REPL_BACKEND UNSTABLE HEAVY)
+add_test_derived(rw_select_insert_big rw_select_insert replication LABELS readwritesplit REPL_BACKEND BIG_REPL_BACKEND UNSTABLE HEAVY)
 
 # Checks connections are distributed equaly among backends
-add_test_derived(rwsplit_conn_num_big rwsplit_conn_num repl_lgc_big LABELS readwritesplit REPL_BACKEND BIG_REPL_BACKEND UNSTABLE HEAVY)
+add_test_derived(rwsplit_conn_num_big rwsplit_conn_num repl_lgc LABELS readwritesplit REPL_BACKEND BIG_REPL_BACKEND UNSTABLE HEAVY)
 
 # Check that there is one connection to Master and one connection to one of slaves
-add_test_derived(rwsplit_connect_big rwsplit_connect replication_big LABELS readwritesplit REPL_BACKEND BIG_REPL_BACKEND UNSTABLE HEAVY)
+add_test_derived(rwsplit_connect_big rwsplit_connect replication LABELS readwritesplit REPL_BACKEND BIG_REPL_BACKEND UNSTABLE HEAVY)
+
+# Regression cases for the bug "Hint filter don't work if listed before regex filter in configuration file"
+# (different filter sequence and configuration, but the same test, see .cnf for details)
+add_test_derived(bug585_big bug587 bug585 LABELS regexfilter BIG_REPL_BACKEND REPL_BACKEND UNSTABLE)
+add_test_derived(bug587_big bug587 bug587 LABELS regexfilter hintfilter BIG_REPL_BACKEND REPL_BACKEND UNSTABLE)
+add_test_derived(bug587_1_big bug587 bug587_1 LABELS regexfilter hintfilter BIG_REPL_BACKEND REPL_BACKEND UNSTABLE)
+# Regression case for the bug "Routing Hints route to server sometimes doesn't work"
+add_test_derived(bug471_big bug471 bug471_big LABELS readwritesplit hintfilter BIG_REPL_BACKEND REPL_BACKEND UNSTABLE)
 
 set_tests_properties(different_size_rwsplit_big PROPERTIES TIMEOUT 3600)
 set_tests_properties(different_size_rwsplit PROPERTIES TIMEOUT 3600)
@@ -1094,6 +1102,10 @@ set_tests_properties(readconnrouter_slave_big PROPERTIES TIMEOUT 3600)
 set_tests_properties(rw_select_insert_big PROPERTIES TIMEOUT 3600)
 set_tests_properties(rwsplit_conn_num_big PROPERTIES TIMEOUT 3600)
 set_tests_properties(rwsplit_connect_big PROPERTIES TIMEOUT 3600)
+set_tests_properties(bug585_big PROPERTIES TIMEOUT 3600)
+set_tests_properties(bug587_big PROPERTIES TIMEOUT 3600)
+set_tests_properties(bug587_1_big PROPERTIES TIMEOUT 3600)
+set_tests_properties(bug471_big PROPERTIES TIMEOUT 3600)
 
 ############################################
 # END: tests with 15 machines backend    #

maxscale-system-test/backend_auth_fail.cpp

@@ -10,7 +10,6 @@ int main(int argc, char** argv)
 {
     MYSQL* mysql[1000];
     TestConnections* Test = new TestConnections(argc, argv);
-    Test->repl->limit_nodes(4);
 
     Test->repl->execute_query_all_nodes((char*) "set global max_connections = 10;");
 
@@ -34,7 +33,7 @@ int main(int argc, char** argv)
 
     // Wait for the connections to clean up
     Test->stop_timeout();
-    sleep(5);
+    sleep(2 * Test->repl->N);
 
     Test->check_maxscale_alive(0);
     int rval = Test->global_result;

maxscale-system-test/bug471.cpp

@@ -152,7 +152,7 @@ using namespace std;
 int main(int argc, char* argv[])
 {
     TestConnections* Test = new TestConnections(argc, argv);
-    Test->repl->limit_nodes(4);
+    Test->repl->limit_nodes(Test->maxscales->get_backend_servers_num(0, "RW-Split-Router"));
     Test->set_timeout(10);
 
     Test->repl->connect();

maxscale-system-test/cnf/maxscale.cnf.template.bug471

@@ -22,7 +22,7 @@ replace=select
 [RW Split Router]
 type=service
 router= readwritesplit
-servers=server1,     server2,              server3,server4
+servers=###server_line###
 user=maxskysql
 password=skysql
 max_slave_connections=100%
@@ -77,6 +77,7 @@ protocol=maxscaled
 
 socket=default
 
+
 [server1]
 type=server
 address=###node_server_IP_1###

maxscale-system-test/cnf/maxscale.cnf.template.bug471_big

@@ -0,0 +1,80 @@
+[maxscale]
+threads=###threads###
+log_warning=1
+
+[MySQL Monitor]
+type=monitor
+module=mysqlmon
+servers=server1,server3,server4,server5,server6,server7,server8,server9,server10,server11,server12,server13,server14,server15
+user=maxskysql
+password= skysql
+
+[hints]
+type=filter
+module=hintfilter
+
+[regex]
+type=filter
+module=regexfilter
+match=fetch
+replace=select
+
+[RW Split Router]
+type=service
+router= readwritesplit
+servers=###server_line###
+user=maxskysql
+password=skysql
+max_slave_connections=100%
+use_sql_variables_in=all
+slave_selection_criteria=LEAST_BEHIND_MASTER
+filters=hints|regex
+
+
+[Read Connection Router Slave]
+type=service
+router=readconnroute
+router_options= slave
+servers=server1,server2,server3,server4
+user=maxskysql
+password=skysql
+
+[Read Connection Router Master]
+type=service
+router=readconnroute
+router_options=master
+servers=server1,server2,server3,server4
+user=maxskysql
+password=skysql
+
+[RW Split Listener]
+type=listener
+service=RW Split Router
+protocol=MySQLClient
+port=4006
+#socket=/tmp/rwsplit.sock
+
+[Read Connection Listener Slave]
+type=listener
+service=Read Connection Router Slave
+protocol=MySQLClient
+port=4009
+
+[Read Connection Listener Master]
+type=listener
+service=Read Connection Router Master
+protocol=MySQLClient
+port=4008
+
+[CLI]
+type=service
+router=cli
+
+[CLI Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+
+socket=default
+
+###server###

maxscale-system-test/cnf/maxscale.cnf.template.bug585

@@ -5,7 +5,7 @@ log_warning=1
 [MySQL Monitor]
 type=monitor
 module=mysqlmon
-servers= server1, server2,server3  ,server4
+servers=###server_line###
 user=maxskysql
 password= skysql
 
@@ -24,7 +24,7 @@ replace=from
 [RW Split Router]
 type=service
 router= readwritesplit
-servers=server1,     server2,              server3,server4
+servers=###server_line###
 user=maxskysql
 password=skysql
 max_slave_connections=100%
@@ -82,27 +82,4 @@ protocol=maxscaled
 
 socket=default
 
-[server1]
+###server###
-type=server
-address=###node_server_IP_1###
-port=###node_server_port_1###
-protocol=MySQLBackend
-
-[server2]
-type=server
-address=###node_server_IP_2###
-port=###node_server_port_2###
-protocol=MySQLBackend
-
-[server3]
-type=server
-address=###node_server_IP_3###
-port=###node_server_port_3###
-protocol=MySQLBackend
-
-[server4]
-type=server
-address=###node_server_IP_4###
-port=###node_server_port_4###
-protocol=MySQLBackend
-

maxscale-system-test/cnf/maxscale.cnf.template.bug587

@@ -5,7 +5,7 @@ log_warning=1
 [MySQL Monitor]
 type=monitor
 module=mysqlmon
-servers= server1, server2,server3  ,server4
+servers=###server_line###
 user=maxskysql
 password= skysql
 
@@ -22,7 +22,7 @@ replace=select
 [RW Split Router]
 type=service
 router= readwritesplit
-servers=server1,     server2,              server3,server4
+servers=###server_line###
 user=maxskysql
 password=skysql
 max_slave_connections=100%
@@ -76,27 +76,4 @@ protocol=maxscaled
 
 socket=default
 
-[server1]
+###server###
-type=server
-address=###node_server_IP_1###
-port=###node_server_port_1###
-protocol=MySQLBackend
-
-[server2]
-type=server
-address=###node_server_IP_2###
-port=###node_server_port_2###
-protocol=MySQLBackend
-
-[server3]
-type=server
-address=###node_server_IP_3###
-port=###node_server_port_3###
-protocol=MySQLBackend
-
-[server4]
-type=server
-address=###node_server_IP_4###
-port=###node_server_port_4###
-protocol=MySQLBackend
-

maxscale-system-test/cnf/maxscale.cnf.template.bug587_1

@@ -5,7 +5,7 @@ log_warning=1
 [MySQL Monitor]
 type=monitor
 module=mysqlmon
-servers= server1, server2,server3  ,server4
+servers=###server_line###
 user=maxskysql
 password= skysql
 
@@ -22,7 +22,7 @@ replace=select
 [RW Split Router]
 type=service
 router= readwritesplit
-servers=server1,     server2,              server3,server4
+servers=###server_line###
 user=maxskysql
 password=skysql
 max_slave_connections=100%
@@ -76,27 +76,4 @@ protocol=maxscaled
 
 socket=default
 
-[server1]
+###server###
-type=server
-address=###node_server_IP_1###
-port=###node_server_port_1###
-protocol=MySQLBackend
-
-[server2]
-type=server
-address=###node_server_IP_2###
-port=###node_server_port_2###
-protocol=MySQLBackend
-
-[server3]
-type=server
-address=###node_server_IP_3###
-port=###node_server_port_3###
-protocol=MySQLBackend
-
-[server4]
-type=server
-address=###node_server_IP_4###
-port=###node_server_port_4###
-protocol=MySQLBackend
-

maxscale-system-test/cnf/maxscale.cnf.template.repl_lgc

@@ -5,7 +5,7 @@ log_warning=1
 [MySQL Monitor]
 type=monitor
 module=mysqlmon
-servers= server1, server2,server3  ,server4
+servers=###server_line###
 user=maxskysql
 password= skysql
 monitor_interval=1000
@@ -13,7 +13,7 @@ monitor_interval=1000
 [RW Split Router]
 type=service
 router= readwritesplit
-servers=server1,     server2,              server3,server4
+servers=###server_line###
 user=maxskysql
 password=skysql
 slave_selection_criteria=LEAST_GLOBAL_CONNECTIONS
@@ -23,7 +23,7 @@ max_slave_connections=1
 type=service
 router=readconnroute
 router_options= slave
-servers=server1,server2,server3,server4
+servers=###server_line###
 user=maxskysql
 password=skysql
 
@@ -31,7 +31,7 @@ password=skysql
 type=service
 router=readconnroute
 router_options=master
-servers=server1,server2,server3,server4
+servers=###server_line###
 user=maxskysql
 password=skysql
 
@@ -65,27 +65,4 @@ protocol=maxscaled
 #address=localhost
 socket=default
 
-[server1]
+###server###
-type=server
-address=###node_server_IP_1###
-port=###node_server_port_1###
-protocol=MySQLBackend
-
-[server2]
-type=server
-address=###node_server_IP_2###
-port=###node_server_port_2###
-protocol=MySQLBackend
-
-[server3]
-type=server
-address=###node_server_IP_3###
-port=###node_server_port_3###
-protocol=MySQLBackend
-
-[server4]
-type=server
-address=###node_server_IP_4###
-port=###node_server_port_4###
-protocol=MySQLBackend
-

maxscale-system-test/cnf/maxscale.cnf.template.replication

@@ -5,7 +5,7 @@ threads=###threads###
 [MySQL-Monitor]
 type=monitor
 module=mysqlmon
-servers=server1,server2,server3,server4
+servers=###server_line###
 user=maxskysql
 password=skysql
 monitor_interval=1000
@@ -15,7 +15,7 @@ detect_standalone_master=false
 [RW-Split-Router]
 type=service
 router=readwritesplit
-servers=server1,server2,server3,server4
+servers=###server_line###
 user=maxskysql
 password=skysql
 slave_selection_criteria=LEAST_GLOBAL_CONNECTIONS
@@ -25,7 +25,7 @@ max_slave_connections=1
 type=service
 router=readconnroute
 router_options=slave
-servers=server1,server2,server3,server4
+servers=###server_line###
 user=maxskysql
 password=skysql
 
@@ -33,7 +33,7 @@ password=skysql
 type=service
 router=readconnroute
 router_options=master
-servers=server1,server2,server3,server4
+servers=###server_line###
 user=maxskysql
 password=skysql
 
@@ -65,26 +65,4 @@ service=CLI
 protocol=maxscaled
 socket=default
 
-[server1]
+###server###
-type=server
-address=###node_server_IP_1###
-port=###node_server_port_1###
-protocol=MySQLBackend
-
-[server2]
-type=server
-address=###node_server_IP_2###
-port=###node_server_port_2###
-protocol=MySQLBackend
-
-[server3]
-type=server
-address=###node_server_IP_3###
-port=###node_server_port_3###
-protocol=MySQLBackend
-
-[server4]
-type=server
-address=###node_server_IP_4###
-port=###node_server_port_4###
-protocol=MySQLBackend

maxscale-system-test/cnf/maxscale.cnf.template.replication.one_slave

@@ -5,7 +5,7 @@ log_warning=1
 [MySQL Monitor]
 type=monitor
 module=mysqlmon
-servers=server1,server2,server3,server4
+servers=###server_line###
 user=maxskysql
 password=skysql
 
@@ -13,7 +13,7 @@ password=skysql
 type=service
 router=readwritesplit
 max_slave_connections=1
-servers=server1,server2,server3,server4
+servers=###server_line###
 user=maxskysql
 password=skysql
 disable_sescmd_history=false
@@ -22,7 +22,7 @@ disable_sescmd_history=false
 type=service
 router=readconnroute
 router_options=slave
-servers=server1,server2,server3,server4
+servers=###server_line###
 user=maxskysql
 password=skysql
 
@@ -30,7 +30,7 @@ password=skysql
 type=service
 router=readconnroute
 router_options=master
-servers=server1,server2,server3,server4
+servers=###server_line###
 user=maxskysql
 password=skysql
 
@@ -63,27 +63,4 @@ service=CLI
 protocol=maxscaled
 socket=default
 
-[server1]
+###server###
-type=server
-address=###node_server_IP_1###
-port=###node_server_port_1###
-protocol=MySQLBackend
-
-[server2]
-type=server
-address=###node_server_IP_2###
-port=###node_server_port_2###
-protocol=MySQLBackend
-
-[server3]
-type=server
-address=###node_server_IP_3###
-port=###node_server_port_3###
-protocol=MySQLBackend
-
-[server4]
-type=server
-address=###node_server_IP_4###
-port=###node_server_port_4###
-protocol=MySQLBackend
-

maxscale-system-test/mariadb_nodes.cpp

@@ -51,6 +51,15 @@ Mariadb_nodes::Mariadb_nodes(const char *pref, const char *test_cwd, bool verbos
     truncate_mariadb_logs();
     flush_hosts();
     close_active_connections();
+    cnf_server_name = std::string(prefix);
+    if (strcmp(prefix, "node") == 0)
+    {
+        cnf_server_name = std::string("server");
+    }
+    if (strcmp(prefix, "galera") == 0)
+    {
+        cnf_server_name = std::string("gserver");
+    }
 }
 
 Mariadb_nodes::~Mariadb_nodes()
@@ -1453,3 +1462,32 @@ void Mariadb_nodes::limit_nodes(int new_N)
         sleep(10);
     }
 }
+
+std::string Mariadb_nodes::cnf_servers()
+{
+    std::string s;
+    for (int i = 0; i < N; i++)
+    {
+        s += std::string("\\n[") +
+                cnf_server_name +
+                std::to_string(i + 1) +
+                std::string("]\\ntype=server\\naddress=") +
+                std::string(IP[i]) +
+                std::string("\\nport=") +
+                std::to_string(port[i]) +
+                std::string("\\nprotocol=MySQLBackend\\n");
+    }
+    return s;
+}
+
+std::string Mariadb_nodes::cnf_servers_line()
+{
+    std::string s = cnf_server_name + std::to_string(1);
+    for (int i = 1; i < N; i++)
+    {
+        s += std::string(",") +
+                cnf_server_name +
+                std::to_string(i + 1);
+    }
+    return s;
+}

maxscale-system-test/mariadb_nodes.h

@@ -482,6 +482,23 @@ public:
      */
     void limit_nodes(int new_N);
 
+    /**
+     * @brief cnf_servers Generates backend servers description for maxscale.cnf
+     * @return Servers description including IPs, ports
+     */
+    std::string cnf_servers();
+
+    /**
+     * @brief cnf_servers_line Generates list of backend servers for serivces definition in maxscale.cnf
+     * @return List of servers, e.g server1,server2,server3,...
+     */
+    std::string cnf_servers_line();
+
+    /**
+     * @brief cnf_server_name Prefix for backend server name ('server', 'gserver')
+     */
+    std::string cnf_server_name;
+
 private:
 
     bool check_master_node(MYSQL* conn);

maxscale-system-test/maxscales.cpp

@@ -332,6 +332,21 @@ int Maxscales::get_maxadmin_param(int m, const char* command, const char* param,
     return exit_code;
 }
 
+int Maxscales::get_backend_servers_num(int m, const char* service)
+{
+    char* buf;
+    int exit_code;
+    int i = 0;
+
+    buf = ssh_node_output_f(m, true, &exit_code, "maxadmin show service %s | grep Name: | grep Protocol: | wc -l", service);
+    if (buf && !exit_code)
+    {
+        sscanf(buf, "%d", &i);
+    }
+    return i;
+}
+
+
 
 long unsigned Maxscales::get_maxscale_memsize(int m)
 {

maxscale-system-test/maxscales.h

@@ -288,8 +288,17 @@ public:
     int check_maxadmin_param(int m, const char* command, const char* param, const char* value);
     int get_maxadmin_param(int m, const char* command, const char* param, char* result);
 
+    /**
+     * @brief get_backend_servers_num Gets number of backend servers configure for service
+     * @param m Number of Maxscale node
+     * @param service Name of service to ask
+     * @return number of backend servers
+     */
+    int get_backend_servers_num(int m, const char* service);
+
     /**
      * @brief get_maxscale_memsize Gets size of the memory consumed by Maxscale process
+     * @param m Number of Maxscale node
      * @return memory size in kilobytes
      */
     long unsigned get_maxscale_memsize(int m = 0);

maxscale-system-test/testconnections.cpp

@@ -774,6 +774,11 @@ void TestConnections::process_template(int m, const char* template_name, const c
                 system(str);
             }
 
+            sprintf(str, "sed -i \"s/###%s###/%s/\" maxscale.cnf", mdn[j]->cnf_server_name.c_str(), mdn[j]->cnf_servers().c_str());
+            system(str);
+            sprintf(str, "sed -i \"s/###%s_line###/%s/\" maxscale.cnf", mdn[j]->cnf_server_name.c_str(), mdn[j]->cnf_servers_line().c_str());
+            system(str);
+
             mdn[j]->connect();
             execute_query(mdn[j]->nodes[0], (char *) "CREATE DATABASE IF NOT EXISTS test");
             mdn[j]->close_connections();