From ed3548b825bf7f5934ae9ad6e43196d257de1913 Mon Sep 17 00:00:00 2001 From: Esa Korhonen Date: Fri, 15 Nov 2019 14:40:34 +0200 Subject: [PATCH] MXS-2770 Fix mxs2167_extra_port 1) Only two backends are set up for extra-port 2) The setting is checked to work by connecting directly to servers 3) The server connections are saturated before starting MaxScale 4) MaxScale logs are checked for extra-port-related messages --- .../maxscale.cnf.template.mxs2167_extra_port | 2 - maxscale-system-test/mxs2167_extra_port.cpp | 216 +++++++++++++----- 2 files changed, 158 insertions(+), 60 deletions(-) diff --git a/maxscale-system-test/cnf/maxscale.cnf.template.mxs2167_extra_port b/maxscale-system-test/cnf/maxscale.cnf.template.mxs2167_extra_port index de10d1d78..71dae8b3e 100755 --- a/maxscale-system-test/cnf/maxscale.cnf.template.mxs2167_extra_port +++ b/maxscale-system-test/cnf/maxscale.cnf.template.mxs2167_extra_port @@ -53,11 +53,9 @@ type=server address=###node_server_IP_3### port=###node_server_port_3### protocol=MySQLBackend -extra_port=33066 [server4] type=server address=###node_server_IP_4### port=###node_server_port_4### protocol=MySQLBackend -extra_port=33066 diff --git a/maxscale-system-test/mxs2167_extra_port.cpp b/maxscale-system-test/mxs2167_extra_port.cpp index 40e6e743c..f2e78bb9b 100644 --- a/maxscale-system-test/mxs2167_extra_port.cpp +++ b/maxscale-system-test/mxs2167_extra_port.cpp @@ -7,89 +7,189 @@ using std::cout; using std::endl; - +using std::string; int main(int argc, char** argv) { TestConnections test(argc, argv); - cout << "Stopping MaxScale" << endl; + // The test requires at least three nodes. + auto N = test.repl->N; + if (N < 3) + { + test.expect(false, "Too few nodes, need at least 3."); + return test.global_result; + } + + test.tprintf("Stopping MaxScale"); test.maxscales->stop(); - cout << "Configuring servers" << endl; - // Add the extra_port parameter to all servers - for (int i = 0; i < test.repl->N; i++) + // Configure extra-port on servers 1&2. + const int N_extra_port = 2; + const int extra_port = 33066; + const string extra_port_str = std::to_string(extra_port); + const string iptables_cmd = + "iptables -I INPUT -p tcp --dport " + extra_port_str + " -j ACCEPT;" + + "ip6tables -I INPUT -p tcp --dport " + extra_port_str + " -j ACCEPT"; + + const string iptables_remove_cmd = + "iptables -D INPUT -p tcp --dport " + extra_port_str + " -j ACCEPT;" + + "ip6tables -D INPUT -p tcp --dport " + extra_port_str + " -j ACCEPT"; + + const string extra_port_sett = "extra_port=" + extra_port_str; + + string user = test.repl->user_name; + string pw = test.repl->password; + + for (int i = 0; i < N_extra_port; i++) { + test.tprintf("Configuring node %i for extra port.", i); // Temporary workaround for firewall issues - test.repl->ssh_node_f(i, true, - "iptables -I INPUT -p tcp --dport 33066 -j ACCEPT;" - "ip6tables -I INPUT -p tcp --dport 33066 -j ACCEPT"); + test.repl->ssh_node_f(i, true, "%s", iptables_cmd.c_str()); test.repl->stash_server_settings(i); - test.repl->add_server_setting(i, "extra_port=33066"); + test.repl->add_server_setting(i, extra_port_sett.c_str()); + test.repl->add_server_setting(i, "extra_max_connections=5"); test.repl->ssh_node_f(i, true, "service mysql restart"); + + // Test a direct connection to the server through the extra port, it should work. + auto conn = open_conn_db_timeout(extra_port, test.repl->ip(i), "", user, pw, 4, false); + int rc = test.try_query(conn, "SELECT 1;"); + test.expect(rc == 0, "Connection from host machine to node %i through port %i failed.", + i, extra_port); + if (rc == 0) + { + test.tprintf("Extra port working on node %i.", i); + } + mysql_close(conn); } - test.repl->connect(); - test.try_query(test.repl->nodes[0], "%s", "CREATE USER 'monitor'@'%' IDENTIFIED BY 'monitor'"); - test.try_query(test.repl->nodes[0], "%s", "GRANT ALL ON *.* TO 'monitor'@'%'"); - test.repl->disconnect(); - - cout << "Starting MaxScale" << endl; - test.maxscales->start(); - - // Stop the monitor to force the connections to be closed - test.maxctrl("stop monitor MySQL-Monitor"); - - // Limit the connections to 20 (is erased on restart) - test.repl->connect(); - test.try_query(test.repl->nodes[0], "SET GLOBAL max_connections=20"); - test.repl->disconnect(); - - std::vector connections; - - // Open connections until we hit the limit - for (int i = 0; i < 40; i++) + if (test.ok()) { - cout << "Opening connection " << i << endl; - - MYSQL* conn = test.maxscales->open_rwsplit_connection(); - - if (execute_query_silent(conn, "SELECT 1") == 0) + // Limit the number of connections on servers 1&2. Resets on restart. + const int max_conns = 10; // 10 is minimum allowed by server. + const string set_max_conns = "SET GLOBAL max_connections=" + std::to_string(max_conns) + ";"; + test.repl->connect(); + for (int i = 0; i < N_extra_port; i++) { - connections.push_back(conn); + test.try_query(test.repl->nodes[i], "%s", set_max_conns.c_str()); + if (test.ok()) + { + test.tprintf("Max connections limit set on node %i.", i); + } } - else + test.repl->disconnect(); + + if (test.ok()) { - mysql_close(conn); - break; + // Then, open connections until the limit is met. Should open a total of 20. + // It seems this setting is not entirely accurate as sometimes one can open a few more. + const int max_conns_limit = max_conns + 5; + std::vector connections; + for (int i = 0; i < N_extra_port; i++) + { + test.tprintf("Opening connections on node %i until maximum reached.", i); + int normal_port = test.repl->port[i]; + auto host = test.repl->ip(i); + + int conn_count = 0; + while (conn_count < max_conns_limit) + { + auto conn = open_conn_db_timeout(normal_port, host, "", user, pw, 4, false); + if (conn && execute_query_silent(conn, "SELECT 1") == 0) + { + connections.push_back(conn); + conn_count++; + } + else + { + break; + } + } + + bool conns_ok = conn_count >= max_conns && conn_count <= max_conns_limit; + if (conns_ok) + { + test.tprintf("Opened %i connections to node %i.", conn_count, i); + } + else + { + test.expect(false, "Opened %i connections to node %i when %i--%i expected.", + conn_count, i, max_conns, max_conns_limit); + } + } + + if (test.ok()) + { + // Finally, start MaxScale. The monitor should use extra port to connect to nodes 0&1, + // and normal port to connect to 2&3. All servers should be running. + cout << "Starting MaxScale" << endl; + test.maxscales->start(); + sleep(3); // Give maxscale some time to start properly. + test.maxscales->wait_for_monitor(2); + for (int i = 0; i < N; i++) + { + string server_name = "server" + std::to_string(i + 1); + auto srv_namez = server_name.c_str(); + auto status = test.maxscales->get_server_status(srv_namez); + bool status_ok = status.count("Running") == 1; + if (status_ok) + { + string status_str; + for (auto s : status) + { + status_str += s + ","; + } + test.tprintf("%s status is: %s", srv_namez, status_str.c_str()); + } + test.expect(status.count("Running") == 1, "Server '%s' is not running or monitor could " + "not connect to it.", srv_namez); + // Also, MaxScale should have used the extra port to connect to nodes 0 & 1. + if (i < N_extra_port) + { + string pat = "Could not connect with normal port to server '" + server_name + + "', using extra_port"; + test.log_includes(0, pat.c_str()); + } + } + + if (test.ok()) + { + // Creating sessions should not work since normal connections cannot be created to + // the master node. + auto conn = test.maxscales->open_rwsplit_connection(); + if (!conn) + { + test.tprintf("Session creation failed, as expected."); + } + else if (execute_query_silent(conn, "SELECT 1;") == 1) + { + test.tprintf("Query failed, as expected."); + } + else + { + test.expect(false, "Routing sessions should not work."); + } + } + } + + // Make sure the old connections still work and close them. + for (auto conn : connections) + { + test.try_query(conn, "SELECT 2"); + mysql_close(conn); + } } } - // Start the monitor to force it to reconnect - test.maxctrl("start monitor MySQL-Monitor"); - test.maxscales->wait_for_monitor(); - - // Make sure the old connections still work - for (auto a : connections) - { - test.try_query(a, "SELECT 2"); - mysql_close(a); - } - - cout << "Stopping MaxScale" << endl; - test.maxscales->stop(); - // Remove extra_port - for (int i = 0; i < test.repl->N; i++) + for (int i = 0; i < N_extra_port; i++) { + test.tprintf("Removing extra port from node %i.", i); + test.repl->ssh_node_f(i, true, "%s", iptables_remove_cmd.c_str()); + test.repl->restore_server_settings(i); test.repl->ssh_node_f(i, true, "service mysql restart"); } - - test.repl->connect(); - test.try_query(test.repl->nodes[0], "%s", "DROP USER 'monitor'@'%'"); - test.repl->disconnect(); - return test.global_result; }