From 20cce59e1b18d8e2a6c8c58718dd418d04cfb27d Mon Sep 17 00:00:00 2001 From: Esa Korhonen Date: Wed, 29 Apr 2020 17:41:04 +0300 Subject: [PATCH] MXS-2976 Reinforce mysqlmon_multimaster test Compares server states as sets. Stops MaxScale during some cluster modifications to ensure consistency. --- .../mariadbmonitor/mysqlmon_multimaster.cnf | 33 +------- .../mariadbmonitor/mysqlmon_multimaster.cpp | 77 +++++++++++++------ 2 files changed, 56 insertions(+), 54 deletions(-) diff --git a/maxscale-system-test/mariadbmonitor/mysqlmon_multimaster.cnf b/maxscale-system-test/mariadbmonitor/mysqlmon_multimaster.cnf index 1da01387d..6beb51262 100644 --- a/maxscale-system-test/mariadbmonitor/mysqlmon_multimaster.cnf +++ b/maxscale-system-test/mariadbmonitor/mysqlmon_multimaster.cnf @@ -5,10 +5,9 @@ log_warning=1 [MySQL-Monitor] type=monitor module=mysqlmon -servers= server1, server2, server3, server4 +servers=server1, server2, server3, server4 user=maxskysql -password= skysql -detect_stale_master=0 +password=skysql monitor_interval=1000 [RW-Split-Router] @@ -20,40 +19,12 @@ password=skysql slave_selection_criteria=LEAST_ROUTER_CONNECTIONS max_slave_replication_lag=1 -[Read-Connection-Router-Slave] -type=service -router=readconnroute -router_options= slave -servers=server1,server2 -user=maxskysql -password=skysql - -[Read-Connection-Router-Master] -type=service -router=readconnroute -router_options=master -servers=server1,server2 -user=maxskysql -password=skysql - [RW-Split-Listener] type=listener service=RW-Split-Router protocol=MySQLClient port=4006 -[Read-Connection-Listener-Slave] -type=listener -service=Read-Connection-Router-Slave -protocol=MySQLClient -port=4009 - -[Read-Connection-Listener-Master] -type=listener -service=Read-Connection-Router-Master -protocol=MySQLClient -port=4008 - [CLI] type=service router=cli diff --git a/maxscale-system-test/mariadbmonitor/mysqlmon_multimaster.cpp b/maxscale-system-test/mariadbmonitor/mysqlmon_multimaster.cpp index 17ded46bc..aecb2fe27 100644 --- a/maxscale-system-test/mariadbmonitor/mysqlmon_multimaster.cpp +++ b/maxscale-system-test/mariadbmonitor/mysqlmon_multimaster.cpp @@ -19,33 +19,47 @@ #include +#include +#include #include "testconnections.h" #include "maxadmin_operations.h" #include "sql_t1.h" #include -#include +#include using std::cout; using std::string; +using std::set; -void check_status(TestConnections& test, const char* server, const char* status) +void check_status(TestConnections& test, const string& server, const set& expected_status) { - char cmd[256]; - char maxadmin_result[1024]; + string cmd = "show server " + server; + char buf[100]; - sprintf(cmd, "show server %s", server); - test.maxscales->get_maxadmin_param(0, cmd, (char*)"Status:", maxadmin_result); + test.maxscales->get_maxadmin_param(0, cmd.c_str(), "Status:", buf); + string maxadmin_result = buf; - if (maxadmin_result == NULL) + if (maxadmin_result.empty()) { - test.add_result(1, "maxadmin execution error\n"); - return; + test.expect(false, "maxadmin execution error\n"); } - - if (strstr(maxadmin_result, status) == NULL) + else { - test.add_result(1, "Test failed, server '%s' status is '%s', expected '%s'\n", - server, maxadmin_result, status); + // First tokenize by comma, then trim. Cannot tokenize by whitespace due to + // "Relay Master"-element. + auto tokens = mxb::strtok(maxadmin_result, ","); + auto trimmer = [](string& s) { + mxb::trim(s); + }; + std::for_each(tokens.begin(), tokens.end(), trimmer); + set tokenset(tokens.begin(), tokens.end()); + + if (tokenset != expected_status) + { + auto expected_joined = mxb::join(expected_status); + test.expect(false, "Wrong states for '%s'. Got '%s', expected '%s'.", + server.c_str(), maxadmin_result.c_str(), expected_joined.c_str()); + } } } @@ -99,7 +113,9 @@ json_t* find_array_elem_json(TestConnections& test, json_t* object, json_t* arr_elem = json_array_get(object, i); json_t* elem_val = json_object_get(arr_elem, key.c_str()); bool is_string = json_is_string(elem_val); - test.expect(is_string, "Key %s was not found in json data or the data is not string.\n", key.c_str()); + test.expect(is_string, + "Key %s was not found in json data or the data is not string.\n", + key.c_str()); if (is_string) { std::string elem_field = json_string_value(elem_val); @@ -164,7 +180,7 @@ void check_rlag(TestConnections& test, const char* server, int min_rlag, int max } } -void change_master(TestConnections& test ,int slave, int master, const string& conn_name = "", +void change_master(TestConnections& test, int slave, int master, const string& conn_name = "", int replication_delay = 0) { const char query[] = "CHANGE MASTER '%s' TO master_host='%s', master_port=%d, " @@ -172,20 +188,25 @@ void change_master(TestConnections& test ,int slave, int master, const string& c "master_user='repl', master_password='repl', master_delay=%d; " "START SLAVE '%s';"; test.try_query(test.repl->nodes[slave], query, conn_name.c_str(), - test.repl->IP_private[master], test.repl->port[master], + test.repl->IP_private[master], test.repl->port[master], replication_delay, conn_name.c_str()); } int main(int argc, char* argv[]) { - const char mm_master_states[] = "Master, Running"; - const char mm_slave_states[] = "Relay Master, Slave, Running"; - const char slave_states[] = "Slave, Running"; - const char running_state[] = "Running"; + string master = "Master"; + string running = "Running"; + string slave = "Slave"; + + set mm_master_states = {master, running}; + set mm_slave_states = {"Relay Master", slave, running}; + set slave_states = {slave, running}; + set running_state = {running}; + const char reset_query[] = "STOP SLAVE; RESET SLAVE ALL; RESET MASTER; SET GLOBAL read_only='OFF'"; const char readonly_on_query[] = "SET GLOBAL read_only='ON'"; - TestConnections::require_repl_version("10.2.3"); // Delayed replication needs this. + TestConnections::require_repl_version("10.2.3"); // Delayed replication needs this. TestConnections test(argc, argv); test.tprintf("Test 1 - Configure all servers into a multi-master ring with one slave"); @@ -237,15 +258,20 @@ int main(int argc, char* argv[]) test.tprintf("Test 3 - Configure nodes 1 and 2 into a master-master pair, make node 0 " "a slave of node 1 and node 3 a slave of node 2"); + test.maxscales->stop_maxscale(); test.set_timeout(120); test.repl->execute_query_all_nodes(reset_query); test.repl->connect(); + change_master(test, 0, 1); change_master(test, 1, 2); change_master(test, 2, 1, "", max_rlag); change_master(test, 3, 2); + test.maxscales->start_maxscale(); + sleep(2); test.maxscales->wait_for_monitor(1); + maxconn = test.maxscales->open_rwsplit_connection(); test.try_query(maxconn, "FLUSH TABLES;"); mysql_close(maxconn); @@ -278,14 +304,18 @@ int main(int argc, char* argv[]) test.tprintf("Test 5 - Create two distinct groups"); + test.maxscales->stop_maxscale(); test.set_timeout(120); test.repl->execute_query_all_nodes(reset_query); test.repl->connect(); + change_master(test, 0, 1); change_master(test, 1, 0); change_master(test, 2, 3); change_master(test, 3, 2); + test.maxscales->start_maxscale(); + sleep(2); test.maxscales->wait_for_monitor(1); // Even though the servers are in two distinct groups, only one of them @@ -330,7 +360,8 @@ int main(int argc, char* argv[]) test.maxscales->wait_for_monitor(1); maxconn = test.maxscales->open_rwsplit_connection(); test.try_query(maxconn, "FLUSH TABLES;"); - test.maxscales->wait_for_monitor(1); + test.maxscales->wait_for_monitor(2); + test.try_query(maxconn, "SHOW DATABASES;"); check_status(test, "server1", slave_states); check_status(test, "server2", mm_slave_states); @@ -346,7 +377,7 @@ int main(int argc, char* argv[]) const char remove_delay[] = "STOP SLAVE '%s'; CHANGE MASTER '%s' TO master_delay=0; START SLAVE '%s';"; test.try_query(test.repl->nodes[0], remove_delay, "a", "a", "a"); - test.maxscales->wait_for_monitor(1); + test.maxscales->wait_for_monitor(2); check_status(test, "server1", slave_states); check_rlag(test, "server1", 0, 0);