/* * Copyright (c) 2016 MariaDB Corporation Ab * * Use of this software is governed by the Business Source License included * in the LICENSE.TXT file and at www.mariadb.com/bsl11. * * Change Date: 2020-01-01 * * On the date above, in accordance with the Business Source License, use * of this software will be governed by version 2 or later of the General * Public License. */ #include #include #include #include #include #include #include "testconnections.h" #include "fail_switch_rejoin_common.cpp" using namespace std; // How often the monitor checks the server state. // NOTE: Ensure this is identical with the value in the configuration file. const time_t MONITOR_INTERVAL = 1; // After how many seconds should the failover/rejoin operation surely have // been performed. Not very critical. const time_t FAILOVER_DURATION = 5; // How long should we keep in running. const time_t TEST_DURATION = 90; #define CMESSAGE(msg) \ do {\ stringstream ss;\ ss << "client(" << m_id << ") : " << msg << "\n";\ cout << ss.str() << flush;\ } while (false) #if !defined(NDEBUG) #define ss_dassert(x) do { if (!(x)) { fprintf(stderr, "Assertion failed: %s", #x); abort(); } } while(false) #define ss_debug(x) x #else #define ss_dassert(s) #define ss_debug(x) #endif namespace { class Client { public: enum { DEFAULT_N_CLIENTS = 4, DEFAULT_N_ROWS = 100 }; static void init(TestConnections& test, size_t nClients, size_t nRows) { s_nClients = nClients; s_nRows = nRows; if (create_tables(test)) { if (insert_data(test)) { cout << "\nSyncing slaves." << endl; test.repl->sync_slaves(); } } } static void start(bool verbose, const char* zHost, int port, const char* zUser, const char* zPassword) { for (size_t i = 0; i < s_nClients; ++i) { s_threads.push_back(std::thread(&Client::thread_main, i, verbose, zHost, port, zUser, zPassword)); } } static void stop() { s_shutdown = true; for (size_t i = 0; i < s_nClients; ++i) { s_threads[i].join(); } } private: Client(int id, bool verbose) : m_id(id) , m_verbose(verbose) , m_value(1) { ss_debug(int rv); unsigned int seed = (time(NULL) << m_id); ss_debug(rv =) initstate_r(seed, m_initstate, sizeof(m_initstate), &m_random_data); ss_dassert(rv == 0); ss_debug(rv=) srandom_r(seed, &m_random_data); ss_dassert(rv == 0); } enum action_t { ACTION_SELECT, ACTION_UPDATE }; action_t action() const { double d = random_decimal_fraction(); // 20% updates // 80% selects if (d <= 0.2) { return ACTION_UPDATE; } else { return ACTION_SELECT; } } bool run(MYSQL* pConn) { bool rv = false; switch (action()) { case ACTION_SELECT: rv = run_select(pConn); break; case ACTION_UPDATE: rv = run_update(pConn); break; default: ss_dassert(!true); } return rv; } bool run_select(MYSQL* pConn) { bool rv = true; string stmt("SELECT * FROM test.t"); stmt += std::to_string(m_id); stmt += " WHERE id="; stmt += std::to_string(get_random_id()); if (mysql_query(pConn, stmt.c_str()) == 0) { flush_response(pConn); } else { if (m_verbose) { CMESSAGE("\"" << stmt << "\" failed: " << mysql_error(pConn)); } rv = false; } return rv; } bool run_update(MYSQL* pConn) { bool rv = true; string stmt("UPDATE test.t"); stmt += std::to_string(m_id); stmt += " SET id="; stmt += std::to_string(m_value); stmt += " WHERE id="; stmt += std::to_string(get_random_id()); m_value = (m_value + 1) % s_nRows; if (mysql_query(pConn, stmt.c_str()) == 0) { flush_response(pConn); } else { if (m_verbose) { CMESSAGE("\"" << stmt << "\" failed: " << mysql_error(pConn)); } rv = false; } return rv; } static void flush_response(MYSQL* pConn) { do { MYSQL_RES* pRes = mysql_store_result(pConn); mysql_free_result(pRes); } while (mysql_next_result(pConn) == 0); } int get_random_id() const { int id = s_nRows * random_decimal_fraction(); ss_dassert(id >= 0); ss_dassert(id <= (int)s_nRows); return id; } double random_decimal_fraction() const { int32_t r; ss_debug(int rv=) random_r(&m_random_data, &r); ss_dassert(rv == 0); return double(r) / RAND_MAX; } void run(const char* zHost, int port, const char* zUser, const char* zPassword) { do { MYSQL* pMysql = mysql_init(NULL); if (pMysql) { unsigned int timeout = 5; mysql_options(pMysql, MYSQL_OPT_CONNECT_TIMEOUT, &timeout); mysql_options(pMysql, MYSQL_OPT_READ_TIMEOUT, &timeout); mysql_options(pMysql, MYSQL_OPT_WRITE_TIMEOUT, &timeout); if (m_verbose) { CMESSAGE("Connecting"); } if (mysql_real_connect(pMysql, zHost, zUser, zPassword, "test", port, NULL, 0)) { if (m_verbose) { CMESSAGE("Connected."); } while (!s_shutdown && run(pMysql)) { ; } } else { if (m_verbose) { CMESSAGE("mysql_real_connect() failed: " << mysql_error(pMysql)); } } if (m_verbose) { CMESSAGE("Closing"); } mysql_close(pMysql); } else { CMESSAGE("mysql_init() failed."); } // To prevent some backend from becoming overwhelmed. sleep(1); } while (!s_shutdown); } static void thread_main(int i, bool verbose, const char* zHost, int port, const char* zUser, const char* zPassword) { if (mysql_thread_init() == 0) { Client client(i, verbose); client.run(zHost, port, zUser, zPassword); mysql_thread_end(); } else { int m_id = i; CMESSAGE("mysql_thread_init() failed."); } } static bool create_tables(TestConnections& test) { cout << "\nCreating tables." << endl; MYSQL* pConn = test.maxscales->conn_rwsplit[0]; string drop_head("DROP TABLE IF EXISTS test.t"); string create_head("CREATE TABLE test.t"); string create_tail(" (id INT)"); for (size_t i = 0; i < s_nClients; ++i) { string drop = drop_head + std::to_string(i); test.try_query(pConn, drop.c_str()); string create = create_head + std::to_string(i) + create_tail; test.try_query(pConn, create.c_str()); } return test.ok(); } static bool insert_data(TestConnections& test) { cout << "\nInserting data." << endl; MYSQL* pConn = test.maxscales->conn_rwsplit[0]; for (size_t i = 0; i < s_nClients; ++i) { string insert("insert into test.t"); insert += std::to_string(i); insert += " values "; for (size_t j = 0; j < s_nRows; ++j) { insert += "("; insert += std::to_string(j); insert += ")"; if (j < s_nRows - 1) { insert += ", "; } } test.try_query(pConn, insert.c_str()); } return test.ok(); } private: enum { INITSTATE_SIZE = 32 }; size_t m_id; bool m_verbose; size_t m_value; char m_initstate[INITSTATE_SIZE]; mutable struct random_data m_random_data; static size_t s_nClients; static size_t s_nRows; static bool s_shutdown; static std::vector s_threads; }; size_t Client::s_nClients; size_t Client::s_nRows; bool Client::s_shutdown; std::vector Client::s_threads; } namespace { void list_servers(TestConnections& test) { test.maxscales->execute_maxadmin_command_print(0, (char*)"list servers"); } void sleep(int s) { cout << "Sleeping " << s << " times 1 second" << flush; do { ::sleep(1); cout << "." << flush; --s; } while (s > 0); cout << endl; } bool check_server_status(TestConnections& test, int id) { bool is_master = false; Mariadb_nodes* pRepl = test.repl; string server = string("server") + std::to_string(id); StringSet statuses = test.get_server_status(server.c_str()); std::ostream_iterator oi(cout, " "); cout << server << ": "; std::copy(statuses.begin(), statuses.end(), oi); cout << " => "; if (statuses.count("Master")) { is_master = true; cout << "OK"; } else if (statuses.count("Slave")) { cout << "OK"; } else if (statuses.count("Running")) { MYSQL* pConn = pRepl->nodes[id - 1]; char result[1024]; if (find_field(pConn, "SHOW SLAVE STATUS", "Last_IO_Error", result) == 0) { const char needle[] = ", which is not in the master's binlog. " "Since the master's binlog contains GTIDs with higher sequence numbers, " "it probably means that the slave has diverged due to executing extra " "erroneous transactions"; if (strstr(result, needle)) { // A rejoin was attempted, but it failed because the node (old master) // had events that were not present in the new master. That is, a rejoin // is not possible in principle without corrective action. cout << "OK (could not be joined due to GTID issue)"; } else { cout << result; test.assert(false, "Merely 'Running' node did not error in expected way."); } } else { test.assert(false, "Could not execute \"SHOW SLAVE STATUS\""); } } else { test.assert(false, "Unexpected server state for %s.", server.c_str()); } cout << endl; return is_master; } void check_server_statuses(TestConnections& test) { int masters = 0; masters += check_server_status(test, 1); masters += check_server_status(test, 2); masters += check_server_status(test, 3); masters += check_server_status(test, 4); if (masters == 0) { test.global_result = 0; test.tprintf("No master, checking that autofail has been turned off."); test.log_includes(0, "disabling automatic failover"); } else if (masters != 1) { test.assert(!true, "Unexpected number of masters: %d", masters); } } void run(TestConnections& test) { cout << "\nConnecting to MaxScale." << endl; test.maxscales->connect_maxscale(); Client::init(test, Client::DEFAULT_N_CLIENTS, Client::DEFAULT_N_ROWS); if (test.ok()) { const char* zHost = test.maxscales->IP[0]; int port = test.maxscales->rwsplit_port[0]; const char* zUser = test.maxscales->user_name; const char* zPassword = test.maxscales->password; cout << "Connecting to " << zHost << ":" << port << " as " << zUser << ":" << zPassword << endl; cout << "Starting clients." << endl; Client::start(test.verbose, zHost, port, zUser, zPassword); time_t start = time(NULL); list_servers(test); while (time(NULL) - start < TEST_DURATION) { test.maxscales->wait_for_monitor(); int master_id = get_master_server_id(test); if (master_id > 0 && master_id <= 4) { cout << "\nStopping node: " << master_id << endl; test.repl->stop_node(master_id - 1); test.maxscales->wait_for_monitor(); list_servers(test); test.maxscales->wait_for_monitor(); list_servers(test); test.maxscales->wait_for_monitor(); cout << "\nStarting node: " << master_id << endl; test.repl->start_node(master_id - 1); test.maxscales->wait_for_monitor(); list_servers(test); test.maxscales->wait_for_monitor(); list_servers(test); } else { test.assert(false, "Unexpected master id: %d"); } } test.maxscales->wait_for_monitor(); cout << "\nStopping clients.\n" << flush; Client::stop(); test.repl->close_connections(); test.repl->connect(); check_server_statuses(test); } } } int main(int argc, char* argv[]) { std::ios::sync_with_stdio(true); Mariadb_nodes::require_gtid(true); TestConnections test(argc, argv); run(test); return test.global_result; }