From 3d06b5129db76835fed82209a9fb213522f9d7d0 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Wed, 3 Oct 2018 09:44:15 +0300 Subject: [PATCH] MXS-1980 Extend test program Now the test program will 1) Write to each node in a Galera cluster and verify that the data ends up in the slave. 2) At the end of 1) execute STOP SLAVE and START SLAVE to check that replication can be stopped and started again (won't work unless each node has the same server_id and value for @@log_bin_basename). 3) Block the node BLR is replicating from and expect it to connect to the next configured master and that replication continues to work. Do that for all nodes. 4) Stop MaxScale and restart it and expect 3) to work. That checks that BLR saves all necessary information in master.ini and is capable of reading it. --- Documentation/Routers/Binlogrouter.md | 4 +- ...cnf.template.mxs1980_blr_galera_server_ids | 1 + .../mxs1980_blr_galera_server_ids.cpp | 260 +++++++++++++----- 3 files changed, 202 insertions(+), 63 deletions(-) diff --git a/Documentation/Routers/Binlogrouter.md b/Documentation/Routers/Binlogrouter.md index 6e71c159c..2e60505f1 100644 --- a/Documentation/Routers/Binlogrouter.md +++ b/Documentation/Routers/Binlogrouter.md @@ -381,7 +381,9 @@ From MaxScale 2.3 onwards it is possible to specify secondary masters that the binlog router can use in case the connection to the default master fails. **Note:** This is _only_ supported for gtid based replication in conjunction -with a Galera cluster. +with a Galera cluster and provided the following holds: +* `@@log_slave_updates` is enabled on all servers, and +* all nodes in the Galera cluster have the *same* `server_id`. The initial setup is performed exactly like when there is but one default master. ``` diff --git a/maxscale-system-test/cnf/maxscale.cnf.template.mxs1980_blr_galera_server_ids b/maxscale-system-test/cnf/maxscale.cnf.template.mxs1980_blr_galera_server_ids index 5ace4280d..2d1897a23 100644 --- a/maxscale-system-test/cnf/maxscale.cnf.template.mxs1980_blr_galera_server_ids +++ b/maxscale-system-test/cnf/maxscale.cnf.template.mxs1980_blr_galera_server_ids @@ -7,6 +7,7 @@ version_string=5.6.15-log master_id=5 server_id=4711 mariadb10_master_gtid=On +heartbeat=2 [BLR-Listener] type=listener diff --git a/maxscale-system-test/mxs1980_blr_galera_server_ids.cpp b/maxscale-system-test/mxs1980_blr_galera_server_ids.cpp index 36562ddbe..4ec876f7f 100644 --- a/maxscale-system-test/mxs1980_blr_galera_server_ids.cpp +++ b/maxscale-system-test/mxs1980_blr_galera_server_ids.cpp @@ -38,10 +38,23 @@ using namespace std; namespace { +void test_sleep(int seconds) +{ + cout << "Sleeping " << seconds << " seconds: " << flush; + while (seconds) + { + cout << "." << flush; + sleep(1); + --seconds; + } + cout << endl; +} + // The amount of time slept between various operations that are // expected to take some time before becoming visible. -const int REPLICATION_SLEEP = 5; // Seconds +const int HEARTBEAT_PERIOD = 2; // Seconds +const int REPLICATION_SLEEP = 6; // Seconds string get_gtid_current_pos(TestConnections& test, MYSQL* pMysql) { @@ -61,39 +74,59 @@ string get_server_id(TestConnections& test, MYSQL* pMysql) return row.empty() ? "" : row[0]; } -// Setup BLR to replicate from galera_000. -bool setup_blr(TestConnections& test, const string& gtid, const char* zHost, int port) +bool setup_secondary_masters(TestConnections& test, MYSQL* pMaxscale) { - test.tprintf("Connecting to BLR at %s:%d", zHost, port); + test.try_query(pMaxscale, "STOP SLAVE"); - MYSQL* pMysql = open_conn_no_db(port, zHost, "repl", "repl"); - test.expect(pMysql, "Could not open connection to %s.", zHost); + Mariadb_nodes& gc = *test.galera; - if (pMysql) + for (int i = 1; i < gc.N; ++i) { - test.try_query(pMysql, "STOP SLAVE"); - test.try_query(pMysql, "SET @@global.gtid_slave_pos='%s'", gtid.c_str()); - - mxb_assert(test.galera); - Mariadb_nodes& gc = *test.galera; - stringstream ss; - ss << "CHANGE MASTER "; - ss << "TO MASTER_HOST='" << gc.IP[0] << "', "; - ss << "MASTER_PORT=" << gc.port[0] << ", "; - ss << "MASTER_USER='repl', MASTER_PASSWORD='repl', MASTER_USE_GTID=Slave_pos"; + ss << "CHANGE MASTER ':" << i + 1 << "' "; + ss << "TO MASTER_HOST='" << gc.IP[i] << "', "; + ss << "MASTER_PORT=" << gc.port[i]; string stmt = ss.str(); cout << stmt << endl; - test.try_query(pMysql, "%s", stmt.c_str()); - test.try_query(pMysql, "START SLAVE"); - - mysql_close(pMysql); + test.try_query(pMaxscale, "%s", stmt.c_str()); } + test.try_query(pMaxscale, "START SLAVE"); + + return test.global_result == 0; +} + +// Setup BLR to replicate from galera_000. +bool setup_blr(TestConnections& test, MYSQL* pMaxscale, const string& gtid) +{ + test.tprintf("Setting up BLR"); + + test.try_query(pMaxscale, "STOP SLAVE"); + test.try_query(pMaxscale, "SET @@global.gtid_slave_pos='%s'", gtid.c_str()); + + mxb_assert(test.galera); + Mariadb_nodes& gc = *test.galera; + + stringstream ss; + + ss << "CHANGE MASTER "; + ss << "TO MASTER_HOST='" << gc.IP[0] << "'"; + ss << ", MASTER_PORT=" << gc.port[0]; + ss << ", MASTER_USER='repl', MASTER_PASSWORD='repl'"; + ss << ", MASTER_USE_GTID=Slave_pos"; + ss << ", MASTER_HEARTBEAT_PERIOD=" << HEARTBEAT_PERIOD; + + string stmt = ss.str(); + + cout << stmt << endl; + + test.try_query(pMaxscale, "%s", stmt.c_str()); + test.try_query(pMaxscale, "START SLAVE"); + return test.global_result == 0; } @@ -104,17 +137,21 @@ bool setup_slave(TestConnections& test, const char* zMaxscale_host, int maxscale_port) { + test.tprintf("Setting up Slave"); + test.try_query(pSlave, "STOP SLAVE"); test.try_query(pSlave, "RESET SLAVE"); - test.try_query(pSlave, "DROP TABLE IF EXISTS test.MXS2011"); + test.try_query(pSlave, "DROP TABLE IF EXISTS test.MXS1980"); test.try_query(pSlave, "SET @@global.gtid_slave_pos='%s'", gtid.c_str()); stringstream ss; ss << "CHANGE MASTER TO "; - ss << "MASTER_HOST='" << zMaxscale_host << "', "; - ss << "MASTER_PORT=" << maxscale_port << ", "; - ss << "MASTER_USER='repl', MASTER_PASSWORD='repl', MASTER_USE_GTID=Slave_pos"; + ss << "MASTER_HOST='" << zMaxscale_host << "'"; + ss << ", MASTER_PORT=" << maxscale_port; + ss << ", MASTER_USER='repl', MASTER_PASSWORD='repl'"; + ss << ", MASTER_USE_GTID=Slave_pos"; + ss << ", MASTER_HEARTBEAT_PERIOD=" << HEARTBEAT_PERIOD; string stmt = ss.str(); @@ -128,8 +165,8 @@ bool setup_slave(TestConnections& test, bool setup_schema(TestConnections& test, MYSQL* pServer) { - test.try_query(pServer, "DROP TABLE IF EXISTS test.MXS2011"); - test.try_query(pServer, "CREATE TABLE test.MXS2011 (i INT)"); + test.try_query(pServer, "DROP TABLE IF EXISTS test.MXS1980"); + test.try_query(pServer, "CREATE TABLE test.MXS1980 (i INT)"); return test.global_result == 0; } @@ -139,7 +176,7 @@ unsigned count = 0; void insert(TestConnections& test, MYSQL* pMaster) { stringstream ss; - ss << "INSERT INTO test.MXS2011 VALUES (" << ++count << ")"; + ss << "INSERT INTO test.MXS1980 VALUES (" << ++count << ")"; string stmt = ss.str(); @@ -150,24 +187,40 @@ void insert(TestConnections& test, MYSQL* pMaster) void select(TestConnections& test, MYSQL* pSlave) { - my_ulonglong nRows; + int attempts = 5; + + my_ulonglong nRows = 0; unsigned long long nResult_sets; + int rc; - int rc = execute_query_num_of_rows(pSlave, "SELECT * FROM test.MXS2011", &nRows, &nResult_sets); - test.expect(rc == 0, "Execution of SELECT failed."); - - if (rc == 0) + do { - mxb_assert(nResult_sets == 1); + --attempts; - test.expect(nRows == count, "Expected %d rows, got %d.", count, (int)nRows); + rc = execute_query_num_of_rows(pSlave, "SELECT * FROM test.MXS1980", &nRows, &nResult_sets); + test.expect(rc == 0, "Execution of SELECT failed."); + + if (rc == 0) + { + mxb_assert(nResult_sets == 1); + + if (nRows != count) + { + // If we don't get the expected result, we sleep a while and retry with the + // assumption that it's just a replication delay. + test_sleep(2); + } + } } + while ((rc == 0) && (nRows != count) && attempts); + + test.expect(nRows == count, "Expected %d rows, got %d.", count, (int)nRows); } bool insert_select(TestConnections& test, MYSQL* pSlave, MYSQL* pMaster) { insert(test, pMaster); - sleep(REPLICATION_SLEEP); // To ensure that the insert reaches the slave. + test_sleep(REPLICATION_SLEEP); // To ensure that the insert reaches the slave. select(test, pSlave); return test.global_result == 0; @@ -206,6 +259,7 @@ void setup_galera(TestConnections& test) { gc.stash_server_settings(i); gc.add_server_setting(i, "log_slave_updates=1"); + gc.add_server_setting(i, "log_bin=galera-cluster"); } } @@ -246,7 +300,7 @@ bool setup_server_ids(TestConnections& test, map* pServer_ids_by_in } } - return test.global_result != 0; + return test.global_result == 0; } void restore_server_ids(TestConnections& test, const map& server_ids_by_index) @@ -264,20 +318,21 @@ void restart_slave(TestConnections& test, MYSQL* pSlave) { Row row; - auto replication_failed = [] (const std::string& column) - { - return column.find("Got fatal error") != string::npos; - }; + auto replication_failed = [] (const std::string& column) { + return column.find("Got fatal error") != string::npos; + }; + cout << "Stopping slave." << endl; test.try_query(pSlave, "STOP SLAVE"); row = get_row(pSlave, "SHOW SLAVE STATUS"); auto it1 = std::find_if(row.begin(), row.end(), replication_failed); test.expect(it1 == row.end(), "Replication failed."); + cout << "Starting slave." << endl; test.try_query(pSlave, "START SLAVE"); - sleep(REPLICATION_SLEEP); + test_sleep(REPLICATION_SLEEP); // With the correct setup: // - log_slave_updates is on, @@ -289,6 +344,40 @@ void restart_slave(TestConnections& test, MYSQL* pSlave) test.expect(it2 == row.end(), "START SLAVE failed."); } +bool test_basics(TestConnections& test, MYSQL* pSlave) +{ + if (insert_select(test, pSlave)) + { + restart_slave(test, pSlave); + } + + return test.global_result == 0; +} + +bool test_multiple_masters(TestConnections& test, MYSQL* pSlave) +{ + Mariadb_nodes& gc = *test.galera; + + for (int i = 0; i < gc.N; ++i) + { + test.tprintf("Blocking Galera node %d", i); + gc.block_node(i); + // Wait a number of times the hearbeat period so as to allow BLR + // enough time to detect the lack of the heartbeat and time + // to take corrective action. + test_sleep(5 * HEARTBEAT_PERIOD); + + MYSQL* pMaster = gc.nodes[(i + 1) % gc.N]; + + insert_select(test, pSlave, pMaster); + + test.tprintf("Unblocking Galera node %d", i); + gc.unblock_node(i); + } + + return test.global_result == 0; +} + } int main(int argc, char* argv[]) @@ -304,8 +393,13 @@ int main(int argc, char* argv[]) test.start_maxscale(0); - setup_galera(test); - test.galera->start_replication(); // Causes restart. + bool dont_setup_galera = getenv("MXS1980_DONT_SETUP_GALERA") ? true : false; + + if (!dont_setup_galera) + { + setup_galera(test); + test.galera->start_replication(); // Causes restart. + } Mariadb_nodes& gc = *test.galera; gc.connect(); @@ -319,11 +413,11 @@ int main(int argc, char* argv[]) const char* zValue; // Env-vars for debugging. - zValue = getenv("MXS2047_BLR_HOST"); + zValue = getenv("MXS1980_BLR_HOST"); const char* zMaxscale_host = (zValue ? zValue : test.maxscales->IP[0]); cout << "MaxScale host: " << zMaxscale_host << endl; - zValue = getenv("MXS2047_BLR_PORT"); + zValue = getenv("MXS1980_BLR_PORT"); int maxscale_port = (zValue ? atoi(zValue) : test.maxscales->binlog_port[0]); cout << "MaxScale port: " << maxscale_port << endl; @@ -331,28 +425,67 @@ int main(int argc, char* argv[]) if (setup_server_ids(test, &server_ids_by_index)) { - if (setup_blr(test, gtid, zMaxscale_host, maxscale_port)) + MYSQL* pMaxscale = open_conn_no_db(maxscale_port, zMaxscale_host, "repl", "repl"); + test.expect(pMaxscale, "Could not open connection to BLR at %s:%d.", zMaxscale_host, maxscale_port); + + if (pMaxscale) { - int slave_index = test.repl->N - 1; // We use the last slave. - - Mariadb_nodes& ms = *test.repl; - ms.connect(slave_index); - - MYSQL* pSlave = ms.nodes[slave_index]; - mxb_assert(pSlave); - - if (setup_slave(test, gtid, pSlave, zMaxscale_host, maxscale_port)) + if (setup_blr(test, pMaxscale, gtid)) { - if (setup_schema(test, gc.nodes[0])) - { - sleep(REPLICATION_SLEEP); + int slave_index = test.repl->N - 1; // We use the last slave. - if (insert_select(test, pSlave)) + Mariadb_nodes& ms = *test.repl; + ms.connect(slave_index); + + MYSQL* pSlave = ms.nodes[slave_index]; + mxb_assert(pSlave); + + if (setup_slave(test, gtid, pSlave, zMaxscale_host, maxscale_port)) + { + if (setup_schema(test, gc.nodes[0])) { - restart_slave(test, pSlave); + test_sleep(REPLICATION_SLEEP); + + if (test.ok()) + { + cout << endl; + test.tprintf("Testing basics."); + test_basics(test, pSlave); + } + + if (test.ok()) + { + cout << endl; + test.tprintf("Testing transparent switching of BLR master."); + + if (setup_secondary_masters(test, pMaxscale)) + { + test_multiple_masters(test, pSlave); + } + } + + if (test.ok()) + { + cout << endl; + test.tprintf("Testing functionality when master.ini is used."); + + cout << "Stopping slave and MaxScale." << endl; + test.try_query(pSlave, "STOP SLAVE"); + test.maxscales->stop(); + + cout << "Starting MaxScale." << endl; + test.maxscales->start(); + test_sleep(5); + cout << "Starting slave." << endl; + test.try_query(pSlave, "START SLAVE"); + test_sleep(3); + test_multiple_masters(test, pSlave); + } } } } + + mysql_close(pMaxscale); } } @@ -360,7 +493,10 @@ int main(int argc, char* argv[]) // of what setup_server_ids() returns. restore_server_ids(test, server_ids_by_index); - restore_galera(test); + if (!dont_setup_galera) + { + restore_galera(test); + } return test.global_result; }