diff --git a/Documentation/Routers/Binlogrouter.md b/Documentation/Routers/Binlogrouter.md index 6e71c159c..2e60505f1 100644 --- a/Documentation/Routers/Binlogrouter.md +++ b/Documentation/Routers/Binlogrouter.md @@ -381,7 +381,9 @@ From MaxScale 2.3 onwards it is possible to specify secondary masters that the binlog router can use in case the connection to the default master fails. **Note:** This is _only_ supported for gtid based replication in conjunction -with a Galera cluster. +with a Galera cluster and provided the following holds: +* `@@log_slave_updates` is enabled on all servers, and +* all nodes in the Galera cluster have the *same* `server_id`. The initial setup is performed exactly like when there is but one default master. ``` diff --git a/maxscale-system-test/cnf/maxscale.cnf.template.mxs1980_blr_galera_server_ids b/maxscale-system-test/cnf/maxscale.cnf.template.mxs1980_blr_galera_server_ids index 5ace4280d..2d1897a23 100644 --- a/maxscale-system-test/cnf/maxscale.cnf.template.mxs1980_blr_galera_server_ids +++ b/maxscale-system-test/cnf/maxscale.cnf.template.mxs1980_blr_galera_server_ids @@ -7,6 +7,7 @@ version_string=5.6.15-log master_id=5 server_id=4711 mariadb10_master_gtid=On +heartbeat=2 [BLR-Listener] type=listener diff --git a/maxscale-system-test/mxs1980_blr_galera_server_ids.cpp b/maxscale-system-test/mxs1980_blr_galera_server_ids.cpp index 36562ddbe..4ec876f7f 100644 --- a/maxscale-system-test/mxs1980_blr_galera_server_ids.cpp +++ b/maxscale-system-test/mxs1980_blr_galera_server_ids.cpp @@ -38,10 +38,23 @@ using namespace std; namespace { +void test_sleep(int seconds) +{ + cout << "Sleeping " << seconds << " seconds: " << flush; + while (seconds) + { + cout << "." << flush; + sleep(1); + --seconds; + } + cout << endl; +} + // The amount of time slept between various operations that are // expected to take some time before becoming visible. -const int REPLICATION_SLEEP = 5; // Seconds +const int HEARTBEAT_PERIOD = 2; // Seconds +const int REPLICATION_SLEEP = 6; // Seconds string get_gtid_current_pos(TestConnections& test, MYSQL* pMysql) { @@ -61,39 +74,59 @@ string get_server_id(TestConnections& test, MYSQL* pMysql) return row.empty() ? "" : row[0]; } -// Setup BLR to replicate from galera_000. -bool setup_blr(TestConnections& test, const string& gtid, const char* zHost, int port) +bool setup_secondary_masters(TestConnections& test, MYSQL* pMaxscale) { - test.tprintf("Connecting to BLR at %s:%d", zHost, port); + test.try_query(pMaxscale, "STOP SLAVE"); - MYSQL* pMysql = open_conn_no_db(port, zHost, "repl", "repl"); - test.expect(pMysql, "Could not open connection to %s.", zHost); + Mariadb_nodes& gc = *test.galera; - if (pMysql) + for (int i = 1; i < gc.N; ++i) { - test.try_query(pMysql, "STOP SLAVE"); - test.try_query(pMysql, "SET @@global.gtid_slave_pos='%s'", gtid.c_str()); - - mxb_assert(test.galera); - Mariadb_nodes& gc = *test.galera; - stringstream ss; - ss << "CHANGE MASTER "; - ss << "TO MASTER_HOST='" << gc.IP[0] << "', "; - ss << "MASTER_PORT=" << gc.port[0] << ", "; - ss << "MASTER_USER='repl', MASTER_PASSWORD='repl', MASTER_USE_GTID=Slave_pos"; + ss << "CHANGE MASTER ':" << i + 1 << "' "; + ss << "TO MASTER_HOST='" << gc.IP[i] << "', "; + ss << "MASTER_PORT=" << gc.port[i]; string stmt = ss.str(); cout << stmt << endl; - test.try_query(pMysql, "%s", stmt.c_str()); - test.try_query(pMysql, "START SLAVE"); - - mysql_close(pMysql); + test.try_query(pMaxscale, "%s", stmt.c_str()); } + test.try_query(pMaxscale, "START SLAVE"); + + return test.global_result == 0; +} + +// Setup BLR to replicate from galera_000. +bool setup_blr(TestConnections& test, MYSQL* pMaxscale, const string& gtid) +{ + test.tprintf("Setting up BLR"); + + test.try_query(pMaxscale, "STOP SLAVE"); + test.try_query(pMaxscale, "SET @@global.gtid_slave_pos='%s'", gtid.c_str()); + + mxb_assert(test.galera); + Mariadb_nodes& gc = *test.galera; + + stringstream ss; + + ss << "CHANGE MASTER "; + ss << "TO MASTER_HOST='" << gc.IP[0] << "'"; + ss << ", MASTER_PORT=" << gc.port[0]; + ss << ", MASTER_USER='repl', MASTER_PASSWORD='repl'"; + ss << ", MASTER_USE_GTID=Slave_pos"; + ss << ", MASTER_HEARTBEAT_PERIOD=" << HEARTBEAT_PERIOD; + + string stmt = ss.str(); + + cout << stmt << endl; + + test.try_query(pMaxscale, "%s", stmt.c_str()); + test.try_query(pMaxscale, "START SLAVE"); + return test.global_result == 0; } @@ -104,17 +137,21 @@ bool setup_slave(TestConnections& test, const char* zMaxscale_host, int maxscale_port) { + test.tprintf("Setting up Slave"); + test.try_query(pSlave, "STOP SLAVE"); test.try_query(pSlave, "RESET SLAVE"); - test.try_query(pSlave, "DROP TABLE IF EXISTS test.MXS2011"); + test.try_query(pSlave, "DROP TABLE IF EXISTS test.MXS1980"); test.try_query(pSlave, "SET @@global.gtid_slave_pos='%s'", gtid.c_str()); stringstream ss; ss << "CHANGE MASTER TO "; - ss << "MASTER_HOST='" << zMaxscale_host << "', "; - ss << "MASTER_PORT=" << maxscale_port << ", "; - ss << "MASTER_USER='repl', MASTER_PASSWORD='repl', MASTER_USE_GTID=Slave_pos"; + ss << "MASTER_HOST='" << zMaxscale_host << "'"; + ss << ", MASTER_PORT=" << maxscale_port; + ss << ", MASTER_USER='repl', MASTER_PASSWORD='repl'"; + ss << ", MASTER_USE_GTID=Slave_pos"; + ss << ", MASTER_HEARTBEAT_PERIOD=" << HEARTBEAT_PERIOD; string stmt = ss.str(); @@ -128,8 +165,8 @@ bool setup_slave(TestConnections& test, bool setup_schema(TestConnections& test, MYSQL* pServer) { - test.try_query(pServer, "DROP TABLE IF EXISTS test.MXS2011"); - test.try_query(pServer, "CREATE TABLE test.MXS2011 (i INT)"); + test.try_query(pServer, "DROP TABLE IF EXISTS test.MXS1980"); + test.try_query(pServer, "CREATE TABLE test.MXS1980 (i INT)"); return test.global_result == 0; } @@ -139,7 +176,7 @@ unsigned count = 0; void insert(TestConnections& test, MYSQL* pMaster) { stringstream ss; - ss << "INSERT INTO test.MXS2011 VALUES (" << ++count << ")"; + ss << "INSERT INTO test.MXS1980 VALUES (" << ++count << ")"; string stmt = ss.str(); @@ -150,24 +187,40 @@ void insert(TestConnections& test, MYSQL* pMaster) void select(TestConnections& test, MYSQL* pSlave) { - my_ulonglong nRows; + int attempts = 5; + + my_ulonglong nRows = 0; unsigned long long nResult_sets; + int rc; - int rc = execute_query_num_of_rows(pSlave, "SELECT * FROM test.MXS2011", &nRows, &nResult_sets); - test.expect(rc == 0, "Execution of SELECT failed."); - - if (rc == 0) + do { - mxb_assert(nResult_sets == 1); + --attempts; - test.expect(nRows == count, "Expected %d rows, got %d.", count, (int)nRows); + rc = execute_query_num_of_rows(pSlave, "SELECT * FROM test.MXS1980", &nRows, &nResult_sets); + test.expect(rc == 0, "Execution of SELECT failed."); + + if (rc == 0) + { + mxb_assert(nResult_sets == 1); + + if (nRows != count) + { + // If we don't get the expected result, we sleep a while and retry with the + // assumption that it's just a replication delay. + test_sleep(2); + } + } } + while ((rc == 0) && (nRows != count) && attempts); + + test.expect(nRows == count, "Expected %d rows, got %d.", count, (int)nRows); } bool insert_select(TestConnections& test, MYSQL* pSlave, MYSQL* pMaster) { insert(test, pMaster); - sleep(REPLICATION_SLEEP); // To ensure that the insert reaches the slave. + test_sleep(REPLICATION_SLEEP); // To ensure that the insert reaches the slave. select(test, pSlave); return test.global_result == 0; @@ -206,6 +259,7 @@ void setup_galera(TestConnections& test) { gc.stash_server_settings(i); gc.add_server_setting(i, "log_slave_updates=1"); + gc.add_server_setting(i, "log_bin=galera-cluster"); } } @@ -246,7 +300,7 @@ bool setup_server_ids(TestConnections& test, map* pServer_ids_by_in } } - return test.global_result != 0; + return test.global_result == 0; } void restore_server_ids(TestConnections& test, const map& server_ids_by_index) @@ -264,20 +318,21 @@ void restart_slave(TestConnections& test, MYSQL* pSlave) { Row row; - auto replication_failed = [] (const std::string& column) - { - return column.find("Got fatal error") != string::npos; - }; + auto replication_failed = [] (const std::string& column) { + return column.find("Got fatal error") != string::npos; + }; + cout << "Stopping slave." << endl; test.try_query(pSlave, "STOP SLAVE"); row = get_row(pSlave, "SHOW SLAVE STATUS"); auto it1 = std::find_if(row.begin(), row.end(), replication_failed); test.expect(it1 == row.end(), "Replication failed."); + cout << "Starting slave." << endl; test.try_query(pSlave, "START SLAVE"); - sleep(REPLICATION_SLEEP); + test_sleep(REPLICATION_SLEEP); // With the correct setup: // - log_slave_updates is on, @@ -289,6 +344,40 @@ void restart_slave(TestConnections& test, MYSQL* pSlave) test.expect(it2 == row.end(), "START SLAVE failed."); } +bool test_basics(TestConnections& test, MYSQL* pSlave) +{ + if (insert_select(test, pSlave)) + { + restart_slave(test, pSlave); + } + + return test.global_result == 0; +} + +bool test_multiple_masters(TestConnections& test, MYSQL* pSlave) +{ + Mariadb_nodes& gc = *test.galera; + + for (int i = 0; i < gc.N; ++i) + { + test.tprintf("Blocking Galera node %d", i); + gc.block_node(i); + // Wait a number of times the hearbeat period so as to allow BLR + // enough time to detect the lack of the heartbeat and time + // to take corrective action. + test_sleep(5 * HEARTBEAT_PERIOD); + + MYSQL* pMaster = gc.nodes[(i + 1) % gc.N]; + + insert_select(test, pSlave, pMaster); + + test.tprintf("Unblocking Galera node %d", i); + gc.unblock_node(i); + } + + return test.global_result == 0; +} + } int main(int argc, char* argv[]) @@ -304,8 +393,13 @@ int main(int argc, char* argv[]) test.start_maxscale(0); - setup_galera(test); - test.galera->start_replication(); // Causes restart. + bool dont_setup_galera = getenv("MXS1980_DONT_SETUP_GALERA") ? true : false; + + if (!dont_setup_galera) + { + setup_galera(test); + test.galera->start_replication(); // Causes restart. + } Mariadb_nodes& gc = *test.galera; gc.connect(); @@ -319,11 +413,11 @@ int main(int argc, char* argv[]) const char* zValue; // Env-vars for debugging. - zValue = getenv("MXS2047_BLR_HOST"); + zValue = getenv("MXS1980_BLR_HOST"); const char* zMaxscale_host = (zValue ? zValue : test.maxscales->IP[0]); cout << "MaxScale host: " << zMaxscale_host << endl; - zValue = getenv("MXS2047_BLR_PORT"); + zValue = getenv("MXS1980_BLR_PORT"); int maxscale_port = (zValue ? atoi(zValue) : test.maxscales->binlog_port[0]); cout << "MaxScale port: " << maxscale_port << endl; @@ -331,28 +425,67 @@ int main(int argc, char* argv[]) if (setup_server_ids(test, &server_ids_by_index)) { - if (setup_blr(test, gtid, zMaxscale_host, maxscale_port)) + MYSQL* pMaxscale = open_conn_no_db(maxscale_port, zMaxscale_host, "repl", "repl"); + test.expect(pMaxscale, "Could not open connection to BLR at %s:%d.", zMaxscale_host, maxscale_port); + + if (pMaxscale) { - int slave_index = test.repl->N - 1; // We use the last slave. - - Mariadb_nodes& ms = *test.repl; - ms.connect(slave_index); - - MYSQL* pSlave = ms.nodes[slave_index]; - mxb_assert(pSlave); - - if (setup_slave(test, gtid, pSlave, zMaxscale_host, maxscale_port)) + if (setup_blr(test, pMaxscale, gtid)) { - if (setup_schema(test, gc.nodes[0])) - { - sleep(REPLICATION_SLEEP); + int slave_index = test.repl->N - 1; // We use the last slave. - if (insert_select(test, pSlave)) + Mariadb_nodes& ms = *test.repl; + ms.connect(slave_index); + + MYSQL* pSlave = ms.nodes[slave_index]; + mxb_assert(pSlave); + + if (setup_slave(test, gtid, pSlave, zMaxscale_host, maxscale_port)) + { + if (setup_schema(test, gc.nodes[0])) { - restart_slave(test, pSlave); + test_sleep(REPLICATION_SLEEP); + + if (test.ok()) + { + cout << endl; + test.tprintf("Testing basics."); + test_basics(test, pSlave); + } + + if (test.ok()) + { + cout << endl; + test.tprintf("Testing transparent switching of BLR master."); + + if (setup_secondary_masters(test, pMaxscale)) + { + test_multiple_masters(test, pSlave); + } + } + + if (test.ok()) + { + cout << endl; + test.tprintf("Testing functionality when master.ini is used."); + + cout << "Stopping slave and MaxScale." << endl; + test.try_query(pSlave, "STOP SLAVE"); + test.maxscales->stop(); + + cout << "Starting MaxScale." << endl; + test.maxscales->start(); + test_sleep(5); + cout << "Starting slave." << endl; + test.try_query(pSlave, "START SLAVE"); + test_sleep(3); + test_multiple_masters(test, pSlave); + } } } } + + mysql_close(pMaxscale); } } @@ -360,7 +493,10 @@ int main(int argc, char* argv[]) // of what setup_server_ids() returns. restore_server_ids(test, server_ids_by_index); - restore_galera(test); + if (!dont_setup_galera) + { + restore_galera(test); + } return test.global_result; }