MXS-1980 Extend test program

Now the test program will

1) Write to each node in a Galera cluster and verify that the data
   ends up in the slave.
2) At the end of 1) execute STOP SLAVE and START SLAVE to check that
   replication can be stopped and started again (won't work unless
   each node has the same server_id and value for @@log_bin_basename).
3) Block the node BLR is replicating from and expect it to connect
   to the next configured master and that replication continues to
   work. Do that for all nodes.
4) Stop MaxScale and restart it and expect 3) to work. That checks
   that BLR saves all necessary information in master.ini and is
   capable of reading it.

Documentation/Routers/Binlogrouter.md

@@ -381,7 +381,9 @@ From MaxScale 2.3 onwards it is possible to specify secondary masters that
 the binlog router can use in case the connection to the default master fails.
 
 **Note:** This is _only_ supported for gtid based replication in conjunction
-with a Galera cluster.
+with a Galera cluster and provided the following holds:
+* `@@log_slave_updates` is enabled on all servers, and
+* all nodes in the Galera cluster have the *same* `server_id`.
 
 The initial setup is performed exactly like when there is but one default master.
 ```

maxscale-system-test/cnf/maxscale.cnf.template.mxs1980_blr_galera_server_ids

@@ -7,6 +7,7 @@ version_string=5.6.15-log
 master_id=5
 server_id=4711
 mariadb10_master_gtid=On
+heartbeat=2
 
 [BLR-Listener]
 type=listener

maxscale-system-test/mxs1980_blr_galera_server_ids.cpp

@@ -38,10 +38,23 @@ using namespace std;
 namespace
 {
 
+void test_sleep(int seconds)
+{
+    cout << "Sleeping " << seconds << " seconds: " << flush;
+    while (seconds)
+    {
+        cout << "." << flush;
+        sleep(1);
+        --seconds;
+    }
+    cout << endl;
+}
+
 // The amount of time slept between various operations that are
 // expected to take some time before becoming visible.
 
-const int REPLICATION_SLEEP = 5; // Seconds
+const int HEARTBEAT_PERIOD  = 2; // Seconds
+const int REPLICATION_SLEEP = 6; // Seconds
 
 string get_gtid_current_pos(TestConnections& test, MYSQL* pMysql)
 {
@@ -61,18 +74,39 @@ string get_server_id(TestConnections& test, MYSQL* pMysql)
     return row.empty() ? "" : row[0];
 }
 
+bool setup_secondary_masters(TestConnections& test, MYSQL* pMaxscale)
+{
+    test.try_query(pMaxscale, "STOP SLAVE");
+
+    Mariadb_nodes& gc = *test.galera;
+
+    for (int i = 1; i < gc.N; ++i)
+    {
+        stringstream ss;
+
+        ss << "CHANGE MASTER ':" << i + 1 << "' ";
+        ss << "TO MASTER_HOST='" << gc.IP[i] << "', ";
+        ss << "MASTER_PORT=" << gc.port[i];
+
+        string stmt = ss.str();
+
+        cout << stmt << endl;
+
+        test.try_query(pMaxscale, "%s", stmt.c_str());
+    }
+
+    test.try_query(pMaxscale, "START SLAVE");
+
+    return test.global_result == 0;
+}
+
 // Setup BLR to replicate from galera_000.
-bool setup_blr(TestConnections& test, const string& gtid, const char* zHost, int port)
+bool setup_blr(TestConnections& test, MYSQL* pMaxscale, const string& gtid)
 {
-    test.tprintf("Connecting to BLR at %s:%d", zHost, port);
+    test.tprintf("Setting up BLR");
 
-    MYSQL* pMysql = open_conn_no_db(port, zHost, "repl", "repl");
+    test.try_query(pMaxscale, "STOP SLAVE");
-    test.expect(pMysql, "Could not open connection to %s.", zHost);
+    test.try_query(pMaxscale, "SET @@global.gtid_slave_pos='%s'", gtid.c_str());
-
-    if (pMysql)
-    {
-        test.try_query(pMysql, "STOP SLAVE");
-        test.try_query(pMysql, "SET @@global.gtid_slave_pos='%s'", gtid.c_str());
 
     mxb_assert(test.galera);
     Mariadb_nodes& gc = *test.galera;
@@ -80,19 +114,18 @@ bool setup_blr(TestConnections& test, const string& gtid, const char* zHost, int
     stringstream ss;
 
     ss << "CHANGE MASTER ";
-        ss << "TO MASTER_HOST='" << gc.IP[0] << "', ";
+    ss << "TO MASTER_HOST='" << gc.IP[0] << "'";
-        ss << "MASTER_PORT=" << gc.port[0] << ", ";
+    ss << ", MASTER_PORT=" << gc.port[0];
-        ss << "MASTER_USER='repl', MASTER_PASSWORD='repl', MASTER_USE_GTID=Slave_pos";
+    ss << ", MASTER_USER='repl', MASTER_PASSWORD='repl'";
+    ss << ", MASTER_USE_GTID=Slave_pos";
+    ss << ", MASTER_HEARTBEAT_PERIOD=" << HEARTBEAT_PERIOD;
 
     string stmt = ss.str();
 
     cout << stmt << endl;
 
-        test.try_query(pMysql, "%s", stmt.c_str());
+    test.try_query(pMaxscale, "%s", stmt.c_str());
-        test.try_query(pMysql, "START SLAVE");
+    test.try_query(pMaxscale, "START SLAVE");
-
-        mysql_close(pMysql);
-    }
 
     return test.global_result == 0;
 }
@@ -104,17 +137,21 @@ bool setup_slave(TestConnections& test,
                  const char* zMaxscale_host,
                  int maxscale_port)
 {
+    test.tprintf("Setting up Slave");
+
     test.try_query(pSlave, "STOP SLAVE");
     test.try_query(pSlave, "RESET SLAVE");
-    test.try_query(pSlave, "DROP TABLE IF EXISTS test.MXS2011");
+    test.try_query(pSlave, "DROP TABLE IF EXISTS test.MXS1980");
     test.try_query(pSlave, "SET @@global.gtid_slave_pos='%s'", gtid.c_str());
 
     stringstream ss;
 
     ss << "CHANGE MASTER TO ";
-    ss << "MASTER_HOST='" << zMaxscale_host << "', ";
+    ss << "MASTER_HOST='" << zMaxscale_host << "'";
-    ss << "MASTER_PORT=" << maxscale_port << ", ";
+    ss << ", MASTER_PORT=" << maxscale_port;
-    ss << "MASTER_USER='repl', MASTER_PASSWORD='repl', MASTER_USE_GTID=Slave_pos";
+    ss << ", MASTER_USER='repl', MASTER_PASSWORD='repl'";
+    ss << ", MASTER_USE_GTID=Slave_pos";
+    ss << ", MASTER_HEARTBEAT_PERIOD=" << HEARTBEAT_PERIOD;
 
     string stmt = ss.str();
 
@@ -128,8 +165,8 @@ bool setup_slave(TestConnections& test,
 
 bool setup_schema(TestConnections& test, MYSQL* pServer)
 {
-    test.try_query(pServer, "DROP TABLE IF EXISTS test.MXS2011");
+    test.try_query(pServer, "DROP TABLE IF EXISTS test.MXS1980");
-    test.try_query(pServer, "CREATE TABLE test.MXS2011 (i INT)");
+    test.try_query(pServer, "CREATE TABLE test.MXS1980 (i INT)");
 
     return test.global_result == 0;
 }
@@ -139,7 +176,7 @@ unsigned count = 0;
 void insert(TestConnections& test, MYSQL* pMaster)
 {
     stringstream ss;
-    ss << "INSERT INTO test.MXS2011 VALUES (" << ++count << ")";
+    ss << "INSERT INTO test.MXS1980 VALUES (" << ++count << ")";
 
     string stmt = ss.str();
 
@@ -150,24 +187,40 @@ void insert(TestConnections& test, MYSQL* pMaster)
 
 void select(TestConnections& test, MYSQL* pSlave)
 {
-    my_ulonglong nRows;
+    int attempts = 5;
-    unsigned long long nResult_sets;
 
-    int rc = execute_query_num_of_rows(pSlave, "SELECT * FROM test.MXS2011", &nRows, &nResult_sets);
+    my_ulonglong nRows = 0;
+    unsigned long long nResult_sets;
+    int rc;
+
+    do
+    {
+        --attempts;
+
+        rc = execute_query_num_of_rows(pSlave, "SELECT * FROM test.MXS1980", &nRows, &nResult_sets);
         test.expect(rc == 0, "Execution of SELECT failed.");
 
         if (rc == 0)
         {
             mxb_assert(nResult_sets == 1);
 
-        test.expect(nRows == count, "Expected %d rows, got %d.", count, (int)nRows);
+            if (nRows != count)
+            {
+                // If we don't get the expected result, we sleep a while and retry with the
+                // assumption that it's just a replication delay.
+                test_sleep(2);
             }
         }
+    }
+    while ((rc == 0) && (nRows != count) && attempts);
+
+    test.expect(nRows == count, "Expected %d rows, got %d.", count, (int)nRows);
+}
 
 bool insert_select(TestConnections& test, MYSQL* pSlave, MYSQL* pMaster)
 {
     insert(test, pMaster);
-    sleep(REPLICATION_SLEEP); // To ensure that the insert reaches the slave.
+    test_sleep(REPLICATION_SLEEP); // To ensure that the insert reaches the slave.
     select(test, pSlave);
 
     return test.global_result == 0;
@@ -206,6 +259,7 @@ void setup_galera(TestConnections& test)
     {
         gc.stash_server_settings(i);
         gc.add_server_setting(i, "log_slave_updates=1");
+        gc.add_server_setting(i, "log_bin=galera-cluster");
     }
 }
 
@@ -246,7 +300,7 @@ bool setup_server_ids(TestConnections& test, map<int, string>* pServer_ids_by_in
         }
     }
 
-    return test.global_result != 0;
+    return test.global_result == 0;
 }
 
 void restore_server_ids(TestConnections& test, const map<int, string>& server_ids_by_index)
@@ -264,20 +318,21 @@ void restart_slave(TestConnections& test, MYSQL* pSlave)
 {
     Row row;
 
-    auto replication_failed = [] (const std::string& column)
+    auto replication_failed = [] (const std::string& column) {
-        {
         return column.find("Got fatal error") != string::npos;
     };
 
+    cout << "Stopping slave." << endl;
     test.try_query(pSlave, "STOP SLAVE");
 
     row = get_row(pSlave, "SHOW SLAVE STATUS");
     auto it1 = std::find_if(row.begin(), row.end(), replication_failed);
     test.expect(it1 == row.end(), "Replication failed.");
 
+    cout << "Starting slave." << endl;
     test.try_query(pSlave, "START SLAVE");
 
-    sleep(REPLICATION_SLEEP);
+    test_sleep(REPLICATION_SLEEP);
 
     // With the correct setup:
     // - log_slave_updates is on,
@@ -289,6 +344,40 @@ void restart_slave(TestConnections& test, MYSQL* pSlave)
     test.expect(it2 == row.end(), "START SLAVE failed.");
 }
 
+bool test_basics(TestConnections& test, MYSQL* pSlave)
+{
+    if (insert_select(test, pSlave))
+    {
+        restart_slave(test, pSlave);
+    }
+
+    return test.global_result == 0;
+}
+
+bool test_multiple_masters(TestConnections& test, MYSQL* pSlave)
+{
+    Mariadb_nodes& gc = *test.galera;
+
+    for (int i = 0; i < gc.N; ++i)
+    {
+        test.tprintf("Blocking Galera node %d", i);
+        gc.block_node(i);
+        // Wait a number of times the hearbeat period so as to allow BLR
+        // enough time to detect the lack of the heartbeat and time
+        // to take corrective action.
+        test_sleep(5 * HEARTBEAT_PERIOD);
+
+        MYSQL* pMaster = gc.nodes[(i + 1) % gc.N];
+
+        insert_select(test, pSlave, pMaster);
+
+        test.tprintf("Unblocking Galera node %d", i);
+        gc.unblock_node(i);
+    }
+
+    return test.global_result == 0;
+}
+
 }
 
 int main(int argc, char* argv[])
@@ -304,8 +393,13 @@ int main(int argc, char* argv[])
 
     test.start_maxscale(0);
 
+    bool dont_setup_galera = getenv("MXS1980_DONT_SETUP_GALERA") ? true : false;
+
+    if (!dont_setup_galera)
+    {
         setup_galera(test);
         test.galera->start_replication(); // Causes restart.
+    }
 
     Mariadb_nodes& gc = *test.galera;
     gc.connect();
@@ -319,11 +413,11 @@ int main(int argc, char* argv[])
     const char* zValue;
 
     // Env-vars for debugging.
-    zValue = getenv("MXS2047_BLR_HOST");
+    zValue = getenv("MXS1980_BLR_HOST");
     const char* zMaxscale_host = (zValue ? zValue : test.maxscales->IP[0]);
     cout << "MaxScale host: " << zMaxscale_host << endl;
 
-    zValue = getenv("MXS2047_BLR_PORT");
+    zValue = getenv("MXS1980_BLR_PORT");
     int maxscale_port = (zValue ? atoi(zValue) : test.maxscales->binlog_port[0]);
     cout << "MaxScale port: " << maxscale_port << endl;
 
@@ -331,7 +425,12 @@ int main(int argc, char* argv[])
 
     if (setup_server_ids(test, &server_ids_by_index))
     {
-        if (setup_blr(test, gtid, zMaxscale_host, maxscale_port))
+        MYSQL* pMaxscale = open_conn_no_db(maxscale_port, zMaxscale_host, "repl", "repl");
+        test.expect(pMaxscale, "Could not open connection to BLR at %s:%d.", zMaxscale_host, maxscale_port);
+
+        if (pMaxscale)
+        {
+            if (setup_blr(test, pMaxscale, gtid))
             {
                 int slave_index = test.repl->N - 1; // We use the last slave.
 
@@ -345,22 +444,59 @@ int main(int argc, char* argv[])
                 {
                     if (setup_schema(test, gc.nodes[0]))
                     {
-                    sleep(REPLICATION_SLEEP);
+                        test_sleep(REPLICATION_SLEEP);
 
-                    if (insert_select(test, pSlave))
+                        if (test.ok())
                         {
-                        restart_slave(test, pSlave);
+                            cout << endl;
+                            test.tprintf("Testing basics.");
+                            test_basics(test, pSlave);
+                        }
+
+                        if (test.ok())
+                        {
+                            cout << endl;
+                            test.tprintf("Testing transparent switching of BLR master.");
+
+                            if (setup_secondary_masters(test, pMaxscale))
+                            {
+                                test_multiple_masters(test, pSlave);
+                            }
+                        }
+
+                        if (test.ok())
+                        {
+                            cout << endl;
+                            test.tprintf("Testing functionality when master.ini is used.");
+
+                            cout << "Stopping slave and MaxScale." << endl;
+                            test.try_query(pSlave, "STOP SLAVE");
+                            test.maxscales->stop();
+
+                            cout << "Starting MaxScale." << endl;
+                            test.maxscales->start();
+                            test_sleep(5);
+                            cout << "Starting slave." << endl;
+                            test.try_query(pSlave, "START SLAVE");
+                            test_sleep(3);
+                            test_multiple_masters(test, pSlave);
                         }
                     }
                 }
             }
+
+            mysql_close(pMaxscale);
+        }
     }
 
     // Since setting the server ids can fail half-way, we run this irrespective
     // of what setup_server_ids() returns.
     restore_server_ids(test, server_ids_by_index);
 
+    if (!dont_setup_galera)
+    {
         restore_galera(test);
+    }
 
     return test.global_result;
 }