MXS-2141: Retry query on master if it times out on slave

With causal_reads enabled, the query would return with an error if the
slave was not able to catch up to the master fast enough. By automatically
retrying the query on the master, we're guaranteed that a valid result is
always returned to the client.

server/modules/routing/readwritesplit/rwsplit_route_stmt.cc

@@ -1023,6 +1023,9 @@ GWBUF* RWSplitSession::add_prefix_wait_gtid(SERVER* server, GWBUF* origin)
         snprintf(prefix_sql, prefix_len, gtid_wait_stmt, wait_func, gtid_position, gtid_wait_timeout);
         GWBUF* prefix_buff = modutil_create_query(prefix_sql);
 
+        // Copy the original query in case it fails on the slave
+        m_current_query.copy_from(origin);
+
         /* Trim origin to sql, Append origin buffer to the prefix buffer */
         uint8_t header[MYSQL_HEADER_LEN];
         gwbuf_copy_data(origin, 0, MYSQL_HEADER_LEN, header);
@@ -1075,6 +1078,9 @@ bool RWSplitSession::handle_got_target(GWBUF* querybuf, SRWBackend& target, bool
         // Perform the causal read only when the query is routed to a slave
         send_buf = add_prefix_wait_gtid(target->server(), send_buf);
         m_wait_gtid = WAITING_FOR_HEADER;
+
+        // The storage for causal reads is done inside add_prefix_wait_gtid
+        store = false;
     }
 
     if (m_qc.load_data_state() != QueryClassifier::LOAD_DATA_ACTIVE

server/modules/routing/readwritesplit/rwsplitsession.cc

@@ -275,7 +275,7 @@ GWBUF* RWSplitSession::discard_master_wait_gtid_result(GWBUF* buffer)
     else if (MYSQL_GET_COMMAND(header_and_command) == MYSQL_REPLY_ERR)
     {
         // The MASTER_WAIT_GTID command failed and no further packets will come
-        m_wait_gtid = NONE;
+        m_wait_gtid = RETRYING_ON_MASTER;
     }
 
     return buffer;
@@ -524,6 +524,10 @@ void RWSplitSession::manage_transactions(SRWBackend& backend, GWBUF* writebuf)
             }
         }
     }
+    else if (m_wait_gtid == RETRYING_ON_MASTER)
+    {
+        // We're retrying the query on the master and we need to keep the current query
+    }
     else
     {
         /** Normal response, reset the currently active query. This is done before
@@ -575,6 +579,23 @@ void RWSplitSession::clientReply(GWBUF* writebuf, DCB* backend_dcb)
         mxb_assert(backend->get_reply_state() == REPLY_STATE_DONE);
         MXS_INFO("Reply complete, last reply from %s", backend->name());
 
+        if (m_wait_gtid == RETRYING_ON_MASTER)
+        {
+            m_wait_gtid = NONE;
+
+            // Discard the error
+            gwbuf_free(writebuf);
+            writebuf = NULL;
+
+            // Retry the query on the master
+            GWBUF* buf = m_current_query.release();
+            buf->hint = hint_create_route(buf->hint, HINT_ROUTE_TO_MASTER, NULL);
+            retry_query(buf, 0);
+
+            // Stop the response processing early
+            return;
+        }
+
         ResponseStat& stat = backend->response_stat();
         stat.query_ended();
         if (stat.is_valid() && (stat.sync_time_reached()

server/modules/routing/readwritesplit/rwsplitsession.hh

@@ -73,6 +73,7 @@ public:
     {
         NONE,
         WAITING_FOR_HEADER,
+        RETRYING_ON_MASTER,
         UPDATING_PACKETS
     };