MXS-1367: Retry interrupted queries

The new `query_retries` parameter controls how many times an interrupted
query is retried. This retrying of interrupted queries will reduce the
rate of false positives that MaxScale monitors detect.

server/core/config.c

@@ -1305,6 +1305,20 @@ handle_global_item(const char *name, const char *value)
     {
         gateway.qc_args = MXS_STRDUP_A(value);
     }
+    else if (strcmp(name, "query_retries") == 0)
+    {
+        char* endptr;
+        int intval = strtol(value, &endptr, 0);
+        if (*endptr == '\0' && intval >= 0)
+        {
+            gateway.query_retries = intval;
+        }
+        else
+        {
+            MXS_ERROR("Invalid timeout value for 'query_retries': %s", value);
+            return 0;
+        }
+    }
     else if (strcmp(name, "log_throttling") == 0)
     {
         if (*value == 0)
@@ -1591,6 +1605,8 @@ global_defaults()
     gateway.auth_read_timeout = DEFAULT_AUTH_READ_TIMEOUT;
     gateway.auth_write_timeout = DEFAULT_AUTH_WRITE_TIMEOUT;
     gateway.skip_permission_checks = false;
+    gateway.query_retries = DEFAULT_QUERY_RETRIES;
+
     if (version_string != NULL)
     {
         gateway.version_string = MXS_STRDUP_A(version_string);

server/core/maxscale/config.h

@@ -25,6 +25,7 @@ MXS_BEGIN_DECLS
 #define DEFAULT_NBPOLLS         3       /**< Default number of non block polls before we block */
 #define DEFAULT_POLLSLEEP       1000    /**< Default poll wait time (milliseconds) */
 #define DEFAULT_NTHREADS        1       /**< Default number of polling threads */
+#define DEFAULT_QUERY_RETRIES   0       /**< Number of retries for interrupted queries */
 
 /**
  * @brief Generate default module parameters

server/core/mysql_utils.c

@@ -21,12 +21,15 @@
  */
 
 #include <maxscale/mysql_utils.h>
+
 #include <string.h>
 #include <stdbool.h>
+#include <errmsg.h>
+
 #include <maxscale/alloc.h>
-#include <maxscale/log_manager.h>
-#include <maxscale/debug.h>
 #include <maxscale/config.h>
+#include <maxscale/debug.h>
+#include <maxscale/log_manager.h>
 
 /**
  * @brief Calculate the length of a length-encoded integer in bytes
@@ -147,16 +150,6 @@ char* mxs_lestr_consume(uint8_t** c, size_t *size)
     return start;
 }
 
-
-
-/**
- * Creates a connection to a MySQL database engine. If necessary, initializes SSL.
- *
- * @param con    A valid MYSQL structure.
- * @param server The server on which the MySQL engine is running.
- * @param user   The MySQL login ID.
- * @param passwd The password for the user.
- */
 MYSQL *mxs_mysql_real_connect(MYSQL *con, SERVER *server, const char *user, const char *passwd)
 {
     SSL_LISTENER *listener = server->server_ssl;
@@ -183,6 +176,38 @@ MYSQL *mxs_mysql_real_connect(MYSQL *con, SERVER *server, const char *user, cons
     return mysql;
 }
 
+static bool is_connection_error(int errcode)
+{
+    switch (errcode)
+    {
+        case CR_SOCKET_CREATE_ERROR:
+        case CR_CONNECTION_ERROR:
+        case CR_CONN_HOST_ERROR:
+        case CR_IPSOCK_ERROR:
+        case CR_SERVER_GONE_ERROR:
+        case CR_TCP_CONNECTION:
+        case CR_SERVER_LOST:
+            return true;
+
+        default:
+            return false;
+    }
+}
+
+int mxs_mysql_query(MYSQL* conn, const char* query)
+{
+    MXS_CONFIG* cnf = config_get_global_options();
+    int rc = mysql_query(conn, query);
+
+    for (int n = 0; rc != 0 && n < cnf->query_retries &&
+         is_connection_error(mysql_errno(conn)); n++)
+    {
+        rc = mysql_query(conn, query);
+    }
+
+    return rc;
+}
+
 bool mxs_mysql_trim_quotes(char *s)
 {
     bool dequoted = true;