MXS-1367: Retry interrupted queries

The new `query_retries` parameter controls how many times an interrupted
query is retried. This retrying of interrupted queries will reduce the
rate of false positives that MaxScale monitors detect.

Documentation/Getting-Started/Configuration-Guide.md

@@ -172,6 +172,15 @@ write or modify the data in the backend server. The default is 2 seconds.
 auth_write_timeout=10
 ```
 
+#### `query_retries`
+
+The number of times an interrupted query will be retried. This feature was added
+in MaxScale 2.1.10 and is disabled by default.
+
+An interrupted query is any query that is interrupted by a network
+error. Connection timeouts will not trigger a reconnection as it is advisable to
+increase the timeouts rather than to try timed out queries again.
+
 #### `ms_timestamp`
 
 Enable or disable the high precision timestamps in logfiles. Enabling this adds

Documentation/Release-Notes/MaxScale-2.1.10-Release-Notes.md

@@ -22,6 +22,18 @@ release notes:
 For any problems you encounter, please consider submitting a bug report at
 [Jira](https://jira.mariadb.org).
 
+## Changed Features
+
+### Internal Query Retries
+
+The internal SQL queries that MaxScale executes to load database users as well
+as monitor the database itself can now be automatically retried if they are
+interrupted. The new global parameter, `query_retries` controls the number of
+retry attempts each query will receive if it fails due to a network problem.
+
+To enable this functionality, add `query_retries=<number-of-retries>` under the
+`[maxscale]` section where _<number-of-retries>_ is a positive integer.
+
 ## Bug fixes
 
 [Here is a list of bugs fixed in MaxScale 2.1.10.](https://jira.mariadb.org/issues/?jql=project%20%3D%20MXS%20AND%20issuetype%20%3D%20Bug%20AND%20status%20%3D%20Closed%20AND%20fixVersion%20%3D%202.1.10)

include/maxscale/config.h

@@ -74,6 +74,7 @@ typedef struct
     bool          skip_permission_checks;              /**< Skip service and monitor permission checks */
     char          qc_name[PATH_MAX];                   /**< The name of the query classifier to load */
     char*         qc_args;                             /**< Arguments for the query classifier */
+    int           query_retries;                       /**< Number of times a interrupted query is retried */
 } MXS_CONFIG;
 
 /**

include/maxscale/mysql_utils.h

@@ -29,7 +29,29 @@ uint64_t mxs_leint_consume(uint8_t ** c);
 char* mxs_lestr_consume_dup(uint8_t** c);
 char* mxs_lestr_consume(uint8_t** c, size_t *size);
 
-MYSQL *mxs_mysql_real_connect(MYSQL *mysql, SERVER *server, const char *user, const char *passwd);
+/**
+ * Creates a connection to a MySQL database engine. If necessary, initializes SSL.
+ *
+ * @param con    A valid MYSQL structure.
+ * @param server The server on which the MySQL engine is running.
+ * @param user   The MySQL login ID.
+ * @param passwd The password for the user.
+ *
+ * @return New connection or NULL on error
+ */
+MYSQL* mxs_mysql_real_connect(MYSQL *mysql, SERVER *server, const char *user, const char *passwd);
+
+/**
+ * Execute a query
+ *
+ * This function wraps mysql_query in a way that automatic query retry is possible.
+ *
+ * @param conn  MySQL connection
+ * @param query Query to execute
+ *
+ * @return return value of mysql_query
+ */
+int mxs_mysql_query(MYSQL* conn, const char* query);
 
 /**
  * Trim MySQL quote characters surrounding a string.

server/core/config.c

@@ -1305,6 +1305,20 @@ handle_global_item(const char *name, const char *value)
     {
         gateway.qc_args = MXS_STRDUP_A(value);
     }
+    else if (strcmp(name, "query_retries") == 0)
+    {
+        char* endptr;
+        int intval = strtol(value, &endptr, 0);
+        if (*endptr == '\0' && intval >= 0)
+        {
+            gateway.query_retries = intval;
+        }
+        else
+        {
+            MXS_ERROR("Invalid timeout value for 'query_retries': %s", value);
+            return 0;
+        }
+    }
     else if (strcmp(name, "log_throttling") == 0)
     {
         if (*value == 0)
@@ -1591,6 +1605,8 @@ global_defaults()
     gateway.auth_read_timeout = DEFAULT_AUTH_READ_TIMEOUT;
     gateway.auth_write_timeout = DEFAULT_AUTH_WRITE_TIMEOUT;
     gateway.skip_permission_checks = false;
+    gateway.query_retries = DEFAULT_QUERY_RETRIES;
+
     if (version_string != NULL)
     {
         gateway.version_string = MXS_STRDUP_A(version_string);

server/core/maxscale/config.h

@@ -25,6 +25,7 @@ MXS_BEGIN_DECLS
 #define DEFAULT_NBPOLLS         3       /**< Default number of non block polls before we block */
 #define DEFAULT_POLLSLEEP       1000    /**< Default poll wait time (milliseconds) */
 #define DEFAULT_NTHREADS        1       /**< Default number of polling threads */
+#define DEFAULT_QUERY_RETRIES   0       /**< Number of retries for interrupted queries */
 
 /**
  * @brief Generate default module parameters

server/core/mysql_utils.c

@@ -21,12 +21,15 @@
  */
 
 #include <maxscale/mysql_utils.h>
+
 #include <string.h>
 #include <stdbool.h>
+#include <errmsg.h>
+
 #include <maxscale/alloc.h>
-#include <maxscale/log_manager.h>
-#include <maxscale/debug.h>
 #include <maxscale/config.h>
+#include <maxscale/debug.h>
+#include <maxscale/log_manager.h>
 
 /**
  * @brief Calculate the length of a length-encoded integer in bytes
@@ -147,16 +150,6 @@ char* mxs_lestr_consume(uint8_t** c, size_t *size)
     return start;
 }
 
-
-
-/**
- * Creates a connection to a MySQL database engine. If necessary, initializes SSL.
- *
- * @param con    A valid MYSQL structure.
- * @param server The server on which the MySQL engine is running.
- * @param user   The MySQL login ID.
- * @param passwd The password for the user.
- */
 MYSQL *mxs_mysql_real_connect(MYSQL *con, SERVER *server, const char *user, const char *passwd)
 {
     SSL_LISTENER *listener = server->server_ssl;
@@ -183,6 +176,38 @@ MYSQL *mxs_mysql_real_connect(MYSQL *con, SERVER *server, const char *user, cons
     return mysql;
 }
 
+static bool is_connection_error(int errcode)
+{
+    switch (errcode)
+    {
+        case CR_SOCKET_CREATE_ERROR:
+        case CR_CONNECTION_ERROR:
+        case CR_CONN_HOST_ERROR:
+        case CR_IPSOCK_ERROR:
+        case CR_SERVER_GONE_ERROR:
+        case CR_TCP_CONNECTION:
+        case CR_SERVER_LOST:
+            return true;
+
+        default:
+            return false;
+    }
+}
+
+int mxs_mysql_query(MYSQL* conn, const char* query)
+{
+    MXS_CONFIG* cnf = config_get_global_options();
+    int rc = mysql_query(conn, query);
+
+    for (int n = 0; rc != 0 && n < cnf->query_retries &&
+         is_connection_error(mysql_errno(conn)); n++)
+    {
+        rc = mysql_query(conn, query);
+    }
+
+    return rc;
+}
+
 bool mxs_mysql_trim_quotes(char *s)
 {
     bool dequoted = true;