MXS-1367: Add timeouts for retried queries

The total timeout for the retrying of interrupted queries can now be
configured with the `query_retry_timeout` parameter. It controls the total
timeout in seconds that the query can take.

The actual connection, read and write timeouts of the connector aren't a
good configuration value to use for abstracted queries as the time that it
takes to execute a query can be composed of both connections, reads and
writes. This is caused by the usage of MYSQL_OPT_RECONNECT that hides the
fact that the connector reconnects to the server when a query is
attempted.

Documentation/Getting-Started/Configuration-Guide.md

@@ -178,8 +178,17 @@ The number of times an interrupted query will be retried. This feature was added
 in MaxScale 2.1.10 and is disabled by default.
 
 An interrupted query is any query that is interrupted by a network
-error. Connection timeouts will not trigger a reconnection as it is advisable to
+error. Connection timeouts are included in network errors and thus is it
-increase the timeouts rather than to try timed out queries again.
+advisable to make sure that the value of `query_retry_timeout` is set to an
+adequate value.
+
+#### `query_retry_timeout`
+
+The total timeout in seconds for any retried queries. The default value is 5
+seconds.
+
+An interrupted query is retried for either the configured amount of attempts or
+until the configured timeout is reached.
 
 #### `ms_timestamp`
 

Documentation/Release-Notes/MaxScale-2.1.10-Release-Notes.md

@@ -30,6 +30,8 @@ The internal SQL queries that MaxScale executes to load database users as well
 as monitor the database itself can now be automatically retried if they are
 interrupted. The new global parameter, `query_retries` controls the number of
 retry attempts each query will receive if it fails due to a network problem.
+The `query_retry_timeout` global parameter controls the total timeout for the
+retries.
 
 To enable this functionality, add `query_retries=<number-of-retries>` under the
 `[maxscale]` section where _<number-of-retries>_ is a positive integer.

include/maxscale/config.h

@@ -21,6 +21,7 @@
 #include <limits.h>
 #include <openssl/sha.h>
 #include <sys/utsname.h>
+#include <time.h>
 
 #include <maxscale/modinfo.h>
 
@@ -75,6 +76,7 @@ typedef struct
     char          qc_name[PATH_MAX];                   /**< The name of the query classifier to load */
     char*         qc_args;                             /**< Arguments for the query classifier */
     int           query_retries;                       /**< Number of times a interrupted query is retried */
+    time_t        query_retry_timeout;                 /**< Timeout for query retries */
 } MXS_CONFIG;
 
 /**

server/core/config.c

@@ -1319,6 +1319,20 @@ handle_global_item(const char *name, const char *value)
             return 0;
         }
     }
+    else if (strcmp(name, "query_retry_timeout") == 0)
+    {
+        char* endptr;
+        int intval = strtol(value, &endptr, 0);
+        if (*endptr == '\0' && intval > 0)
+        {
+            gateway.query_retries = intval;
+        }
+        else
+        {
+            MXS_ERROR("Invalid timeout value for 'query_retry_timeout': %s", value);
+            return 0;
+        }
+    }
     else if (strcmp(name, "log_throttling") == 0)
     {
         if (*value == 0)
@@ -1606,6 +1620,7 @@ global_defaults()
     gateway.auth_write_timeout = DEFAULT_AUTH_WRITE_TIMEOUT;
     gateway.skip_permission_checks = false;
     gateway.query_retries = DEFAULT_QUERY_RETRIES;
+    gateway.query_retry_timeout = DEFAULT_QUERY_RETRY_TIMEOUT;
 
     if (version_string != NULL)
     {

server/core/maxscale/config.h

@@ -22,10 +22,11 @@
 
 MXS_BEGIN_DECLS
 
-#define DEFAULT_NBPOLLS         3       /**< Default number of non block polls before we block */
+#define DEFAULT_NBPOLLS             3    /**< Default number of non block polls before we block */
-#define DEFAULT_POLLSLEEP       1000    /**< Default poll wait time (milliseconds) */
+#define DEFAULT_POLLSLEEP           1000 /**< Default poll wait time (milliseconds) */
-#define DEFAULT_NTHREADS        1       /**< Default number of polling threads */
+#define DEFAULT_NTHREADS            1    /**< Default number of polling threads */
-#define DEFAULT_QUERY_RETRIES   0       /**< Number of retries for interrupted queries */
+#define DEFAULT_QUERY_RETRIES       0    /**< Number of retries for interrupted queries */
+#define DEFAULT_QUERY_RETRY_TIMEOUT 5    /**< Timeout for query retries */
 
 /**
  * @brief Generate default module parameters

server/core/mysql_utils.c

@@ -197,10 +197,12 @@ static bool is_connection_error(int errcode)
 int mxs_mysql_query(MYSQL* conn, const char* query)
 {
     MXS_CONFIG* cnf = config_get_global_options();
+    time_t start = time(NULL);
     int rc = mysql_query(conn, query);
 
     for (int n = 0; rc != 0 && n < cnf->query_retries &&
-         is_connection_error(mysql_errno(conn)); n++)
+         is_connection_error(mysql_errno(conn)) &&
+         time(NULL) - start < cnf->query_retry_timeout; n++)
     {
         rc = mysql_query(conn, query);
     }