Added detection of connection timeout when monitors connect to backend servers.

server/modules/monitor/monitor_common.c

@@ -413,3 +413,92 @@ monitor_event_t mon_name_to_event(char* tok)
 
     }
 
+/**
+ * Connect to a database. This will always leave a valid database handle in the
+ * database->con pointer. This allows the user to call MySQL C API functions to
+ * find out the reason of the failure.
+ * @param mon Monitor
+ * @param database Monitored database
+ * @return MONITOR_CONN_OK if the connection is OK else the reason for the failure
+ */
+connect_result_t mon_connect_to_db(MONITOR* mon, MONITOR_SERVERS *database)
+{
+    connect_result_t rval = MONITOR_CONN_OK;
+
+    /** Return if the connection is OK */
+    if (database->con && mysql_ping(database->con) == 0)
+    {
+        return rval;
+    }
+
+    int connect_timeout = mon->connect_timeout;
+    int read_timeout = mon->read_timeout;
+    int write_timeout = mon->write_timeout;
+    char *uname = database->server->monuser ? database->server->monuser : mon->user;
+    char *passwd = database->server->monpw ? database->server->monpw : mon->password;
+    char *dpwd = decryptPassword(passwd);
+
+    if (database->con)
+    {
+        mysql_close(database->con);
+    }
+    database->con = mysql_init(NULL);
+
+    mysql_options(database->con, MYSQL_OPT_CONNECT_TIMEOUT, (void *) &connect_timeout);
+    mysql_options(database->con, MYSQL_OPT_READ_TIMEOUT, (void *) &read_timeout);
+    mysql_options(database->con, MYSQL_OPT_WRITE_TIMEOUT, (void *) &write_timeout);
+
+    time_t start = time(NULL);
+    bool result = (mysql_real_connect(database->con,
+                                      database->server->name,
+                                      uname,
+                                      dpwd,
+                                      NULL,
+                                      database->server->port,
+                                      NULL,
+                                      0) != NULL);
+    time_t end = time(NULL);
+
+    if (!result)
+    {
+        if ((int) difftime(end, start) >= connect_timeout)
+        {
+            rval = MONITOR_CONN_TIMEOUT;
+        }
+        else
+        {
+            rval = MONITOR_CONN_REFUSED;
+        }
+    }
+
+    free(dpwd);
+    return rval;
+}
+
+/**
+ * Log an error about the failure to connect to a backend server
+ * and why it happened.
+ * @param database Backend database
+ * @param rval Return value of mon_connect_to_db
+ */
+void mon_log_connect_error(MONITOR_SERVERS* database, connect_result_t rval)
+{
+    if (rval == MONITOR_CONN_TIMEOUT)
+    {
+        skygw_log_write_flush(LOGFILE_ERROR,
+                              "Error : Monitor timed out when connecting to "
+                              "server %s:%d : \"%s\"",
+                              database->server->name,
+                              database->server->port,
+                              mysql_error(database->con));
+    }
+    else
+    {
+        skygw_log_write_flush(LOGFILE_ERROR,
+                              "Error : Monitor was unable to connect to "
+                              "server %s:%d : \"%s\"",
+                              database->server->name,
+                              database->server->port,
+                              mysql_error(database->con));
+    }
+}

server/modules/monitor/monitor_common.h

@@ -23,6 +23,7 @@
 #include <monitor.h>
 #include <log_manager.h>
 #include <externcmd.h>
+#include <secrets.h>
 /**
  * @file monitor_common.h - The generic monitor structures all monitors use
  *
@@ -63,6 +64,14 @@ typedef enum {
   NEW_NDB_EVENT,
   MAX_MONITOR_EVENT
 }monitor_event_t;
+
+typedef enum
+{
+    MONITOR_CONN_OK,
+    MONITOR_CONN_REFUSED,
+    MONITOR_CONN_TIMEOUT
+} connect_result_t;
+
 void mon_append_node_names(MONITOR_SERVERS* start,char* str, int len);
 monitor_event_t mon_get_event_type(MONITOR_SERVERS* node);
 char* mon_get_event_name(MONITOR_SERVERS* node);
@@ -72,4 +81,6 @@ bool mon_status_changed(MONITOR_SERVERS* mon_srv);
 bool mon_print_fail_status(MONITOR_SERVERS* mon_srv);
 void monitor_launch_script(MONITOR* mon,MONITOR_SERVERS* ptr, char* script);
 int mon_parse_event_string(bool* events, size_t count,char* string);
+connect_result_t mon_connect_to_db(MONITOR* mon, MONITOR_SERVERS *database);
+void mon_log_connect_error(MONITOR_SERVERS* database, connect_result_t rval);
 #endif

server/modules/monitor/mysql_mon.c

@@ -287,40 +287,6 @@ static void diagnostics(DCB *dcb, void *arg)
     }
     dcb_printf(dcb, "\n");
 }
-/**
- * Connect to a database
- * @param mon Monitor
- * @param database Monitored database
- * @return true if connection was successful, false if there was an error
- */
-static inline bool connect_to_db(MONITOR* mon,MONITOR_SERVERS *database)
-{
-    char		  *uname  = mon->user;
-    char              *passwd = mon->password;
-    char *dpwd = decryptPassword(passwd);
-    int  connect_timeout = mon->connect_timeout;
-    int  read_timeout = mon->read_timeout;
-    int  write_timeout = mon->write_timeout;
-
-    if(database->con)
-	mysql_close(database->con);
-    database->con = mysql_init(NULL);
-
-    mysql_options(database->con, MYSQL_OPT_CONNECT_TIMEOUT, (void *)&connect_timeout);
-    mysql_options(database->con, MYSQL_OPT_READ_TIMEOUT, (void *)&read_timeout);
-    mysql_options(database->con, MYSQL_OPT_WRITE_TIMEOUT, (void *)&write_timeout);
-
-    bool result = (mysql_real_connect(database->con,
-			       database->server->name,
-			       uname,
-			       dpwd,
-			       NULL,
-			       database->server->port,
-			       NULL,
-			       0) != NULL);
-	free(dpwd);
-	return result;
-}
 
 static inline void monitor_mysql100_db(MONITOR_SERVERS* database)
 {
@@ -641,7 +607,8 @@ monitorDatabase(MONITOR *mon, MONITOR_SERVERS *database)
 
     if (database->con == NULL || mysql_ping(database->con) != 0)
     {
-	if(connect_to_db(mon,database))
+	connect_result_t rval;
+	if ((rval = mon_connect_to_db(mon, database)) == MONITOR_CONN_OK)
 	{
 	    server_clear_status(database->server, SERVER_AUTH_ERROR);
 	    monitor_clear_pending_status(database, SERVER_AUTH_ERROR);
@@ -676,13 +643,7 @@ monitorDatabase(MONITOR *mon, MONITOR_SERVERS *database)
 	    /* Log connect failure only once */
 	    if (mon_status_changed(database) && mon_print_fail_status(database))
         {
-		LOGIF(LE, (skygw_log_write_flush(
+            mon_log_connect_error(database, rval);
-			LOGFILE_ERROR,
-						 "Error : Monitor was unable to connect to "
-			"server %s:%d : \"%s\"",
-						 database->server->name,
-						 database->server->port,
-						 mysql_error(database->con))));
         }
 
 	    return;