Bug #443, http://bugs.skysql.com/show_bug.cgi?id=443

monitors used mysql_ping without MYSQL_OPT_READ_TIMEOUT which caused read to block. Fixed in mysql and galera monitor.
Added log writing per each status change of each server and repeatedly if server is not running.

Removed SERVER_IS_JOINED checks from rwsplit router.

server/modules/include/readwritesplit.h

@@ -229,7 +229,6 @@ typedef struct router_instance {
 } ROUTER_INSTANCE;
 
 #define BACKEND_TYPE(b) (SERVER_IS_MASTER((b)->backend_server) ? BE_MASTER :    \
-        (SERVER_IS_SLAVE((b)->backend_server) ? BE_SLAVE :                      \
+        (SERVER_IS_SLAVE((b)->backend_server) ? BE_SLAVE :  BE_UNDEFINED));
-        (SERVER_IS_JOINED((b)->backend_server) ? BE_JOINED : BE_UNDEFINED)));
 
 #endif /*< _RWSPLITROUTER_H */

server/modules/monitor/galera_mon.c

@@ -317,10 +317,22 @@ char 			*server_string;
 	if (database->con == NULL || mysql_ping(database->con) != 0)
 	{
 		char *dpwd = decryptPassword(passwd);
+		int rc;
+		int read_timeout = 1;
+
 		database->con = mysql_init(NULL);
+		rc = mysql_options(database->con, MYSQL_OPT_READ_TIMEOUT, (void *)&read_timeout);
+
 		if (mysql_real_connect(database->con, database->server->name,
 			uname, dpwd, NULL, database->server->port, NULL, 0) == NULL)
 		{
+			LOGIF(LE, (skygw_log_write_flush(
+				LOGFILE_ERROR,
+				"Error : Monitor was unable to connect to "
+				"server %s:%d : \"%s\"",
+				database->server->name,
+				database->server->port,
+				mysql_error(database->con))));
 			server_clear_status(database->server, SERVER_RUNNING);
 			database->server->node_id = -1;
 			free(dpwd);
@@ -416,6 +428,7 @@ long master_id;
 
 		while (ptr)
 		{
+			unsigned int prev_status = ptr->server->status;
 			monitorDatabase(ptr, handle->defaultUser, handle->defaultPasswd);
 
 			/* set master_id to the lowest value of ptr->server->node_id */
@@ -433,6 +446,16 @@ long master_id;
 				server_clear_status(ptr->server, SERVER_SLAVE);
                 		server_clear_status(ptr->server, SERVER_MASTER);
 			}
+			if (ptr->server->status != prev_status ||
+				SERVER_IS_DOWN(ptr->server))
+			{
+				LOGIF(LM, (skygw_log_write_flush(
+					LOGFILE_MESSAGE,
+					"Backend server %s:%d state : %s",
+					ptr->server->name,
+					ptr->server->port,
+					STRSRVSTATUS(ptr->server))));
+			}
 			ptr = ptr->next;
 		}
 

server/modules/monitor/mysql_mon.c

@@ -302,7 +302,7 @@ char		*sep;
  * Monitor an individual server
  *
  * @param handle        The MySQL Monitor object
- * @param database      The database to probe
+ * @param database	The database to probe
  */
 static void
 monitorDatabase(MYSQL_MONITOR *handle, MONITOR_SERVERS *database)
@@ -332,7 +332,11 @@ int			replication_heartbeat = handle->replicationHeartbeat;
 	if (database->con == NULL || mysql_ping(database->con) != 0)
 	{
 		char *dpwd = decryptPassword(passwd);
+                int  rc;
+                int  read_timeout = 1;
 		database->con = mysql_init(NULL);
+                rc = mysql_options(database->con, MYSQL_OPT_READ_TIMEOUT, (void *)&read_timeout);
+                
 		if (mysql_real_connect(database->con,
                                        database->server->name,
                                        uname,
@@ -342,6 +346,14 @@ int			replication_heartbeat = handle->replicationHeartbeat;
                                        NULL,
                                        0) == NULL)
 		{
+                        LOGIF(LE, (skygw_log_write_flush(
+                                LOGFILE_ERROR,
+                                "Error : Monitor was unable to connect to "
+                                "server %s:%d : \"%s\"",
+                                database->server->name,
+                                database->server->port,
+                                mysql_error(database->con))));
+                        
 			free(dpwd);
 			server_clear_status(database->server, SERVER_RUNNING);
 			return;
@@ -626,7 +638,6 @@ int			replication_heartbeat = handle->replicationHeartbeat;
 		server_clear_status(database->server, SERVER_SLAVE);
 		server_clear_status(database->server, SERVER_MASTER);
 	}
-	
 }
 
 /**
@@ -661,7 +672,21 @@ MONITOR_SERVERS	*ptr;
 		ptr = handle->databases;
 		while (ptr)
 		{
+                        unsigned int prev_status = ptr->server->status;
+                        
 			monitorDatabase(handle, ptr);
+                        
+                        if (ptr->server->status != prev_status ||
+                                SERVER_IS_DOWN(ptr->server))
+                        {
+                                LOGIF(LM, (skygw_log_write_flush(
+                                        LOGFILE_MESSAGE,
+                                        "Backend server %s:%d state : %s",
+                                        ptr->server->name,
+                                        ptr->server->port,
+                                        STRSRVSTATUS(ptr->server))));
+                        }
+                        
 			ptr = ptr->next;
 		}
 		thread_millisleep(handle->interval);
@@ -676,10 +701,10 @@ MONITOR_SERVERS	*ptr;
  */
 static void
 defaultId(void *arg, unsigned long id)
-{
+                        {
 MYSQL_MONITOR   *handle = (MYSQL_MONITOR *)arg;
 	memcpy(&handle->id, &id, sizeof(unsigned long));
-}
+                        }
                         
 /**
  * Set the monitor sampling interval.
@@ -692,7 +717,7 @@ setInterval(void *arg, unsigned long interval)
 {
 MYSQL_MONITOR   *handle = (MYSQL_MONITOR *)arg;
 	memcpy(&handle->interval, &interval, sizeof(unsigned long));
-}
+	}
 
 /**
  * Enable/Disable the MySQL Replication hearbeat, detecting slave lag behind master.

server/modules/routing/readconnroute.c

@@ -352,7 +352,7 @@ int			master_host = -1;
 				inst->bitmask)));
 		}
 
-		if (SERVER_IN_MAINT(inst->server))
+		if (SERVER_IN_MAINT(inst->servers[i]->server))
 			continue;
 
 		/*

server/modules/routing/readwritesplit/readwritesplit.c

@@ -808,7 +808,7 @@ static bool get_dcb(
                 }                        
                 ss_dassert(succp);
         }
-        else if (btype == BE_MASTER || BE_JOINED)
+        else if (btype == BE_MASTER)
         {
                 for (i=0; i<rses->rses_nbackends; i++)
                 {

utils/skygw_debug.h

@@ -229,6 +229,12 @@ typedef enum skygw_chk_t {
                         ((c) == LEAST_ROUTER_CONNECTIONS ? "LEAST_ROUTER_CONNECTIONS" : \
                         ((c) == LEAST_BEHIND_MASTER ? "LEAST_BEHIND_MASTER" : "Unknown criteria"))))
 
+#define STRSRVSTATUS(s) ((SERVER_IS_RUNNING(s) && SERVER_IS_MASTER(s)) ? "RUNNING MASTER" :     \
+                        ((SERVER_IS_RUNNING(s) && SERVER_IS_SLAVE(s)) ? "RUNNING SLAVE" :       \
+                        ((SERVER_IS_RUNNING(s) && SERVER_IS_JOINED(s)) ? "RUNNING JOINED" :     \
+                        ((SERVER_IS_RUNNING(s) && SERVER_IS_MAINT(s)) ? "RUNNING MAINTENANCE" : \
+                        (SERVER_IS_RUNNING(s) ? "RUNNING (only)" : "NO STATUS")))))
+
 #define CHK_MLIST(l) {                                                  \
             ss_info_dassert((l->mlist_chk_top ==  CHK_NUM_MLIST &&      \
                              l->mlist_chk_tail == CHK_NUM_MLIST),       \