Merge branch 'MAX-316' into develop

server/core/config.c

@@ -38,6 +38,7 @@
  * 09/09/14	Massimiliano Pinto	Added localhost_match_wildcard_host parameter
  * 12/09/14	Mark Riddoch		Addition of checks on servers list and
  *					internal router suppression of messages
+ * 30/10/14	Massimiliano Pinto	Added disable_master_failback parameter
  * 07/11/14	Massimiliano Pinto	Addition of monitor timeouts for connect/read/write
  *
  * @endverbatim
@@ -774,6 +775,7 @@ int			error_count = 0;
 			unsigned long interval = 0;
 			int replication_heartbeat = 0;
 			int detect_stale_master = 0;
+			int disable_master_failback = 0;
 			int connect_timeout = 0;
 			int read_timeout = 0;
 			int write_timeout = 0;
@@ -794,6 +796,10 @@ int			error_count = 0;
 				detect_stale_master = atoi(config_get_value(obj->parameters, "detect_stale_master"));
 			}
 
+			if (config_get_value(obj->parameters, "disable_master_failback")) {
+				disable_master_failback = atoi(config_get_value(obj->parameters, "disable_master_failback"));
+			}
+
 			if (config_get_value(obj->parameters, "backend_connect_timeout")) {
 				connect_timeout = atoi(config_get_value(obj->parameters, "backend_connect_timeout"));
 			}
@@ -804,7 +810,6 @@ int			error_count = 0;
 				write_timeout = atoi(config_get_value(obj->parameters, "backend_write_timeout"));
 			}
 			
-
                         if (module)
 			{
 				obj->element = monitor_alloc(obj->object, module);
@@ -832,6 +837,10 @@ int			error_count = 0;
 					if(detect_stale_master == 1)
 						monitorDetectStaleMaster(obj->element, detect_stale_master);
 
+					/* disable master failback */
+					if(disable_master_failback == 1)
+						monitorDisableMasterFailback(obj->element, disable_master_failback);
+
 					/* set timeouts */
 					if (connect_timeout > 0)
 						monitorSetNetworkTimeout(obj->element, MONITOR_CONNECT_TIMEOUT, connect_timeout);
@@ -1649,6 +1658,7 @@ static char *monitor_params[] =
 		"monitor_interval",
 		"detect_replication_lag",
 		"detect_stale_master",
+		"disable_master_failback",
 		"backend_connect_timeout",
 		"backend_read_timeout",
 		"backend_write_timeout",

server/core/monitor.c

@@ -26,6 +26,7 @@
  * 08/07/13	Mark Riddoch		Initial implementation
  * 23/05/14	Massimiliano Pinto	Addition of monitor_interval parameter
  * 					and monitor id
+ * 30/10/14	Massimiliano Pinto	Addition of disable_master_failback parameter
  * 07/11/14	Massimiliano Pinto	Addition of monitor network timeouts
  *
  * @endverbatim
@@ -331,6 +332,19 @@ monitorDetectStaleMaster(MONITOR *mon, int enable)
 	}
 }
 
+/**
+ * Disable Master Failback
+ *
+ * @param mon		The monitor instance
+ * @param disable	The value 1 disable the failback, 0 keeps it
+ */
+void
+monitorDisableMasterFailback(MONITOR *mon, int disable)
+{
+	if (mon->module->disableMasterFailback != NULL) {
+		mon->module->disableMasterFailback(mon->handle, disable);
+}
+
 /**
  * Set Monitor timeouts for connect/read/write
  *

server/core/server.c

@@ -30,7 +30,8 @@
  * 28/05/14	Massimiliano Pinto	Addition of rlagd and node_ts fields
  * 20/06/14	Massimiliano Pinto	Addition of master_id, depth, slaves fields
  * 26/06/14	Mark Riddoch		Addition of server parameters
- * 30/08/14	Massimiliano Pinto	Addition of new service status description
+ * 30/08/14	Massimiliano Pinto	Addition of new service status description 
+ * 30/10/14	Massimiliano Pinto	Addition of SERVER_MASTER_STICKINESS description
  *
  * @endverbatim
  */
@@ -424,6 +425,8 @@ char	*status = NULL;
 		strcat(status, "Slave of External Server, ");
 	if (server->status & SERVER_STALE_STATUS)
 		strcat(status, "Stale Status, ");
+	if (server->status & SERVER_MASTER_STICKINESS)
+		strcat(status, "Master Stickiness, ");
 	if (server->status & SERVER_AUTH_ERROR)
 		strcat(status, "Auth Error, ");
 	if (server->status & SERVER_RUNNING)

server/include/monitor.h

@@ -33,6 +33,7 @@
  * 23/05/14	Massimiliano Pinto	Addition of defaultId and setInterval
  * 23/06/14	Massimiliano Pinto	Addition of replicationHeartbeat
  * 28/08/14	Massimiliano Pinto	Addition of detectStaleMaster
+ * 30/10/14	Massimiliano Pinto	Addition of disableMasterFailback
  * 07/11/14	Massimiliano Pinto	Addition of setNetworkTimeout
  *
  * @endverbatim
@@ -75,6 +76,7 @@ typedef struct {
 	void	(*defaultId)(void *, unsigned long);
 	void	(*replicationHeartbeat)(void *, int);
 	void	(*detectStaleMaster)(void *, int);
+	void	(*disableMasterFailback)(void *, int);
 } MONITOR_OBJECT;
 
 /**
@@ -135,5 +137,6 @@ extern void     monitorSetId(MONITOR *, unsigned long);
 extern void     monitorSetInterval (MONITOR *, unsigned long);
 extern void     monitorSetReplicationHeartbeat(MONITOR *, int);
 extern void     monitorDetectStaleMaster(MONITOR *, int);
+extern void     monitorDisableMasterFailback(MONITOR *, int);
 extern void     monitorSetNetworkTimeout(MONITOR *, int, int);
 #endif

server/include/server.h

@@ -40,6 +40,7 @@
  * 26/06/14	Mark Riddoch		Adidtion of server parameters
  * 30/07/14	Massimiliano Pinto	Addition of NDB status for MySQL Cluster
  * 30/08/14	Massimiliano Pinto	Addition of SERVER_STALE_STATUS
+ * 27/10/14	Massimiliano Pinto	Addition of SERVER_MASTER_STICKINESS
  *
  * @endverbatim
  */
@@ -105,6 +106,7 @@ typedef struct server {
 #define SERVER_MAINT		0x0020	/**<< Server is in maintenance mode */
 #define SERVER_SLAVE_OF_EXTERNAL_MASTER  0x0040	/**<< Server is slave of a Master outside the provided replication topology */
 #define SERVER_STALE_STATUS	0x0080	/**<< Server stale status, monitor didn't update it */
+#define SERVER_MASTER_STICKINESS	0x0100	/**<< Server Master stickiness */
 #define SERVER_AUTH_ERROR	0x1000	/**<< Authentication erorr from monitor */
 
 /**

server/modules/monitor/galera_mon.c

@@ -30,6 +30,7 @@
  * 					Interval is printed in diagnostics.
  * 03/06/14	Mark Riddoch		Add support for maintenance mode
  * 24/06/14	Massimiliano Pinto	Added depth level 0 for each node
+ * 30/10/14	Massimiliano Pinto	Added disableMasterFailback feature
  *
  * @endverbatim
  */
@@ -52,7 +53,7 @@ extern int lm_enabled_logfiles_bitmask;
 
 static	void	monitorMain(void *);
 
-static char *version_str = "V1.2.0";
+static char *version_str = "V1.3.0";
 
 MODULE_INFO	info = {
 	MODULE_API_MONITOR,
@@ -68,6 +69,9 @@ static	void	unregisterServer(void *, SERVER *);
 static	void	defaultUsers(void *, char *, char *);
 static	void	diagnostics(DCB *, void *);
 static  void    setInterval(void *, size_t);
+static MONITOR_SERVERS *get_candidate_master(MONITOR_SERVERS *);
+static MONITOR_SERVERS *set_cluster_master(MONITOR_SERVERS *, MONITOR_SERVERS *, int);
+static	void	disableMasterFailback(void *, int);
 
 static MONITOR_OBJECT MyObject = { 
 	startMonitor, 
@@ -80,7 +84,8 @@ static MONITOR_OBJECT MyObject = {
 	NULL,
 	NULL, 
 	NULL, 
-	NULL
+	NULL,
+	disableMasterFailback
 };
 
 /**
@@ -148,6 +153,8 @@ MYSQL_MONITOR *handle;
 		handle->defaultPasswd = NULL;
 		handle->id = MONITOR_DEFAULT_ID;
 		handle->interval = MONITOR_INTERVAL;
+		handle->disableMasterFailback = 0;
+		handle->master = NULL;
 		spinlock_init(&handle->lock);
 	}
 	handle->tid = (THREAD)thread_start(monitorMain, handle);
@@ -265,6 +272,7 @@ char		*sep;
 	}
 
 	dcb_printf(dcb,"\tSampling interval:\t%lu milliseconds\n", handle->interval);
+	dcb_printf(dcb,"\tMaster Failback:\t%s\n", (handle->disableMasterFailback == 1) ? "off" : "on");
 	dcb_printf(dcb, "\tMonitored servers:	");
 
 	db = handle->databases;
@@ -332,8 +340,10 @@ char 			*server_string;
 		char *dpwd = decryptPassword(passwd);
 		int rc;
 		int read_timeout = 1;
+		int connect_timeout = 2;
 
 		database->con = mysql_init(NULL);
+		rc = mysql_options(database->con, MYSQL_OPT_CONNECT_TIMEOUT, (void *)&connect_timeout);
 		rc = mysql_options(database->con, MYSQL_OPT_READ_TIMEOUT, (void *)&read_timeout);
 
 		if (mysql_real_connect(database->con, database->server->name,
@@ -346,7 +356,15 @@ char 			*server_string;
 				database->server->name,
 				database->server->port,
 				mysql_error(database->con))));
+
 			server_clear_status(database->server, SERVER_RUNNING);
+
+			/* Also clear Joined, M/S and Stickiness bits */
+			server_clear_status(database->server, SERVER_JOINED);
+			server_clear_status(database->server, SERVER_SLAVE);
+			server_clear_status(database->server, SERVER_MASTER);
+			server_clear_status(database->server, SERVER_MASTER_STICKINESS);
+
 			if (mysql_errno(database->con) == ER_ACCESS_DENIED_ERROR)
 			{
 				server_set_status(database->server, SERVER_AUTH_ERROR);
@@ -422,10 +440,11 @@ char 			*server_string;
 static void
 monitorMain(void *arg)
 {
-MYSQL_MONITOR	*handle = (MYSQL_MONITOR *)arg;
+MYSQL_MONITOR		*handle = (MYSQL_MONITOR *)arg;
-MONITOR_SERVERS	*ptr;
+MONITOR_SERVERS		*ptr;
-long master_id;
+size_t			nrounds = 0;
-size_t nrounds = 0;
+MONITOR_SERVERS		*candidate_master = NULL;
+int			master_stickiness = handle->disableMasterFailback;
 
 	if (mysql_thread_init())
 	{
@@ -454,39 +473,35 @@ size_t nrounds = 0;
 		 * interval, then skip monitoring checks. Excluding the first
 		 * round.
 		 */ 
-		if (nrounds != 0 && 
+
-			((nrounds*MON_BASE_INTERVAL_MS)%handle->interval) >= 
+		if (nrounds != 0 && ((nrounds*MON_BASE_INTERVAL_MS)%handle->interval) >= MON_BASE_INTERVAL_MS) 
-			MON_BASE_INTERVAL_MS) 
 		{
 			nrounds += 1;
 			continue;
 		}
+
 		nrounds += 1;
-		master_id = -1;
 		ptr = handle->databases;
 
 		while (ptr)
 		{
 			unsigned int prev_status = ptr->server->status;
+
 			monitorDatabase(ptr, handle->defaultUser, handle->defaultPasswd);
 
-			/* set master_id to the lowest value of ptr->server->node_id */
+			/* clear bits for non member nodes */
+			if ( ! SERVER_IN_MAINT(ptr->server) && (ptr->server->node_id < 0 || ! SERVER_IS_JOINED(ptr->server))) {
+				ptr->server->depth = -1;
 
-			if ((! SERVER_IN_MAINT(ptr->server))  && ptr->server->node_id >= 0 && SERVER_IS_JOINED(ptr->server)) {
-				ptr->server->depth = 0;
-				if (ptr->server->node_id < master_id && master_id >= 0) {
-					master_id = ptr->server->node_id;
-				} else {
-					if (master_id < 0) {
-						master_id = ptr->server->node_id;
-					}
-				}
-			} else if (!SERVER_IN_MAINT(ptr->server)) {
 				/* clear M/S status */
 				server_clear_status(ptr->server, SERVER_SLAVE);
                 		server_clear_status(ptr->server, SERVER_MASTER);
-				ptr->server->depth = -1;
+				
+				/* clear master sticky status */
+				server_clear_status(ptr->server, SERVER_MASTER_STICKINESS);
 			}
+
+			/* Log server status change */
 			if (ptr->server->status != prev_status ||
 				SERVER_IS_DOWN(ptr->server))
 			{
@@ -497,24 +512,50 @@ size_t nrounds = 0;
 					ptr->server->port,
 					STRSRVSTATUS(ptr->server))));
 			}
+
 			ptr = ptr->next;
 		}
 
+		/*
+		 * Let's select a master server:
+		 * it could be the candidate master following MIN(node_id) rule or
+		 * the server that was master in the previous monitor polling cycle
+		 * Decision depends on master_stickiness value set in configuration
+		 */
+
+		/* get the candidate master, followinf MIN(node_id) rule */
+		candidate_master = get_candidate_master(handle->databases);
+
+		/* Select the master, based on master_stickiness */
+		handle->master = set_cluster_master(handle->master, candidate_master, master_stickiness);
+
 		ptr = handle->databases;
 
-		/* this server loop sets Master and Slave roles */
+		while (ptr && handle->master) {
-		while (ptr)
+			if (!SERVER_IS_JOINED(ptr->server) || SERVER_IN_MAINT(ptr->server)) {
-		{
+				ptr = ptr->next;
-			if ((! SERVER_IN_MAINT(ptr->server)) && ptr->server->node_id >= 0 && master_id >= 0) {
+				continue;
-				/* set the Master role */
+			}
-				if (SERVER_IS_JOINED(ptr->server) && (ptr->server->node_id == master_id)) {
+
-                			server_set_status(ptr->server, SERVER_MASTER);
+			if (ptr != handle->master) {
-                			server_clear_status(ptr->server, SERVER_SLAVE);
-				} else if (SERVER_IS_JOINED(ptr->server) && (ptr->server->node_id > master_id)) {
 				/* set the Slave role */
-                			server_set_status(ptr->server, SERVER_SLAVE);
+				server_set_status(ptr->server, SERVER_SLAVE);
-                			server_clear_status(ptr->server, SERVER_MASTER);
+				server_clear_status(ptr->server, SERVER_MASTER);
-				}
+
+				/* clear master stickyness */
+				server_clear_status(ptr->server, SERVER_MASTER_STICKINESS);
+			} else {
+				/* set the Master role */
+				server_set_status(handle->master->server, SERVER_MASTER);
+				server_clear_status(handle->master->server, SERVER_SLAVE);
+
+				if (candidate_master && handle->master->server->node_id != candidate_master->server->node_id) {
+					/* set master stickyness */
+					server_set_status(handle->master->server, SERVER_MASTER_STICKINESS);
+				} else {
+					/* clear master stickyness */
+					server_clear_status(ptr->server, SERVER_MASTER_STICKINESS);
+				}			
 			}
 
 			ptr = ptr->next;
@@ -534,3 +575,90 @@ setInterval(void *arg, size_t interval)
 MYSQL_MONITOR   *handle = (MYSQL_MONITOR *)arg;
 	memcpy(&handle->interval, &interval, sizeof(unsigned long));
 }
+
+/**
+ * get candidate master from all nodes
+ *
+ * The current available rule: get the server with MIN(node_id)
+ * node_id comes from 'wsrep_local_index' variable
+ *
+ * @param	servers The monitored servers list
+ * @return	The candidate master on success, NULL on failure
+ */
+static MONITOR_SERVERS *get_candidate_master(MONITOR_SERVERS *servers) {
+	MONITOR_SERVERS *ptr = servers;
+	MONITOR_SERVERS *candidate_master = NULL;
+	long min_id = -1;
+	
+	/* set min_id to the lowest value of ptr->server->node_id */
+	while(ptr) {
+		if ((! SERVER_IN_MAINT(ptr->server)) && ptr->server->node_id >= 0 && SERVER_IS_JOINED(ptr->server)) {
+			ptr->server->depth = 0;
+			if ((ptr->server->node_id < min_id) && min_id >= 0) {
+				min_id = ptr->server->node_id;
+				candidate_master = ptr;
+			} else {
+				if (min_id < 0) {
+					min_id = ptr->server->node_id;
+					candidate_master = ptr;
+				}
+			}
+		}
+
+		ptr = ptr->next;
+	}
+
+	return candidate_master;
+}
+
+/**
+ * set the master server in the cluster
+ *
+ * master could be the last one from previous monitor cycle Iis running) or
+ * the candidate master.
+ * The selection is based on the configuration option mapped to master_stickiness
+ * The candidate master may change over time due to
+ * 'wsrep_local_index' value change in the Galera Cluster
+ * Enabling master_stickiness will avoid master change unless a failure is spotted
+ *
+ * @param	current_master Previous master server
+ * @param	candidate_master The candidate master server accordingly to the selection rule
+ * @return	The  master node pointer (could be NULL)
+ */
+static MONITOR_SERVERS *set_cluster_master(MONITOR_SERVERS *current_master, MONITOR_SERVERS *candidate_master, int master_stickiness) {
+	/*
+	 * if current master is not set or master_stickiness is not enable
+	 * just return candidate_master.
+	 */
+	if (current_master == NULL || master_stickiness == 0) {
+		return candidate_master;
+	} else {
+		/*
+		 * if current_master is still a cluster member use it
+		 *
+		 */
+		if (SERVER_IS_JOINED(current_master->server) && (! SERVER_IN_MAINT(current_master->server))) {
+			return current_master;
+		} else
+			return candidate_master;
+	}
+}
+
+/**
+ * Disable/Enable the Master failback in a Galera Cluster.
+ *
+ * A restarted / rejoined node may get back the previous 'wsrep_local_index'
+ * from Cluster: if the value is the lowest in the cluster it will be selected as Master
+ * This will cause a Master change even if there is no failure.
+ * The option if set to 1 will avoid this situation, keeping the current Master (if running) available
+ *
+ * @param arg           The handle allocated by startMonitor
+ * @param disable       To disable it use 1, 0 keeps failback
+ */
+static void
+disableMasterFailback(void *arg, int disable)
+{
+MYSQL_MONITOR   *handle = (MYSQL_MONITOR *)arg;
+        memcpy(&handle->disableMasterFailback, &disable, sizeof(int));
+}
+

server/modules/monitor/mm_mon.c

@@ -80,7 +80,8 @@ static MONITOR_OBJECT MyObject = {
 	NULL,
 	NULL,
 	NULL,
-	detectStaleMaster
+	detectStaleMaster,
+	NULL
 };
 
 /**

server/modules/monitor/mysql_mon.c

@@ -107,7 +107,8 @@ static MONITOR_OBJECT MyObject = {
 	NULL,
 	defaultId, 
 	replicationHeartbeat, 
-	detectStaleMaster
+	detectStaleMaster,
+	NULL
 };
 
 /**

server/modules/monitor/mysqlmon.h

@@ -65,6 +65,7 @@ typedef struct {
 	unsigned long         id;	/**< Monitor ID */
 	int	replicationHeartbeat;	/**< Monitor flag for MySQL replication heartbeat */
 	int	detectStaleMaster;	/**< Monitor flag for MySQL replication Stale Master detection */
+	int	disableMasterFailback;	/**< Monitor flag for Galera Cluster Master failback */
 	MONITOR_SERVERS *master;	/**< Master server for MySQL Master/Slave replication */
 	MONITOR_SERVERS	*databases;     /**< Linked list of servers to monitor */
 } MYSQL_MONITOR;

server/modules/monitor/ndbcluster_mon.c

@@ -74,6 +74,7 @@ static MONITOR_OBJECT MyObject = {
 	NULL, 
 	NULL, 
 	NULL,
+	NULL,
 	NULL
 };