diff --git a/server/core/config.c b/server/core/config.c index 9617104f1..b0aa25178 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -38,6 +38,7 @@ * 09/09/14 Massimiliano Pinto Added localhost_match_wildcard_host parameter * 12/09/14 Mark Riddoch Addition of checks on servers list and * internal router suppression of messages + * 30/10/14 Massimiliano Pinto Added disable_master_failback parameter * 07/11/14 Massimiliano Pinto Addition of monitor timeouts for connect/read/write * * @endverbatim @@ -774,6 +775,7 @@ int error_count = 0; unsigned long interval = 0; int replication_heartbeat = 0; int detect_stale_master = 0; + int disable_master_failback = 0; int connect_timeout = 0; int read_timeout = 0; int write_timeout = 0; @@ -794,6 +796,10 @@ int error_count = 0; detect_stale_master = atoi(config_get_value(obj->parameters, "detect_stale_master")); } + if (config_get_value(obj->parameters, "disable_master_failback")) { + disable_master_failback = atoi(config_get_value(obj->parameters, "disable_master_failback")); + } + if (config_get_value(obj->parameters, "backend_connect_timeout")) { connect_timeout = atoi(config_get_value(obj->parameters, "backend_connect_timeout")); } @@ -804,7 +810,6 @@ int error_count = 0; write_timeout = atoi(config_get_value(obj->parameters, "backend_write_timeout")); } - if (module) { obj->element = monitor_alloc(obj->object, module); @@ -832,6 +837,10 @@ int error_count = 0; if(detect_stale_master == 1) monitorDetectStaleMaster(obj->element, detect_stale_master); + /* disable master failback */ + if(disable_master_failback == 1) + monitorDisableMasterFailback(obj->element, disable_master_failback); + /* set timeouts */ if (connect_timeout > 0) monitorSetNetworkTimeout(obj->element, MONITOR_CONNECT_TIMEOUT, connect_timeout); @@ -1649,6 +1658,7 @@ static char *monitor_params[] = "monitor_interval", "detect_replication_lag", "detect_stale_master", + "disable_master_failback", "backend_connect_timeout", "backend_read_timeout", "backend_write_timeout", diff --git a/server/core/monitor.c b/server/core/monitor.c index 03d26e9ca..8089e14d3 100644 --- a/server/core/monitor.c +++ b/server/core/monitor.c @@ -26,6 +26,7 @@ * 08/07/13 Mark Riddoch Initial implementation * 23/05/14 Massimiliano Pinto Addition of monitor_interval parameter * and monitor id + * 30/10/14 Massimiliano Pinto Addition of disable_master_failback parameter * 07/11/14 Massimiliano Pinto Addition of monitor network timeouts * * @endverbatim @@ -331,6 +332,19 @@ monitorDetectStaleMaster(MONITOR *mon, int enable) } } +/** + * Disable Master Failback + * + * @param mon The monitor instance + * @param disable The value 1 disable the failback, 0 keeps it + */ +void +monitorDisableMasterFailback(MONITOR *mon, int disable) +{ + if (mon->module->disableMasterFailback != NULL) { + mon->module->disableMasterFailback(mon->handle, disable); +} + /** * Set Monitor timeouts for connect/read/write * diff --git a/server/core/server.c b/server/core/server.c index 3fabd2b34..dd0e8819d 100644 --- a/server/core/server.c +++ b/server/core/server.c @@ -30,7 +30,8 @@ * 28/05/14 Massimiliano Pinto Addition of rlagd and node_ts fields * 20/06/14 Massimiliano Pinto Addition of master_id, depth, slaves fields * 26/06/14 Mark Riddoch Addition of server parameters - * 30/08/14 Massimiliano Pinto Addition of new service status description + * 30/08/14 Massimiliano Pinto Addition of new service status description + * 30/10/14 Massimiliano Pinto Addition of SERVER_MASTER_STICKINESS description * * @endverbatim */ @@ -424,6 +425,8 @@ char *status = NULL; strcat(status, "Slave of External Server, "); if (server->status & SERVER_STALE_STATUS) strcat(status, "Stale Status, "); + if (server->status & SERVER_MASTER_STICKINESS) + strcat(status, "Master Stickiness, "); if (server->status & SERVER_AUTH_ERROR) strcat(status, "Auth Error, "); if (server->status & SERVER_RUNNING) diff --git a/server/include/monitor.h b/server/include/monitor.h index 23a16539d..c5a9d314d 100644 --- a/server/include/monitor.h +++ b/server/include/monitor.h @@ -33,6 +33,7 @@ * 23/05/14 Massimiliano Pinto Addition of defaultId and setInterval * 23/06/14 Massimiliano Pinto Addition of replicationHeartbeat * 28/08/14 Massimiliano Pinto Addition of detectStaleMaster + * 30/10/14 Massimiliano Pinto Addition of disableMasterFailback * 07/11/14 Massimiliano Pinto Addition of setNetworkTimeout * * @endverbatim @@ -75,6 +76,7 @@ typedef struct { void (*defaultId)(void *, unsigned long); void (*replicationHeartbeat)(void *, int); void (*detectStaleMaster)(void *, int); + void (*disableMasterFailback)(void *, int); } MONITOR_OBJECT; /** @@ -135,5 +137,6 @@ extern void monitorSetId(MONITOR *, unsigned long); extern void monitorSetInterval (MONITOR *, unsigned long); extern void monitorSetReplicationHeartbeat(MONITOR *, int); extern void monitorDetectStaleMaster(MONITOR *, int); +extern void monitorDisableMasterFailback(MONITOR *, int); extern void monitorSetNetworkTimeout(MONITOR *, int, int); #endif diff --git a/server/include/server.h b/server/include/server.h index b8ca445f0..1e4aa25ab 100644 --- a/server/include/server.h +++ b/server/include/server.h @@ -40,6 +40,7 @@ * 26/06/14 Mark Riddoch Adidtion of server parameters * 30/07/14 Massimiliano Pinto Addition of NDB status for MySQL Cluster * 30/08/14 Massimiliano Pinto Addition of SERVER_STALE_STATUS + * 27/10/14 Massimiliano Pinto Addition of SERVER_MASTER_STICKINESS * * @endverbatim */ @@ -105,6 +106,7 @@ typedef struct server { #define SERVER_MAINT 0x0020 /**<< Server is in maintenance mode */ #define SERVER_SLAVE_OF_EXTERNAL_MASTER 0x0040 /**<< Server is slave of a Master outside the provided replication topology */ #define SERVER_STALE_STATUS 0x0080 /**<< Server stale status, monitor didn't update it */ +#define SERVER_MASTER_STICKINESS 0x0100 /**<< Server Master stickiness */ #define SERVER_AUTH_ERROR 0x1000 /**<< Authentication erorr from monitor */ /** diff --git a/server/modules/monitor/galera_mon.c b/server/modules/monitor/galera_mon.c index 68091c724..e63bf9bfa 100644 --- a/server/modules/monitor/galera_mon.c +++ b/server/modules/monitor/galera_mon.c @@ -30,6 +30,7 @@ * Interval is printed in diagnostics. * 03/06/14 Mark Riddoch Add support for maintenance mode * 24/06/14 Massimiliano Pinto Added depth level 0 for each node + * 30/10/14 Massimiliano Pinto Added disableMasterFailback feature * * @endverbatim */ @@ -52,7 +53,7 @@ extern int lm_enabled_logfiles_bitmask; static void monitorMain(void *); -static char *version_str = "V1.2.0"; +static char *version_str = "V1.3.0"; MODULE_INFO info = { MODULE_API_MONITOR, @@ -68,6 +69,9 @@ static void unregisterServer(void *, SERVER *); static void defaultUsers(void *, char *, char *); static void diagnostics(DCB *, void *); static void setInterval(void *, size_t); +static MONITOR_SERVERS *get_candidate_master(MONITOR_SERVERS *); +static MONITOR_SERVERS *set_cluster_master(MONITOR_SERVERS *, MONITOR_SERVERS *, int); +static void disableMasterFailback(void *, int); static MONITOR_OBJECT MyObject = { startMonitor, @@ -80,7 +84,8 @@ static MONITOR_OBJECT MyObject = { NULL, NULL, NULL, - NULL + NULL, + disableMasterFailback }; /** @@ -148,6 +153,8 @@ MYSQL_MONITOR *handle; handle->defaultPasswd = NULL; handle->id = MONITOR_DEFAULT_ID; handle->interval = MONITOR_INTERVAL; + handle->disableMasterFailback = 0; + handle->master = NULL; spinlock_init(&handle->lock); } handle->tid = (THREAD)thread_start(monitorMain, handle); @@ -265,6 +272,7 @@ char *sep; } dcb_printf(dcb,"\tSampling interval:\t%lu milliseconds\n", handle->interval); + dcb_printf(dcb,"\tMaster Failback:\t%s\n", (handle->disableMasterFailback == 1) ? "off" : "on"); dcb_printf(dcb, "\tMonitored servers: "); db = handle->databases; @@ -332,8 +340,10 @@ char *server_string; char *dpwd = decryptPassword(passwd); int rc; int read_timeout = 1; + int connect_timeout = 2; database->con = mysql_init(NULL); + rc = mysql_options(database->con, MYSQL_OPT_CONNECT_TIMEOUT, (void *)&connect_timeout); rc = mysql_options(database->con, MYSQL_OPT_READ_TIMEOUT, (void *)&read_timeout); if (mysql_real_connect(database->con, database->server->name, @@ -346,7 +356,15 @@ char *server_string; database->server->name, database->server->port, mysql_error(database->con)))); + server_clear_status(database->server, SERVER_RUNNING); + + /* Also clear Joined, M/S and Stickiness bits */ + server_clear_status(database->server, SERVER_JOINED); + server_clear_status(database->server, SERVER_SLAVE); + server_clear_status(database->server, SERVER_MASTER); + server_clear_status(database->server, SERVER_MASTER_STICKINESS); + if (mysql_errno(database->con) == ER_ACCESS_DENIED_ERROR) { server_set_status(database->server, SERVER_AUTH_ERROR); @@ -422,10 +440,11 @@ char *server_string; static void monitorMain(void *arg) { -MYSQL_MONITOR *handle = (MYSQL_MONITOR *)arg; -MONITOR_SERVERS *ptr; -long master_id; -size_t nrounds = 0; +MYSQL_MONITOR *handle = (MYSQL_MONITOR *)arg; +MONITOR_SERVERS *ptr; +size_t nrounds = 0; +MONITOR_SERVERS *candidate_master = NULL; +int master_stickiness = handle->disableMasterFailback; if (mysql_thread_init()) { @@ -454,39 +473,35 @@ size_t nrounds = 0; * interval, then skip monitoring checks. Excluding the first * round. */ - if (nrounds != 0 && - ((nrounds*MON_BASE_INTERVAL_MS)%handle->interval) >= - MON_BASE_INTERVAL_MS) + + if (nrounds != 0 && ((nrounds*MON_BASE_INTERVAL_MS)%handle->interval) >= MON_BASE_INTERVAL_MS) { nrounds += 1; continue; } + nrounds += 1; - master_id = -1; ptr = handle->databases; while (ptr) { unsigned int prev_status = ptr->server->status; + monitorDatabase(ptr, handle->defaultUser, handle->defaultPasswd); - /* set master_id to the lowest value of ptr->server->node_id */ + /* clear bits for non member nodes */ + if ( ! SERVER_IN_MAINT(ptr->server) && (ptr->server->node_id < 0 || ! SERVER_IS_JOINED(ptr->server))) { + ptr->server->depth = -1; - if ((! SERVER_IN_MAINT(ptr->server)) && ptr->server->node_id >= 0 && SERVER_IS_JOINED(ptr->server)) { - ptr->server->depth = 0; - if (ptr->server->node_id < master_id && master_id >= 0) { - master_id = ptr->server->node_id; - } else { - if (master_id < 0) { - master_id = ptr->server->node_id; - } - } - } else if (!SERVER_IN_MAINT(ptr->server)) { /* clear M/S status */ server_clear_status(ptr->server, SERVER_SLAVE); server_clear_status(ptr->server, SERVER_MASTER); - ptr->server->depth = -1; + + /* clear master sticky status */ + server_clear_status(ptr->server, SERVER_MASTER_STICKINESS); } + + /* Log server status change */ if (ptr->server->status != prev_status || SERVER_IS_DOWN(ptr->server)) { @@ -497,24 +512,50 @@ size_t nrounds = 0; ptr->server->port, STRSRVSTATUS(ptr->server)))); } + ptr = ptr->next; } + /* + * Let's select a master server: + * it could be the candidate master following MIN(node_id) rule or + * the server that was master in the previous monitor polling cycle + * Decision depends on master_stickiness value set in configuration + */ + + /* get the candidate master, followinf MIN(node_id) rule */ + candidate_master = get_candidate_master(handle->databases); + + /* Select the master, based on master_stickiness */ + handle->master = set_cluster_master(handle->master, candidate_master, master_stickiness); + ptr = handle->databases; - /* this server loop sets Master and Slave roles */ - while (ptr) - { - if ((! SERVER_IN_MAINT(ptr->server)) && ptr->server->node_id >= 0 && master_id >= 0) { - /* set the Master role */ - if (SERVER_IS_JOINED(ptr->server) && (ptr->server->node_id == master_id)) { - server_set_status(ptr->server, SERVER_MASTER); - server_clear_status(ptr->server, SERVER_SLAVE); - } else if (SERVER_IS_JOINED(ptr->server) && (ptr->server->node_id > master_id)) { + while (ptr && handle->master) { + if (!SERVER_IS_JOINED(ptr->server) || SERVER_IN_MAINT(ptr->server)) { + ptr = ptr->next; + continue; + } + + if (ptr != handle->master) { /* set the Slave role */ - server_set_status(ptr->server, SERVER_SLAVE); - server_clear_status(ptr->server, SERVER_MASTER); - } + server_set_status(ptr->server, SERVER_SLAVE); + server_clear_status(ptr->server, SERVER_MASTER); + + /* clear master stickyness */ + server_clear_status(ptr->server, SERVER_MASTER_STICKINESS); + } else { + /* set the Master role */ + server_set_status(handle->master->server, SERVER_MASTER); + server_clear_status(handle->master->server, SERVER_SLAVE); + + if (candidate_master && handle->master->server->node_id != candidate_master->server->node_id) { + /* set master stickyness */ + server_set_status(handle->master->server, SERVER_MASTER_STICKINESS); + } else { + /* clear master stickyness */ + server_clear_status(ptr->server, SERVER_MASTER_STICKINESS); + } } ptr = ptr->next; @@ -534,3 +575,90 @@ setInterval(void *arg, size_t interval) MYSQL_MONITOR *handle = (MYSQL_MONITOR *)arg; memcpy(&handle->interval, &interval, sizeof(unsigned long)); } + +/** + * get candidate master from all nodes + * + * The current available rule: get the server with MIN(node_id) + * node_id comes from 'wsrep_local_index' variable + * + * @param servers The monitored servers list + * @return The candidate master on success, NULL on failure + */ +static MONITOR_SERVERS *get_candidate_master(MONITOR_SERVERS *servers) { + MONITOR_SERVERS *ptr = servers; + MONITOR_SERVERS *candidate_master = NULL; + long min_id = -1; + + /* set min_id to the lowest value of ptr->server->node_id */ + while(ptr) { + if ((! SERVER_IN_MAINT(ptr->server)) && ptr->server->node_id >= 0 && SERVER_IS_JOINED(ptr->server)) { + ptr->server->depth = 0; + if ((ptr->server->node_id < min_id) && min_id >= 0) { + min_id = ptr->server->node_id; + candidate_master = ptr; + } else { + if (min_id < 0) { + min_id = ptr->server->node_id; + candidate_master = ptr; + } + } + } + + ptr = ptr->next; + } + + return candidate_master; +} + +/** + * set the master server in the cluster + * + * master could be the last one from previous monitor cycle Iis running) or + * the candidate master. + * The selection is based on the configuration option mapped to master_stickiness + * The candidate master may change over time due to + * 'wsrep_local_index' value change in the Galera Cluster + * Enabling master_stickiness will avoid master change unless a failure is spotted + * + * @param current_master Previous master server + * @param candidate_master The candidate master server accordingly to the selection rule + * @return The master node pointer (could be NULL) + */ +static MONITOR_SERVERS *set_cluster_master(MONITOR_SERVERS *current_master, MONITOR_SERVERS *candidate_master, int master_stickiness) { + /* + * if current master is not set or master_stickiness is not enable + * just return candidate_master. + */ + if (current_master == NULL || master_stickiness == 0) { + return candidate_master; + } else { + /* + * if current_master is still a cluster member use it + * + */ + if (SERVER_IS_JOINED(current_master->server) && (! SERVER_IN_MAINT(current_master->server))) { + return current_master; + } else + return candidate_master; + } +} + +/** + * Disable/Enable the Master failback in a Galera Cluster. + * + * A restarted / rejoined node may get back the previous 'wsrep_local_index' + * from Cluster: if the value is the lowest in the cluster it will be selected as Master + * This will cause a Master change even if there is no failure. + * The option if set to 1 will avoid this situation, keeping the current Master (if running) available + * + * @param arg The handle allocated by startMonitor + * @param disable To disable it use 1, 0 keeps failback + */ +static void +disableMasterFailback(void *arg, int disable) +{ +MYSQL_MONITOR *handle = (MYSQL_MONITOR *)arg; + memcpy(&handle->disableMasterFailback, &disable, sizeof(int)); +} + diff --git a/server/modules/monitor/mm_mon.c b/server/modules/monitor/mm_mon.c index 44b5ccd05..b4f62e8b0 100644 --- a/server/modules/monitor/mm_mon.c +++ b/server/modules/monitor/mm_mon.c @@ -80,7 +80,8 @@ static MONITOR_OBJECT MyObject = { NULL, NULL, NULL, - detectStaleMaster + detectStaleMaster, + NULL }; /** diff --git a/server/modules/monitor/mysql_mon.c b/server/modules/monitor/mysql_mon.c index 3dd3fe77c..a89d46d84 100644 --- a/server/modules/monitor/mysql_mon.c +++ b/server/modules/monitor/mysql_mon.c @@ -107,7 +107,8 @@ static MONITOR_OBJECT MyObject = { NULL, defaultId, replicationHeartbeat, - detectStaleMaster + detectStaleMaster, + NULL }; /** diff --git a/server/modules/monitor/mysqlmon.h b/server/modules/monitor/mysqlmon.h index 018d695ff..54658325a 100644 --- a/server/modules/monitor/mysqlmon.h +++ b/server/modules/monitor/mysqlmon.h @@ -65,6 +65,7 @@ typedef struct { unsigned long id; /**< Monitor ID */ int replicationHeartbeat; /**< Monitor flag for MySQL replication heartbeat */ int detectStaleMaster; /**< Monitor flag for MySQL replication Stale Master detection */ + int disableMasterFailback; /**< Monitor flag for Galera Cluster Master failback */ MONITOR_SERVERS *master; /**< Master server for MySQL Master/Slave replication */ MONITOR_SERVERS *databases; /**< Linked list of servers to monitor */ } MYSQL_MONITOR; diff --git a/server/modules/monitor/ndbcluster_mon.c b/server/modules/monitor/ndbcluster_mon.c index 704ca1b5a..d273fe32f 100644 --- a/server/modules/monitor/ndbcluster_mon.c +++ b/server/modules/monitor/ndbcluster_mon.c @@ -74,6 +74,7 @@ static MONITOR_OBJECT MyObject = { NULL, NULL, NULL, + NULL, NULL };