hcq/MaxScale
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MXS-2426 Do not permanently disable automatic cluster operations when they fail

Browse Source 
Only disabled for "failcount" monitor ticks. Also turns some related log
messages to notices.
This commit is contained in:
Esa Korhonen
2019-04-05 14:30:19 +03:00
parent b9aec60d7a
commit 5ba305c2c1
2 changed files with 82 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

163
server/modules/monitor/mariadbmon/mariadbmon.cc
View File

@ -33,8 +33,6 @@
#include <maxscale/modutil.h> #include <maxscale/modutil.h>
#include <maxscale/mysql_utils.h> #include <maxscale/mysql_utils.h>
#include <maxscale/utils.h> #include <maxscale/utils.h>
// TODO: For monitorAddParameters
#include "../../../core/internal/monitor.h"
/** Column positions for SHOW SLAVE STATUS */ /** Column positions for SHOW SLAVE STATUS */
#define MYSQL55_STATUS_MASTER_LOG_POS 5 #define MYSQL55_STATUS_MASTER_LOG_POS 5
@ -95,7 +93,7 @@ static int add_slave_to_master(long *, int, long);
static bool isMySQLEvent(mxs_monitor_event_t event); static bool isMySQLEvent(mxs_monitor_event_t event);
void check_maxscale_schema_replication(MXS_MONITOR *monitor); void check_maxscale_schema_replication(MXS_MONITOR *monitor);
static MySqlServerInfo* get_server_info(const MYSQL_MONITOR* handle, const MXS_MONITORED_SERVER* db); static MySqlServerInfo* get_server_info(const MYSQL_MONITOR* handle, const MXS_MONITORED_SERVER* db);
static bool mon_process_failover(MYSQL_MONITOR*, uint32_t, bool*); static void mon_process_failover(MYSQL_MONITOR* monitor);
static bool do_failover(MYSQL_MONITOR* mon, json_t** output); static bool do_failover(MYSQL_MONITOR* mon, json_t** output);
static bool do_switchover(MYSQL_MONITOR* mon, MXS_MONITORED_SERVER* current_master, static bool do_switchover(MYSQL_MONITOR* mon, MXS_MONITORED_SERVER* current_master,
MXS_MONITORED_SERVER* new_master, json_t** err_out); MXS_MONITORED_SERVER* new_master, json_t** err_out);
@ -109,7 +107,8 @@ static bool server_is_rejoin_suspect(MYSQL_MONITOR* mon, MXS_MONITORED_SERVER* s
static bool get_joinable_servers(MYSQL_MONITOR* mon, ServerVector* output); static bool get_joinable_servers(MYSQL_MONITOR* mon, ServerVector* output);
static uint32_t do_rejoin(MYSQL_MONITOR* mon, const ServerVector& servers, json_t** output); static uint32_t do_rejoin(MYSQL_MONITOR* mon, const ServerVector& servers, json_t** output);
static bool join_cluster(MXS_MONITORED_SERVER* server, const char* change_cmd); static bool join_cluster(MXS_MONITORED_SERVER* server, const char* change_cmd);
static void disable_setting(MYSQL_MONITOR* mon, const char* setting); static void delay_auto_cluster_ops(MYSQL_MONITOR* mon);
static bool can_perform_cluster_ops(MYSQL_MONITOR* mon);
static bool cluster_can_be_joined(MYSQL_MONITOR* mon); static bool cluster_can_be_joined(MYSQL_MONITOR* mon);
static bool can_replicate_from(MYSQL_MONITOR* mon, static bool can_replicate_from(MYSQL_MONITOR* mon,
MXS_MONITORED_SERVER* slave, MySqlServerInfo* slave_info, MXS_MONITORED_SERVER* slave, MySqlServerInfo* slave_info,
@ -552,15 +551,9 @@ bool mysql_switchover(MXS_MONITOR* mon, MXS_MONITORED_SERVER* new_master,
} }
else else
{ {
string format = "Switchover %s -> %s failed"; const char format[] = "Switchover %s -> %s failed.";
bool failover = config_get_bool(mon->parameters, CN_AUTO_FAILOVER); PRINT_MXS_JSON_ERROR(error_out, format, curr_master_name, new_master_name);
if (failover) delay_auto_cluster_ops(handle);
{
disable_setting(handle, CN_AUTO_FAILOVER);
format += ", failover has been disabled.";
}
format += ".";
PRINT_MXS_JSON_ERROR(error_out, format.c_str(), curr_master_name, new_master_name);
} }
} }
@ -2206,7 +2199,8 @@ monitorMain(void *arg)
size_t nrounds = 0; size_t nrounds = 0;
int log_no_master = 1; int log_no_master = 1;
bool heartbeat_checked = false; bool heartbeat_checked = false;
handle->cluster_op_performed = false;
handle->cluster_operation_disable_timer = 0;
replication_heartbeat = handle->replicationHeartbeat; replication_heartbeat = handle->replicationHeartbeat;
detect_stale_master = handle->detectStaleMaster; detect_stale_master = handle->detectStaleMaster;
@ -2255,6 +2249,12 @@ monitorMain(void *arg)
num_servers = 0; num_servers = 0;
atomic_add_uint64(&mon->ticks, 1); atomic_add_uint64(&mon->ticks, 1);
handle->cluster_op_performed = false;
if (handle->cluster_operation_disable_timer > 0)
{
handle->cluster_operation_disable_timer--;
}
lock_monitor_servers(mon); lock_monitor_servers(mon);
servers_status_pending_to_current(mon); servers_status_pending_to_current(mon);
@ -2559,34 +2559,25 @@ monitorMain(void *arg)
* need to be launched. * need to be launched.
*/ */
mon_process_state_changes(mon, handle->script, handle->events); mon_process_state_changes(mon, handle->script, handle->events);
bool cluster_modified = false; // Has an automatic failover/rejoin been performed this loop?
if (handle->auto_failover) if (handle->auto_failover && can_perform_cluster_ops(handle))
{ {
const char RE_ENABLE_FMT[] = "%s To re-enable failover, manually set '%s' to 'true' for monitor "
"'%s' via MaxAdmin or the REST API, or restart MaxScale.";
if (failover_not_possible(handle)) if (failover_not_possible(handle))
{ {
const char PROBLEMS[] = "Failover is not possible due to one or more problems in the " const char PROBLEMS[] = "Failover is not possible due to one or more problems in the "
"replication configuration, disabling automatic failover. Failover " "replication configuration.";
"should only be enabled after the replication configuration has been " MXS_ERROR(PROBLEMS);
"fixed."; delay_auto_cluster_ops(handle);
MXS_ERROR(RE_ENABLE_FMT, PROBLEMS, CN_AUTO_FAILOVER, mon->name);
handle->auto_failover = false;
disable_setting(handle, CN_AUTO_FAILOVER);
} }
// If master seems to be down, check if slaves are receiving events. // If master seems to be down, check if slaves are receiving events.
else if (handle->verify_master_failure && handle->master && else if (handle->verify_master_failure && handle->master &&
SERVER_IS_DOWN(handle->master->server) && slave_receiving_events(handle)) SERVER_IS_DOWN(handle->master->server) && slave_receiving_events(handle))
{ {
MXS_INFO("Master failure not yet confirmed by slaves, delaying failover."); MXS_NOTICE("A slave is still receiving data from master, delaying failover.");
} }
else if (!mon_process_failover(handle, handle->failover_timeout, &cluster_modified)) else
{ {
const char FAILED[] = "Failed to perform failover, disabling automatic failover."; mon_process_failover(handle);
MXS_ERROR(RE_ENABLE_FMT, FAILED, CN_AUTO_FAILOVER, mon->name);
handle->auto_failover = false;
disable_setting(handle, CN_AUTO_FAILOVER);
} }
} }
@ -2651,10 +2642,9 @@ monitorMain(void *arg)
// Do not auto-join servers on this monitor loop if a failover (or any other cluster modification) // Do not auto-join servers on this monitor loop if a failover (or any other cluster modification)
// has been performed, as server states have not been updated yet. It will happen next iteration. // has been performed, as server states have not been updated yet. It will happen next iteration.
if (!config_get_global_options()->passive && handle->auto_rejoin && if (handle->auto_rejoin && cluster_can_be_joined(handle) && can_perform_cluster_ops(handle))
!cluster_modified && cluster_can_be_joined(handle))
{ {
// Check if any servers should be autojoined to the cluster // Check if any servers should be autojoined to the cluster.
ServerVector joinable_servers; ServerVector joinable_servers;
if (get_joinable_servers(handle, &joinable_servers)) if (get_joinable_servers(handle, &joinable_servers))
{ {
@ -2662,7 +2652,7 @@ monitorMain(void *arg)
if (joins > 0) if (joins > 0)
{ {
MXS_NOTICE("%d server(s) redirected or rejoined the cluster.", joins); MXS_NOTICE("%d server(s) redirected or rejoined the cluster.", joins);
cluster_modified = true; handle->cluster_op_performed = true;
} }
} }
else else
@ -2675,7 +2665,7 @@ monitorMain(void *arg)
/* Check if any slave servers have read-only off and turn it on if user so wishes. Again, do not /* Check if any slave servers have read-only off and turn it on if user so wishes. Again, do not
* perform this if cluster has been modified this loop since it may not be clear which server * perform this if cluster has been modified this loop since it may not be clear which server
* should be a slave. */ * should be a slave. */
if (!config_get_global_options()->passive && handle->enforce_read_only_slaves && !cluster_modified) if (handle->enforce_read_only_slaves && can_perform_cluster_ops(handle))
{ {
enforce_read_only_on_slaves(handle); enforce_read_only_on_slaves(handle);
} }
@ -3384,30 +3374,19 @@ void check_maxscale_schema_replication(MXS_MONITOR *monitor)
} }
/** /**
* @brief Process possible failover event * If a master has failed, performs failover. This function only works with flat replication topologies
* Should be called immediately after @c mon_process_state_changes.
* *
* If a master failure has occurred and MaxScale is configured with failover * @param monitor Monitor whose cluster is processed
* functionality, this fuction executes an external failover program to elect
* a new master server.
*
* This function should be called immediately after @c mon_process_state_changes.
*
* @param monitor Monitor whose cluster is processed
* @param failover_timeout Timeout in seconds for the failover
* @param cluster_modified_out Set to true if modifying cluster
* @return True on success, false on error
*
* @todo Currently this only works with flat replication topologies and
* needs to be moved inside mysqlmon as it is MariaDB specific code.
*/ */
bool mon_process_failover(MYSQL_MONITOR* monitor, uint32_t failover_timeout, bool* cluster_modified_out) void mon_process_failover(MYSQL_MONITOR* monitor)
{ {
ss_dassert(*cluster_modified_out == false); ss_dassert(monitor->cluster_op_performed == false);
if (config_get_global_options()->passive || if (monitor->master && SERVER_IS_MASTER(monitor->master->server))
(monitor->master && SERVER_IS_MASTER(monitor->master->server)))
{ {
return true; return;
} }
int failover_timeout = monitor->failover_timeout;
MXS_MONITORED_SERVER* failed_master = NULL; MXS_MONITORED_SERVER* failed_master = NULL;
for (MXS_MONITORED_SERVER *ptr = monitor->monitor->monitored_servers; ptr; ptr = ptr->next) for (MXS_MONITORED_SERVER *ptr = monitor->monitor->monitored_servers; ptr; ptr = ptr->next)
@ -3416,12 +3395,11 @@ bool mon_process_failover(MYSQL_MONITOR* monitor, uint32_t failover_timeout, boo
{ {
if (failed_master) if (failed_master)
{ {
MXS_ALERT("Multiple failed master servers detected: " MXS_ERROR("Multiple failed master servers detected: '%s' is the first master to fail "
"'%s' is the first master to fail but server " "but server '%s' has also triggered a master_down event.",
"'%s' has also triggered a master_down event.", failed_master->server->unique_name, ptr->server->unique_name);
failed_master->server->unique_name, delay_auto_cluster_ops(monitor);
ptr->server->unique_name); return;
return false;
} }
if (ptr->server->active_event) if (ptr->server->active_event)
@ -3442,7 +3420,7 @@ bool mon_process_failover(MYSQL_MONITOR* monitor, uint32_t failover_timeout, boo
if (t > timeout) if (t > timeout)
{ {
MXS_WARNING("Failover of server '%s' did not take place within " MXS_WARNING("Failover of server '%s' did not take place within "
"%u seconds, failover needs to be re-triggered", "%i seconds, failover needs to be re-triggered",
ptr->server->unique_name, failover_timeout); ptr->server->unique_name, failover_timeout);
failed_master = ptr; failed_master = ptr;
} }
@ -3450,7 +3428,6 @@ bool mon_process_failover(MYSQL_MONITOR* monitor, uint32_t failover_timeout, boo
} }
} }
bool rval = true;
if (failed_master) if (failed_master)
{ {
int failcount = monitor->failcount; int failcount = monitor->failcount;
@ -3468,12 +3445,15 @@ bool mon_process_failover(MYSQL_MONITOR* monitor, uint32_t failover_timeout, boo
monitor->warn_failover_precond = true; monitor->warn_failover_precond = true;
MXS_NOTICE("Performing automatic failover to replace failed master '%s'.", MXS_NOTICE("Performing automatic failover to replace failed master '%s'.",
failed_master->server->unique_name); failed_master->server->unique_name);
failed_master->new_event = false; if (do_failover(monitor, NULL))
// If this fails, auto_failover is disabled.
rval = do_failover(monitor, NULL);
if (rval)
{ {
*cluster_modified_out = true; monitor->cluster_op_performed = true;
failed_master->new_event = false;
}
else
{
MXS_ERROR("Automatic failover failed.");
delay_auto_cluster_ops(monitor);
} }
} }
else else
@ -3493,8 +3473,6 @@ bool mon_process_failover(MYSQL_MONITOR* monitor, uint32_t failover_timeout, boo
{ {
monitor->warn_failover_precond = true; monitor->warn_failover_precond = true;
} }
return rval;
} }
/** /**
@ -3755,8 +3733,8 @@ bool failover_wait_relay_log(MYSQL_MONITOR* mon, MXS_MONITORED_SERVER* new_maste
io_pos_stable && io_pos_stable &&
difftime(time(NULL), begin) < seconds_remaining) difftime(time(NULL), begin) < seconds_remaining)
{ {
MXS_INFO("Relay log of server '%s' not yet empty, waiting to clear %" PRId64 " events.", MXS_NOTICE("Relay log of server '%s' not yet empty, waiting to clear %" PRId64 " events.",
new_master->server->unique_name, master_info->relay_log_events()); new_master->server->unique_name, master_info->relay_log_events());
thread_millisleep(1000); // Sleep for a while before querying server again. thread_millisleep(1000); // Sleep for a while before querying server again.
// Todo: check server version before entering failover. // Todo: check server version before entering failover.
Gtid old_gtid_io_pos = master_info->slave_status.gtid_io_pos; Gtid old_gtid_io_pos = master_info->slave_status.gtid_io_pos;
@ -4916,6 +4894,7 @@ static uint32_t do_rejoin(MYSQL_MONITOR* mon, const ServerVector& joinable_serve
{ {
SERVER* master = mon->master->server; SERVER* master = mon->master->server;
uint32_t servers_joined = 0; uint32_t servers_joined = 0;
bool rejoin_error = false;
if (!joinable_servers.empty()) if (!joinable_servers.empty())
{ {
string change_cmd = generate_change_master_cmd(mon, master->name, master->port); string change_cmd = generate_change_master_cmd(mon, master->name, master->port);
@ -4947,6 +4926,10 @@ static uint32_t do_rejoin(MYSQL_MONITOR* mon, const ServerVector& joinable_serve
MXS_NOTICE("Server '%s' is replicating from a server other than '%s', " MXS_NOTICE("Server '%s' is replicating from a server other than '%s', "
"redirecting it to '%s'.", name, master_name, master_name); "redirecting it to '%s'.", name, master_name, master_name);
op_success = redirect_one_slave(joinable, change_cmd.c_str()); op_success = redirect_one_slave(joinable, change_cmd.c_str());
if (!op_success)
{
rejoin_error = true;
}
} }
if (op_success) if (op_success)
@ -4955,6 +4938,11 @@ static uint32_t do_rejoin(MYSQL_MONITOR* mon, const ServerVector& joinable_serve
} }
} }
} }
if (rejoin_error)
{
delay_auto_cluster_ops(mon);
}
return servers_joined; return servers_joined;
} }
@ -4993,19 +4981,21 @@ static bool join_cluster(MXS_MONITORED_SERVER* server, const char* change_cmd)
return success; return success;
} }
/** static void delay_auto_cluster_ops(MYSQL_MONITOR* mon)
* Set a monitor config parameter to "false". The effect persists over stopMonitor/startMonitor but not
* MaxScale restart. Only use on boolean config settings.
*
* @param mon Cluster monitor
* @param setting_name Setting to disable
*/
static void disable_setting(MYSQL_MONITOR* mon, const char* setting)
{ {
MXS_CONFIG_PARAMETER p = {}; if (mon->auto_failover || mon->auto_rejoin || mon->enforce_read_only_slaves)
p.name = const_cast<char*>(setting); {
p.value = const_cast<char*>("false"); const char DISABLING_AUTO_OPS[] = "Disabling automatic cluster operations for %i monitor ticks.";
monitorAddParameters(mon->monitor, &p); MXS_NOTICE(DISABLING_AUTO_OPS, mon->failcount);
}
// + 1 is because the start of next tick substracts 1.
mon->cluster_operation_disable_timer = mon->failcount + 1;
}
static bool can_perform_cluster_ops(MYSQL_MONITOR* mon)
{
return (!config_get_global_options()->passive && mon->cluster_operation_disable_timer <= 0 &&
!mon->cluster_op_performed);
} }
/** /**
@ -5132,6 +5122,7 @@ static bool check_sql_files(MYSQL_MONITOR* mon)
static void enforce_read_only_on_slaves(MYSQL_MONITOR* mon) static void enforce_read_only_on_slaves(MYSQL_MONITOR* mon)
{ {
const char QUERY[] = "SET GLOBAL read_only=1;"; const char QUERY[] = "SET GLOBAL read_only=1;";
bool error = false;
for (MXS_MONITORED_SERVER* mon_srv = mon->monitor->monitored_servers; mon_srv; mon_srv = mon_srv->next) for (MXS_MONITORED_SERVER* mon_srv = mon->monitor->monitored_servers; mon_srv; mon_srv = mon_srv->next)
{ {
MySqlServerInfo* serv_info = get_server_info(mon, mon_srv); MySqlServerInfo* serv_info = get_server_info(mon, mon_srv);
@ -5145,7 +5136,13 @@ static void enforce_read_only_on_slaves(MYSQL_MONITOR* mon)
else else
{ {
MXS_ERROR("Setting read_only on server '%s' failed: '%s.", name, mysql_error(mon_srv->con)); MXS_ERROR("Setting read_only on server '%s' failed: '%s.", name, mysql_error(mon_srv->con));
error = true;
} }
} }
} }
if (error)
{
delay_auto_cluster_ops(mon);
}
} }

2
server/modules/monitor/mysqlmon.h
View File

@ -82,6 +82,8 @@ typedef struct
MXS_MONITORED_SERVER** excluded_servers; /**< Servers banned for master promotion during auto-failover. */ MXS_MONITORED_SERVER** excluded_servers; /**< Servers banned for master promotion during auto-failover. */
const char* promote_sql_file; /**< File with sql commands which are ran to a server being promoted. */ const char* promote_sql_file; /**< File with sql commands which are ran to a server being promoted. */
const char* demote_sql_file; /**< File with sql commands which are ran to a server being demoted. */ const char* demote_sql_file; /**< File with sql commands which are ran to a server being demoted. */
int cluster_operation_disable_timer; /**< Counter for temporary automatic cluster operation disabling. */
bool cluster_op_performed; /**< Has an automatic failover/rejoin been performed this loop? */
MXS_MONITOR* monitor; MXS_MONITOR* monitor;
} MYSQL_MONITOR; } MYSQL_MONITOR;