hcq/MaxScale
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
0215ad32a16e73c9132366a138d91c9946270f90
MaxScale/server/core/monitor.c
Esa Korhonen 7e9db7ed0c Have server status updates applied during monitor loop 
Previously, server status changes from MaxAdmin would be set immediately
as long as the server lock could be acquired. This meant that it might take
several seconds until the next monitor pass is executed. Usually, this was
fine but in some situations we would want the monitor to run immediately
after the change (MXS-740 and Galera). This patch changes the logic of
setting and clearing status bits to a delayed mode: changes are first applied
to a "status_pending"-variable, and only once the monitor runs will the
setting be applied. To reduce the delay, the monitor now has a flag
which is checked during sleep (between short 0.1s naps). If set, the
sleep is cut short.

If a server is not monitored, the status bits are set directly.

There is a small possibility of a race condition: If a monitor is stopped or
destroyed before the pending change is applied, the change is forgotten.
2016-12-19 10:19:23 +02:00

1500 lines
40 KiB
C
Raw Blame History

/*
* Copyright (c) 2016 MariaDB Corporation Ab
*
* Use of this software is governed by the Business Source License included
* in the LICENSE.TXT file and at www.mariadb.com/bsl.
*
* Change Date: 2019-07-01
*
* On the date above, in accordance with the Business Source License, use
* of this software will be governed by version 2 or later of the General
* Public License.
*/
/**
* @file monitor.c - The monitor module management routines
*
* @verbatim
* Revision History
*
* Date Who Description
* 08/07/13 Mark Riddoch Initial implementation
* 23/05/14 Massimiliano Pinto Addition of monitor_interval parameter
* and monitor id
* 30/10/14 Massimiliano Pinto Addition of disable_master_failback parameter
* 07/11/14 Massimiliano Pinto Addition of monitor network timeouts
* 08/05/15 Markus Makela Moved common monitor variables to MONITOR struct
*
* @endverbatim
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <maxscale/monitor.h>
#include <maxscale/spinlock.h>
#include <maxscale/modules.h>
#include <maxscale/log_manager.h>
#include <maxscale/secrets.h>
#include <maxscale/pcre2.h>
#include <maxscale/externcmd.h>
#include <mysqld_error.h>
#include <maxscale/mysql_utils.h>
#include <maxscale/alloc.h>
#include <maxscale/gwdirs.h>
/*
* Create declarations of the enum for monitor events and also the array of
* structs containing the matching names. The data is taken from def_monitor_event.h
*
*/
#undef ADDITEM
#define ADDITEM( _event_type, _event_name ) { #_event_name }
const monitor_def_t monitor_event_definitions[MAX_MONITOR_EVENT] =
{
#include <maxscale/def_monitor_event.h>
};
#undef ADDITEM
static MONITOR *allMonitors = NULL;
static SPINLOCK monLock = SPINLOCK_INIT;
static void monitor_server_free_all(MONITOR_SERVERS *servers);
/**
* Allocate a new monitor, load the associated module for the monitor
* and start execution on the monitor.
*
* @param name The name of the monitor module to load
* @param module The module to load
* @return The newly created monitor
*/
MONITOR *
monitor_alloc(char *name, char *module)
{
name = MXS_STRDUP(name);
char *my_module = MXS_STRDUP(module);
MONITOR *mon = (MONITOR *)MXS_MALLOC(sizeof(MONITOR));
if (!name || !mon || !my_module)
{
MXS_FREE(name);
MXS_FREE(mon);
MXS_FREE(my_module);
return NULL;
}
if ((mon->module = load_module(module, MODULE_MONITOR)) == NULL)
{
MXS_ERROR("Unable to load monitor module '%s'.", name);
MXS_FREE(name);
MXS_FREE(mon);
return NULL;
}
mon->state = MONITOR_STATE_ALLOC;
mon->name = name;
mon->module_name = my_module;
mon->handle = NULL;
mon->databases = NULL;
*mon->password = '\0';
*mon->user = '\0';
mon->read_timeout = DEFAULT_READ_TIMEOUT;
mon->write_timeout = DEFAULT_WRITE_TIMEOUT;
mon->connect_timeout = DEFAULT_CONNECT_TIMEOUT;
mon->interval = MONITOR_INTERVAL;
mon->parameters = NULL;
mon->created_online = false;
mon->server_pending_changes = false;
spinlock_init(&mon->lock);
spinlock_acquire(&monLock);
mon->next = allMonitors;
allMonitors = mon;
spinlock_release(&monLock);
return mon;
}
/**
* Free a monitor, first stop the monitor and then remove the monitor from
* the chain of monitors and free the memory.
*
* @param mon The monitor to free
*/
void
monitor_free(MONITOR *mon)
{
MONITOR *ptr;
mon->module->stopMonitor(mon);
mon->state = MONITOR_STATE_FREED;
spinlock_acquire(&monLock);
if (allMonitors == mon)
{
allMonitors = mon->next;
}
else
{
ptr = allMonitors;
while (ptr->next && ptr->next != mon)
{
ptr = ptr->next;
}
if (ptr->next)
{
ptr->next = mon->next;
}
}
spinlock_release(&monLock);
config_parameter_free(mon->parameters);
monitor_server_free_all(mon->databases);
MXS_FREE(mon->name);
MXS_FREE(mon->module_name);
MXS_FREE(mon);
}
/**
* Start an individual monitor that has previously been stopped.
*
* @param monitor The Monitor that should be started
*/
void
monitorStart(MONITOR *monitor, void* params)
{
spinlock_acquire(&monitor->lock);
if ((monitor->handle = (*monitor->module->startMonitor)(monitor, params)))
{
monitor->state = MONITOR_STATE_RUNNING;
}
else
{
MXS_ERROR("Failed to start monitor '%s'.", monitor->name);
}
spinlock_release(&monitor->lock);
}
/**
* Start all monitors
*/
void monitorStartAll()
{
MONITOR *ptr;
spinlock_acquire(&monLock);
ptr = allMonitors;
while (ptr)
{
monitorStart(ptr, ptr->parameters);
ptr = ptr->next;
}
spinlock_release(&monLock);
}
/**
* Stop a given monitor
*
* @param monitor The monitor to stop
*/
void
monitorStop(MONITOR *monitor)
{
spinlock_acquire(&monitor->lock);
/** Only stop the monitor if it is running */
if (monitor->state == MONITOR_STATE_RUNNING)
{
monitor->state = MONITOR_STATE_STOPPING;
monitor->module->stopMonitor(monitor);
monitor->state = MONITOR_STATE_STOPPED;
MONITOR_SERVERS* db = monitor->databases;
while (db)
{
// TODO: Create a generic entry point for this or move it inside stopMonitor
mysql_close(db->con);
db->con = NULL;
db = db->next;
}
}
spinlock_release(&monitor->lock);
}
/**
* Shutdown all running monitors
*/
void
monitorStopAll()
{
MONITOR *ptr;
spinlock_acquire(&monLock);
ptr = allMonitors;
while (ptr)
{
monitorStop(ptr);
ptr = ptr->next;
}
spinlock_release(&monLock);
}
/**
* Add a server to a monitor. Simply register the server that needs to be
* monitored to the running monitor module.
*
* @param mon The Monitor instance
* @param server The Server to add to the monitoring
*/
bool monitorAddServer(MONITOR *mon, SERVER *server)
{
bool rval = false;
if (monitor_server_in_use(server))
{
MXS_ERROR("Server '%s' is already monitored.", server->unique_name);
}
else
{
rval = true;
MONITOR_SERVERS *db = (MONITOR_SERVERS *)MXS_MALLOC(sizeof(MONITOR_SERVERS));
MXS_ABORT_IF_NULL(db);
db->server = server;
db->con = NULL;
db->next = NULL;
db->mon_err_count = 0;
db->log_version_err = true;
/** Server status is uninitialized */
db->mon_prev_status = -1;
/* pending status is updated by get_replication_tree */
db->pending_status = 0;
monitor_state_t old_state = mon->state;
if (old_state == MONITOR_STATE_RUNNING)
{
monitorStop(mon);
}
spinlock_acquire(&mon->lock);
if (mon->databases == NULL)
{
mon->databases = db;
}
else
{
MONITOR_SERVERS *ptr = mon->databases;
while (ptr->next != NULL)
{
ptr = ptr->next;
}
ptr->next = db;
}
spinlock_release(&mon->lock);
if (old_state == MONITOR_STATE_RUNNING)
{
monitorStart(mon, mon->parameters);
}
}
return rval;
}
static void monitor_server_free(MONITOR_SERVERS *tofree)
{
if (tofree)
{
if (tofree->con)
{
mysql_close(tofree->con);
}
MXS_FREE(tofree);
}
}
/**
* Free monitor server list
* @param servers Servers to free
*/
static void monitor_server_free_all(MONITOR_SERVERS *servers)
{
while (servers)
{
MONITOR_SERVERS *tofree = servers;
servers = servers->next;
monitor_server_free(tofree);
}
}
/**
* Remove a server from a monitor.
*
* @param mon The Monitor instance
* @param server The Server to remove
*/
void monitorRemoveServer(MONITOR *mon, SERVER *server)
{
monitor_state_t old_state = mon->state;
if (old_state == MONITOR_STATE_RUNNING)
{
monitorStop(mon);
}
spinlock_acquire(&mon->lock);
MONITOR_SERVERS *ptr = mon->databases;
if (ptr && ptr->server == server)
{
mon->databases = mon->databases->next;
}
else
{
MONITOR_SERVERS *prev = ptr;
while (ptr)
{
if (ptr->server == server)
{
prev->next = ptr->next;
break;
}
prev = ptr;
ptr = ptr->next;
}
}
spinlock_release(&mon->lock);
if (ptr)
{
monitor_server_free(ptr);
}
if (old_state == MONITOR_STATE_RUNNING)
{
monitorStart(mon, mon->parameters);
}
}
/**
* Add a default user to the monitor. This user is used to connect to the
* monitored databases but may be overriden on a per server basis.
*
* @param mon The monitor instance
* @param user The default username to use when connecting
* @param passwd The default password associated to the default user.
*/
void
monitorAddUser(MONITOR *mon, char *user, char *passwd)
{
if (user != mon->user)
{
snprintf(mon->user, sizeof(mon->user), "%s", user);
}
if (passwd != mon->password)
{
snprintf(mon->password, sizeof(mon->password), "%s", passwd);
}
}
/**
* Show all monitors
*
* @param dcb DCB for printing output
*/
void
monitorShowAll(DCB *dcb)
{
MONITOR *ptr;
spinlock_acquire(&monLock);
ptr = allMonitors;
while (ptr)
{
monitorShow(dcb, ptr);
ptr = ptr->next;
}
spinlock_release(&monLock);
}
/**
* Show a single monitor
*
* @param dcb DCB for printing output
*/
void
monitorShow(DCB *dcb, MONITOR *monitor)
{
const char *state;
switch (monitor->state)
{
case MONITOR_STATE_RUNNING:
state = "Running";
break;
case MONITOR_STATE_STOPPING:
state = "Stopping";
break;
case MONITOR_STATE_STOPPED:
state = "Stopped";
break;
case MONITOR_STATE_ALLOC:
state = "Allocated";
break;
default:
state = "Unknown";
break;
}
dcb_printf(dcb, "Monitor: %p\n", monitor);
dcb_printf(dcb, "Name: %s\n", monitor->name);
dcb_printf(dcb, "State: %s\n", state);
dcb_printf(dcb, "Sampling interval: %lu milliseconds\n", monitor->interval);
dcb_printf(dcb, "Connect Timeout: %i seconds\n", monitor->connect_timeout);
dcb_printf(dcb, "Read Timeout: %i seconds\n", monitor->read_timeout);
dcb_printf(dcb, "Write Timeout: %i seconds\n", monitor->write_timeout);
dcb_printf(dcb, "Monitored servers: ");
const char *sep = "";
for (MONITOR_SERVERS *db = monitor->databases; db; db = db->next)
{
dcb_printf(dcb, "%s%s:%d", sep, db->server->name, db->server->port);
sep = ", ";
}
dcb_printf(dcb, "\n");
if (monitor->handle)
{
if (monitor->module->diagnostics)
{
monitor->module->diagnostics(dcb, monitor);
}
else
{
dcb_printf(dcb, "\t(no diagnostics)\n");
}
}
else
{
dcb_printf(dcb, "\tMonitor failed\n");
}
dcb_printf(dcb, "\n");
}
/**
* List all the monitors
*
* @param dcb DCB for printing output
*/
void
monitorList(DCB *dcb)
{
MONITOR *ptr;
spinlock_acquire(&monLock);
ptr = allMonitors;
dcb_printf(dcb, "---------------------+---------------------\n");
dcb_printf(dcb, "%-20s | Status\n", "Monitor");
dcb_printf(dcb, "---------------------+---------------------\n");
while (ptr)
{
dcb_printf(dcb, "%-20s | %s\n", ptr->name,
ptr->state & MONITOR_STATE_RUNNING
? "Running" : "Stopped");
ptr = ptr->next;
}
dcb_printf(dcb, "---------------------+---------------------\n");
spinlock_release(&monLock);
}
/**
* Find a monitor by name
*
* @param name The name of the monitor
* @return Pointer to the monitor or NULL
*/
MONITOR *
monitor_find(const char *name)
{
MONITOR *ptr;
spinlock_acquire(&monLock);
ptr = allMonitors;
while (ptr)
{
if (!strcmp(ptr->name, name))
{
break;
}
ptr = ptr->next;
}
spinlock_release(&monLock);
return ptr;
}
/**
* Set the monitor sampling interval.
*
* @param mon The monitor instance
* @param interval The sampling interval in milliseconds
*/
void
monitorSetInterval(MONITOR *mon, unsigned long interval)
{
mon->interval = interval;
}
/**
* Set Monitor timeouts for connect/read/write
*
* @param mon The monitor instance
* @param type The timeout handling type
* @param value The timeout to set
*/
bool
monitorSetNetworkTimeout(MONITOR *mon, int type, int value)
{
bool rval = true;
if (value > 0)
{
switch (type)
{
case MONITOR_CONNECT_TIMEOUT:
mon->connect_timeout = value;
break;
case MONITOR_READ_TIMEOUT:
mon->read_timeout = value;
break;
case MONITOR_WRITE_TIMEOUT:
mon->write_timeout = value;
break;
default:
MXS_ERROR("Monitor setNetworkTimeout received an unsupported action type %i", type);
rval = false;
break;
}
}
else
{
MXS_ERROR("Negative value for monitor timeout.");
rval = false;
}
return rval;
}
/**
* Provide a row to the result set that defines the set of monitors
*
* @param set The result set
* @param data The index of the row to send
* @return The next row or NULL
*/
static RESULT_ROW *
monitorRowCallback(RESULTSET *set, void *data)
{
int *rowno = (int *)data;
int i = 0;;
char buf[20];
RESULT_ROW *row;
MONITOR *ptr;
spinlock_acquire(&monLock);
ptr = allMonitors;
while (i < *rowno && ptr)
{
i++;
ptr = ptr->next;
}
if (ptr == NULL)
{
spinlock_release(&monLock);
MXS_FREE(data);
return NULL;
}
(*rowno)++;
row = resultset_make_row(set);
resultset_row_set(row, 0, ptr->name);
resultset_row_set(row, 1, ptr->state & MONITOR_STATE_RUNNING
? "Running" : "Stopped");
spinlock_release(&monLock);
return row;
}
/**
* Return a resultset that has the current set of monitors in it
*
* @return A Result set
*/
RESULTSET *
monitorGetList()
{
RESULTSET *set;
int *data;
if ((data = (int *)MXS_MALLOC(sizeof(int))) == NULL)
{
return NULL;
}
*data = 0;
if ((set = resultset_create(monitorRowCallback, data)) == NULL)
{
MXS_FREE(data);
return NULL;
}
resultset_add_column(set, "Monitor", 20, COL_TYPE_VARCHAR);
resultset_add_column(set, "Status", 10, COL_TYPE_VARCHAR);
return set;
}
/**
* @brief Check if the monitor user has all required permissions to operate properly.
*
* @param service Monitor to inspect
* @param query Query to execute
* @return True on success, false if monitor credentials lack permissions
*/
bool check_monitor_permissions(MONITOR* monitor, const char* query)
{
if (monitor->databases == NULL || // No servers to check
config_get_global_options()->skip_permission_checks)
{
return true;
}
char *user = monitor->user;
char *dpasswd = decryptPassword(monitor->password);
GATEWAY_CONF* cnf = config_get_global_options();
bool rval = false;
for (MONITOR_SERVERS *mondb = monitor->databases; mondb; mondb = mondb->next)
{
if (mon_connect_to_db(monitor, mondb) != MONITOR_CONN_OK)
{
MXS_ERROR("[%s] Failed to connect to server '%s' (%s:%d) when"
" checking monitor user credentials and permissions: %s",
monitor->name, mondb->server->unique_name, mondb->server->name,
mondb->server->port, mysql_error(mondb->con));
switch (mysql_errno(mondb->con))
{
case ER_ACCESS_DENIED_ERROR:
case ER_DBACCESS_DENIED_ERROR:
case ER_ACCESS_DENIED_NO_PASSWORD_ERROR:
break;
default:
rval = true;
break;
}
}
else if (mysql_query(mondb->con, query) != 0)
{
switch (mysql_errno(mondb->con))
{
case ER_TABLEACCESS_DENIED_ERROR:
case ER_COLUMNACCESS_DENIED_ERROR:
case ER_SPECIFIC_ACCESS_DENIED_ERROR:
case ER_PROCACCESS_DENIED_ERROR:
case ER_KILL_DENIED_ERROR:
rval = false;
break;
default:
rval = true;
break;
}
MXS_ERROR("[%s] Failed to execute query '%s' with user '%s'. MySQL error message: %s",
monitor->name, query, user, mysql_error(mondb->con));
}
else
{
rval = true;
MYSQL_RES *res = mysql_use_result(mondb->con);
if (res == NULL)
{
MXS_ERROR("[%s] Result retrieval failed when checking monitor permissions: %s",
monitor->name, mysql_error(mondb->con));
}
else
{
mysql_free_result(res);
}
}
}
MXS_FREE(dpasswd);
return rval;
}
/**
* Add parameters to the monitor
* @param monitor Monitor
* @param params Config parameters
*/
void monitorAddParameters(MONITOR *monitor, CONFIG_PARAMETER *params)
{
while (params)
{
CONFIG_PARAMETER* clone = config_clone_param(params);
if (clone)
{
clone->next = monitor->parameters;
monitor->parameters = clone;
}
params = params->next;
}
}
bool monitorRemoveParameter(MONITOR *monitor, const char *key)
{
CONFIG_PARAMETER *prev = NULL;
for (CONFIG_PARAMETER *p = monitor->parameters; p; p = p->next)
{
if (strcmp(p->name, key) == 0)
{
if (p == monitor->parameters)
{
monitor->parameters = monitor->parameters->next;
p->next = NULL;
}
else
{
prev->next = p->next;
p->next = NULL;
}
config_parameter_free(p);
return true;
}
prev = p;
}
return false;
}
/**
* Set a pending status bit in the monitor server
*
* @param server The server to update
* @param bit The bit to clear for the server
*/
void
monitor_set_pending_status(MONITOR_SERVERS *ptr, int bit)
{
ptr->pending_status |= bit;
}
/**
* Clear a pending status bit in the monitor server
*
* @param server The server to update
* @param bit The bit to clear for the server
*/
void
monitor_clear_pending_status(MONITOR_SERVERS *ptr, int bit)
{
ptr->pending_status &= ~bit;
}
/*
* Determine a monitor event, defined by the difference between the old
* status of a server and the new status.
*
* @param node The monitor server data for a particular server
* @result monitor_event_t A monitor event (enum)
*/
monitor_event_t
mon_get_event_type(MONITOR_SERVERS* node)
{
typedef enum
{
DOWN_EVENT,
UP_EVENT,
LOSS_EVENT,
NEW_EVENT,
UNSUPPORTED_EVENT
} general_event_type;
general_event_type event_type = UNSUPPORTED_EVENT;
unsigned int prev = node->mon_prev_status
& (SERVER_RUNNING | SERVER_MASTER | SERVER_SLAVE | SERVER_JOINED | SERVER_NDB);
unsigned int present = node->server->status
& (SERVER_RUNNING | SERVER_MASTER | SERVER_SLAVE | SERVER_JOINED | SERVER_NDB);
if (prev == present)
{
/* No change in the bits we're interested in */
return UNDEFINED_MONITOR_EVENT;
}
if ((prev & SERVER_RUNNING) == 0)
{
/* The server was not running previously */
if ((present & SERVER_RUNNING) != 0)
{
event_type = UP_EVENT;
}
/* Otherwise, was not running and still is not running */
/* - this is not a recognised event */
}
else
{
/* Previous state must have been running */
if ((present & SERVER_RUNNING) == 0)
{
event_type = DOWN_EVENT;
}
else
{
/** These are used to detect whether we actually lost something or
* just transitioned from one state to another */
unsigned int prev_bits = prev & (SERVER_MASTER | SERVER_SLAVE);
unsigned int present_bits = present & (SERVER_MASTER | SERVER_SLAVE);
/* Was running and still is */
if ((!prev_bits || !present_bits || prev_bits == present_bits) &&
prev & (SERVER_MASTER | SERVER_SLAVE | SERVER_JOINED | SERVER_NDB))
{
/* We used to know what kind of server it was */
event_type = LOSS_EVENT;
}
else
{
/* We didn't know what kind of server it was, now we do*/
event_type = NEW_EVENT;
}
}
}
switch (event_type)
{
case UP_EVENT:
return (present & SERVER_MASTER) ? MASTER_UP_EVENT :
(present & SERVER_SLAVE) ? SLAVE_UP_EVENT :
(present & SERVER_JOINED) ? SYNCED_UP_EVENT :
(present & SERVER_NDB) ? NDB_UP_EVENT :
SERVER_UP_EVENT;
case DOWN_EVENT:
return (prev & SERVER_MASTER) ? MASTER_DOWN_EVENT :
(prev & SERVER_SLAVE) ? SLAVE_DOWN_EVENT :
(prev & SERVER_JOINED) ? SYNCED_DOWN_EVENT :
(prev & SERVER_NDB) ? NDB_DOWN_EVENT :
SERVER_DOWN_EVENT;
case LOSS_EVENT:
return (prev & SERVER_MASTER) ? LOST_MASTER_EVENT :
(prev & SERVER_SLAVE) ? LOST_SLAVE_EVENT :
(prev & SERVER_JOINED) ? LOST_SYNCED_EVENT :
LOST_NDB_EVENT;
case NEW_EVENT:
return (present & SERVER_MASTER) ? NEW_MASTER_EVENT :
(present & SERVER_SLAVE) ? NEW_SLAVE_EVENT :
(present & SERVER_JOINED) ? NEW_SYNCED_EVENT :
NEW_NDB_EVENT;
default:
return UNDEFINED_MONITOR_EVENT;
}
}
/*
* Given a monitor event (enum) provide a text string equivalent
* @param node The monitor server data whose event is wanted
* @result string The name of the monitor event for the server
*/
const char*
mon_get_event_name(MONITOR_SERVERS* node)
{
return monitor_event_definitions[mon_get_event_type(node)].name;
}
/*
* Given the text version of a monitor event, determine the event (enum)
*
* @param event_name String containing the event name
* @result monitor_event_t Monitor event corresponding to name
*/
monitor_event_t
mon_name_to_event(const char *event_name)
{
monitor_event_t event;
for (event = 0; event < MAX_MONITOR_EVENT; event++)
{
if (0 == strcasecmp(monitor_event_definitions[event].name, event_name))
{
return event;
}
}
return UNDEFINED_MONITOR_EVENT;
}
/**
* Create a list of running servers
*
* @param servers Monitored servers
* @param dest Destination where the string is appended, must be null terminated
* @param len Length of @c dest
*/
static void mon_append_node_names(MONITOR_SERVERS* servers, char* dest, int len, int status)
{
char *separator = "";
char arr[MAX_SERVER_NAME_LEN + 32]; // Some extra space for port
dest[0] = '\0';
while (servers && strlen(dest) < (len - strlen(separator)))
{
if (status == 0 || servers->server->status & status)
{
strncat(dest, separator, len);
separator = ",";
snprintf(arr, sizeof(arr), "%s:%d", servers->server->name, servers->server->port);
strncat(dest, arr, len - strlen(dest) - 1);
}
servers = servers->next;
}
}
/**
* Check if current monitored server status has changed
*
* @param mon_srv The monitored server
* @return true if status has changed or false
*/
bool
mon_status_changed(MONITOR_SERVERS* mon_srv)
{
/* Previous status is -1 if not yet set */
return (mon_srv->mon_prev_status != -1
&& mon_srv->mon_prev_status != mon_srv->server->status
/** If the server is going into maintenance or coming out of it, don't trigger a state change */
&& ((mon_srv->mon_prev_status | mon_srv->server->status) & SERVER_MAINT) == 0);
}
/**
* Check if current monitored server has a loggable failure status
*
* @param mon_srv The monitored server
* @return true if failed status can be logged or false
*/
bool
mon_print_fail_status(MONITOR_SERVERS* mon_srv)
{
return (SERVER_IS_DOWN(mon_srv->server) && mon_srv->mon_err_count == 0);
}
/**
* Launch a script
* @param mon Owning monitor
* @param ptr The server which has changed state
* @param script Script to execute
*/
void
monitor_launch_script(MONITOR* mon, MONITOR_SERVERS* ptr, char* script)
{
EXTERNCMD* cmd = externcmd_allocate(script);
if (cmd == NULL)
{
MXS_ERROR("Failed to initialize script '%s'. See previous errors for the "
"cause of this failure.", script);
return;
}
if (externcmd_matches(cmd, "$INITIATOR"))
{
char initiator[strlen(ptr->server->name) + 24]; // Extra space for port
snprintf(initiator, sizeof(initiator), "%s:%d", ptr->server->name, ptr->server->port);
externcmd_substitute_arg(cmd, "[$]INITIATOR", initiator);
}
if (externcmd_matches(cmd, "$EVENT"))
{
externcmd_substitute_arg(cmd, "[$]EVENT", mon_get_event_name(ptr));
}
char nodelist[PATH_MAX + MON_ARG_MAX + 1] = {'\0'};
if (externcmd_matches(cmd, "$NODELIST"))
{
mon_append_node_names(mon->databases, nodelist, sizeof(nodelist), SERVER_RUNNING);
externcmd_substitute_arg(cmd, "[$]NODELIST", nodelist);
}
if (externcmd_matches(cmd, "$LIST"))
{
mon_append_node_names(mon->databases, nodelist, sizeof(nodelist), 0);
externcmd_substitute_arg(cmd, "[$]LIST", nodelist);
}
if (externcmd_matches(cmd, "$MASTERLIST"))
{
mon_append_node_names(mon->databases, nodelist, sizeof(nodelist), SERVER_MASTER);
externcmd_substitute_arg(cmd, "[$]MASTERLIST", nodelist);
}
if (externcmd_matches(cmd, "$SLAVELIST"))
{
mon_append_node_names(mon->databases, nodelist, sizeof(nodelist), SERVER_SLAVE);
externcmd_substitute_arg(cmd, "[$]SLAVELIST", nodelist);
}
if (externcmd_matches(cmd, "$SYNCEDLIST"))
{
mon_append_node_names(mon->databases, nodelist, sizeof(nodelist), SERVER_JOINED);
externcmd_substitute_arg(cmd, "[$]SYNCEDLIST", nodelist);
}
if (externcmd_execute(cmd))
{
MXS_ERROR("Failed to execute script '%s' on server state change event '%s'.",
script, mon_get_event_name(ptr));
}
else
{
ss_dassert(cmd->argv != NULL && cmd->argv[0] != NULL);
// Construct a string with the script + arguments
char *scriptStr = NULL;
int totalStrLen = 0;
bool memError = false;
for (int i = 0; cmd->argv[i]; i++)
{
totalStrLen += strlen(cmd->argv[i]) + 1; // +1 for space and one \0
}
int spaceRemaining = totalStrLen;
if ((scriptStr = MXS_CALLOC(totalStrLen, sizeof(char))) != NULL)
{
char *currentPos = scriptStr;
// The script name should not begin with a space
int len = snprintf(currentPos, spaceRemaining, "%s", cmd->argv[0]);
currentPos += len;
spaceRemaining -= len;
for (int i = 1; cmd->argv[i]; i++)
{
if ((cmd->argv[i])[0] == '\0')
{
continue; // Empty argument, print nothing
}
len = snprintf(currentPos, spaceRemaining, " %s", cmd->argv[i]);
currentPos += len;
spaceRemaining -= len;
}
ss_dassert(spaceRemaining > 0);
*currentPos = '\0';
}
else
{
memError = true;
scriptStr = cmd->argv[0]; // print at least something
}
MXS_NOTICE("Executed monitor script '%s' on event '%s'.",
scriptStr, mon_get_event_name(ptr));
if (!memError)
{
MXS_FREE(scriptStr);
}
}
externcmd_free(cmd);
}
/**
* Parse a string of event names to an array with enabled events.
* @param events Pointer to an array of boolean values
* @param count Size of the array
* @param string String to parse
* @return 0 on success. 1 when an error has occurred or an unexpected event was
* found.
*/
int
mon_parse_event_string(bool* events, size_t count, char* given_string)
{
char *tok, *saved, *string = MXS_STRDUP(given_string);
MXS_ABORT_IF_NULL(string);
monitor_event_t event;
tok = strtok_r(string, ",| ", &saved);
if (tok == NULL)
{
MXS_FREE(string);
return -1;
}
while (tok)
{
event = mon_name_to_event(tok);
if (event == UNDEFINED_MONITOR_EVENT)
{
MXS_ERROR("Invalid event name %s", tok);
MXS_FREE(string);
return -1;
}
if (event < count)
{
events[event] = true;
tok = strtok_r(NULL, ",| ", &saved);
}
}
MXS_FREE(string);
return 0;
}
/**
* Connect to a database. This will always leave a valid database handle in the
* database->con pointer. This allows the user to call MySQL C API functions to
* find out the reason of the failure.
* @param mon Monitor
* @param database Monitored database
* @return MONITOR_CONN_OK if the connection is OK else the reason for the failure
*/
connect_result_t
mon_connect_to_db(MONITOR* mon, MONITOR_SERVERS *database)
{
connect_result_t rval = MONITOR_CONN_OK;
/** Return if the connection is OK */
if (database->con && mysql_ping(database->con) == 0)
{
return rval;
}
if (database->con)
{
mysql_close(database->con);
}
if ((database->con = mysql_init(NULL)))
{
char *uname = mon->user;
char *passwd = mon->password;
if (database->server->monuser[0] && database->server->monpw[0])
{
uname = database->server->monuser;
passwd = database->server->monpw;
}
char *dpwd = decryptPassword(passwd);
mysql_options(database->con, MYSQL_OPT_CONNECT_TIMEOUT, (void *) &mon->connect_timeout);
mysql_options(database->con, MYSQL_OPT_READ_TIMEOUT, (void *) &mon->read_timeout);
mysql_options(database->con, MYSQL_OPT_WRITE_TIMEOUT, (void *) &mon->write_timeout);
time_t start = time(NULL);
bool result = (mxs_mysql_real_connect(database->con, database->server, uname, dpwd) != NULL);
time_t end = time(NULL);
if (!result)
{
if ((int) difftime(end, start) >= mon->connect_timeout)
{
rval = MONITOR_CONN_TIMEOUT;
}
else
{
rval = MONITOR_CONN_REFUSED;
}
}
MXS_FREE(dpwd);
}
else
{
rval = MONITOR_CONN_REFUSED;
}
return rval;
}
/**
* Log an error about the failure to connect to a backend server
* and why it happened.
* @param database Backend database
* @param rval Return value of mon_connect_to_db
*/
void
mon_log_connect_error(MONITOR_SERVERS* database, connect_result_t rval)
{
MXS_ERROR(rval == MONITOR_CONN_TIMEOUT ?
"Monitor timed out when connecting to server %s:%d : \"%s\"" :
"Monitor was unable to connect to server %s:%d : \"%s\"",
database->server->name, database->server->port,
mysql_error(database->con));
}
void mon_log_state_change(MONITOR_SERVERS *ptr)
{
SERVER srv;
srv.status = ptr->mon_prev_status;
char *prev = server_status(&srv);
char *next = server_status(ptr->server);
MXS_NOTICE("Server changed state: %s[%s:%u]: %s. [%s] -> [%s]",
ptr->server->unique_name, ptr->server->name, ptr->server->port,
mon_get_event_name(ptr), prev, next);
MXS_FREE(prev);
MXS_FREE(next);
}
MONITOR* monitor_server_in_use(const SERVER *server)
{
MONITOR *rval = NULL;
spinlock_acquire(&monLock);
for (MONITOR *mon = allMonitors; mon && !rval; mon = mon->next)
{
spinlock_acquire(&mon->lock);
for (MONITOR_SERVERS *db = mon->databases; db && !rval; db = db->next)
{
if (db->server == server)
{
rval = mon;
}
}
spinlock_release(&mon->lock);
}
spinlock_release(&monLock);
return rval;
}
/**
* Creates a monitor configuration at the location pointed by @c filename
*
* @param monitor Monitor to serialize into a configuration
* @param filename Filename where configuration is written
* @return True on success, false on error
*/
static bool create_monitor_server_config(const MONITOR *monitor, const char *filename)
{
int file = open(filename, O_EXCL | O_CREAT | O_WRONLY, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
if (file == -1)
{
char errbuf[MXS_STRERROR_BUFLEN];
MXS_ERROR("Failed to open file '%s' when serializing monitor '%s': %d, %s",
filename, monitor->name, errno, strerror_r(errno, errbuf, sizeof(errbuf)));
return false;
}
/**
* Only additional parameters are added to the configuration. This prevents
* duplication or addition of parameters that don't support it.
*
* TODO: Check for return values on all of the dprintf calls
*/
dprintf(file, "[%s]\n", monitor->name);
if (monitor->databases)
{
dprintf(file, "servers=");
for (MONITOR_SERVERS *db = monitor->databases; db; db = db->next)
{
if (db != monitor->databases)
{
dprintf(file, ",");
}
dprintf(file, "%s", db->server->unique_name);
}
dprintf(file, "\n");
}
close(file);
return true;
}
static bool create_monitor_config(const MONITOR *monitor, const char *filename)
{
int file = open(filename, O_EXCL | O_CREAT | O_WRONLY, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
if (file == -1)
{
char errbuf[MXS_STRERROR_BUFLEN];
MXS_ERROR("Failed to open file '%s' when serializing monitor '%s': %d, %s",
filename, monitor->name, errno, strerror_r(errno, errbuf, sizeof(errbuf)));
return false;
}
/**
* Only additional parameters are added to the configuration. This prevents
* duplication or addition of parameters that don't support it.
*
* TODO: Check for return values on all of the dprintf calls
*/
dprintf(file, "[%s]\n", monitor->name);
dprintf(file, "type=monitor\n");
dprintf(file, "module=%s\n", monitor->module_name);
dprintf(file, "user=%s\n", monitor->user);
dprintf(file, "password=%s\n", monitor->password);
dprintf(file, "monitor_interval=%lu\n", monitor->interval);
dprintf(file, "backend_connect_timeout=%d\n", monitor->connect_timeout);
dprintf(file, "backend_write_timeout=%d\n", monitor->write_timeout);
dprintf(file, "backend_read_timeout=%d\n", monitor->read_timeout);
close(file);
return true;
}
bool monitor_serialize_servers(const MONITOR *monitor)
{
bool rval = false;
char filename[PATH_MAX];
snprintf(filename, sizeof(filename), "%s/%s.cnf.tmp", get_config_persistdir(),
monitor->name);
if (unlink(filename) == -1 && errno != ENOENT)
{
char err[MXS_STRERROR_BUFLEN];
MXS_ERROR("Failed to remove temporary monitor configuration at '%s': %d, %s",
filename, errno, strerror_r(errno, err, sizeof(err)));
}
else if (create_monitor_server_config(monitor, filename))
{
char final_filename[PATH_MAX];
strcpy(final_filename, filename);
char *dot = strrchr(final_filename, '.');
ss_dassert(dot);
*dot = '\0';
if (rename(filename, final_filename) == 0)
{
rval = true;
}
else
{
char err[MXS_STRERROR_BUFLEN];
MXS_ERROR("Failed to rename temporary monitor configuration at '%s': %d, %s",
filename, errno, strerror_r(errno, err, sizeof(err)));
}
}
return rval;
}
bool monitor_serialize(const MONITOR *monitor)
{
bool rval = false;
char filename[PATH_MAX];
snprintf(filename, sizeof(filename), "%s/%s.cnf.tmp", get_config_persistdir(),
monitor->name);
if (unlink(filename) == -1 && errno != ENOENT)
{
char err[MXS_STRERROR_BUFLEN];
MXS_ERROR("Failed to remove temporary monitor configuration at '%s': %d, %s",
filename, errno, strerror_r(errno, err, sizeof(err)));
}
else if (create_monitor_config(monitor, filename))
{
char final_filename[PATH_MAX];
strcpy(final_filename, filename);
char *dot = strrchr(final_filename, '.');
ss_dassert(dot);
*dot = '\0';
if (rename(filename, final_filename) == 0)
{
rval = true;
}
else
{
char err[MXS_STRERROR_BUFLEN];
MXS_ERROR("Failed to rename temporary monitor configuration at '%s': %d, %s",
filename, errno, strerror_r(errno, err, sizeof(err)));
}
}
return rval;
}
void mon_hangup_failed_servers(MONITOR *monitor)
{
for (MONITOR_SERVERS *ptr = monitor->databases; ptr; ptr = ptr->next)
{
if (mon_status_changed(ptr) &&
(!(SERVER_IS_RUNNING(ptr->server)) ||
!(SERVER_IS_IN_CLUSTER(ptr->server))))
{
dcb_hangup_foreach(ptr->server);
}
}
}
/**
* Acquire locks on all servers monitored by this monitor. There should
* only be max 1 monitor per server.
* @param monitor The target monitor
*/
void lock_monitor_servers(MONITOR *monitor)
{
MONITOR_SERVERS *ptr = monitor->databases;
while (ptr)
{
spinlock_acquire(&ptr->server->lock);
ptr = ptr->next;
}
}
/**
* Release locks on all servers monitored by this monitor. There should
* only be max 1 monitor per server.
* @param monitor The target monitor
*/
void release_monitor_servers(MONITOR *monitor)
{
MONITOR_SERVERS *ptr = monitor->databases;
while (ptr)
{
spinlock_release(&ptr->server->lock);
ptr = ptr->next;
}
}
/**
* Sets the current status of all servers monitored by this monitor to
* the pending status. This should only be called at the beginning of
* a monitor loop, after the servers are locked.
* @param monitor The target monitor
*/
void servers_status_pending_to_current(MONITOR *monitor)
{
MONITOR_SERVERS *ptr = monitor->databases;
while (ptr)
{
ptr->server->status = ptr->server->status_pending;
ptr = ptr->next;
}
monitor->server_pending_changes = false;
}
/**
* Sets the pending status of all servers monitored by this monitor to
* the current status. This should only be called at the end of
* a monitor loop, before the servers are released.
* @param monitor The target monitor
*/
void servers_status_current_to_pending(MONITOR *monitor)
{
MONITOR_SERVERS *ptr = monitor->databases;
while (ptr)
{
ptr->server->status_pending = ptr->server->status;
ptr = ptr->next;
}
}
Reference in New Issue View Git Blame Copy Permalink