Merge branch 'develop' into MAX-167
Conflicts: server/modules/monitor/mysql_mon.c server/modules/routing/readwritesplit/readwritesplit.c
This commit is contained in:
@ -33,6 +33,13 @@
|
||||
* 28/05/14 Massimiliano Pinto Added set Id and configuration options (setInverval)
|
||||
* Parameters are now printed in diagnostics
|
||||
* 03/06/14 Mark Ridoch Add support for maintenance mode
|
||||
* 17/06/14 Massimiliano Pinto Addition of getServerByNodeId routine
|
||||
* and first implementation for depth of replication for nodes.
|
||||
* 23/06/14 Massimiliano Pinto Added replication consistency after replication tree computation
|
||||
* 27/06/14 Massimiliano Pinto Added replication pending status in monitored server, storing there
|
||||
* the status to update in server status field before
|
||||
* starting the replication consistency check.
|
||||
* This will also give routers a consistent "status" of all servers
|
||||
*
|
||||
* @endverbatim
|
||||
*/
|
||||
@ -75,6 +82,14 @@ static void defaultId(void *, unsigned long);
|
||||
static void replicationHeartbeat(void *, int);
|
||||
static bool mon_status_changed(MONITOR_SERVERS* mon_srv);
|
||||
static bool mon_print_fail_status(MONITOR_SERVERS* mon_srv);
|
||||
static MONITOR_SERVERS *getServerByNodeId(MONITOR_SERVERS *, long);
|
||||
static MONITOR_SERVERS *getSlaveOfNodeId(MONITOR_SERVERS *, long);
|
||||
static MONITOR_SERVERS *get_replication_tree(MYSQL_MONITOR *, int);
|
||||
static void set_master_heartbeat(MYSQL_MONITOR *, MONITOR_SERVERS *);
|
||||
static void set_slave_heartbeat(MYSQL_MONITOR *, MONITOR_SERVERS *);
|
||||
static int add_slave_to_master(long *, int, long);
|
||||
static void monitor_set_pending_status(MONITOR_SERVERS *, int);
|
||||
static void monitor_clear_pending_status(MONITOR_SERVERS *, int);
|
||||
|
||||
static MONITOR_OBJECT MyObject = { startMonitor, stopMonitor, registerServer, unregisterServer, defaultUser, diagnostics, setInterval, defaultId, replicationHeartbeat };
|
||||
|
||||
@ -145,6 +160,7 @@ MYSQL_MONITOR *handle;
|
||||
handle->id = MONITOR_DEFAULT_ID;
|
||||
handle->interval = MONITOR_INTERVAL;
|
||||
handle->replicationHeartbeat = 0;
|
||||
handle->master = NULL;
|
||||
spinlock_init(&handle->lock);
|
||||
}
|
||||
handle->tid = (THREAD)thread_start(monitorMain, handle);
|
||||
@ -185,6 +201,9 @@ MONITOR_SERVERS *ptr, *db;
|
||||
db->next = NULL;
|
||||
db->mon_err_count = 0;
|
||||
db->mon_prev_status = 0;
|
||||
/* pending status is updated by get_replication_tree */
|
||||
db->pending_status = 0;
|
||||
|
||||
spinlock_acquire(&handle->lock);
|
||||
|
||||
if (handle->databases == NULL)
|
||||
@ -316,7 +335,6 @@ monitorDatabase(MYSQL_MONITOR *handle, MONITOR_SERVERS *database)
|
||||
MYSQL_ROW row;
|
||||
MYSQL_RES *result;
|
||||
int num_fields;
|
||||
int ismaster = 0;
|
||||
int isslave = 0;
|
||||
char *uname = handle->defaultUser;
|
||||
char *passwd = handle->defaultPasswd;
|
||||
@ -372,15 +390,28 @@ int replication_heartbeat = handle->replicationHeartbeat;
|
||||
database->server->port,
|
||||
mysql_error(database->con))));
|
||||
}
|
||||
/** Store current status */
|
||||
|
||||
/* The current server is not running
|
||||
*
|
||||
* Store server NOT running in server and monitor server pending struct
|
||||
*
|
||||
*/
|
||||
server_clear_status(database->server, SERVER_RUNNING);
|
||||
|
||||
monitor_clear_pending_status(database, SERVER_RUNNING);
|
||||
|
||||
/* Also clear M/S state in both server and monitor server pending struct */
|
||||
server_clear_status(database->server, SERVER_SLAVE);
|
||||
server_clear_status(database->server, SERVER_MASTER);
|
||||
monitor_clear_pending_status(database, SERVER_SLAVE);
|
||||
monitor_clear_pending_status(database, SERVER_MASTER);
|
||||
|
||||
return;
|
||||
}
|
||||
free(dpwd);
|
||||
}
|
||||
/** Store current status */
|
||||
server_set_status(database->server, SERVER_RUNNING);
|
||||
}
|
||||
/* Store current status in both server and monitor server pending struct */
|
||||
server_set_status(database->server, SERVER_RUNNING);
|
||||
monitor_set_pending_status(database, SERVER_RUNNING);
|
||||
|
||||
/* get server version from current server */
|
||||
server_version = mysql_get_server_version(database->con);
|
||||
@ -410,121 +441,6 @@ int replication_heartbeat = handle->replicationHeartbeat;
|
||||
mysql_free_result(result);
|
||||
}
|
||||
|
||||
/* Check SHOW SLAVE HOSTS - if we get rows then we are a master */
|
||||
if (mysql_query(database->con, "SHOW SLAVE HOSTS"))
|
||||
{
|
||||
if (mysql_errno(database->con) == ER_SPECIFIC_ACCESS_DENIED_ERROR)
|
||||
{
|
||||
/* Log lack of permission */
|
||||
}
|
||||
|
||||
database->server->rlag = -1;
|
||||
} else if ((result = mysql_store_result(database->con)) != NULL) {
|
||||
num_fields = mysql_num_fields(result);
|
||||
while ((row = mysql_fetch_row(result)))
|
||||
{
|
||||
ismaster = 1;
|
||||
}
|
||||
mysql_free_result(result);
|
||||
|
||||
if (ismaster && replication_heartbeat == 1) {
|
||||
time_t heartbeat;
|
||||
time_t purge_time;
|
||||
char heartbeat_insert_query[128]="";
|
||||
char heartbeat_purge_query[128]="";
|
||||
|
||||
handle->master_id = database->server->node_id;
|
||||
|
||||
/* create the maxscale_schema database */
|
||||
if (mysql_query(database->con, "CREATE DATABASE IF NOT EXISTS maxscale_schema")) {
|
||||
LOGIF(LE, (skygw_log_write_flush(
|
||||
LOGFILE_ERROR,
|
||||
"[mysql_mon]: Error creating maxscale_schema database in Master server"
|
||||
": %s", mysql_error(database->con))));
|
||||
|
||||
database->server->rlag = -1;
|
||||
}
|
||||
|
||||
/* create repl_heartbeat table in maxscale_schema database */
|
||||
if (mysql_query(database->con, "CREATE TABLE IF NOT EXISTS "
|
||||
"maxscale_schema.replication_heartbeat "
|
||||
"(maxscale_id INT NOT NULL, "
|
||||
"master_server_id INT NOT NULL, "
|
||||
"master_timestamp INT UNSIGNED NOT NULL, "
|
||||
"PRIMARY KEY ( master_server_id, maxscale_id ) ) "
|
||||
"ENGINE=MYISAM DEFAULT CHARSET=latin1")) {
|
||||
LOGIF(LE, (skygw_log_write_flush(
|
||||
LOGFILE_ERROR,
|
||||
"[mysql_mon]: Error creating maxscale_schema.replication_heartbeat table in Master server"
|
||||
": %s", mysql_error(database->con))));
|
||||
|
||||
database->server->rlag = -1;
|
||||
}
|
||||
|
||||
/* auto purge old values after 48 hours*/
|
||||
purge_time = time(0) - (3600 * 48);
|
||||
|
||||
sprintf(heartbeat_purge_query, "DELETE FROM maxscale_schema.replication_heartbeat WHERE master_timestamp < %lu", purge_time);
|
||||
|
||||
if (mysql_query(database->con, heartbeat_purge_query)) {
|
||||
LOGIF(LE, (skygw_log_write_flush(
|
||||
LOGFILE_ERROR,
|
||||
"[mysql_mon]: Error deleting from maxscale_schema.replication_heartbeat table: [%s], %s",
|
||||
heartbeat_purge_query,
|
||||
mysql_error(database->con))));
|
||||
}
|
||||
|
||||
heartbeat = time(0);
|
||||
|
||||
/* set node_ts for master as time(0) */
|
||||
database->server->node_ts = heartbeat;
|
||||
|
||||
sprintf(heartbeat_insert_query, "UPDATE maxscale_schema.replication_heartbeat SET master_timestamp = %lu WHERE master_server_id = %i AND maxscale_id = %lu", heartbeat, handle->master_id, id);
|
||||
|
||||
/* Try to insert MaxScale timestamp into master */
|
||||
if (mysql_query(database->con, heartbeat_insert_query)) {
|
||||
|
||||
database->server->rlag = -1;
|
||||
|
||||
LOGIF(LE, (skygw_log_write_flush(
|
||||
LOGFILE_ERROR,
|
||||
"[mysql_mon]: Error updating maxscale_schema.replication_heartbeat table: [%s], %s",
|
||||
heartbeat_insert_query,
|
||||
mysql_error(database->con))));
|
||||
} else {
|
||||
if (mysql_affected_rows(database->con) == 0) {
|
||||
heartbeat = time(0);
|
||||
sprintf(heartbeat_insert_query, "REPLACE INTO maxscale_schema.replication_heartbeat (master_server_id, maxscale_id, master_timestamp ) VALUES ( %i, %lu, %lu)", handle->master_id, id, heartbeat);
|
||||
|
||||
if (mysql_query(database->con, heartbeat_insert_query)) {
|
||||
|
||||
database->server->rlag = -1;
|
||||
|
||||
LOGIF(LE, (skygw_log_write_flush(
|
||||
LOGFILE_ERROR,
|
||||
"[mysql_mon]: Error inserting into maxscale_schema.replication_heartbeat table: [%s], %s",
|
||||
heartbeat_insert_query,
|
||||
mysql_error(database->con))));
|
||||
} else {
|
||||
/* Set replication lag to 0 for the master */
|
||||
database->server->rlag = 0;
|
||||
|
||||
LOGIF(LD, (skygw_log_write_flush(
|
||||
LOGFILE_DEBUG,
|
||||
"[mysql_mon]: heartbeat table inserted data for %s:%i", database->server->name, database->server->port)));
|
||||
}
|
||||
} else {
|
||||
/* Set replication lag as 0 for the master */
|
||||
database->server->rlag = 0;
|
||||
|
||||
LOGIF(LD, (skygw_log_write_flush(
|
||||
LOGFILE_DEBUG,
|
||||
"[mysql_mon]: heartbeat table updated for %s:%i", database->server->name, database->server->port)));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Check if the Slave_SQL_Running and Slave_IO_Running status is
|
||||
* set to Yes
|
||||
*/
|
||||
@ -536,17 +452,28 @@ int replication_heartbeat = handle->replicationHeartbeat;
|
||||
&& (result = mysql_store_result(database->con)) != NULL)
|
||||
{
|
||||
int i = 0;
|
||||
long master_id = -1;
|
||||
num_fields = mysql_num_fields(result);
|
||||
while ((row = mysql_fetch_row(result)))
|
||||
{
|
||||
/* get Slave_IO_Running and Slave_SQL_Running values*/
|
||||
if (strncmp(row[12], "Yes", 3) == 0
|
||||
&& strncmp(row[13], "Yes", 3) == 0) {
|
||||
isslave += 1;
|
||||
|
||||
/* get Master_Server_Id values */
|
||||
master_id = atol(row[41]);
|
||||
if (master_id == 0)
|
||||
master_id = -1;
|
||||
}
|
||||
i++;
|
||||
}
|
||||
/* store master_id of current node */
|
||||
memcpy(&database->server->master_id, &master_id, sizeof(long));
|
||||
|
||||
mysql_free_result(result);
|
||||
|
||||
/* If all configured slaves are running set this node as slave */
|
||||
if (isslave > 0 && isslave == i)
|
||||
isslave = 1;
|
||||
else
|
||||
@ -556,105 +483,45 @@ int replication_heartbeat = handle->replicationHeartbeat;
|
||||
if (mysql_query(database->con, "SHOW SLAVE STATUS") == 0
|
||||
&& (result = mysql_store_result(database->con)) != NULL)
|
||||
{
|
||||
long master_id = -1;
|
||||
num_fields = mysql_num_fields(result);
|
||||
while ((row = mysql_fetch_row(result)))
|
||||
{
|
||||
/* get Slave_IO_Running and Slave_SQL_Running values*/
|
||||
if (strncmp(row[10], "Yes", 3) == 0
|
||||
&& strncmp(row[11], "Yes", 3) == 0)
|
||||
&& strncmp(row[11], "Yes", 3) == 0) {
|
||||
isslave = 1;
|
||||
|
||||
/* get Master_Server_Id values */
|
||||
master_id = atol(row[39]);
|
||||
if (master_id == 0)
|
||||
master_id = -1;
|
||||
}
|
||||
}
|
||||
/* store master_id of current node */
|
||||
memcpy(&database->server->master_id, &master_id, sizeof(long));
|
||||
|
||||
mysql_free_result(result);
|
||||
}
|
||||
}
|
||||
|
||||
/* Get the master_timestamp value from maxscale_schema.replication_heartbeat table */
|
||||
if (isslave && replication_heartbeat == 1) {
|
||||
time_t heartbeat;
|
||||
char select_heartbeat_query[256] = "";
|
||||
/* Remove addition info */
|
||||
monitor_clear_pending_status(database, SERVER_SLAVE_OF_EXTERNAL_MASTER);
|
||||
|
||||
sprintf(select_heartbeat_query, "SELECT master_timestamp "
|
||||
"FROM maxscale_schema.replication_heartbeat "
|
||||
"WHERE maxscale_id = %lu AND master_server_id = %i",
|
||||
id, handle->master_id);
|
||||
/* Please note, the MASTER status and SERVER_SLAVE_OF_EXTERNAL_MASTER
|
||||
* will be assigned in the monitorMain() via get_replication_tree() routine
|
||||
*/
|
||||
|
||||
/* if there is a master then send the query to the slave with master_id*/
|
||||
if (handle->master_id >= 0 && (mysql_query(database->con, select_heartbeat_query) == 0
|
||||
&& (result = mysql_store_result(database->con)) != NULL)) {
|
||||
num_fields = mysql_num_fields(result);
|
||||
|
||||
while ((row = mysql_fetch_row(result))) {
|
||||
int rlag = -1;
|
||||
time_t slave_read;
|
||||
|
||||
heartbeat = time(0);
|
||||
slave_read = strtoul(row[0], NULL, 10);
|
||||
|
||||
if ((errno == ERANGE && (slave_read == LONG_MAX || slave_read == LONG_MIN)) || (errno != 0 && slave_read == 0)) {
|
||||
slave_read = 0;
|
||||
}
|
||||
|
||||
if (slave_read) {
|
||||
/* set the replication lag */
|
||||
rlag = heartbeat - slave_read;
|
||||
}
|
||||
|
||||
/* set this node_ts as master_timestamp read from replication_heartbeat table */
|
||||
database->server->node_ts = slave_read;
|
||||
|
||||
if (rlag >= 0) {
|
||||
/* store rlag only if greater than monitor sampling interval */
|
||||
database->server->rlag = (rlag > (handle->interval / 1000)) ? rlag : 0;
|
||||
} else {
|
||||
database->server->rlag = -1;
|
||||
}
|
||||
|
||||
LOGIF(LD, (skygw_log_write_flush(
|
||||
LOGFILE_DEBUG,
|
||||
"[mysql_mon]: replication heartbeat: "
|
||||
"server %s:%i is %i seconds behind master",
|
||||
database->server->name,
|
||||
database->server->port,
|
||||
database->server->rlag)));
|
||||
}
|
||||
mysql_free_result(result);
|
||||
} else {
|
||||
database->server->rlag = -1;
|
||||
database->server->node_ts = 0;
|
||||
|
||||
if (handle->master_id < 0) {
|
||||
LOGIF(LE, (skygw_log_write_flush(
|
||||
LOGFILE_ERROR,
|
||||
"[mysql_mon]: error: replication heartbeat: "
|
||||
"master_server_id NOT available for %s:%i",
|
||||
database->server->name,
|
||||
database->server->port)));
|
||||
} else {
|
||||
LOGIF(LE, (skygw_log_write_flush(
|
||||
LOGFILE_ERROR,
|
||||
"[mysql_mon]: error: replication heartbeat: "
|
||||
"failed selecting from hearthbeat table of %s:%i : [%s], %s",
|
||||
database->server->name,
|
||||
database->server->port,
|
||||
select_heartbeat_query,
|
||||
mysql_error(database->con))));
|
||||
}
|
||||
}
|
||||
}
|
||||
/** Store current status */
|
||||
if (ismaster)
|
||||
/* Set the Slave Role */
|
||||
if (isslave)
|
||||
{
|
||||
server_set_status(database->server, SERVER_MASTER);
|
||||
server_clear_status(database->server, SERVER_SLAVE);
|
||||
}
|
||||
else if (isslave)
|
||||
{
|
||||
server_set_status(database->server, SERVER_SLAVE);
|
||||
server_clear_status(database->server, SERVER_MASTER);
|
||||
}
|
||||
if (ismaster == 0 && isslave == 0)
|
||||
{
|
||||
server_clear_status(database->server, SERVER_SLAVE);
|
||||
server_clear_status(database->server, SERVER_MASTER);
|
||||
monitor_set_pending_status(database, SERVER_SLAVE);
|
||||
/* Avoid any possible stale Master state */
|
||||
monitor_clear_pending_status(database, SERVER_MASTER);
|
||||
} else {
|
||||
/* Avoid any possible Master/Slave stale state */
|
||||
monitor_clear_pending_status(database, SERVER_SLAVE);
|
||||
monitor_clear_pending_status(database, SERVER_MASTER);
|
||||
}
|
||||
}
|
||||
|
||||
@ -668,6 +535,9 @@ monitorMain(void *arg)
|
||||
{
|
||||
MYSQL_MONITOR *handle = (MYSQL_MONITOR *)arg;
|
||||
MONITOR_SERVERS *ptr;
|
||||
int replication_heartbeat = handle->replicationHeartbeat;
|
||||
int num_servers=0;
|
||||
MONITOR_SERVERS *root_master;
|
||||
|
||||
if (mysql_thread_init())
|
||||
{
|
||||
@ -690,8 +560,21 @@ MONITOR_SERVERS *ptr;
|
||||
ptr = handle->databases;
|
||||
while (ptr)
|
||||
{
|
||||
/* copy server status into monitor pending_status */
|
||||
ptr->pending_status = ptr->server->status;
|
||||
|
||||
/* monitor current node */
|
||||
monitorDatabase(handle, ptr);
|
||||
|
||||
/* reset the slave list of current node */
|
||||
if (ptr->server->slaves) {
|
||||
free(ptr->server->slaves);
|
||||
}
|
||||
/* create a new slave list */
|
||||
ptr->server->slaves = (long *) calloc(MONITOR_MAX_NUM_SLAVES, sizeof(long));
|
||||
|
||||
num_servers++;
|
||||
|
||||
if (mon_status_changed(ptr))
|
||||
{
|
||||
dcb_call_foreach(DCB_REASON_NOT_RESPONDING);
|
||||
@ -717,8 +600,40 @@ MONITOR_SERVERS *ptr;
|
||||
/** Reset this server's error count */
|
||||
ptr->mon_err_count = 0;
|
||||
}
|
||||
|
||||
ptr = ptr->next;
|
||||
}
|
||||
|
||||
/* Compute the replication tree */
|
||||
root_master = get_replication_tree(handle, num_servers);
|
||||
|
||||
/* Update server status from monitor pending status on that server*/
|
||||
|
||||
ptr = handle->databases;
|
||||
while (ptr)
|
||||
{
|
||||
if (! SERVER_IN_MAINT(ptr->server)) {
|
||||
ptr->server->status = ptr->pending_status;
|
||||
}
|
||||
ptr = ptr->next;
|
||||
}
|
||||
|
||||
/* Do now the heartbeat replication set/get for MySQL Replication Consistency */
|
||||
if (replication_heartbeat && root_master && (SERVER_IS_MASTER(root_master->server) || SERVER_IS_RELAY_SERVER(root_master->server))) {
|
||||
set_master_heartbeat(handle, root_master);
|
||||
ptr = handle->databases;
|
||||
while (ptr) {
|
||||
if( (! SERVER_IN_MAINT(ptr->server)) && SERVER_IS_RUNNING(ptr->server))
|
||||
{
|
||||
if (ptr->server->node_id != root_master->server->node_id && (SERVER_IS_SLAVE(ptr->server) || SERVER_IS_RELAY_SERVER(ptr->server))) {
|
||||
set_slave_heartbeat(handle, ptr);
|
||||
}
|
||||
}
|
||||
ptr = ptr->next;
|
||||
}
|
||||
}
|
||||
|
||||
/* wait for the configured interval */
|
||||
thread_millisleep(handle->interval);
|
||||
}
|
||||
}
|
||||
@ -797,3 +712,380 @@ static bool mon_print_fail_status(
|
||||
}
|
||||
return succp;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a MySQL node by node_id
|
||||
*
|
||||
* @param ptr The list of servers to monitor
|
||||
* @param node_id The MySQL server_id to fetch
|
||||
* @return The server with the required server_id
|
||||
*/
|
||||
static MONITOR_SERVERS *
|
||||
getServerByNodeId(MONITOR_SERVERS *ptr, long node_id) {
|
||||
SERVER *current;
|
||||
while (ptr)
|
||||
{
|
||||
current = ptr->server;
|
||||
if (current->node_id == node_id) {
|
||||
return ptr;
|
||||
}
|
||||
ptr = ptr->next;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a MySQL slave node from a node_id
|
||||
*
|
||||
* @param ptr The list of servers to monitor
|
||||
* @param node_id The MySQL server_id to fetch
|
||||
* @return The slave server of this node_id
|
||||
*/
|
||||
static MONITOR_SERVERS *
|
||||
getSlaveOfNodeId(MONITOR_SERVERS *ptr, long node_id) {
|
||||
SERVER *current;
|
||||
while (ptr)
|
||||
{
|
||||
current = ptr->server;
|
||||
if (current->master_id == node_id) {
|
||||
return ptr;
|
||||
}
|
||||
ptr = ptr->next;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/*******
|
||||
* This function sets the replication heartbeat
|
||||
* into the maxscale_schema.replication_heartbeat table in the current master.
|
||||
* The inserted values will be seen from all slaves replication from this master.
|
||||
*
|
||||
* @param handle The monitor handle
|
||||
* @param database The number database server
|
||||
*/
|
||||
static void set_master_heartbeat(MYSQL_MONITOR *handle, MONITOR_SERVERS *database) {
|
||||
unsigned long id = handle->id;
|
||||
time_t heartbeat;
|
||||
time_t purge_time;
|
||||
char heartbeat_insert_query[128]="";
|
||||
char heartbeat_purge_query[128]="";
|
||||
|
||||
/* create the maxscale_schema database */
|
||||
if (mysql_query(database->con, "CREATE DATABASE IF NOT EXISTS maxscale_schema")) {
|
||||
LOGIF(LE, (skygw_log_write_flush(
|
||||
LOGFILE_ERROR,
|
||||
"[mysql_mon]: Error creating maxscale_schema database in Master server"
|
||||
": %s", mysql_error(database->con))));
|
||||
|
||||
database->server->rlag = -1;
|
||||
}
|
||||
|
||||
/* create repl_heartbeat table in maxscale_schema database */
|
||||
if (mysql_query(database->con, "CREATE TABLE IF NOT EXISTS "
|
||||
"maxscale_schema.replication_heartbeat "
|
||||
"(maxscale_id INT NOT NULL, "
|
||||
"master_server_id INT NOT NULL, "
|
||||
"master_timestamp INT UNSIGNED NOT NULL, "
|
||||
"PRIMARY KEY ( master_server_id, maxscale_id ) ) "
|
||||
"ENGINE=MYISAM DEFAULT CHARSET=latin1")) {
|
||||
LOGIF(LE, (skygw_log_write_flush(
|
||||
LOGFILE_ERROR,
|
||||
"[mysql_mon]: Error creating maxscale_schema.replication_heartbeat table in Master server"
|
||||
": %s", mysql_error(database->con))));
|
||||
|
||||
database->server->rlag = -1;
|
||||
}
|
||||
|
||||
/* auto purge old values after 48 hours*/
|
||||
purge_time = time(0) - (3600 * 48);
|
||||
|
||||
sprintf(heartbeat_purge_query, "DELETE FROM maxscale_schema.replication_heartbeat WHERE master_timestamp < %lu", purge_time);
|
||||
|
||||
if (mysql_query(database->con, heartbeat_purge_query)) {
|
||||
LOGIF(LE, (skygw_log_write_flush(
|
||||
LOGFILE_ERROR,
|
||||
"[mysql_mon]: Error deleting from maxscale_schema.replication_heartbeat table: [%s], %s",
|
||||
heartbeat_purge_query,
|
||||
mysql_error(database->con))));
|
||||
}
|
||||
|
||||
heartbeat = time(0);
|
||||
|
||||
/* set node_ts for master as time(0) */
|
||||
database->server->node_ts = heartbeat;
|
||||
|
||||
sprintf(heartbeat_insert_query, "UPDATE maxscale_schema.replication_heartbeat SET master_timestamp = %lu WHERE master_server_id = %li AND maxscale_id = %lu", heartbeat, handle->master->server->node_id, id);
|
||||
|
||||
/* Try to insert MaxScale timestamp into master */
|
||||
if (mysql_query(database->con, heartbeat_insert_query)) {
|
||||
|
||||
database->server->rlag = -1;
|
||||
|
||||
LOGIF(LE, (skygw_log_write_flush(
|
||||
LOGFILE_ERROR,
|
||||
"[mysql_mon]: Error updating maxscale_schema.replication_heartbeat table: [%s], %s",
|
||||
heartbeat_insert_query,
|
||||
mysql_error(database->con))));
|
||||
} else {
|
||||
if (mysql_affected_rows(database->con) == 0) {
|
||||
heartbeat = time(0);
|
||||
sprintf(heartbeat_insert_query, "REPLACE INTO maxscale_schema.replication_heartbeat (master_server_id, maxscale_id, master_timestamp ) VALUES ( %li, %lu, %lu)", handle->master->server->node_id, id, heartbeat);
|
||||
|
||||
if (mysql_query(database->con, heartbeat_insert_query)) {
|
||||
|
||||
database->server->rlag = -1;
|
||||
|
||||
LOGIF(LE, (skygw_log_write_flush(
|
||||
LOGFILE_ERROR,
|
||||
"[mysql_mon]: Error inserting into maxscale_schema.replication_heartbeat table: [%s], %s",
|
||||
heartbeat_insert_query,
|
||||
mysql_error(database->con))));
|
||||
} else {
|
||||
/* Set replication lag to 0 for the master */
|
||||
database->server->rlag = 0;
|
||||
|
||||
LOGIF(LD, (skygw_log_write_flush(
|
||||
LOGFILE_DEBUG,
|
||||
"[mysql_mon]: heartbeat table inserted data for %s:%i", database->server->name, database->server->port)));
|
||||
}
|
||||
} else {
|
||||
/* Set replication lag as 0 for the master */
|
||||
database->server->rlag = 0;
|
||||
|
||||
LOGIF(LD, (skygw_log_write_flush(
|
||||
LOGFILE_DEBUG,
|
||||
"[mysql_mon]: heartbeat table updated for Master %s:%i", database->server->name, database->server->port)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*******
|
||||
* This function gets the replication heartbeat
|
||||
* from the maxscale_schema.replication_heartbeat table in the current slave
|
||||
* and stores the timestamp and replication lag in the slave server struct
|
||||
*
|
||||
* @param handle The monitor handle
|
||||
* @param database The number database server
|
||||
*/
|
||||
static void set_slave_heartbeat(MYSQL_MONITOR *handle, MONITOR_SERVERS *database) {
|
||||
unsigned long id = handle->id;
|
||||
time_t heartbeat;
|
||||
char select_heartbeat_query[256] = "";
|
||||
MYSQL_ROW row;
|
||||
MYSQL_RES *result;
|
||||
int num_fields;
|
||||
|
||||
/* Get the master_timestamp value from maxscale_schema.replication_heartbeat table */
|
||||
|
||||
sprintf(select_heartbeat_query, "SELECT master_timestamp "
|
||||
"FROM maxscale_schema.replication_heartbeat "
|
||||
"WHERE maxscale_id = %lu AND master_server_id = %li",
|
||||
id, handle->master->server->node_id);
|
||||
|
||||
/* if there is a master then send the query to the slave with master_id */
|
||||
if (handle->master !=NULL && (mysql_query(database->con, select_heartbeat_query) == 0
|
||||
&& (result = mysql_store_result(database->con)) != NULL)) {
|
||||
num_fields = mysql_num_fields(result);
|
||||
|
||||
while ((row = mysql_fetch_row(result))) {
|
||||
int rlag = -1;
|
||||
time_t slave_read;
|
||||
|
||||
heartbeat = time(0);
|
||||
slave_read = strtoul(row[0], NULL, 10);
|
||||
|
||||
if ((errno == ERANGE && (slave_read == LONG_MAX || slave_read == LONG_MIN)) || (errno != 0 && slave_read == 0)) {
|
||||
slave_read = 0;
|
||||
}
|
||||
|
||||
if (slave_read) {
|
||||
/* set the replication lag */
|
||||
rlag = heartbeat - slave_read;
|
||||
}
|
||||
|
||||
/* set this node_ts as master_timestamp read from replication_heartbeat table */
|
||||
database->server->node_ts = slave_read;
|
||||
|
||||
if (rlag >= 0) {
|
||||
/* store rlag only if greater than monitor sampling interval */
|
||||
database->server->rlag = (rlag > (handle->interval / 1000)) ? rlag : 0;
|
||||
} else {
|
||||
database->server->rlag = -1;
|
||||
}
|
||||
|
||||
LOGIF(LD, (skygw_log_write_flush(
|
||||
LOGFILE_DEBUG,
|
||||
"[mysql_mon]: replication heartbeat: "
|
||||
"Slave %s:%i has %i seconds lag",
|
||||
database->server->name,
|
||||
database->server->port,
|
||||
database->server->rlag)));
|
||||
}
|
||||
mysql_free_result(result);
|
||||
} else {
|
||||
database->server->rlag = -1;
|
||||
database->server->node_ts = 0;
|
||||
|
||||
if (handle->master->server->node_id < 0) {
|
||||
LOGIF(LE, (skygw_log_write_flush(
|
||||
LOGFILE_ERROR,
|
||||
"[mysql_mon]: error: replication heartbeat: "
|
||||
"master_server_id NOT available for %s:%i",
|
||||
database->server->name,
|
||||
database->server->port)));
|
||||
} else {
|
||||
LOGIF(LE, (skygw_log_write_flush(
|
||||
LOGFILE_ERROR,
|
||||
"[mysql_mon]: error: replication heartbeat: "
|
||||
"failed selecting from hearthbeat table of %s:%i : [%s], %s",
|
||||
database->server->name,
|
||||
database->server->port,
|
||||
select_heartbeat_query,
|
||||
mysql_error(database->con))));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*******
|
||||
* This function computes the replication tree
|
||||
* from a set of MySQL Master/Slave monitored servers
|
||||
* and returns the root server with SERVER_MASTER bit.
|
||||
* The tree is computed even for servers in 'maintenance' mode.
|
||||
*
|
||||
* @param handle The monitor handle
|
||||
* @param num_servers The number of servers monitored
|
||||
* @return The server at root level with SERVER_MASTER bit
|
||||
*/
|
||||
|
||||
static MONITOR_SERVERS *get_replication_tree(MYSQL_MONITOR *handle, int num_servers) {
|
||||
MONITOR_SERVERS *ptr;
|
||||
MONITOR_SERVERS *backend;
|
||||
SERVER *current;
|
||||
int depth=0;
|
||||
long node_id;
|
||||
int root_level;
|
||||
|
||||
ptr = handle->databases;
|
||||
root_level = num_servers;
|
||||
|
||||
while (ptr)
|
||||
{
|
||||
/* The server could be in SERVER_IN_MAINT
|
||||
* that means SERVER_IS_RUNNING returns 0
|
||||
* Let's check only for SERVER_IS_DOWN: server is not running
|
||||
*/
|
||||
if (SERVER_IS_DOWN(ptr->server)) {
|
||||
ptr = ptr->next;
|
||||
continue;
|
||||
}
|
||||
depth = 0;
|
||||
current = ptr->server;
|
||||
|
||||
node_id = current->master_id;
|
||||
if (node_id < 1) {
|
||||
MONITOR_SERVERS *find_slave;
|
||||
find_slave = getSlaveOfNodeId(handle->databases, current->node_id);
|
||||
|
||||
if (find_slave == NULL) {
|
||||
current->depth = -1;
|
||||
ptr = ptr->next;
|
||||
|
||||
continue;
|
||||
} else {
|
||||
current->depth = 0;
|
||||
}
|
||||
} else {
|
||||
depth++;
|
||||
}
|
||||
|
||||
while(depth <= num_servers) {
|
||||
/* set the root master at lowest depth level */
|
||||
if (current->depth > -1 && current->depth < root_level) {
|
||||
root_level = current->depth;
|
||||
handle->master = ptr;
|
||||
}
|
||||
backend = getServerByNodeId(handle->databases, node_id);
|
||||
|
||||
if (backend) {
|
||||
node_id = backend->server->master_id;
|
||||
} else {
|
||||
node_id = -1;
|
||||
}
|
||||
|
||||
if (node_id > 0) {
|
||||
current->depth = depth + 1;
|
||||
depth++;
|
||||
|
||||
} else {
|
||||
MONITOR_SERVERS *master;
|
||||
current->depth = depth;
|
||||
|
||||
master = getServerByNodeId(handle->databases, current->master_id);
|
||||
if (master && master->server && master->server->node_id > 0) {
|
||||
add_slave_to_master(master->server->slaves, MONITOR_MAX_NUM_SLAVES, current->node_id);
|
||||
master->server->depth = current->depth -1;
|
||||
monitor_set_pending_status(master, SERVER_MASTER);
|
||||
} else {
|
||||
if (current->master_id > 0) {
|
||||
monitor_set_pending_status(ptr, SERVER_SLAVE_OF_EXTERNAL_MASTER);
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
ptr = ptr->next;
|
||||
}
|
||||
|
||||
/* If root master is in MAINT, return NULL */
|
||||
if (SERVER_IN_MAINT(handle->master->server)) {
|
||||
return NULL;
|
||||
} else {
|
||||
return handle->master;
|
||||
}
|
||||
}
|
||||
|
||||
/*******
|
||||
* This function add a slave id into the slaves server field
|
||||
* of its master server
|
||||
*
|
||||
* @param slaves_list The slave list array of the master server
|
||||
* @param list_size The size of the slave list
|
||||
* @param node_id The node_id of the slave to be inserted
|
||||
* @return 1 for inserted value and 0 otherwise
|
||||
*/
|
||||
static int add_slave_to_master(long *slaves_list, int list_size, long node_id) {
|
||||
int i;
|
||||
for (i = 0; i< list_size; i++) {
|
||||
if (slaves_list[i] == 0) {
|
||||
memcpy(&slaves_list[i], &node_id, sizeof(long));
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set a pending status bit in the monior server
|
||||
*
|
||||
* @param server The server to update
|
||||
* @param bit The bit to clear for the server
|
||||
*/
|
||||
static void
|
||||
monitor_set_pending_status(MONITOR_SERVERS *ptr, int bit)
|
||||
{
|
||||
ptr->pending_status |= bit;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear a pending status bit in the monior server
|
||||
*
|
||||
* @param server The server to update
|
||||
* @param bit The bit to clear for the server
|
||||
*/
|
||||
static void
|
||||
monitor_clear_pending_status(MONITOR_SERVERS *ptr, int bit)
|
||||
{
|
||||
ptr->pending_status &= ~bit;
|
||||
}
|
||||
|
Reference in New Issue
Block a user