From bf00ca8aba2ece20e0e93e25fd84a7c04ccc5b88 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Mon, 26 May 2014 10:29:34 +0200 Subject: [PATCH 1/5] GaleraHA support - R/W Splitting with Galera GaleraHA support - R/W Splitting with Galera --- server/core/server.c | 8 +++- server/include/server.h | 12 +++-- server/modules/monitor/galera_mon.c | 71 ++++++++++++++++++++++++++++- 3 files changed, 82 insertions(+), 9 deletions(-) diff --git a/server/core/server.c b/server/core/server.c index bf5b3174f..27ed23b75 100644 --- a/server/core/server.c +++ b/server/core/server.c @@ -22,8 +22,9 @@ * @verbatim * Revision History * - * Date Who Description - * 18/06/13 Mark Riddoch Initial implementation + * Date Who Description + * 18/06/13 Mark Riddoch Initial implementation + * 21/05/14 Massimiliano Pinto Addition of node_id * * @endverbatim */ @@ -67,6 +68,7 @@ SERVER *server; server->nextdb = NULL; server->monuser = NULL; server->monpw = NULL; + server->node_id = -1; spinlock_acquire(&server_spin); server->next = allServers; @@ -197,6 +199,7 @@ char *stat; free(stat); dcb_printf(dcb, "\tProtocol: %s\n", ptr->protocol); dcb_printf(dcb, "\tPort: %d\n", ptr->port); + dcb_printf(dcb, "\tNode Id: %d\n", ptr->node_id); dcb_printf(dcb, "\tNumber of connections: %d\n", ptr->stats.n_connections); dcb_printf(dcb, "\tCurrent no. of connections: %d\n", ptr->stats.n_current); ptr = ptr->next; @@ -222,6 +225,7 @@ char *stat; free(stat); dcb_printf(dcb, "\tProtocol: %s\n", server->protocol); dcb_printf(dcb, "\tPort: %d\n", server->port); + dcb_printf(dcb, "\tNode Id: %d\n", server->node_id); dcb_printf(dcb, "\tNumber of connections: %d\n", server->stats.n_connections); dcb_printf(dcb, "\tCurrent No. of connections: %d\n", server->stats.n_current); } diff --git a/server/include/server.h b/server/include/server.h index e355e8c49..6e7b146d1 100644 --- a/server/include/server.h +++ b/server/include/server.h @@ -27,10 +27,11 @@ * @verbatim * Revision History * - * Date Who Description - * 14/06/13 Mark Riddoch Initial implementation - * 21/06/13 Mark Riddoch Addition of server status flags - * 22/07/13 Mark Riddoch Addition of JOINED status for Galera + * Date Who Description + * 14/06/13 Mark Riddoch Initial implementation + * 21/06/13 Mark Riddoch Addition of server status flags + * 22/07/13 Mark Riddoch Addition of JOINED status for Galera + * 20/05/14 Massimiliano Pinto Addition of node_id field * * @endverbatim */ @@ -60,6 +61,7 @@ typedef struct server { SERVER_STATS stats; /**< The server statistics */ struct server *next; /**< Next server */ struct server *nextdb; /**< Next server in list attached to a service */ + long node_id; /**< Node id, server_id for M/S or local_index for Galera */ } SERVER; /** @@ -99,7 +101,7 @@ typedef struct server { * Is the server joined Galera node? The server must be running and joined. */ #define SERVER_IS_JOINED(server) \ - (((server)->status & (SERVER_RUNNING|SERVER_MASTER|SERVER_SLAVE|SERVER_JOINED)) == (SERVER_RUNNING|SERVER_JOINED)) + (((server)->status & (SERVER_RUNNING|SERVER_JOINED)) == (SERVER_RUNNING|SERVER_JOINED)) extern SERVER *server_alloc(char *, char *, unsigned short); extern int server_free(SERVER *); diff --git a/server/modules/monitor/galera_mon.c b/server/modules/monitor/galera_mon.c index 1f277dcbc..15d07eda9 100644 --- a/server/modules/monitor/galera_mon.c +++ b/server/modules/monitor/galera_mon.c @@ -22,8 +22,10 @@ * @verbatim * Revision History * - * Date Who Description - * 22/07/13 Mark Riddoch Initial implementation + * Date Who Description + * 22/07/13 Mark Riddoch Initial implementation + * 21/05/14 Massimiliano Pinto Monitor sets a master server + * that has the lowest value of wsrep_local_index * * @endverbatim */ @@ -280,6 +282,8 @@ MYSQL_RES *result; int num_fields; int isjoined = 0; char *uname = defaultUser, *passwd = defaultPasswd; +unsigned long int server_version = 0; +char *server_string; if (database->server->monuser != NULL) { @@ -297,6 +301,7 @@ char *uname = defaultUser, *passwd = defaultPasswd; uname, dpwd, NULL, database->server->port, NULL, 0) == NULL) { server_clear_status(database->server, SERVER_RUNNING); + database->server->node_id = -1; free(dpwd); return; } @@ -306,6 +311,15 @@ char *uname = defaultUser, *passwd = defaultPasswd; /* If we get this far then we have a working connection */ server_set_status(database->server, SERVER_RUNNING); + /* get server version from current server */ + server_version = mysql_get_server_version(database->con); + + /* get server version string */ + server_string = (char *)mysql_get_server_info(database->con); + if (server_string) { + database->server->server_string = strdup(server_string); + } + /* Check if the the Galera FSM shows this node is joined to the cluster */ if (mysql_query(database->con, "SHOW STATUS LIKE 'wsrep_local_state_comment'") == 0 && (result = mysql_store_result(database->con)) != NULL) @@ -319,6 +333,25 @@ char *uname = defaultUser, *passwd = defaultPasswd; mysql_free_result(result); } + /* Check the the Galera node index in the cluster */ + if (mysql_query(database->con, "SHOW STATUS LIKE 'wsrep_local_index'") == 0 + && (result = mysql_store_result(database->con)) != NULL) + { + long local_index = -1; + num_fields = mysql_num_fields(result); + while ((row = mysql_fetch_row(result))) + { + local_index = strtol(row[1], NULL, 10); + if ((errno == ERANGE && (local_index == LONG_MAX + || local_index == LONG_MIN)) || (errno != 0 && local_index == 0)) + { + local_index = -1; + } + database->server->node_id = local_index; + } + mysql_free_result(result); + } + if (isjoined) server_set_status(database->server, SERVER_JOINED); else @@ -335,6 +368,7 @@ monitorMain(void *arg) { MYSQL_MONITOR *handle = (MYSQL_MONITOR *)arg; MONITOR_SERVERS *ptr; +long master_id; if (mysql_thread_init()) { @@ -347,6 +381,8 @@ MONITOR_SERVERS *ptr; handle->status = MONITOR_RUNNING; while (1) { + master_id = -1; + if (handle->shutdown) { handle->status = MONITOR_STOPPING; @@ -358,8 +394,39 @@ MONITOR_SERVERS *ptr; while (ptr) { monitorDatabase(ptr, handle->defaultUser, handle->defaultPasswd); + if (ptr->server->node_id >= 0 && SERVER_IS_JOINED(ptr->server)) { + if (ptr->server->node_id < master_id && master_id >= 0) { + master_id = ptr->server->node_id; + } else { + if (master_id < 0) { + master_id = ptr->server->node_id; + } + } + } else { + server_clear_status(ptr->server, SERVER_SLAVE); + server_clear_status(ptr->server, SERVER_MASTER); + } ptr = ptr->next; } + + ptr = handle->databases; + + /* this server loop sets Master and Slave roles */ + while (ptr) + { + if (ptr->server->node_id >= 0 && master_id >= 0) { + if (SERVER_IS_JOINED(ptr->server) && (ptr->server->node_id == master_id)) { + server_set_status(ptr->server, SERVER_MASTER); + server_clear_status(ptr->server, SERVER_SLAVE); + } else if (SERVER_IS_JOINED(ptr->server) && (ptr->server->node_id > master_id)) { + server_set_status(ptr->server, SERVER_SLAVE); + server_clear_status(ptr->server, SERVER_MASTER); + } + } + + ptr = ptr->next; + } + thread_millisleep(10000); } } From b17744fd46aae497b7a0f9e8824913f613b41c44 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Mon, 26 May 2014 14:07:04 +0200 Subject: [PATCH 2/5] Monitor Version Update Monitor Version Update --- server/modules/monitor/galera_mon.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/modules/monitor/galera_mon.c b/server/modules/monitor/galera_mon.c index 15d07eda9..7f74fedb6 100644 --- a/server/modules/monitor/galera_mon.c +++ b/server/modules/monitor/galera_mon.c @@ -47,7 +47,7 @@ extern int lm_enabled_logfiles_bitmask; static void monitorMain(void *); -static char *version_str = "V1.0.0"; +static char *version_str = "V1.1.0"; static void *startMonitor(void *); static void stopMonitor(void *); From 330c0f6bf7fc162245ccfb102e3e5eeb4cd0fef0 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Mon, 26 May 2014 14:13:01 +0200 Subject: [PATCH 3/5] MONITOR_INTERVAL default added MONITOR_INTERVAL default added --- server/modules/monitor/galera_mon.c | 10 +++++++++- server/modules/monitor/mysqlmon.h | 2 ++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/server/modules/monitor/galera_mon.c b/server/modules/monitor/galera_mon.c index 7f74fedb6..56b2c0d32 100644 --- a/server/modules/monitor/galera_mon.c +++ b/server/modules/monitor/galera_mon.c @@ -390,10 +390,15 @@ long master_id; handle->status = MONITOR_STOPPED; return; } + ptr = handle->databases; + while (ptr) { monitorDatabase(ptr, handle->defaultUser, handle->defaultPasswd); + + /* set master_id to the lowest value of ptr->server->node_id */ + if (ptr->server->node_id >= 0 && SERVER_IS_JOINED(ptr->server)) { if (ptr->server->node_id < master_id && master_id >= 0) { master_id = ptr->server->node_id; @@ -403,6 +408,7 @@ long master_id; } } } else { + /* clear M/S status */ server_clear_status(ptr->server, SERVER_SLAVE); server_clear_status(ptr->server, SERVER_MASTER); } @@ -415,10 +421,12 @@ long master_id; while (ptr) { if (ptr->server->node_id >= 0 && master_id >= 0) { + /* set the Master role */ if (SERVER_IS_JOINED(ptr->server) && (ptr->server->node_id == master_id)) { server_set_status(ptr->server, SERVER_MASTER); server_clear_status(ptr->server, SERVER_SLAVE); } else if (SERVER_IS_JOINED(ptr->server) && (ptr->server->node_id > master_id)) { + /* set the Slave role */ server_set_status(ptr->server, SERVER_SLAVE); server_clear_status(ptr->server, SERVER_MASTER); } @@ -427,6 +435,6 @@ long master_id; ptr = ptr->next; } - thread_millisleep(10000); + thread_millisleep(MONITOR_INTERVAL); } } diff --git a/server/modules/monitor/mysqlmon.h b/server/modules/monitor/mysqlmon.h index c2aacd21e..e06d095c8 100644 --- a/server/modules/monitor/mysqlmon.h +++ b/server/modules/monitor/mysqlmon.h @@ -61,4 +61,6 @@ typedef struct { #define MONITOR_STOPPING 2 #define MONITOR_STOPPED 3 +#define MONITOR_INTERVAL 10000 // in milliseconds + #endif From b54ca8e87f5f9b5d9a6b64302baba649ba0ad90e Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Mon, 26 May 2014 14:16:56 +0200 Subject: [PATCH 4/5] Default values for MONITOR_INTERVAL Default values for MONITOR_INTERVAL --- server/modules/monitor/mysqlmon.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/modules/monitor/mysqlmon.h b/server/modules/monitor/mysqlmon.h index e06d095c8..a2c2e364c 100644 --- a/server/modules/monitor/mysqlmon.h +++ b/server/modules/monitor/mysqlmon.h @@ -27,8 +27,9 @@ * @verbatim * Revision History * - * Date Who Description - * 08/07/13 Mark Riddoch Initial implementation + * Date Who Description + * 08/07/13 Mark Riddoch Initial implementation + * 26/05/14 Massimiliano Pinto Default values for MONITOR_INTERVAL * * @endverbatim */ From 933025b0176f33f44b77f0bd9d897eceb1212e5f Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Mon, 26 May 2014 16:49:10 +0200 Subject: [PATCH 5/5] Update Update --- server/core/server.c | 11 +++++++++-- server/include/server.h | 1 + 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/server/core/server.c b/server/core/server.c index 27ed23b75..1a81ddb6a 100644 --- a/server/core/server.c +++ b/server/core/server.c @@ -68,6 +68,7 @@ SERVER *server; server->nextdb = NULL; server->monuser = NULL; server->monpw = NULL; + server->server_string = NULL; server->node_id = -1; spinlock_acquire(&server_spin); @@ -111,6 +112,8 @@ SERVER *ptr; /* Clean up session and free the memory */ free(server->name); free(server->protocol); + if (server->server_string) + free(server->server_string); free(server); return 1; } @@ -199,9 +202,11 @@ char *stat; free(stat); dcb_printf(dcb, "\tProtocol: %s\n", ptr->protocol); dcb_printf(dcb, "\tPort: %d\n", ptr->port); + if (ptr->server_string) + dcb_printf(dcb, "\tServer Version:\t\t%s\n", ptr->server_string); dcb_printf(dcb, "\tNode Id: %d\n", ptr->node_id); dcb_printf(dcb, "\tNumber of connections: %d\n", ptr->stats.n_connections); - dcb_printf(dcb, "\tCurrent no. of connections: %d\n", ptr->stats.n_current); + dcb_printf(dcb, "\tCurrent no. of conns: %d\n", ptr->stats.n_current); ptr = ptr->next; } spinlock_release(&server_spin); @@ -225,9 +230,11 @@ char *stat; free(stat); dcb_printf(dcb, "\tProtocol: %s\n", server->protocol); dcb_printf(dcb, "\tPort: %d\n", server->port); + if (server->server_string) + dcb_printf(dcb, "\tServer Version:\t\t%s\n", server->server_string); dcb_printf(dcb, "\tNode Id: %d\n", server->node_id); dcb_printf(dcb, "\tNumber of connections: %d\n", server->stats.n_connections); - dcb_printf(dcb, "\tCurrent No. of connections: %d\n", server->stats.n_current); + dcb_printf(dcb, "\tCurrent No. of conns: %d\n", server->stats.n_current); } /** diff --git a/server/include/server.h b/server/include/server.h index 6e7b146d1..f6de85ab7 100644 --- a/server/include/server.h +++ b/server/include/server.h @@ -61,6 +61,7 @@ typedef struct server { SERVER_STATS stats; /**< The server statistics */ struct server *next; /**< Next server */ struct server *nextdb; /**< Next server in list attached to a service */ + char *server_string; /**< Server version string, i.e. MySQL server version */ long node_id; /**< Node id, server_id for M/S or local_index for Galera */ } SERVER;