From bf00ca8aba2ece20e0e93e25fd84a7c04ccc5b88 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Mon, 26 May 2014 10:29:34 +0200 Subject: [PATCH 1/8] GaleraHA support - R/W Splitting with Galera GaleraHA support - R/W Splitting with Galera --- server/core/server.c | 8 +++- server/include/server.h | 12 +++-- server/modules/monitor/galera_mon.c | 71 ++++++++++++++++++++++++++++- 3 files changed, 82 insertions(+), 9 deletions(-) diff --git a/server/core/server.c b/server/core/server.c index bf5b3174f..27ed23b75 100644 --- a/server/core/server.c +++ b/server/core/server.c @@ -22,8 +22,9 @@ * @verbatim * Revision History * - * Date Who Description - * 18/06/13 Mark Riddoch Initial implementation + * Date Who Description + * 18/06/13 Mark Riddoch Initial implementation + * 21/05/14 Massimiliano Pinto Addition of node_id * * @endverbatim */ @@ -67,6 +68,7 @@ SERVER *server; server->nextdb = NULL; server->monuser = NULL; server->monpw = NULL; + server->node_id = -1; spinlock_acquire(&server_spin); server->next = allServers; @@ -197,6 +199,7 @@ char *stat; free(stat); dcb_printf(dcb, "\tProtocol: %s\n", ptr->protocol); dcb_printf(dcb, "\tPort: %d\n", ptr->port); + dcb_printf(dcb, "\tNode Id: %d\n", ptr->node_id); dcb_printf(dcb, "\tNumber of connections: %d\n", ptr->stats.n_connections); dcb_printf(dcb, "\tCurrent no. of connections: %d\n", ptr->stats.n_current); ptr = ptr->next; @@ -222,6 +225,7 @@ char *stat; free(stat); dcb_printf(dcb, "\tProtocol: %s\n", server->protocol); dcb_printf(dcb, "\tPort: %d\n", server->port); + dcb_printf(dcb, "\tNode Id: %d\n", server->node_id); dcb_printf(dcb, "\tNumber of connections: %d\n", server->stats.n_connections); dcb_printf(dcb, "\tCurrent No. of connections: %d\n", server->stats.n_current); } diff --git a/server/include/server.h b/server/include/server.h index e355e8c49..6e7b146d1 100644 --- a/server/include/server.h +++ b/server/include/server.h @@ -27,10 +27,11 @@ * @verbatim * Revision History * - * Date Who Description - * 14/06/13 Mark Riddoch Initial implementation - * 21/06/13 Mark Riddoch Addition of server status flags - * 22/07/13 Mark Riddoch Addition of JOINED status for Galera + * Date Who Description + * 14/06/13 Mark Riddoch Initial implementation + * 21/06/13 Mark Riddoch Addition of server status flags + * 22/07/13 Mark Riddoch Addition of JOINED status for Galera + * 20/05/14 Massimiliano Pinto Addition of node_id field * * @endverbatim */ @@ -60,6 +61,7 @@ typedef struct server { SERVER_STATS stats; /**< The server statistics */ struct server *next; /**< Next server */ struct server *nextdb; /**< Next server in list attached to a service */ + long node_id; /**< Node id, server_id for M/S or local_index for Galera */ } SERVER; /** @@ -99,7 +101,7 @@ typedef struct server { * Is the server joined Galera node? The server must be running and joined. */ #define SERVER_IS_JOINED(server) \ - (((server)->status & (SERVER_RUNNING|SERVER_MASTER|SERVER_SLAVE|SERVER_JOINED)) == (SERVER_RUNNING|SERVER_JOINED)) + (((server)->status & (SERVER_RUNNING|SERVER_JOINED)) == (SERVER_RUNNING|SERVER_JOINED)) extern SERVER *server_alloc(char *, char *, unsigned short); extern int server_free(SERVER *); diff --git a/server/modules/monitor/galera_mon.c b/server/modules/monitor/galera_mon.c index 1f277dcbc..15d07eda9 100644 --- a/server/modules/monitor/galera_mon.c +++ b/server/modules/monitor/galera_mon.c @@ -22,8 +22,10 @@ * @verbatim * Revision History * - * Date Who Description - * 22/07/13 Mark Riddoch Initial implementation + * Date Who Description + * 22/07/13 Mark Riddoch Initial implementation + * 21/05/14 Massimiliano Pinto Monitor sets a master server + * that has the lowest value of wsrep_local_index * * @endverbatim */ @@ -280,6 +282,8 @@ MYSQL_RES *result; int num_fields; int isjoined = 0; char *uname = defaultUser, *passwd = defaultPasswd; +unsigned long int server_version = 0; +char *server_string; if (database->server->monuser != NULL) { @@ -297,6 +301,7 @@ char *uname = defaultUser, *passwd = defaultPasswd; uname, dpwd, NULL, database->server->port, NULL, 0) == NULL) { server_clear_status(database->server, SERVER_RUNNING); + database->server->node_id = -1; free(dpwd); return; } @@ -306,6 +311,15 @@ char *uname = defaultUser, *passwd = defaultPasswd; /* If we get this far then we have a working connection */ server_set_status(database->server, SERVER_RUNNING); + /* get server version from current server */ + server_version = mysql_get_server_version(database->con); + + /* get server version string */ + server_string = (char *)mysql_get_server_info(database->con); + if (server_string) { + database->server->server_string = strdup(server_string); + } + /* Check if the the Galera FSM shows this node is joined to the cluster */ if (mysql_query(database->con, "SHOW STATUS LIKE 'wsrep_local_state_comment'") == 0 && (result = mysql_store_result(database->con)) != NULL) @@ -319,6 +333,25 @@ char *uname = defaultUser, *passwd = defaultPasswd; mysql_free_result(result); } + /* Check the the Galera node index in the cluster */ + if (mysql_query(database->con, "SHOW STATUS LIKE 'wsrep_local_index'") == 0 + && (result = mysql_store_result(database->con)) != NULL) + { + long local_index = -1; + num_fields = mysql_num_fields(result); + while ((row = mysql_fetch_row(result))) + { + local_index = strtol(row[1], NULL, 10); + if ((errno == ERANGE && (local_index == LONG_MAX + || local_index == LONG_MIN)) || (errno != 0 && local_index == 0)) + { + local_index = -1; + } + database->server->node_id = local_index; + } + mysql_free_result(result); + } + if (isjoined) server_set_status(database->server, SERVER_JOINED); else @@ -335,6 +368,7 @@ monitorMain(void *arg) { MYSQL_MONITOR *handle = (MYSQL_MONITOR *)arg; MONITOR_SERVERS *ptr; +long master_id; if (mysql_thread_init()) { @@ -347,6 +381,8 @@ MONITOR_SERVERS *ptr; handle->status = MONITOR_RUNNING; while (1) { + master_id = -1; + if (handle->shutdown) { handle->status = MONITOR_STOPPING; @@ -358,8 +394,39 @@ MONITOR_SERVERS *ptr; while (ptr) { monitorDatabase(ptr, handle->defaultUser, handle->defaultPasswd); + if (ptr->server->node_id >= 0 && SERVER_IS_JOINED(ptr->server)) { + if (ptr->server->node_id < master_id && master_id >= 0) { + master_id = ptr->server->node_id; + } else { + if (master_id < 0) { + master_id = ptr->server->node_id; + } + } + } else { + server_clear_status(ptr->server, SERVER_SLAVE); + server_clear_status(ptr->server, SERVER_MASTER); + } ptr = ptr->next; } + + ptr = handle->databases; + + /* this server loop sets Master and Slave roles */ + while (ptr) + { + if (ptr->server->node_id >= 0 && master_id >= 0) { + if (SERVER_IS_JOINED(ptr->server) && (ptr->server->node_id == master_id)) { + server_set_status(ptr->server, SERVER_MASTER); + server_clear_status(ptr->server, SERVER_SLAVE); + } else if (SERVER_IS_JOINED(ptr->server) && (ptr->server->node_id > master_id)) { + server_set_status(ptr->server, SERVER_SLAVE); + server_clear_status(ptr->server, SERVER_MASTER); + } + } + + ptr = ptr->next; + } + thread_millisleep(10000); } } From b17744fd46aae497b7a0f9e8824913f613b41c44 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Mon, 26 May 2014 14:07:04 +0200 Subject: [PATCH 2/8] Monitor Version Update Monitor Version Update --- server/modules/monitor/galera_mon.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/modules/monitor/galera_mon.c b/server/modules/monitor/galera_mon.c index 15d07eda9..7f74fedb6 100644 --- a/server/modules/monitor/galera_mon.c +++ b/server/modules/monitor/galera_mon.c @@ -47,7 +47,7 @@ extern int lm_enabled_logfiles_bitmask; static void monitorMain(void *); -static char *version_str = "V1.0.0"; +static char *version_str = "V1.1.0"; static void *startMonitor(void *); static void stopMonitor(void *); From 330c0f6bf7fc162245ccfb102e3e5eeb4cd0fef0 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Mon, 26 May 2014 14:13:01 +0200 Subject: [PATCH 3/8] MONITOR_INTERVAL default added MONITOR_INTERVAL default added --- server/modules/monitor/galera_mon.c | 10 +++++++++- server/modules/monitor/mysqlmon.h | 2 ++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/server/modules/monitor/galera_mon.c b/server/modules/monitor/galera_mon.c index 7f74fedb6..56b2c0d32 100644 --- a/server/modules/monitor/galera_mon.c +++ b/server/modules/monitor/galera_mon.c @@ -390,10 +390,15 @@ long master_id; handle->status = MONITOR_STOPPED; return; } + ptr = handle->databases; + while (ptr) { monitorDatabase(ptr, handle->defaultUser, handle->defaultPasswd); + + /* set master_id to the lowest value of ptr->server->node_id */ + if (ptr->server->node_id >= 0 && SERVER_IS_JOINED(ptr->server)) { if (ptr->server->node_id < master_id && master_id >= 0) { master_id = ptr->server->node_id; @@ -403,6 +408,7 @@ long master_id; } } } else { + /* clear M/S status */ server_clear_status(ptr->server, SERVER_SLAVE); server_clear_status(ptr->server, SERVER_MASTER); } @@ -415,10 +421,12 @@ long master_id; while (ptr) { if (ptr->server->node_id >= 0 && master_id >= 0) { + /* set the Master role */ if (SERVER_IS_JOINED(ptr->server) && (ptr->server->node_id == master_id)) { server_set_status(ptr->server, SERVER_MASTER); server_clear_status(ptr->server, SERVER_SLAVE); } else if (SERVER_IS_JOINED(ptr->server) && (ptr->server->node_id > master_id)) { + /* set the Slave role */ server_set_status(ptr->server, SERVER_SLAVE); server_clear_status(ptr->server, SERVER_MASTER); } @@ -427,6 +435,6 @@ long master_id; ptr = ptr->next; } - thread_millisleep(10000); + thread_millisleep(MONITOR_INTERVAL); } } diff --git a/server/modules/monitor/mysqlmon.h b/server/modules/monitor/mysqlmon.h index c2aacd21e..e06d095c8 100644 --- a/server/modules/monitor/mysqlmon.h +++ b/server/modules/monitor/mysqlmon.h @@ -61,4 +61,6 @@ typedef struct { #define MONITOR_STOPPING 2 #define MONITOR_STOPPED 3 +#define MONITOR_INTERVAL 10000 // in milliseconds + #endif From b54ca8e87f5f9b5d9a6b64302baba649ba0ad90e Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Mon, 26 May 2014 14:16:56 +0200 Subject: [PATCH 4/8] Default values for MONITOR_INTERVAL Default values for MONITOR_INTERVAL --- server/modules/monitor/mysqlmon.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/modules/monitor/mysqlmon.h b/server/modules/monitor/mysqlmon.h index e06d095c8..a2c2e364c 100644 --- a/server/modules/monitor/mysqlmon.h +++ b/server/modules/monitor/mysqlmon.h @@ -27,8 +27,9 @@ * @verbatim * Revision History * - * Date Who Description - * 08/07/13 Mark Riddoch Initial implementation + * Date Who Description + * 08/07/13 Mark Riddoch Initial implementation + * 26/05/14 Massimiliano Pinto Default values for MONITOR_INTERVAL * * @endverbatim */ From 933025b0176f33f44b77f0bd9d897eceb1212e5f Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Mon, 26 May 2014 16:49:10 +0200 Subject: [PATCH 5/8] Update Update --- server/core/server.c | 11 +++++++++-- server/include/server.h | 1 + 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/server/core/server.c b/server/core/server.c index 27ed23b75..1a81ddb6a 100644 --- a/server/core/server.c +++ b/server/core/server.c @@ -68,6 +68,7 @@ SERVER *server; server->nextdb = NULL; server->monuser = NULL; server->monpw = NULL; + server->server_string = NULL; server->node_id = -1; spinlock_acquire(&server_spin); @@ -111,6 +112,8 @@ SERVER *ptr; /* Clean up session and free the memory */ free(server->name); free(server->protocol); + if (server->server_string) + free(server->server_string); free(server); return 1; } @@ -199,9 +202,11 @@ char *stat; free(stat); dcb_printf(dcb, "\tProtocol: %s\n", ptr->protocol); dcb_printf(dcb, "\tPort: %d\n", ptr->port); + if (ptr->server_string) + dcb_printf(dcb, "\tServer Version:\t\t%s\n", ptr->server_string); dcb_printf(dcb, "\tNode Id: %d\n", ptr->node_id); dcb_printf(dcb, "\tNumber of connections: %d\n", ptr->stats.n_connections); - dcb_printf(dcb, "\tCurrent no. of connections: %d\n", ptr->stats.n_current); + dcb_printf(dcb, "\tCurrent no. of conns: %d\n", ptr->stats.n_current); ptr = ptr->next; } spinlock_release(&server_spin); @@ -225,9 +230,11 @@ char *stat; free(stat); dcb_printf(dcb, "\tProtocol: %s\n", server->protocol); dcb_printf(dcb, "\tPort: %d\n", server->port); + if (server->server_string) + dcb_printf(dcb, "\tServer Version:\t\t%s\n", server->server_string); dcb_printf(dcb, "\tNode Id: %d\n", server->node_id); dcb_printf(dcb, "\tNumber of connections: %d\n", server->stats.n_connections); - dcb_printf(dcb, "\tCurrent No. of connections: %d\n", server->stats.n_current); + dcb_printf(dcb, "\tCurrent No. of conns: %d\n", server->stats.n_current); } /** diff --git a/server/include/server.h b/server/include/server.h index 6e7b146d1..f6de85ab7 100644 --- a/server/include/server.h +++ b/server/include/server.h @@ -61,6 +61,7 @@ typedef struct server { SERVER_STATS stats; /**< The server statistics */ struct server *next; /**< Next server */ struct server *nextdb; /**< Next server in list attached to a service */ + char *server_string; /**< Server version string, i.e. MySQL server version */ long node_id; /**< Node id, server_id for M/S or local_index for Galera */ } SERVER; From 70115d35c9899745e90b0876ade4fc600ba878b8 Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Tue, 27 May 2014 10:08:17 +0300 Subject: [PATCH 6/8] Fix to bug #438 http://bugs.skysql.com/show_bug.cgi?id=438 Fixed some compiler warnings, added header includes, return values etc. --- server/core/Makefile | 4 ++-- server/core/dbusers.c | 1 + server/core/dcb.c | 1 + server/core/poll.c | 1 + server/core/secrets.c | 1 + server/core/service.c | 2 +- server/modules/monitor/mysql_mon.c | 4 ++-- server/modules/protocol/mysql_backend.c | 7 ++++++- 8 files changed, 15 insertions(+), 6 deletions(-) diff --git a/server/core/Makefile b/server/core/Makefile index 3f862a475..bd29fbae8 100644 --- a/server/core/Makefile +++ b/server/core/Makefile @@ -41,7 +41,7 @@ UTILSPATH := $(ROOT_PATH)/utils CC=cc -CFLAGS=-c -I/usr/include -I../include -I../inih \ +CFLAGS=-c -I/usr/include -I../include -I../modules/include -I../inih \ $(MYSQL_HEADERS) \ -I$(LOGPATH) -I$(UTILSPATH) \ -Wall -g @@ -59,7 +59,7 @@ SRCS= atomic.c buffer.c spinlock.c gateway.c \ monitor.c adminusers.c secrets.c HDRS= ../include/atomic.h ../include/buffer.h ../include/dcb.h \ - ../include/gw.h ../include/mysql_protocol.h \ + ../include/gw.h ../modules/include/mysql_client_server_protocol.h \ ../include/session.h ../include/spinlock.h ../include/thread.h \ ../include/modules.h ../include/poll.h ../include/config.h \ ../include/users.h ../include/hashtable.h ../include/gwbitmask.h \ diff --git a/server/core/dbusers.c b/server/core/dbusers.c index 4847c4c8b..41dda5a92 100644 --- a/server/core/dbusers.c +++ b/server/core/dbusers.c @@ -42,6 +42,7 @@ #include #include #include +#include #define USERS_QUERY_NO_ROOT " AND user NOT IN ('root')" #define LOAD_MYSQL_USERS_QUERY "SELECT user, host, password, concat(user,host,password) AS userdata FROM mysql.user WHERE user IS NOT NULL AND user <> ''" diff --git a/server/core/dcb.c b/server/core/dcb.c index f2e5e183a..c499e067b 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -68,6 +68,7 @@ #include #include #include +#include extern int lm_enabled_logfiles_bitmask; diff --git a/server/core/poll.c b/server/core/poll.c index f1c65ebea..0b23a6b05 100644 --- a/server/core/poll.c +++ b/server/core/poll.c @@ -27,6 +27,7 @@ #include #include #include +#include extern int lm_enabled_logfiles_bitmask; diff --git a/server/core/secrets.c b/server/core/secrets.c index a11b0276d..c4d1822f5 100644 --- a/server/core/secrets.c +++ b/server/core/secrets.c @@ -21,6 +21,7 @@ #include #include #include +#include extern int lm_enabled_logfiles_bitmask; /** diff --git a/server/core/service.c b/server/core/service.c index 79522513c..efc441173 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -702,7 +702,7 @@ SERVICE *ptr; * @param dcb DCB to print data to * @param service The service to print */ -dprintService(DCB *dcb, SERVICE *service) +void dprintService(DCB *dcb, SERVICE *service) { SERVER *server = service->databases; diff --git a/server/modules/monitor/mysql_mon.c b/server/modules/monitor/mysql_mon.c index 4cf7e6938..554f3810f 100644 --- a/server/modules/monitor/mysql_mon.c +++ b/server/modules/monitor/mysql_mon.c @@ -128,7 +128,7 @@ MYSQL_MONITOR *handle; handle->defaultPasswd = NULL; spinlock_init(&handle->lock); } - handle->tid = thread_start(monitorMain, handle); + handle->tid = (THREAD)thread_start(monitorMain, handle); return handle; } @@ -143,7 +143,7 @@ stopMonitor(void *arg) MYSQL_MONITOR *handle = (MYSQL_MONITOR *)arg; handle->shutdown = 1; - thread_wait(handle->tid); + thread_wait((void *)handle->tid); } /** diff --git a/server/modules/protocol/mysql_backend.c b/server/modules/protocol/mysql_backend.c index 2ba8402b8..e4b461dad 100644 --- a/server/modules/protocol/mysql_backend.c +++ b/server/modules/protocol/mysql_backend.c @@ -283,6 +283,11 @@ static int gw_read_backend_event(DCB *dcb) { } if (backend_protocol->state == MYSQL_AUTH_FAILED) { + /** + * protocol state won't change anymore, + * lock can be freed + */ + spinlock_release(&dcb->authlock); spinlock_acquire(&dcb->delayqlock); /*< * vraa : errorHandle @@ -351,7 +356,7 @@ static int gw_read_backend_event(DCB *dcb) { /* close router_session */ router->closeSession(router_instance, rsession); rc = 1; - goto return_with_lock; + goto return_rc; } else { From 78a02f52972d814ace50b45a5c02a4732507f58c Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Wed, 28 May 2014 09:39:33 +0200 Subject: [PATCH 7/8] Compile warning fixed in galera_mon.c Compile warning fixed in galera_mon.c --- server/modules/monitor/galera_mon.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/modules/monitor/galera_mon.c b/server/modules/monitor/galera_mon.c index a00a61253..96b891d58 100644 --- a/server/modules/monitor/galera_mon.c +++ b/server/modules/monitor/galera_mon.c @@ -123,7 +123,7 @@ MYSQL_MONITOR *handle; handle->defaultPasswd = NULL; spinlock_init(&handle->lock); } - handle->tid = thread_start(monitorMain, handle); + handle->tid = (THREAD)thread_start(monitorMain, handle); return handle; } @@ -138,7 +138,7 @@ stopMonitor(void *arg) MYSQL_MONITOR *handle = (MYSQL_MONITOR *)arg; handle->shutdown = 1; - thread_wait(handle->tid); + thread_wait((void *)handle->tid); } /** From f985e1cac532fad8329cbb02c67da82df6387bb3 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Wed, 28 May 2014 10:43:01 +0200 Subject: [PATCH 8/8] server_id for MySQL replication set for each node server_id for MySQL replication is now set for each node and dprintServer* routines can print it as well --- server/core/server.c | 4 ++-- server/modules/monitor/mysql_mon.c | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/server/core/server.c b/server/core/server.c index 7358897fd..eff77c937 100644 --- a/server/core/server.c +++ b/server/core/server.c @@ -246,7 +246,7 @@ char *stat; dcb_printf(dcb, "\tPort: %d\n", ptr->port); if (ptr->server_string) dcb_printf(dcb, "\tServer Version:\t\t%s\n", ptr->server_string); - dcb_printf(dcb, "\tNode Id: %d\n", ptr->node_id); + dcb_printf(dcb, "\tNode Id: %d\n", ptr->node_id); dcb_printf(dcb, "\tNumber of connections: %d\n", ptr->stats.n_connections); dcb_printf(dcb, "\tCurrent no. of conns: %d\n", ptr->stats.n_current); ptr = ptr->next; @@ -274,7 +274,7 @@ char *stat; dcb_printf(dcb, "\tPort: %d\n", server->port); if (server->server_string) dcb_printf(dcb, "\tServer Version:\t\t%s\n", server->server_string); - dcb_printf(dcb, "\tNode Id: %d\n", server->node_id); + dcb_printf(dcb, "\tNode Id: %d\n", server->node_id); dcb_printf(dcb, "\tNumber of connections: %d\n", server->stats.n_connections); dcb_printf(dcb, "\tCurrent No. of conns: %d\n", server->stats.n_current); } diff --git a/server/modules/monitor/mysql_mon.c b/server/modules/monitor/mysql_mon.c index 554f3810f..ddf1f7cbc 100644 --- a/server/modules/monitor/mysql_mon.c +++ b/server/modules/monitor/mysql_mon.c @@ -334,6 +334,25 @@ char *server_string; database->server->server_string = strdup(server_string); } + /* get server_id form current node */ + if (mysql_query(database->con, "SELECT @@server_id") == 0 + && (result = mysql_store_result(database->con)) != NULL) + { + long server_id = -1; + num_fields = mysql_num_fields(result); + while ((row = mysql_fetch_row(result))) + { + server_id = strtol(row[0], NULL, 10); + if ((errno == ERANGE && (server_id == LONG_MAX + || server_id == LONG_MIN)) || (errno != 0 && server_id == 0)) + { + server_id = -1; + } + database->server->node_id = server_id; + } + mysql_free_result(result); + } + /* Check SHOW SLAVE HOSTS - if we get rows then we are a master */ if (mysql_query(database->con, "SHOW SLAVE HOSTS")) {