From bf00ca8aba2ece20e0e93e25fd84a7c04ccc5b88 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Mon, 26 May 2014 10:29:34 +0200 Subject: [PATCH 1/9] GaleraHA support - R/W Splitting with Galera GaleraHA support - R/W Splitting with Galera --- server/core/server.c | 8 +++- server/include/server.h | 12 +++-- server/modules/monitor/galera_mon.c | 71 ++++++++++++++++++++++++++++- 3 files changed, 82 insertions(+), 9 deletions(-) diff --git a/server/core/server.c b/server/core/server.c index bf5b3174f..27ed23b75 100644 --- a/server/core/server.c +++ b/server/core/server.c @@ -22,8 +22,9 @@ * @verbatim * Revision History * - * Date Who Description - * 18/06/13 Mark Riddoch Initial implementation + * Date Who Description + * 18/06/13 Mark Riddoch Initial implementation + * 21/05/14 Massimiliano Pinto Addition of node_id * * @endverbatim */ @@ -67,6 +68,7 @@ SERVER *server; server->nextdb = NULL; server->monuser = NULL; server->monpw = NULL; + server->node_id = -1; spinlock_acquire(&server_spin); server->next = allServers; @@ -197,6 +199,7 @@ char *stat; free(stat); dcb_printf(dcb, "\tProtocol: %s\n", ptr->protocol); dcb_printf(dcb, "\tPort: %d\n", ptr->port); + dcb_printf(dcb, "\tNode Id: %d\n", ptr->node_id); dcb_printf(dcb, "\tNumber of connections: %d\n", ptr->stats.n_connections); dcb_printf(dcb, "\tCurrent no. of connections: %d\n", ptr->stats.n_current); ptr = ptr->next; @@ -222,6 +225,7 @@ char *stat; free(stat); dcb_printf(dcb, "\tProtocol: %s\n", server->protocol); dcb_printf(dcb, "\tPort: %d\n", server->port); + dcb_printf(dcb, "\tNode Id: %d\n", server->node_id); dcb_printf(dcb, "\tNumber of connections: %d\n", server->stats.n_connections); dcb_printf(dcb, "\tCurrent No. of connections: %d\n", server->stats.n_current); } diff --git a/server/include/server.h b/server/include/server.h index e355e8c49..6e7b146d1 100644 --- a/server/include/server.h +++ b/server/include/server.h @@ -27,10 +27,11 @@ * @verbatim * Revision History * - * Date Who Description - * 14/06/13 Mark Riddoch Initial implementation - * 21/06/13 Mark Riddoch Addition of server status flags - * 22/07/13 Mark Riddoch Addition of JOINED status for Galera + * Date Who Description + * 14/06/13 Mark Riddoch Initial implementation + * 21/06/13 Mark Riddoch Addition of server status flags + * 22/07/13 Mark Riddoch Addition of JOINED status for Galera + * 20/05/14 Massimiliano Pinto Addition of node_id field * * @endverbatim */ @@ -60,6 +61,7 @@ typedef struct server { SERVER_STATS stats; /**< The server statistics */ struct server *next; /**< Next server */ struct server *nextdb; /**< Next server in list attached to a service */ + long node_id; /**< Node id, server_id for M/S or local_index for Galera */ } SERVER; /** @@ -99,7 +101,7 @@ typedef struct server { * Is the server joined Galera node? The server must be running and joined. */ #define SERVER_IS_JOINED(server) \ - (((server)->status & (SERVER_RUNNING|SERVER_MASTER|SERVER_SLAVE|SERVER_JOINED)) == (SERVER_RUNNING|SERVER_JOINED)) + (((server)->status & (SERVER_RUNNING|SERVER_JOINED)) == (SERVER_RUNNING|SERVER_JOINED)) extern SERVER *server_alloc(char *, char *, unsigned short); extern int server_free(SERVER *); diff --git a/server/modules/monitor/galera_mon.c b/server/modules/monitor/galera_mon.c index 1f277dcbc..15d07eda9 100644 --- a/server/modules/monitor/galera_mon.c +++ b/server/modules/monitor/galera_mon.c @@ -22,8 +22,10 @@ * @verbatim * Revision History * - * Date Who Description - * 22/07/13 Mark Riddoch Initial implementation + * Date Who Description + * 22/07/13 Mark Riddoch Initial implementation + * 21/05/14 Massimiliano Pinto Monitor sets a master server + * that has the lowest value of wsrep_local_index * * @endverbatim */ @@ -280,6 +282,8 @@ MYSQL_RES *result; int num_fields; int isjoined = 0; char *uname = defaultUser, *passwd = defaultPasswd; +unsigned long int server_version = 0; +char *server_string; if (database->server->monuser != NULL) { @@ -297,6 +301,7 @@ char *uname = defaultUser, *passwd = defaultPasswd; uname, dpwd, NULL, database->server->port, NULL, 0) == NULL) { server_clear_status(database->server, SERVER_RUNNING); + database->server->node_id = -1; free(dpwd); return; } @@ -306,6 +311,15 @@ char *uname = defaultUser, *passwd = defaultPasswd; /* If we get this far then we have a working connection */ server_set_status(database->server, SERVER_RUNNING); + /* get server version from current server */ + server_version = mysql_get_server_version(database->con); + + /* get server version string */ + server_string = (char *)mysql_get_server_info(database->con); + if (server_string) { + database->server->server_string = strdup(server_string); + } + /* Check if the the Galera FSM shows this node is joined to the cluster */ if (mysql_query(database->con, "SHOW STATUS LIKE 'wsrep_local_state_comment'") == 0 && (result = mysql_store_result(database->con)) != NULL) @@ -319,6 +333,25 @@ char *uname = defaultUser, *passwd = defaultPasswd; mysql_free_result(result); } + /* Check the the Galera node index in the cluster */ + if (mysql_query(database->con, "SHOW STATUS LIKE 'wsrep_local_index'") == 0 + && (result = mysql_store_result(database->con)) != NULL) + { + long local_index = -1; + num_fields = mysql_num_fields(result); + while ((row = mysql_fetch_row(result))) + { + local_index = strtol(row[1], NULL, 10); + if ((errno == ERANGE && (local_index == LONG_MAX + || local_index == LONG_MIN)) || (errno != 0 && local_index == 0)) + { + local_index = -1; + } + database->server->node_id = local_index; + } + mysql_free_result(result); + } + if (isjoined) server_set_status(database->server, SERVER_JOINED); else @@ -335,6 +368,7 @@ monitorMain(void *arg) { MYSQL_MONITOR *handle = (MYSQL_MONITOR *)arg; MONITOR_SERVERS *ptr; +long master_id; if (mysql_thread_init()) { @@ -347,6 +381,8 @@ MONITOR_SERVERS *ptr; handle->status = MONITOR_RUNNING; while (1) { + master_id = -1; + if (handle->shutdown) { handle->status = MONITOR_STOPPING; @@ -358,8 +394,39 @@ MONITOR_SERVERS *ptr; while (ptr) { monitorDatabase(ptr, handle->defaultUser, handle->defaultPasswd); + if (ptr->server->node_id >= 0 && SERVER_IS_JOINED(ptr->server)) { + if (ptr->server->node_id < master_id && master_id >= 0) { + master_id = ptr->server->node_id; + } else { + if (master_id < 0) { + master_id = ptr->server->node_id; + } + } + } else { + server_clear_status(ptr->server, SERVER_SLAVE); + server_clear_status(ptr->server, SERVER_MASTER); + } ptr = ptr->next; } + + ptr = handle->databases; + + /* this server loop sets Master and Slave roles */ + while (ptr) + { + if (ptr->server->node_id >= 0 && master_id >= 0) { + if (SERVER_IS_JOINED(ptr->server) && (ptr->server->node_id == master_id)) { + server_set_status(ptr->server, SERVER_MASTER); + server_clear_status(ptr->server, SERVER_SLAVE); + } else if (SERVER_IS_JOINED(ptr->server) && (ptr->server->node_id > master_id)) { + server_set_status(ptr->server, SERVER_SLAVE); + server_clear_status(ptr->server, SERVER_MASTER); + } + } + + ptr = ptr->next; + } + thread_millisleep(10000); } } From b17744fd46aae497b7a0f9e8824913f613b41c44 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Mon, 26 May 2014 14:07:04 +0200 Subject: [PATCH 2/9] Monitor Version Update Monitor Version Update --- server/modules/monitor/galera_mon.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/modules/monitor/galera_mon.c b/server/modules/monitor/galera_mon.c index 15d07eda9..7f74fedb6 100644 --- a/server/modules/monitor/galera_mon.c +++ b/server/modules/monitor/galera_mon.c @@ -47,7 +47,7 @@ extern int lm_enabled_logfiles_bitmask; static void monitorMain(void *); -static char *version_str = "V1.0.0"; +static char *version_str = "V1.1.0"; static void *startMonitor(void *); static void stopMonitor(void *); From 330c0f6bf7fc162245ccfb102e3e5eeb4cd0fef0 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Mon, 26 May 2014 14:13:01 +0200 Subject: [PATCH 3/9] MONITOR_INTERVAL default added MONITOR_INTERVAL default added --- server/modules/monitor/galera_mon.c | 10 +++++++++- server/modules/monitor/mysqlmon.h | 2 ++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/server/modules/monitor/galera_mon.c b/server/modules/monitor/galera_mon.c index 7f74fedb6..56b2c0d32 100644 --- a/server/modules/monitor/galera_mon.c +++ b/server/modules/monitor/galera_mon.c @@ -390,10 +390,15 @@ long master_id; handle->status = MONITOR_STOPPED; return; } + ptr = handle->databases; + while (ptr) { monitorDatabase(ptr, handle->defaultUser, handle->defaultPasswd); + + /* set master_id to the lowest value of ptr->server->node_id */ + if (ptr->server->node_id >= 0 && SERVER_IS_JOINED(ptr->server)) { if (ptr->server->node_id < master_id && master_id >= 0) { master_id = ptr->server->node_id; @@ -403,6 +408,7 @@ long master_id; } } } else { + /* clear M/S status */ server_clear_status(ptr->server, SERVER_SLAVE); server_clear_status(ptr->server, SERVER_MASTER); } @@ -415,10 +421,12 @@ long master_id; while (ptr) { if (ptr->server->node_id >= 0 && master_id >= 0) { + /* set the Master role */ if (SERVER_IS_JOINED(ptr->server) && (ptr->server->node_id == master_id)) { server_set_status(ptr->server, SERVER_MASTER); server_clear_status(ptr->server, SERVER_SLAVE); } else if (SERVER_IS_JOINED(ptr->server) && (ptr->server->node_id > master_id)) { + /* set the Slave role */ server_set_status(ptr->server, SERVER_SLAVE); server_clear_status(ptr->server, SERVER_MASTER); } @@ -427,6 +435,6 @@ long master_id; ptr = ptr->next; } - thread_millisleep(10000); + thread_millisleep(MONITOR_INTERVAL); } } diff --git a/server/modules/monitor/mysqlmon.h b/server/modules/monitor/mysqlmon.h index c2aacd21e..e06d095c8 100644 --- a/server/modules/monitor/mysqlmon.h +++ b/server/modules/monitor/mysqlmon.h @@ -61,4 +61,6 @@ typedef struct { #define MONITOR_STOPPING 2 #define MONITOR_STOPPED 3 +#define MONITOR_INTERVAL 10000 // in milliseconds + #endif From b54ca8e87f5f9b5d9a6b64302baba649ba0ad90e Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Mon, 26 May 2014 14:16:56 +0200 Subject: [PATCH 4/9] Default values for MONITOR_INTERVAL Default values for MONITOR_INTERVAL --- server/modules/monitor/mysqlmon.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/modules/monitor/mysqlmon.h b/server/modules/monitor/mysqlmon.h index e06d095c8..a2c2e364c 100644 --- a/server/modules/monitor/mysqlmon.h +++ b/server/modules/monitor/mysqlmon.h @@ -27,8 +27,9 @@ * @verbatim * Revision History * - * Date Who Description - * 08/07/13 Mark Riddoch Initial implementation + * Date Who Description + * 08/07/13 Mark Riddoch Initial implementation + * 26/05/14 Massimiliano Pinto Default values for MONITOR_INTERVAL * * @endverbatim */ From 933025b0176f33f44b77f0bd9d897eceb1212e5f Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Mon, 26 May 2014 16:49:10 +0200 Subject: [PATCH 5/9] Update Update --- server/core/server.c | 11 +++++++++-- server/include/server.h | 1 + 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/server/core/server.c b/server/core/server.c index 27ed23b75..1a81ddb6a 100644 --- a/server/core/server.c +++ b/server/core/server.c @@ -68,6 +68,7 @@ SERVER *server; server->nextdb = NULL; server->monuser = NULL; server->monpw = NULL; + server->server_string = NULL; server->node_id = -1; spinlock_acquire(&server_spin); @@ -111,6 +112,8 @@ SERVER *ptr; /* Clean up session and free the memory */ free(server->name); free(server->protocol); + if (server->server_string) + free(server->server_string); free(server); return 1; } @@ -199,9 +202,11 @@ char *stat; free(stat); dcb_printf(dcb, "\tProtocol: %s\n", ptr->protocol); dcb_printf(dcb, "\tPort: %d\n", ptr->port); + if (ptr->server_string) + dcb_printf(dcb, "\tServer Version:\t\t%s\n", ptr->server_string); dcb_printf(dcb, "\tNode Id: %d\n", ptr->node_id); dcb_printf(dcb, "\tNumber of connections: %d\n", ptr->stats.n_connections); - dcb_printf(dcb, "\tCurrent no. of connections: %d\n", ptr->stats.n_current); + dcb_printf(dcb, "\tCurrent no. of conns: %d\n", ptr->stats.n_current); ptr = ptr->next; } spinlock_release(&server_spin); @@ -225,9 +230,11 @@ char *stat; free(stat); dcb_printf(dcb, "\tProtocol: %s\n", server->protocol); dcb_printf(dcb, "\tPort: %d\n", server->port); + if (server->server_string) + dcb_printf(dcb, "\tServer Version:\t\t%s\n", server->server_string); dcb_printf(dcb, "\tNode Id: %d\n", server->node_id); dcb_printf(dcb, "\tNumber of connections: %d\n", server->stats.n_connections); - dcb_printf(dcb, "\tCurrent No. of connections: %d\n", server->stats.n_current); + dcb_printf(dcb, "\tCurrent No. of conns: %d\n", server->stats.n_current); } /** diff --git a/server/include/server.h b/server/include/server.h index 6e7b146d1..f6de85ab7 100644 --- a/server/include/server.h +++ b/server/include/server.h @@ -61,6 +61,7 @@ typedef struct server { SERVER_STATS stats; /**< The server statistics */ struct server *next; /**< Next server */ struct server *nextdb; /**< Next server in list attached to a service */ + char *server_string; /**< Server version string, i.e. MySQL server version */ long node_id; /**< Node id, server_id for M/S or local_index for Galera */ } SERVER; From 78a02f52972d814ace50b45a5c02a4732507f58c Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Wed, 28 May 2014 09:39:33 +0200 Subject: [PATCH 6/9] Compile warning fixed in galera_mon.c Compile warning fixed in galera_mon.c --- server/modules/monitor/galera_mon.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/modules/monitor/galera_mon.c b/server/modules/monitor/galera_mon.c index a00a61253..96b891d58 100644 --- a/server/modules/monitor/galera_mon.c +++ b/server/modules/monitor/galera_mon.c @@ -123,7 +123,7 @@ MYSQL_MONITOR *handle; handle->defaultPasswd = NULL; spinlock_init(&handle->lock); } - handle->tid = thread_start(monitorMain, handle); + handle->tid = (THREAD)thread_start(monitorMain, handle); return handle; } @@ -138,7 +138,7 @@ stopMonitor(void *arg) MYSQL_MONITOR *handle = (MYSQL_MONITOR *)arg; handle->shutdown = 1; - thread_wait(handle->tid); + thread_wait((void *)handle->tid); } /** From f985e1cac532fad8329cbb02c67da82df6387bb3 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Wed, 28 May 2014 10:43:01 +0200 Subject: [PATCH 7/9] server_id for MySQL replication set for each node server_id for MySQL replication is now set for each node and dprintServer* routines can print it as well --- server/core/server.c | 4 ++-- server/modules/monitor/mysql_mon.c | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/server/core/server.c b/server/core/server.c index 7358897fd..eff77c937 100644 --- a/server/core/server.c +++ b/server/core/server.c @@ -246,7 +246,7 @@ char *stat; dcb_printf(dcb, "\tPort: %d\n", ptr->port); if (ptr->server_string) dcb_printf(dcb, "\tServer Version:\t\t%s\n", ptr->server_string); - dcb_printf(dcb, "\tNode Id: %d\n", ptr->node_id); + dcb_printf(dcb, "\tNode Id: %d\n", ptr->node_id); dcb_printf(dcb, "\tNumber of connections: %d\n", ptr->stats.n_connections); dcb_printf(dcb, "\tCurrent no. of conns: %d\n", ptr->stats.n_current); ptr = ptr->next; @@ -274,7 +274,7 @@ char *stat; dcb_printf(dcb, "\tPort: %d\n", server->port); if (server->server_string) dcb_printf(dcb, "\tServer Version:\t\t%s\n", server->server_string); - dcb_printf(dcb, "\tNode Id: %d\n", server->node_id); + dcb_printf(dcb, "\tNode Id: %d\n", server->node_id); dcb_printf(dcb, "\tNumber of connections: %d\n", server->stats.n_connections); dcb_printf(dcb, "\tCurrent No. of conns: %d\n", server->stats.n_current); } diff --git a/server/modules/monitor/mysql_mon.c b/server/modules/monitor/mysql_mon.c index 554f3810f..ddf1f7cbc 100644 --- a/server/modules/monitor/mysql_mon.c +++ b/server/modules/monitor/mysql_mon.c @@ -334,6 +334,25 @@ char *server_string; database->server->server_string = strdup(server_string); } + /* get server_id form current node */ + if (mysql_query(database->con, "SELECT @@server_id") == 0 + && (result = mysql_store_result(database->con)) != NULL) + { + long server_id = -1; + num_fields = mysql_num_fields(result); + while ((row = mysql_fetch_row(result))) + { + server_id = strtol(row[0], NULL, 10); + if ((errno == ERANGE && (server_id == LONG_MAX + || server_id == LONG_MIN)) || (errno != 0 && server_id == 0)) + { + server_id = -1; + } + database->server->node_id = server_id; + } + mysql_free_result(result); + } + /* Check SHOW SLAVE HOSTS - if we get rows then we are a master */ if (mysql_query(database->con, "SHOW SLAVE HOSTS")) { From 509379df7047433fa83d2793f04a739a372fb18b Mon Sep 17 00:00:00 2001 From: Mark Riddoch Date: Wed, 28 May 2014 11:00:50 +0100 Subject: [PATCH 8/9] Fix for NULL buffer past to gwbuf_length --- server/core/buffer.c | 6 +++++- server/core/dcb.c | 22 +++++++++++++++++----- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/server/core/buffer.c b/server/core/buffer.c index b21cf216c..11fb5b556 100644 --- a/server/core/buffer.c +++ b/server/core/buffer.c @@ -284,7 +284,11 @@ unsigned int gwbuf_length(GWBUF *head) { int rval = 0; - CHK_GWBUF(head); + + if (head) + { + CHK_GWBUF(head); + } while (head) { rval += GWBUF_LENGTH(head); diff --git a/server/core/dcb.c b/server/core/dcb.c index f2e5e183a..3e7a50674 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -758,7 +758,10 @@ int below_water; * the routine that drains the queue data, so we should * not have a race condition on the event. */ - qlen = gwbuf_length(queue); + if (queue) + qlen = gwbuf_length(queue); + else + qlen = 0; atomic_add(&dcb->writeqlen, qlen); dcb->writeq = gwbuf_append(dcb->writeq, queue); dcb->stats.n_buffered++; @@ -872,7 +875,14 @@ int below_water; * for suspended write. */ dcb->writeq = queue; - qlen = gwbuf_length(queue); + if (queue) + { + qlen = gwbuf_length(queue); + } + else + { + qlen = 0; + } atomic_add(&dcb->writeqlen, qlen); if (queue != NULL) @@ -1075,7 +1085,8 @@ printDCB(DCB *dcb) printf("\tDCB state: %s\n", gw_dcb_state2string(dcb->state)); if (dcb->remote) printf("\tConnected to: %s\n", dcb->remote); - printf("\tQueued write data: %d\n", gwbuf_length(dcb->writeq)); + if (dcb->writeq) + printf("\tQueued write data: %d\n",gwbuf_length(dcb->writeq)); printf("\tStatistics:\n"); printf("\t\tNo. of Reads: %d\n", dcb->stats.n_reads); printf("\t\tNo. of Writes: %d\n", dcb->stats.n_writes); @@ -1150,7 +1161,8 @@ dprintDCB(DCB *pdcb, DCB *dcb) if (dcb->remote) dcb_printf(pdcb, "\tConnected to: %s\n", dcb->remote); dcb_printf(pdcb, "\tOwning Session: %d\n", dcb->session); - dcb_printf(pdcb, "\tQueued write data: %d\n", gwbuf_length(dcb->writeq)); + if (dcb->writeq) + dcb_printf(pdcb, "\tQueued write data: %d\n", gwbuf_length(dcb->writeq)); dcb_printf(pdcb, "\tStatistics:\n"); dcb_printf(pdcb, "\t\tNo. of Reads: %d\n", dcb->stats.n_reads); dcb_printf(pdcb, "\t\tNo. of Writes: %d\n", dcb->stats.n_writes); @@ -1552,7 +1564,7 @@ int rval = 1; /** * Remove a callback from the callback list for the DCB * - * Searches down the linked list to find he callback with a matching reason, function + * Searches down the linked list to find the callback with a matching reason, function * and userdata. * * @param dcb The DCB to add the callback to From a42e7b5702c3e51c1ba8cc739860df3fe529966d Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Wed, 28 May 2014 14:03:59 +0300 Subject: [PATCH 9/9] Bug #438, http://bugs.skysql.com/show_bug.cgi?id=438 try to complete the fix. dcb->authlock was double-freed. --- server/modules/protocol/mysql_backend.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/modules/protocol/mysql_backend.c b/server/modules/protocol/mysql_backend.c index e4b461dad..fff9a2604 100644 --- a/server/modules/protocol/mysql_backend.c +++ b/server/modules/protocol/mysql_backend.c @@ -326,7 +326,7 @@ static int gw_read_backend_event(DCB *dcb) { if (session->client->session == NULL) { rc = 1; - goto return_with_lock; + goto return_rc; } usleep(1); }