From 6dc75d4b9c4313e8b0e507dd4c0b6bec8dadc268 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Mon, 12 Sep 2016 05:06:42 +0300 Subject: [PATCH 01/36] MXS-860: Detect whether replication is configured The `detect_stale_slave` functionality used to only work when MaxScale had the knowledge that a master server has existed and that replication was working at some point in time. This might be a "safe" way to do it in regards to staleness of the data but in practice it is preferrable to always allow slave to be used for reads. This change adds the missing functionality to the monitor by assigning slave status to all servers which are configured as replication slaves when no master can be found. The new member variable that was added to the SERVER should be removed in 2.1 where the server_info offers the same functionalty without "polluting" the SERVER type. --- Documentation/Monitors/MySQL-Monitor.md | 4 ---- .../Release-Notes/MaxScale-2.0.1-Release-Notes.md | 1 + server/core/server.c | 1 + server/include/server.h | 2 ++ server/modules/monitor/mysql_mon.c | 14 ++++++++++++++ 5 files changed, 18 insertions(+), 4 deletions(-) diff --git a/Documentation/Monitors/MySQL-Monitor.md b/Documentation/Monitors/MySQL-Monitor.md index e0941b561..2b825e5d8 100644 --- a/Documentation/Monitors/MySQL-Monitor.md +++ b/Documentation/Monitors/MySQL-Monitor.md @@ -78,10 +78,6 @@ With this parameter, slaves that have lost their master but have been slaves of a master server can retain their slave status even without a master. This means that when a slave loses its master, it can still be used for reads. -If MaxScale loses the connection to the slave, the slave will lose the stale -slave state because MaxScale doesn't know if the slave has had recent contact -with the master server. - If this feature is disabled, a server is considered a valid slave if and only if it has a running master server monitored by this monitor. diff --git a/Documentation/Release-Notes/MaxScale-2.0.1-Release-Notes.md b/Documentation/Release-Notes/MaxScale-2.0.1-Release-Notes.md index fe6675efc..195bacf53 100644 --- a/Documentation/Release-Notes/MaxScale-2.0.1-Release-Notes.md +++ b/Documentation/Release-Notes/MaxScale-2.0.1-Release-Notes.md @@ -120,6 +120,7 @@ Please consult * [MXS-845](https://jira.mariadb.org/browse/MXS-845): "Server down" event is re-triggered after maintenance mode is repeated * [MXS-842](https://jira.mariadb.org/browse/MXS-842): Unexpected / undocumented behaviour when multiple available masters from mmmon monitor * [MXS-846](https://jira.mariadb.org/browse/MXS-846): MMMon: Maintenance mode on slave logs error message every second +* [MXS-860](https://jira.mariadb.org/browse/MXS-860): I want to access the web site if master server is down. ## Known Issues and Limitations diff --git a/server/core/server.c b/server/core/server.c index c4ef6426b..a29198791 100644 --- a/server/core/server.c +++ b/server/core/server.c @@ -88,6 +88,7 @@ server_alloc(char *servname, char *protocol, unsigned short port) server->persistmax = 0; server->persistmaxtime = 0; server->persistpoolmax = 0; + server->slave_configured = false; spinlock_init(&server->persistlock); spinlock_acquire(&server_spin); diff --git a/server/include/server.h b/server/include/server.h index 81a6a2313..5dfe79cfc 100644 --- a/server/include/server.h +++ b/server/include/server.h @@ -102,6 +102,8 @@ typedef struct server int depth; /**< Replication level in the tree */ long slaves[MAX_NUM_SLAVES]; /**< Slaves of this node */ bool master_err_is_logged; /*< If node failed, this indicates whether it is logged */ + bool slave_configured; /**< Server is configured as a replication slave + * TODO: Remove this for 2.1 */ DCB *persistent; /**< List of unused persistent connections to the server */ SPINLOCK persistlock; /**< Lock for adjusting the persistent connections list */ long persistpoolmax; /**< Maximum size of persistent connections pool */ diff --git a/server/modules/monitor/mysql_mon.c b/server/modules/monitor/mysql_mon.c index 6948175c1..4305505c2 100644 --- a/server/modules/monitor/mysql_mon.c +++ b/server/modules/monitor/mysql_mon.c @@ -323,8 +323,11 @@ static inline void monitor_mysql100_db(MONITOR_SERVERS* database) return; } + database->server->slave_configured = false; + while ((row = mysql_fetch_row(result))) { + database->server->slave_configured = true; /* get Slave_IO_Running and Slave_SQL_Running values*/ if (strncmp(row[12], "Yes", 3) == 0 && strncmp(row[13], "Yes", 3) == 0) @@ -407,8 +410,11 @@ static inline void monitor_mysql55_db(MONITOR_SERVERS* database) return; } + database->server->slave_configured = false; + while ((row = mysql_fetch_row(result))) { + database->server->slave_configured = true; /* get Slave_IO_Running and Slave_SQL_Running values*/ if (strncmp(row[10], "Yes", 3) == 0 && strncmp(row[11], "Yes", 3) == 0) @@ -479,8 +485,11 @@ static inline void monitor_mysql51_db(MONITOR_SERVERS* database) return; } + database->server->slave_configured = false; + while ((row = mysql_fetch_row(result))) { + database->server->slave_configured = true; /* get Slave_IO_Running and Slave_SQL_Running values*/ if (strncmp(row[10], "Yes", 3) == 0 && strncmp(row[11], "Yes", 3) == 0) @@ -994,6 +1003,11 @@ monitorMain(void *arg) { ptr->pending_status |= SERVER_SLAVE; } + else if (root_master == NULL && ptr->server->slave_configured) + { + /** TODO: Change this in 2.1 to use the server_info mechanism */ + ptr->pending_status |= SERVER_SLAVE; + } } ptr->server->status = ptr->pending_status; From 029e6574da1ace10f367a7848cdfb607f7f7ba56 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Mon, 12 Sep 2016 15:35:15 +0300 Subject: [PATCH 02/36] MXS-812: Always reset counters when backend is closed The active operation counters are now closed every time a backend referece is taken out of use. This should fix a few debug assertions that were hit in tests. --- server/modules/routing/readwritesplit/readwritesplit.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index f6a60f788..dd2b8a7e7 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -3135,6 +3135,10 @@ static bool select_connect_backend_servers(backend_ref_t **p_master_ref, ss_dassert(backend_ref[i].bref_backend->backend_conn_count > 0); /** disconnect opened connections */ + if (BREF_IS_WAITING_RESULT(&backend_ref[i])) + { + bref_clear_state(&backend_ref[i], BREF_WAITING_RESULT); + } bref_clear_state(&backend_ref[i], BREF_IN_USE); /** Decrease backend's connection counter. */ atomic_add(&backend_ref[i].bref_backend->backend_conn_count, -1); @@ -4299,6 +4303,10 @@ static void handleError(ROUTER *instance, void *router_session, CHK_BACKEND_REF(bref); bref_clear_state(bref, BREF_IN_USE); bref_set_state(bref, BREF_CLOSED); + if (BREF_IS_WAITING_RESULT(bref)) + { + bref_clear_state(bref, BREF_WAITING_RESULT); + } } else { From 7bd0b19b59dfa77a6b72a5f1202e7d2ad94964c3 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Thu, 15 Sep 2016 23:12:50 +0300 Subject: [PATCH 03/36] Update MaxScale version number --- VERSION.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION.cmake b/VERSION.cmake index f7abd4919..0693bfd8b 100644 --- a/VERSION.cmake +++ b/VERSION.cmake @@ -5,10 +5,10 @@ set(MAXSCALE_VERSION_MAJOR "2" CACHE STRING "Major version") set(MAXSCALE_VERSION_MINOR "0" CACHE STRING "Minor version") -set(MAXSCALE_VERSION_PATCH "0" CACHE STRING "Patch version") +set(MAXSCALE_VERSION_PATCH "1" CACHE STRING "Patch version") # This should only be incremented if a package is rebuilt set(MAXSCALE_BUILD_NUMBER 1 CACHE STRING "Release number") set(MAXSCALE_VERSION_NUMERIC "${MAXSCALE_VERSION_MAJOR}.${MAXSCALE_VERSION_MINOR}.${MAXSCALE_VERSION_PATCH}") -set(MAXSCALE_VERSION "beta-${MAXSCALE_VERSION_MAJOR}.${MAXSCALE_VERSION_MINOR}.${MAXSCALE_VERSION_PATCH}") +set(MAXSCALE_VERSION "${MAXSCALE_VERSION_MAJOR}.${MAXSCALE_VERSION_MINOR}.${MAXSCALE_VERSION_PATCH}") From 2a4addc298261dad2d0e52789ad345a2a2a1db9c Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 16 Sep 2016 01:13:52 +0300 Subject: [PATCH 04/36] Clear waiting results flag on client errors When a backend causes an error and it should be sent to the client, the backend reference was closed but the waiting results state was not cleared. This caused a debug assertion to be hit. --- server/modules/routing/readwritesplit/readwritesplit.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index dd2b8a7e7..07b768e8b 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -4395,6 +4395,10 @@ static void handle_error_reply_client(SESSION *ses, ROUTER_CLIENT_SES *rses, CHK_BACKEND_REF(bref); bref_clear_state(bref, BREF_IN_USE); bref_set_state(bref, BREF_CLOSED); + if (BREF_IS_WAITING_RESULT(bref)) + { + bref_clear_state(bref, BREF_WAITING_RESULT); + } } if (sesstate == SESSION_STATE_ROUTER_READY) From 89f9f4a42f0ecfbb40ad2ce8ee8dbf3e64168525 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 16 Sep 2016 01:11:44 +0300 Subject: [PATCH 05/36] Lock writeq before inspecting it Looking at the contents of the writeq should be done under a spinlock otherwise it is possible that another thread grabs the queue. --- server/modules/protocol/mysql_backend.c | 50 ++++++++++--------------- 1 file changed, 20 insertions(+), 30 deletions(-) diff --git a/server/modules/protocol/mysql_backend.c b/server/modules/protocol/mysql_backend.c index 178d36c61..5addc27b8 100644 --- a/server/modules/protocol/mysql_backend.c +++ b/server/modules/protocol/mysql_backend.c @@ -1184,48 +1184,38 @@ static int gw_write_backend_event(DCB *dcb) */ if (dcb->state != DCB_STATE_POLLING) { - uint8_t* data; + uint8_t* data = NULL; + bool com_quit = false; - if (dcb->writeq != NULL) + spinlock_acquire(&dcb->writeqlock); + if (dcb->writeq) { data = (uint8_t *) GWBUF_DATA(dcb->writeq); + com_quit = MYSQL_IS_COM_QUIT(data); + rc = 0; + } + spinlock_release(&dcb->writeqlock); - if (dcb->session->client_dcb == NULL) - { - rc = 0; - } - else if (!(MYSQL_IS_COM_QUIT(data))) - { - /*< vraa : errorHandle */ - mysql_send_custom_error(dcb->session->client_dcb, - 1, - 0, - "Writing to backend failed due invalid Maxscale " - "state."); - MXS_DEBUG("%lu [gw_write_backend_event] Write to backend " - "dcb %p fd %d " - "failed due invalid state %s.", - pthread_self(), - dcb, - dcb->fd, - STRDCBSTATE(dcb->state)); - MXS_ERROR("Attempt to write buffered data to backend " - "failed " - "due internal inconsistent state."); + if (data && !com_quit) + { + mysql_send_custom_error(dcb->session->client_dcb, 1, 0, + "Writing to backend failed due invalid Maxscale state."); + MXS_DEBUG("%lu [gw_write_backend_event] Write to backend " + "dcb %p fd %d failed due invalid state %s.", + pthread_self(), dcb, dcb->fd, STRDCBSTATE(dcb->state)); - rc = 0; - } + MXS_ERROR("Attempt to write buffered data to backend " + "failed due internal inconsistent state."); } else { MXS_DEBUG("%lu [gw_write_backend_event] Dcb %p in state %s " - "but there's nothing to write either.", - pthread_self(), - dcb, - STRDCBSTATE(dcb->state)); + "but there's nothing to write either.", + pthread_self(), dcb, STRDCBSTATE(dcb->state)); rc = 1; } + goto return_rc; } From 7fc7249349623d2c63176a84726afe7bdcc7c73e Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Thu, 15 Sep 2016 21:28:29 +0300 Subject: [PATCH 06/36] Update installation instructions --- .../MariaDB-MaxScale-Installation-Guide.md | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/Documentation/Getting-Started/MariaDB-MaxScale-Installation-Guide.md b/Documentation/Getting-Started/MariaDB-MaxScale-Installation-Guide.md index 75f1412ae..3a1ad5c20 100644 --- a/Documentation/Getting-Started/MariaDB-MaxScale-Installation-Guide.md +++ b/Documentation/Getting-Started/MariaDB-MaxScale-Installation-Guide.md @@ -6,21 +6,33 @@ In this introduction to MariaDB MaxScale the aim is to take the reader from the ## Installation -The simplest way to install MariaDB MaxScale is to use one of the binary packages that are available for download from the MariaDB website. +MariaDB MaxScale can be installed either using the MariaDB Enterprise Repository or directly from a downloaded package. -* Simply go to [http://www.mariadb.com/my_portal/download](http://www.mariadb.com/my_portal/download) +### Using the MariaDB Enterprise Repository -* Sign in to MariaDB.com +* Go to [https://mariadb.com/my_portal/download](https://mariadb.com/my_portal/download). -* Follow the instructions at the top of the page. +* Sign in or create an account for you. -![image alt text](images/getting_started.png) +* Select your operating system and follow the instructions. -If you want to install only MariaDB MaxScale, further down you will find the product specific download pages. Click on the MariaDB MaxScale link and follow the distribution specific instructions. +### From a Downloaded Package -![image alt text](images/getting_started2.png) +The MaxScale package can be downloaded from the following locations: -After you have installed MariaDB MaxScale, you can start it. +* [https://mariadb.com/my_portal/download/maxscale](https://mariadb.com/my_portal/download/maxscale) + +* [https://mariadb.com/downloads/maxscale](https://mariadb.com/downloads/maxscale) + +Select your operating system and download the package. + +Once you have downloaded the package, install it using the package tool relevant for your operating system. + +### Starting MariaDB MaxScale + +Before starting MariaDB MaxScale, you need to create a configuration file for it; please see further [down](#configuring-mariadb-maxscale). + +Once a configuration file has been created you can start MariaDB MaxScale: ``` systemctl start maxscale.service From 92ef33327edbd95f3908850f31ed777ce225b429 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Mon, 19 Sep 2016 05:34:03 +0300 Subject: [PATCH 07/36] MXS-870: Handle non-contiguous session command responses Session command responses with multiple packets could be spread across multiple, non-contiguous buffers. If a buffer contained a complete packet and some extra data but it wasn't contiguous, the debug assertion in gwbuf_clone_portion would fail. With release builds, it would cause eventual out-of-bounds memory access when the response would be sent to the client. --- server/core/buffer.c | 6 +++--- server/core/test/testbuffer.c | 9 --------- server/include/buffer.h | 1 - server/modules/protocol/mysql_backend.c | 23 +++++++++-------------- 4 files changed, 12 insertions(+), 27 deletions(-) diff --git a/server/core/buffer.c b/server/core/buffer.c index ce4b794b9..b2dfa4c72 100644 --- a/server/core/buffer.c +++ b/server/core/buffer.c @@ -384,9 +384,9 @@ GWBUF* gwbuf_clone_all(GWBUF* buf) } -GWBUF *gwbuf_clone_portion(GWBUF *buf, - size_t start_offset, - size_t length) +static GWBUF *gwbuf_clone_portion(GWBUF *buf, + size_t start_offset, + size_t length) { GWBUF* clonebuf; diff --git a/server/core/test/testbuffer.c b/server/core/test/testbuffer.c index 8ec09d6d3..ad26f0a5d 100644 --- a/server/core/test/testbuffer.c +++ b/server/core/test/testbuffer.c @@ -375,15 +375,6 @@ test1() gwbuf_free(clone); ss_dfprintf(stderr, "Freed cloned buffer"); ss_dfprintf(stderr, "\t..done\n"); - partclone = gwbuf_clone_portion(buffer, 25, 50); - buflen = GWBUF_LENGTH(partclone); - ss_dfprintf(stderr, "Part cloned buffer length is now %d", buflen); - ss_info_dassert(50 == buflen, "Incorrect buffer size"); - ss_info_dassert(0 == GWBUF_EMPTY(partclone), "Part cloned buffer should not be empty"); - ss_dfprintf(stderr, "\t..done\n"); - gwbuf_free(partclone); - ss_dfprintf(stderr, "Freed part cloned buffer"); - ss_dfprintf(stderr, "\t..done\n"); buffer = gwbuf_consume(buffer, bite1); ss_info_dassert(NULL != buffer, "Buffer should not be null"); buflen = GWBUF_LENGTH(buffer); diff --git a/server/include/buffer.h b/server/include/buffer.h index 42d0aa016..2a55f4963 100644 --- a/server/include/buffer.h +++ b/server/include/buffer.h @@ -194,7 +194,6 @@ extern unsigned int gwbuf_length(GWBUF *head); extern int gwbuf_count(GWBUF *head); extern size_t gwbuf_copy_data(GWBUF *buffer, size_t offset, size_t bytes, uint8_t* dest); -extern GWBUF *gwbuf_clone_portion(GWBUF *head, size_t offset, size_t len); extern GWBUF *gwbuf_split(GWBUF **buf, size_t length); extern GWBUF *gwbuf_clone_transform(GWBUF *head, gwbuf_type_t type); extern GWBUF *gwbuf_clone_all(GWBUF* head); diff --git a/server/modules/protocol/mysql_backend.c b/server/modules/protocol/mysql_backend.c index 5addc27b8..2a9bbbb7d 100644 --- a/server/modules/protocol/mysql_backend.c +++ b/server/modules/protocol/mysql_backend.c @@ -2011,22 +2011,17 @@ static GWBUF* process_response_data(DCB* dcb, outbuf = gwbuf_append(outbuf, readbuf); readbuf = NULL; } - /** - * Packet was read. There should be more since bytes were - * left over. - * Move the next packet to its own buffer and add that next - * to the prev packet's buffer. - */ - else /*< nbytes_left < nbytes_to_process */ + /** + * Buffer contains more data than we need. Split the complete packet and + * the extra data into two separate buffers. + */ + else { - ss_dassert(nbytes_left >= 0); - nbytes_to_process -= nbytes_left; - - /** Move the prefix of the buffer to outbuf from redbuf */ - outbuf = gwbuf_append(outbuf, - gwbuf_clone_portion(readbuf, 0, (size_t) nbytes_left)); - readbuf = gwbuf_consume(readbuf, (size_t) nbytes_left); + ss_dassert(nbytes_left < nbytes_to_process); + ss_dassert(nbytes_left > 0); ss_dassert(npackets_left > 0); + outbuf = gwbuf_append(outbuf, gwbuf_split(&readbuf, nbytes_left)); + nbytes_to_process -= nbytes_left; npackets_left -= 1; nbytes_left = 0; } From ec42413db8f591d40e47308bd29168776cfc734b Mon Sep 17 00:00:00 2001 From: counterpoint Date: Wed, 13 May 2015 08:57:32 +0100 Subject: [PATCH 08/36] Initial changes to implement test before creating maxscale_schema. --- server/modules/monitor/mysqlmon/mysql_mon.c | 41 ++++++++++++--------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/server/modules/monitor/mysqlmon/mysql_mon.c b/server/modules/monitor/mysqlmon/mysql_mon.c index c3f332d18..e5e5db637 100644 --- a/server/modules/monitor/mysqlmon/mysql_mon.c +++ b/server/modules/monitor/mysqlmon/mysql_mon.c @@ -1441,27 +1441,34 @@ static void set_master_heartbeat(MYSQL_MONITOR *handle, MONITOR_SERVERS *databas } /* create the maxscale_schema database */ - if (mysql_query(database->con, "CREATE DATABASE IF NOT EXISTS maxscale_schema")) - { - MXS_ERROR("[mysql_mon]: Error creating maxscale_schema database in Master server" - ": %s", mysql_error(database->con)); - + if (mysql_query(database->con, "SELECT table_name FROM information_schema.tables WHERE table_schema = 'maxscale_schema' AND table_name = 'replication_heartbeat'")) { + MXS_ERROR( "[mysql_mon]: Error checking for replication_heartbeat in Master server" + ": %s", mysql_error(database->con)); database->server->rlag = -1; } - - /* create repl_heartbeat table in maxscale_schema database */ - if (mysql_query(database->con, "CREATE TABLE IF NOT EXISTS " - "maxscale_schema.replication_heartbeat " - "(maxscale_id INT NOT NULL, " - "master_server_id INT NOT NULL, " - "master_timestamp INT UNSIGNED NOT NULL, " - "PRIMARY KEY ( master_server_id, maxscale_id ) ) " - "ENGINE=MYISAM DEFAULT CHARSET=latin1")) + + if (0 == mysql_num_rows(mysql_store_result(database->con))) { - MXS_ERROR("[mysql_mon]: Error creating maxscale_schema.replication_heartbeat " - "table in Master server: %s", mysql_error(database->con)); + if (mysql_query(database->con, "CREATE DATABASE IF NOT EXISTS maxscale_schema")) { + MXS_ERROR( "[mysql_mon]: Error creating maxscale_schema database in Master server" + ": %s", mysql_error(database->con)); + database->server->rlag = -1; + } - database->server->rlag = -1; + /* create repl_heartbeat table in maxscale_schema database */ + if (mysql_query(database->con, "CREATE TABLE IF NOT EXISTS " + "maxscale_schema.replication_heartbeat " + "(maxscale_id INT NOT NULL, " + "master_server_id INT NOT NULL, " + "master_timestamp INT UNSIGNED NOT NULL, " + "PRIMARY KEY ( master_server_id, maxscale_id ) ) " + "ENGINE=MYISAM DEFAULT CHARSET=latin1")) + { + MXS_ERROR("[mysql_mon]: Error creating maxscale_schema.replication_heartbeat " + "table in Master server: %s", mysql_error(database->con)); + + database->server->rlag = -1; + } } /* auto purge old values after 48 hours*/ From 73c974c2864e90c9505af6f8f8ecc369a3ab6d20 Mon Sep 17 00:00:00 2001 From: counterpoint Date: Wed, 13 May 2015 11:09:39 +0100 Subject: [PATCH 09/36] Remove automatic create database - requires too powerful database user. --- server/modules/monitor/mysqlmon/mysql_mon.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/server/modules/monitor/mysqlmon/mysql_mon.c b/server/modules/monitor/mysqlmon/mysql_mon.c index e5e5db637..98130fd19 100644 --- a/server/modules/monitor/mysqlmon/mysql_mon.c +++ b/server/modules/monitor/mysqlmon/mysql_mon.c @@ -1449,12 +1449,6 @@ static void set_master_heartbeat(MYSQL_MONITOR *handle, MONITOR_SERVERS *databas if (0 == mysql_num_rows(mysql_store_result(database->con))) { - if (mysql_query(database->con, "CREATE DATABASE IF NOT EXISTS maxscale_schema")) { - MXS_ERROR( "[mysql_mon]: Error creating maxscale_schema database in Master server" - ": %s", mysql_error(database->con)); - database->server->rlag = -1; - } - /* create repl_heartbeat table in maxscale_schema database */ if (mysql_query(database->con, "CREATE TABLE IF NOT EXISTS " "maxscale_schema.replication_heartbeat " From 48456833da2a78468ebbad479d1c2fe0e199f13e Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 17 Mar 2016 16:43:12 +0100 Subject: [PATCH 10/36] fixed memleak and potential call of mysql_num_rows with NULL --- server/modules/monitor/mysqlmon/mysql_mon.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/server/modules/monitor/mysqlmon/mysql_mon.c b/server/modules/monitor/mysqlmon/mysql_mon.c index 98130fd19..4e30c5cfe 100644 --- a/server/modules/monitor/mysqlmon/mysql_mon.c +++ b/server/modules/monitor/mysqlmon/mysql_mon.c @@ -1433,6 +1433,8 @@ static void set_master_heartbeat(MYSQL_MONITOR *handle, MONITOR_SERVERS *databas time_t purge_time; char heartbeat_insert_query[512] = ""; char heartbeat_purge_query[512] = ""; + MYSQL_RES *result; + long returned_rows; if (handle->master == NULL) { @@ -1440,14 +1442,26 @@ static void set_master_heartbeat(MYSQL_MONITOR *handle, MONITOR_SERVERS *databas return; } - /* create the maxscale_schema database */ + /* check if the maxscale_schema database and replication_heartbeat table exist */ if (mysql_query(database->con, "SELECT table_name FROM information_schema.tables WHERE table_schema = 'maxscale_schema' AND table_name = 'replication_heartbeat'")) { MXS_ERROR( "[mysql_mon]: Error checking for replication_heartbeat in Master server" ": %s", mysql_error(database->con)); database->server->rlag = -1; } - - if (0 == mysql_num_rows(mysql_store_result(database->con))) + + result = mysql_store_result(database->con); + + if (result == NULL) + { + returned_rows = 0; + } + else + { + returned_rows = mysql_num_rows(result); + mysql_free_result(result); + } + + if (0 == returned_rows) { /* create repl_heartbeat table in maxscale_schema database */ if (mysql_query(database->con, "CREATE TABLE IF NOT EXISTS " From e849297fc8090d82f092ca3f13596e3ec539e5fd Mon Sep 17 00:00:00 2001 From: TheTuxKeeper Date: Sun, 3 Apr 2016 19:35:03 +0200 Subject: [PATCH 11/36] line length now less than 110 characters --- server/modules/monitor/mysqlmon/mysql_mon.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/modules/monitor/mysqlmon/mysql_mon.c b/server/modules/monitor/mysqlmon/mysql_mon.c index 4e30c5cfe..c171699f9 100644 --- a/server/modules/monitor/mysqlmon/mysql_mon.c +++ b/server/modules/monitor/mysqlmon/mysql_mon.c @@ -1443,7 +1443,10 @@ static void set_master_heartbeat(MYSQL_MONITOR *handle, MONITOR_SERVERS *databas } /* check if the maxscale_schema database and replication_heartbeat table exist */ - if (mysql_query(database->con, "SELECT table_name FROM information_schema.tables WHERE table_schema = 'maxscale_schema' AND table_name = 'replication_heartbeat'")) { + if (mysql_query(database->con, + "SELECT table_name FROM information_schema.tables " + "WHERE table_schema = 'maxscale_schema' AND table_name = 'replication_heartbeat'")) + { MXS_ERROR( "[mysql_mon]: Error checking for replication_heartbeat in Master server" ": %s", mysql_error(database->con)); database->server->rlag = -1; From 60955ba70d55bf201551ac0d47ae12c0028e1966 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 13 Sep 2016 09:34:38 +0300 Subject: [PATCH 12/36] Clean up mysqlmon after pull request merge The pull request introduced some minor whitespace errors. --- server/modules/monitor/mysqlmon/mysql_mon.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/server/modules/monitor/mysqlmon/mysql_mon.c b/server/modules/monitor/mysqlmon/mysql_mon.c index c171699f9..b59777054 100644 --- a/server/modules/monitor/mysqlmon/mysql_mon.c +++ b/server/modules/monitor/mysqlmon/mysql_mon.c @@ -1443,17 +1443,16 @@ static void set_master_heartbeat(MYSQL_MONITOR *handle, MONITOR_SERVERS *databas } /* check if the maxscale_schema database and replication_heartbeat table exist */ - if (mysql_query(database->con, - "SELECT table_name FROM information_schema.tables " - "WHERE table_schema = 'maxscale_schema' AND table_name = 'replication_heartbeat'")) + if (mysql_query(database->con, "SELECT table_name FROM information_schema.tables " + "WHERE table_schema = 'maxscale_schema' AND table_name = 'replication_heartbeat'")) { MXS_ERROR( "[mysql_mon]: Error checking for replication_heartbeat in Master server" - ": %s", mysql_error(database->con)); + ": %s", mysql_error(database->con)); database->server->rlag = -1; } - + result = mysql_store_result(database->con); - + if (result == NULL) { returned_rows = 0; @@ -1463,7 +1462,7 @@ static void set_master_heartbeat(MYSQL_MONITOR *handle, MONITOR_SERVERS *databas returned_rows = mysql_num_rows(result); mysql_free_result(result); } - + if (0 == returned_rows) { /* create repl_heartbeat table in maxscale_schema database */ @@ -1476,7 +1475,7 @@ static void set_master_heartbeat(MYSQL_MONITOR *handle, MONITOR_SERVERS *databas "ENGINE=MYISAM DEFAULT CHARSET=latin1")) { MXS_ERROR("[mysql_mon]: Error creating maxscale_schema.replication_heartbeat " - "table in Master server: %s", mysql_error(database->con)); + "table in Master server: %s", mysql_error(database->con)); database->server->rlag = -1; } From e4543881d231cd93fc09e7b1cd081715636fa098 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Tue, 20 Sep 2016 09:53:27 +0300 Subject: [PATCH 13/36] Add instructions for installing a package --- .../MariaDB-MaxScale-Installation-Guide.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Documentation/Getting-Started/MariaDB-MaxScale-Installation-Guide.md b/Documentation/Getting-Started/MariaDB-MaxScale-Installation-Guide.md index 3a1ad5c20..a8e0f7bd2 100644 --- a/Documentation/Getting-Started/MariaDB-MaxScale-Installation-Guide.md +++ b/Documentation/Getting-Started/MariaDB-MaxScale-Installation-Guide.md @@ -26,7 +26,17 @@ The MaxScale package can be downloaded from the following locations: Select your operating system and download the package. -Once you have downloaded the package, install it using the package tool relevant for your operating system. +Depending on your OS, the package will either be a _deb_ or an _rpm_. + +An _rpm_ is installed as follows +``` +$ sudo yum install path-to-maxscale-package.rpm +``` +and a _deb_ as follows +``` +$ sudo dpkg -i path-to-maxscale-package.deb +$ sudo apt-get install -f +``` ### Starting MariaDB MaxScale From 35d4be14d2a86a4f6e9d2f86673f2c781ac08499 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 13 Sep 2016 21:22:36 +0300 Subject: [PATCH 14/36] Make service and monitor permissions checks optional MaxScale shouldn't require the service and monitor user checks. It makes sense to disable the checks to speed up the startup process when the user knows that the permissions are OK. --- .../Getting-Started/Configuration-Guide.md | 38 +++++++++++++++++++ server/core/config.c | 5 +++ server/core/dbusers.c | 3 +- server/core/monitor.c | 5 +++ server/include/maxconfig.h | 1 + 5 files changed, 51 insertions(+), 1 deletion(-) diff --git a/Documentation/Getting-Started/Configuration-Guide.md b/Documentation/Getting-Started/Configuration-Guide.md index 33ea2ab46..cd8b50456 100644 --- a/Documentation/Getting-Started/Configuration-Guide.md +++ b/Documentation/Getting-Started/Configuration-Guide.md @@ -95,14 +95,26 @@ It should be noted that additional threads will be created to execute other inte The connection timeout in seconds for the MySQL connections to the backend server when user authentication data is fetched. Increasing the value of this parameter will cause MariaDB MaxScale to wait longer for a response from the backend server before aborting the authentication process. The default is 3 seconds. +``` +auth_connect_timeout=10 +``` + #### `auth_read_timeout` The read timeout in seconds for the MySQL connection to the backend database when user authentication data is fetched. Increasing the value of this parameter will cause MariaDB MaxScale to wait longer for a response from the backend server when user data is being actively fetched. If the authentication is failing and you either have a large number of database users and grants or the connection to the backend servers is slow, it is a good idea to increase this value. The default is 1 second. +``` +auth_read_timeout=10 +``` + #### `auth_write_timeout` The write timeout in seconds for the MySQL connection to the backend database when user authentication data is fetched. Currently MariaDB MaxScale does not write or modify the data in the backend server. The default is 2 seconds. +``` +auth_write_timeout=10 +``` + #### `ms_timestamp` Enable or disable the high precision timestamps in logfiles. Enabling this adds millisecond precision to all logfile timestamps. @@ -113,10 +125,28 @@ Enable or disable the high precision timestamps in logfiles. Enabling this adds ms_timestamp=1 ``` +#### `skip_permission_checks` + +Skip service and monitor user permission checks. This is useful when you know +the permissions are OK and you want to speed up the startup process. This +parameter takes a boolean value and is disabled by default. + +It is recommended to not disable the permission checks so that any missing +privileges are detected when maxscale is starting up. If you are experiencing a +slow startup of MaxScale due to large amounts of connection timeouts when +permissions are checked, disabling the permission checks could speed up the +startup process. + +``` +skip_permission_checks=true +``` + #### `syslog` + Enable or disable the logging of messages to *syslog*. By default logging to *syslog* is enabled. + ``` # Valid options are: # syslog=<0|1> @@ -127,9 +157,11 @@ To enable logging to syslog use the value 1 and to disable use the value 0. #### `maxlog` + Enable to disable to logging of messages to MariaDB MaxScale's log file. By default logging to *maxlog* is enabled. + ``` # Valid options are: # syslog=<0|1> @@ -140,6 +172,7 @@ To enable logging to the MariaDB MaxScale log file use the value 1 and to disable use the value 0. #### `log_to_shm` + Enable or disable the writing of the *maxscale.log* file to shared memory. If enabled, then the actual log file will be created under `/dev/shm` and a symbolic link to that file will be created in the *MaxScale* log directory. @@ -169,6 +202,7 @@ To enable logging to shared memory use the value 1 and to disable use the value 0. #### `log_warning` + Enable or disable the logging of messages whose syslog priority is *warning*. Messages of this priority are enabled by default. @@ -181,6 +215,7 @@ log_warning=0 To disable these messages use the value 0 and to enable them use the value 1. #### `log_notice` + Enable or disable the logging of messages whose syslog priority is *notice*. Messages of this priority provide information about the functioning of MariaDB MaxScale and are enabled by default. @@ -267,10 +302,12 @@ times in one second, the logging of that error is suppressed for the following 10 seconds. To disable log throttling, add an entry with an empty value + ``` log_throttling= ``` or one where any of the integers is 0. + ``` log_throttling=0, 0, 0 ``` @@ -358,6 +395,7 @@ An integer argument taking the following values: query_classifier=qc_sqlite query_classifier_args=log_unrecognized_statements=1 ``` + This will log all statements that cannot be parsed completely. This may be useful if you suspect that MariaDB MaxScale routes statements to the wrong server (e.g. to a slave instead of to a master). diff --git a/server/core/config.c b/server/core/config.c index b1cbb41e8..baaec3596 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -957,6 +957,10 @@ handle_global_item(const char *name, const char *value) { mxs_log_set_highprecision_enabled(config_truth_value((char*)value)); } + else if (strcmp(name, "skip_permission_checks") == 0) + { + gateway.skip_permission_checks = config_truth_value((char*)value); + } else if (strcmp(name, "auth_connect_timeout") == 0) { char* endptr; @@ -1301,6 +1305,7 @@ global_defaults() gateway.auth_conn_timeout = DEFAULT_AUTH_CONNECT_TIMEOUT; gateway.auth_read_timeout = DEFAULT_AUTH_READ_TIMEOUT; gateway.auth_write_timeout = DEFAULT_AUTH_WRITE_TIMEOUT; + gateway.skip_permission_checks = false; if (version_string != NULL) { gateway.version_string = MXS_STRDUP_A(version_string); diff --git a/server/core/dbusers.c b/server/core/dbusers.c index d49a9698e..7927e1a90 100644 --- a/server/core/dbusers.c +++ b/server/core/dbusers.c @@ -2689,7 +2689,8 @@ static bool check_server_permissions(SERVICE *service, SERVER* server, */ bool check_service_permissions(SERVICE* service) { - if (is_internal_service(service->routerModule)) + if (is_internal_service(service->routerModule) || + config_get_global_options()->skip_permission_checks) { return true; } diff --git a/server/core/monitor.c b/server/core/monitor.c index 3741c3071..e2143b319 100644 --- a/server/core/monitor.c +++ b/server/core/monitor.c @@ -543,6 +543,11 @@ bool check_monitor_permissions(MONITOR* monitor, const char* query) return false; } + if (config_get_global_options()->skip_permission_checks) + { + return true; + } + char *user = monitor->user; char *dpasswd = decryptPassword(monitor->password); GATEWAY_CONF* cnf = config_get_global_options(); diff --git a/server/include/maxconfig.h b/server/include/maxconfig.h index 2e7970657..684af2376 100644 --- a/server/include/maxconfig.h +++ b/server/include/maxconfig.h @@ -121,6 +121,7 @@ typedef struct unsigned int auth_conn_timeout; /**< Connection timeout for the user authentication */ unsigned int auth_read_timeout; /**< Read timeout for the user authentication */ unsigned int auth_write_timeout; /**< Write timeout for the user authentication */ + bool skip_permission_checks; /**< Skip service and monitor permission checks */ char qc_name[PATH_MAX]; /**< The name of the query classifier to load */ char* qc_args; /**< Arguments for the query classifier */ } GATEWAY_CONF; From 1001654987b20fee97f71a2199f7a8ab2d0d5619 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Wed, 14 Sep 2016 09:48:13 +0300 Subject: [PATCH 15/36] Add utility scripts to make installation easier The `create_grants` scripts allow users to be easily "copied" to MaxScale. It queries the backend for grants for all users and converts them into similar grants for the MaxScale host. The `create_roles.sql` is a small set of queries which creates two utility roles, `proxy_authenticator` and `proxy_monitor`. These roles can be assigned to the actual service and monitor users with a single grant command. --- CMakeLists.txt | 2 + script/create_grants | 84 +++++++++++++++++++++++++++++++++++++++++ script/create_roles.sql | 7 ++++ 3 files changed, 93 insertions(+) create mode 100755 script/create_grants create mode 100644 script/create_roles.sql diff --git a/CMakeLists.txt b/CMakeLists.txt index 9119f5ad7..9d0773e25 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -205,6 +205,8 @@ install_file(${CMAKE_BINARY_DIR}/ReleaseNotes.txt core) install_file(${CMAKE_BINARY_DIR}/UpgradingToMaxScale12.txt core) install_file(server/maxscale.cnf.template core) install_file(server/maxscale_binlogserver_template.cnf core) +install_program(script/create_grants core) +install_file(script/create_roles.sql core) # Install the template into /etc if(WITH_MAXSCALE_CNF AND (NOT TARGET_COMPONENT OR "core" STREQUAL "${TARGET_COMPONENT}")) diff --git a/script/create_grants b/script/create_grants new file mode 100755 index 000000000..8dce0518d --- /dev/null +++ b/script/create_grants @@ -0,0 +1,84 @@ +#!/bin/bash + +# Copyright (c) 2016 MariaDB Corporation Ab +# +# Use of this software is governed by the Business Source License included +# in the LICENSE.TXT file and at www.mariadb.com/bsl. +# +# Change Date: 2019-07-01 +# +# On the date above, in accordance with the Business Source License, use +# of this software will be governed by version 2 or later of the General +# Public License. + +function runQuery(){ + mysql -s -s -h "$host" -P "$port" -u "$user" -p"$password" -e "$1" + if [ $? -ne 0 ] + then + echo "Failed to execute query: $1" + exit + fi +} + +# Transform grants to from external hosts to MaxScale's host +function getGrants(){ + result=$(runQuery "show grants for $1"|sed -e "s/@[^ ]*/@'$maxscalehost'/" -e "s/ *IDENTIFIED BY.*//" -e "s/$/;/") + echo "$result" +} + +user=$(whoami) +host=$(hostname) +port=3306 +include_root="and user <> 'root'" + +if [ "$1" == "--help" ] || [ $# -eq 0 ] +then + echo "Transform grants from original host to this host" + echo "" + echo "This script queries the backend database for a list of grants and outputs " + echo "copies of them with the hostnames replaced with the current server's hostname." + echo "The value of the hostname is the same you would get by doing a 'SELECT USER()'" + echo "query from this server." + echo "" + echo "Usage: $0 -u USER -p PASSWORD -h HOST -P PORT [-r]" + echo "-u USER Username" + echo "-p PASSWORD Password" + echo "-h HOST Database address ($host)" + echo "-P PORT Database port ($port)" + echo "-r Include root user in the grants" + exit +fi + +while getopts "u:p:h:P:r" var +do + case $var in + u) + user=$OPTARG + ;; + + p) + password=$OPTARG + ;; + + h) + host=$OPTARG + ;; + + P) + port=$OPTARG + ;; + r) + include_root="" + ;; + esac +done + +# Get the MaxScale hostname from the backend server +maxscalehost=$(runQuery "select user()") +maxscalehost=${maxscalehost#*@} + +# List all the users +runQuery "select concat(\"'\", user, \"'\", '@', \"'\", host, \"'\") from mysql.user where user <> '' and host <> '%' $include_root"|while read i +do + getGrants "$i" +done diff --git a/script/create_roles.sql b/script/create_roles.sql new file mode 100644 index 000000000..98d473c1c --- /dev/null +++ b/script/create_roles.sql @@ -0,0 +1,7 @@ +CREATE ROLE proxy_authenticator; +GRANT SELECT ON mysql.user TO proxy_authenticator; +GRANT SELECT ON mysql.db TO proxy_authenticator; +GRANT SELECT ON mysql.tables_priv TO proxy_authenticator; +GRANT SHOW DATABASES ON *.* TO proxy_authenticator; +CREATE ROLE proxy_monitor; +GRANT REPLICATION CLIENT ON *.* TO proxy_monitor; From 7e2a21de9e3156e5423fd114d7a1b1ff7629be50 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Wed, 14 Sep 2016 13:26:05 +0300 Subject: [PATCH 16/36] Cache: Assume we need to handle one packet at a time --- server/modules/filter/cache/cache.c | 80 +++++++------------ .../storage/storage_rocksdb/rocksdbstorage.cc | 3 +- .../storage_rocksdb/storage_rocksdb.cc | 14 ++++ 3 files changed, 43 insertions(+), 54 deletions(-) diff --git a/server/modules/filter/cache/cache.c b/server/modules/filter/cache/cache.c index 314891261..8b1de1434 100644 --- a/server/modules/filter/cache/cache.c +++ b/server/modules/filter/cache/cache.c @@ -364,84 +364,58 @@ static int routeQuery(FILTER *instance, void *sdata, GWBUF *packets) if (csdata->packets) { - C_DEBUG("Old packets exist."); gwbuf_append(csdata->packets, packets); } else { - C_DEBUG("NO old packets exist."); csdata->packets = packets; } - packets = modutil_get_complete_packets(&csdata->packets); + GWBUF *packet = modutil_get_next_MySQL_packet(&csdata->packets); int rv; - if (packets) + if (packet) { - C_DEBUG("At least one complete packet exist."); - GWBUF *packet; + bool use_default = true; - // TODO: Is it really possible to get more that one packet - // TODO: is this loop? If so, can those packets be sent - // TODO: after one and other, or do we need to wait for - // TODO: a replies? If there are more complete packets - // TODO: than one, then either CACHE_SESSION_DATA::key - // TODO: needs to be a queue - - // TODO: modutil_get_next_MySQL_packet *copies* the data. - while ((packet = modutil_get_next_MySQL_packet(&packets))) + // TODO: This returns the wrong result if GWBUF_LENGTH(packet) is < 5. + if (modutil_is_SQL(packet)) { - C_DEBUG("Processing packet."); - bool use_default = true; + packet = gwbuf_make_contiguous(packet); - if (modutil_is_SQL(packet)) + // We do not care whether the query was fully parsed or not. + // If a query cannot be fully parsed, the worst thing that can + // happen is that caching is not used, even though it would be + // possible. + + if (qc_get_operation(packet) == QUERY_OP_SELECT) { - C_DEBUG("Is SQL."); - // We do not care whether the query was fully parsed or not. - // If a query cannot be fully parsed, the worst thing that can - // happen is that caching is not used, even though it would be - // possible. + GWBUF *result; + use_default = !route_using_cache(cinstance, csdata, packet, &result); - if (qc_get_operation(packet) == QUERY_OP_SELECT) + if (!use_default) { - C_DEBUG("Is a SELECT"); + C_DEBUG("Using data from cache."); + gwbuf_free(packet); + DCB *dcb = csdata->session->client_dcb; - GWBUF *result; - use_default = !route_using_cache(cinstance, csdata, packet, &result); - - if (!use_default) - { - C_DEBUG("Using data from cache."); - gwbuf_free(packet); - DCB *dcb = csdata->session->client_dcb; - - // TODO: This is not ok. Any filters before this filter, will not - // TODO: see this data. - rv = dcb->func.write(dcb, result); - } - } - else - { - C_DEBUG("Is NOT a SELECT"); + // TODO: This is not ok. Any filters before this filter, will not + // TODO: see this data. + rv = dcb->func.write(dcb, result); } } - else - { - C_DEBUG("Is NOT SQL."); - } + } - if (use_default) - { - C_DEBUG("Using default processing."); - rv = csdata->down.routeQuery(csdata->down.instance, csdata->down.session, packet); - } + if (use_default) + { + C_DEBUG("Using default processing."); + rv = csdata->down.routeQuery(csdata->down.instance, csdata->down.session, packet); } } else { - C_DEBUG("Not even one complete packet exist; more data needed."); - // Ok, we need more data before we can do something. + // We need more data before we can do something. rv = 1; } diff --git a/server/modules/filter/cache/storage/storage_rocksdb/rocksdbstorage.cc b/server/modules/filter/cache/storage/storage_rocksdb/rocksdbstorage.cc index 96e049f07..84d593257 100644 --- a/server/modules/filter/cache/storage/storage_rocksdb/rocksdbstorage.cc +++ b/server/modules/filter/cache/storage/storage_rocksdb/rocksdbstorage.cc @@ -168,6 +168,7 @@ cache_result_t RocksDBStorage::getValue(const char* pKey, GWBUF** ppResult) else { MXS_NOTICE("Cache item is stale, not using."); + result = CACHE_RESULT_NOT_FOUND; } } else @@ -190,7 +191,7 @@ cache_result_t RocksDBStorage::getValue(const char* pKey, GWBUF** ppResult) cache_result_t RocksDBStorage::putValue(const char* pKey, const GWBUF* pValue) { - // ss_dassert(gwbuf_is_contiguous(pValue)); + ss_dassert(GWBUF_IS_CONTIGUOUS(pValue)); rocksdb::Slice key(pKey, ROCKSDB_KEY_LENGTH); rocksdb::Slice value(static_cast(GWBUF_DATA(pValue)), GWBUF_LENGTH(pValue)); diff --git a/server/modules/filter/cache/storage/storage_rocksdb/storage_rocksdb.cc b/server/modules/filter/cache/storage/storage_rocksdb/storage_rocksdb.cc index 98390d8a4..efa285450 100644 --- a/server/modules/filter/cache/storage/storage_rocksdb/storage_rocksdb.cc +++ b/server/modules/filter/cache/storage/storage_rocksdb/storage_rocksdb.cc @@ -25,6 +25,8 @@ bool initialize() CACHE_STORAGE* createInstance(const char* zName, uint32_t ttl, int argc, char* argv[]) { + ss_dassert(zName); + CACHE_STORAGE* pStorage = 0; try @@ -56,6 +58,10 @@ cache_result_t getKey(CACHE_STORAGE* pStorage, const GWBUF* pQuery, char* pKey) { + ss_dassert(pStorage); + ss_dassert(pQuery); + ss_dassert(pKey); + cache_result_t result = CACHE_RESULT_ERROR; try @@ -80,6 +86,10 @@ cache_result_t getKey(CACHE_STORAGE* pStorage, cache_result_t getValue(CACHE_STORAGE* pStorage, const char* pKey, GWBUF** ppResult) { + ss_dassert(pStorage); + ss_dassert(pKey); + ss_dassert(ppResult); + cache_result_t result = CACHE_RESULT_ERROR; try @@ -106,6 +116,10 @@ cache_result_t putValue(CACHE_STORAGE* pStorage, const char* pKey, const GWBUF* pValue) { + ss_dassert(pStorage); + ss_dassert(pKey); + ss_dassert(pValue); + cache_result_t result = CACHE_RESULT_ERROR; try From 4d1eb6fe85f82ff28cd5129c53d91085cdc8f3ea Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Mon, 19 Sep 2016 13:30:05 +0300 Subject: [PATCH 17/36] cache: Process response to COM_QUERY When a query has been sent to a backend, the response is now processed to the extent that the cache is capable of figuring out how many rows are being returned, so that the cache setting `max_resultset_rows` can be processed. The code is now also written in such a manner that it should be insensitive to how a package has been split up into a chain of GWBUFs. --- Documentation/Filters/Cache.md | 15 +- server/modules/filter/cache/cache.c | 540 +++++++++++++++++++++++----- server/modules/filter/cache/cache.h | 33 ++ 3 files changed, 501 insertions(+), 87 deletions(-) create mode 100644 server/modules/filter/cache/cache.h diff --git a/Documentation/Filters/Cache.md b/Documentation/Filters/Cache.md index d0e0b5e70..bdd5fa555 100644 --- a/Documentation/Filters/Cache.md +++ b/Documentation/Filters/Cache.md @@ -87,15 +87,26 @@ The setting can be changed to `any`, provided fully qualified names are always used or if the names of tables in different databases are different. +#### `max_resultset_rows` + +Specifies the maximum number of rows a resultset can have in order to be +stored in the cache. A resultset larger than this, will not be stored. +``` +max_resultset_rows=1000 +``` +Zero or a negative value is interpreted as no limitation. + +The default value is `-1`. + #### `max_resultset_size` Specifies the maximum size a resultset can have, measured in kibibytes, in order to be stored in the cache. A resultset larger than this, will not be stored. ``` -max_resultset_size=64 +max_resultset_size=128 ``` -The default value is TBD. +The default value is 64. #### `ttl` diff --git a/server/modules/filter/cache/cache.c b/server/modules/filter/cache/cache.c index 8b1de1434..4fc9b17ec 100644 --- a/server/modules/filter/cache/cache.c +++ b/server/modules/filter/cache/cache.c @@ -17,23 +17,13 @@ #include #include #include +#include #include +#include "cache.h" #include "storage.h" static char VERSION_STRING[] = "V1.0.0"; -typedef enum cache_references -{ - CACHE_REFERENCES_ANY, - CACHE_REFERENCES_QUALIFIED -} cache_references_t; - -#define DEFAULT_ALLOWED_REFERENCES CACHE_REFERENCES_QUALIFIED -// Bytes -#define DEFAULT_MAX_RESULTSET_SIZE 64 * 1024 -// Seconds -#define DEFAULT_TTL 10 - static FILTER *createInstance(const char *name, char **options, FILTER_PARAMETER **); static void *newSession(FILTER *instance, SESSION *session); static void closeSession(FILTER *instance, void *sdata); @@ -101,12 +91,23 @@ FILTER_OBJECT *GetModuleObject() typedef struct cache_config { cache_references_t allowed_references; + uint32_t max_resultset_rows; uint32_t max_resultset_size; const char *storage; const char *storage_args; uint32_t ttl; } CACHE_CONFIG; +static const CACHE_CONFIG DEFAULT_CONFIG = +{ + CACHE_DEFAULT_ALLOWED_REFERENCES, + CACHE_DEFAULT_MAX_RESULTSET_ROWS, + CACHE_DEFAULT_MAX_RESULTSET_SIZE, + NULL, + NULL, + CACHE_DEFAULT_TTL +}; + typedef struct cache_instance { const char *name; @@ -115,31 +116,59 @@ typedef struct cache_instance CACHE_STORAGE *storage; } CACHE_INSTANCE; -static const CACHE_CONFIG DEFAULT_CONFIG = +typedef enum cache_session_state { - DEFAULT_ALLOWED_REFERENCES, - DEFAULT_MAX_RESULTSET_SIZE, - NULL, - NULL, - DEFAULT_TTL -}; + CACHE_EXPECTING_RESPONSE, // A select has been sent, and we are waiting for the response. + CACHE_EXPECTING_FIELDS, // A select has been sent, and we want more fields. + CACHE_EXPECTING_ROWS, // A select has been sent, and we want more rows. + CACHE_EXPECTING_NOTHING, // We are not expecting anything from the server. + CACHE_IGNORING_RESPONSE, // We are not interested in the data received from the server. +} cache_session_state_t; + +typedef struct cache_request_state +{ + GWBUF* data; /**< Request data, possibly incomplete. */ +} CACHE_REQUEST_STATE; + +typedef struct cache_response_state +{ + GWBUF* data; /**< Response data, possibly incomplete. */ + size_t n_totalfields; /**< The number of fields a resultset contains. */ + size_t n_fields; /**< How many fields we have received, <= n_totalfields. */ + size_t n_rows; /**< How many rows we have received. */ + size_t offset; /**< Where we are in the response buffer. */ +} CACHE_RESPONSE_STATE; + +static void cache_response_state_reset(CACHE_RESPONSE_STATE *state); typedef struct cache_session_data { - CACHE_STORAGE_API *api; /**< The storage API to be used. */ - CACHE_STORAGE *storage; /**< The storage to be used with this session data. */ - DOWNSTREAM down; /**< The previous filter or equivalent. */ - UPSTREAM up; /**< The next filter or equivalent. */ - GWBUF *packets; /**< A possible incomplete packet. */ - SESSION *session; /**< The session this data is associated with. */ - char key[CACHE_KEY_MAXLEN]; /**< Key storage. */ - char *used_key; /**< A key if one is ued. */ + CACHE_INSTANCE *instance; /**< The cache instance the session is associated with. */ + CACHE_STORAGE_API *api; /**< The storage API to be used. */ + CACHE_STORAGE *storage; /**< The storage to be used with this session data. */ + DOWNSTREAM down; /**< The previous filter or equivalent. */ + UPSTREAM up; /**< The next filter or equivalent. */ + CACHE_REQUEST_STATE req; /**< The request state. */ + CACHE_RESPONSE_STATE res; /**< The response state. */ + SESSION *session; /**< The session this data is associated with. */ + char key[CACHE_KEY_MAXLEN]; /**< Key storage. */ + cache_session_state_t state; } CACHE_SESSION_DATA; -static bool route_using_cache(CACHE_INSTANCE *instance, - CACHE_SESSION_DATA *sdata, - const GWBUF *key, - GWBUF **value); +static CACHE_SESSION_DATA *cache_session_data_create(CACHE_INSTANCE *instance, SESSION *session); +static void cache_session_data_free(CACHE_SESSION_DATA *data); + +static int handle_expecting_fields(CACHE_SESSION_DATA *csdata); +static int handle_expecting_nothing(CACHE_SESSION_DATA *csdata); +static int handle_expecting_response(CACHE_SESSION_DATA *csdata); +static int handle_expecting_rows(CACHE_SESSION_DATA *csdata); +static int handle_ignoring_response(CACHE_SESSION_DATA *csdata); + +static bool route_using_cache(CACHE_SESSION_DATA *sdata, const GWBUF *key, GWBUF **value); + +static int send_upstream(CACHE_SESSION_DATA *csdata); + +static void store_result(CACHE_SESSION_DATA *csdata); // // API BEGIN @@ -181,13 +210,26 @@ static FILTER *createInstance(const char *name, char **options, FILTER_PARAMETER error = true; } } + else if (strcmp(param->name, "max_resultset_rows") == 0) + { + int v = atoi(param->value); + + if (v > 0) + { + config.max_resultset_rows = v; + } + else + { + config.max_resultset_rows = CACHE_DEFAULT_MAX_RESULTSET_ROWS; + } + } else if (strcmp(param->name, "max_resultset_size") == 0) { int v = atoi(param->value); if (v > 0) { - config.max_resultset_size = v; + config.max_resultset_size = v * 1024; } else { @@ -282,14 +324,7 @@ static FILTER *createInstance(const char *name, char **options, FILTER_PARAMETER static void *newSession(FILTER *instance, SESSION *session) { CACHE_INSTANCE *cinstance = (CACHE_INSTANCE*)instance; - CACHE_SESSION_DATA *csdata = (CACHE_SESSION_DATA*)MXS_CALLOC(1, sizeof(CACHE_SESSION_DATA)); - - if (csdata) - { - csdata->api = cinstance->module->api; - csdata->storage = cinstance->storage; - csdata->session = session; - } + CACHE_SESSION_DATA *csdata = cache_session_data_create(cinstance, session); return csdata; } @@ -317,7 +352,7 @@ static void freeSession(FILTER *instance, void *sdata) CACHE_INSTANCE *cinstance = (CACHE_INSTANCE*)instance; CACHE_SESSION_DATA *csdata = (CACHE_SESSION_DATA*)sdata; - MXS_FREE(csdata); + cache_session_data_free(csdata); } /** @@ -357,21 +392,21 @@ static void setUpstream(FILTER *instance, void *sdata, UPSTREAM *up) * @param sdata The filter session data * @param packets The query data */ -static int routeQuery(FILTER *instance, void *sdata, GWBUF *packets) +static int routeQuery(FILTER *instance, void *sdata, GWBUF *data) { CACHE_INSTANCE *cinstance = (CACHE_INSTANCE*)instance; CACHE_SESSION_DATA *csdata = (CACHE_SESSION_DATA*)sdata; - if (csdata->packets) + if (csdata->req.data) { - gwbuf_append(csdata->packets, packets); + gwbuf_append(csdata->req.data, data); } else { - csdata->packets = packets; + csdata->req.data = data; } - GWBUF *packet = modutil_get_next_MySQL_packet(&csdata->packets); + GWBUF *packet = modutil_get_next_MySQL_packet(&csdata->req.data); int rv; @@ -379,6 +414,9 @@ static int routeQuery(FILTER *instance, void *sdata, GWBUF *packets) { bool use_default = true; + cache_response_state_reset(&csdata->res); + csdata->state = CACHE_IGNORING_RESPONSE; + // TODO: This returns the wrong result if GWBUF_LENGTH(packet) is < 5. if (modutil_is_SQL(packet)) { @@ -392,10 +430,15 @@ static int routeQuery(FILTER *instance, void *sdata, GWBUF *packets) if (qc_get_operation(packet) == QUERY_OP_SELECT) { GWBUF *result; - use_default = !route_using_cache(cinstance, csdata, packet, &result); + use_default = !route_using_cache(csdata, packet, &result); - if (!use_default) + if (use_default) { + csdata->state = CACHE_EXPECTING_RESPONSE; + } + else + { + csdata->state = CACHE_EXPECTING_NOTHING; C_DEBUG("Using data from cache."); gwbuf_free(packet); DCB *dcb = csdata->session->client_dcb; @@ -429,33 +472,66 @@ static int routeQuery(FILTER *instance, void *sdata, GWBUF *packets) * @param sdata The filter session data * @param queue The query data */ -static int clientReply(FILTER *instance, void *sdata, GWBUF *queue) +static int clientReply(FILTER *instance, void *sdata, GWBUF *data) { CACHE_INSTANCE *cinstance = (CACHE_INSTANCE*)instance; CACHE_SESSION_DATA *csdata = (CACHE_SESSION_DATA*)sdata; - // TODO: queue can be put to the cache only if it is a complete - // TODO: response. If it isn't, then we need to stash it and wait - // TODO: we get a complete response. - // TODO: Since we will know from the first queue how big the - // TODO: entire response will be, this is also where we can decide - // TODO: that something is too large to cache. If it is, an existing - // TODO: item must be deleted. + int rv; - if (csdata->used_key) + if (csdata->res.data) { - C_DEBUG("Key available, storing result."); + gwbuf_append(csdata->res.data, data); + } + else + { + csdata->res.data = data; + } - cache_result_t result = csdata->api->putValue(csdata->storage, csdata->used_key, queue); - csdata->used_key = NULL; - - if (result != CACHE_RESULT_OK) + if (csdata->state != CACHE_IGNORING_RESPONSE) + { + if (gwbuf_length(csdata->res.data) > csdata->instance->config.max_resultset_size) { - MXS_ERROR("Could not store cache item."); + C_DEBUG("Current size %uB of resultset, at least as much " + "as maximum allowed size %uKiB. Not caching.", + gwbuf_length(csdata->res.data), + csdata->instance->config.max_resultset_size / 1024); + + csdata->state = CACHE_IGNORING_RESPONSE; } } - return csdata->up.clientReply(csdata->up.instance, csdata->up.session, queue); + switch (csdata->state) + { + case CACHE_EXPECTING_FIELDS: + rv = handle_expecting_fields(csdata); + break; + + case CACHE_EXPECTING_NOTHING: + rv = handle_expecting_nothing(csdata); + break; + + case CACHE_EXPECTING_RESPONSE: + rv = handle_expecting_response(csdata); + break; + + case CACHE_EXPECTING_ROWS: + rv = handle_expecting_rows(csdata); + break; + + case CACHE_IGNORING_RESPONSE: + rv = handle_ignoring_response(csdata); + break; + + default: + MXS_ERROR("Internal cache logic broken, unexpected state: %d", csdata->state); + ss_dassert(!true); + rv = send_upstream(csdata); + cache_response_state_reset(&csdata->res); + csdata->state = CACHE_IGNORING_RESPONSE; + } + + return rv; } /** @@ -480,47 +556,341 @@ static void diagnostics(FILTER *instance, void *sdata, DCB *dcb) // API END // +/** + * Reset cache response state + * + * @param state Pointer to object. + */ +static void cache_response_state_reset(CACHE_RESPONSE_STATE *state) +{ + state->data = NULL; + state->n_totalfields = 0; + state->n_fields = 0; + state->n_rows = 0; + state->offset = 0; +} + +/** + * Create cache session data + * + * @param instance The cache instance this data is associated with. + * + * @return Session data or NULL if creation fails. + */ +static CACHE_SESSION_DATA *cache_session_data_create(CACHE_INSTANCE *instance, + SESSION* session) +{ + CACHE_SESSION_DATA *data = (CACHE_SESSION_DATA*)MXS_CALLOC(1, sizeof(CACHE_SESSION_DATA)); + + if (data) + { + data->instance = instance; + data->api = instance->module->api; + data->storage = instance->storage; + data->session = session; + data->state = CACHE_EXPECTING_NOTHING; + } + + return data; +} + +/** + * Free cache session data. + * + * @param A cache session data previously allocated using session_data_create(). + */ +static void cache_session_data_free(CACHE_SESSION_DATA* data) +{ + if (data) + { + MXS_FREE(data); + } +} + +/** + * Called when resultset field information is handled. + * + * @param csdata The cache session data. + */ +static int handle_expecting_fields(CACHE_SESSION_DATA *csdata) +{ + ss_dassert(csdata->state == CACHE_EXPECTING_FIELDS); + ss_dassert(csdata->res.data); + + int rv = 1; + + bool insufficient = false; + + size_t buflen = gwbuf_length(csdata->res.data); + + while (!insufficient && (buflen - csdata->res.offset >= MYSQL_HEADER_LEN)) + { + uint8_t header[MYSQL_HEADER_LEN + 1]; + gwbuf_copy_data(csdata->res.data, csdata->res.offset, MYSQL_HEADER_LEN + 1, header); + + size_t packetlen = MYSQL_HEADER_LEN + MYSQL_GET_PACKET_LEN(header); + + if (csdata->res.offset + packetlen <= buflen) + { + // We have at least one complete packet. + int command = (int)MYSQL_GET_COMMAND(header); + + switch (command) + { + case 0xfe: // EOF, the one after the fields. + csdata->res.offset += packetlen; + csdata->state = CACHE_EXPECTING_ROWS; + rv = handle_expecting_rows(csdata); + break; + + default: // Field information. + csdata->res.offset += packetlen; + ++csdata->res.n_fields; + ss_dassert(csdata->res.n_fields <= csdata->res.n_totalfields); + break; + } + } + else + { + // We need more data + insufficient = true; + } + } + + return rv; +} + +/** + * Called when data is received (even if nothing is expected) from the server. + * + * @param csdata The cache session data. + */ +static int handle_expecting_nothing(CACHE_SESSION_DATA *csdata) +{ + ss_dassert(csdata->state == CACHE_EXPECTING_NOTHING); + ss_dassert(csdata->res.data); + MXS_ERROR("Received data from the backend althoug we were expecting nothing."); + ss_dassert(!true); + + return send_upstream(csdata); +} + +/** + * Called when a response is received from the server. + * + * @param csdata The cache session data. + */ +static int handle_expecting_response(CACHE_SESSION_DATA *csdata) +{ + ss_dassert(csdata->state == CACHE_EXPECTING_RESPONSE); + ss_dassert(csdata->res.data); + + int rv = 1; + + size_t buflen = gwbuf_length(csdata->res.data); + + if (buflen >= MYSQL_HEADER_LEN + 1) // We need the command byte. + { + // Reserve enough space to accomodate for the largest length encoded integer, + // which is type field + 8 bytes. + uint8_t header[MYSQL_HEADER_LEN + 1 + 8]; + gwbuf_copy_data(csdata->res.data, 0, MYSQL_HEADER_LEN + 1, header); + + switch ((int)MYSQL_GET_COMMAND(header)) + { + case 0x00: // OK + case 0xff: // ERR + C_DEBUG("OK or ERR"); + store_result(csdata); + + rv = send_upstream(csdata); + csdata->state = CACHE_EXPECTING_NOTHING; + break; + + case 0xfb: // GET_MORE_CLIENT_DATA/SEND_MORE_CLIENT_DATA + C_DEBUG("GET_MORE_CLIENT_DATA"); + rv = send_upstream(csdata); + csdata->state = CACHE_IGNORING_RESPONSE; + break; + + default: + C_DEBUG("RESULTSET"); + + if (csdata->res.n_totalfields != 0) + { + // We've seen the header and have figured out how many fields there are. + csdata->state = CACHE_EXPECTING_FIELDS; + rv = handle_expecting_fields(csdata); + } + else + { + // leint_bytes() returns the length of the int type field + the size of the + // integer. + size_t n_bytes = leint_bytes(&header[4]); + + if (MYSQL_HEADER_LEN + n_bytes <= buflen) + { + // Now we can figure out how many fields there are, but first we + // need to copy some more data. + gwbuf_copy_data(csdata->res.data, + MYSQL_HEADER_LEN + 1, n_bytes - 1, &header[MYSQL_HEADER_LEN + 1]); + + csdata->res.n_totalfields = leint_value(&header[4]); + csdata->res.offset = MYSQL_HEADER_LEN + n_bytes; + + csdata->state = CACHE_EXPECTING_FIELDS; + rv = handle_expecting_fields(csdata); + } + else + { + // We need more data. We will be called again, when data is available. + } + } + break; + } + } + + return rv; +} + +/** + * Called when resultset rows are handled. + * + * @param csdata The cache session data. + */ +static int handle_expecting_rows(CACHE_SESSION_DATA *csdata) +{ + ss_dassert(csdata->state == CACHE_EXPECTING_ROWS); + ss_dassert(csdata->res.data); + + int rv = 1; + + bool insufficient = false; + + size_t buflen = gwbuf_length(csdata->res.data); + + while (!insufficient && (buflen - csdata->res.offset >= MYSQL_HEADER_LEN)) + { + uint8_t header[MYSQL_HEADER_LEN + 1]; + gwbuf_copy_data(csdata->res.data, csdata->res.offset, MYSQL_HEADER_LEN + 1, header); + + size_t packetlen = MYSQL_HEADER_LEN + MYSQL_GET_PACKET_LEN(header); + + if (csdata->res.offset + packetlen <= buflen) + { + // We have at least one complete packet. + int command = (int)MYSQL_GET_COMMAND(header); + + switch (command) + { + case 0xfe: // EOF, the one after the rows. + csdata->res.offset += packetlen; + ss_dassert(csdata->res.offset == buflen); + + store_result(csdata); + + rv = send_upstream(csdata); + csdata->state = CACHE_EXPECTING_NOTHING; + break; + + case 0xfb: // NULL + default: // length-encoded-string + csdata->res.offset += packetlen; + ++csdata->res.n_rows; + + if (csdata->res.n_rows > csdata->instance->config.max_resultset_rows) + { + C_DEBUG("Max rows %lu reached, not caching result.", csdata->res.n_rows); + rv = send_upstream(csdata); + csdata->res.offset = buflen; // To abort the loop. + csdata->state = CACHE_IGNORING_RESPONSE; + } + break; + } + } + else + { + // We need more data + insufficient = true; + } + } + + return rv; +} + +/** + * Called when all data from the server is ignored. + * + * @param csdata The cache session data. + */ +static int handle_ignoring_response(CACHE_SESSION_DATA *csdata) +{ + ss_dassert(csdata->state == CACHE_IGNORING_RESPONSE); + ss_dassert(csdata->res.data); + + return send_upstream(csdata); +} + /** * Route a query via the cache. * - * @param instance The filter instance. - * @param sdata Session data + * @param csdata Session data * @param key A SELECT packet. * @param value The result. * @return True if the query was satisfied from the query. */ -static bool route_using_cache(CACHE_INSTANCE *instance, - CACHE_SESSION_DATA *csdata, +static bool route_using_cache(CACHE_SESSION_DATA *csdata, const GWBUF *query, GWBUF **value) { - // TODO: This works *only* if only one request/response is handled at a time. - // TODO: Is that the case, or is it not? - cache_result_t result = csdata->api->getKey(csdata->storage, query, csdata->key); if (result == CACHE_RESULT_OK) { result = csdata->api->getValue(csdata->storage, csdata->key, value); - - switch (result) - { - case CACHE_RESULT_OK: - csdata->used_key = NULL; - break; - - default: - MXS_ERROR("Could not get value from cache storage."); - case CACHE_RESULT_NOT_FOUND: - csdata->used_key = csdata->key; - break; - } } else { MXS_ERROR("Could not create cache key."); - csdata->used_key = NULL; } return result == CACHE_RESULT_OK; } + +/** + * Send data upstream. + * + * @param csdata Session data + * + * @return Whatever the upstream returns. + */ +static int send_upstream(CACHE_SESSION_DATA *csdata) +{ + ss_dassert(csdata->res.data != NULL); + + int rv = csdata->up.clientReply(csdata->up.instance, csdata->up.session, csdata->res.data); + csdata->res.data = NULL; + + return rv; +} + +/** + * Store the data. + * + * @param csdata Session data + */ +static void store_result(CACHE_SESSION_DATA *csdata) +{ + ss_dassert(csdata->res.data); + + csdata->res.data = gwbuf_make_contiguous(csdata->res.data); + + cache_result_t result = csdata->api->putValue(csdata->storage, + csdata->key, + csdata->res.data); + + if (result != CACHE_RESULT_OK) + { + MXS_ERROR("Could not store cache item."); + } +} diff --git a/server/modules/filter/cache/cache.h b/server/modules/filter/cache/cache.h new file mode 100644 index 000000000..ce904dad0 --- /dev/null +++ b/server/modules/filter/cache/cache.h @@ -0,0 +1,33 @@ +#ifndef CACHE_H +#define CACHE_H +/* + * Copyright (c) 2016 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl. + * + * Change Date: 2019-07-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ + +#include + + +typedef enum cache_references +{ + CACHE_REFERENCES_ANY, // select * from tbl; + CACHE_REFERENCES_QUALIFIED // select * from db.tbl; +} cache_references_t; + +#define CACHE_DEFAULT_ALLOWED_REFERENCES CACHE_REFERENCES_QUALIFIED +// Count +#define CACHE_DEFAULT_MAX_RESULTSET_ROWS UINT_MAX +// Bytes +#define CACHE_DEFAULT_MAX_RESULTSET_SIZE 64 * 1024 +// Seconds +#define CACHE_DEFAULT_TTL 10 + +#endif From 0ab4f04d7bffc76a121967c46eaf911c0c2c682a Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Wed, 14 Sep 2016 14:19:44 +0300 Subject: [PATCH 18/36] Add authentication plugin name to authenticator API The authenticators can now declare the authentication plugin name. Right now this is only relevant for MySQL authentication but for example the HTTP module could implement both Basic and Digest authentication. --- server/include/gw_authenticator.h | 2 ++ server/modules/authenticator/cdc_plain_auth.c | 3 ++- server/modules/authenticator/http_auth.c | 3 ++- server/modules/authenticator/max_admin_auth.c | 3 ++- server/modules/authenticator/mysql_auth.c | 3 ++- server/modules/authenticator/null_auth_allow.c | 3 ++- server/modules/authenticator/null_auth_deny.c | 3 ++- server/modules/include/mysql_client_server_protocol.h | 2 ++ server/modules/protocol/MySQLClient/mysql_client.c | 10 +++++++--- 9 files changed, 23 insertions(+), 9 deletions(-) diff --git a/server/include/gw_authenticator.h b/server/include/gw_authenticator.h index e004c93d0..461126bbc 100644 --- a/server/include/gw_authenticator.h +++ b/server/include/gw_authenticator.h @@ -47,6 +47,7 @@ struct servlistener; * authenticate Carry out the authentication * free Free extracted data * loadusers Load or update authenticator user data + * plugin_name The protocol specific name of the authentication plugin. * @endverbatim * * This forms the "module object" for authenticator modules within the gateway. @@ -60,6 +61,7 @@ typedef struct gw_authenticator int (*authenticate)(struct dcb *); void (*free)(struct dcb *); int (*loadusers)(struct servlistener *); + const char* plugin_name; } GWAUTHENTICATOR; /** Return values for the loadusers entry point */ diff --git a/server/modules/authenticator/cdc_plain_auth.c b/server/modules/authenticator/cdc_plain_auth.c index b94d25277..cc2d2d3a0 100644 --- a/server/modules/authenticator/cdc_plain_auth.c +++ b/server/modules/authenticator/cdc_plain_auth.c @@ -71,7 +71,8 @@ static GWAUTHENTICATOR MyObject = cdc_auth_is_client_ssl_capable, /* Check if client supports SSL */ cdc_auth_authenticate, /* Authenticate user credentials */ cdc_auth_free_client_data, /* Free the client data held in DCB */ - cdc_replace_users + cdc_replace_users, + NULL }; static int cdc_auth_check( diff --git a/server/modules/authenticator/http_auth.c b/server/modules/authenticator/http_auth.c index 2f067acd2..9d902afbd 100644 --- a/server/modules/authenticator/http_auth.c +++ b/server/modules/authenticator/http_auth.c @@ -63,7 +63,8 @@ static GWAUTHENTICATOR MyObject = http_auth_is_client_ssl_capable, /* Check if client supports SSL */ http_auth_authenticate, /* Authenticate user credentials */ http_auth_free_client_data, /* Free the client data held in DCB */ - users_default_loadusers + users_default_loadusers, + NULL }; typedef struct http_auth diff --git a/server/modules/authenticator/max_admin_auth.c b/server/modules/authenticator/max_admin_auth.c index 4c0741d38..ce5b85e5d 100644 --- a/server/modules/authenticator/max_admin_auth.c +++ b/server/modules/authenticator/max_admin_auth.c @@ -63,7 +63,8 @@ static GWAUTHENTICATOR MyObject = max_admin_auth_is_client_ssl_capable, /* Check if client supports SSL */ max_admin_auth_authenticate, /* Authenticate user credentials */ max_admin_auth_free_client_data, /* Free the client data held in DCB */ - users_default_loadusers + users_default_loadusers, + NULL }; /** diff --git a/server/modules/authenticator/mysql_auth.c b/server/modules/authenticator/mysql_auth.c index 4e357faad..f50165b42 100644 --- a/server/modules/authenticator/mysql_auth.c +++ b/server/modules/authenticator/mysql_auth.c @@ -66,7 +66,8 @@ static GWAUTHENTICATOR MyObject = mysql_auth_is_client_ssl_capable, /* Check if client supports SSL */ mysql_auth_authenticate, /* Authenticate user credentials */ mysql_auth_free_client_data, /* Free the client data held in DCB */ - mysql_auth_load_users /* Load users from backend databases */ + mysql_auth_load_users, /* Load users from backend databases */ + "mysql_native_password" }; static int combined_auth_check( diff --git a/server/modules/authenticator/null_auth_allow.c b/server/modules/authenticator/null_auth_allow.c index e2378d277..ff3b595b5 100644 --- a/server/modules/authenticator/null_auth_allow.c +++ b/server/modules/authenticator/null_auth_allow.c @@ -62,7 +62,8 @@ static GWAUTHENTICATOR MyObject = null_auth_is_client_ssl_capable, /* Check if client supports SSL */ null_auth_authenticate, /* Authenticate user credentials */ null_auth_free_client_data, /* Free the client data held in DCB */ - users_default_loadusers + users_default_loadusers, + NULL }; /** diff --git a/server/modules/authenticator/null_auth_deny.c b/server/modules/authenticator/null_auth_deny.c index 6563eb956..59d11edcf 100644 --- a/server/modules/authenticator/null_auth_deny.c +++ b/server/modules/authenticator/null_auth_deny.c @@ -62,7 +62,8 @@ static GWAUTHENTICATOR MyObject = null_auth_is_client_ssl_capable, /* Check if client supports SSL */ null_auth_authenticate, /* Authenticate user credentials */ null_auth_free_client_data, /* Free the client data held in DCB */ - users_default_loadusers + users_default_loadusers, + NULL }; /** diff --git a/server/modules/include/mysql_client_server_protocol.h b/server/modules/include/mysql_client_server_protocol.h index 31262b2a2..e82988f39 100644 --- a/server/modules/include/mysql_client_server_protocol.h +++ b/server/modules/include/mysql_client_server_protocol.h @@ -79,6 +79,8 @@ #define GW_MYSQL_SCRAMBLE_SIZE 20 #define GW_SCRAMBLE_LENGTH_323 8 +#define DEFAULT_AUTH_PLUGIN_NAME "mysql_native_password" + /** Maximum length of a MySQL packet */ #define MYSQL_PACKET_LENGTH_MAX 0x00ffffff diff --git a/server/modules/protocol/MySQLClient/mysql_client.c b/server/modules/protocol/MySQLClient/mysql_client.c index 73bf741a1..3528d464c 100644 --- a/server/modules/protocol/MySQLClient/mysql_client.c +++ b/server/modules/protocol/MySQLClient/mysql_client.c @@ -311,11 +311,15 @@ int MySQLSendHandshake(DCB* dcb) memcpy(mysql_plugin_data, server_scramble + 8, 12); + const char* plugin_name = dcb->authfunc.plugin_name ? + dcb->authfunc.plugin_name : DEFAULT_AUTH_PLUGIN_NAME; + int plugin_name_len = strlen(plugin_name); + mysql_payload_size = sizeof(mysql_protocol_version) + (len_version_string + 1) + sizeof(mysql_thread_id_num) + 8 + sizeof(/* mysql_filler */ uint8_t) + sizeof(mysql_server_capabilities_one) + sizeof(mysql_server_language) + sizeof(mysql_server_status) + sizeof(mysql_server_capabilities_two) + sizeof(mysql_scramble_len) + - sizeof(mysql_filler_ten) + 12 + sizeof(/* mysql_last_byte */ uint8_t) + strlen("mysql_native_password") + + sizeof(mysql_filler_ten) + 12 + sizeof(/* mysql_last_byte */ uint8_t) + plugin_name_len + sizeof(/* mysql_last_byte */ uint8_t); // allocate memory for packet header + payload @@ -407,8 +411,8 @@ int MySQLSendHandshake(DCB* dcb) mysql_handshake_payload++; // to be understanded ???? - memcpy(mysql_handshake_payload, "mysql_native_password", strlen("mysql_native_password")); - mysql_handshake_payload = mysql_handshake_payload + strlen("mysql_native_password"); + memcpy(mysql_handshake_payload, plugin_name, plugin_name_len); + mysql_handshake_payload = mysql_handshake_payload + plugin_name_len; //write last byte, 0 *mysql_handshake_payload = 0x00; From b1b2e5b770fd24a77160e91a07bfb306e065e0f0 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 20 Sep 2016 11:17:37 +0300 Subject: [PATCH 19/36] Don't free the shared shard maps When client sessions are closed, the shared shard maps should not be freed. --- server/modules/routing/schemarouter/schemarouter.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/server/modules/routing/schemarouter/schemarouter.c b/server/modules/routing/schemarouter/schemarouter.c index 16b867ae2..29f17d5c3 100644 --- a/server/modules/routing/schemarouter/schemarouter.c +++ b/server/modules/routing/schemarouter/schemarouter.c @@ -1381,11 +1381,6 @@ static void freeSession(ROUTER* router_instance, void* router_client_session) * all the memory and other resources associated * to the client session. */ - if (router_cli_ses->shardmap) - { - hashtable_free(router_cli_ses->shardmap->hash); - free(router_cli_ses->shardmap); - } free(router_cli_ses->rses_backend_ref); free(router_cli_ses); return; From 39921353258118f58776e07442f866f03e426b45 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 20 Sep 2016 10:59:41 +0300 Subject: [PATCH 20/36] Move authentication return codes to gw_authenticator.h The MYSQL_* authentication return codes are now in gw_authenticator.h so that all authenticators can use them. Also dropped the MYSQL_ prefix from the return codes and added AUTH_INCOMPLETE for a generic authentication-in-progress return code. --- server/core/service.c | 4 +- server/core/users.c | 2 +- server/include/gw_authenticator.h | 13 +++++- server/modules/authenticator/cdc_plain_auth.c | 6 +-- server/modules/authenticator/mysql_auth.c | 46 +++++++++---------- .../include/mysql_client_server_protocol.h | 7 --- .../protocol/MySQLClient/mysql_client.c | 25 +++++----- server/modules/protocol/mysql_common.c | 4 +- 8 files changed, 56 insertions(+), 51 deletions(-) diff --git a/server/core/service.c b/server/core/service.c index 9cbf73ea4..10535c157 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -300,7 +300,7 @@ serviceStartPort(SERVICE *service, SERV_LISTENER *port) /** Load the authentication users before before starting the listener */ if (port->listener->authfunc.loadusers && (service->router->getCapabilities() & RCAP_TYPE_NO_USERS_INIT) == 0 && - port->listener->authfunc.loadusers(port) != AUTH_LOADUSERS_OK) + port->listener->authfunc.loadusers(port) != MXS_AUTH_LOADUSERS_OK) { MXS_ERROR("[%s] Failed to load users for listener '%s', authentication might not work.", service->name, port->name); @@ -1457,7 +1457,7 @@ int service_refresh_users(SERVICE *service) for (SERV_LISTENER *port = service->ports; port; port = port->next) { - if (port->listener->authfunc.loadusers(port) != AUTH_LOADUSERS_OK) + if (port->listener->authfunc.loadusers(port) != MXS_AUTH_LOADUSERS_OK) { MXS_ERROR("[%s] Failed to load users for listener '%s', authentication might not work.", service->name, port->name); diff --git a/server/core/users.c b/server/core/users.c index 53f7d9cd1..66ebe3001 100644 --- a/server/core/users.c +++ b/server/core/users.c @@ -224,5 +224,5 @@ int users_default_loadusers(SERV_LISTENER *port) { users_free(port->users); port->users = users_alloc(); - return AUTH_LOADUSERS_OK; + return MXS_AUTH_LOADUSERS_OK; } diff --git a/server/include/gw_authenticator.h b/server/include/gw_authenticator.h index 461126bbc..a0cae4ce7 100644 --- a/server/include/gw_authenticator.h +++ b/server/include/gw_authenticator.h @@ -64,9 +64,18 @@ typedef struct gw_authenticator const char* plugin_name; } GWAUTHENTICATOR; +/** Return values for extract and authenticate entry points */ +#define MXS_AUTH_SUCCEEDED 0 /**< Authentication was successful */ +#define MXS_AUTH_FAILED 1 /**< Authentication failed */ +#define MXS_AUTH_FAILED_DB 2 +#define MXS_AUTH_FAILED_SSL 3 +#define MXS_AUTH_INCOMPLETE 4 /**< Authentication is not yet complete */ +#define MXS_AUTH_SSL_INCOMPLETE 5 /**< SSL connection is not yet complete */ +#define MXS_AUTH_NO_SESSION 6 + /** Return values for the loadusers entry point */ -#define AUTH_LOADUSERS_OK 0 /**< Users loaded successfully */ -#define AUTH_LOADUSERS_ERROR 1 /**< Failed to load users */ +#define MXS_AUTH_LOADUSERS_OK 0 /**< Users loaded successfully */ +#define MXS_AUTH_LOADUSERS_ERROR 1 /**< Failed to load users */ /** * The GWAUTHENTICATOR version data. The following should be updated whenever diff --git a/server/modules/authenticator/cdc_plain_auth.c b/server/modules/authenticator/cdc_plain_auth.c index cc2d2d3a0..84777bdd7 100644 --- a/server/modules/authenticator/cdc_plain_auth.c +++ b/server/modules/authenticator/cdc_plain_auth.c @@ -181,7 +181,7 @@ cdc_auth_authenticate(DCB *dcb) auth_ret = cdc_auth_check(dcb, protocol, client_data->user, client_data->auth_data, client_data->flags); /* On failed authentication try to reload users and authenticate again */ - if (CDC_STATE_AUTH_OK != auth_ret && cdc_replace_users(dcb->listener) == AUTH_LOADUSERS_OK) + if (CDC_STATE_AUTH_OK != auth_ret && cdc_replace_users(dcb->listener) == MXS_AUTH_LOADUSERS_OK) { auth_ret = cdc_auth_check(dcb, protocol, client_data->user, client_data->auth_data, client_data->flags); } @@ -483,7 +483,7 @@ cdc_read_users(USERS *users, char *usersfile) */ int cdc_replace_users(SERV_LISTENER *listener) { - int rc = AUTH_LOADUSERS_ERROR; + int rc = MXS_AUTH_LOADUSERS_ERROR; USERS *newusers = users_alloc(); if (newusers) @@ -501,7 +501,7 @@ int cdc_replace_users(SERV_LISTENER *listener) /** Successfully loaded at least one user */ oldusers = listener->users; listener->users = newusers; - rc = AUTH_LOADUSERS_OK; + rc = MXS_AUTH_LOADUSERS_OK; } else if (listener->users) { diff --git a/server/modules/authenticator/mysql_auth.c b/server/modules/authenticator/mysql_auth.c index f50165b42..253b337f3 100644 --- a/server/modules/authenticator/mysql_auth.c +++ b/server/modules/authenticator/mysql_auth.c @@ -152,23 +152,23 @@ mysql_auth_authenticate(DCB *dcb) if (0 != ssl_ret) { - auth_ret = (SSL_ERROR_CLIENT_NOT_SSL == ssl_ret) ? MYSQL_FAILED_AUTH_SSL : MYSQL_FAILED_AUTH; + auth_ret = (SSL_ERROR_CLIENT_NOT_SSL == ssl_ret) ? MXS_AUTH_FAILED_SSL : MXS_AUTH_FAILED; } else if (!health_after) { - auth_ret = MYSQL_AUTH_SSL_INCOMPLETE; + auth_ret = MXS_AUTH_SSL_INCOMPLETE; } else if (!health_before && health_after) { - auth_ret = MYSQL_AUTH_SSL_INCOMPLETE; + auth_ret = MXS_AUTH_SSL_INCOMPLETE; poll_add_epollin_event_to_dcb(dcb, NULL); } else if (0 == strlen(client_data->user)) { - auth_ret = MYSQL_FAILED_AUTH; + auth_ret = MXS_AUTH_FAILED; } else @@ -181,14 +181,14 @@ mysql_auth_authenticate(DCB *dcb) /* On failed authentication try to load user table from backend database */ /* Success for service_refresh_users returns 0 */ - if (MYSQL_AUTH_SUCCEEDED != auth_ret && 0 == service_refresh_users(dcb->service)) + if (MXS_AUTH_SUCCEEDED != auth_ret && 0 == service_refresh_users(dcb->service)) { auth_ret = combined_auth_check(dcb, client_data->auth_token, client_data->auth_token_len, protocol, client_data->user, client_data->client_sha1, client_data->db); } /* on successful authentication, set user into dcb field */ - if (MYSQL_AUTH_SUCCEEDED == auth_ret) + if (MXS_AUTH_SUCCEEDED == auth_ret) { dcb->user = MXS_STRDUP_A(client_data->user); } @@ -247,7 +247,7 @@ mysql_auth_set_protocol_data(DCB *dcb, GWBUF *buf) { if (NULL == (client_data = (MYSQL_session *)MXS_CALLOC(1, sizeof(MYSQL_session)))) { - return MYSQL_FAILED_AUTH; + return MXS_AUTH_FAILED; } #if defined(SS_DEBUG) client_data->myses_chk_top = CHK_NUM_MYSQLSES; @@ -279,7 +279,7 @@ mysql_auth_set_protocol_data(DCB *dcb, GWBUF *buf) if (client_auth_packet_size < (4 + 4 + 4 + 1 + 23)) { /* Packet is not big enough */ - return MYSQL_FAILED_AUTH; + return MXS_AUTH_FAILED; } return mysql_auth_set_client_data(client_data, protocol, buf); @@ -339,7 +339,7 @@ mysql_auth_set_client_data( else { /* Packet has incomplete or too long username */ - return MYSQL_FAILED_AUTH; + return MXS_AUTH_FAILED; } if (client_auth_packet_size > (auth_packet_base_size + user_length + 1)) { @@ -364,13 +364,13 @@ mysql_auth_set_client_data( else { /* Failed to allocate space for authentication token string */ - return MYSQL_FAILED_AUTH; + return MXS_AUTH_FAILED; } } else { /* Packet was too small to contain authentication token */ - return MYSQL_FAILED_AUTH; + return MXS_AUTH_FAILED; } packet_length_used += 1 + client_data->auth_token_len; /* @@ -392,12 +392,12 @@ mysql_auth_set_client_data( { /* Packet is too short to contain database string */ /* or database string in packet is too long */ - return MYSQL_FAILED_AUTH; + return MXS_AUTH_FAILED; } } } } - return MYSQL_AUTH_SUCCEEDED; + return MXS_AUTH_SUCCEEDED; } /** @@ -615,7 +615,7 @@ gw_check_mysql_scramble_data(DCB *dcb, if ((username == NULL) || (mxs_scramble == NULL) || (stage1_hash == NULL)) { - return MYSQL_FAILED_AUTH; + return MXS_AUTH_FAILED; } /*< @@ -633,7 +633,7 @@ gw_check_mysql_scramble_data(DCB *dcb, memcpy(stage1_hash, (char *)"_", 1); } - return MYSQL_FAILED_AUTH; + return MXS_AUTH_FAILED; } if (token && token_len) @@ -649,7 +649,7 @@ gw_check_mysql_scramble_data(DCB *dcb, { /* check if the password is not set in the user table */ return memcmp(password, null_client_sha1, MYSQL_SCRAMBLE_LEN) ? - MYSQL_FAILED_AUTH : MYSQL_AUTH_SUCCEEDED; + MXS_AUTH_FAILED : MXS_AUTH_SUCCEEDED; } /*< @@ -704,7 +704,7 @@ gw_check_mysql_scramble_data(DCB *dcb, /* now compare SHA1(SHA1(gateway_password)) and check_hash: return 0 is MYSQL_AUTH_OK */ return (0 == memcmp(password, check_hash, SHA_DIGEST_LENGTH)) ? - MYSQL_AUTH_SUCCEEDED : MYSQL_FAILED_AUTH; + MXS_AUTH_SUCCEEDED : MXS_AUTH_FAILED; } /** @@ -746,14 +746,14 @@ check_db_name_after_auth(DCB *dcb, char *database, int auth_ret) db_exists = -1; } - if (db_exists == 0 && auth_ret == MYSQL_AUTH_SUCCEEDED) + if (db_exists == 0 && auth_ret == MXS_AUTH_SUCCEEDED) { - auth_ret = MYSQL_FAILED_AUTH_DB; + auth_ret = MXS_AUTH_FAILED_DB; } - if (db_exists < 0 && auth_ret == MYSQL_AUTH_SUCCEEDED) + if (db_exists < 0 && auth_ret == MXS_AUTH_SUCCEEDED) { - auth_ret = MYSQL_FAILED_AUTH; + auth_ret = MXS_AUTH_FAILED; } } @@ -830,7 +830,7 @@ mysql_auth_free_client_data(DCB *dcb) */ static int mysql_auth_load_users(SERV_LISTENER *port) { - int rc = AUTH_LOADUSERS_OK; + int rc = MXS_AUTH_LOADUSERS_OK; SERVICE *service = port->listener->service; int loaded = replace_mysql_users(port); @@ -847,7 +847,7 @@ static int mysql_auth_load_users(SERV_LISTENER *port) if ((loaded = dbusers_load(port->users, path)) == -1) { MXS_ERROR("[%s] Failed to load cached users from '%s'.", service->name, path);; - rc = AUTH_LOADUSERS_ERROR; + rc = MXS_AUTH_LOADUSERS_ERROR; } else { diff --git a/server/modules/include/mysql_client_server_protocol.h b/server/modules/include/mysql_client_server_protocol.h index e82988f39..715283ff3 100644 --- a/server/modules/include/mysql_client_server_protocol.h +++ b/server/modules/include/mysql_client_server_protocol.h @@ -97,13 +97,6 @@ #define COM_QUIT_PACKET_SIZE (4+1) struct dcb; -#define MYSQL_AUTH_SUCCEEDED 0 -#define MYSQL_FAILED_AUTH 1 -#define MYSQL_FAILED_AUTH_DB 2 -#define MYSQL_FAILED_AUTH_SSL 3 -#define MYSQL_AUTH_SSL_INCOMPLETE 4 -#define MYSQL_AUTH_NO_SESSION 5 - typedef enum { MYSQL_ALLOC, /* Initial state of protocol auth state */ diff --git a/server/modules/protocol/MySQLClient/mysql_client.c b/server/modules/protocol/MySQLClient/mysql_client.c index 3528d464c..553a86a2f 100644 --- a/server/modules/protocol/MySQLClient/mysql_client.c +++ b/server/modules/protocol/MySQLClient/mysql_client.c @@ -577,7 +577,7 @@ gw_read_do_authentication(DCB *dcb, GWBUF *read_buffer, int nbytes_read) * data extraction succeeds, then a call is made to the actual * authenticate function to carry out the user checks. */ - if (MYSQL_AUTH_SUCCEEDED == ( + if (MXS_AUTH_SUCCEEDED == ( auth_val = dcb->authfunc.extract(dcb, read_buffer))) { /* @@ -597,7 +597,7 @@ gw_read_do_authentication(DCB *dcb, GWBUF *read_buffer, int nbytes_read) * non-null session) then the whole process has succeeded. In all * other cases an error return is made. */ - if (MYSQL_AUTH_SUCCEEDED == auth_val) + if (MXS_AUTH_SUCCEEDED == auth_val) { SESSION *session; @@ -628,14 +628,17 @@ gw_read_do_authentication(DCB *dcb, GWBUF *read_buffer, int nbytes_read) } else { - auth_val = MYSQL_AUTH_NO_SESSION; + auth_val = MXS_AUTH_NO_SESSION; } } /** - * If we did not get success throughout, then the protocol state is updated, - * the client is notified of the failure and the DCB is closed. + * If we did not get success throughout or authentication is not yet complete, + * then the protocol state is updated, the client is notified of the failure + * and the DCB is closed. */ - if (MYSQL_AUTH_SUCCEEDED != auth_val && MYSQL_AUTH_SSL_INCOMPLETE != auth_val) + if (MXS_AUTH_SUCCEEDED != auth_val && + MXS_AUTH_INCOMPLETE != auth_val && + MXS_AUTH_SSL_INCOMPLETE != auth_val) { protocol->protocol_auth_state = MYSQL_AUTH_FAILED; mysql_client_auth_error_handling(dcb, auth_val); @@ -974,7 +977,7 @@ mysql_client_auth_error_handling(DCB *dcb, int auth_val) switch (auth_val) { - case MYSQL_AUTH_NO_SESSION: + case MXS_AUTH_NO_SESSION: MXS_DEBUG("%lu [gw_read_client_event] session " "creation failed. fd %d, " "state = MYSQL_AUTH_NO_SESSION.", @@ -987,7 +990,7 @@ mysql_client_auth_error_handling(DCB *dcb, int auth_val) 0, "failed to create new session"); break; - case MYSQL_FAILED_AUTH_DB: + case MXS_AUTH_FAILED_DB: MXS_DEBUG("%lu [gw_read_client_event] database " "specified was not valid. fd %d, " "state = MYSQL_FAILED_AUTH_DB.", @@ -1003,7 +1006,7 @@ mysql_client_auth_error_handling(DCB *dcb, int auth_val) modutil_send_mysql_err_packet(dcb, packet_number, 0, 1049, "42000", fail_str); break; - case MYSQL_FAILED_AUTH_SSL: + case MXS_AUTH_FAILED_SSL: MXS_DEBUG("%lu [gw_read_client_event] client is " "not SSL capable for SSL listener. fd %d, " "state = MYSQL_FAILED_AUTH_SSL.", @@ -1016,7 +1019,7 @@ mysql_client_auth_error_handling(DCB *dcb, int auth_val) 0, "failed to complete SSL authentication"); break; - case MYSQL_AUTH_SSL_INCOMPLETE: + case MXS_AUTH_SSL_INCOMPLETE: MXS_DEBUG("%lu [gw_read_client_event] unable to " "complete SSL authentication. fd %d, " "state = MYSQL_AUTH_SSL_INCOMPLETE.", @@ -1029,7 +1032,7 @@ mysql_client_auth_error_handling(DCB *dcb, int auth_val) 0, "failed to complete SSL authentication"); break; - case MYSQL_FAILED_AUTH: + case MXS_AUTH_FAILED: MXS_DEBUG("%lu [gw_read_client_event] authentication failed. fd %d, " "state = MYSQL_FAILED_AUTH.", pthread_self(), diff --git a/server/modules/protocol/mysql_common.c b/server/modules/protocol/mysql_common.c index bfef46bca..9da47aa58 100644 --- a/server/modules/protocol/mysql_common.c +++ b/server/modules/protocol/mysql_common.c @@ -959,7 +959,7 @@ char *create_auth_fail_str(char *username, { ferrstr = "Access denied for user '%s'@'%s' (using password: %s) to database '%s'"; } - else if (errcode == MYSQL_FAILED_AUTH_SSL) + else if (errcode == MXS_AUTH_FAILED_SSL) { ferrstr = "Access without SSL denied"; } @@ -980,7 +980,7 @@ char *create_auth_fail_str(char *username, { sprintf(errstr, ferrstr, username, hostaddr, (*sha1 == '\0' ? "NO" : "YES"), db); } - else if (errcode == MYSQL_FAILED_AUTH_SSL) + else if (errcode == MXS_AUTH_FAILED_SSL) { sprintf(errstr, "%s", ferrstr); } From 9a9511fc5868e2467a183fb2aca0307ea0b928d2 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Tue, 20 Sep 2016 10:53:51 +0300 Subject: [PATCH 21/36] Update tarball instructions --- .../Install-MariaDB-MaxScale-Using-a-Tarball.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Documentation/Getting-Started/Install-MariaDB-MaxScale-Using-a-Tarball.md b/Documentation/Getting-Started/Install-MariaDB-MaxScale-Using-a-Tarball.md index d9ceb1854..5aca16d0b 100644 --- a/Documentation/Getting-Started/Install-MariaDB-MaxScale-Using-a-Tarball.md +++ b/Documentation/Getting-Started/Install-MariaDB-MaxScale-Using-a-Tarball.md @@ -1,6 +1,6 @@ # Installing MariaDB MaxScale using a tarball -MariaDB MaxScale is also made available as a tarball, which is named like `maxscale-X.Y.X.tar.gz` where `X.Y.Z` is the same as the corresponding version, e.g. `maxscale-2.0.1.tar.gz`. +MariaDB MaxScale is also made available as a tarball, which is named like `maxscale-x.y.z.OS.tar.gz` where `x.y.z` is the same as the corresponding version and `OS` identifies the operating system, e.g. `maxscale-2.0.1.centos.7.tar.gz`. The tarball has been built with the assumption that it will be installed in `/usr/local`. However, it is possible to install it in any directory, but in that case MariaDB MaxScale must be invoked with a flag. @@ -13,8 +13,8 @@ The required steps are as follows: $ sudo groupadd maxscale $ sudo useradd -g maxscale maxscale $ cd /usr/local - $ sudo tar -xzvf maxscale-X.Y.Z.tar.gz - $ sudo ln -s maxscale-X.Y.Z maxscale + $ sudo tar -xzvf maxscale-x.y.z.OS.tar.gz + $ sudo ln -s maxscale-x.y.z.OS.maxscale $ cd maxscale $ chown -R maxscale var @@ -34,13 +34,13 @@ If you want to place the configuration file somewhere else but in `/etc` you can Enter a directory where you have the right to create a subdirectory. Then do as follows. - $ tar -xzvf maxscale-X.Y.Z.tar.gz + $ tar -xzvf maxscale-x.y.z.OS.tar.gz -The next step is to create the MaxScale configuration file `maxscale-X.Y.Z/etc/maxscale.cnf`. The file `maxscale-X.Y.Z/etc/maxscale.cnf.template` can be used as a base. Please refer to [Configuration Guide](Configuration-Guide.md) for details. +The next step is to create the MaxScale configuration file `maxscale-x.y.z/etc/maxscale.cnf`. The file `maxscale-x.y.z/etc/maxscale.cnf.template` can be used as a base. Please refer to [Configuration Guide](Configuration-Guide.md) for details. When the configuration file has been created, MariaDB MaxScale can be started. - $ cd maxscale-X.Y.Z + $ cd maxscale-x.y.z $ LD_LIBRARY_PATH=lib/maxscale bin/maxscale -d --basedir=. With the flag `--basedir`, MariaDB MaxScale is told where the `bin`, `etc`, `lib` From 6ec999851ff3e3756fb81a473f2a37ababc8c2ad Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Tue, 20 Sep 2016 12:58:05 +0300 Subject: [PATCH 22/36] Update 2.0.1 release notes --- .../Release-Notes/MaxScale-2.0.1-Release-Notes.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/Documentation/Release-Notes/MaxScale-2.0.1-Release-Notes.md b/Documentation/Release-Notes/MaxScale-2.0.1-Release-Notes.md index 195bacf53..509cffad5 100644 --- a/Documentation/Release-Notes/MaxScale-2.0.1-Release-Notes.md +++ b/Documentation/Release-Notes/MaxScale-2.0.1-Release-Notes.md @@ -113,14 +113,17 @@ Please consult ## Bug fixes -[Here is a list of bugs fixed since the release of MaxScale 2.0.1.](https://jira.mariadb.org/issues/?jql=project%20%3D%20MXS%20AND%20issuetype%20%3D%20Bug%20AND%20status%20%3D%20Closed%20AND%20fixVersion%20in%20(2.0.0%2C%202.0.1)%20AND%20resolved%20%3E%3D%20-21d%20ORDER%20BY%20priority%20DESC%2C%20updated%20DESC) +[Here is a list of bugs fixed since the release of MaxScale 2.0.0.](https://jira.mariadb.org/browse/MXS-860?jql=project%20%3D%20MXS%20AND%20issuetype%20%3D%20Bug%20AND%20status%20%3D%20Closed%20AND%20fixVersion%20in%20(2.0.1)%20AND%20resolved%20%3E%3D%20-21d%20AND%20(resolution%20%3D%20Done%20OR%20resolution%20%3D%20Fixed)%20ORDER%20BY%20priority%20DESC) -* [MXS-812](https://jira.mariadb.org/browse/MXS-812): Number of conns not matching number of operations -* [MXS-847](https://jira.mariadb.org/browse/MXS-847): server_down event is executed 8 times due to putting sever into maintenance mode +* [MXS-860](https://jira.mariadb.org/browse/MXS-860): I want to access the web site if master server is down +* [MXS-870](https://jira.mariadb.org/browse/MXS-870): Assertion of Buffer Overflow * [MXS-845](https://jira.mariadb.org/browse/MXS-845): "Server down" event is re-triggered after maintenance mode is repeated -* [MXS-842](https://jira.mariadb.org/browse/MXS-842): Unexpected / undocumented behaviour when multiple available masters from mmmon monitor -* [MXS-846](https://jira.mariadb.org/browse/MXS-846): MMMon: Maintenance mode on slave logs error message every second -* [MXS-860](https://jira.mariadb.org/browse/MXS-860): I want to access the web site if master server is down. +* [MXS-836](https://jira.mariadb.org/browse/MXS-836): "Failed to start all MaxScale services" without retrying +* [MXS-835](https://jira.mariadb.org/browse/MXS-835): Please reinstate remote access to maxscaled protocol +* [MXS-773](https://jira.mariadb.org/browse/MXS-773): 100% CPU on idle MaxScale with MaxInfo +* [MXS-812](https://jira.mariadb.org/browse/MXS-812): Number of conns not matching number of operations +* [MXS-856](https://jira.mariadb.org/browse/MXS-856): If config file cannot be accessed and creation of log file fails, MaxScale crashes with SIGSEGV +* [MXS-829](https://jira.mariadb.org/browse/MXS-829): When the config file isn't readable or doesn't exist, maxscale silently ends ## Known Issues and Limitations From 923761159f7c55f0b462675c186e925e9d9faea8 Mon Sep 17 00:00:00 2001 From: counterpoint Date: Tue, 20 Sep 2016 11:55:02 +0100 Subject: [PATCH 23/36] Move MXS-807 refactor read-write split into develop. --- server/modules/include/rwsplit_internal.h | 142 + .../routing/readwritesplit/CMakeLists.txt | 2 +- .../routing/readwritesplit/readwritesplit.c | 4336 +++-------------- .../routing/readwritesplit/rwsplit_mysql.c | 541 ++ .../readwritesplit/rwsplit_route_stmt.c | 1304 +++++ .../readwritesplit/rwsplit_select_backends.c | 473 ++ .../readwritesplit/rwsplit_session_cmd.c | 480 ++ .../readwritesplit/rwsplit_tmp_table_multi.c | 408 ++ 8 files changed, 4066 insertions(+), 3620 deletions(-) create mode 100644 server/modules/include/rwsplit_internal.h create mode 100644 server/modules/routing/readwritesplit/rwsplit_mysql.c create mode 100644 server/modules/routing/readwritesplit/rwsplit_route_stmt.c create mode 100644 server/modules/routing/readwritesplit/rwsplit_select_backends.c create mode 100644 server/modules/routing/readwritesplit/rwsplit_session_cmd.c create mode 100644 server/modules/routing/readwritesplit/rwsplit_tmp_table_multi.c diff --git a/server/modules/include/rwsplit_internal.h b/server/modules/include/rwsplit_internal.h new file mode 100644 index 000000000..af0619945 --- /dev/null +++ b/server/modules/include/rwsplit_internal.h @@ -0,0 +1,142 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ + +/* + * File: rwsplit_internal.h + * Author: mbrampton + * + * Created on 08 August 2016, 11:54 + */ + +#ifndef RWSPLIT_INTERNAL_H +#define RWSPLIT_INTERNAL_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +/* This needs to be removed along with dependency on it - see the + * rwsplit_tmp_table_multi functions + */ +#include + +/* + * The following are implemented in rwsplit_mysql.c + */ +bool route_single_stmt(ROUTER_INSTANCE *inst, ROUTER_CLIENT_SES *rses, + GWBUF *querybuf); +void closed_session_reply(GWBUF *querybuf); +void live_session_reply(GWBUF **querybuf, ROUTER_CLIENT_SES *rses); +void print_error_packet(ROUTER_CLIENT_SES *rses, GWBUF *buf, DCB *dcb); +void check_session_command_reply(GWBUF *writebuf, sescmd_cursor_t *scur, backend_ref_t *bref); +bool execute_sescmd_in_backend(backend_ref_t *backend_ref); +bool handle_target_is_all(route_target_t route_target, + ROUTER_INSTANCE *inst, ROUTER_CLIENT_SES *rses, + GWBUF *querybuf, int packet_type, qc_query_type_t qtype); +int determine_packet_type(GWBUF *querybuf, bool *non_empty_packet); +void log_transaction_status(ROUTER_CLIENT_SES *rses, GWBUF *querybuf, qc_query_type_t qtype); +void session_lock_failure_handling(GWBUF *querybuf, int packet_type, qc_query_type_t qtype); +bool is_packet_a_one_way_message(int packet_type); +sescmd_cursor_t *backend_ref_get_sescmd_cursor(backend_ref_t *bref); +bool is_packet_a_query(int packet_type); +bool send_readonly_error(DCB *dcb); + +/* + * The following are implemented in readwritesplit.c + */ +bool rses_begin_locked_router_action(ROUTER_CLIENT_SES *rses); +void rses_end_locked_router_action(ROUTER_CLIENT_SES *rses); +void bref_clear_state(backend_ref_t *bref, bref_state_t state); +void bref_set_state(backend_ref_t *bref, bref_state_t state); +int router_handle_state_switch(DCB *dcb, DCB_REASON reason, void *data); +backend_ref_t *get_bref_from_dcb(ROUTER_CLIENT_SES *rses, DCB *dcb); +void rses_property_done(rses_property_t *prop); +int rses_get_max_slavecount(ROUTER_CLIENT_SES *rses, int router_nservers); +int rses_get_max_replication_lag(ROUTER_CLIENT_SES *rses); + +/* + * The following are implemented in rwsplit_route_stmt.c + */ + +bool route_single_stmt(ROUTER_INSTANCE *inst, ROUTER_CLIENT_SES *rses, + GWBUF *querybuf); +int rwsplit_hashkeyfun(const void *key); +int rwsplit_hashcmpfun(const void *v1, const void *v2); +void *rwsplit_hstrdup(const void *fval); +void rwsplit_hfree(void *fval); +bool rwsplit_get_dcb(DCB **p_dcb, ROUTER_CLIENT_SES *rses, backend_type_t btype, + char *name, int max_rlag); +route_target_t get_route_target(ROUTER_CLIENT_SES *rses, + qc_query_type_t qtype, HINT *hint); +rses_property_t *rses_property_init(rses_property_type_t prop_type); +int rses_property_add(ROUTER_CLIENT_SES *rses, rses_property_t *prop); +void handle_multi_temp_and_load(ROUTER_CLIENT_SES *rses, GWBUF *querybuf, + int packet_type, int *qtype); +bool handle_hinted_target(ROUTER_CLIENT_SES *rses, GWBUF *querybuf, + route_target_t route_target, DCB **target_dcb); +bool handle_slave_is_target(ROUTER_INSTANCE *inst, ROUTER_CLIENT_SES *rses, + DCB **target_dcb); +bool handle_master_is_target(ROUTER_INSTANCE *inst, ROUTER_CLIENT_SES *rses, + DCB **target_dcb); +bool handle_got_target(ROUTER_INSTANCE *inst, ROUTER_CLIENT_SES *rses, + GWBUF *querybuf, DCB *target_dcb); +bool route_session_write(ROUTER_CLIENT_SES *router_cli_ses, + GWBUF *querybuf, ROUTER_INSTANCE *inst, + int packet_type, + qc_query_type_t qtype); + +/* + * The following are implemented in rwsplit_session_cmd.c +*/ +mysql_sescmd_t *rses_property_get_sescmd(rses_property_t *prop); +mysql_sescmd_t *mysql_sescmd_init(rses_property_t *rses_prop, + GWBUF *sescmd_buf, + unsigned char packet_type, + ROUTER_CLIENT_SES *rses); +void mysql_sescmd_done(mysql_sescmd_t *sescmd); +mysql_sescmd_t *sescmd_cursor_get_command(sescmd_cursor_t *scur); +bool sescmd_cursor_is_active(sescmd_cursor_t *sescmd_cursor); +void sescmd_cursor_set_active(sescmd_cursor_t *sescmd_cursor, + bool value); +bool execute_sescmd_history(backend_ref_t *bref); +GWBUF *sescmd_cursor_clone_querybuf(sescmd_cursor_t *scur); +GWBUF *sescmd_cursor_process_replies(GWBUF *replybuf, + backend_ref_t *bref, + bool *reconnect); + +/* + * The following are implemented in rwsplit_select_backends.c + */ +bool select_connect_backend_servers(backend_ref_t **p_master_ref, + backend_ref_t *backend_ref, + int router_nservers, int max_nslaves, + int max_rlag, + select_criteria_t select_criteria, + SESSION *session, + ROUTER_INSTANCE *router); + +/* + * The following are implemented in rwsplit_tmp_table_multi.c + */ +void check_drop_tmp_table(ROUTER_CLIENT_SES *router_cli_ses, + GWBUF *querybuf, + mysql_server_cmd_t packet_type); +qc_query_type_t is_read_tmp_table(ROUTER_CLIENT_SES *router_cli_ses, + GWBUF *querybuf, + qc_query_type_t type); +void check_create_tmp_table(ROUTER_CLIENT_SES *router_cli_ses, + GWBUF *querybuf, qc_query_type_t type); +bool check_for_multi_stmt(GWBUF *buf, void *protocol, mysql_server_cmd_t packet_type); +qc_query_type_t determine_query_type(GWBUF *querybuf, int packet_type, bool non_empty_packet); + +#ifdef __cplusplus +} +#endif + +#endif /* RWSPLIT_INTERNAL_H */ + diff --git a/server/modules/routing/readwritesplit/CMakeLists.txt b/server/modules/routing/readwritesplit/CMakeLists.txt index 7c61efe5a..bdb6efdf5 100644 --- a/server/modules/routing/readwritesplit/CMakeLists.txt +++ b/server/modules/routing/readwritesplit/CMakeLists.txt @@ -1,4 +1,4 @@ -add_library(readwritesplit SHARED readwritesplit.c) +add_library(readwritesplit SHARED readwritesplit.c rwsplit_mysql.c rwsplit_route_stmt.c rwsplit_select_backends.c rwsplit_session_cmd.c rwsplit_tmp_table_multi.c) target_link_libraries(readwritesplit maxscale-common) set_target_properties(readwritesplit PROPERTIES VERSION "1.0.2") install_module(readwritesplit core) diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index 0fb07c35a..67f818003 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -4,13 +4,12 @@ * Use of this software is governed by the Business Source License included * in the LICENSE.TXT file and at www.mariadb.com/bsl. * - * Change Date: 2019-07-01 + * Change Date: 2019-01-01 * * On the date above, in accordance with the Business Source License, use * of this software will be governed by version 2 or later of the General * Public License. */ -#include #include #include #include @@ -19,8 +18,8 @@ #include #include +#include -#include #include #include #include @@ -28,8 +27,6 @@ #include #include #include -#include -#include #include MODULE_INFO info = @@ -38,18 +35,16 @@ MODULE_INFO info = "A Read/Write splitting router for enhancement read scalability" }; -#if defined(SS_DEBUG) -#include -#endif - -#define RWSPLIT_TRACE_MSG_LEN 1000 - /** * @file readwritesplit.c The entry points for the read/write query splitting * router module. * * This file contains the entry points that comprise the API to the read write - * query splitting router. + * query splitting router. It also contains functions that are directly called + * by the entry point functions. Some of these are used by functions in other + * modules of the read write split router, others are used only within this + * module. + * * @verbatim * Revision History * @@ -66,95 +61,36 @@ MODULE_INFO info = * 09/09/2015 Martin Brampton Modify error handler * 25/09/2015 Martin Brampton Block callback processing when no router * session in the DCB + * 03/08/2016 Martin Brampton Extract the API functions, move the rest * * @endverbatim */ static char *version_str = "V1.1.0"; +/* + * The functions that implement the router module API + */ + static ROUTER *createInstance(SERVICE *service, char **options); static void *newSession(ROUTER *instance, SESSION *session); static void closeSession(ROUTER *instance, void *session); static void freeSession(ROUTER *instance, void *session); static int routeQuery(ROUTER *instance, void *session, GWBUF *queue); -static void diagnostic(ROUTER *instance, DCB *dcb); - +static void diagnostics(ROUTER *instance, DCB *dcb); static void clientReply(ROUTER *instance, void *router_session, GWBUF *queue, DCB *backend_dcb); - static void handleError(ROUTER *instance, void *router_session, GWBUF *errmsgbuf, DCB *backend_dcb, error_action_t action, bool *succp); - -static void print_error_packet(ROUTER_CLIENT_SES *rses, GWBUF *buf, DCB *dcb); -static int router_get_servercount(ROUTER_INSTANCE *router); -static int rses_get_max_slavecount(ROUTER_CLIENT_SES *rses, - int router_nservers); -static int rses_get_max_replication_lag(ROUTER_CLIENT_SES *rses); -static backend_ref_t *get_bref_from_dcb(ROUTER_CLIENT_SES *rses, DCB *dcb); -static DCB *rses_get_client_dcb(ROUTER_CLIENT_SES *rses); - -static backend_ref_t *check_candidate_bref(backend_ref_t *candidate_bref, - backend_ref_t *new_bref, - select_criteria_t sc); - -static qc_query_type_t is_read_tmp_table(ROUTER_CLIENT_SES *router_cli_ses, - GWBUF *querybuf, qc_query_type_t type); - -static void check_create_tmp_table(ROUTER_CLIENT_SES *router_cli_ses, - GWBUF *querybuf, qc_query_type_t type); - -static bool route_single_stmt(ROUTER_INSTANCE *inst, ROUTER_CLIENT_SES *rses, - GWBUF *querybuf); - static int getCapabilities(); -#if defined(NOT_USED) -static bool router_option_configured(ROUTER_INSTANCE *router, - const char *optionstr, void *data); -#endif - -#if defined(PREP_STMT_CACHING) -static prep_stmt_t *prep_stmt_init(prep_stmt_type_t type, void *id); -static void prep_stmt_done(prep_stmt_t *pstmt); -#endif /*< PREP_STMT_CACHING */ - -int bref_cmp_global_conn(const void *bref1, const void *bref2); - -int bref_cmp_router_conn(const void *bref1, const void *bref2); - -int bref_cmp_behind_master(const void *bref1, const void *bref2); - -int bref_cmp_current_load(const void *bref1, const void *bref2); - -/** - * The order of functions _must_ match with the order the select criteria are - * listed in select_criteria_t definition in readwritesplit.h +/* + * End of the API functions; now the module structure that links to them. + * Note that the function names are chosen to exactly match the names used in + * the definition of ROUTER_OBJECT. This is not obligatory, but is done to + * make it easier to track the connection between calls and functions. */ -int (*criteria_cmpfun[LAST_CRITERIA])(const void *, const void *) = -{ - NULL, - bref_cmp_global_conn, - bref_cmp_router_conn, - bref_cmp_behind_master, - bref_cmp_current_load -}; - -static bool select_connect_backend_servers(backend_ref_t **p_master_ref, - backend_ref_t *backend_ref, - int router_nservers, int max_nslaves, - int max_rlag, - select_criteria_t select_criteria, - SESSION *session, - ROUTER_INSTANCE *router); - -static bool get_dcb(DCB **dcb, ROUTER_CLIENT_SES *rses, backend_type_t btype, - char *name, int max_rlag); - -static bool rwsplit_process_router_options(ROUTER_INSTANCE *router, - char **options); - - static ROUTER_OBJECT MyObject = { @@ -163,127 +99,37 @@ static ROUTER_OBJECT MyObject = closeSession, freeSession, routeQuery, - diagnostic, + diagnostics, clientReply, handleError, getCapabilities }; -static bool rses_begin_locked_router_action(ROUTER_CLIENT_SES *rses); - -static void rses_end_locked_router_action(ROUTER_CLIENT_SES *rses); - -static void mysql_sescmd_done(mysql_sescmd_t *sescmd); - -static mysql_sescmd_t *mysql_sescmd_init(rses_property_t *rses_prop, - GWBUF *sescmd_buf, - unsigned char packet_type, - ROUTER_CLIENT_SES *rses); - -static rses_property_t *mysql_sescmd_get_property(mysql_sescmd_t *scmd); - -static rses_property_t *rses_property_init(rses_property_type_t prop_type); - -static int rses_property_add(ROUTER_CLIENT_SES *rses, rses_property_t *prop); - -static void rses_property_done(rses_property_t *prop); - -static mysql_sescmd_t *rses_property_get_sescmd(rses_property_t *prop); - -static bool execute_sescmd_history(backend_ref_t *bref); - -static bool execute_sescmd_in_backend(backend_ref_t *backend_ref); - -static void sescmd_cursor_reset(sescmd_cursor_t *scur); - -static bool sescmd_cursor_history_empty(sescmd_cursor_t *scur); - -static void sescmd_cursor_set_active(sescmd_cursor_t *sescmd_cursor, - bool value); - -static bool sescmd_cursor_is_active(sescmd_cursor_t *sescmd_cursor); - -static GWBUF *sescmd_cursor_clone_querybuf(sescmd_cursor_t *scur); - -static mysql_sescmd_t *sescmd_cursor_get_command(sescmd_cursor_t *scur); - -static bool sescmd_cursor_next(sescmd_cursor_t *scur); - -static GWBUF *sescmd_cursor_process_replies(GWBUF *replybuf, - backend_ref_t *bref, bool *); - -static void tracelog_routed_query(ROUTER_CLIENT_SES *rses, char *funcname, - backend_ref_t *bref, GWBUF *buf); - -static bool route_session_write(ROUTER_CLIENT_SES *router_client_ses, - GWBUF *querybuf, ROUTER_INSTANCE *inst, - unsigned char packet_type, - qc_query_type_t qtype); - -static void refreshInstance(ROUTER_INSTANCE *router, CONFIG_PARAMETER *param); - -static void bref_clear_state(backend_ref_t *bref, bref_state_t state); -static void bref_set_state(backend_ref_t *bref, bref_state_t state); -static sescmd_cursor_t *backend_ref_get_sescmd_cursor(backend_ref_t *bref); - -static int router_handle_state_switch(DCB *dcb, DCB_REASON reason, void *data); -static bool handle_error_new_connection(ROUTER_INSTANCE *inst, - ROUTER_CLIENT_SES **rses, - DCB *backend_dcb, GWBUF *errmsg); -static void handle_error_reply_client(SESSION *ses, ROUTER_CLIENT_SES *rses, - DCB *backend_dcb, GWBUF *errmsg); - -static backend_ref_t *get_root_master_bref(ROUTER_CLIENT_SES *rses); - -static BACKEND *get_root_master(backend_ref_t *servers, int router_nservers); - -static bool have_enough_servers(ROUTER_CLIENT_SES **rses, const int nsrv, - int router_nsrv, ROUTER_INSTANCE *router); +/* + * A couple of static variables that are used throughout the router + */ static SPINLOCK instlock; static ROUTER_INSTANCE *instances; -static int hashkeyfun(const void *key); -static int hashcmpfun(const void *, const void *); -static bool check_for_multi_stmt(ROUTER_CLIENT_SES *rses, GWBUF *buf, - mysql_server_cmd_t packet_type); -static bool send_readonly_error(DCB *dcb); +/* + * Declaration of functions that are used only within this module, and are + * not part of the API. + */ -static int hashkeyfun(const void *key) -{ - if (key == NULL) - { - return 0; - } - - unsigned int hash = 0, c = 0; - const char *ptr = (const char *)key; - - while ((c = *ptr++)) - { - hash = c + (hash << 6) + (hash << 16) - hash; - } - return hash; -} - -static int hashcmpfun(const void *v1, const void *v2) -{ - const char *i1 = (const char *)v1; - const char *i2 = (const char *)v2; - - return strcmp(i1, i2); -} - -static void *hstrdup(const void *fval) -{ - char *str = (char *)fval; - return MXS_STRDUP(str); -} - -static void hfree(void *fval) -{ - MXS_FREE(fval); -} +static void refreshInstance(ROUTER_INSTANCE *router, + CONFIG_PARAMETER *singleparam); +static void free_rwsplit_instance(ROUTER_INSTANCE *router); +static bool rwsplit_process_router_options(ROUTER_INSTANCE *router, + char **options); +static void handle_error_reply_client(SESSION *ses, ROUTER_CLIENT_SES *rses, + DCB *backend_dcb, GWBUF *errmsg); +static bool handle_error_new_connection(ROUTER_INSTANCE *inst, + ROUTER_CLIENT_SES **rses, + DCB *backend_dcb, GWBUF *errmsg); +static int router_get_servercount(ROUTER_INSTANCE *inst); +static bool have_enough_servers(ROUTER_CLIENT_SES **p_rses, const int min_nsrv, + int router_nsrv, ROUTER_INSTANCE *router); /** * Implementation of the mandatory version entry point @@ -296,19 +142,19 @@ char *version() } /** - * The module initialisation routine, called when the module + * The module initialization routine, called when the module * is first loaded. */ void ModuleInit() { - MXS_NOTICE("Initializing statemend-based read/write split router module."); + MXS_NOTICE("Initializing statement-based read/write split router module."); spinlock_init(&instlock); instances = NULL; } /** * The module entry point routine. It is this routine that - * must populate the structure that is referred to as the + * must return the structure that is referred to as the * "module object", this is a structure with the set of * external entry points for this module. * @@ -319,166 +165,17 @@ ROUTER_OBJECT *GetModuleObject() return &MyObject; } -/** - * Refresh the instance by the given parameter value. - * - * @param router Router instance - * @param singleparam Parameter fo be reloaded - * - * Note: this part is not done. Needs refactoring. +/* + * Now we implement the API functions */ -static void refreshInstance(ROUTER_INSTANCE *router, - CONFIG_PARAMETER *singleparam) -{ - CONFIG_PARAMETER *param; - bool refresh_single; - config_param_type_t paramtype; - - if (singleparam != NULL) - { - param = singleparam; - refresh_single = true; - } - else - { - param = router->service->svc_config_param; - refresh_single = false; - } - paramtype = config_get_paramtype(param); - - while (param != NULL) - { - /** Catch unused parameter types */ - ss_dassert(paramtype == COUNT_TYPE || paramtype == PERCENT_TYPE || - paramtype == SQLVAR_TARGET_TYPE); - - if (paramtype == COUNT_TYPE) - { - if (strncmp(param->name, "max_slave_connections", MAX_PARAM_LEN) == 0) - { - int val; - bool succp; - - router->rwsplit_config.rw_max_slave_conn_percent = 0; - - succp = config_get_valint(&val, param, NULL, paramtype); - - if (succp) - { - router->rwsplit_config.rw_max_slave_conn_count = val; - } - } - else if (strncmp(param->name, "max_slave_replication_lag", - MAX_PARAM_LEN) == 0) - { - int val; - bool succp; - - succp = config_get_valint(&val, param, NULL, paramtype); - - if (succp) - { - router->rwsplit_config.rw_max_slave_replication_lag = val; - } - } - } - else if (paramtype == PERCENT_TYPE) - { - if (strncmp(param->name, "max_slave_connections", MAX_PARAM_LEN) == 0) - { - int val; - bool succp; - - router->rwsplit_config.rw_max_slave_conn_count = 0; - - succp = config_get_valint(&val, param, NULL, paramtype); - - if (succp) - { - router->rwsplit_config.rw_max_slave_conn_percent = val; - } - } - } - else if (paramtype == SQLVAR_TARGET_TYPE) - { - if (strncmp(param->name, "use_sql_variables_in", MAX_PARAM_LEN) == 0) - { - target_t valtarget; - bool succp; - - succp = config_get_valtarget(&valtarget, param, NULL, paramtype); - - if (succp) - { - router->rwsplit_config.rw_use_sql_variables_in = valtarget; - } - } - } - - if (refresh_single) - { - break; - } - param = param->next; - } - -#if defined(NOT_USED) /*< can't read monitor config parameters */ - if ((*router->servers)->backend_server->rlag == -2) - { - rlag_enabled = false; - } - else - { - rlag_enabled = true; - } - /** - * If replication lag detection is not enabled the measure can't be - * used in slave selection. - */ - if (!rlag_enabled) - { - if (rlag_limited) - { - MXS_WARNING("Configuration Failed, max_slave_replication_lag " - "is set to %d,\n\t\t but detect_replication_lag " - "is not enabled. Replication lag will not be checked.", - router->rwsplit_config.rw_max_slave_replication_lag); - } - - if (router->rwsplit_config.rw_slave_select_criteria == - LEAST_BEHIND_MASTER) - { - MXS_WARNING("Configuration Failed, router option " - "\n\t\t slave_selection_criteria=LEAST_BEHIND_MASTER " - "is specified, but detect_replication_lag " - "is not enabled.\n\t\t " - "slave_selection_criteria=%s will be used instead.", - STRCRITERIA(DEFAULT_CRITERIA)); - - router->rwsplit_config.rw_slave_select_criteria = DEFAULT_CRITERIA; - } - } -#endif /*< NOT_USED */ -} - -static inline void free_rwsplit_instance(ROUTER_INSTANCE *router) -{ - if (router) - { - if (router->servers) - { - for (int i = 0; router->servers[i]; i++) - { - MXS_FREE(router->servers[i]); - } - } - MXS_FREE(router->servers); - MXS_FREE(router); - } -} /** - * Create an instance of read/write statement router within the MaxScale. + * @brief Create an instance of the read/write router (API). + * + * Create an instance of read/write statement router within the MaxScale. One + * instance of the router is required for each service that is defined in the + * configuration as using this router. One instance of the router will handle + * multiple connections (or router sessions). * * @param service The service this router is being create for * @param options The options for this query router @@ -702,14 +399,21 @@ static ROUTER *createInstance(SERVICE *service, char **options) } /** - * Associate a new session with this instance of the router. + * @brief Associate a new session with this instance of the router (API). * - * The session is used to store all the data required for a particular - * client connection. + * The session is used to store all the data required by the router for a + * particular client connection. The instance of the router that relates to a + * particular service is passed as the first parameter. The second parameter is + * the session that has been created in response to the request from a client + * for a connection. The passed session contains generic information; this + * function creates the session structure that holds router specific data. + * There is often a one to one relationship between sessions and router + * sessions, although it is possible to create configurations where a + * connection is handled by multiple routers, one after another. * * @param instance The router instance data - * @param session The session itself - * @return Session specific data for this session + * @param session The MaxScale session (generic connection data) + * @return Session specific data for this session, i.e. a router session */ static void *newSession(ROUTER *router_inst, SESSION *session) { @@ -885,11 +589,16 @@ return_rses: } /** - * Close a session with the router, this is the mechanism - * by which a router may cleanup data structure etc. + * @brief Close a router session (API). + * + * Close a session with the router, this is the mechanism by which a router + * may cleanup data structure etc. The instance of the router that relates to + * the relevant service is passed, along with the router session that is to + * be closed. Typically the function is used in conjunction with freeSession + * which will release the resources used by a router session (see below). * * @param instance The router instance data - * @param session The session being closed + * @param session The router session being closed */ static void closeSession(ROUTER *instance, void *router_session) { @@ -942,7 +651,7 @@ static void closeSession(ROUTER *instance, void *router_session) } #endif /** Clean operation counter in bref and in SERVER */ - if (BREF_IS_WAITING_RESULT(bref)) + while (BREF_IS_WAITING_RESULT(bref)) { bref_clear_state(bref, BREF_WAITING_RESULT); } @@ -955,10 +664,6 @@ static void closeSession(ROUTER *instance, void *router_session) /** decrease server current connection counters */ atomic_add(&bref->bref_backend->backend_conn_count, -1); } - else - { - ss_dassert(!BREF_IS_WAITING_RESULT(bref)); - } } /** Unlock */ rses_end_locked_router_action(router_cli_ses); @@ -966,8 +671,10 @@ static void closeSession(ROUTER *instance, void *router_session) } /** - * When router session is closed, freeSession can be called to free allocated - * resources. + * @brief Free a router session (API). + * + * When a router session has been closed, freeSession can be called to free + * allocated resources. * * @param router_instance The router instance the session belongs to * @param router_client_session Client session @@ -1031,754 +738,9 @@ static void freeSession(ROUTER *router_instance, void *router_client_session) } /** - * Provide the router with a pointer to a suitable backend dcb. + * @brief The main routing entry point for a query (API) * - * Detect failures in server statuses and reselect backends if necessary. - * If name is specified, server name becomes primary selection criteria. - * Similarly, if max replication lag is specified, skip backends which lag too - * much. - * - * @param p_dcb Address of the pointer to the resulting DCB - * @param rses Pointer to router client session - * @param btype Backend type - * @param name Name of the backend which is primarily searched. May be NULL. - * - * @return True if proper DCB was found, false otherwise. - */ -static bool get_dcb(DCB **p_dcb, ROUTER_CLIENT_SES *rses, backend_type_t btype, - char *name, int max_rlag) -{ - backend_ref_t *backend_ref; - backend_ref_t *master_bref; - int i; - bool succp = false; - - CHK_CLIENT_RSES(rses); - ss_dassert(p_dcb != NULL && *(p_dcb) == NULL); - - if (p_dcb == NULL) - { - goto return_succp; - } - backend_ref = rses->rses_backend_ref; - - /** get root master from available servers */ - master_bref = get_root_master_bref(rses); - - if (name != NULL) /*< Choose backend by name from a hint */ - { - ss_dassert(btype != BE_MASTER); /*< Master dominates and no name should be passed with it */ - - for (i = 0; i < rses->rses_nbackends; i++) - { - BACKEND *b = backend_ref[i].bref_backend; - SERVER server; - server.status = backend_ref[i].bref_backend->backend_server->status; - /** - * To become chosen: - * backend must be in use, name must match, - * backend's role must be either slave, relay - * server, or master. - */ - if (BREF_IS_IN_USE((&backend_ref[i])) && - (strncasecmp(name, b->backend_server->unique_name, PATH_MAX) == 0) && - (SERVER_IS_SLAVE(&server) || SERVER_IS_RELAY_SERVER(&server) || - SERVER_IS_MASTER(&server))) - { - *p_dcb = backend_ref[i].bref_dcb; - succp = true; - ss_dassert(backend_ref[i].bref_dcb->state != DCB_STATE_ZOMBIE); - break; - } - } - if (succp) - { - goto return_succp; - } - else - { - btype = BE_SLAVE; - } - } - - if (btype == BE_SLAVE) - { - backend_ref_t *candidate_bref = NULL; - - for (i = 0; i < rses->rses_nbackends; i++) - { - BACKEND *b = (&backend_ref[i])->bref_backend; - SERVER server; - SERVER candidate; - server.status = backend_ref[i].bref_backend->backend_server->status; - /** - * Unused backend or backend which is not master nor - * slave can't be used - */ - if (!BREF_IS_IN_USE(&backend_ref[i]) || - (!SERVER_IS_MASTER(&server) && !SERVER_IS_SLAVE(&server))) - { - continue; - } - /** - * If there are no candidates yet accept both master or - * slave. - */ - else if (candidate_bref == NULL) - { - /** - * Ensure that master has not changed dunring - * session and abort if it has. - */ - if (SERVER_IS_MASTER(&server) && &backend_ref[i] == master_bref) - { - /** found master */ - candidate_bref = &backend_ref[i]; - candidate.status = candidate_bref->bref_backend->backend_server->status; - succp = true; - } - /** - * Ensure that max replication lag is not set - * or that candidate's lag doesn't exceed the - * maximum allowed replication lag. - */ - else if (max_rlag == MAX_RLAG_UNDEFINED || - (b->backend_server->rlag != MAX_RLAG_NOT_AVAILABLE && - b->backend_server->rlag <= max_rlag)) - { - /** found slave */ - candidate_bref = &backend_ref[i]; - candidate.status = candidate_bref->bref_backend->backend_server->status; - succp = true; - } - } - /** - * If candidate is master, any slave which doesn't break - * replication lag limits replaces it. - */ - else if (SERVER_IS_MASTER(&candidate) && SERVER_IS_SLAVE(&server) && - (max_rlag == MAX_RLAG_UNDEFINED || - (b->backend_server->rlag != MAX_RLAG_NOT_AVAILABLE && - b->backend_server->rlag <= max_rlag)) && - !rses->rses_config.rw_master_reads) - { - /** found slave */ - candidate_bref = &backend_ref[i]; - candidate.status = candidate_bref->bref_backend->backend_server->status; - succp = true; - } - /** - * When candidate exists, compare it against the current - * backend and update assign it to new candidate if - * necessary. - */ - else if (SERVER_IS_SLAVE(&server)) - { - if (max_rlag == MAX_RLAG_UNDEFINED || - (b->backend_server->rlag != MAX_RLAG_NOT_AVAILABLE && - b->backend_server->rlag <= max_rlag)) - { - candidate_bref = - check_candidate_bref(candidate_bref, &backend_ref[i], - rses->rses_config.rw_slave_select_criteria); - candidate.status = - candidate_bref->bref_backend->backend_server->status; - } - else - { - MXS_INFO("Server %s:%d is too much behind the " - "master, %d s. and can't be chosen.", - b->backend_server->name, b->backend_server->port, - b->backend_server->rlag); - } - } - } /*< for */ - /** Assign selected DCB's pointer value */ - if (candidate_bref != NULL) - { - *p_dcb = candidate_bref->bref_dcb; - } - - goto return_succp; - } /*< if (btype == BE_SLAVE) */ - /** - * If target was originally master only then the execution jumps - * directly here. - */ - if (btype == BE_MASTER) - { - if (master_bref) - { - /** It is possible for the server status to change at any point in time - * so copying it locally will make possible error messages - * easier to understand */ - SERVER server; - server.status = master_bref->bref_backend->backend_server->status; - if (BREF_IS_IN_USE(master_bref) && SERVER_IS_MASTER(&server)) - { - *p_dcb = master_bref->bref_dcb; - succp = true; - /** if bref is in use DCB should not be closed */ - ss_dassert(master_bref->bref_dcb->state != DCB_STATE_ZOMBIE); - } - else - { - MXS_ERROR("Server at %s:%d should be master but " - "is %s instead and can't be chosen to master.", - master_bref->bref_backend->backend_server->name, - master_bref->bref_backend->backend_server->port, - STRSRVSTATUS(&server)); - succp = false; - } - } - } - -return_succp: - return succp; -} - -/** - * Find out which of the two backend servers has smaller value for select - * criteria property. - * - * @param cand previously selected candidate - * @param new challenger - * @param sc select criteria - * - * @return pointer to backend reference of that backend server which has smaller - * value in selection criteria. If either reference pointer is NULL then the - * other reference pointer value is returned. - */ -static backend_ref_t *check_candidate_bref(backend_ref_t *cand, - backend_ref_t *new, - select_criteria_t sc) -{ - int (*p)(const void *, const void *); - /** get compare function */ - p = criteria_cmpfun[sc]; - - if (new == NULL) - { - return cand; - } - else if (cand == NULL || (p((void *)cand, (void *)new) > 0)) - { - return new; - } - else - { - return cand; - } -} - -/** - * Examine the query type, transaction state and routing hints. Find out the - * target for query routing. - * - * @param qtype Type of query - * @param trx_active Is transacation active or not - * @param hint Pointer to list of hints attached to the query buffer - * - * @return bitfield including the routing target, or the target server name - * if the query would otherwise be routed to slave. - */ -static route_target_t get_route_target(ROUTER_CLIENT_SES *rses, - qc_query_type_t qtype, HINT *hint) -{ - bool trx_active = rses->rses_transaction_active; - bool load_active = rses->rses_load_active; - target_t use_sql_variables_in = rses->rses_config.rw_use_sql_variables_in; - route_target_t target = TARGET_UNDEFINED; - - if (rses->rses_config.rw_strict_multi_stmt && rses->forced_node && - rses->forced_node == rses->rses_master_ref) - { - target = TARGET_MASTER; - } - /** - * These queries are not affected by hints - */ - else if (!load_active && - (QUERY_IS_TYPE(qtype, QUERY_TYPE_SESSION_WRITE) || - /** Configured to allow writing variables to all nodes */ - (use_sql_variables_in == TYPE_ALL && - QUERY_IS_TYPE(qtype, QUERY_TYPE_GSYSVAR_WRITE)) || - /** enable or disable autocommit are always routed to all */ - QUERY_IS_TYPE(qtype, QUERY_TYPE_ENABLE_AUTOCOMMIT) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_DISABLE_AUTOCOMMIT))) - { - /** - * This is problematic query because it would be routed to all - * backends but since this is SELECT that is not possible: - * 1. response set is not handled correctly in clientReply and - * 2. multiple results can degrade performance. - * - * Prepared statements are an exception to this since they do not - * actually do anything but only prepare the statement to be used. - * They can be safely routed to all backends since the execution - * is done later. - * - * With prepared statement caching the task of routing - * the execution of the prepared statements to the right server would be - * an easy one. Currently this is not supported. - */ - if (QUERY_IS_TYPE(qtype, QUERY_TYPE_READ) && - !(QUERY_IS_TYPE(qtype, QUERY_TYPE_PREPARE_STMT) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_PREPARE_NAMED_STMT))) - { - MXS_WARNING("The query can't be routed to all " - "backend servers because it includes SELECT and " - "SQL variable modifications which is not supported. " - "Set use_sql_variables_in=master or split the " - "query to two, where SQL variable modifications " - "are done in the first and the SELECT in the " - "second one."); - - target = TARGET_MASTER; - } - target |= TARGET_ALL; - } - /** - * Hints may affect on routing of the following queries - */ - else if (!trx_active && !load_active && - !QUERY_IS_TYPE(qtype, QUERY_TYPE_WRITE) && - (QUERY_IS_TYPE(qtype, QUERY_TYPE_READ) || /*< any SELECT */ - QUERY_IS_TYPE(qtype, QUERY_TYPE_SHOW_TABLES) || /*< 'SHOW TABLES' */ - QUERY_IS_TYPE(qtype, - QUERY_TYPE_USERVAR_READ) || /*< read user var */ - QUERY_IS_TYPE(qtype, QUERY_TYPE_SYSVAR_READ) || /*< read sys var */ - QUERY_IS_TYPE(qtype, - QUERY_TYPE_EXEC_STMT) || /*< prepared stmt exec */ - QUERY_IS_TYPE(qtype, QUERY_TYPE_PREPARE_STMT) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_PREPARE_NAMED_STMT) || - QUERY_IS_TYPE(qtype, - QUERY_TYPE_GSYSVAR_READ))) /*< read global sys var */ - { - /** First set expected targets before evaluating hints */ - if (!QUERY_IS_TYPE(qtype, QUERY_TYPE_MASTER_READ) && - (QUERY_IS_TYPE(qtype, QUERY_TYPE_READ) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_SHOW_TABLES) || /*< 'SHOW TABLES' */ - /** Configured to allow reading variables from slaves */ - (use_sql_variables_in == TYPE_ALL && - (QUERY_IS_TYPE(qtype, QUERY_TYPE_USERVAR_READ) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_SYSVAR_READ) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_GSYSVAR_READ))))) - { - target = TARGET_SLAVE; - } - - if (QUERY_IS_TYPE(qtype, QUERY_TYPE_MASTER_READ) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_EXEC_STMT) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_PREPARE_STMT) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_PREPARE_NAMED_STMT) || - /** Configured not to allow reading variables from slaves */ - (use_sql_variables_in == TYPE_MASTER && - (QUERY_IS_TYPE(qtype, QUERY_TYPE_USERVAR_READ) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_SYSVAR_READ)))) - { - target = TARGET_MASTER; - } - - /** If nothing matches then choose the master */ - if ((target & (TARGET_ALL | TARGET_SLAVE | TARGET_MASTER)) == 0) - { - target = TARGET_MASTER; - } - } - else - { - ss_dassert(trx_active || - (QUERY_IS_TYPE(qtype, QUERY_TYPE_WRITE) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_MASTER_READ) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_SESSION_WRITE) || - (QUERY_IS_TYPE(qtype, QUERY_TYPE_USERVAR_READ) && - use_sql_variables_in == TYPE_MASTER) || - (QUERY_IS_TYPE(qtype, QUERY_TYPE_SYSVAR_READ) && - use_sql_variables_in == TYPE_MASTER) || - (QUERY_IS_TYPE(qtype, QUERY_TYPE_GSYSVAR_READ) && - use_sql_variables_in == TYPE_MASTER) || - (QUERY_IS_TYPE(qtype, QUERY_TYPE_GSYSVAR_WRITE) && - use_sql_variables_in == TYPE_MASTER) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_BEGIN_TRX) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_ENABLE_AUTOCOMMIT) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_DISABLE_AUTOCOMMIT) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_ROLLBACK) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_COMMIT) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_EXEC_STMT) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_CREATE_TMP_TABLE) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_READ_TMP_TABLE) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_UNKNOWN))); - target = TARGET_MASTER; - } - - /** process routing hints */ - while (hint != NULL) - { - if (hint->type == HINT_ROUTE_TO_MASTER) - { - target = TARGET_MASTER; /*< override */ - MXS_DEBUG("%lu [get_route_target] Hint: route to master.", - pthread_self()); - break; - } - else if (hint->type == HINT_ROUTE_TO_NAMED_SERVER) - { - /** - * Searching for a named server. If it can't be - * found, the oroginal target is chosen. - */ - target |= TARGET_NAMED_SERVER; - MXS_DEBUG("%lu [get_route_target] Hint: route to " - "named server : ", - pthread_self()); - } - else if (hint->type == HINT_ROUTE_TO_UPTODATE_SERVER) - { - /** not implemented */ - } - else if (hint->type == HINT_ROUTE_TO_ALL) - { - /** not implemented */ - } - else if (hint->type == HINT_PARAMETER) - { - if (strncasecmp((char *)hint->data, "max_slave_replication_lag", - strlen("max_slave_replication_lag")) == 0) - { - target |= TARGET_RLAG_MAX; - } - else - { - MXS_ERROR("Unknown hint parameter " - "'%s' when 'max_slave_replication_lag' " - "was expected.", - (char *)hint->data); - } - } - else if (hint->type == HINT_ROUTE_TO_SLAVE) - { - target = TARGET_SLAVE; - MXS_DEBUG("%lu [get_route_target] Hint: route to " - "slave.", - pthread_self()); - } - hint = hint->next; - } /*< while (hint != NULL) */ - -#if defined(SS_EXTRA_DEBUG) - MXS_INFO("Selected target \"%s\"", STRTARGET(target)); -#endif - return target; -} - -/** - * Check if the query is a DROP TABLE... query and - * if it targets a temporary table, remove it from the hashtable. - * @param router_cli_ses Router client session - * @param querybuf GWBUF containing the query - * @param type The type of the query resolved so far - */ -void check_drop_tmp_table(ROUTER_CLIENT_SES *router_cli_ses, GWBUF *querybuf, - qc_query_type_t type) -{ - - int tsize = 0, klen = 0, i; - char **tbl = NULL; - char *hkey, *dbname; - MYSQL_session *data; - rses_property_t *rses_prop_tmp; - - if (router_cli_ses == NULL || querybuf == NULL) - { - MXS_ERROR("[%s] Error: NULL parameters passed: %p %p", __FUNCTION__, - router_cli_ses, querybuf); - return; - } - - if (router_cli_ses->client_dcb == NULL) - { - MXS_ERROR("[%s] Error: Client DCB is NULL.", __FUNCTION__); - return; - } - - rses_prop_tmp = router_cli_ses->rses_properties[RSES_PROP_TYPE_TMPTABLES]; - data = (MYSQL_session *)router_cli_ses->client_dcb->data; - - if (data == NULL) - { - MXS_ERROR("[%s] Error: User data in master server DBC is NULL.", - __FUNCTION__); - return; - } - - dbname = (char *)data->db; - - if (qc_is_drop_table_query(querybuf)) - { - tbl = qc_get_table_names(querybuf, &tsize, false); - if (tbl != NULL) - { - for (i = 0; i < tsize; i++) - { - klen = strlen(dbname) + strlen(tbl[i]) + 2; - hkey = MXS_CALLOC(klen, sizeof(char)); - MXS_ABORT_IF_NULL(hkey); - strcpy(hkey, dbname); - strcat(hkey, "."); - strcat(hkey, tbl[i]); - - if (rses_prop_tmp && rses_prop_tmp->rses_prop_data.temp_tables) - { - if (hashtable_delete(rses_prop_tmp->rses_prop_data.temp_tables, - (void *)hkey)) - { - MXS_INFO("Temporary table dropped: %s", hkey); - } - } - MXS_FREE(tbl[i]); - MXS_FREE(hkey); - } - - MXS_FREE(tbl); - } - } -} - -/** - * Check if the query targets a temporary table. - * @param router_cli_ses Router client session - * @param querybuf GWBUF containing the query - * @param type The type of the query resolved so far - * @return The type of the query - */ -static qc_query_type_t is_read_tmp_table(ROUTER_CLIENT_SES *router_cli_ses, - GWBUF *querybuf, - qc_query_type_t type) -{ - - bool target_tmp_table = false; - int tsize = 0, klen = 0, i; - char **tbl = NULL; - char *dbname; - char hkey[MYSQL_DATABASE_MAXLEN + MYSQL_TABLE_MAXLEN + 2]; - MYSQL_session *data; - qc_query_type_t qtype = type; - rses_property_t *rses_prop_tmp; - - if (router_cli_ses == NULL || querybuf == NULL) - { - MXS_ERROR("[%s] Error: NULL parameters passed: %p %p", __FUNCTION__, - router_cli_ses, querybuf); - return type; - } - - if (router_cli_ses->client_dcb == NULL) - { - MXS_ERROR("[%s] Error: Client DCB is NULL.", __FUNCTION__); - return type; - } - - rses_prop_tmp = router_cli_ses->rses_properties[RSES_PROP_TYPE_TMPTABLES]; - data = (MYSQL_session *)router_cli_ses->client_dcb->data; - - if (data == NULL) - { - MXS_ERROR("[%s] Error: User data in client DBC is NULL.", __FUNCTION__); - return qtype; - } - - dbname = (char *)data->db; - - if (QUERY_IS_TYPE(qtype, QUERY_TYPE_READ) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_LOCAL_READ) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_USERVAR_READ) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_SYSVAR_READ) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_GSYSVAR_READ)) - { - tbl = qc_get_table_names(querybuf, &tsize, false); - - if (tbl != NULL && tsize > 0) - { - /** Query targets at least one table */ - for (i = 0; i < tsize && !target_tmp_table && tbl[i]; i++) - { - sprintf(hkey, "%s.%s", dbname, tbl[i]); - if (rses_prop_tmp && rses_prop_tmp->rses_prop_data.temp_tables) - { - if (hashtable_fetch(rses_prop_tmp->rses_prop_data.temp_tables, hkey)) - { - /**Query target is a temporary table*/ - qtype = QUERY_TYPE_READ_TMP_TABLE; - MXS_INFO("Query targets a temporary table: %s", hkey); - break; - } - } - } - } - } - - if (tbl != NULL) - { - for (i = 0; i < tsize; i++) - { - MXS_FREE(tbl[i]); - } - MXS_FREE(tbl); - } - - return qtype; -} - -/** - * If query is of type QUERY_TYPE_CREATE_TMP_TABLE then find out - * the database and table name, create a hashvalue and - * add it to the router client session's property. If property - * doesn't exist then create it first. - * @param router_cli_ses Router client session - * @param querybuf GWBUF containing the query - * @param type The type of the query resolved so far - */ -static void check_create_tmp_table(ROUTER_CLIENT_SES *router_cli_ses, - GWBUF *querybuf, qc_query_type_t type) -{ - if (!QUERY_IS_TYPE(type, QUERY_TYPE_CREATE_TMP_TABLE)) - { - return; - } - - int klen = 0; - char *hkey, *dbname; - MYSQL_session *data; - rses_property_t *rses_prop_tmp; - HASHTABLE *h; - - if (router_cli_ses == NULL || querybuf == NULL) - { - MXS_ERROR("[%s] Error: NULL parameters passed: %p %p", __FUNCTION__, - router_cli_ses, querybuf); - return; - } - - if (router_cli_ses->client_dcb == NULL) - { - MXS_ERROR("[%s] Error: Client DCB is NULL.", __FUNCTION__); - return; - } - - router_cli_ses->have_tmp_tables = true; - rses_prop_tmp = router_cli_ses->rses_properties[RSES_PROP_TYPE_TMPTABLES]; - data = (MYSQL_session *)router_cli_ses->client_dcb->data; - - if (data == NULL) - { - MXS_ERROR("[%s] Error: User data in master server DBC is NULL.", - __FUNCTION__); - return; - } - - dbname = (char *)data->db; - - bool is_temp = true; - char *tblname = NULL; - - tblname = qc_get_created_table_name(querybuf); - - if (tblname && strlen(tblname) > 0) - { - klen = strlen(dbname) + strlen(tblname) + 2; - hkey = MXS_CALLOC(klen, sizeof(char)); - MXS_ABORT_IF_NULL(hkey); - strcpy(hkey, dbname); - strcat(hkey, "."); - strcat(hkey, tblname); - } - else - { - hkey = NULL; - } - - if (rses_prop_tmp == NULL) - { - if ((rses_prop_tmp = (rses_property_t *)MXS_CALLOC(1, sizeof(rses_property_t)))) - { -#if defined(SS_DEBUG) - rses_prop_tmp->rses_prop_chk_top = CHK_NUM_ROUTER_PROPERTY; - rses_prop_tmp->rses_prop_chk_tail = CHK_NUM_ROUTER_PROPERTY; -#endif - rses_prop_tmp->rses_prop_rsession = router_cli_ses; - rses_prop_tmp->rses_prop_refcount = 1; - rses_prop_tmp->rses_prop_next = NULL; - rses_prop_tmp->rses_prop_type = RSES_PROP_TYPE_TMPTABLES; - router_cli_ses->rses_properties[RSES_PROP_TYPE_TMPTABLES] = rses_prop_tmp; - } - } - if (rses_prop_tmp) - { - if (rses_prop_tmp->rses_prop_data.temp_tables == NULL) - { - h = hashtable_alloc(7, hashkeyfun, hashcmpfun); - hashtable_memory_fns(h, hstrdup, NULL, hfree, NULL); - if (h != NULL) - { - rses_prop_tmp->rses_prop_data.temp_tables = h; - } - else - { - MXS_ERROR("Failed to allocate a new hashtable."); - } - } - - if (hkey && rses_prop_tmp->rses_prop_data.temp_tables && - hashtable_add(rses_prop_tmp->rses_prop_data.temp_tables, (void *)hkey, - (void *)is_temp) == 0) /*< Conflict in hash table */ - { - MXS_INFO("Temporary table conflict in hashtable: %s", hkey); - } -#if defined(SS_DEBUG) - { - bool retkey = hashtable_fetch(rses_prop_tmp->rses_prop_data.temp_tables, hkey); - if (retkey) - { - MXS_INFO("Temporary table added: %s", hkey); - } - } -#endif - } - - MXS_FREE(hkey); - MXS_FREE(tblname); -} - -/** - * Get client DCB pointer of the router client session. - * This routine must be protected by Router client session lock. - * - * @param rses Router client session pointer - * - * @return Pointer to client DCB - */ -static DCB *rses_get_client_dcb(ROUTER_CLIENT_SES *rses) -{ - DCB *dcb = NULL; - int i; - - for (i = 0; i < rses->rses_nbackends; i++) - { - if ((dcb = rses->rses_backend_ref[i].bref_dcb) != NULL && - BREF_IS_IN_USE(&rses->rses_backend_ref[i]) && dcb->session != NULL && - dcb->session->client_dcb != NULL) - { - return dcb->session->client_dcb; - } - } - return NULL; -} - -/** - * @brief The main routing entry point - * - * The routeQuery will make the routing decision based on the contents + * The routeQuery function will make the routing decision based on the contents * of the instance, session and the query itself. The query always represents * a complete MariaDB/MySQL packet because we define the RCAP_TYPE_STMT_INPUT in * getCapabilities(). @@ -1798,34 +760,11 @@ static int routeQuery(ROUTER *instance, void *router_session, GWBUF *querybuf) if (rses->rses_closed) { - uint8_t* data = GWBUF_DATA(querybuf); - - if (GWBUF_LENGTH(querybuf) >= 5 && !MYSQL_IS_COM_QUIT(data)) - { - char *query_str = modutil_get_query(querybuf); - MXS_ERROR("Can't route %s:\"%s\" to backend server. Router is closed.", - STRPACKETTYPE(data[4]), query_str ? query_str : "(empty)"); - MXS_FREE(query_str); - } + closed_session_reply(querybuf); } else { - if (GWBUF_IS_TYPE_UNDEFINED(querybuf)) - { - GWBUF *tmpbuf = querybuf; - - querybuf = modutil_get_complete_packets(&tmpbuf); - if (tmpbuf) - { - rses->client_dcb->dcb_readqueue = gwbuf_append(rses->client_dcb->dcb_readqueue, tmpbuf); - } - querybuf = gwbuf_make_contiguous(querybuf); - - /** Mark buffer to as MySQL type */ - gwbuf_set_type(querybuf, GWBUF_TYPE_MYSQL); - gwbuf_set_type(querybuf, GWBUF_TYPE_SINGLE_STMT); - } - + live_session_reply(&querybuf, rses); if (route_single_stmt(inst, rses, querybuf)) { rval = 1; @@ -1841,603 +780,14 @@ static int routeQuery(ROUTER *instance, void *router_session, GWBUF *querybuf) } /** - * Routing function. Find out query type, backend type, and target DCB(s). - * Then route query to found target(s). - * @param inst router instance - * @param rses router session - * @param querybuf GWBUF including the query - * - * @return true if routing succeed or if it failed due to unsupported query. - * false if backend failure was encountered. - */ -static bool route_single_stmt(ROUTER_INSTANCE *inst, ROUTER_CLIENT_SES *rses, - GWBUF *querybuf) -{ - qc_query_type_t qtype = QUERY_TYPE_UNKNOWN; - mysql_server_cmd_t packet_type = MYSQL_COM_UNDEFINED; - uint8_t *packet; - size_t packet_len; - int ret = 0; - DCB *target_dcb = NULL; - route_target_t route_target; - bool succp = false; - int rlag_max = MAX_RLAG_UNDEFINED; - backend_type_t btype; /*< target backend type */ - - ss_dassert(querybuf->next == NULL); // The buffer must be contiguous. - ss_dassert(!GWBUF_IS_TYPE_UNDEFINED(querybuf)); - - packet = GWBUF_DATA(querybuf); - packet_len = gw_mysql_get_byte3(packet); - - if (packet_len == 0) - { - /** Empty packet signals end of LOAD DATA LOCAL INFILE, send it to master*/ - route_target = TARGET_MASTER; - packet_type = MYSQL_COM_UNDEFINED; - rses->rses_load_active = false; - route_target = TARGET_MASTER; - MXS_INFO("> LOAD DATA LOCAL INFILE finished: %lu bytes sent.", - rses->rses_load_data_sent + gwbuf_length(querybuf)); - } - else - { - packet_type = packet[4]; - - switch (packet_type) - { - case MYSQL_COM_QUIT: /*< 1 QUIT will close all sessions */ - case MYSQL_COM_INIT_DB: /*< 2 DDL must go to the master */ - case MYSQL_COM_REFRESH: /*< 7 - I guess this is session but not sure */ - case MYSQL_COM_DEBUG: /*< 0d all servers dump debug info to stdout */ - case MYSQL_COM_PING: /*< 0e all servers are pinged */ - case MYSQL_COM_CHANGE_USER: /*< 11 all servers change it accordingly */ - qtype = QUERY_TYPE_SESSION_WRITE; - break; - - case MYSQL_COM_CREATE_DB: /**< 5 DDL must go to the master */ - case MYSQL_COM_DROP_DB: /**< 6 DDL must go to the master */ - case MYSQL_COM_STMT_CLOSE: /*< free prepared statement */ - case MYSQL_COM_STMT_SEND_LONG_DATA: /*< send data to column */ - case MYSQL_COM_STMT_RESET: /*< resets the data of a prepared statement */ - qtype = QUERY_TYPE_WRITE; - break; - - case MYSQL_COM_QUERY: - qtype = qc_get_type(querybuf); - break; - - case MYSQL_COM_STMT_PREPARE: - qtype = qc_get_type(querybuf); - qtype |= QUERY_TYPE_PREPARE_STMT; - break; - - case MYSQL_COM_STMT_EXECUTE: - /** Parsing is not needed for this type of packet */ - qtype = QUERY_TYPE_EXEC_STMT; - break; - - case MYSQL_COM_SHUTDOWN: /**< 8 where should shutdown be routed ? */ - case MYSQL_COM_STATISTICS: /**< 9 ? */ - case MYSQL_COM_PROCESS_INFO: /**< 0a ? */ - case MYSQL_COM_CONNECT: /**< 0b ? */ - case MYSQL_COM_PROCESS_KILL: /**< 0c ? */ - case MYSQL_COM_TIME: /**< 0f should this be run in gateway ? */ - case MYSQL_COM_DELAYED_INSERT: /**< 10 ? */ - case MYSQL_COM_DAEMON: /**< 1d ? */ - default: - break; - } /**< switch by packet type */ - - /** This might not be absolutely necessary as some parts of the code - * can only be executed by one thread at a time. */ - if (!rses_begin_locked_router_action(rses)) - { - succp = false; - goto retblock; - } - - /** Check for multi-statement queries. If no master server is available - * and a multi-statement is issued, an error is returned to the client - * when the query is routed. - * - * If we do not have a master node, assigning the forced node is not - * effective since we don't have a node to force queries to. In this - * situation, assigning QUERY_TYPE_WRITE for the query will trigger - * the error processing. */ - if (check_for_multi_stmt(rses, querybuf, packet_type) && - rses->rses_master_ref == NULL) - { - qtype |= QUERY_TYPE_WRITE; - } - - /** - * Check if the query has anything to do with temporary tables. - */ - if (rses->have_tmp_tables && - (packet_type == MYSQL_COM_QUERY || packet_type == MYSQL_COM_DROP_DB)) - { - check_drop_tmp_table(rses, querybuf, qtype); - if (packet_type == MYSQL_COM_QUERY) - { - qtype = is_read_tmp_table(rses, querybuf, qtype); - } - } - check_create_tmp_table(rses, querybuf, qtype); - - /** - * Check if this is a LOAD DATA LOCAL INFILE query. If so, send all queries - * to the master until the last, empty packet arrives. - */ - if (rses->rses_load_active) - { - rses->rses_load_data_sent += gwbuf_length(querybuf); - } - else if (packet_type == MYSQL_COM_QUERY) - { - qc_query_op_t queryop = qc_get_operation(querybuf); - if (queryop == QUERY_OP_LOAD) - { - rses->rses_load_active = true; - rses->rses_load_data_sent = 0; - } - } - - rses_end_locked_router_action(rses); - /** - * If autocommit is disabled or transaction is explicitly started - * transaction becomes active and master gets all statements until - * transaction is committed and autocommit is enabled again. - */ - if (rses->rses_autocommit_enabled && - QUERY_IS_TYPE(qtype, QUERY_TYPE_DISABLE_AUTOCOMMIT)) - { - rses->rses_autocommit_enabled = false; - - if (!rses->rses_transaction_active) - { - rses->rses_transaction_active = true; - } - } - else if (!rses->rses_transaction_active && - QUERY_IS_TYPE(qtype, QUERY_TYPE_BEGIN_TRX)) - { - rses->rses_transaction_active = true; - } - /** - * Explicit COMMIT and ROLLBACK, implicit COMMIT. - */ - if (rses->rses_autocommit_enabled && rses->rses_transaction_active && - (QUERY_IS_TYPE(qtype, QUERY_TYPE_COMMIT) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_ROLLBACK))) - { - rses->rses_transaction_active = false; - } - else if (!rses->rses_autocommit_enabled && - QUERY_IS_TYPE(qtype, QUERY_TYPE_ENABLE_AUTOCOMMIT)) - { - rses->rses_autocommit_enabled = true; - rses->rses_transaction_active = false; - } - - if (MXS_LOG_PRIORITY_IS_ENABLED(LOG_INFO)) - { - if (!rses->rses_load_active) - { - uint8_t *packet = GWBUF_DATA(querybuf); - unsigned char ptype = packet[4]; - size_t len = MIN(GWBUF_LENGTH(querybuf), - MYSQL_GET_PACKET_LEN((unsigned char *)querybuf->start) - 1); - char *data = (char *)&packet[5]; - char *contentstr = strndup(data, MIN(len, RWSPLIT_TRACE_MSG_LEN)); - char *qtypestr = qc_get_qtype_str(qtype); - MXS_INFO("> Autocommit: %s, trx is %s, cmd: %s, type: %s, stmt: %s%s %s", - (rses->rses_autocommit_enabled ? "[enabled]" : "[disabled]"), - (rses->rses_transaction_active ? "[open]" : "[not open]"), - STRPACKETTYPE(ptype), (qtypestr == NULL ? "N/A" : qtypestr), - contentstr, (querybuf->hint == NULL ? "" : ", Hint:"), - (querybuf->hint == NULL ? "" : STRHINTTYPE(querybuf->hint->type))); - MXS_FREE(contentstr); - MXS_FREE(qtypestr); - } - else - { - MXS_INFO("> Processing LOAD DATA LOCAL INFILE: %lu bytes sent.", - rses->rses_load_data_sent); - } - } - /** - * Find out where to route the query. Result may not be clear; it is - * possible to have a hint for routing to a named server which can - * be either slave or master. - * If query would otherwise be routed to slave then the hint determines - * actual target server if it exists. - * - * route_target is a bitfield and may include : - * TARGET_ALL - * - route to all connected backend servers - * TARGET_SLAVE[|TARGET_NAMED_SERVER|TARGET_RLAG_MAX] - * - route primarily according to hints, then to slave and if those - * failed, eventually to master - * TARGET_MASTER[|TARGET_NAMED_SERVER|TARGET_RLAG_MAX] - * - route primarily according to the hints and if they failed, - * eventually to master - */ - route_target = get_route_target(rses, qtype, querybuf->hint); - - if (TARGET_IS_ALL(route_target)) - { - /** Multiple, conflicting routing target. Return error */ - if (TARGET_IS_MASTER(route_target) || TARGET_IS_SLAVE(route_target)) - { - backend_ref_t *bref = rses->rses_backend_ref; - - char *query_str = modutil_get_query(querybuf); - char *qtype_str = qc_get_qtype_str(qtype); - - MXS_ERROR("Can't route %s:%s:\"%s\". SELECT with session data " - "modification is not supported if configuration parameter " - "use_sql_variables_in=all .", STRPACKETTYPE(packet_type), - qtype_str, (query_str == NULL ? "(empty)" : query_str)); - - MXS_INFO("Unable to route the query without losing session data " - "modification from other servers. <"); - - while (bref != NULL && !BREF_IS_IN_USE(bref)) - { - bref++; - } - - if (bref != NULL && BREF_IS_IN_USE(bref)) - { - /** Create and add MySQL error to eventqueue */ - modutil_reply_parse_error(bref->bref_dcb, - MXS_STRDUP_A("Routing query to backend failed. " - "See the error log for further " - "details."), 0); - succp = true; - } - else - { - /** - * If there were no available backend references - * available return false - session will be closed - */ - MXS_ERROR("Sending error message to client " - "failed. Router doesn't have any " - "available backends. Session will be " - "closed."); - succp = false; - } - if (query_str) - { - MXS_FREE(query_str); - } - if (qtype_str) - { - MXS_FREE(qtype_str); - } - goto retblock; - } - /** - * It is not sure if the session command in question requires - * response. Statement is examined in route_session_write. - * Router locking is done inside the function. - */ - succp = route_session_write(rses, gwbuf_clone(querybuf), inst, - packet_type, qtype); - - if (succp) - { - atomic_add(&inst->stats.n_all, 1); - } - goto retblock; - } - } - /** Lock router session */ - if (!rses_begin_locked_router_action(rses)) - { - if (packet_type != MYSQL_COM_QUIT) - { - char *query_str = modutil_get_query(querybuf); - - MXS_ERROR("Can't route %s:%s:\"%s\" to " - "backend server. Router is closed.", - STRPACKETTYPE(packet_type), STRQTYPE(qtype), - (query_str == NULL ? "(empty)" : query_str)); - MXS_FREE(query_str); - } - succp = false; - goto retblock; - } - - DCB *master_dcb = rses->rses_master_ref ? rses->rses_master_ref->bref_dcb : NULL; - - /** - * There is a hint which either names the target backend or - * hint which sets maximum allowed replication lag for the - * backend. - */ - if (TARGET_IS_NAMED_SERVER(route_target) || - TARGET_IS_RLAG_MAX(route_target)) - { - HINT *hint; - char *named_server = NULL; - - hint = querybuf->hint; - - while (hint != NULL) - { - if (hint->type == HINT_ROUTE_TO_NAMED_SERVER) - { - /** - * Set the name of searched - * backend server. - */ - named_server = hint->data; - MXS_INFO("Hint: route to server " - "'%s'", - named_server); - } - else if (hint->type == HINT_PARAMETER && - (strncasecmp((char *)hint->data, "max_slave_replication_lag", - strlen("max_slave_replication_lag")) == 0)) - { - int val = (int)strtol((char *)hint->value, (char **)NULL, 10); - - if (val != 0 || errno == 0) - { - /** Set max. acceptable replication lag value for backend srv */ - rlag_max = val; - MXS_INFO("Hint: max_slave_replication_lag=%d", rlag_max); - } - } - hint = hint->next; - } /*< while */ - - if (rlag_max == MAX_RLAG_UNDEFINED) /*< no rlag max hint, use config */ - { - rlag_max = rses_get_max_replication_lag(rses); - } - - /** target may be master or slave */ - btype = route_target & TARGET_SLAVE ? BE_SLAVE : BE_MASTER; - - /** - * Search backend server by name or replication lag. - * If it fails, then try to find valid slave or master. - */ - succp = get_dcb(&target_dcb, rses, btype, named_server, rlag_max); - - if (!succp) - { - if (TARGET_IS_NAMED_SERVER(route_target)) - { - MXS_INFO("Was supposed to route to named server " - "%s but couldn't find the server in a " - "suitable state.", named_server); - } - else if (TARGET_IS_RLAG_MAX(route_target)) - { - MXS_INFO("Was supposed to route to server with " - "replication lag at most %d but couldn't " - "find such a slave.", rlag_max); - } - } - } - else if (TARGET_IS_SLAVE(route_target)) - { - btype = BE_SLAVE; - - if (rlag_max == MAX_RLAG_UNDEFINED) /*< no rlag max hint, use config */ - { - rlag_max = rses_get_max_replication_lag(rses); - } - /** - * Search suitable backend server, get DCB in target_dcb - */ - succp = get_dcb(&target_dcb, rses, BE_SLAVE, NULL, rlag_max); - - if (succp) - { -#if defined(SS_EXTRA_DEBUG) - MXS_INFO("Found DCB for slave."); -#endif - atomic_add(&inst->stats.n_slave, 1); - } - else - { - MXS_INFO("Was supposed to route to slave but finding suitable one failed."); - } - } - else if (TARGET_IS_MASTER(route_target)) - { - DCB *curr_master_dcb = NULL; - - succp = get_dcb(&curr_master_dcb, rses, BE_MASTER, NULL, MAX_RLAG_UNDEFINED); - - if (succp && master_dcb == curr_master_dcb) - { - atomic_add(&inst->stats.n_master, 1); - target_dcb = master_dcb; - } - else - { - if (succp && master_dcb != curr_master_dcb) - { - MXS_INFO("Was supposed to route to master but master has changed."); - } - else - { - MXS_INFO("Was supposed to route to master but couldn't find master" - " in a suitable state."); - } - - if (rses->rses_config.rw_master_failure_mode == RW_ERROR_ON_WRITE) - { - /** Old master is no longer available */ - succp = send_readonly_error(rses->client_dcb); - } - else - { - MXS_WARNING("[%s] Write query received from %s@%s when no master is " - "available, closing client connection.", inst->service->name, - rses->client_dcb->user, rses->client_dcb->remote); - succp = false; - } - - rses_end_locked_router_action(rses); - goto retblock; - } - } - - if (succp) /*< Have DCB of the target backend */ - { - backend_ref_t *bref; - sescmd_cursor_t *scur; - - bref = get_bref_from_dcb(rses, target_dcb); - scur = &bref->bref_sescmd_cur; - - ss_dassert(target_dcb != NULL); - - MXS_INFO("Route query to %s \t%s:%d <", - (SERVER_IS_MASTER(bref->bref_backend->backend_server) ? "master" - : "slave"), bref->bref_backend->backend_server->name, - bref->bref_backend->backend_server->port); - /** - * Store current stmt if execution of previous session command - * haven't completed yet. - * - * !!! Note that according to MySQL protocol - * there can only be one such non-sescmd stmt at the time. - * It is possible that bref->bref_pending_cmd includes a pending - * command if rwsplit is parent or child for another router, - * which runs all the same commands. - * - * If the assertion below traps, pending queries are treated - * somehow wrong, or client is sending more queries before - * previous is received. - */ - if (sescmd_cursor_is_active(scur)) - { - ss_dassert(bref->bref_pending_cmd == NULL); - bref->bref_pending_cmd = gwbuf_clone(querybuf); - - rses_end_locked_router_action(rses); - goto retblock; - } - - if ((ret = target_dcb->func.write(target_dcb, gwbuf_clone(querybuf))) == 1) - { - backend_ref_t *bref; - - atomic_add(&inst->stats.n_queries, 1); - /** - * Add one query response waiter to backend reference - */ - bref = get_bref_from_dcb(rses, target_dcb); - bref_set_state(bref, BREF_QUERY_ACTIVE); - bref_set_state(bref, BREF_WAITING_RESULT); - } - else - { - MXS_ERROR("Routing query failed."); - succp = false; - } - } - rses_end_locked_router_action(rses); - -retblock : -#if defined(SS_DEBUG2) - { - char *canonical_query_str; - - canonical_query_str = skygw_get_canonical(querybuf); - - if (canonical_query_str != NULL) - { - MXS_INFO("Canonical version: %s", canonical_query_str); - MXS_FREE(canonical_query_str); - } - } -#endif - return succp; -} - -/** - * @node Acquires lock to router client session if it is not closed. - * - * Parameters: - * @param rses - in, use - * - * - * @return true if router session was not closed. If return value is true - * it means that router is locked, and must be unlocked later. False, if - * router was closed before lock was acquired. - * - * - * @details (write detailed description here) - * - */ -static bool rses_begin_locked_router_action(ROUTER_CLIENT_SES *rses) -{ - bool succp = false; - - if (rses == NULL) - { - return false; - } - - CHK_CLIENT_RSES(rses); - - if (rses->rses_closed) - { - - goto return_succp; - } - spinlock_acquire(&rses->rses_lock); - if (rses->rses_closed) - { - spinlock_release(&rses->rses_lock); - goto return_succp; - } - succp = true; - -return_succp: - return succp; -} - -/** to be inline'd */ - -/** - * @node Releases router client session lock. - * - * Parameters: - * @param rses - - * - * - * @return void - * - * - * @details (write detailed description here) - * - */ -static void rses_end_locked_router_action(ROUTER_CLIENT_SES *rses) -{ - CHK_CLIENT_RSES(rses); - spinlock_release(&rses->rses_lock); -} - -/** - * Diagnostics routine + * @brief Diagnostics routine (API) * * Print query router statistics to the DCB passed in * * @param instance The router instance * @param dcb The DCB for diagnostic output */ -static void diagnostic(ROUTER *instance, DCB *dcb) +static void diagnostics(ROUTER *instance, DCB *dcb) { ROUTER_CLIENT_SES *router_cli_ses; ROUTER_INSTANCE *router = (ROUTER_INSTANCE *)instance; @@ -2495,7 +845,7 @@ static void diagnostic(ROUTER *instance, DCB *dcb) } /** - * Client Reply routine + * @brief Client Reply routine (API) * * The routine will reply to client for session change with master server data * @@ -2573,25 +923,8 @@ static void clientReply(ROUTER *instance, void *router_session, GWBUF *writebuf, */ if (sescmd_cursor_is_active(scur)) { - if (MXS_LOG_PRIORITY_IS_ENABLED(LOG_ERR) && - MYSQL_IS_ERROR_PACKET(((uint8_t *)GWBUF_DATA(writebuf)))) - { - uint8_t *buf = (uint8_t *)GWBUF_DATA((scur->scmd_cur_cmd->my_sescmd_buf)); - uint8_t *replybuf = (uint8_t *)GWBUF_DATA(writebuf); - size_t len = MYSQL_GET_PACKET_LEN(buf); - size_t replylen = MYSQL_GET_PACKET_LEN(replybuf); - char *err = strndup(&((char *)replybuf)[8], 5); - char *replystr = strndup(&((char *)replybuf)[13], replylen - 4 - 5); - - ss_dassert(len + 4 == GWBUF_LENGTH(scur->scmd_cur_cmd->my_sescmd_buf)); - - MXS_ERROR("Failed to execute session command in %s:%d. Error was: %s %s", - bref->bref_backend->backend_server->name, - bref->bref_backend->backend_server->port, err, replystr); - MXS_FREE(err); - MXS_FREE(replystr); - } - + check_session_command_reply(writebuf, scur, bref); + if (GWBUF_IS_TYPE_SESCMD_RESPONSE(writebuf)) { /** @@ -2707,1529 +1040,12 @@ lock_failed: return; } -/** Compare nunmber of connections from this router in backend servers */ -int bref_cmp_router_conn(const void *bref1, const void *bref2) -{ - BACKEND *b1 = ((backend_ref_t *)bref1)->bref_backend; - BACKEND *b2 = ((backend_ref_t *)bref2)->bref_backend; - - if (b1->weight == 0 && b2->weight == 0) - { - return b1->backend_server->stats.n_current - - b2->backend_server->stats.n_current; - } - else if (b1->weight == 0) - { - return 1; - } - else if (b2->weight == 0) - { - return -1; - } - - return ((1000 + 1000 * b1->backend_conn_count) / b1->weight) - - ((1000 + 1000 * b2->backend_conn_count) / b2->weight); -} - -/** Compare nunmber of global connections in backend servers */ -int bref_cmp_global_conn(const void *bref1, const void *bref2) -{ - BACKEND *b1 = ((backend_ref_t *)bref1)->bref_backend; - BACKEND *b2 = ((backend_ref_t *)bref2)->bref_backend; - - if (b1->weight == 0 && b2->weight == 0) - { - return b1->backend_server->stats.n_current - - b2->backend_server->stats.n_current; - } - else if (b1->weight == 0) - { - return 1; - } - else if (b2->weight == 0) - { - return -1; - } - - return ((1000 + 1000 * b1->backend_server->stats.n_current) / b1->weight) - - ((1000 + 1000 * b2->backend_server->stats.n_current) / b2->weight); -} - -/** Compare relication lag between backend servers */ -int bref_cmp_behind_master(const void *bref1, const void *bref2) -{ - BACKEND *b1 = ((backend_ref_t *)bref1)->bref_backend; - BACKEND *b2 = ((backend_ref_t *)bref2)->bref_backend; - - return ((b1->backend_server->rlag < b2->backend_server->rlag) ? -1 - : ((b1->backend_server->rlag > b2->backend_server->rlag) ? 1 : 0)); -} - -/** Compare nunmber of current operations in backend servers */ -int bref_cmp_current_load(const void *bref1, const void *bref2) -{ - SERVER *s1 = ((backend_ref_t *)bref1)->bref_backend->backend_server; - SERVER *s2 = ((backend_ref_t *)bref2)->bref_backend->backend_server; - BACKEND *b1 = ((backend_ref_t *)bref1)->bref_backend; - BACKEND *b2 = ((backend_ref_t *)bref2)->bref_backend; - - if (b1->weight == 0 && b2->weight == 0) - { - return b1->backend_server->stats.n_current - - b2->backend_server->stats.n_current; - } - else if (b1->weight == 0) - { - return 1; - } - else if (b2->weight == 0) - { - return -1; - } - - return ((1000 * s1->stats.n_current_ops) - b1->weight) - - ((1000 * s2->stats.n_current_ops) - b2->weight); -} - -static void bref_clear_state(backend_ref_t *bref, bref_state_t state) -{ - if (bref == NULL) - { - MXS_ERROR("[%s] Error: NULL parameter.", __FUNCTION__); - return; - } - - if ((state & BREF_WAITING_RESULT) && (bref->bref_state & BREF_WAITING_RESULT)) - { - int prev1; - int prev2; - - /** Decrease waiter count */ - prev1 = atomic_add(&bref->bref_num_result_wait, -1); - - if (prev1 <= 0) - { - atomic_add(&bref->bref_num_result_wait, 1); - } - else - { - /** Decrease global operation count */ - prev2 = atomic_add(&bref->bref_backend->backend_server->stats.n_current_ops, -1); - ss_dassert(prev2 > 0); - if (prev2 <= 0) - { - MXS_ERROR("[%s] Error: negative current operation count in backend %s:%u", - __FUNCTION__, bref->bref_backend->backend_server->name, - bref->bref_backend->backend_server->port); - } - } - } - - bref->bref_state &= ~state; -} - -static void bref_set_state(backend_ref_t *bref, bref_state_t state) -{ - if (bref == NULL) - { - MXS_ERROR("[%s] Error: NULL parameter.", __FUNCTION__); - return; - } - - if ((state & BREF_WAITING_RESULT) && (bref->bref_state & BREF_WAITING_RESULT) == 0) - { - int prev1; - int prev2; - - /** Increase waiter count */ - prev1 = atomic_add(&bref->bref_num_result_wait, 1); - ss_dassert(prev1 >= 0); - if (prev1 < 0) - { - MXS_ERROR("[%s] Error: negative number of connections waiting for " - "results in backend %s:%u", - __FUNCTION__, bref->bref_backend->backend_server->name, - bref->bref_backend->backend_server->port); - } - /** Increase global operation count */ - prev2 = - atomic_add(&bref->bref_backend->backend_server->stats.n_current_ops, 1); - ss_dassert(prev2 >= 0); - if (prev2 < 0) - { - MXS_ERROR("[%s] Error: negative current operation count in backend %s:%u", - __FUNCTION__, bref->bref_backend->backend_server->name, - bref->bref_backend->backend_server->port); - } - } - - bref->bref_state |= state; -} - -/** - * @brief Connect a server - * - * Connects to a server, adds callbacks to the created DCB and updates - * router statistics. If @p execute_history is true, the session command - * history will be executed on this server. - * - * @param b Router's backend structure for the server - * @param session Client's session object - * @param execute_history Execute session command history - * @return True if successful, false if an error occurred - */ -bool connect_server(backend_ref_t *bref, SESSION *session, bool execute_history) -{ - SERVER *serv = bref->bref_backend->backend_server; - bool rval = false; - - bref->bref_dcb = dcb_connect(serv, session, serv->protocol); - - if (bref->bref_dcb != NULL) - { - if (!execute_history || execute_sescmd_history(bref)) - { - /** Add a callback for unresponsive server */ - dcb_add_callback(bref->bref_dcb, DCB_REASON_NOT_RESPONDING, - &router_handle_state_switch, (void *) bref); - bref->bref_state = 0; - bref_set_state(bref, BREF_IN_USE); - atomic_add(&bref->bref_backend->backend_conn_count, 1); - rval = true; - } - else - { - MXS_ERROR("Failed to execute session command in %s (%s:%d). See earlier " - "errors for more details.", - bref->bref_backend->backend_server->unique_name, - bref->bref_backend->backend_server->name, - bref->bref_backend->backend_server->port); - dcb_close(bref->bref_dcb); - bref->bref_dcb = NULL; - } - } - else - { - MXS_ERROR("Unable to establish connection with server %s:%d", - serv->name, serv->port); - } - - return rval; -} - -/** - * @brief Log server connections - * - * @param select_criteria Slave selection criteria - * @param backend_ref Backend reference array - * @param router_nservers Number of backends in @p backend_ref - */ -void log_server_connections(select_criteria_t select_criteria, - backend_ref_t *backend_ref, int router_nservers) -{ - if (select_criteria == LEAST_GLOBAL_CONNECTIONS || - select_criteria == LEAST_ROUTER_CONNECTIONS || - select_criteria == LEAST_BEHIND_MASTER || - select_criteria == LEAST_CURRENT_OPERATIONS) - { - MXS_INFO("Servers and %s connection counts:", - select_criteria == LEAST_GLOBAL_CONNECTIONS ? "all MaxScale" - : "router"); - - for (int i = 0; i < router_nservers; i++) - { - BACKEND *b = backend_ref[i].bref_backend; - - switch (select_criteria) - { - case LEAST_GLOBAL_CONNECTIONS: - MXS_INFO("MaxScale connections : %d in \t%s:%d %s", - b->backend_server->stats.n_current, b->backend_server->name, - b->backend_server->port, STRSRVSTATUS(b->backend_server)); - break; - - case LEAST_ROUTER_CONNECTIONS: - MXS_INFO("RWSplit connections : %d in \t%s:%d %s", - b->backend_conn_count, b->backend_server->name, - b->backend_server->port, STRSRVSTATUS(b->backend_server)); - break; - - case LEAST_CURRENT_OPERATIONS: - MXS_INFO("current operations : %d in \t%s:%d %s", - b->backend_server->stats.n_current_ops, - b->backend_server->name, b->backend_server->port, - STRSRVSTATUS(b->backend_server)); - break; - - case LEAST_BEHIND_MASTER: - MXS_INFO("replication lag : %d in \t%s:%d %s", - b->backend_server->rlag, b->backend_server->name, - b->backend_server->port, STRSRVSTATUS(b->backend_server)); - default: - break; - } - } - } -} - -/** - * @brief Search suitable backend servers from those of router instance - * - * It is assumed that there is only one master among servers of a router instance. - * As a result, the first master found is chosen. There will possibly be more - * backend references than connected backends because only those in correct state - * are connected to. - * - * @param p_master_ref Pointer to location where master's backend reference is to be stored - * @param backend_ref Pointer to backend server reference object array - * @param router_nservers Number of backend server pointers pointed to by @p backend_ref - * @param max_nslaves Upper limit for the number of slaves - * @param max_slave_rlag Maximum allowed replication lag for any slave - * @param select_criteria Slave selection criteria - * @param session Client session - * @param router Router instance - * @return true, if at least one master and one slave was found. - */ -static bool select_connect_backend_servers(backend_ref_t **p_master_ref, - backend_ref_t *backend_ref, - int router_nservers, int max_nslaves, - int max_slave_rlag, - select_criteria_t select_criteria, - SESSION *session, - ROUTER_INSTANCE *router) -{ - if (p_master_ref == NULL || backend_ref == NULL) - { - MXS_ERROR("Master reference (%p) or backend reference (%p) is NULL.", - p_master_ref, backend_ref); - ss_dassert(false); - return false; - } - - /* get the root Master */ - BACKEND *master_host = get_root_master(backend_ref, router_nservers); - - if (router->rwsplit_config.rw_master_failure_mode == RW_FAIL_INSTANTLY && - (master_host == NULL || SERVER_IS_DOWN(master_host->backend_server))) - { - MXS_ERROR("Couldn't find suitable Master from %d candidates.", router_nservers); - return false; - } - - /** - * Existing session : master is already chosen and connected. - * The function was called because new slave must be selected to replace - * failed one. - */ - bool master_connected = *p_master_ref != NULL; - - /** Check slave selection criteria and set compare function */ - int (*p)(const void *, const void *) = criteria_cmpfun[select_criteria]; - ss_dassert(p); - - /** Sort the pointer list to servers according to slave selection criteria. - * The servers that match the criteria the best are at the beginning of - * the list. */ - qsort(backend_ref, (size_t) router_nservers, sizeof(backend_ref_t), p); - - if (MXS_LOG_PRIORITY_IS_ENABLED(LOG_INFO)) - { - log_server_connections(select_criteria, backend_ref, router_nservers); - } - - int slaves_found = 0; - int slaves_connected = 0; - const int min_nslaves = 0; /*< not configurable at the time */ - bool succp = false; - - /** - * Choose at least 1+min_nslaves (master and slave) and at most 1+max_nslaves - * servers from the sorted list. First master found is selected. - */ - for (int i = 0; i < router_nservers && - (slaves_connected < max_nslaves || !master_connected); i++) - { - SERVER *serv = backend_ref[i].bref_backend->backend_server; - - if (!BREF_HAS_FAILED(&backend_ref[i]) && SERVER_IS_RUNNING(serv)) - { - /* check also for relay servers and don't take the master_host */ - if (slaves_found < max_nslaves && - (max_slave_rlag == MAX_RLAG_UNDEFINED || - (serv->rlag != MAX_RLAG_NOT_AVAILABLE && - serv->rlag <= max_slave_rlag)) && - (SERVER_IS_SLAVE(serv) || SERVER_IS_RELAY_SERVER(serv)) && - (master_host == NULL || (serv != master_host->backend_server))) - { - slaves_found += 1; - - if (BREF_IS_IN_USE((&backend_ref[i])) || - connect_server(&backend_ref[i], session, true)) - { - slaves_connected += 1; - } - } - /* take the master_host for master */ - else if (master_host && (serv == master_host->backend_server)) - { - /** p_master_ref must be assigned with this backend_ref pointer - * because its original value may have been lost when backend - * references were sorted with qsort. */ - *p_master_ref = &backend_ref[i]; - - if (!master_connected) - { - if (connect_server(&backend_ref[i], session, false)) - { - master_connected = true; - } - } - } - } - } /*< for */ - - /** - * Successful cases - */ - if (slaves_connected >= min_nslaves && slaves_connected <= max_nslaves) - { - succp = true; - - if (MXS_LOG_PRIORITY_IS_ENABLED(LOG_INFO)) - { - if (slaves_connected < max_nslaves) - { - MXS_INFO("Couldn't connect to maximum number of " - "slaves. Connected successfully to %d slaves " - "of %d of them.", slaves_connected, slaves_found); - } - - for (int i = 0; i < router_nservers; i++) - { - if (BREF_IS_IN_USE((&backend_ref[i]))) - { - MXS_INFO("Selected %s in \t%s:%d", - STRSRVSTATUS(backend_ref[i].bref_backend->backend_server), - backend_ref[i].bref_backend->backend_server->name, - backend_ref[i].bref_backend->backend_server->port); - } - } /* for */ - } - } - /** Failure cases */ - else - { - if (slaves_connected < min_nslaves) - { - MXS_ERROR("Couldn't establish required amount of " - "slave connections for router session."); - } - - /** Clean up connections */ - for (int i = 0; i < router_nservers; i++) - { - if (BREF_IS_IN_USE((&backend_ref[i]))) - { - ss_dassert(backend_ref[i].bref_backend->backend_conn_count > 0); - - /** disconnect opened connections */ - bref_clear_state(&backend_ref[i], BREF_IN_USE); - /** Decrease backend's connection counter. */ - atomic_add(&backend_ref[i].bref_backend->backend_conn_count, -1); - dcb_close(backend_ref[i].bref_dcb); - } - } - } - - return succp; -} - -/** - * Create a generic router session property strcture. - */ -static rses_property_t *rses_property_init(rses_property_type_t prop_type) -{ - rses_property_t *prop; - - prop = (rses_property_t *)MXS_CALLOC(1, sizeof(rses_property_t)); - if (prop == NULL) - { - return NULL; - } - prop->rses_prop_type = prop_type; -#if defined(SS_DEBUG) - prop->rses_prop_chk_top = CHK_NUM_ROUTER_PROPERTY; - prop->rses_prop_chk_tail = CHK_NUM_ROUTER_PROPERTY; -#endif - - CHK_RSES_PROP(prop); - return prop; -} - -/** - * Property is freed at the end of router client session. - */ -static void rses_property_done(rses_property_t *prop) -{ - if (prop == NULL) - { - MXS_ERROR("[%s] Error: NULL parameter.", __FUNCTION__); - return; - } - CHK_RSES_PROP(prop); - - switch (prop->rses_prop_type) - { - case RSES_PROP_TYPE_SESCMD: - mysql_sescmd_done(&prop->rses_prop_data.sescmd); - break; - - case RSES_PROP_TYPE_TMPTABLES: - hashtable_free(prop->rses_prop_data.temp_tables); - break; - - default: - MXS_DEBUG("%lu [rses_property_done] Unknown property type %d " - "in property %p", pthread_self(), prop->rses_prop_type, prop); - - ss_dassert(false); - break; - } - MXS_FREE(prop); -} - -/** - * Add property to the router_client_ses structure's rses_properties - * array. The slot is determined by the type of property. - * In each slot there is a list of properties of similar type. - * - * Router client session must be locked. - */ -static int rses_property_add(ROUTER_CLIENT_SES *rses, rses_property_t *prop) -{ - if (rses == NULL) - { - MXS_ERROR("Router client session is NULL. (%s:%d)", __FILE__, __LINE__); - return -1; - } - if (prop == NULL) - { - MXS_ERROR("Router client session property is NULL. (%s:%d)", __FILE__, __LINE__); - return -1; - } - rses_property_t *p; - - CHK_CLIENT_RSES(rses); - CHK_RSES_PROP(prop); - ss_dassert(SPINLOCK_IS_LOCKED(&rses->rses_lock)); - - prop->rses_prop_rsession = rses; - p = rses->rses_properties[prop->rses_prop_type]; - - if (p == NULL) - { - rses->rses_properties[prop->rses_prop_type] = prop; - } - else - { - while (p->rses_prop_next != NULL) - { - p = p->rses_prop_next; - } - p->rses_prop_next = prop; - } - return 0; -} - -/** - * Router session must be locked. - * Return session command pointer if succeed, NULL if failed. - */ -static mysql_sescmd_t *rses_property_get_sescmd(rses_property_t *prop) -{ - mysql_sescmd_t *sescmd; - - if (prop == NULL) - { - MXS_ERROR("[%s] Error: NULL parameter.", __FUNCTION__); - return NULL; - } - - CHK_RSES_PROP(prop); - ss_dassert(prop->rses_prop_rsession == NULL || - SPINLOCK_IS_LOCKED(&prop->rses_prop_rsession->rses_lock)); - - sescmd = &prop->rses_prop_data.sescmd; - - if (sescmd != NULL) - { - CHK_MYSQL_SESCMD(sescmd); - } - return sescmd; -} - -/** - * Create session command property. - */ -static mysql_sescmd_t *mysql_sescmd_init(rses_property_t *rses_prop, - GWBUF *sescmd_buf, - unsigned char packet_type, - ROUTER_CLIENT_SES *rses) -{ - mysql_sescmd_t *sescmd; - - CHK_RSES_PROP(rses_prop); - /** Can't call rses_property_get_sescmd with uninitialized sescmd */ - sescmd = &rses_prop->rses_prop_data.sescmd; - sescmd->my_sescmd_prop = rses_prop; /*< reference to owning property */ -#if defined(SS_DEBUG) - sescmd->my_sescmd_chk_top = CHK_NUM_MY_SESCMD; - sescmd->my_sescmd_chk_tail = CHK_NUM_MY_SESCMD; -#endif - /** Set session command buffer */ - sescmd->my_sescmd_buf = sescmd_buf; - sescmd->my_sescmd_packet_type = packet_type; - sescmd->position = atomic_add(&rses->pos_generator, 1); - - return sescmd; -} - -static void mysql_sescmd_done(mysql_sescmd_t *sescmd) -{ - if (sescmd == NULL) - { - MXS_ERROR("[%s] Error: NULL parameter.", __FUNCTION__); - return; - } - CHK_RSES_PROP(sescmd->my_sescmd_prop); - gwbuf_free(sescmd->my_sescmd_buf); - memset(sescmd, 0, sizeof(mysql_sescmd_t)); -} - -/** - * All cases where backend message starts at least with one response to session - * command are handled here. - * Read session commands from property list. If command is already replied, - * discard packet. Else send reply to client. In both cases move cursor forward - * until all session command replies are handled. - * - * Cases that are expected to happen and which are handled: - * s = response not yet replied to client, S = already replied response, - * q = query - * 1. q+ for example : select * from mysql.user - * 2. s+ for example : set autocommit=1 - * 3. S+ - * 4. sq+ - * 5. Sq+ - * 6. Ss+ - * 7. Ss+q+ - * 8. S+q+ - * 9. s+q+ - */ -static GWBUF *sescmd_cursor_process_replies(GWBUF *replybuf, - backend_ref_t *bref, - bool *reconnect) -{ - mysql_sescmd_t *scmd; - sescmd_cursor_t *scur; - ROUTER_CLIENT_SES *ses; - - scur = &bref->bref_sescmd_cur; - ss_dassert(SPINLOCK_IS_LOCKED(&(scur->scmd_cur_rses->rses_lock))); - scmd = sescmd_cursor_get_command(scur); - ses = (*scur->scmd_cur_ptr_property)->rses_prop_rsession; - CHK_GWBUF(replybuf); - - /** - * Walk through packets in the message and the list of session - * commands. - */ - while (scmd != NULL && replybuf != NULL) - { - bref->reply_cmd = *((unsigned char *)replybuf->start + 4); - scur->position = scmd->position; - /** Faster backend has already responded to client : discard */ - if (scmd->my_sescmd_is_replied) - { - bool last_packet = false; - - CHK_GWBUF(replybuf); - - while (!last_packet) - { - int buflen; - - buflen = GWBUF_LENGTH(replybuf); - last_packet = GWBUF_IS_TYPE_RESPONSE_END(replybuf); - /** discard packet */ - replybuf = gwbuf_consume(replybuf, buflen); - } - /** Set response status received */ - bref_clear_state(bref, BREF_WAITING_RESULT); - - if (bref->reply_cmd != scmd->reply_cmd) - { - MXS_ERROR("Slave server '%s': response differs from master's response. " - "Closing connection due to inconsistent session state.", - bref->bref_backend->backend_server->unique_name); - sescmd_cursor_set_active(scur, false); - bref_clear_state(bref, BREF_QUERY_ACTIVE); - bref_clear_state(bref, BREF_IN_USE); - bref_set_state(bref, BREF_CLOSED); - bref_set_state(bref, BREF_SESCMD_FAILED); - if (bref->bref_dcb) - { - dcb_close(bref->bref_dcb); - } - *reconnect = true; - gwbuf_free(replybuf); - replybuf = NULL; - } - } - /** This is a response from the master and it is the "right" one. - * A slave server's response will be compared to this and if - * their response differs from the master server's response, they - * are dropped from the valid list of backend servers. - * Response is in the buffer and it will be sent to client. - * - * If we have no master server, the first slave's response is considered - * the "right" one. */ - else if (ses->rses_master_ref == NULL || - !BREF_IS_IN_USE(ses->rses_master_ref) || - ses->rses_master_ref->bref_dcb == bref->bref_dcb) - { - /** Mark the rest session commands as replied */ - scmd->my_sescmd_is_replied = true; - scmd->reply_cmd = *((unsigned char *)replybuf->start + 4); - - MXS_INFO("Server '%s' responded to a session command, sending the response " - "to the client.", bref->bref_backend->backend_server->unique_name); - - for (int i = 0; i < ses->rses_nbackends; i++) - { - if (!BREF_IS_WAITING_RESULT(&ses->rses_backend_ref[i])) - { - /** This backend has already received a response */ - if (ses->rses_backend_ref[i].reply_cmd != scmd->reply_cmd && - !BREF_IS_CLOSED(&ses->rses_backend_ref[i]) && - BREF_IS_IN_USE(&ses->rses_backend_ref[i])) - { - bref_clear_state(&ses->rses_backend_ref[i], BREF_QUERY_ACTIVE); - bref_clear_state(&ses->rses_backend_ref[i], BREF_IN_USE); - bref_set_state(&ses->rses_backend_ref[i], BREF_CLOSED); - bref_set_state(bref, BREF_SESCMD_FAILED); - if (ses->rses_backend_ref[i].bref_dcb) - { - dcb_close(ses->rses_backend_ref[i].bref_dcb); - } - *reconnect = true; - MXS_INFO("Disabling slave %s:%d, result differs from " - "master's result. Master: %d Slave: %d", - ses->rses_backend_ref[i].bref_backend->backend_server->name, - ses->rses_backend_ref[i].bref_backend->backend_server->port, - bref->reply_cmd, ses->rses_backend_ref[i].reply_cmd); - } - } - } - - } - else - { - MXS_INFO("Slave '%s' responded before master to a session command. Result: %d", - bref->bref_backend->backend_server->unique_name, - (int)bref->reply_cmd); - if (bref->reply_cmd == 0xff) - { - SERVER *serv = bref->bref_backend->backend_server; - MXS_ERROR("Slave '%s' (%s:%u) failed to execute session command.", - serv->unique_name, serv->name, serv->port); - } - - gwbuf_free(replybuf); - replybuf = NULL; - } - - if (sescmd_cursor_next(scur)) - { - scmd = sescmd_cursor_get_command(scur); - } - else - { - scmd = NULL; - /** All session commands are replied */ - scur->scmd_cur_active = false; - } - } - ss_dassert(replybuf == NULL || *scur->scmd_cur_ptr_property == NULL); - - return replybuf; -} - -/** - * Get the address of current session command. - * - * Router session must be locked */ -static mysql_sescmd_t *sescmd_cursor_get_command(sescmd_cursor_t *scur) -{ - mysql_sescmd_t *scmd; - - ss_dassert(SPINLOCK_IS_LOCKED(&(scur->scmd_cur_rses->rses_lock))); - scur->scmd_cur_cmd = rses_property_get_sescmd(*scur->scmd_cur_ptr_property); - - CHK_MYSQL_SESCMD(scur->scmd_cur_cmd); - - scmd = scur->scmd_cur_cmd; - - return scmd; -} - -/** router must be locked */ -static bool sescmd_cursor_is_active(sescmd_cursor_t *sescmd_cursor) -{ - bool succp; - - if (sescmd_cursor == NULL) - { - MXS_ERROR("[%s] Error: NULL parameter.", __FUNCTION__); - return false; - } - ss_dassert(SPINLOCK_IS_LOCKED(&sescmd_cursor->scmd_cur_rses->rses_lock)); - - succp = sescmd_cursor->scmd_cur_active; - return succp; -} - -/** router must be locked */ -static void sescmd_cursor_set_active(sescmd_cursor_t *sescmd_cursor, - bool value) -{ - ss_dassert(SPINLOCK_IS_LOCKED(&sescmd_cursor->scmd_cur_rses->rses_lock)); - /** avoid calling unnecessarily */ - ss_dassert(sescmd_cursor->scmd_cur_active != value); - sescmd_cursor->scmd_cur_active = value; -} - -/** - * Clone session command's command buffer. - * Router session must be locked - */ -static GWBUF *sescmd_cursor_clone_querybuf(sescmd_cursor_t *scur) -{ - GWBUF *buf; - if (scur == NULL) - { - MXS_ERROR("[%s] Error: NULL parameter.", __FUNCTION__); - return NULL; - } - ss_dassert(scur->scmd_cur_cmd != NULL); - - buf = gwbuf_clone_all(scur->scmd_cur_cmd->my_sescmd_buf); - - CHK_GWBUF(buf); - return buf; -} - -static bool sescmd_cursor_history_empty(sescmd_cursor_t *scur) -{ - bool succp; - - if (scur == NULL) - { - MXS_ERROR("[%s] Error: NULL parameter.", __FUNCTION__); - return true; - } - CHK_SESCMD_CUR(scur); - - if (scur->scmd_cur_rses->rses_properties[RSES_PROP_TYPE_SESCMD] == NULL) - { - succp = true; - } - else - { - succp = false; - } - - return succp; -} - -static void sescmd_cursor_reset(sescmd_cursor_t *scur) -{ - ROUTER_CLIENT_SES *rses; - if (scur == NULL) - { - MXS_ERROR("[%s] Error: NULL parameter.", __FUNCTION__); - return; - } - CHK_SESCMD_CUR(scur); - CHK_CLIENT_RSES(scur->scmd_cur_rses); - rses = scur->scmd_cur_rses; - - scur->scmd_cur_ptr_property = &rses->rses_properties[RSES_PROP_TYPE_SESCMD]; - - CHK_RSES_PROP((*scur->scmd_cur_ptr_property)); - scur->scmd_cur_active = false; - scur->scmd_cur_cmd = &(*scur->scmd_cur_ptr_property)->rses_prop_data.sescmd; -} - -static bool execute_sescmd_history(backend_ref_t *bref) -{ - bool succp = true; - sescmd_cursor_t *scur; - if (bref == NULL) - { - MXS_ERROR("[%s] Error: NULL parameter.", __FUNCTION__); - return false; - } - CHK_BACKEND_REF(bref); - - scur = &bref->bref_sescmd_cur; - CHK_SESCMD_CUR(scur); - - if (!sescmd_cursor_history_empty(scur)) - { - sescmd_cursor_reset(scur); - succp = execute_sescmd_in_backend(bref); - } - - return succp; -} - -/** - * If session command cursor is passive, sends the command to backend for - * execution. - * - * Returns true if command was sent or added successfully to the queue. - * Returns false if command sending failed or if there are no pending session - * commands. - * - * Router session must be locked. - */ -static bool execute_sescmd_in_backend(backend_ref_t *backend_ref) -{ - DCB *dcb; - bool succp; - int rc = 0; - sescmd_cursor_t *scur; - GWBUF *buf; - if (backend_ref == NULL) - { - MXS_ERROR("[%s] Error: NULL parameter.", __FUNCTION__); - return false; - } - if (BREF_IS_CLOSED(backend_ref)) - { - succp = false; - goto return_succp; - } - dcb = backend_ref->bref_dcb; - - CHK_DCB(dcb); - CHK_BACKEND_REF(backend_ref); - - /** - * Get cursor pointer and copy of command buffer to cursor. - */ - scur = &backend_ref->bref_sescmd_cur; - - /** Return if there are no pending ses commands */ - if (sescmd_cursor_get_command(scur) == NULL) - { - succp = true; - MXS_INFO("Cursor had no pending session commands."); - - goto return_succp; - } - - if (!sescmd_cursor_is_active(scur)) - { - /** Cursor is left active when function returns. */ - sescmd_cursor_set_active(scur, true); - } - - switch (scur->scmd_cur_cmd->my_sescmd_packet_type) - { - case MYSQL_COM_CHANGE_USER: - /** This makes it possible to handle replies correctly */ - gwbuf_set_type(scur->scmd_cur_cmd->my_sescmd_buf, GWBUF_TYPE_SESCMD); - buf = sescmd_cursor_clone_querybuf(scur); - rc = dcb->func.auth(dcb, NULL, dcb->session, buf); - break; - - case MYSQL_COM_INIT_DB: - { - /** - * Record database name and store to session. - */ - GWBUF *tmpbuf; - MYSQL_session *data; - unsigned int qlen; - - data = dcb->session->client_dcb->data; - *data->db = 0; - tmpbuf = scur->scmd_cur_cmd->my_sescmd_buf; - qlen = MYSQL_GET_PACKET_LEN((unsigned char *) GWBUF_DATA(tmpbuf)); - if (qlen) - { - --qlen; // The COM_INIT_DB byte - if (qlen > MYSQL_DATABASE_MAXLEN) - { - MXS_ERROR("Too long a database name received in COM_INIT_DB, " - "trailing data will be cut."); - qlen = MYSQL_DATABASE_MAXLEN; - } - - memcpy(data->db, (char*)GWBUF_DATA(tmpbuf) + 5, qlen); - data->db[qlen] = 0; - } - } - /** Fallthrough */ - case MYSQL_COM_QUERY: - default: - /** - * Mark session command buffer, it triggers writing - * MySQL command to protocol - */ - - gwbuf_set_type(scur->scmd_cur_cmd->my_sescmd_buf, GWBUF_TYPE_SESCMD); - buf = sescmd_cursor_clone_querybuf(scur); - rc = dcb->func.write(dcb, buf); - break; - } - - if (rc == 1) - { - succp = true; - } - else - { - succp = false; - } -return_succp: - return succp; -} - -/** - * Moves cursor to next property and copied address of its sescmd to cursor. - * Current propery must be non-null. - * If current property is the last on the list, *scur->scmd_ptr_property == NULL - * - * Router session must be locked - */ -static bool sescmd_cursor_next(sescmd_cursor_t *scur) -{ - bool succp = false; - rses_property_t *prop_curr; - rses_property_t *prop_next; - - if (scur == NULL) - { - MXS_ERROR("[%s] Error: NULL parameter.", __FUNCTION__); - return false; - } - - ss_dassert(scur != NULL); - ss_dassert(*(scur->scmd_cur_ptr_property) != NULL); - ss_dassert(SPINLOCK_IS_LOCKED( - &(*(scur->scmd_cur_ptr_property))->rses_prop_rsession->rses_lock)); - - /** Illegal situation */ - if (scur == NULL || *scur->scmd_cur_ptr_property == NULL || - scur->scmd_cur_cmd == NULL) - { - /** Log error */ - goto return_succp; - } - prop_curr = *(scur->scmd_cur_ptr_property); - - CHK_MYSQL_SESCMD(scur->scmd_cur_cmd); - ss_dassert(prop_curr == mysql_sescmd_get_property(scur->scmd_cur_cmd)); - CHK_RSES_PROP(prop_curr); - - /** Copy address of pointer to next property */ - scur->scmd_cur_ptr_property = &(prop_curr->rses_prop_next); - prop_next = *scur->scmd_cur_ptr_property; - ss_dassert(prop_next == *(scur->scmd_cur_ptr_property)); - - /** If there is a next property move forward */ - if (prop_next != NULL) - { - CHK_RSES_PROP(prop_next); - CHK_RSES_PROP((*(scur->scmd_cur_ptr_property))); - - /** Get pointer to next property's sescmd */ - scur->scmd_cur_cmd = rses_property_get_sescmd(prop_next); - - ss_dassert(prop_next == scur->scmd_cur_cmd->my_sescmd_prop); - CHK_MYSQL_SESCMD(scur->scmd_cur_cmd); - CHK_RSES_PROP(scur->scmd_cur_cmd->my_sescmd_prop); - } - else - { - /** No more properties, can't proceed. */ - goto return_succp; - } - - if (scur->scmd_cur_cmd != NULL) - { - succp = true; - } - else - { - ss_dassert(false); /*< Log error, sescmd shouldn't be NULL */ - } -return_succp: - return succp; -} - -static rses_property_t *mysql_sescmd_get_property(mysql_sescmd_t *scmd) -{ - CHK_MYSQL_SESCMD(scmd); - return scmd->my_sescmd_prop; -} - -static void tracelog_routed_query(ROUTER_CLIENT_SES *rses, char *funcname, - backend_ref_t *bref, GWBUF *buf) -{ - uint8_t *packet = GWBUF_DATA(buf); - unsigned char packet_type = packet[4]; - size_t len; - size_t buflen = GWBUF_LENGTH(buf); - char *querystr; - char *startpos = (char *)&packet[5]; - BACKEND *b; - backend_type_t be_type; - DCB *dcb; - - CHK_BACKEND_REF(bref); - b = bref->bref_backend; - CHK_BACKEND(b); - dcb = bref->bref_dcb; - CHK_DCB(dcb); - - be_type = BACKEND_TYPE(b); - - if (GWBUF_IS_TYPE_MYSQL(buf)) - { - len = packet[0]; - len += 256 * packet[1]; - len += 256 * 256 * packet[2]; - - if (packet_type == '\x03') - { - querystr = (char *)MXS_MALLOC(len); - MXS_ABORT_IF_NULL(querystr); - memcpy(querystr, startpos, len - 1); - querystr[len - 1] = '\0'; - MXS_DEBUG("%lu [%s] %d bytes long buf, \"%s\" -> %s:%d %s dcb %p", - pthread_self(), funcname, (int)buflen, querystr, - b->backend_server->name, b->backend_server->port, - STRBETYPE(be_type), dcb); - MXS_FREE(querystr); - } - else if (packet_type == '\x22' || packet_type == 0x22 || - packet_type == '\x26' || packet_type == 0x26 || true) - { - querystr = (char *)MXS_MALLOC(len); - MXS_ABORT_IF_NULL(querystr); - memcpy(querystr, startpos, len - 1); - querystr[len - 1] = '\0'; - MXS_DEBUG("%lu [%s] %d bytes long buf, \"%s\" -> %s:%d %s dcb %p", - pthread_self(), funcname, (int)buflen, querystr, - b->backend_server->name, b->backend_server->port, - STRBETYPE(be_type), dcb); - MXS_FREE(querystr); - } - } - gwbuf_free(buf); -} - -/** - * Return RCAP_TYPE_STMT_INPUT. - */ -static int getCapabilities() -{ - return RCAP_TYPE_STMT_INPUT; -} - -/** - * Execute in backends used by current router session. - * Save session variable commands to router session property - * struct. Thus, they can be replayed in backends which are - * started and joined later. - * - * Suppress redundant OK packets sent by backends. - * - * The first OK packet is replied to the client. - * - * @param router_cli_ses Client's router session pointer - * @param querybuf GWBUF including the query to be routed - * @param inst Router instance - * @param packet_type Type of MySQL packet - * @param qtype Query type from query_classifier - * - * @return True if at least one backend is used and routing succeed to all - * backends being used, otherwise false. - * - */ -static bool route_session_write(ROUTER_CLIENT_SES *router_cli_ses, - GWBUF *querybuf, ROUTER_INSTANCE *inst, - unsigned char packet_type, - qc_query_type_t qtype) -{ - bool succp; - rses_property_t *prop; - backend_ref_t *backend_ref; - int i; - int max_nslaves; - int nbackends; - int nsucc; - - MXS_INFO("Session write, routing to all servers."); - /** Maximum number of slaves in this router client session */ - max_nslaves = - rses_get_max_slavecount(router_cli_ses, router_cli_ses->rses_nbackends); - nsucc = 0; - nbackends = 0; - backend_ref = router_cli_ses->rses_backend_ref; - - /** - * These are one-way messages and server doesn't respond to them. - * Therefore reply processing is unnecessary and session - * command property is not needed. It is just routed to all available - * backends. - */ - if (packet_type == MYSQL_COM_STMT_SEND_LONG_DATA || - packet_type == MYSQL_COM_QUIT || packet_type == MYSQL_COM_STMT_CLOSE) - { - int rc; - - /** Lock router session */ - if (!rses_begin_locked_router_action(router_cli_ses)) - { - goto return_succp; - } - - for (i = 0; i < router_cli_ses->rses_nbackends; i++) - { - DCB *dcb = backend_ref[i].bref_dcb; - - if (MXS_LOG_PRIORITY_IS_ENABLED(LOG_INFO) && - BREF_IS_IN_USE((&backend_ref[i]))) - { - MXS_INFO("Route query to %s \t%s:%d%s", - (SERVER_IS_MASTER(backend_ref[i].bref_backend->backend_server) - ? "master" : "slave"), - backend_ref[i].bref_backend->backend_server->name, - backend_ref[i].bref_backend->backend_server->port, - (i + 1 == router_cli_ses->rses_nbackends ? " <" : " ")); - } - - if (BREF_IS_IN_USE((&backend_ref[i]))) - { - nbackends += 1; - if ((rc = dcb->func.write(dcb, gwbuf_clone(querybuf))) == 1) - { - nsucc += 1; - } - } - } - rses_end_locked_router_action(router_cli_ses); - gwbuf_free(querybuf); - goto return_succp; - } - /** Lock router session */ - if (!rses_begin_locked_router_action(router_cli_ses)) - { - goto return_succp; - } - - if (router_cli_ses->rses_nbackends <= 0) - { - MXS_INFO("Router session doesn't have any backends in use. Routing failed. <"); - goto return_succp; - } - - if (router_cli_ses->rses_config.rw_max_sescmd_history_size > 0 && - router_cli_ses->rses_nsescmd >= - router_cli_ses->rses_config.rw_max_sescmd_history_size) - { - MXS_WARNING("Router session exceeded session command history limit. " - "Slave recovery is disabled and only slave servers with " - "consistent session state are used " - "for the duration of the session."); - router_cli_ses->rses_config.rw_disable_sescmd_hist = true; - router_cli_ses->rses_config.rw_max_sescmd_history_size = 0; - } - - if (router_cli_ses->rses_config.rw_disable_sescmd_hist) - { - rses_property_t *prop, *tmp; - backend_ref_t *bref; - bool conflict; - - prop = router_cli_ses->rses_properties[RSES_PROP_TYPE_SESCMD]; - while (prop) - { - conflict = false; - - for (i = 0; i < router_cli_ses->rses_nbackends; i++) - { - bref = &backend_ref[i]; - if (BREF_IS_IN_USE(bref)) - { - - if (bref->bref_sescmd_cur.position <= - prop->rses_prop_data.sescmd.position + 1) - { - conflict = true; - break; - } - } - } - - if (conflict) - { - break; - } - - tmp = prop; - router_cli_ses->rses_properties[RSES_PROP_TYPE_SESCMD] = prop->rses_prop_next; - rses_property_done(tmp); - prop = router_cli_ses->rses_properties[RSES_PROP_TYPE_SESCMD]; - } - } - - /** - * Additional reference is created to querybuf to - * prevent it from being released before properties - * are cleaned up as a part of router sessionclean-up. - */ - if ((prop = rses_property_init(RSES_PROP_TYPE_SESCMD)) == NULL) - { - MXS_ERROR("Router session property initialization failed"); - rses_end_locked_router_action(router_cli_ses); - return false; - } - - mysql_sescmd_init(prop, querybuf, packet_type, router_cli_ses); - - /** Add sescmd property to router client session */ - if (rses_property_add(router_cli_ses, prop) != 0) - { - MXS_ERROR("Session property addition failed."); - rses_end_locked_router_action(router_cli_ses); - return false; - } - - for (i = 0; i < router_cli_ses->rses_nbackends; i++) - { - if (BREF_IS_IN_USE((&backend_ref[i]))) - { - sescmd_cursor_t *scur; - - nbackends += 1; - - if (MXS_LOG_PRIORITY_IS_ENABLED(LOG_INFO)) - { - MXS_INFO("Route query to %s \t%s:%d%s", - (SERVER_IS_MASTER(backend_ref[i].bref_backend->backend_server) - ? "master" : "slave"), - backend_ref[i].bref_backend->backend_server->name, - backend_ref[i].bref_backend->backend_server->port, - (i + 1 == router_cli_ses->rses_nbackends ? " <" : " ")); - } - - scur = backend_ref_get_sescmd_cursor(&backend_ref[i]); - - /** - * Add one waiter to backend reference. - */ - bref_set_state(get_bref_from_dcb(router_cli_ses, backend_ref[i].bref_dcb), - BREF_WAITING_RESULT); - /** - * Start execution if cursor is not already executing. - * Otherwise, cursor will execute pending commands - * when it completes with previous commands. - */ - if (sescmd_cursor_is_active(scur)) - { - nsucc += 1; - MXS_INFO("Backend %s:%d already executing sescmd.", - backend_ref[i].bref_backend->backend_server->name, - backend_ref[i].bref_backend->backend_server->port); - } - else - { - if (execute_sescmd_in_backend(&backend_ref[i])) - { - nsucc += 1; - } - else - { - MXS_ERROR("Failed to execute session command in %s:%d", - backend_ref[i].bref_backend->backend_server->name, - backend_ref[i].bref_backend->backend_server->port); - } - } - } - } - - atomic_add(&router_cli_ses->rses_nsescmd, 1); - - /** Unlock router session */ - rses_end_locked_router_action(router_cli_ses); - -return_succp: - /** - * Routing must succeed to all backends that are used. - * There must be at leas one and at most max_nslaves+1 backends. - */ - succp = (nbackends > 0 && nsucc == nbackends && nbackends <= max_nslaves + 1); - return succp; -} - -#if defined(NOT_USED) - -static bool router_option_configured(ROUTER_INSTANCE *router, - const char *optionstr, void *data) -{ - bool succp = false; - char **option; - - option = router->service->routerOptions; - - while (option != NULL) - { - char *value; - - if ((value = strchr(options[i], '=')) == NULL) - { - break; - } - else - { - *value = 0; - value++; - if (strcmp(options[i], "slave_selection_criteria") == 0) - { - if (GET_SELECT_CRITERIA(value) == (select_criteria_t *)*data) - { - succp = true; - break; - } - } - } - } - return succp; -} -#endif /*< NOT_USED */ - -/** - * @brief Process router options - * - * @param router Router instance - * @param options Router options - * @return True on success, false if a configuration error was found - */ -static bool rwsplit_process_router_options(ROUTER_INSTANCE *router, - char **options) -{ - int i; - char *value; - select_criteria_t c; - - if (options == NULL) - { - return true; - } - - bool success = true; - - for (i = 0; options[i]; i++) - { - if ((value = strchr(options[i], '=')) == NULL) - { - MXS_ERROR("Unsupported router option \"%s\" for readwritesplit router.", options[i]); - success = false; - } - else - { - *value = 0; - value++; - if (strcmp(options[i], "slave_selection_criteria") == 0) - { - c = GET_SELECT_CRITERIA(value); - ss_dassert(c == LEAST_GLOBAL_CONNECTIONS || - c == LEAST_ROUTER_CONNECTIONS || c == LEAST_BEHIND_MASTER || - c == LEAST_CURRENT_OPERATIONS || c == UNDEFINED_CRITERIA); - - if (c == UNDEFINED_CRITERIA) - { - MXS_ERROR("Unknown slave selection criteria \"%s\". " - "Allowed values are LEAST_GLOBAL_CONNECTIONS, " - "LEAST_ROUTER_CONNECTIONS, LEAST_BEHIND_MASTER," - "and LEAST_CURRENT_OPERATIONS.", - STRCRITERIA(router->rwsplit_config.rw_slave_select_criteria)); - success = false; - } - else - { - router->rwsplit_config.rw_slave_select_criteria = c; - } - } - else if (strcmp(options[i], "max_sescmd_history") == 0) - { - router->rwsplit_config.rw_max_sescmd_history_size = atoi(value); - } - else if (strcmp(options[i], "disable_sescmd_history") == 0) - { - router->rwsplit_config.rw_disable_sescmd_hist = config_truth_value(value); - } - else if (strcmp(options[i], "master_accept_reads") == 0) - { - router->rwsplit_config.rw_master_reads = config_truth_value(value); - } - else if (strcmp(options[i], "strict_multi_stmt") == 0) - { - router->rwsplit_config.rw_strict_multi_stmt = config_truth_value(value); - } - else if (strcmp(options[i], "master_failure_mode") == 0) - { - if (strcasecmp(value, "fail_instantly") == 0) - { - router->rwsplit_config.rw_master_failure_mode = RW_FAIL_INSTANTLY; - } - else if (strcasecmp(value, "fail_on_write") == 0) - { - router->rwsplit_config.rw_master_failure_mode = RW_FAIL_ON_WRITE; - } - else if (strcasecmp(value, "error_on_write") == 0) - { - router->rwsplit_config.rw_master_failure_mode = RW_ERROR_ON_WRITE; - } - else - { - MXS_ERROR("Unknown value for 'master_failure_mode': %s", value); - success = false; - } - } - else - { - MXS_ERROR("Unknown router option \"%s=%s\" for readwritesplit router.", - options[i], value); - success = false; - } - } - } /*< for */ - - return success; -} - /** + * @brief Router error handling routine (API) + * * Error Handler routine to resolve _backend_ failures. If it succeeds then - * there - * are enough operative backends available and connected. Otherwise it fails, - * and session is terminated. + * there are enough operative backends available and connected. Otherwise it + * fails, and session is terminated. * * @param instance The router instance * @param router_session The router session @@ -4375,6 +1191,479 @@ static void handleError(ROUTER *instance, void *router_session, dcb_close(problem_dcb); } +/** + * @brief Get router capabilities (API) + * + * Return a bit map indicating the characteristics of this particular router. + * In this case, the only bit set indicates that the router wants to receive + * data for routing as whole SQL statements. + * + * @return int RCAP_TYPE_STMT_INPUT. + */ +static int getCapabilities() +{ + return RCAP_TYPE_STMT_INPUT; +} + +/* + * This is the end of the API functions, and the start of functions that are + * used by the API functions and also used in other modules of the router + * code. Their prototypes are included in rwsplit_internal.h since these + * functions are not intended for use outside the read write split router. + */ + +/** + * @brief Acquires lock to router client session if it is not closed. + * + * Parameters: + * @param rses - in, use + * + * + * @return true if router session was not closed. If return value is true + * it means that router is locked, and must be unlocked later. False, if + * router was closed before lock was acquired. + * + */ +bool rses_begin_locked_router_action(ROUTER_CLIENT_SES *rses) +{ + bool succp = false; + + if (rses == NULL) + { + return false; + } + + CHK_CLIENT_RSES(rses); + + if (rses->rses_closed) + { + + goto return_succp; + } + spinlock_acquire(&rses->rses_lock); + if (rses->rses_closed) + { + spinlock_release(&rses->rses_lock); + goto return_succp; + } + succp = true; + +return_succp: + return succp; +} + +/** to be inline'd */ + +/** + * @brief Releases router client session lock. + * + * Parameters: + * @param rses - + * + * + * @return void + * + */ +void rses_end_locked_router_action(ROUTER_CLIENT_SES *rses) +{ + CHK_CLIENT_RSES(rses); + spinlock_release(&rses->rses_lock); +} + +/* + * @brief Clear one or more bits in the backend reference state + * + * The router session holds details of the backend servers that are + * involved in the routing for this particular service. Each backend + * server has a state bit string, and this function (along with + * bref_set_state) is used to manage the state. + * + * @param bref The backend reference to be modified + * @param state A bit string where the 1 bits indicate bits that should + * be turned off in the bref state. + * + */ +void bref_clear_state(backend_ref_t *bref, bref_state_t state) +{ + if (bref == NULL) + { + MXS_ERROR("[%s] Error: NULL parameter.", __FUNCTION__); + return; + } + if (state != BREF_WAITING_RESULT) + { + bref->bref_state &= ~state; + } + else + { + int prev1; + int prev2; + + /** Decrease waiter count */ + prev1 = atomic_add(&bref->bref_num_result_wait, -1); + + if (prev1 <= 0) + { + atomic_add(&bref->bref_num_result_wait, 1); + } + else + { + /** Decrease global operation count */ + prev2 = atomic_add(&bref->bref_backend->backend_server->stats.n_current_ops, -1); + ss_dassert(prev2 > 0); + if (prev2 <= 0) + { + MXS_ERROR("[%s] Error: negative current operation count in backend %s:%u", + __FUNCTION__, bref->bref_backend->backend_server->name, + bref->bref_backend->backend_server->port); + } + } + } +} + +/* + * @brief Set one or more bits in the backend reference state + * + * The router session holds details of the backend servers that are + * involved in the routing for this particular service. Each backend + * server has a state bit string, and this function (along with + * bref_clear_state) is used to manage the state. + * + * @param bref The backend reference to be modified + * @param state A bit string where the 1 bits indicate bits that should + * be turned on in the bref state. + * + */ +void bref_set_state(backend_ref_t *bref, bref_state_t state) +{ + if (bref == NULL) + { + MXS_ERROR("[%s] Error: NULL parameter.", __FUNCTION__); + return; + } + if (state != BREF_WAITING_RESULT) + { + bref->bref_state |= state; + } + else + { + int prev1; + int prev2; + + /** Increase waiter count */ + prev1 = atomic_add(&bref->bref_num_result_wait, 1); + ss_dassert(prev1 >= 0); + if (prev1 < 0) + { + MXS_ERROR("[%s] Error: negative number of connections waiting for " + "results in backend %s:%u", + __FUNCTION__, bref->bref_backend->backend_server->name, + bref->bref_backend->backend_server->port); + } + /** Increase global operation count */ + prev2 = + atomic_add(&bref->bref_backend->backend_server->stats.n_current_ops, 1); + ss_dassert(prev2 >= 0); + if (prev2 < 0) + { + MXS_ERROR("[%s] Error: negative current operation count in backend %s:%u", + __FUNCTION__, bref->bref_backend->backend_server->name, + bref->bref_backend->backend_server->port); + } + } +} + +/** + * @brief Free resources belonging to a property + * + * Property is freed at the end of router client session. + * + * @param prop The property whose resources are to be released + */ +void rses_property_done(rses_property_t *prop) +{ + if (prop == NULL) + { + MXS_ERROR("[%s] Error: NULL parameter.", __FUNCTION__); + return; + } + CHK_RSES_PROP(prop); + + switch (prop->rses_prop_type) + { + case RSES_PROP_TYPE_SESCMD: + mysql_sescmd_done(&prop->rses_prop_data.sescmd); + break; + + case RSES_PROP_TYPE_TMPTABLES: + hashtable_free(prop->rses_prop_data.temp_tables); + break; + + default: + MXS_DEBUG("%lu [rses_property_done] Unknown property type %d " + "in property %p", pthread_self(), prop->rses_prop_type, prop); + + ss_dassert(false); + break; + } + MXS_FREE(prop); +} + +/** + * @brief Get count of backend servers that are slaves. + * + * Find out the number of read backend servers. + * Depending on the configuration value type, either copy direct count + * of slave connections or calculate the count from percentage value. + * + * @param rses Router client session + * @param router_nservers The number of backend servers in total + */ +int rses_get_max_slavecount(ROUTER_CLIENT_SES *rses, + int router_nservers) +{ + int conf_max_nslaves; + int max_nslaves; + + CHK_CLIENT_RSES(rses); + + if (rses->rses_config.rw_max_slave_conn_count > 0) + { + conf_max_nslaves = rses->rses_config.rw_max_slave_conn_count; + } + else + { + conf_max_nslaves = (router_nservers * rses->rses_config.rw_max_slave_conn_percent) / 100; + } + max_nslaves = MIN(router_nservers - 1, MAX(1, conf_max_nslaves)); + + return max_nslaves; +} + +/* + * @brief Get the maximum replication lag for this router + * + * @param rses Router client session + * @return Replication lag from configuration or very large number + */ +int rses_get_max_replication_lag(ROUTER_CLIENT_SES *rses) +{ + int conf_max_rlag; + + CHK_CLIENT_RSES(rses); + + /** if there is no configured value, then longest possible int is used */ + if (rses->rses_config.rw_max_slave_replication_lag > 0) + { + conf_max_rlag = rses->rses_config.rw_max_slave_replication_lag; + } + else + { + conf_max_rlag = ~(1 << 31); + } + + return conf_max_rlag; +} + +/** + * @brief Find a back end reference that matches the given DCB + * + * Finds out if there is a backend reference pointing at the DCB given as + * parameter. + * + * @param rses router client session + * @param dcb DCB + * + * @return backend reference pointer if succeed or NULL + */ +backend_ref_t *get_bref_from_dcb(ROUTER_CLIENT_SES *rses, DCB *dcb) +{ + backend_ref_t *bref; + int i = 0; + CHK_DCB(dcb); + CHK_CLIENT_RSES(rses); + + bref = rses->rses_backend_ref; + + while (i < rses->rses_nbackends) + { + if (bref->bref_dcb == dcb) + { + break; + } + bref++; + i += 1; + } + + if (i == rses->rses_nbackends) + { + bref = NULL; + } + return bref; +} + +/** + * @brief Call hang up function + * + * Calls hang-up function for DCB if it is not both running and in + * master/slave/joined/ndb role. Called by DCB's callback routine. + */ +int router_handle_state_switch(DCB *dcb, DCB_REASON reason, void *data) +{ + backend_ref_t *bref; + int rc = 1; + SERVER *srv; + CHK_DCB(dcb); + + if (NULL == dcb->session->router_session) + { + /* + * The following processing will fail if there is no router session, + * because the "data" parameter will not contain meaningful data, + * so we have no choice but to stop here. + */ + return 0; + } + bref = (backend_ref_t *)data; + CHK_BACKEND_REF(bref); + + srv = bref->bref_backend->backend_server; + + if (SERVER_IS_RUNNING(srv) && SERVER_IS_IN_CLUSTER(srv)) + { + goto return_rc; + } + + MXS_DEBUG("%lu [router_handle_state_switch] %s %s:%d in state %s", + pthread_self(), STRDCBREASON(reason), srv->name, srv->port, + STRSRVSTATUS(srv)); + CHK_SESSION(((SESSION *)dcb->session)); + if (dcb->session->router_session) + { + CHK_CLIENT_RSES(((ROUTER_CLIENT_SES *)dcb->session->router_session)); + } + + switch (reason) + { + case DCB_REASON_NOT_RESPONDING: + dcb->func.hangup(dcb); + break; + + default: + break; + } + +return_rc: + return rc; +} + +/* + * The end of the functions used here and elsewhere in the router; start of + * functions that are purely internal to this module, i.e. are called directly + * or indirectly by the API functions and not used elsewhere. + */ + +/** + * @brief Process router options + * + * @param router Router instance + * @param options Router options + * @return True on success, false if a configuration error was found + */ +static bool rwsplit_process_router_options(ROUTER_INSTANCE *router, + char **options) +{ + int i; + char *value; + select_criteria_t c; + + if (options == NULL) + { + return true; + } + + bool success = true; + + for (i = 0; options[i]; i++) + { + if ((value = strchr(options[i], '=')) == NULL) + { + MXS_ERROR("Unsupported router option \"%s\" for readwritesplit router.", options[i]); + success = false; + } + else + { + *value = 0; + value++; + if (strcmp(options[i], "slave_selection_criteria") == 0) + { + c = GET_SELECT_CRITERIA(value); + ss_dassert(c == LEAST_GLOBAL_CONNECTIONS || + c == LEAST_ROUTER_CONNECTIONS || c == LEAST_BEHIND_MASTER || + c == LEAST_CURRENT_OPERATIONS || c == UNDEFINED_CRITERIA); + + if (c == UNDEFINED_CRITERIA) + { + MXS_ERROR("Unknown slave selection criteria \"%s\". " + "Allowed values are LEAST_GLOBAL_CONNECTIONS, " + "LEAST_ROUTER_CONNECTIONS, LEAST_BEHIND_MASTER," + "and LEAST_CURRENT_OPERATIONS.", + STRCRITERIA(router->rwsplit_config.rw_slave_select_criteria)); + success = false; + } + else + { + router->rwsplit_config.rw_slave_select_criteria = c; + } + } + else if (strcmp(options[i], "max_sescmd_history") == 0) + { + router->rwsplit_config.rw_max_sescmd_history_size = atoi(value); + } + else if (strcmp(options[i], "disable_sescmd_history") == 0) + { + router->rwsplit_config.rw_disable_sescmd_hist = config_truth_value(value); + } + else if (strcmp(options[i], "master_accept_reads") == 0) + { + router->rwsplit_config.rw_master_reads = config_truth_value(value); + } + else if (strcmp(options[i], "strict_multi_stmt") == 0) + { + router->rwsplit_config.rw_strict_multi_stmt = config_truth_value(value); + } + else if (strcmp(options[i], "master_failure_mode") == 0) + { + if (strcasecmp(value, "fail_instantly") == 0) + { + router->rwsplit_config.rw_master_failure_mode = RW_FAIL_INSTANTLY; + } + else if (strcasecmp(value, "fail_on_write") == 0) + { + router->rwsplit_config.rw_master_failure_mode = RW_FAIL_ON_WRITE; + } + else if (strcasecmp(value, "error_on_write") == 0) + { + router->rwsplit_config.rw_master_failure_mode = RW_ERROR_ON_WRITE; + } + else + { + MXS_ERROR("Unknown value for 'master_failure_mode': %s", value); + success = false; + } + } + else + { + MXS_ERROR("Unknown router option \"%s=%s\" for readwritesplit router.", + options[i], value); + success = false; + } + } + } /*< for */ + + return success; +} + static void handle_error_reply_client(SESSION *ses, ROUTER_CLIENT_SES *rses, DCB *backend_dcb, GWBUF *errmsg) { @@ -4504,53 +1793,6 @@ return_succp: return succp; } -static void print_error_packet(ROUTER_CLIENT_SES *rses, GWBUF *buf, DCB *dcb) -{ -#if defined(SS_DEBUG) - if (GWBUF_IS_TYPE_MYSQL(buf)) - { - while (gwbuf_length(buf) > 0) - { - /** - * This works with MySQL protocol only ! - * Protocol specific packet print functions would be nice. - */ - uint8_t *ptr = GWBUF_DATA(buf); - size_t len = MYSQL_GET_PACKET_LEN(ptr); - - if (MYSQL_GET_COMMAND(ptr) == 0xff) - { - SERVER *srv = NULL; - backend_ref_t *bref = rses->rses_backend_ref; - int i; - char *bufstr; - - for (i = 0; i < rses->rses_nbackends; i++) - { - if (bref[i].bref_dcb == dcb) - { - srv = bref[i].bref_backend->backend_server; - } - } - ss_dassert(srv != NULL); - char *str = (char *)&ptr[7]; - bufstr = strndup(str, len - 3); - - MXS_ERROR("Backend server %s:%d responded with " - "error : %s", - srv->name, srv->port, bufstr); - MXS_FREE(bufstr); - } - buf = gwbuf_consume(buf, len + 4); - } - } - else - { - gwbuf_free(buf); - } -#endif /*< SS_DEBUG */ -} - static int router_get_servercount(ROUTER_INSTANCE *inst) { int router_nservers = 0; @@ -4617,149 +1859,6 @@ static bool have_enough_servers(ROUTER_CLIENT_SES **p_rses, const int min_nsrv, return succp; } -/** - * Find out the number of read backend servers. - * Depending on the configuration value type, either copy direct count - * of slave connections or calculate the count from percentage value. - */ -static int rses_get_max_slavecount(ROUTER_CLIENT_SES *rses, - int router_nservers) -{ - int conf_max_nslaves; - int max_nslaves; - - CHK_CLIENT_RSES(rses); - - if (rses->rses_config.rw_max_slave_conn_count > 0) - { - conf_max_nslaves = rses->rses_config.rw_max_slave_conn_count; - } - else - { - conf_max_nslaves = (router_nservers * rses->rses_config.rw_max_slave_conn_percent) / 100; - } - max_nslaves = MIN(router_nservers - 1, MAX(1, conf_max_nslaves)); - - return max_nslaves; -} - -static int rses_get_max_replication_lag(ROUTER_CLIENT_SES *rses) -{ - int conf_max_rlag; - - CHK_CLIENT_RSES(rses); - - /** if there is no configured value, then longest possible int is used */ - if (rses->rses_config.rw_max_slave_replication_lag > 0) - { - conf_max_rlag = rses->rses_config.rw_max_slave_replication_lag; - } - else - { - conf_max_rlag = ~(1 << 31); - } - - return conf_max_rlag; -} - -/** - * Finds out if there is a backend reference pointing at the DCB given as - * parameter. - * @param rses router client session - * @param dcb DCB - * - * @return backend reference pointer if succeed or NULL - */ -static backend_ref_t *get_bref_from_dcb(ROUTER_CLIENT_SES *rses, DCB *dcb) -{ - backend_ref_t *bref; - int i = 0; - CHK_DCB(dcb); - CHK_CLIENT_RSES(rses); - - bref = rses->rses_backend_ref; - - while (i < rses->rses_nbackends) - { - if (bref->bref_dcb == dcb) - { - break; - } - bref++; - i += 1; - } - - if (i == rses->rses_nbackends) - { - bref = NULL; - } - return bref; -} - -/** - * Calls hang-up function for DCB if it is not both running and in - * master/slave/joined/ndb role. Called by DCB's callback routine. - */ -static int router_handle_state_switch(DCB *dcb, DCB_REASON reason, void *data) -{ - backend_ref_t *bref; - int rc = 1; - SERVER *srv; - CHK_DCB(dcb); - - if (NULL == dcb->session->router_session) - { - /* - * The following processing will fail if there is no router session, - * because the "data" parameter will not contain meaningful data, - * so we have no choice but to stop here. - */ - return 0; - } - bref = (backend_ref_t *)data; - CHK_BACKEND_REF(bref); - - srv = bref->bref_backend->backend_server; - - if (SERVER_IS_RUNNING(srv) && SERVER_IS_IN_CLUSTER(srv)) - { - goto return_rc; - } - - MXS_DEBUG("%lu [router_handle_state_switch] %s %s:%d in state %s", - pthread_self(), STRDCBREASON(reason), srv->name, srv->port, - STRSRVSTATUS(srv)); - CHK_SESSION(((SESSION *)dcb->session)); - if (dcb->session->router_session) - { - CHK_CLIENT_RSES(((ROUTER_CLIENT_SES *)dcb->session->router_session)); - } - - switch (reason) - { - case DCB_REASON_NOT_RESPONDING: - dcb->func.hangup(dcb); - break; - - default: - break; - } - -return_rc: - return rc; -} - -static sescmd_cursor_t *backend_ref_get_sescmd_cursor(backend_ref_t *bref) -{ - sescmd_cursor_t *scur; - CHK_BACKEND_REF(bref); - - scur = &bref->bref_sescmd_cur; - CHK_SESCMD_CUR(scur); - - return scur; -} - #if defined(PREP_STMT_CACHING) #define MAX_STMT_LEN 1024 @@ -4811,172 +1910,171 @@ static bool prep_stmt_drop(prep_stmt_t *pstmt) } #endif /*< PREP_STMT_CACHING */ -/******************************** - * This routine returns the root master server from MySQL replication tree - * Get the root Master rule: - * - * find server with the lowest replication depth level - * and the SERVER_MASTER bitval - * Servers are checked even if they are in 'maintenance' - * - * @param servers The list of servers - * @param router_nservers The number of servers - * @return The Master found - * - */ -static BACKEND *get_root_master(backend_ref_t *servers, int router_nservers) -{ - int i = 0; - BACKEND *master_host = NULL; - - for (i = 0; i < router_nservers; i++) - { - BACKEND *b; - - if (servers[i].bref_backend == NULL) - { - continue; - } - - b = servers[i].bref_backend; - - if ((b->backend_server->status & (SERVER_MASTER | SERVER_MAINT)) == - SERVER_MASTER) - { - if (master_host == NULL || - (b->backend_server->depth < master_host->backend_server->depth)) - { - master_host = b; - } - } - } - return master_host; -} - -/******************************** - * This routine returns the root master server from MySQL replication tree - * Get the root Master rule: - * - * find server with the lowest replication depth level - * and the SERVER_MASTER bitval - * Servers are checked even if they are in 'maintenance' - * - * @param rses pointer to router session - * @return pointer to backend reference of the root master or NULL - * - */ -static backend_ref_t *get_root_master_bref(ROUTER_CLIENT_SES *rses) -{ - backend_ref_t *bref; - backend_ref_t *candidate_bref = NULL; - SERVER master = {}; - - for (int i = 0; i < rses->rses_nbackends; i++) - { - bref = &rses->rses_backend_ref[i]; - if (bref && BREF_IS_IN_USE(bref)) - { - if (bref == rses->rses_master_ref) - { - /** Store master state for better error reporting */ - master.status = bref->bref_backend->backend_server->status; - } - - if (bref->bref_backend->backend_server->status & SERVER_MASTER) - { - if (candidate_bref == NULL || - (bref->bref_backend->backend_server->depth < - candidate_bref->bref_backend->backend_server->depth)) - { - candidate_bref = bref; - } - } - } - } - - if (candidate_bref == NULL && rses->rses_config.rw_master_failure_mode == RW_FAIL_INSTANTLY) - { - MXS_ERROR("Could not find master among the backend " - "servers. Previous master's state : %s", - rses->rses_master_ref == NULL ? "No master found" : - (!BREF_IS_IN_USE(rses->rses_master_ref) ? "Master is not in use" : - STRSRVSTATUS(&master))); - } - - return candidate_bref; -} - /** - * @brief Detect multi-statement queries + * @brief Refresh the instance by the given parameter value. * - * It is possible that the session state is modified inside a multi-statement - * query which would leave any slave sessions in an inconsistent state. Due to - * this, for the duration of this session, all queries will be sent to the - * master - * if the current query contains a multi-statement query. - * @param rses Router client session - * @param buf Buffer containing the full query - * @return True if the query contains multiple statements + * Used by createInstance and newSession + * + * @param router Router instance + * @param singleparam Parameter fo be reloaded + * + * Note: this part is not done. Needs refactoring. */ -static bool check_for_multi_stmt(ROUTER_CLIENT_SES *rses, GWBUF *buf, - mysql_server_cmd_t packet_type) +static void refreshInstance(ROUTER_INSTANCE *router, + CONFIG_PARAMETER *singleparam) { - MySQLProtocol *proto = (MySQLProtocol *)rses->client_dcb->protocol; - bool rval = false; + CONFIG_PARAMETER *param; + bool refresh_single; + config_param_type_t paramtype; - if (proto->client_capabilities & GW_MYSQL_CAPABILITIES_MULTI_STATEMENTS && - packet_type == MYSQL_COM_QUERY && - (rses->forced_node == NULL || rses->forced_node != rses->rses_master_ref)) + if (singleparam != NULL) { - char *ptr, *data = GWBUF_DATA(buf) + 5; - /** Payload size without command byte */ - int buflen = gw_mysql_get_byte3((uint8_t *)GWBUF_DATA(buf)) - 1; - - if ((ptr = strnchr_esc_mysql(data, ';', buflen))) - { - /** Skip stored procedures etc. */ - while (ptr && is_mysql_sp_end(ptr, buflen - (ptr - data))) - { - ptr = strnchr_esc_mysql(ptr + 1, ';', buflen - (ptr - data) - 1); - } - - if (ptr) - { - if (ptr < data + buflen && - !is_mysql_statement_end(ptr, buflen - (ptr - data))) - { - rses->forced_node = rses->rses_master_ref; - rval = true; - MXS_INFO("Multi-statement query, routing all future queries to master."); - } - } - } - } - - return rval; -} - -/** - * Send an error message to the client telling that the server is in read only mode - * @param dcb Client DCB - * @return True if sending the message was successful, false if an error occurred - */ -static bool send_readonly_error(DCB *dcb) -{ - bool succp = false; - const char* errmsg = "The MariaDB server is running with the --read-only" - " option so it cannot execute this statement"; - GWBUF* err = modutil_create_mysql_err_msg(1, 0, ER_OPTION_PREVENTS_STATEMENT, - "HY000", errmsg); - - if (err) - { - succp = dcb->func.write(dcb, err); + param = singleparam; + refresh_single = true; } else { - MXS_ERROR("Memory allocation failed when creating client error message."); + param = router->service->svc_config_param; + refresh_single = false; + } + paramtype = config_get_paramtype(param); + + while (param != NULL) + { + /** Catch unused parameter types */ + ss_dassert(paramtype == COUNT_TYPE || paramtype == PERCENT_TYPE || + paramtype == SQLVAR_TARGET_TYPE); + + if (paramtype == COUNT_TYPE) + { + if (strncmp(param->name, "max_slave_connections", MAX_PARAM_LEN) == 0) + { + int val; + bool succp; + + router->rwsplit_config.rw_max_slave_conn_percent = 0; + + succp = config_get_valint(&val, param, NULL, paramtype); + + if (succp) + { + router->rwsplit_config.rw_max_slave_conn_count = val; + } + } + else if (strncmp(param->name, "max_slave_replication_lag", + MAX_PARAM_LEN) == 0) + { + int val; + bool succp; + + succp = config_get_valint(&val, param, NULL, paramtype); + + if (succp) + { + router->rwsplit_config.rw_max_slave_replication_lag = val; + } + } + } + else if (paramtype == PERCENT_TYPE) + { + if (strncmp(param->name, "max_slave_connections", MAX_PARAM_LEN) == 0) + { + int val; + bool succp; + + router->rwsplit_config.rw_max_slave_conn_count = 0; + + succp = config_get_valint(&val, param, NULL, paramtype); + + if (succp) + { + router->rwsplit_config.rw_max_slave_conn_percent = val; + } + } + } + else if (paramtype == SQLVAR_TARGET_TYPE) + { + if (strncmp(param->name, "use_sql_variables_in", MAX_PARAM_LEN) == 0) + { + target_t valtarget; + bool succp; + + succp = config_get_valtarget(&valtarget, param, NULL, paramtype); + + if (succp) + { + router->rwsplit_config.rw_use_sql_variables_in = valtarget; + } + } + } + + if (refresh_single) + { + break; + } + param = param->next; } - return succp; +#if defined(NOT_USED) /*< can't read monitor config parameters */ + if ((*router->servers)->backend_server->rlag == -2) + { + rlag_enabled = false; + } + else + { + rlag_enabled = true; + } + /** + * If replication lag detection is not enabled the measure can't be + * used in slave selection. + */ + if (!rlag_enabled) + { + if (rlag_limited) + { + MXS_WARNING("Configuration Failed, max_slave_replication_lag " + "is set to %d,\n\t\t but detect_replication_lag " + "is not enabled. Replication lag will not be checked.", + router->rwsplit_config.rw_max_slave_replication_lag); + } + + if (router->rwsplit_config.rw_slave_select_criteria == + LEAST_BEHIND_MASTER) + { + MXS_WARNING("Configuration Failed, router option " + "\n\t\t slave_selection_criteria=LEAST_BEHIND_MASTER " + "is specified, but detect_replication_lag " + "is not enabled.\n\t\t " + "slave_selection_criteria=%s will be used instead.", + STRCRITERIA(DEFAULT_CRITERIA)); + + router->rwsplit_config.rw_slave_select_criteria = DEFAULT_CRITERIA; + } + } +#endif /*< NOT_USED */ } + +/* + * @brief Release resources when createInstance fails to complete + * + * Internal to createInstance + * + * @param router Router instance + * + */ +static void free_rwsplit_instance(ROUTER_INSTANCE *router) +{ + if (router) + { + if (router->servers) + { + for (int i = 0; router->servers[i]; i++) + { + MXS_FREE(router->servers[i]); + } + } + MXS_FREE(router->servers); + MXS_FREE(router); + } +} + diff --git a/server/modules/routing/readwritesplit/rwsplit_mysql.c b/server/modules/routing/readwritesplit/rwsplit_mysql.c new file mode 100644 index 000000000..bb6c30633 --- /dev/null +++ b/server/modules/routing/readwritesplit/rwsplit_mysql.c @@ -0,0 +1,541 @@ +/* + * Copyright (c) 2016 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl. + * + * Change Date: 2019-01-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(SS_DEBUG) +#include +#endif + +#define RWSPLIT_TRACE_MSG_LEN 1000 + +/** + * @file rwsplit_mysql.c Functions within the read-write split router that + * are specific to MySQL. The aim is to either remove these into a separate + * module or to move them into the MySQL protocol modules. + * + * @verbatim + * Revision History + * + * Date Who Description + * 08/08/2016 Martin Brampton Initial implementation + * + * @endverbatim + */ + +/* + * The following functions are called from elsewhere in the router and + * are defined in rwsplit_internal.h. They are not intended to be called + * from outside this router. + */ + +/* This could be placed in the protocol, with a new API entry point + * It is certainly MySQL specific. + * */ +int +determine_packet_type(GWBUF *querybuf, bool *non_empty_packet) +{ + mysql_server_cmd_t packet_type; + uint8_t *packet = GWBUF_DATA(querybuf); + + if (gw_mysql_get_byte3(packet) == 0) + { + /** Empty packet signals end of LOAD DATA LOCAL INFILE, send it to master*/ + *non_empty_packet = false; + packet_type = MYSQL_COM_UNDEFINED; + } + else + { + *non_empty_packet = true; + packet_type = packet[4]; + } + return (int)packet_type; +} + +/* + * This appears to be MySQL specific + */ +bool +is_packet_a_query(int packet_type) +{ + return (packet_type == MYSQL_COM_QUERY); +} + +/* + * This looks MySQL specific + */ +bool +is_packet_a_one_way_message(int packet_type) +{ + return (packet_type == MYSQL_COM_STMT_SEND_LONG_DATA || + packet_type == MYSQL_COM_QUIT || packet_type == MYSQL_COM_STMT_CLOSE); +} + +/* + * This one is problematic because it is MySQL specific, but also router + * specific. + */ +void +log_transaction_status(ROUTER_CLIENT_SES *rses, GWBUF *querybuf, qc_query_type_t qtype) +{ + if (!rses->rses_load_active) + { + uint8_t *packet = GWBUF_DATA(querybuf); + unsigned char ptype = packet[4]; + size_t len = MIN(GWBUF_LENGTH(querybuf), + MYSQL_GET_PACKET_LEN((unsigned char *)querybuf->start) - 1); + char *data = (char *)&packet[5]; + char *contentstr = strndup(data, MIN(len, RWSPLIT_TRACE_MSG_LEN)); + char *qtypestr = qc_get_qtype_str(qtype); + MXS_INFO("> Autocommit: %s, trx is %s, cmd: %s, type: %s, stmt: %s%s %s", + (rses->rses_autocommit_enabled ? "[enabled]" : "[disabled]"), + (rses->rses_transaction_active ? "[open]" : "[not open]"), + STRPACKETTYPE(ptype), (qtypestr == NULL ? "N/A" : qtypestr), + contentstr, (querybuf->hint == NULL ? "" : ", Hint:"), + (querybuf->hint == NULL ? "" : STRHINTTYPE(querybuf->hint->type))); + MXS_FREE(contentstr); + MXS_FREE(qtypestr); + } + else + { + MXS_INFO("> Processing LOAD DATA LOCAL INFILE: %lu bytes sent.", + rses->rses_load_data_sent); + } +} + +/* + * This is mostly router code, but it contains MySQL specific operations that + * maybe could be moved to the protocol module. The modutil functions are mostly + * MySQL specific and could migrate to the MySQL protocol; likewise the + * utility to convert packet type to a string. The aim is for most of this + * code to remain as part of the router. + */ +bool +handle_target_is_all(route_target_t route_target, + ROUTER_INSTANCE *inst, ROUTER_CLIENT_SES *rses, + GWBUF *querybuf, int packet_type, qc_query_type_t qtype) +{ + bool result; + + /** Multiple, conflicting routing target. Return error */ + if (TARGET_IS_MASTER(route_target) || TARGET_IS_SLAVE(route_target)) + { + backend_ref_t *bref = rses->rses_backend_ref; + + /* NOTE: modutil_get_query is MySQL specific */ + char *query_str = modutil_get_query(querybuf); + char *qtype_str = qc_get_qtype_str(qtype); + + /* NOTE: packet_type is MySQL specific */ + MXS_ERROR("Can't route %s:%s:\"%s\". SELECT with session data " + "modification is not supported if configuration parameter " + "use_sql_variables_in=all .", STRPACKETTYPE(packet_type), + qtype_str, (query_str == NULL ? "(empty)" : query_str)); + + MXS_INFO("Unable to route the query without losing session data " + "modification from other servers. <"); + + while (bref != NULL && !BREF_IS_IN_USE(bref)) + { + bref++; + } + + if (bref != NULL && BREF_IS_IN_USE(bref)) + { + /** Create and add MySQL error to eventqueue */ + modutil_reply_parse_error(bref->bref_dcb, + MXS_STRDUP_A("Routing query to backend failed. " + "See the error log for further " + "details."), 0); + result = true; + } + else + { + /** + * If there were no available backend references + * available return false - session will be closed + */ + MXS_ERROR("Sending error message to client " + "failed. Router doesn't have any " + "available backends. Session will be " + "closed."); + result = false; + } + /* Test shouldn't be needed */ + if (query_str) + { + MXS_FREE(query_str); + } + if (qtype_str) + { + MXS_FREE(qtype_str); + } + return result; + } + /** + * It is not sure if the session command in question requires + * response. Statement is examined in route_session_write. + * Router locking is done inside the function. + */ + result = route_session_write(rses, gwbuf_clone(querybuf), inst, + packet_type, qtype); + + if (result) + { + atomic_add(&inst->stats.n_all, 1); + } + return result; +} + +/* This is MySQL specific */ +void +session_lock_failure_handling(GWBUF *querybuf, int packet_type, qc_query_type_t qtype) +{ + if (packet_type != MYSQL_COM_QUIT) + { + /* NOTE: modutil_get_query is MySQL specific */ + char *query_str = modutil_get_query(querybuf); + + MXS_ERROR("Can't route %s:%s:\"%s\" to " + "backend server. Router is closed.", + STRPACKETTYPE(packet_type), STRQTYPE(qtype), + (query_str == NULL ? "(empty)" : query_str)); + MXS_FREE(query_str); + } +} + +/* + * Probably MySQL specific because of modutil function + */ +void closed_session_reply(GWBUF *querybuf) +{ + uint8_t* data = GWBUF_DATA(querybuf); + + if (GWBUF_LENGTH(querybuf) >= 5 && !MYSQL_IS_COM_QUIT(data)) + { + /* Note that most modutil functions are MySQL specific */ + char *query_str = modutil_get_query(querybuf); + MXS_ERROR("Can't route %s:\"%s\" to backend server. Router is closed.", + STRPACKETTYPE(data[4]), query_str ? query_str : "(empty)"); + MXS_FREE(query_str); + } +} + +/* + * Probably MySQL specific because of modutil function + */ +void live_session_reply(GWBUF **querybuf, ROUTER_CLIENT_SES *rses) +{ + GWBUF *tmpbuf = *querybuf; + if (GWBUF_IS_TYPE_UNDEFINED(tmpbuf)) + { + /* Note that many modutil functions are MySQL specific */ + *querybuf = modutil_get_complete_packets(&tmpbuf); + if (tmpbuf) + { + rses->client_dcb->dcb_readqueue = gwbuf_append(rses->client_dcb->dcb_readqueue, tmpbuf); + } + *querybuf = gwbuf_make_contiguous(*querybuf); + + /** Mark buffer to as MySQL type */ + gwbuf_set_type(*querybuf, GWBUF_TYPE_MYSQL); + gwbuf_set_type(*querybuf, GWBUF_TYPE_SINGLE_STMT); + } +} + +/* + * Uses MySQL specific mechanisms + */ +void print_error_packet(ROUTER_CLIENT_SES *rses, GWBUF *buf, DCB *dcb) +{ +#if defined(SS_DEBUG) + if (GWBUF_IS_TYPE_MYSQL(buf)) + { + while (gwbuf_length(buf) > 0) + { + /** + * This works with MySQL protocol only ! + * Protocol specific packet print functions would be nice. + */ + uint8_t *ptr = GWBUF_DATA(buf); + size_t len = MYSQL_GET_PACKET_LEN(ptr); + + if (MYSQL_GET_COMMAND(ptr) == 0xff) + { + SERVER *srv = NULL; + backend_ref_t *bref = rses->rses_backend_ref; + int i; + char *bufstr; + + for (i = 0; i < rses->rses_nbackends; i++) + { + if (bref[i].bref_dcb == dcb) + { + srv = bref[i].bref_backend->backend_server; + } + } + ss_dassert(srv != NULL); + char *str = (char *)&ptr[7]; + bufstr = strndup(str, len - 3); + + MXS_ERROR("Backend server %s:%d responded with " + "error : %s", + srv->name, srv->port, bufstr); + MXS_FREE(bufstr); + } + buf = gwbuf_consume(buf, len + 4); + } + } + else + { + gwbuf_free(buf); + } +#endif /*< SS_DEBUG */ +} + +/* + * Uses MySQL specific mechanisms + */ +void check_session_command_reply(GWBUF *writebuf, sescmd_cursor_t *scur, backend_ref_t *bref) +{ + if (MXS_LOG_PRIORITY_IS_ENABLED(LOG_ERR) && + MYSQL_IS_ERROR_PACKET(((uint8_t *)GWBUF_DATA(writebuf)))) + { + uint8_t *buf = (uint8_t *)GWBUF_DATA((scur->scmd_cur_cmd->my_sescmd_buf)); + uint8_t *replybuf = (uint8_t *)GWBUF_DATA(writebuf); + size_t len = MYSQL_GET_PACKET_LEN(buf); + size_t replylen = MYSQL_GET_PACKET_LEN(replybuf); + char *err = strndup(&((char *)replybuf)[8], 5); + char *replystr = strndup(&((char *)replybuf)[13], replylen - 4 - 5); + + ss_dassert(len + 4 == GWBUF_LENGTH(scur->scmd_cur_cmd->my_sescmd_buf)); + + MXS_ERROR("Failed to execute session command in %s:%d. Error was: %s %s", + bref->bref_backend->backend_server->name, + bref->bref_backend->backend_server->port, err, replystr); + MXS_FREE(err); + MXS_FREE(replystr); + } +} + +/** + * If session command cursor is passive, sends the command to backend for + * execution. + * + * Returns true if command was sent or added successfully to the queue. + * Returns false if command sending failed or if there are no pending session + * commands. + * + * Router session must be locked. + */ +/* + * Uses MySQL specific values in the large switch statement, although it + * may be possible to generalize them. + */ +bool execute_sescmd_in_backend(backend_ref_t *backend_ref) +{ + DCB *dcb; + bool succp; + int rc = 0; + sescmd_cursor_t *scur; + GWBUF *buf; + if (backend_ref == NULL) + { + MXS_ERROR("[%s] Error: NULL parameter.", __FUNCTION__); + return false; + } + if (BREF_IS_CLOSED(backend_ref)) + { + succp = false; + goto return_succp; + } + dcb = backend_ref->bref_dcb; + + CHK_DCB(dcb); + CHK_BACKEND_REF(backend_ref); + + /** + * Get cursor pointer and copy of command buffer to cursor. + */ + scur = &backend_ref->bref_sescmd_cur; + + /** Return if there are no pending ses commands */ + if (sescmd_cursor_get_command(scur) == NULL) + { + succp = true; + MXS_INFO("Cursor had no pending session commands."); + + goto return_succp; + } + + if (!sescmd_cursor_is_active(scur)) + { + /** Cursor is left active when function returns. */ + sescmd_cursor_set_active(scur, true); + } + + switch (scur->scmd_cur_cmd->my_sescmd_packet_type) + { + case MYSQL_COM_CHANGE_USER: + /** This makes it possible to handle replies correctly */ + gwbuf_set_type(scur->scmd_cur_cmd->my_sescmd_buf, GWBUF_TYPE_SESCMD); + buf = sescmd_cursor_clone_querybuf(scur); + rc = dcb->func.auth(dcb, NULL, dcb->session, buf); + break; + + case MYSQL_COM_INIT_DB: + { + /** + * Record database name and store to session. + */ + GWBUF *tmpbuf; + MYSQL_session *data; + unsigned int qlen; + + data = dcb->session->client_dcb->data; + *data->db = 0; + tmpbuf = scur->scmd_cur_cmd->my_sescmd_buf; + qlen = MYSQL_GET_PACKET_LEN((unsigned char *) GWBUF_DATA(tmpbuf)); + if (qlen) + { + --qlen; // The COM_INIT_DB byte + if (qlen > MYSQL_DATABASE_MAXLEN) + { + MXS_ERROR("Too long a database name received in COM_INIT_DB, " + "trailing data will be cut."); + qlen = MYSQL_DATABASE_MAXLEN; + } + + memcpy(data->db, (char*)GWBUF_DATA(tmpbuf) + 5, qlen); + data->db[qlen] = 0; + } + } + /** Fallthrough */ + case MYSQL_COM_QUERY: + default: + /** + * Mark session command buffer, it triggers writing + * MySQL command to protocol + */ + + gwbuf_set_type(scur->scmd_cur_cmd->my_sescmd_buf, GWBUF_TYPE_SESCMD); + buf = sescmd_cursor_clone_querybuf(scur); + rc = dcb->func.write(dcb, buf); + break; + } + + if (rc == 1) + { + succp = true; + } + else + { + succp = false; + } +return_succp: + return succp; +} + +/* + * End of functions called from other router modules; start of functions that + * are internal to this module + */ + +/** + * Get client DCB pointer of the router client session. + * This routine must be protected by Router client session lock. + * + * APPEARS TO NEVER BE USED!! + * + * @param rses Router client session pointer + * + * @return Pointer to client DCB + */ +static DCB *rses_get_client_dcb(ROUTER_CLIENT_SES *rses) +{ + DCB *dcb = NULL; + int i; + + for (i = 0; i < rses->rses_nbackends; i++) + { + if ((dcb = rses->rses_backend_ref[i].bref_dcb) != NULL && + BREF_IS_IN_USE(&rses->rses_backend_ref[i]) && dcb->session != NULL && + dcb->session->client_dcb != NULL) + { + return dcb->session->client_dcb; + } + } + return NULL; +} + +/* + * The following are internal (directly or indirectly) to routing a statement + * and should be moved to rwsplit_route_cmd.c if the MySQL specific code can + * be removed. + */ + +sescmd_cursor_t *backend_ref_get_sescmd_cursor(backend_ref_t *bref) +{ + sescmd_cursor_t *scur; + CHK_BACKEND_REF(bref); + + scur = &bref->bref_sescmd_cur; + CHK_SESCMD_CUR(scur); + + return scur; +} + +/** + * Send an error message to the client telling that the server is in read only mode + * @param dcb Client DCB + * @return True if sending the message was successful, false if an error occurred + */ +bool send_readonly_error(DCB *dcb) +{ + bool succp = false; + const char* errmsg = "The MariaDB server is running with the --read-only" + " option so it cannot execute this statement"; + GWBUF* err = modutil_create_mysql_err_msg(1, 0, ER_OPTION_PREVENTS_STATEMENT, + "HY000", errmsg); + + if (err) + { + succp = dcb->func.write(dcb, err); + } + else + { + MXS_ERROR("Memory allocation failed when creating client error message."); + } + + return succp; +} diff --git a/server/modules/routing/readwritesplit/rwsplit_route_stmt.c b/server/modules/routing/readwritesplit/rwsplit_route_stmt.c new file mode 100644 index 000000000..247c52064 --- /dev/null +++ b/server/modules/routing/readwritesplit/rwsplit_route_stmt.c @@ -0,0 +1,1304 @@ +/* + * Copyright (c) 2016 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl. + * + * Change Date: 2019-01-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ +#include +#include +#include +#include +#include +#include + +#include +#include +#include +/** + * @file rwsplit_route_stmt.c The functions that support the routing of + * queries to back end servers. All the functions in this module are internal + * to the read write split router, and not intended to be called from + * anywhere else. + * + * @verbatim + * Revision History + * + * Date Who Description + * 08/08/2016 Martin Brampton Initial implementation + * + * @endverbatim + */ + +extern int (*criteria_cmpfun[LAST_CRITERIA])(const void *, const void *); + +static backend_ref_t *check_candidate_bref(backend_ref_t *cand, + backend_ref_t *new, + select_criteria_t sc); +static backend_ref_t *get_root_master_bref(ROUTER_CLIENT_SES *rses); + +/** + * Routing function. Find out query type, backend type, and target DCB(s). + * Then route query to found target(s). + * @param inst router instance + * @param rses router session + * @param querybuf GWBUF including the query + * + * @return true if routing succeed or if it failed due to unsupported query. + * false if backend failure was encountered. + */ +bool route_single_stmt(ROUTER_INSTANCE *inst, ROUTER_CLIENT_SES *rses, + GWBUF *querybuf) +{ + qc_query_type_t qtype = QUERY_TYPE_UNKNOWN; + int packet_type; + DCB *target_dcb = NULL; + route_target_t route_target; + bool succp = false; + bool non_empty_packet; + + ss_dassert(querybuf->next == NULL); // The buffer must be contiguous. + ss_dassert(!GWBUF_IS_TYPE_UNDEFINED(querybuf)); + + /* packet_type is a problem as it is MySQL specific */ + packet_type = determine_packet_type(querybuf, &non_empty_packet); + qtype = determine_query_type(querybuf, packet_type, non_empty_packet); + if (non_empty_packet) + { + /** This might not be absolutely necessary as some parts of the code + * can only be executed by one thread at a time. */ + if (!rses_begin_locked_router_action(rses)) + { + return false; + } + handle_multi_temp_and_load(rses, querybuf, packet_type, (int *)&qtype); + rses_end_locked_router_action(rses); + /** + * If autocommit is disabled or transaction is explicitly started + * transaction becomes active and master gets all statements until + * transaction is committed and autocommit is enabled again. + */ + if (rses->rses_autocommit_enabled && + QUERY_IS_TYPE(qtype, QUERY_TYPE_DISABLE_AUTOCOMMIT)) + { + rses->rses_autocommit_enabled = false; + + if (!rses->rses_transaction_active) + { + rses->rses_transaction_active = true; + } + } + else if (!rses->rses_transaction_active && + QUERY_IS_TYPE(qtype, QUERY_TYPE_BEGIN_TRX)) + { + rses->rses_transaction_active = true; + } + /** + * Explicit COMMIT and ROLLBACK, implicit COMMIT. + */ + if (rses->rses_autocommit_enabled && rses->rses_transaction_active && + (QUERY_IS_TYPE(qtype, QUERY_TYPE_COMMIT) || + QUERY_IS_TYPE(qtype, QUERY_TYPE_ROLLBACK))) + { + rses->rses_transaction_active = false; + } + else if (!rses->rses_autocommit_enabled && + QUERY_IS_TYPE(qtype, QUERY_TYPE_ENABLE_AUTOCOMMIT)) + { + rses->rses_autocommit_enabled = true; + rses->rses_transaction_active = false; + } + + if (MXS_LOG_PRIORITY_IS_ENABLED(LOG_INFO)) + { + log_transaction_status(rses, querybuf, qtype); + } + /** + * Find out where to route the query. Result may not be clear; it is + * possible to have a hint for routing to a named server which can + * be either slave or master. + * If query would otherwise be routed to slave then the hint determines + * actual target server if it exists. + * + * route_target is a bitfield and may include : + * TARGET_ALL + * - route to all connected backend servers + * TARGET_SLAVE[|TARGET_NAMED_SERVER|TARGET_RLAG_MAX] + * - route primarily according to hints, then to slave and if those + * failed, eventually to master + * TARGET_MASTER[|TARGET_NAMED_SERVER|TARGET_RLAG_MAX] + * - route primarily according to the hints and if they failed, + * eventually to master + */ + route_target = get_route_target(rses, qtype, querybuf->hint); + } + else + { + route_target = TARGET_MASTER; + /** Empty packet signals end of LOAD DATA LOCAL INFILE, send it to master*/ + rses->rses_load_active = false; + MXS_INFO("> LOAD DATA LOCAL INFILE finished: %lu bytes sent.", + rses->rses_load_data_sent + gwbuf_length(querybuf)); + } + if (TARGET_IS_ALL(route_target)) + { + succp = handle_target_is_all(route_target, inst, rses, querybuf, packet_type, qtype); + } + else if (rses_begin_locked_router_action(rses)) + { + /* Now we have a lock on the router session */ + DCB *master_dcb = rses->rses_master_ref ? rses->rses_master_ref->bref_dcb : NULL; + + /** + * There is a hint which either names the target backend or + * hint which sets maximum allowed replication lag for the + * backend. + */ + if (TARGET_IS_NAMED_SERVER(route_target) || + TARGET_IS_RLAG_MAX(route_target)) + { + succp = handle_hinted_target(rses, querybuf, route_target, &target_dcb); + } + else if (TARGET_IS_SLAVE(route_target)) + { + succp = handle_slave_is_target(inst, rses, &target_dcb); + } + else if (TARGET_IS_MASTER(route_target)) + { + succp = handle_master_is_target(inst, rses, &target_dcb); + } + + if (target_dcb && succp) /*< Have DCB of the target backend */ + { + handle_got_target(inst, rses, querybuf, target_dcb); + } + rses_end_locked_router_action(rses); + } + else + { + session_lock_failure_handling(querybuf, packet_type, qtype); + succp = false; + } + + return succp; +} /* route_single_stmt */ + +/** + * Execute in backends used by current router session. + * Save session variable commands to router session property + * struct. Thus, they can be replayed in backends which are + * started and joined later. + * + * Suppress redundant OK packets sent by backends. + * + * The first OK packet is replied to the client. + * + * @param router_cli_ses Client's router session pointer + * @param querybuf GWBUF including the query to be routed + * @param inst Router instance + * @param packet_type Type of MySQL packet + * @param qtype Query type from query_classifier + * + * @return True if at least one backend is used and routing succeed to all + * backends being used, otherwise false. + * + */ +bool route_session_write(ROUTER_CLIENT_SES *router_cli_ses, + GWBUF *querybuf, ROUTER_INSTANCE *inst, + int packet_type, + qc_query_type_t qtype) +{ + bool succp; + rses_property_t *prop; + backend_ref_t *backend_ref; + int i; + int max_nslaves; + int nbackends; + int nsucc; + + MXS_INFO("Session write, routing to all servers."); + /** Maximum number of slaves in this router client session */ + max_nslaves = + rses_get_max_slavecount(router_cli_ses, router_cli_ses->rses_nbackends); + nsucc = 0; + nbackends = 0; + backend_ref = router_cli_ses->rses_backend_ref; + + /** + * These are one-way messages and server doesn't respond to them. + * Therefore reply processing is unnecessary and session + * command property is not needed. It is just routed to all available + * backends. + */ + if (is_packet_a_one_way_message(packet_type)) + { + int rc; + + /** Lock router session */ + if (!rses_begin_locked_router_action(router_cli_ses)) + { + goto return_succp; + } + + for (i = 0; i < router_cli_ses->rses_nbackends; i++) + { + DCB *dcb = backend_ref[i].bref_dcb; + + if (MXS_LOG_PRIORITY_IS_ENABLED(LOG_INFO) && + BREF_IS_IN_USE((&backend_ref[i]))) + { + MXS_INFO("Route query to %s \t%s:%d%s", + (SERVER_IS_MASTER(backend_ref[i].bref_backend->backend_server) + ? "master" : "slave"), + backend_ref[i].bref_backend->backend_server->name, + backend_ref[i].bref_backend->backend_server->port, + (i + 1 == router_cli_ses->rses_nbackends ? " <" : " ")); + } + + if (BREF_IS_IN_USE((&backend_ref[i]))) + { + nbackends += 1; + if ((rc = dcb->func.write(dcb, gwbuf_clone(querybuf))) == 1) + { + nsucc += 1; + } + } + } + rses_end_locked_router_action(router_cli_ses); + gwbuf_free(querybuf); + goto return_succp; + } + /** Lock router session */ + if (!rses_begin_locked_router_action(router_cli_ses)) + { + goto return_succp; + } + + if (router_cli_ses->rses_nbackends <= 0) + { + MXS_INFO("Router session doesn't have any backends in use. Routing failed. <"); + goto return_succp; + } + + if (router_cli_ses->rses_config.rw_max_sescmd_history_size > 0 && + router_cli_ses->rses_nsescmd >= + router_cli_ses->rses_config.rw_max_sescmd_history_size) + { + MXS_WARNING("Router session exceeded session command history limit. " + "Slave recovery is disabled and only slave servers with " + "consistent session state are used " + "for the duration of the session."); + router_cli_ses->rses_config.rw_disable_sescmd_hist = true; + router_cli_ses->rses_config.rw_max_sescmd_history_size = 0; + } + + if (router_cli_ses->rses_config.rw_disable_sescmd_hist) + { + rses_property_t *prop, *tmp; + backend_ref_t *bref; + bool conflict; + + prop = router_cli_ses->rses_properties[RSES_PROP_TYPE_SESCMD]; + while (prop) + { + conflict = false; + + for (i = 0; i < router_cli_ses->rses_nbackends; i++) + { + bref = &backend_ref[i]; + if (BREF_IS_IN_USE(bref)) + { + + if (bref->bref_sescmd_cur.position <= + prop->rses_prop_data.sescmd.position + 1) + { + conflict = true; + break; + } + } + } + + if (conflict) + { + break; + } + + tmp = prop; + router_cli_ses->rses_properties[RSES_PROP_TYPE_SESCMD] = prop->rses_prop_next; + rses_property_done(tmp); + prop = router_cli_ses->rses_properties[RSES_PROP_TYPE_SESCMD]; + } + } + + /** + * Additional reference is created to querybuf to + * prevent it from being released before properties + * are cleaned up as a part of router sessionclean-up. + */ + if ((prop = rses_property_init(RSES_PROP_TYPE_SESCMD)) == NULL) + { + MXS_ERROR("Router session property initialization failed"); + rses_end_locked_router_action(router_cli_ses); + return false; + } + + mysql_sescmd_init(prop, querybuf, packet_type, router_cli_ses); + + /** Add sescmd property to router client session */ + if (rses_property_add(router_cli_ses, prop) != 0) + { + MXS_ERROR("Session property addition failed."); + rses_end_locked_router_action(router_cli_ses); + return false; + } + + for (i = 0; i < router_cli_ses->rses_nbackends; i++) + { + if (BREF_IS_IN_USE((&backend_ref[i]))) + { + sescmd_cursor_t *scur; + + nbackends += 1; + + if (MXS_LOG_PRIORITY_IS_ENABLED(LOG_INFO)) + { + MXS_INFO("Route query to %s \t%s:%d%s", + (SERVER_IS_MASTER(backend_ref[i].bref_backend->backend_server) + ? "master" : "slave"), + backend_ref[i].bref_backend->backend_server->name, + backend_ref[i].bref_backend->backend_server->port, + (i + 1 == router_cli_ses->rses_nbackends ? " <" : " ")); + } + + scur = backend_ref_get_sescmd_cursor(&backend_ref[i]); + + /** + * Add one waiter to backend reference. + */ + bref_set_state(get_bref_from_dcb(router_cli_ses, backend_ref[i].bref_dcb), + BREF_WAITING_RESULT); + /** + * Start execution if cursor is not already executing. + * Otherwise, cursor will execute pending commands + * when it completes with previous commands. + */ + if (sescmd_cursor_is_active(scur)) + { + nsucc += 1; + MXS_INFO("Backend %s:%d already executing sescmd.", + backend_ref[i].bref_backend->backend_server->name, + backend_ref[i].bref_backend->backend_server->port); + } + else + { + if (execute_sescmd_in_backend(&backend_ref[i])) + { + nsucc += 1; + } + else + { + MXS_ERROR("Failed to execute session command in %s:%d", + backend_ref[i].bref_backend->backend_server->name, + backend_ref[i].bref_backend->backend_server->port); + } + } + } + } + + atomic_add(&router_cli_ses->rses_nsescmd, 1); + + /** Unlock router session */ + rses_end_locked_router_action(router_cli_ses); + +return_succp: + /** + * Routing must succeed to all backends that are used. + * There must be at least one and at most max_nslaves+1 backends. + */ + succp = (nbackends > 0 && nsucc == nbackends && nbackends <= max_nslaves + 1); + return succp; +} + +int rwsplit_hashkeyfun(const void *key) +{ + if (key == NULL) + { + return 0; + } + + unsigned int hash = 0, c = 0; + const char *ptr = (const char *)key; + + while ((c = *ptr++)) + { + hash = c + (hash << 6) + (hash << 16) - hash; + } + return hash; +} + +int rwsplit_hashcmpfun(const void *v1, const void *v2) +{ + const char *i1 = (const char *)v1; + const char *i2 = (const char *)v2; + + return strcmp(i1, i2); +} + +void *rwsplit_hstrdup(const void *fval) +{ + char *str = (char *)fval; + return MXS_STRDUP(str); +} + +void rwsplit_hfree(void *fval) +{ + MXS_FREE(fval); +} + +/** + * Provide the router with a pointer to a suitable backend dcb. + * + * Detect failures in server statuses and reselect backends if necessary. + * If name is specified, server name becomes primary selection criteria. + * Similarly, if max replication lag is specified, skip backends which lag too + * much. + * + * @param p_dcb Address of the pointer to the resulting DCB + * @param rses Pointer to router client session + * @param btype Backend type + * @param name Name of the backend which is primarily searched. May be NULL. + * + * @return True if proper DCB was found, false otherwise. + */ +bool rwsplit_get_dcb(DCB **p_dcb, ROUTER_CLIENT_SES *rses, backend_type_t btype, + char *name, int max_rlag) +{ + backend_ref_t *backend_ref; + backend_ref_t *master_bref; + int i; + bool succp = false; + + CHK_CLIENT_RSES(rses); + ss_dassert(p_dcb != NULL && *(p_dcb) == NULL); + + if (p_dcb == NULL) + { + goto return_succp; + } + backend_ref = rses->rses_backend_ref; + + /** get root master from available servers */ + master_bref = get_root_master_bref(rses); + + if (name != NULL) /*< Choose backend by name from a hint */ + { + ss_dassert(btype != BE_MASTER); /*< Master dominates and no name should be passed with it */ + + for (i = 0; i < rses->rses_nbackends; i++) + { + BACKEND *b = backend_ref[i].bref_backend; + SERVER server; + server.status = backend_ref[i].bref_backend->backend_server->status; + /** + * To become chosen: + * backend must be in use, name must match, + * backend's role must be either slave, relay + * server, or master. + */ + if (BREF_IS_IN_USE((&backend_ref[i])) && + (strncasecmp(name, b->backend_server->unique_name, PATH_MAX) == 0) && + (SERVER_IS_SLAVE(&server) || SERVER_IS_RELAY_SERVER(&server) || + SERVER_IS_MASTER(&server))) + { + *p_dcb = backend_ref[i].bref_dcb; + succp = true; + ss_dassert(backend_ref[i].bref_dcb->state != DCB_STATE_ZOMBIE); + break; + } + } + if (succp) + { + goto return_succp; + } + else + { + btype = BE_SLAVE; + } + } + + if (btype == BE_SLAVE) + { + backend_ref_t *candidate_bref = NULL; + + for (i = 0; i < rses->rses_nbackends; i++) + { + BACKEND *b = (&backend_ref[i])->bref_backend; + SERVER server; + SERVER candidate; + server.status = backend_ref[i].bref_backend->backend_server->status; + /** + * Unused backend or backend which is not master nor + * slave can't be used + */ + if (!BREF_IS_IN_USE(&backend_ref[i]) || + (!SERVER_IS_MASTER(&server) && !SERVER_IS_SLAVE(&server))) + { + continue; + } + /** + * If there are no candidates yet accept both master or + * slave. + */ + else if (candidate_bref == NULL) + { + /** + * Ensure that master has not changed dunring + * session and abort if it has. + */ + if (SERVER_IS_MASTER(&server) && &backend_ref[i] == master_bref) + { + /** found master */ + candidate_bref = &backend_ref[i]; + candidate.status = candidate_bref->bref_backend->backend_server->status; + succp = true; + } + /** + * Ensure that max replication lag is not set + * or that candidate's lag doesn't exceed the + * maximum allowed replication lag. + */ + else if (max_rlag == MAX_RLAG_UNDEFINED || + (b->backend_server->rlag != MAX_RLAG_NOT_AVAILABLE && + b->backend_server->rlag <= max_rlag)) + { + /** found slave */ + candidate_bref = &backend_ref[i]; + candidate.status = candidate_bref->bref_backend->backend_server->status; + succp = true; + } + } + /** + * If candidate is master, any slave which doesn't break + * replication lag limits replaces it. + */ + else if (SERVER_IS_MASTER(&candidate) && SERVER_IS_SLAVE(&server) && + (max_rlag == MAX_RLAG_UNDEFINED || + (b->backend_server->rlag != MAX_RLAG_NOT_AVAILABLE && + b->backend_server->rlag <= max_rlag)) && + !rses->rses_config.rw_master_reads) + { + /** found slave */ + candidate_bref = &backend_ref[i]; + candidate.status = candidate_bref->bref_backend->backend_server->status; + succp = true; + } + /** + * When candidate exists, compare it against the current + * backend and update assign it to new candidate if + * necessary. + */ + else if (SERVER_IS_SLAVE(&server)) + { + if (max_rlag == MAX_RLAG_UNDEFINED || + (b->backend_server->rlag != MAX_RLAG_NOT_AVAILABLE && + b->backend_server->rlag <= max_rlag)) + { + candidate_bref = + check_candidate_bref(candidate_bref, &backend_ref[i], + rses->rses_config.rw_slave_select_criteria); + candidate.status = + candidate_bref->bref_backend->backend_server->status; + } + else + { + MXS_INFO("Server %s:%d is too much behind the " + "master, %d s. and can't be chosen.", + b->backend_server->name, b->backend_server->port, + b->backend_server->rlag); + } + } + } /*< for */ + /** Assign selected DCB's pointer value */ + if (candidate_bref != NULL) + { + *p_dcb = candidate_bref->bref_dcb; + } + + goto return_succp; + } /*< if (btype == BE_SLAVE) */ + /** + * If target was originally master only then the execution jumps + * directly here. + */ + if (btype == BE_MASTER) + { + if (master_bref) + { + /** It is possible for the server status to change at any point in time + * so copying it locally will make possible error messages + * easier to understand */ + SERVER server; + server.status = master_bref->bref_backend->backend_server->status; + if (BREF_IS_IN_USE(master_bref) && SERVER_IS_MASTER(&server)) + { + *p_dcb = master_bref->bref_dcb; + succp = true; + /** if bref is in use DCB should not be closed */ + ss_dassert(master_bref->bref_dcb->state != DCB_STATE_ZOMBIE); + } + else + { + MXS_ERROR("Server at %s:%d should be master but " + "is %s instead and can't be chosen to master.", + master_bref->bref_backend->backend_server->name, + master_bref->bref_backend->backend_server->port, + STRSRVSTATUS(&server)); + succp = false; + } + } + } + +return_succp: + return succp; +} + +/** + * Examine the query type, transaction state and routing hints. Find out the + * target for query routing. + * + * @param qtype Type of query + * @param trx_active Is transacation active or not + * @param hint Pointer to list of hints attached to the query buffer + * + * @return bitfield including the routing target, or the target server name + * if the query would otherwise be routed to slave. + */ +route_target_t get_route_target(ROUTER_CLIENT_SES *rses, + qc_query_type_t qtype, HINT *hint) +{ + bool trx_active = rses->rses_transaction_active; + bool load_active = rses->rses_load_active; + target_t use_sql_variables_in = rses->rses_config.rw_use_sql_variables_in; + route_target_t target = TARGET_UNDEFINED; + + if (rses->rses_config.rw_strict_multi_stmt && rses->forced_node && + rses->forced_node == rses->rses_master_ref) + { + target = TARGET_MASTER; + } + /** + * These queries are not affected by hints + */ + else if (!load_active && + (QUERY_IS_TYPE(qtype, QUERY_TYPE_SESSION_WRITE) || + /** Configured to allow writing variables to all nodes */ + (use_sql_variables_in == TYPE_ALL && + QUERY_IS_TYPE(qtype, QUERY_TYPE_GSYSVAR_WRITE)) || + /** enable or disable autocommit are always routed to all */ + QUERY_IS_TYPE(qtype, QUERY_TYPE_ENABLE_AUTOCOMMIT) || + QUERY_IS_TYPE(qtype, QUERY_TYPE_DISABLE_AUTOCOMMIT))) + { + /** + * This is problematic query because it would be routed to all + * backends but since this is SELECT that is not possible: + * 1. response set is not handled correctly in clientReply and + * 2. multiple results can degrade performance. + * + * Prepared statements are an exception to this since they do not + * actually do anything but only prepare the statement to be used. + * They can be safely routed to all backends since the execution + * is done later. + * + * With prepared statement caching the task of routing + * the execution of the prepared statements to the right server would be + * an easy one. Currently this is not supported. + */ + if (QUERY_IS_TYPE(qtype, QUERY_TYPE_READ) && + !(QUERY_IS_TYPE(qtype, QUERY_TYPE_PREPARE_STMT) || + QUERY_IS_TYPE(qtype, QUERY_TYPE_PREPARE_NAMED_STMT))) + { + MXS_WARNING("The query can't be routed to all " + "backend servers because it includes SELECT and " + "SQL variable modifications which is not supported. " + "Set use_sql_variables_in=master or split the " + "query to two, where SQL variable modifications " + "are done in the first and the SELECT in the " + "second one."); + + target = TARGET_MASTER; + } + target |= TARGET_ALL; + } + /** + * Hints may affect on routing of the following queries + */ + else if (!trx_active && !load_active && + !QUERY_IS_TYPE(qtype, QUERY_TYPE_WRITE) && + (QUERY_IS_TYPE(qtype, QUERY_TYPE_READ) || /*< any SELECT */ + QUERY_IS_TYPE(qtype, QUERY_TYPE_SHOW_TABLES) || /*< 'SHOW TABLES' */ + QUERY_IS_TYPE(qtype, + QUERY_TYPE_USERVAR_READ) || /*< read user var */ + QUERY_IS_TYPE(qtype, QUERY_TYPE_SYSVAR_READ) || /*< read sys var */ + QUERY_IS_TYPE(qtype, + QUERY_TYPE_EXEC_STMT) || /*< prepared stmt exec */ + QUERY_IS_TYPE(qtype, QUERY_TYPE_PREPARE_STMT) || + QUERY_IS_TYPE(qtype, QUERY_TYPE_PREPARE_NAMED_STMT) || + QUERY_IS_TYPE(qtype, + QUERY_TYPE_GSYSVAR_READ))) /*< read global sys var */ + { + /** First set expected targets before evaluating hints */ + if (!QUERY_IS_TYPE(qtype, QUERY_TYPE_MASTER_READ) && + (QUERY_IS_TYPE(qtype, QUERY_TYPE_READ) || + QUERY_IS_TYPE(qtype, QUERY_TYPE_SHOW_TABLES) || /*< 'SHOW TABLES' */ + /** Configured to allow reading variables from slaves */ + (use_sql_variables_in == TYPE_ALL && + (QUERY_IS_TYPE(qtype, QUERY_TYPE_USERVAR_READ) || + QUERY_IS_TYPE(qtype, QUERY_TYPE_SYSVAR_READ) || + QUERY_IS_TYPE(qtype, QUERY_TYPE_GSYSVAR_READ))))) + { + target = TARGET_SLAVE; + } + + if (QUERY_IS_TYPE(qtype, QUERY_TYPE_MASTER_READ) || + QUERY_IS_TYPE(qtype, QUERY_TYPE_EXEC_STMT) || + QUERY_IS_TYPE(qtype, QUERY_TYPE_PREPARE_STMT) || + QUERY_IS_TYPE(qtype, QUERY_TYPE_PREPARE_NAMED_STMT) || + /** Configured not to allow reading variables from slaves */ + (use_sql_variables_in == TYPE_MASTER && + (QUERY_IS_TYPE(qtype, QUERY_TYPE_USERVAR_READ) || + QUERY_IS_TYPE(qtype, QUERY_TYPE_SYSVAR_READ)))) + { + target = TARGET_MASTER; + } + + /** If nothing matches then choose the master */ + if ((target & (TARGET_ALL | TARGET_SLAVE | TARGET_MASTER)) == 0) + { + target = TARGET_MASTER; + } + } + else + { + /** hints don't affect on routing */ + ss_dassert(trx_active || + (QUERY_IS_TYPE(qtype, QUERY_TYPE_WRITE) || + QUERY_IS_TYPE(qtype, QUERY_TYPE_MASTER_READ) || + QUERY_IS_TYPE(qtype, QUERY_TYPE_SESSION_WRITE) || + (QUERY_IS_TYPE(qtype, QUERY_TYPE_USERVAR_READ) && + use_sql_variables_in == TYPE_MASTER) || + (QUERY_IS_TYPE(qtype, QUERY_TYPE_SYSVAR_READ) && + use_sql_variables_in == TYPE_MASTER) || + (QUERY_IS_TYPE(qtype, QUERY_TYPE_GSYSVAR_READ) && + use_sql_variables_in == TYPE_MASTER) || + (QUERY_IS_TYPE(qtype, QUERY_TYPE_GSYSVAR_WRITE) && + use_sql_variables_in == TYPE_MASTER) || + QUERY_IS_TYPE(qtype, QUERY_TYPE_BEGIN_TRX) || + QUERY_IS_TYPE(qtype, QUERY_TYPE_ENABLE_AUTOCOMMIT) || + QUERY_IS_TYPE(qtype, QUERY_TYPE_DISABLE_AUTOCOMMIT) || + QUERY_IS_TYPE(qtype, QUERY_TYPE_ROLLBACK) || + QUERY_IS_TYPE(qtype, QUERY_TYPE_COMMIT) || + QUERY_IS_TYPE(qtype, QUERY_TYPE_EXEC_STMT) || + QUERY_IS_TYPE(qtype, QUERY_TYPE_CREATE_TMP_TABLE) || + QUERY_IS_TYPE(qtype, QUERY_TYPE_READ_TMP_TABLE) || + QUERY_IS_TYPE(qtype, QUERY_TYPE_UNKNOWN))); + target = TARGET_MASTER; + } + + /** process routing hints */ + while (hint != NULL) + { + if (hint->type == HINT_ROUTE_TO_MASTER) + { + target = TARGET_MASTER; /*< override */ + MXS_DEBUG("%lu [get_route_target] Hint: route to master.", + pthread_self()); + break; + } + else if (hint->type == HINT_ROUTE_TO_NAMED_SERVER) + { + /** + * Searching for a named server. If it can't be + * found, the oroginal target is chosen. + */ + target |= TARGET_NAMED_SERVER; + MXS_DEBUG("%lu [get_route_target] Hint: route to " + "named server : ", + pthread_self()); + } + else if (hint->type == HINT_ROUTE_TO_UPTODATE_SERVER) + { + /** not implemented */ + } + else if (hint->type == HINT_ROUTE_TO_ALL) + { + /** not implemented */ + } + else if (hint->type == HINT_PARAMETER) + { + if (strncasecmp((char *)hint->data, "max_slave_replication_lag", + strlen("max_slave_replication_lag")) == 0) + { + target |= TARGET_RLAG_MAX; + } + else + { + MXS_ERROR("Unknown hint parameter " + "'%s' when 'max_slave_replication_lag' " + "was expected.", + (char *)hint->data); + } + } + else if (hint->type == HINT_ROUTE_TO_SLAVE) + { + target = TARGET_SLAVE; + MXS_DEBUG("%lu [get_route_target] Hint: route to " + "slave.", + pthread_self()); + } + hint = hint->next; + } /*< while (hint != NULL) */ + +#if defined(SS_EXTRA_DEBUG) + MXS_INFO("Selected target \"%s\"", STRTARGET(target)); +#endif + return target; +} + +void +handle_multi_temp_and_load(ROUTER_CLIENT_SES *rses, GWBUF *querybuf, + int packet_type, int *qtype) +{ + /** Check for multi-statement queries. If no master server is available + * and a multi-statement is issued, an error is returned to the client + * when the query is routed. + * + * If we do not have a master node, assigning the forced node is not + * effective since we don't have a node to force queries to. In this + * situation, assigning QUERY_TYPE_WRITE for the query will trigger + * the error processing. */ + if ((rses->forced_node == NULL || rses->forced_node != rses->rses_master_ref) && + check_for_multi_stmt(querybuf, rses->client_dcb->protocol, packet_type)) + { + if (rses->rses_master_ref) + { + rses->forced_node = rses->rses_master_ref; + MXS_INFO("Multi-statement query, routing all future queries to master."); + } + else + { + *qtype |= QUERY_TYPE_WRITE; + } + } + + /* + * Make checks prior to calling temp tables functions + */ + + if (rses == NULL || querybuf == NULL || + rses->client_dcb == NULL || rses->client_dcb->data == NULL) + { + if (rses == NULL || querybuf == NULL) + { + MXS_ERROR("[%s] Error: NULL variables for temp table checks: %p %p", __FUNCTION__, + rses, querybuf); + } + + if (rses->client_dcb == NULL) + { + MXS_ERROR("[%s] Error: Client DCB is NULL.", __FUNCTION__); + } + + if (rses->client_dcb->data == NULL) + { + MXS_ERROR("[%s] Error: User data in master server DBC is NULL.", + __FUNCTION__); + } + } + + else + { + /** + * Check if the query has anything to do with temporary tables. + */ + if (rses->have_tmp_tables) + { + rses_property_t *rses_prop_tmp; + + rses_prop_tmp = rses->rses_properties[RSES_PROP_TYPE_TMPTABLES]; + check_drop_tmp_table(rses, querybuf, packet_type); + if (is_packet_a_query(packet_type)) + { + *qtype = is_read_tmp_table(rses, querybuf, *qtype); + } + } + check_create_tmp_table(rses, querybuf, *qtype); + } + + /** + * Check if this is a LOAD DATA LOCAL INFILE query. If so, send all queries + * to the master until the last, empty packet arrives. + */ + if (rses->rses_load_active) + { + rses->rses_load_data_sent += gwbuf_length(querybuf); + } + else if (is_packet_a_query(packet_type)) + { + qc_query_op_t queryop = qc_get_operation(querybuf); + if (queryop == QUERY_OP_LOAD) + { + rses->rses_load_active = true; + rses->rses_load_data_sent = 0; + } + } +} + +bool handle_hinted_target(ROUTER_CLIENT_SES *rses, GWBUF *querybuf, + route_target_t route_target, DCB **target_dcb) +{ + HINT *hint; + char *named_server = NULL; + backend_type_t btype; /*< target backend type */ + int rlag_max = MAX_RLAG_UNDEFINED; + bool succp; + + hint = querybuf->hint; + + while (hint != NULL) + { + if (hint->type == HINT_ROUTE_TO_NAMED_SERVER) + { + /** + * Set the name of searched + * backend server. + */ + named_server = hint->data; + MXS_INFO("Hint: route to server " + "'%s'", + named_server); + } + else if (hint->type == HINT_PARAMETER && + (strncasecmp((char *)hint->data, "max_slave_replication_lag", + strlen("max_slave_replication_lag")) == 0)) + { + int val = (int)strtol((char *)hint->value, (char **)NULL, 10); + + if (val != 0 || errno == 0) + { + /** Set max. acceptable replication lag value for backend srv */ + rlag_max = val; + MXS_INFO("Hint: max_slave_replication_lag=%d", rlag_max); + } + } + hint = hint->next; + } /*< while */ + + if (rlag_max == MAX_RLAG_UNDEFINED) /*< no rlag max hint, use config */ + { + rlag_max = rses_get_max_replication_lag(rses); + } + + /** target may be master or slave */ + btype = route_target & TARGET_SLAVE ? BE_SLAVE : BE_MASTER; + + /** + * Search backend server by name or replication lag. + * If it fails, then try to find valid slave or master. + */ + succp = rwsplit_get_dcb(target_dcb, rses, btype, named_server, rlag_max); + + if (!succp) + { + if (TARGET_IS_NAMED_SERVER(route_target)) + { + MXS_INFO("Was supposed to route to named server " + "%s but couldn't find the server in a " + "suitable state.", named_server); + } + else if (TARGET_IS_RLAG_MAX(route_target)) + { + MXS_INFO("Was supposed to route to server with " + "replication lag at most %d but couldn't " + "find such a slave.", rlag_max); + } + } + return succp; +} + +bool handle_slave_is_target(ROUTER_INSTANCE *inst, ROUTER_CLIENT_SES *rses, + DCB **target_dcb) +{ + int rlag_max = rses_get_max_replication_lag(rses); + + /** + * Search suitable backend server, get DCB in target_dcb + */ + if (rwsplit_get_dcb(target_dcb, rses, BE_SLAVE, NULL, rlag_max)) + { +#if defined(SS_EXTRA_DEBUG) + MXS_INFO("Found DCB for slave."); +#endif + atomic_add(&inst->stats.n_slave, 1); + return true; + } + else + { + MXS_INFO("Was supposed to route to slave but finding suitable one failed."); + return false; + } +} + +bool handle_master_is_target(ROUTER_INSTANCE *inst, ROUTER_CLIENT_SES *rses, + DCB **target_dcb) +{ + DCB *master_dcb = rses->rses_master_ref ? rses->rses_master_ref->bref_dcb : NULL; + DCB *curr_master_dcb = NULL; + bool succp = rwsplit_get_dcb(&curr_master_dcb, rses, BE_MASTER, NULL, MAX_RLAG_UNDEFINED); + + if (succp && master_dcb == curr_master_dcb) + { + atomic_add(&inst->stats.n_master, 1); + *target_dcb = master_dcb; + } + else + { + if (succp && master_dcb != curr_master_dcb) + { + MXS_INFO("Was supposed to route to master but master has changed."); + } + else + { + MXS_INFO("Was supposed to route to master but couldn't find master" + " in a suitable state."); + } + + if (rses->rses_config.rw_master_failure_mode == RW_ERROR_ON_WRITE) + { + /** Old master is no longer available */ + succp = send_readonly_error(rses->client_dcb); + } + else + { + MXS_WARNING("[%s] Write query received from %s@%s when no master is " + "available, closing client connection.", inst->service->name, + rses->client_dcb->user, rses->client_dcb->remote); + succp = false; + } + } + return succp; +} + +bool +handle_got_target(ROUTER_INSTANCE *inst, ROUTER_CLIENT_SES *rses, + GWBUF *querybuf, DCB *target_dcb) +{ + backend_ref_t *bref; + sescmd_cursor_t *scur; + + bref = get_bref_from_dcb(rses, target_dcb); + scur = &bref->bref_sescmd_cur; + + ss_dassert(target_dcb != NULL); + + MXS_INFO("Route query to %s \t%s:%d <", + (SERVER_IS_MASTER(bref->bref_backend->backend_server) ? "master" + : "slave"), bref->bref_backend->backend_server->name, + bref->bref_backend->backend_server->port); + /** + * Store current stmt if execution of previous session command + * haven't completed yet. + * + * !!! Note that according to MySQL protocol + * there can only be one such non-sescmd stmt at the time. + * It is possible that bref->bref_pending_cmd includes a pending + * command if rwsplit is parent or child for another router, + * which runs all the same commands. + * + * If the assertion below traps, pending queries are treated + * somehow wrong, or client is sending more queries before + * previous is received. + */ + if (sescmd_cursor_is_active(scur)) + { + ss_dassert(bref->bref_pending_cmd == NULL); + bref->bref_pending_cmd = gwbuf_clone(querybuf); + + return true; + } + + if (target_dcb->func.write(target_dcb, gwbuf_clone(querybuf)) == 1) + { + backend_ref_t *bref; + + atomic_add(&inst->stats.n_queries, 1); + /** + * Add one query response waiter to backend reference + */ + bref = get_bref_from_dcb(rses, target_dcb); + bref_set_state(bref, BREF_QUERY_ACTIVE); + bref_set_state(bref, BREF_WAITING_RESULT); + return true; + } + else + { + MXS_ERROR("Routing query failed."); + return false; + } +} + +/** + * Create a generic router session property structure. + */ +rses_property_t *rses_property_init(rses_property_type_t prop_type) +{ + rses_property_t *prop; + + prop = (rses_property_t *)MXS_CALLOC(1, sizeof(rses_property_t)); + if (prop == NULL) + { + return NULL; + } + prop->rses_prop_type = prop_type; +#if defined(SS_DEBUG) + prop->rses_prop_chk_top = CHK_NUM_ROUTER_PROPERTY; + prop->rses_prop_chk_tail = CHK_NUM_ROUTER_PROPERTY; +#endif + + CHK_RSES_PROP(prop); + return prop; +} + +/** + * Add property to the router_client_ses structure's rses_properties + * array. The slot is determined by the type of property. + * In each slot there is a list of properties of similar type. + * + * Router client session must be locked. + */ +int rses_property_add(ROUTER_CLIENT_SES *rses, rses_property_t *prop) +{ + if (rses == NULL) + { + MXS_ERROR("Router client session is NULL. (%s:%d)", __FILE__, __LINE__); + return -1; + } + if (prop == NULL) + { + MXS_ERROR("Router client session property is NULL. (%s:%d)", __FILE__, __LINE__); + return -1; + } + rses_property_t *p; + + CHK_CLIENT_RSES(rses); + CHK_RSES_PROP(prop); + ss_dassert(SPINLOCK_IS_LOCKED(&rses->rses_lock)); + + prop->rses_prop_rsession = rses; + p = rses->rses_properties[prop->rses_prop_type]; + + if (p == NULL) + { + rses->rses_properties[prop->rses_prop_type] = prop; + } + else + { + while (p->rses_prop_next != NULL) + { + p = p->rses_prop_next; + } + p->rses_prop_next = prop; + } + return 0; +} + +/** + * Find out which of the two backend servers has smaller value for select + * criteria property. + * + * @param cand previously selected candidate + * @param new challenger + * @param sc select criteria + * + * @return pointer to backend reference of that backend server which has smaller + * value in selection criteria. If either reference pointer is NULL then the + * other reference pointer value is returned. + */ +static backend_ref_t *check_candidate_bref(backend_ref_t *cand, + backend_ref_t *new, + select_criteria_t sc) +{ + int (*p)(const void *, const void *); + /** get compare function */ + p = criteria_cmpfun[sc]; + + if (new == NULL) + { + return cand; + } + else if (cand == NULL || (p((void *)cand, (void *)new) > 0)) + { + return new; + } + else + { + return cand; + } +} + +/******************************** + * This routine returns the root master server from MySQL replication tree + * Get the root Master rule: + * + * find server with the lowest replication depth level + * and the SERVER_MASTER bitval + * Servers are checked even if they are in 'maintenance' + * + * @param rses pointer to router session + * @return pointer to backend reference of the root master or NULL + * + */ +static backend_ref_t *get_root_master_bref(ROUTER_CLIENT_SES *rses) +{ + backend_ref_t *bref; + backend_ref_t *candidate_bref = NULL; + SERVER master = {}; + + for (int i = 0; i < rses->rses_nbackends; i++) + { + bref = &rses->rses_backend_ref[i]; + if (bref && BREF_IS_IN_USE(bref)) + { + if (bref == rses->rses_master_ref) + { + /** Store master state for better error reporting */ + master.status = bref->bref_backend->backend_server->status; + } + + if (bref->bref_backend->backend_server->status & SERVER_MASTER) + { + if (candidate_bref == NULL || + (bref->bref_backend->backend_server->depth < + candidate_bref->bref_backend->backend_server->depth)) + { + candidate_bref = bref; + } + } + } + } + + if (candidate_bref == NULL && rses->rses_config.rw_master_failure_mode == RW_FAIL_INSTANTLY) + { + MXS_ERROR("Could not find master among the backend " + "servers. Previous master's state : %s", + rses->rses_master_ref == NULL ? "No master found" : + (!BREF_IS_IN_USE(rses->rses_master_ref) ? "Master is not in use" : + STRSRVSTATUS(&master))); + } + + return candidate_bref; +} diff --git a/server/modules/routing/readwritesplit/rwsplit_select_backends.c b/server/modules/routing/readwritesplit/rwsplit_select_backends.c new file mode 100644 index 000000000..093557a37 --- /dev/null +++ b/server/modules/routing/readwritesplit/rwsplit_select_backends.c @@ -0,0 +1,473 @@ +/* + * Copyright (c) 2016 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl. + * + * Change Date: 2019-01-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ +#include +#include +#include +#include +#include + +#include +#include +#include +/** + * @file rwsplit_select_backends.c The functions that implement back end + * selection for the read write split router. All of these functions are + * internal to that router and not intended to be called from elsewhere. + * + * @verbatim + * Revision History + * + * Date Who Description + * 08/08/2016 Martin Brampton Initial implementation + * + * @endverbatim + */ + +static bool connect_server(backend_ref_t *bref, SESSION *session, bool execute_history); + +static void log_server_connections(select_criteria_t select_criteria, + backend_ref_t *backend_ref, int router_nservers); + +static BACKEND *get_root_master(backend_ref_t *servers, int router_nservers); + +static int bref_cmp_global_conn(const void *bref1, const void *bref2); + +static int bref_cmp_router_conn(const void *bref1, const void *bref2); + +static int bref_cmp_behind_master(const void *bref1, const void *bref2); + +static int bref_cmp_current_load(const void *bref1, const void *bref2); + +/** + * The order of functions _must_ match with the order the select criteria are + * listed in select_criteria_t definition in readwritesplit.h + */ +int (*criteria_cmpfun[LAST_CRITERIA])(const void *, const void *) = +{ + NULL, + bref_cmp_global_conn, + bref_cmp_router_conn, + bref_cmp_behind_master, + bref_cmp_current_load +}; + +/* + * The following function is the only one that is called from elsewhere in + * the read write split router. It is not intended for use from outside this + * router. Other functions in this module are internal and are called + * directly or indirectly by this function. + */ + +/** + * @brief Search suitable backend servers from those of router instance + * + * It is assumed that there is only one master among servers of a router instance. + * As a result, the first master found is chosen. There will possibly be more + * backend references than connected backends because only those in correct state + * are connected to. + * + * @param p_master_ref Pointer to location where master's backend reference is to be stored + * @param backend_ref Pointer to backend server reference object array + * @param router_nservers Number of backend server pointers pointed to by @p backend_ref + * @param max_nslaves Upper limit for the number of slaves + * @param max_slave_rlag Maximum allowed replication lag for any slave + * @param select_criteria Slave selection criteria + * @param session Client session + * @param router Router instance + * @return true, if at least one master and one slave was found. + */ +bool select_connect_backend_servers(backend_ref_t **p_master_ref, + backend_ref_t *backend_ref, + int router_nservers, int max_nslaves, + int max_slave_rlag, + select_criteria_t select_criteria, + SESSION *session, + ROUTER_INSTANCE *router) +{ + if (p_master_ref == NULL || backend_ref == NULL) + { + MXS_ERROR("Master reference (%p) or backend reference (%p) is NULL.", + p_master_ref, backend_ref); + ss_dassert(false); + return false; + } + + /* get the root Master */ + BACKEND *master_host = get_root_master(backend_ref, router_nservers); + + if (router->rwsplit_config.rw_master_failure_mode == RW_FAIL_INSTANTLY && + (master_host == NULL || SERVER_IS_DOWN(master_host->backend_server))) + { + MXS_ERROR("Couldn't find suitable Master from %d candidates.", router_nservers); + return false; + } + + /** + * Existing session : master is already chosen and connected. + * The function was called because new slave must be selected to replace + * failed one. + */ + bool master_connected = *p_master_ref != NULL; + + /** Check slave selection criteria and set compare function */ + int (*p)(const void *, const void *) = criteria_cmpfun[select_criteria]; + ss_dassert(p); + + /** Sort the pointer list to servers according to slave selection criteria. + * The servers that match the criteria the best are at the beginning of + * the list. */ + qsort(backend_ref, (size_t) router_nservers, sizeof(backend_ref_t), p); + + if (MXS_LOG_PRIORITY_IS_ENABLED(LOG_INFO)) + { + log_server_connections(select_criteria, backend_ref, router_nservers); + } + + int slaves_found = 0; + int slaves_connected = 0; + const int min_nslaves = 0; /*< not configurable at the time */ + bool succp = false; + + /** + * Choose at least 1+min_nslaves (master and slave) and at most 1+max_nslaves + * servers from the sorted list. First master found is selected. + */ + for (int i = 0; i < router_nservers && + (slaves_connected < max_nslaves || !master_connected); i++) + { + SERVER *serv = backend_ref[i].bref_backend->backend_server; + + if (!BREF_HAS_FAILED(&backend_ref[i]) && SERVER_IS_RUNNING(serv)) + { + /* check also for relay servers and don't take the master_host */ + if (slaves_found < max_nslaves && + (max_slave_rlag == MAX_RLAG_UNDEFINED || + (serv->rlag != MAX_RLAG_NOT_AVAILABLE && + serv->rlag <= max_slave_rlag)) && + (SERVER_IS_SLAVE(serv) || SERVER_IS_RELAY_SERVER(serv)) && + (master_host == NULL || (serv != master_host->backend_server))) + { + slaves_found += 1; + + if (BREF_IS_IN_USE((&backend_ref[i])) || + connect_server(&backend_ref[i], session, true)) + { + slaves_connected += 1; + } + } + /* take the master_host for master */ + else if (master_host && (serv == master_host->backend_server)) + { + /** p_master_ref must be assigned with this backend_ref pointer + * because its original value may have been lost when backend + * references were sorted with qsort. */ + *p_master_ref = &backend_ref[i]; + + if (!master_connected) + { + if (connect_server(&backend_ref[i], session, false)) + { + master_connected = true; + } + } + } + } + } /*< for */ + + /** + * Successful cases + */ + if (slaves_connected >= min_nslaves && slaves_connected <= max_nslaves) + { + succp = true; + + if (MXS_LOG_PRIORITY_IS_ENABLED(LOG_INFO)) + { + if (slaves_connected < max_nslaves) + { + MXS_INFO("Couldn't connect to maximum number of " + "slaves. Connected successfully to %d slaves " + "of %d of them.", slaves_connected, slaves_found); + } + + for (int i = 0; i < router_nservers; i++) + { + if (BREF_IS_IN_USE((&backend_ref[i]))) + { + MXS_INFO("Selected %s in \t%s:%d", + STRSRVSTATUS(backend_ref[i].bref_backend->backend_server), + backend_ref[i].bref_backend->backend_server->name, + backend_ref[i].bref_backend->backend_server->port); + } + } /* for */ + } + } + /** Failure cases */ + else + { + if (slaves_connected < min_nslaves) + { + MXS_ERROR("Couldn't establish required amount of " + "slave connections for router session."); + } + + /** Clean up connections */ + for (int i = 0; i < router_nservers; i++) + { + if (BREF_IS_IN_USE((&backend_ref[i]))) + { + ss_dassert(backend_ref[i].bref_backend->backend_conn_count > 0); + + /** disconnect opened connections */ + bref_clear_state(&backend_ref[i], BREF_IN_USE); + /** Decrease backend's connection counter. */ + atomic_add(&backend_ref[i].bref_backend->backend_conn_count, -1); + dcb_close(backend_ref[i].bref_dcb); + } + } + } + + return succp; +} + +/** Compare number of connections from this router in backend servers */ +static int bref_cmp_router_conn(const void *bref1, const void *bref2) +{ + BACKEND *b1 = ((backend_ref_t *)bref1)->bref_backend; + BACKEND *b2 = ((backend_ref_t *)bref2)->bref_backend; + + if (b1->weight == 0 && b2->weight == 0) + { + return b1->backend_server->stats.n_current - + b2->backend_server->stats.n_current; + } + else if (b1->weight == 0) + { + return 1; + } + else if (b2->weight == 0) + { + return -1; + } + + return ((1000 + 1000 * b1->backend_conn_count) / b1->weight) - + ((1000 + 1000 * b2->backend_conn_count) / b2->weight); +} + +/** Compare number of global connections in backend servers */ +static int bref_cmp_global_conn(const void *bref1, const void *bref2) +{ + BACKEND *b1 = ((backend_ref_t *)bref1)->bref_backend; + BACKEND *b2 = ((backend_ref_t *)bref2)->bref_backend; + + if (b1->weight == 0 && b2->weight == 0) + { + return b1->backend_server->stats.n_current - + b2->backend_server->stats.n_current; + } + else if (b1->weight == 0) + { + return 1; + } + else if (b2->weight == 0) + { + return -1; + } + + return ((1000 + 1000 * b1->backend_server->stats.n_current) / b1->weight) - + ((1000 + 1000 * b2->backend_server->stats.n_current) / b2->weight); +} + +/** Compare replication lag between backend servers */ +static int bref_cmp_behind_master(const void *bref1, const void *bref2) +{ + BACKEND *b1 = ((backend_ref_t *)bref1)->bref_backend; + BACKEND *b2 = ((backend_ref_t *)bref2)->bref_backend; + + return ((b1->backend_server->rlag < b2->backend_server->rlag) ? -1 + : ((b1->backend_server->rlag > b2->backend_server->rlag) ? 1 : 0)); +} + +/** Compare number of current operations in backend servers */ +static int bref_cmp_current_load(const void *bref1, const void *bref2) +{ + SERVER *s1 = ((backend_ref_t *)bref1)->bref_backend->backend_server; + SERVER *s2 = ((backend_ref_t *)bref2)->bref_backend->backend_server; + BACKEND *b1 = ((backend_ref_t *)bref1)->bref_backend; + BACKEND *b2 = ((backend_ref_t *)bref2)->bref_backend; + + if (b1->weight == 0 && b2->weight == 0) + { + return b1->backend_server->stats.n_current - + b2->backend_server->stats.n_current; + } + else if (b1->weight == 0) + { + return 1; + } + else if (b2->weight == 0) + { + return -1; + } + + return ((1000 * s1->stats.n_current_ops) - b1->weight) - + ((1000 * s2->stats.n_current_ops) - b2->weight); +} + +/** + * @brief Connect a server + * + * Connects to a server, adds callbacks to the created DCB and updates + * router statistics. If @p execute_history is true, the session command + * history will be executed on this server. + * + * @param b Router's backend structure for the server + * @param session Client's session object + * @param execute_history Execute session command history + * @return True if successful, false if an error occurred + */ +static bool connect_server(backend_ref_t *bref, SESSION *session, bool execute_history) +{ + SERVER *serv = bref->bref_backend->backend_server; + bool rval = false; + + bref->bref_dcb = dcb_connect(serv, session, serv->protocol); + + if (bref->bref_dcb != NULL) + { + if (!execute_history || execute_sescmd_history(bref)) + { + /** Add a callback for unresponsive server */ + dcb_add_callback(bref->bref_dcb, DCB_REASON_NOT_RESPONDING, + &router_handle_state_switch, (void *) bref); + bref->bref_state = 0; + bref_set_state(bref, BREF_IN_USE); + atomic_add(&bref->bref_backend->backend_conn_count, 1); + rval = true; + } + else + { + MXS_ERROR("Failed to execute session command in %s (%s:%d). See earlier " + "errors for more details.", + bref->bref_backend->backend_server->unique_name, + bref->bref_backend->backend_server->name, + bref->bref_backend->backend_server->port); + dcb_close(bref->bref_dcb); + bref->bref_dcb = NULL; + } + } + else + { + MXS_ERROR("Unable to establish connection with server %s:%d", + serv->name, serv->port); + } + + return rval; +} + +/** + * @brief Log server connections + * + * @param select_criteria Slave selection criteria + * @param backend_ref Backend reference array + * @param router_nservers Number of backends in @p backend_ref + */ +static void log_server_connections(select_criteria_t select_criteria, + backend_ref_t *backend_ref, int router_nservers) +{ + if (select_criteria == LEAST_GLOBAL_CONNECTIONS || + select_criteria == LEAST_ROUTER_CONNECTIONS || + select_criteria == LEAST_BEHIND_MASTER || + select_criteria == LEAST_CURRENT_OPERATIONS) + { + MXS_INFO("Servers and %s connection counts:", + select_criteria == LEAST_GLOBAL_CONNECTIONS ? "all MaxScale" + : "router"); + + for (int i = 0; i < router_nservers; i++) + { + BACKEND *b = backend_ref[i].bref_backend; + + switch (select_criteria) + { + case LEAST_GLOBAL_CONNECTIONS: + MXS_INFO("MaxScale connections : %d in \t%s:%d %s", + b->backend_server->stats.n_current, b->backend_server->name, + b->backend_server->port, STRSRVSTATUS(b->backend_server)); + break; + + case LEAST_ROUTER_CONNECTIONS: + MXS_INFO("RWSplit connections : %d in \t%s:%d %s", + b->backend_conn_count, b->backend_server->name, + b->backend_server->port, STRSRVSTATUS(b->backend_server)); + break; + + case LEAST_CURRENT_OPERATIONS: + MXS_INFO("current operations : %d in \t%s:%d %s", + b->backend_server->stats.n_current_ops, + b->backend_server->name, b->backend_server->port, + STRSRVSTATUS(b->backend_server)); + break; + + case LEAST_BEHIND_MASTER: + MXS_INFO("replication lag : %d in \t%s:%d %s", + b->backend_server->rlag, b->backend_server->name, + b->backend_server->port, STRSRVSTATUS(b->backend_server)); + default: + break; + } + } + } +} + +/******************************** + * This routine returns the root master server from MySQL replication tree + * Get the root Master rule: + * + * find server with the lowest replication depth level + * and the SERVER_MASTER bitval + * Servers are checked even if they are in 'maintenance' + * + * @param servers The list of servers + * @param router_nservers The number of servers + * @return The Master found + * + */ +static BACKEND *get_root_master(backend_ref_t *servers, int router_nservers) +{ + int i = 0; + BACKEND *master_host = NULL; + + for (i = 0; i < router_nservers; i++) + { + BACKEND *b; + + if (servers[i].bref_backend == NULL) + { + continue; + } + + b = servers[i].bref_backend; + + if ((b->backend_server->status & (SERVER_MASTER | SERVER_MAINT)) == + SERVER_MASTER) + { + if (master_host == NULL || + (b->backend_server->depth < master_host->backend_server->depth)) + { + master_host = b; + } + } + } + return master_host; +} diff --git a/server/modules/routing/readwritesplit/rwsplit_session_cmd.c b/server/modules/routing/readwritesplit/rwsplit_session_cmd.c new file mode 100644 index 000000000..d66e7f66f --- /dev/null +++ b/server/modules/routing/readwritesplit/rwsplit_session_cmd.c @@ -0,0 +1,480 @@ +/* + * Copyright (c) 2016 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl. + * + * Change Date: 2019-01-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ +#include +#include +#include +#include +#include + +#include +#include +#include + +/** + * @file rwsplit_session_cmd.c The functions that provide session command + * handling for the read write split router. + * + * @verbatim + * Revision History + * + * Date Who Description + * 08/08/2016 Martin Brampton Initial implementation + * + * @endverbatim + */ + +static bool sescmd_cursor_history_empty(sescmd_cursor_t *scur); +static void sescmd_cursor_reset(sescmd_cursor_t *scur); +static bool sescmd_cursor_next(sescmd_cursor_t *scur); +static rses_property_t *mysql_sescmd_get_property(mysql_sescmd_t *scmd); + +/* + * The following functions, all to do with the handling of session commands, + * are called from other modules of the read write split router: + */ + +/** + * Router session must be locked. + * Return session command pointer if succeed, NULL if failed. + */ +mysql_sescmd_t *rses_property_get_sescmd(rses_property_t *prop) +{ + mysql_sescmd_t *sescmd; + + if (prop == NULL) + { + MXS_ERROR("[%s] Error: NULL parameter.", __FUNCTION__); + return NULL; + } + + CHK_RSES_PROP(prop); + ss_dassert(prop->rses_prop_rsession == NULL || + SPINLOCK_IS_LOCKED(&prop->rses_prop_rsession->rses_lock)); + + sescmd = &prop->rses_prop_data.sescmd; + + if (sescmd != NULL) + { + CHK_MYSQL_SESCMD(sescmd); + } + return sescmd; +} + +/** + * Create session command property. + */ +mysql_sescmd_t *mysql_sescmd_init(rses_property_t *rses_prop, + GWBUF *sescmd_buf, + unsigned char packet_type, + ROUTER_CLIENT_SES *rses) +{ + mysql_sescmd_t *sescmd; + + CHK_RSES_PROP(rses_prop); + /** Can't call rses_property_get_sescmd with uninitialized sescmd */ + sescmd = &rses_prop->rses_prop_data.sescmd; + sescmd->my_sescmd_prop = rses_prop; /*< reference to owning property */ +#if defined(SS_DEBUG) + sescmd->my_sescmd_chk_top = CHK_NUM_MY_SESCMD; + sescmd->my_sescmd_chk_tail = CHK_NUM_MY_SESCMD; +#endif + /** Set session command buffer */ + sescmd->my_sescmd_buf = sescmd_buf; + sescmd->my_sescmd_packet_type = packet_type; + sescmd->position = atomic_add(&rses->pos_generator, 1); + + return sescmd; +} + +void mysql_sescmd_done(mysql_sescmd_t *sescmd) +{ + if (sescmd == NULL) + { + MXS_ERROR("[%s] Error: NULL parameter.", __FUNCTION__); + return; + } + CHK_RSES_PROP(sescmd->my_sescmd_prop); + gwbuf_free(sescmd->my_sescmd_buf); + memset(sescmd, 0, sizeof(mysql_sescmd_t)); +} + +/** + * All cases where backend message starts at least with one response to session + * command are handled here. + * Read session commands from property list. If command is already replied, + * discard packet. Else send reply to client. In both cases move cursor forward + * until all session command replies are handled. + * + * Cases that are expected to happen and which are handled: + * s = response not yet replied to client, S = already replied response, + * q = query + * 1. q+ for example : select * from mysql.user + * 2. s+ for example : set autocommit=1 + * 3. S+ + * 4. sq+ + * 5. Sq+ + * 6. Ss+ + * 7. Ss+q+ + * 8. S+q+ + * 9. s+q+ + */ +GWBUF *sescmd_cursor_process_replies(GWBUF *replybuf, + backend_ref_t *bref, + bool *reconnect) +{ + mysql_sescmd_t *scmd; + sescmd_cursor_t *scur; + ROUTER_CLIENT_SES *ses; + + scur = &bref->bref_sescmd_cur; + ss_dassert(SPINLOCK_IS_LOCKED(&(scur->scmd_cur_rses->rses_lock))); + scmd = sescmd_cursor_get_command(scur); + ses = (*scur->scmd_cur_ptr_property)->rses_prop_rsession; + CHK_GWBUF(replybuf); + + /** + * Walk through packets in the message and the list of session + * commands. + */ + while (scmd != NULL && replybuf != NULL) + { + bref->reply_cmd = *((unsigned char *)replybuf->start + 4); + scur->position = scmd->position; + /** Faster backend has already responded to client : discard */ + if (scmd->my_sescmd_is_replied) + { + bool last_packet = false; + + CHK_GWBUF(replybuf); + + while (!last_packet) + { + int buflen; + + buflen = GWBUF_LENGTH(replybuf); + last_packet = GWBUF_IS_TYPE_RESPONSE_END(replybuf); + /** discard packet */ + replybuf = gwbuf_consume(replybuf, buflen); + } + /** Set response status received */ + bref_clear_state(bref, BREF_WAITING_RESULT); + + if (bref->reply_cmd != scmd->reply_cmd) + { + MXS_ERROR("Slave server '%s': response differs from master's response. " + "Closing connection due to inconsistent session state.", + bref->bref_backend->backend_server->unique_name); + sescmd_cursor_set_active(scur, false); + bref_clear_state(bref, BREF_QUERY_ACTIVE); + bref_clear_state(bref, BREF_IN_USE); + bref_set_state(bref, BREF_CLOSED); + bref_set_state(bref, BREF_SESCMD_FAILED); + if (bref->bref_dcb) + { + dcb_close(bref->bref_dcb); + } + *reconnect = true; + gwbuf_free(replybuf); + replybuf = NULL; + } + } + /** This is a response from the master and it is the "right" one. + * A slave server's response will be compared to this and if + * their response differs from the master server's response, they + * are dropped from the valid list of backend servers. + * Response is in the buffer and it will be sent to client. + * + * If we have no master server, the first slave's response is considered + * the "right" one. */ + else if (ses->rses_master_ref == NULL || + !BREF_IS_IN_USE(ses->rses_master_ref) || + ses->rses_master_ref->bref_dcb == bref->bref_dcb) + { + /** Mark the rest session commands as replied */ + scmd->my_sescmd_is_replied = true; + scmd->reply_cmd = *((unsigned char *)replybuf->start + 4); + + MXS_INFO("Server '%s' responded to a session command, sending the response " + "to the client.", bref->bref_backend->backend_server->unique_name); + + for (int i = 0; i < ses->rses_nbackends; i++) + { + if (!BREF_IS_WAITING_RESULT(&ses->rses_backend_ref[i])) + { + /** This backend has already received a response */ + if (ses->rses_backend_ref[i].reply_cmd != scmd->reply_cmd && + !BREF_IS_CLOSED(&ses->rses_backend_ref[i])) + { + bref_clear_state(&ses->rses_backend_ref[i], BREF_QUERY_ACTIVE); + bref_clear_state(&ses->rses_backend_ref[i], BREF_IN_USE); + bref_set_state(&ses->rses_backend_ref[i], BREF_CLOSED); + bref_set_state(bref, BREF_SESCMD_FAILED); + if (ses->rses_backend_ref[i].bref_dcb) + { + dcb_close(ses->rses_backend_ref[i].bref_dcb); + } + *reconnect = true; + MXS_INFO("Disabling slave %s:%d, result differs from " + "master's result. Master: %d Slave: %d", + ses->rses_backend_ref[i].bref_backend->backend_server->name, + ses->rses_backend_ref[i].bref_backend->backend_server->port, + bref->reply_cmd, ses->rses_backend_ref[i].reply_cmd); + } + } + } + + } + else + { + MXS_INFO("Slave '%s' responded before master to a session command. Result: %d", + bref->bref_backend->backend_server->unique_name, + (int)bref->reply_cmd); + if (bref->reply_cmd == 0xff) + { + SERVER *serv = bref->bref_backend->backend_server; + MXS_ERROR("Slave '%s' (%s:%u) failed to execute session command.", + serv->unique_name, serv->name, serv->port); + } + + gwbuf_free(replybuf); + replybuf = NULL; + } + + if (sescmd_cursor_next(scur)) + { + scmd = sescmd_cursor_get_command(scur); + } + else + { + scmd = NULL; + /** All session commands are replied */ + scur->scmd_cur_active = false; + } + } + ss_dassert(replybuf == NULL || *scur->scmd_cur_ptr_property == NULL); + + return replybuf; +} + +/** + * Get the address of current session command. + * + * Router session must be locked */ +mysql_sescmd_t *sescmd_cursor_get_command(sescmd_cursor_t *scur) +{ + mysql_sescmd_t *scmd; + + ss_dassert(SPINLOCK_IS_LOCKED(&(scur->scmd_cur_rses->rses_lock))); + scur->scmd_cur_cmd = rses_property_get_sescmd(*scur->scmd_cur_ptr_property); + + CHK_MYSQL_SESCMD(scur->scmd_cur_cmd); + + scmd = scur->scmd_cur_cmd; + + return scmd; +} + +/** router must be locked */ +bool sescmd_cursor_is_active(sescmd_cursor_t *sescmd_cursor) +{ + bool succp; + + if (sescmd_cursor == NULL) + { + MXS_ERROR("[%s] Error: NULL parameter.", __FUNCTION__); + return false; + } + ss_dassert(SPINLOCK_IS_LOCKED(&sescmd_cursor->scmd_cur_rses->rses_lock)); + + succp = sescmd_cursor->scmd_cur_active; + return succp; +} + +/** router must be locked */ +void sescmd_cursor_set_active(sescmd_cursor_t *sescmd_cursor, + bool value) +{ + ss_dassert(SPINLOCK_IS_LOCKED(&sescmd_cursor->scmd_cur_rses->rses_lock)); + /** avoid calling unnecessarily */ + ss_dassert(sescmd_cursor->scmd_cur_active != value); + sescmd_cursor->scmd_cur_active = value; +} + +/** + * Clone session command's command buffer. + * Router session must be locked + */ +GWBUF *sescmd_cursor_clone_querybuf(sescmd_cursor_t *scur) +{ + GWBUF *buf; + if (scur == NULL) + { + MXS_ERROR("[%s] Error: NULL parameter.", __FUNCTION__); + return NULL; + } + ss_dassert(scur->scmd_cur_cmd != NULL); + + buf = gwbuf_clone_all(scur->scmd_cur_cmd->my_sescmd_buf); + + CHK_GWBUF(buf); + return buf; +} + +bool execute_sescmd_history(backend_ref_t *bref) +{ + bool succp = true; + sescmd_cursor_t *scur; + if (bref == NULL) + { + MXS_ERROR("[%s] Error: NULL parameter.", __FUNCTION__); + return false; + } + CHK_BACKEND_REF(bref); + + scur = &bref->bref_sescmd_cur; + CHK_SESCMD_CUR(scur); + + if (!sescmd_cursor_history_empty(scur)) + { + sescmd_cursor_reset(scur); + succp = execute_sescmd_in_backend(bref); + } + + return succp; +} + +static bool sescmd_cursor_history_empty(sescmd_cursor_t *scur) +{ + bool succp; + + if (scur == NULL) + { + MXS_ERROR("[%s] Error: NULL parameter.", __FUNCTION__); + return true; + } + CHK_SESCMD_CUR(scur); + + if (scur->scmd_cur_rses->rses_properties[RSES_PROP_TYPE_SESCMD] == NULL) + { + succp = true; + } + else + { + succp = false; + } + + return succp; +} + +/* + * End of functions called from other modules of the read write split router; + * start of functions that are internal to this module. + */ + +static void sescmd_cursor_reset(sescmd_cursor_t *scur) +{ + ROUTER_CLIENT_SES *rses; + if (scur == NULL) + { + MXS_ERROR("[%s] Error: NULL parameter.", __FUNCTION__); + return; + } + CHK_SESCMD_CUR(scur); + CHK_CLIENT_RSES(scur->scmd_cur_rses); + rses = scur->scmd_cur_rses; + + scur->scmd_cur_ptr_property = &rses->rses_properties[RSES_PROP_TYPE_SESCMD]; + + CHK_RSES_PROP((*scur->scmd_cur_ptr_property)); + scur->scmd_cur_active = false; + scur->scmd_cur_cmd = &(*scur->scmd_cur_ptr_property)->rses_prop_data.sescmd; +} + +/** + * Moves cursor to next property and copied address of its sescmd to cursor. + * Current propery must be non-null. + * If current property is the last on the list, *scur->scmd_ptr_property == NULL + * + * Router session must be locked + */ +static bool sescmd_cursor_next(sescmd_cursor_t *scur) +{ + bool succp = false; + rses_property_t *prop_curr; + rses_property_t *prop_next; + + if (scur == NULL) + { + MXS_ERROR("[%s] Error: NULL parameter.", __FUNCTION__); + return false; + } + + ss_dassert(scur != NULL); + ss_dassert(*(scur->scmd_cur_ptr_property) != NULL); + ss_dassert(SPINLOCK_IS_LOCKED( + &(*(scur->scmd_cur_ptr_property))->rses_prop_rsession->rses_lock)); + + /** Illegal situation */ + if (scur == NULL || *scur->scmd_cur_ptr_property == NULL || + scur->scmd_cur_cmd == NULL) + { + /** Log error */ + goto return_succp; + } + prop_curr = *(scur->scmd_cur_ptr_property); + + CHK_MYSQL_SESCMD(scur->scmd_cur_cmd); + ss_dassert(prop_curr == mysql_sescmd_get_property(scur->scmd_cur_cmd)); + CHK_RSES_PROP(prop_curr); + + /** Copy address of pointer to next property */ + scur->scmd_cur_ptr_property = &(prop_curr->rses_prop_next); + prop_next = *scur->scmd_cur_ptr_property; + ss_dassert(prop_next == *(scur->scmd_cur_ptr_property)); + + /** If there is a next property move forward */ + if (prop_next != NULL) + { + CHK_RSES_PROP(prop_next); + CHK_RSES_PROP((*(scur->scmd_cur_ptr_property))); + + /** Get pointer to next property's sescmd */ + scur->scmd_cur_cmd = rses_property_get_sescmd(prop_next); + + ss_dassert(prop_next == scur->scmd_cur_cmd->my_sescmd_prop); + CHK_MYSQL_SESCMD(scur->scmd_cur_cmd); + CHK_RSES_PROP(scur->scmd_cur_cmd->my_sescmd_prop); + } + else + { + /** No more properties, can't proceed. */ + goto return_succp; + } + + if (scur->scmd_cur_cmd != NULL) + { + succp = true; + } + else + { + ss_dassert(false); /*< Log error, sescmd shouldn't be NULL */ + } +return_succp: + return succp; +} + +static rses_property_t *mysql_sescmd_get_property(mysql_sescmd_t *scmd) +{ + CHK_MYSQL_SESCMD(scmd); + return scmd->my_sescmd_prop; +} diff --git a/server/modules/routing/readwritesplit/rwsplit_tmp_table_multi.c b/server/modules/routing/readwritesplit/rwsplit_tmp_table_multi.c new file mode 100644 index 000000000..996583416 --- /dev/null +++ b/server/modules/routing/readwritesplit/rwsplit_tmp_table_multi.c @@ -0,0 +1,408 @@ +/* + * Copyright (c) 2016 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl. + * + * Change Date: 2019-01-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ +#include +#include +#include +#include +#include +#include + +/* Note that modutil contains much MySQL specific code */ +#include + +#include +#include +#include +/** + * @file rwsplit_tmp_table.c The functions that carry out checks on + * statements to see if they involve various operations involving temporary + * tables or multi-statement queries. + * + * @verbatim + * Revision History + * + * Date Who Description + * 08/08/2016 Martin Brampton Initial implementation + * + * @endverbatim + */ + +/* + * The following are to do with checking whether the statement refers to + * temporary tables, or is a multi-statement request. Maybe they belong + * somewhere else, outside this router. Perhaps in the query classifier? + */ + +/** + * Check if the query is a DROP TABLE... query and + * if it targets a temporary table, remove it from the hashtable. + * @param router_cli_ses Router client session + * @param querybuf GWBUF containing the query + * @param type The type of the query resolved so far + */ +void check_drop_tmp_table(ROUTER_CLIENT_SES *router_cli_ses, GWBUF *querybuf, + mysql_server_cmd_t packet_type) +{ + if (packet_type != MYSQL_COM_QUERY && packet_type != MYSQL_COM_DROP_DB) + { + return; + } + + int tsize = 0, klen = 0, i; + char **tbl = NULL; + char *hkey, *dbname; + MYSQL_session *my_data; + rses_property_t *rses_prop_tmp; + MYSQL_session *data = (MYSQL_session *)router_cli_ses->client_dcb->data; + + rses_prop_tmp = router_cli_ses->rses_properties[RSES_PROP_TYPE_TMPTABLES]; + dbname = (char *)data->db; + + if (qc_is_drop_table_query(querybuf)) + { + tbl = qc_get_table_names(querybuf, &tsize, false); + if (tbl != NULL) + { + for (i = 0; i < tsize; i++) + { + /* Not clear why the next six lines are outside the if block */ + klen = strlen(dbname) + strlen(tbl[i]) + 2; + hkey = MXS_CALLOC(klen, sizeof(char)); + MXS_ABORT_IF_NULL(hkey); + strcpy(hkey, dbname); + strcat(hkey, "."); + strcat(hkey, tbl[i]); + + if (rses_prop_tmp && rses_prop_tmp->rses_prop_data.temp_tables) + { + if (hashtable_delete(rses_prop_tmp->rses_prop_data.temp_tables, + (void *)hkey)) + { + MXS_INFO("Temporary table dropped: %s", hkey); + } + } + MXS_FREE(tbl[i]); + MXS_FREE(hkey); + } + + MXS_FREE(tbl); + } + } +} + +/** + * Check if the query targets a temporary table. + * @param router_cli_ses Router client session + * @param querybuf GWBUF containing the query + * @param type The type of the query resolved so far + * @return The type of the query + */ +qc_query_type_t is_read_tmp_table(ROUTER_CLIENT_SES *router_cli_ses, + GWBUF *querybuf, + qc_query_type_t type) +{ + + bool target_tmp_table = false; + int tsize = 0, klen = 0, i; + char **tbl = NULL; + char *dbname; + char hkey[MYSQL_DATABASE_MAXLEN + MYSQL_TABLE_MAXLEN + 2]; + MYSQL_session *data; + qc_query_type_t qtype = type; + rses_property_t *rses_prop_tmp; + + if (router_cli_ses == NULL || querybuf == NULL) + { + MXS_ERROR("[%s] Error: NULL parameters passed: %p %p", __FUNCTION__, + router_cli_ses, querybuf); + return type; + } + + if (router_cli_ses->client_dcb == NULL) + { + MXS_ERROR("[%s] Error: Client DCB is NULL.", __FUNCTION__); + return type; + } + + rses_prop_tmp = router_cli_ses->rses_properties[RSES_PROP_TYPE_TMPTABLES]; + data = (MYSQL_session *)router_cli_ses->client_dcb->data; + + if (data == NULL) + { + MXS_ERROR("[%s] Error: User data in client DBC is NULL.", __FUNCTION__); + return qtype; + } + + dbname = (char *)data->db; + + if (QUERY_IS_TYPE(qtype, QUERY_TYPE_READ) || + QUERY_IS_TYPE(qtype, QUERY_TYPE_LOCAL_READ) || + QUERY_IS_TYPE(qtype, QUERY_TYPE_USERVAR_READ) || + QUERY_IS_TYPE(qtype, QUERY_TYPE_SYSVAR_READ) || + QUERY_IS_TYPE(qtype, QUERY_TYPE_GSYSVAR_READ)) + { + tbl = qc_get_table_names(querybuf, &tsize, false); + + if (tbl != NULL && tsize > 0) + { + /** Query targets at least one table */ + for (i = 0; i < tsize && !target_tmp_table && tbl[i]; i++) + { + sprintf(hkey, "%s.%s", dbname, tbl[i]); + if (rses_prop_tmp && rses_prop_tmp->rses_prop_data.temp_tables) + { + if (hashtable_fetch(rses_prop_tmp->rses_prop_data.temp_tables, hkey)) + { + /**Query target is a temporary table*/ + qtype = QUERY_TYPE_READ_TMP_TABLE; + MXS_INFO("Query targets a temporary table: %s", hkey); + break; + } + } + } + } + } + + if (tbl != NULL) + { + for (i = 0; i < tsize; i++) + { + MXS_FREE(tbl[i]); + } + MXS_FREE(tbl); + } + + return qtype; +} + +/** + * If query is of type QUERY_TYPE_CREATE_TMP_TABLE then find out + * the database and table name, create a hashvalue and + * add it to the router client session's property. If property + * doesn't exist then create it first. + * @param router_cli_ses Router client session + * @param querybuf GWBUF containing the query + * @param type The type of the query resolved so far + */ +void check_create_tmp_table(ROUTER_CLIENT_SES *router_cli_ses, + GWBUF *querybuf, qc_query_type_t type) +{ + if (!QUERY_IS_TYPE(type, QUERY_TYPE_CREATE_TMP_TABLE)) + { + return; + } + + int klen = 0; + char *hkey, *dbname; + MYSQL_session *data; + rses_property_t *rses_prop_tmp; + HASHTABLE *h; + + if (router_cli_ses == NULL || querybuf == NULL) + { + MXS_ERROR("[%s] Error: NULL parameters passed: %p %p", __FUNCTION__, + router_cli_ses, querybuf); + return; + } + + if (router_cli_ses->client_dcb == NULL) + { + MXS_ERROR("[%s] Error: Client DCB is NULL.", __FUNCTION__); + return; + } + + router_cli_ses->have_tmp_tables = true; + rses_prop_tmp = router_cli_ses->rses_properties[RSES_PROP_TYPE_TMPTABLES]; + data = (MYSQL_session *)router_cli_ses->client_dcb->data; + + if (data == NULL) + { + MXS_ERROR("[%s] Error: User data in master server DBC is NULL.", + __FUNCTION__); + return; + } + + dbname = (char *)data->db; + + bool is_temp = true; + char *tblname = NULL; + + tblname = qc_get_created_table_name(querybuf); + + if (tblname && strlen(tblname) > 0) + { + klen = strlen(dbname) + strlen(tblname) + 2; + hkey = MXS_CALLOC(klen, sizeof(char)); + MXS_ABORT_IF_NULL(hkey); + strcpy(hkey, dbname); + strcat(hkey, "."); + strcat(hkey, tblname); + } + else + { + hkey = NULL; + } + + if (rses_prop_tmp == NULL) + { + if ((rses_prop_tmp = (rses_property_t *)MXS_CALLOC(1, sizeof(rses_property_t)))) + { +#if defined(SS_DEBUG) + rses_prop_tmp->rses_prop_chk_top = CHK_NUM_ROUTER_PROPERTY; + rses_prop_tmp->rses_prop_chk_tail = CHK_NUM_ROUTER_PROPERTY; +#endif + rses_prop_tmp->rses_prop_rsession = router_cli_ses; + rses_prop_tmp->rses_prop_refcount = 1; + rses_prop_tmp->rses_prop_next = NULL; + rses_prop_tmp->rses_prop_type = RSES_PROP_TYPE_TMPTABLES; + router_cli_ses->rses_properties[RSES_PROP_TYPE_TMPTABLES] = rses_prop_tmp; + } + } + if (rses_prop_tmp) + { + if (rses_prop_tmp->rses_prop_data.temp_tables == NULL) + { + h = hashtable_alloc(7, rwsplit_hashkeyfun, rwsplit_hashcmpfun); + hashtable_memory_fns(h, rwsplit_hstrdup, NULL, rwsplit_hfree, NULL); + if (h != NULL) + { + rses_prop_tmp->rses_prop_data.temp_tables = h; + } + else + { + MXS_ERROR("Failed to allocate a new hashtable."); + } + } + + if (hkey && rses_prop_tmp->rses_prop_data.temp_tables && + hashtable_add(rses_prop_tmp->rses_prop_data.temp_tables, (void *)hkey, + (void *)is_temp) == 0) /*< Conflict in hash table */ + { + MXS_INFO("Temporary table conflict in hashtable: %s", hkey); + } +#if defined(SS_DEBUG) + { + bool retkey = hashtable_fetch(rses_prop_tmp->rses_prop_data.temp_tables, hkey); + if (retkey) + { + MXS_INFO("Temporary table added: %s", hkey); + } + } +#endif + } + + MXS_FREE(hkey); + MXS_FREE(tblname); +} + +/** + * @brief Detect multi-statement queries + * + * It is possible that the session state is modified inside a multi-statement + * query which would leave any slave sessions in an inconsistent state. Due to + * this, for the duration of this session, all queries will be sent to the + * master + * if the current query contains a multi-statement query. + * @param rses Router client session + * @param buf Buffer containing the full query + * @return True if the query contains multiple statements + */ +bool check_for_multi_stmt(GWBUF *buf, void *protocol, mysql_server_cmd_t packet_type) +{ + MySQLProtocol *proto = (MySQLProtocol *)protocol; + bool rval = false; + + if (proto->client_capabilities & GW_MYSQL_CAPABILITIES_MULTI_STATEMENTS && + packet_type == MYSQL_COM_QUERY) + { + char *ptr, *data = GWBUF_DATA(buf) + 5; + /** Payload size without command byte */ + int buflen = gw_mysql_get_byte3((uint8_t *)GWBUF_DATA(buf)) - 1; + + if ((ptr = strnchr_esc_mysql(data, ';', buflen))) + { + /** Skip stored procedures etc. */ + while (ptr && is_mysql_sp_end(ptr, buflen - (ptr - data))) + { + ptr = strnchr_esc_mysql(ptr + 1, ';', buflen - (ptr - data) - 1); + } + + if (ptr) + { + if (ptr < data + buflen && + !is_mysql_statement_end(ptr, buflen - (ptr - data))) + { + rval = true; + } + } + } + } + + return rval; +} + +qc_query_type_t +determine_query_type(GWBUF *querybuf, int packet_type, bool non_empty_packet) +{ + qc_query_type_t qtype = QUERY_TYPE_UNKNOWN; + + if (non_empty_packet) + { + mysql_server_cmd_t my_packet_type = (mysql_server_cmd_t)packet_type; + switch (my_packet_type) + { + case MYSQL_COM_QUIT: /*< 1 QUIT will close all sessions */ + case MYSQL_COM_INIT_DB: /*< 2 DDL must go to the master */ + case MYSQL_COM_REFRESH: /*< 7 - I guess this is session but not sure */ + case MYSQL_COM_DEBUG: /*< 0d all servers dump debug info to stdout */ + case MYSQL_COM_PING: /*< 0e all servers are pinged */ + case MYSQL_COM_CHANGE_USER: /*< 11 all servers change it accordingly */ + qtype = QUERY_TYPE_SESSION_WRITE; + break; + + case MYSQL_COM_CREATE_DB: /**< 5 DDL must go to the master */ + case MYSQL_COM_DROP_DB: /**< 6 DDL must go to the master */ + case MYSQL_COM_STMT_CLOSE: /*< free prepared statement */ + case MYSQL_COM_STMT_SEND_LONG_DATA: /*< send data to column */ + case MYSQL_COM_STMT_RESET: /*< resets the data of a prepared statement */ + qtype = QUERY_TYPE_WRITE; + break; + + case MYSQL_COM_QUERY: + qtype = qc_get_type(querybuf); + break; + + case MYSQL_COM_STMT_PREPARE: + qtype = qc_get_type(querybuf); + qtype |= QUERY_TYPE_PREPARE_STMT; + break; + + case MYSQL_COM_STMT_EXECUTE: + /** Parsing is not needed for this type of packet */ + qtype = QUERY_TYPE_EXEC_STMT; + break; + + case MYSQL_COM_SHUTDOWN: /**< 8 where should shutdown be routed ? */ + case MYSQL_COM_STATISTICS: /**< 9 ? */ + case MYSQL_COM_PROCESS_INFO: /**< 0a ? */ + case MYSQL_COM_CONNECT: /**< 0b ? */ + case MYSQL_COM_PROCESS_KILL: /**< 0c ? */ + case MYSQL_COM_TIME: /**< 0f should this be run in gateway ? */ + case MYSQL_COM_DELAYED_INSERT: /**< 10 ? */ + case MYSQL_COM_DAEMON: /**< 1d ? */ + default: + break; + } /**< switch by packet type */ + } + return qtype; +} From 0f6d4e4cdb6e867f43f7579935dd1268f600afaf Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Tue, 20 Sep 2016 13:58:28 +0300 Subject: [PATCH 24/36] Fix typo in tarball documentation --- .../Getting-Started/Install-MariaDB-MaxScale-Using-a-Tarball.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/Getting-Started/Install-MariaDB-MaxScale-Using-a-Tarball.md b/Documentation/Getting-Started/Install-MariaDB-MaxScale-Using-a-Tarball.md index 5aca16d0b..6dd5cd3c8 100644 --- a/Documentation/Getting-Started/Install-MariaDB-MaxScale-Using-a-Tarball.md +++ b/Documentation/Getting-Started/Install-MariaDB-MaxScale-Using-a-Tarball.md @@ -14,7 +14,7 @@ The required steps are as follows: $ sudo useradd -g maxscale maxscale $ cd /usr/local $ sudo tar -xzvf maxscale-x.y.z.OS.tar.gz - $ sudo ln -s maxscale-x.y.z.OS.maxscale + $ sudo ln -s maxscale-x.y.z maxscale $ cd maxscale $ chown -R maxscale var From dfec3c8552b67b31e413bd45298c0897661db8b3 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 20 Sep 2016 13:58:30 +0300 Subject: [PATCH 25/36] Install maxbinlogcheck in the right place Maxbinlogcheck was installed in the wrong place. --- server/modules/routing/binlog/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/modules/routing/binlog/CMakeLists.txt b/server/modules/routing/binlog/CMakeLists.txt index 8b9298b22..780a7a564 100644 --- a/server/modules/routing/binlog/CMakeLists.txt +++ b/server/modules/routing/binlog/CMakeLists.txt @@ -7,7 +7,7 @@ install(TARGETS binlogrouter DESTINATION ${MAXSCALE_LIBDIR}) add_executable(maxbinlogcheck maxbinlogcheck.c blr_file.c blr_cache.c blr_master.c blr_slave.c blr.c) target_link_libraries(maxbinlogcheck maxscale-common ${PCRE_LINK_FLAGS} uuid) -install(TARGETS maxbinlogcheck DESTINATION bin) +install(TARGETS maxbinlogcheck DESTINATION ${MAXSCALE_BINDIR}) if(BUILD_TESTS) add_subdirectory(test) From 1331bf9ae817f21430a417b87b0e999f5b955579 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 20 Sep 2016 14:11:50 +0300 Subject: [PATCH 26/36] Add CPACK_PACKAGE_FILE_NAME to CMake cache Adding CPACK_PACKAGE_FILE_NAME to the CMake cache allows the user to control it. With this, custom tarball names can be easily generated. --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6da4daeea..ea6506d7e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -265,9 +265,9 @@ if(PACKAGE) set(CPACK_PACKAGE_VERSION_PATCH "${MAXSCALE_VERSION_PATCH}") set(CPACK_PACKAGE_CONTACT "MariaDB Corporation Ab") if(DISTRIB_SUFFIX) - set(CPACK_PACKAGE_FILE_NAME "maxscale-${MAXSCALE_VERSION}-${MAXSCALE_BUILD_NUMBER}.${DISTRIB_SUFFIX}.${CPACK_PACKAGE_ARCHITECTURE}") + set(CPACK_PACKAGE_FILE_NAME "maxscale-${MAXSCALE_VERSION}-${MAXSCALE_BUILD_NUMBER}.${DISTRIB_SUFFIX}.${CPACK_PACKAGE_ARCHITECTURE}" CACHE STRING "MaxScale package filename") else() - set(CPACK_PACKAGE_FILE_NAME "maxscale-${MAXSCALE_VERSION}-${MAXSCALE_BUILD_NUMBER}.${CPACK_PACKAGE_ARCHITECTURE}") + set(CPACK_PACKAGE_FILE_NAME "maxscale-${MAXSCALE_VERSION}-${MAXSCALE_BUILD_NUMBER}.${CPACK_PACKAGE_ARCHITECTURE}" CACHE STRING "MaxScale package filename") endif() set(CPACK_PACKAGE_NAME "maxscale") set(CPACK_PACKAGE_VENDOR "MariaDB Corporation Ab") From 715f978051d8452edbfa3edc57add45d76364f35 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 20 Sep 2016 14:36:16 +0300 Subject: [PATCH 27/36] Add TARBALL variable to CMake The TARBALL variable controls if a special tar.gz package is built when packages are generated. This package has a different directory structure compared to the RPM/DEB packages. If RPM/DEB packages are built, tar.gz packages are not built. This makes RPM/DEB generation faster and allows tarballs to be built separately with a proper directory structures. --- CMakeLists.txt | 102 ++++++++++-------- .../Building-MaxScale-from-Source-Code.md | 6 +- cmake/package_tgz.cmake | 19 ++++ 3 files changed, 80 insertions(+), 47 deletions(-) create mode 100644 cmake/package_tgz.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index ea6506d7e..148b915ce 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -98,6 +98,61 @@ endif() set(CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_RPATH}:${CMAKE_INSTALL_PREFIX}/${MAXSCALE_LIBDIR}) +# Only do packaging if configured +if(PACKAGE) + + execute_process(COMMAND uname -m COMMAND tr -d '\n' OUTPUT_VARIABLE CPACK_PACKAGE_ARCHITECTURE) + # Install the files copied by the postinst script into the share folder + install(PROGRAMS ${CMAKE_BINARY_DIR}/maxscale DESTINATION ${MAXSCALE_SHAREDIR}) + install(FILES ${CMAKE_BINARY_DIR}/maxscale.conf DESTINATION ${MAXSCALE_SHAREDIR}) + install(PROGRAMS ${CMAKE_BINARY_DIR}/postinst DESTINATION ${MAXSCALE_SHAREDIR}) + install(PROGRAMS ${CMAKE_BINARY_DIR}/postrm DESTINATION ${MAXSCALE_SHAREDIR}) + if(${CMAKE_VERSION} VERSION_LESS 2.8.12) + message(WARNING "CMake version is ${CMAKE_VERSION}. Building of packages requires version 2.8.12 or greater.") + else() + + # Generic CPack configuration variables + SET(CPACK_SET_DESTDIR ON) + set(CPACK_STRIP_FILES FALSE) + set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "MaxScale") + set(CPACK_PACKAGE_VERSION_MAJOR "${MAXSCALE_VERSION_MAJOR}") + set(CPACK_PACKAGE_VERSION_MINOR "${MAXSCALE_VERSION_MINOR}") + set(CPACK_PACKAGE_VERSION_PATCH "${MAXSCALE_VERSION_PATCH}") + set(CPACK_PACKAGE_CONTACT "MariaDB Corporation Ab") + if(DISTRIB_SUFFIX) + set(CPACK_PACKAGE_FILE_NAME "maxscale-${MAXSCALE_VERSION}-${MAXSCALE_BUILD_NUMBER}.${DISTRIB_SUFFIX}.${CPACK_PACKAGE_ARCHITECTURE}" CACHE STRING "MaxScale package filename") + else() + set(CPACK_PACKAGE_FILE_NAME "maxscale-${MAXSCALE_VERSION}-${MAXSCALE_BUILD_NUMBER}.${CPACK_PACKAGE_ARCHITECTURE}" CACHE STRING "MaxScale package filename") + endif() + set(CPACK_PACKAGE_NAME "maxscale") + set(CPACK_PACKAGE_VENDOR "MariaDB Corporation Ab") + set(CPACK_PACKAGE_DESCRIPTION_FILE ${CMAKE_SOURCE_DIR}/etc/DESCRIPTION) + set(CPACK_PACKAGING_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}") + + # See if we are on a RPM-capable or DEB-capable system + find_program(RPMBUILD rpmbuild) + find_program(DEBBUILD dpkg-buildpackage) + + if(TARBALL) + include(cmake/package_tgz.cmake) + + elseif (NOT ( ${RPMBUILD} STREQUAL "RPMBUILD-NOTFOUND" ) OR NOT ( ${DEBBUILD} STREQUAL "DEBBUILD-NOTFOUND" )) + if(NOT ( ${RPMBUILD} STREQUAL "RPMBUILD-NOTFOUND" ) ) + include(cmake/package_rpm.cmake) + message(STATUS "Generating RPM packages") + endif() + if(NOT ( ${DEBBUILD} STREQUAL "DEBBUILD-NOTFOUND" ) ) + include(cmake/package_deb.cmake) + message(STATUS "Generating DEB packages for ${DEB_ARCHITECTURE}") + endif() + else() + message(FATAL_ERROR "Could not automatically resolve the package generator and no generators " + "defined on the command line. Please install distribution specific packaging software or " + "define -DTARBALL=Y to build tar.gz packages.") + endif() + endif() +endif() + # Make sure the release notes for this release are present if it is a stable one if(${MAXSCALE_VERSION} MATCHES "-stable") file(GLOB ${CMAKE_SOURCE_DIR}/Documentation/Release-Notes RELEASE_NOTES *${MAXSCALE_VERSION_NUMERIC}*.md) @@ -243,53 +298,8 @@ if(WITH_SCRIPTS) endif() endif() -# Only do packaging if configured if(PACKAGE) - - execute_process(COMMAND uname -m COMMAND tr -d '\n' OUTPUT_VARIABLE CPACK_PACKAGE_ARCHITECTURE) - # Install the files copied by the postinst script into the share folder - install(PROGRAMS ${CMAKE_BINARY_DIR}/maxscale DESTINATION ${MAXSCALE_SHAREDIR}) - install(FILES ${CMAKE_BINARY_DIR}/maxscale.conf DESTINATION ${MAXSCALE_SHAREDIR}) - install(PROGRAMS ${CMAKE_BINARY_DIR}/postinst DESTINATION ${MAXSCALE_SHAREDIR}) - install(PROGRAMS ${CMAKE_BINARY_DIR}/postrm DESTINATION ${MAXSCALE_SHAREDIR}) - if(${CMAKE_VERSION} VERSION_LESS 2.8.12) - message(WARNING "CMake version is ${CMAKE_VERSION}. Building of packages requires version 2.8.12 or greater.") - else() - - # Generic CPack configuration variables - SET(CPACK_SET_DESTDIR ON) - set(CPACK_STRIP_FILES FALSE) - set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "MaxScale") - set(CPACK_PACKAGE_VERSION_MAJOR "${MAXSCALE_VERSION_MAJOR}") - set(CPACK_PACKAGE_VERSION_MINOR "${MAXSCALE_VERSION_MINOR}") - set(CPACK_PACKAGE_VERSION_PATCH "${MAXSCALE_VERSION_PATCH}") - set(CPACK_PACKAGE_CONTACT "MariaDB Corporation Ab") - if(DISTRIB_SUFFIX) - set(CPACK_PACKAGE_FILE_NAME "maxscale-${MAXSCALE_VERSION}-${MAXSCALE_BUILD_NUMBER}.${DISTRIB_SUFFIX}.${CPACK_PACKAGE_ARCHITECTURE}" CACHE STRING "MaxScale package filename") - else() - set(CPACK_PACKAGE_FILE_NAME "maxscale-${MAXSCALE_VERSION}-${MAXSCALE_BUILD_NUMBER}.${CPACK_PACKAGE_ARCHITECTURE}" CACHE STRING "MaxScale package filename") - endif() - set(CPACK_PACKAGE_NAME "maxscale") - set(CPACK_PACKAGE_VENDOR "MariaDB Corporation Ab") - set(CPACK_PACKAGE_DESCRIPTION_FILE ${CMAKE_SOURCE_DIR}/etc/DESCRIPTION) - set(CPACK_PACKAGING_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}") - - # See if we are on a RPM-capable or DEB-capable system - find_program(RPMBUILD rpmbuild) - find_program(DEBBUILD dpkg-buildpackage) - set(CPACK_GENERATOR "TGZ") - - if(NOT ( ${RPMBUILD} STREQUAL "RPMBUILD-NOTFOUND" ) ) - include(cmake/package_rpm.cmake) - message(STATUS "Generating RPM packages") - endif() - if(NOT ( ${DEBBUILD} STREQUAL "DEBBUILD-NOTFOUND" ) ) - include(cmake/package_deb.cmake) - message(STATUS "Generating DEB packages for ${DEB_ARCHITECTURE}") - endif() - - include(CPack) - endif() + include(CPack) endif() # uninstall target diff --git a/Documentation/Getting-Started/Building-MaxScale-from-Source-Code.md b/Documentation/Getting-Started/Building-MaxScale-from-Source-Code.md index c2a9a409e..afd1da429 100644 --- a/Documentation/Getting-Started/Building-MaxScale-from-Source-Code.md +++ b/Documentation/Getting-Started/Building-MaxScale-from-Source-Code.md @@ -87,6 +87,7 @@ _NAME_=_VALUE_ format (e.g. `-DBUILD_TESTS=Y`). |BUILD_TESTS|Build tests| |WITH_SCRIPTS|Install systemd and init.d scripts| |PACKAGE|Enable building of packages| +|TARBALL|Build tar.gz packages, requires PACKAGE=Y| **Note**: You can look into [defaults.cmake](../../cmake/defaults.cmake) for a list of the CMake variables. @@ -152,7 +153,10 @@ make test make package ``` -This will create a tarball and a RPM/DEB package. +This will create a RPM/DEB package. + +To build a tarball, add `-DTARBALL=Y` to the cmake invokation. This will create +a _maxscale-x.y.z.tar.gz_ file where _x.y.z_ is the version number. Some Debian and Ubuntu systems suffer from a bug where `make package` fails with errors from dpkg-shlibdeps. This can be fixed by running `make` before diff --git a/cmake/package_tgz.cmake b/cmake/package_tgz.cmake new file mode 100644 index 000000000..f99417748 --- /dev/null +++ b/cmake/package_tgz.cmake @@ -0,0 +1,19 @@ +# Tarball package configuration +message(STATUS "Generating tar.gz packages") +set(CMAKE_INSTALL_RPATH_USE_LINK_PATH FALSE) +set(MAXSCALE_BINDIR /bin CACHE PATH "" FORCE) +set(MAXSCALE_LIBDIR /lib CACHE PATH "" FORCE) +set(MAXSCALE_SHAREDIR /share CACHE PATH "" FORCE) +set(MAXSCALE_DOCDIR /share CACHE PATH "" FORCE) +set(MAXSCALE_VARDIR /var CACHE PATH "" FORCE) +set(MAXSCALE_CONFDIR /etc CACHE PATH "" FORCE) +set(CMAKE_INSTALL_PREFIX "/" CACHE PATH "" FORCE) +set(CMAKE_INSTALL_RPATH "$ORIGIN/../lib" CACHE PATH "" FORCE) +set(CMAKE_INSTALL_DATADIR /share CACHE PATH "" FORCE) +set(CPACK_GENERATOR "TGZ") + +if(DISTRIB_SUFFIX) + set(CPACK_PACKAGE_FILE_NAME "maxscale-${MAXSCALE_VERSION}.${DISTRIB_SUFFIX}") +else() + set(CPACK_PACKAGE_FILE_NAME "maxscale-${MAXSCALE_VERSION}") +endif() From ba2cafc65ec6120481b41bb62dae68c8c7edee91 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Tue, 20 Sep 2016 16:13:34 +0300 Subject: [PATCH 28/36] Update tarball instructions --- .../Install-MariaDB-MaxScale-Using-a-Tarball.md | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/Documentation/Getting-Started/Install-MariaDB-MaxScale-Using-a-Tarball.md b/Documentation/Getting-Started/Install-MariaDB-MaxScale-Using-a-Tarball.md index 6dd5cd3c8..ac2682979 100644 --- a/Documentation/Getting-Started/Install-MariaDB-MaxScale-Using-a-Tarball.md +++ b/Documentation/Getting-Started/Install-MariaDB-MaxScale-Using-a-Tarball.md @@ -18,7 +18,9 @@ The required steps are as follows: $ cd maxscale $ chown -R maxscale var -Creating the symbolic link is necessary, since MariaDB MaxScale has been built with with the assumption that its base-directory is `/usr/local/maxscale`. It also makes it easy to switch between different versions of MariaDB MaxScale that have been installed side by side in `/usr/local`; just make the symbolic link point to another installation. +Creating the symbolic link is necessary, since MariaDB MaxScale has been built with with the assumption that its base-directory, that is, the directory under which all its sub-directories are found, is `/usr/local/maxscale`. + +The symbolic link also makes it easy to switch between different versions of MariaDB MaxScale that have been installed side by side in `/usr/local`; just make the symbolic link point to another installation. The following step is to create the MariaDB MaxScale configuration file `/etc/maxscale.cnf`. The file `etc/maxscale.cnf.template` can be used as a base. Please refer to [Configuration Guide](Configuration-Guide.md) for details. @@ -41,16 +43,12 @@ The next step is to create the MaxScale configuration file `maxscale-x.y.z/etc/m When the configuration file has been created, MariaDB MaxScale can be started. $ cd maxscale-x.y.z - $ LD_LIBRARY_PATH=lib/maxscale bin/maxscale -d --basedir=. + $ bin/maxscale -d --basedir=. -With the flag `--basedir`, MariaDB MaxScale is told where the `bin`, `etc`, `lib` -and `var` directories are found. Unless it is specified, MariaDB MaxScale assumes -the directories are found in `/usr/local/maxscale` and the configuration -file in `/etc`. +With the flag `--basedir`, MariaDB MaxScale is told where the `bin`, `etc`, `lib` and `var` directories are found. Unless it is specified, MariaDB MaxScale assumes the directories are found in `/usr/local/maxscale` and the configuration file in `/etc`. -It is also possible to specify the directories and the location of the -configuration file individually. Invoke MaxScale like +It is also possible to specify the directories and the location of the configuration file individually. Invoke MaxScale like - $ LD_LIBRARY_PATH=lib/maxscale bin/maxscale --help + $ bin/maxscale --help to find out the appropriate flags. From 62ab834e8a1ea7c6001623239990e42aaeee1e76 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 20 Sep 2016 21:04:10 +0300 Subject: [PATCH 29/36] Remove unused variables from readwritesplit There were some unused variables in readwritesplit that caused builds to fail. --- server/modules/routing/readwritesplit/rwsplit_route_stmt.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/modules/routing/readwritesplit/rwsplit_route_stmt.c b/server/modules/routing/readwritesplit/rwsplit_route_stmt.c index 247c52064..f8ca27c4d 100644 --- a/server/modules/routing/readwritesplit/rwsplit_route_stmt.c +++ b/server/modules/routing/readwritesplit/rwsplit_route_stmt.c @@ -927,9 +927,6 @@ handle_multi_temp_and_load(ROUTER_CLIENT_SES *rses, GWBUF *querybuf, */ if (rses->have_tmp_tables) { - rses_property_t *rses_prop_tmp; - - rses_prop_tmp = rses->rses_properties[RSES_PROP_TYPE_TMPTABLES]; check_drop_tmp_table(rses, querybuf, packet_type); if (is_packet_a_query(packet_type)) { From f6888ef2056478b5a38f2f54eae72d3c8a5ac381 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Wed, 21 Sep 2016 09:40:15 +0300 Subject: [PATCH 30/36] Install all /var directories The /var/lib/maxscale directory wasn't installed and tarballs didn't have any /var directories. --- CMakeLists.txt | 3 ++- cmake/package_tgz.cmake | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 148b915ce..b37d03670 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -370,8 +370,9 @@ endif() # NOTE: If you make changes here, ensure they are compatible with the # situation in gwdirs.h.in. -if (NOT CMAKE_INSTALL_PREFIX EQUAL "/usr") +if (NOT PACKAGE) install(DIRECTORY DESTINATION var/cache/maxscale) install(DIRECTORY DESTINATION var/log/maxscale) install(DIRECTORY DESTINATION var/run/maxscale) + install(DIRECTORY DESTINATION var/lib/maxscale) endif() diff --git a/cmake/package_tgz.cmake b/cmake/package_tgz.cmake index f99417748..d96756502 100644 --- a/cmake/package_tgz.cmake +++ b/cmake/package_tgz.cmake @@ -12,6 +12,12 @@ set(CMAKE_INSTALL_RPATH "$ORIGIN/../lib" CACHE PATH "" FORCE) set(CMAKE_INSTALL_DATADIR /share CACHE PATH "" FORCE) set(CPACK_GENERATOR "TGZ") +# Include the var directories in the tarball +install(DIRECTORY DESTINATION var/cache/maxscale) +install(DIRECTORY DESTINATION var/log/maxscale) +install(DIRECTORY DESTINATION var/run/maxscale) +install(DIRECTORY DESTINATION var/lib/maxscale) + if(DISTRIB_SUFFIX) set(CPACK_PACKAGE_FILE_NAME "maxscale-${MAXSCALE_VERSION}.${DISTRIB_SUFFIX}") else() From 08d980b4336e9b598259cc74746762e32c99da69 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 20 Sep 2016 20:58:42 +0300 Subject: [PATCH 31/36] Trim and squeeze whitespace when canonicalizing queries The canonical form of the query should ignore changes in whitespace as the semantics of the query stays the same regardless of the amount of whitespace. --- server/core/query_classifier.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/server/core/query_classifier.c b/server/core/query_classifier.c index 442dd9e9b..2e3de362d 100644 --- a/server/core/query_classifier.c +++ b/server/core/query_classifier.c @@ -183,14 +183,24 @@ char* qc_get_canonical(GWBUF* query) { QC_TRACE(); ss_dassert(classifier); + + char *rval; + if (classifier->qc_get_canonical) { - return classifier->qc_get_canonical(query); + rval = classifier->qc_get_canonical(query); } else { - return modutil_get_canonical(query); + rval = modutil_get_canonical(query); } + + if (rval) + { + squeeze_whitespace(rval); + } + + return rval; } bool qc_query_has_clause(GWBUF* query) From bd60fbde7ecf6b002f8702538c07af0d89a1a529 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Wed, 21 Sep 2016 10:53:08 +0300 Subject: [PATCH 32/36] Create dummy files in /var directories for tar.gz packages These files allow seemingly empty directories to be installed on various platforms. Some platforms had problems installing empty directories. --- cmake/package_tgz.cmake | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/cmake/package_tgz.cmake b/cmake/package_tgz.cmake index d96756502..06ccf201e 100644 --- a/cmake/package_tgz.cmake +++ b/cmake/package_tgz.cmake @@ -13,10 +13,15 @@ set(CMAKE_INSTALL_DATADIR /share CACHE PATH "" FORCE) set(CPACK_GENERATOR "TGZ") # Include the var directories in the tarball -install(DIRECTORY DESTINATION var/cache/maxscale) -install(DIRECTORY DESTINATION var/log/maxscale) -install(DIRECTORY DESTINATION var/run/maxscale) -install(DIRECTORY DESTINATION var/lib/maxscale) +# +# On some platforms with certain CMake versions, installing empty directories +# with tarballs does not work. As a workaround, the .cmake-tgz-workaround file +# is installed into the would-be empty directories. +file(WRITE ${CMAKE_BINARY_DIR}/.cmake-tgz-workaround "") +install(FILES ${CMAKE_BINARY_DIR}/.cmake-tgz-workaround DESTINATION var/cache/maxscale) +install(FILES ${CMAKE_BINARY_DIR}/.cmake-tgz-workaround DESTINATION var/log/maxscale) +install(FILES ${CMAKE_BINARY_DIR}/.cmake-tgz-workaround DESTINATION var/run/maxscale) +install(FILES ${CMAKE_BINARY_DIR}/.cmake-tgz-workaround DESTINATION var/lib/maxscale) if(DISTRIB_SUFFIX) set(CPACK_PACKAGE_FILE_NAME "maxscale-${MAXSCALE_VERSION}.${DISTRIB_SUFFIX}") From 578f21e757d68806d1812a25bdc81c8ccc6c20d9 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Wed, 21 Sep 2016 10:08:19 +0300 Subject: [PATCH 33/36] MXS-874: Clear closed state before reconnecting to a server The backend reference states should be cleared when a reconnection attempt is made. Should the creation of a new DCB succeed, the backend should no longer be closed. --- server/modules/routing/readwritesplit/readwritesplit.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index 07b768e8b..ac5a8781d 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -2889,6 +2889,8 @@ bool connect_server(backend_ref_t *bref, SESSION *session, bool execute_history) if (bref->bref_dcb != NULL) { + bref_clear_state(bref, BREF_CLOSED); + if (!execute_history || execute_sescmd_history(bref)) { /** Add a callback for unresponsive server */ From 4422cd87ebb4790fb41f3b15aa4d1c3aa355d721 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Wed, 21 Sep 2016 12:48:38 +0300 Subject: [PATCH 34/36] Update tarball installation instructions The /var/[log|lib|run|cache]/maxscale directory must be created manually. --- ...nstall-MariaDB-MaxScale-Using-a-Tarball.md | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/Documentation/Getting-Started/Install-MariaDB-MaxScale-Using-a-Tarball.md b/Documentation/Getting-Started/Install-MariaDB-MaxScale-Using-a-Tarball.md index ac2682979..25ec59a86 100644 --- a/Documentation/Getting-Started/Install-MariaDB-MaxScale-Using-a-Tarball.md +++ b/Documentation/Getting-Started/Install-MariaDB-MaxScale-Using-a-Tarball.md @@ -14,14 +14,28 @@ The required steps are as follows: $ sudo useradd -g maxscale maxscale $ cd /usr/local $ sudo tar -xzvf maxscale-x.y.z.OS.tar.gz - $ sudo ln -s maxscale-x.y.z maxscale + $ sudo ln -s maxscale-x.y.z.OS maxscale $ cd maxscale - $ chown -R maxscale var + $ sudo chown -R maxscale var Creating the symbolic link is necessary, since MariaDB MaxScale has been built with with the assumption that its base-directory, that is, the directory under which all its sub-directories are found, is `/usr/local/maxscale`. The symbolic link also makes it easy to switch between different versions of MariaDB MaxScale that have been installed side by side in `/usr/local`; just make the symbolic link point to another installation. +In addition, the first time you install MariaDB MaxScale from a tarball you need to create the following directories: + + $ sudo mkdir /var/log/maxscale + $ sudo mkdir /var/lib/maxscale + $ sudo mkdir /var/run/maxscale + $ sudo mkdir /var/cache/maxscale + +and make `maxscale` the owner of them: + + $ sudo chown maxscale /var/log/maxscale + $ sudo chown maxscale /var/lib/maxscale + $ sudo chown maxscale /var/run/maxscale + $ sudo chown maxscale /var/cache/maxscale + The following step is to create the MariaDB MaxScale configuration file `/etc/maxscale.cnf`. The file `etc/maxscale.cnf.template` can be used as a base. Please refer to [Configuration Guide](Configuration-Guide.md) for details. When the configuration file has been created, MariaDB MaxScale can be started. @@ -42,10 +56,10 @@ The next step is to create the MaxScale configuration file `maxscale-x.y.z/etc/m When the configuration file has been created, MariaDB MaxScale can be started. - $ cd maxscale-x.y.z + $ cd maxscale-x.y.z.OS $ bin/maxscale -d --basedir=. -With the flag `--basedir`, MariaDB MaxScale is told where the `bin`, `etc`, `lib` and `var` directories are found. Unless it is specified, MariaDB MaxScale assumes the directories are found in `/usr/local/maxscale` and the configuration file in `/etc`. +With the flag `--basedir`, MariaDB MaxScale is told where the `lib`, `etc` and `var` directories are found. Unless it is specified, MariaDB MaxScale assumes the `lib` directory is found in `/usr/local/maxscale`, and the `var` and `etc` directories in `/`. It is also possible to specify the directories and the location of the configuration file individually. Invoke MaxScale like From 0882541c8095e6c1888743be394c1f5877dc0870 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Wed, 21 Sep 2016 13:49:54 +0300 Subject: [PATCH 35/36] Make the default directories configurable The default directories can now be changed at build time. This allows tarballs to look for libraries in a more sensible place. --- cmake/install_layout.cmake | 21 +++++++++++++++++++++ cmake/package_tgz.cmake | 6 ++++-- server/include/gwdirs.h.in | 37 +++++++++++++++++++------------------ 3 files changed, 44 insertions(+), 20 deletions(-) diff --git a/cmake/install_layout.cmake b/cmake/install_layout.cmake index a14ef05c5..4e3095e8e 100644 --- a/cmake/install_layout.cmake +++ b/cmake/install_layout.cmake @@ -9,3 +9,24 @@ set(MAXSCALE_DOCDIR ${CMAKE_INSTALL_DOCDIR}/maxscale CACHE PATH "Documentation i # These are the only hard-coded absolute paths set(MAXSCALE_VARDIR /var CACHE PATH "Data file path (usually /var/)") set(MAXSCALE_CONFDIR /etc CACHE PATH "Configuration file installation path (/etc/)") + +# Default values for directories and subpaths where files are searched. These +# are used in `server/include/gwdirs.h.in`. + +set(DEFAULT_PID_SUBPATH "run/maxscale" CACHE PATH "Default PID file subpath") +set(DEFAULT_LOG_SUBPATH "log/maxscale" CACHE PATH "Default log subpath") +set(DEFAULT_DATA_SUBPATH "lib/maxscale" CACHE PATH "Default datadir subpath") +set(DEFAULT_LIB_SUBPATH "${MAXSCALE_LIBDIR}" CACHE PATH "Default library subpath") +set(DEFAULT_CACHE_SUBPATH "cache/maxscale" CACHE PATH "Default cache subpath") +set(DEFAULT_LANG_SUBPATH "lib/maxscale" CACHE PATH "Default language file subpath") +set(DEFAULT_EXEC_SUBPATH "${MAXSCALE_BINDIR}" CACHE PATH "Default executable subpath") +set(DEFAULT_CONFIG_SUBPATH "etc" CACHE PATH "Default configuration subpath") + +set(DEFAULT_PIDDIR ${MAXSCALE_VARDIR}/${DEFAULT_PID_SUBPATH} CACHE PATH "Default PID file directory") +set(DEFAULT_LOGDIR ${MAXSCALE_VARDIR}/${DEFAULT_LOG_SUBPATH} CACHE PATH "Default log directory") +set(DEFAULT_DATADIR ${MAXSCALE_VARDIR}/${DEFAULT_DATA_SUBPATH} CACHE PATH "Default datadir path") +set(DEFAULT_LIBDIR ${CMAKE_INSTALL_PREFIX}/${DEFAULT_LIB_SUBPATH}/ CACHE PATH "Default library path") +set(DEFAULT_CACHEDIR ${MAXSCALE_VARDIR}/${DEFAULT_CACHE_SUBPATH} CACHE PATH "Default cache directory") +set(DEFAULT_LANGDIR ${MAXSCALE_VARDIR}/${DEFAULT_LANG_SUBPATH} CACHE PATH "Default language file directory") +set(DEFAULT_EXECDIR ${CMAKE_INSTALL_PREFIX}/${DEFAULT_EXEC_SUBPATH} CACHE PATH "Default executable directory") +set(DEFAULT_CONFIGDIR /${DEFAULT_CONFIG_SUBPATH} CACHE PATH "Default configuration directory") diff --git a/cmake/package_tgz.cmake b/cmake/package_tgz.cmake index 06ccf201e..cd30b7ce2 100644 --- a/cmake/package_tgz.cmake +++ b/cmake/package_tgz.cmake @@ -2,14 +2,16 @@ message(STATUS "Generating tar.gz packages") set(CMAKE_INSTALL_RPATH_USE_LINK_PATH FALSE) set(MAXSCALE_BINDIR /bin CACHE PATH "" FORCE) -set(MAXSCALE_LIBDIR /lib CACHE PATH "" FORCE) +set(MAXSCALE_LIBDIR /lib/maxscale CACHE PATH "" FORCE) set(MAXSCALE_SHAREDIR /share CACHE PATH "" FORCE) set(MAXSCALE_DOCDIR /share CACHE PATH "" FORCE) set(MAXSCALE_VARDIR /var CACHE PATH "" FORCE) set(MAXSCALE_CONFDIR /etc CACHE PATH "" FORCE) set(CMAKE_INSTALL_PREFIX "/" CACHE PATH "" FORCE) -set(CMAKE_INSTALL_RPATH "$ORIGIN/../lib" CACHE PATH "" FORCE) +set(CMAKE_INSTALL_RPATH "$ORIGIN/../lib/maxscale/" CACHE PATH "" FORCE) set(CMAKE_INSTALL_DATADIR /share CACHE PATH "" FORCE) +set(DEFAULT_LIB_SUBPATH /lib/maxscale CACHE PATH "" FORCE) +set(DEFAULT_LIBDIR "/usr/local/maxscale/lib/maxscale" CACHE PATH "" FORCE) set(CPACK_GENERATOR "TGZ") # Include the var directories in the tarball diff --git a/server/include/gwdirs.h.in b/server/include/gwdirs.h.in index 0a2c0dab1..d607b021d 100644 --- a/server/include/gwdirs.h.in +++ b/server/include/gwdirs.h.in @@ -21,30 +21,31 @@ EXTERN_C_BLOCK_BEGIN -// NOTE: If you make changes here, ensure they are compatible with the -// situation in /CMakeLists.txt, where directories are installed. -#define MXS_DEFAULT_PID_SUBPATH "run/maxscale" -#define MXS_DEFAULT_LOG_SUBPATH "log/maxscale" -#define MXS_DEFAULT_DATA_SUBPATH "lib/maxscale" -#define MXS_DEFAULT_LIB_SUBPATH "@MAXSCALE_LIBDIR@" -#define MXS_DEFAULT_CACHE_SUBPATH "cache/maxscale" -#define MXS_DEFAULT_LANG_SUBPATH "lib/maxscale" -#define MXS_DEFAULT_EXEC_SUBPATH "@MAXSCALE_BINDIR@" -#define MXS_DEFAULT_CONFIG_SUBPATH "etc" +/** + * All of the following DEFAULT_* variables are defined in cmake/install_layout.cmake + */ +#define MXS_DEFAULT_PID_SUBPATH "@DEFAULT_PID_SUBPATH@" +#define MXS_DEFAULT_LOG_SUBPATH "@DEFAULT_LOG_SUBPATH@" +#define MXS_DEFAULT_DATA_SUBPATH "@DEFAULT_DATA_SUBPATH@" +#define MXS_DEFAULT_LIB_SUBPATH "@DEFAULT_LIB_SUBPATH@" +#define MXS_DEFAULT_CACHE_SUBPATH "@DEFAULT_CACHE_SUBPATH@" +#define MXS_DEFAULT_LANG_SUBPATH "@DEFAULT_LANG_SUBPATH@" +#define MXS_DEFAULT_EXEC_SUBPATH "@DEFAULT_EXEC_SUBPATH@" +#define MXS_DEFAULT_CONFIG_SUBPATH "@DEFAULT_CONFIG_SUBPATH@" /** Default file locations, configured by CMake */ static const char* default_cnf_fname = "maxscale.cnf"; -static const char* default_configdir = "/" MXS_DEFAULT_CONFIG_SUBPATH; +static const char* default_configdir = "@DEFAULT_CONFIGDIR@"; /*< This should be changed to just /run eventually, * the /var/run folder is an old standard and the newer FSH 3.0 * uses /run for PID files.*/ -static const char* default_piddir = "@MAXSCALE_VARDIR@/" MXS_DEFAULT_PID_SUBPATH; -static const char* default_logdir = "@MAXSCALE_VARDIR@/" MXS_DEFAULT_LOG_SUBPATH; -static const char* default_datadir = "@MAXSCALE_VARDIR@/" MXS_DEFAULT_DATA_SUBPATH; -static const char* default_libdir = "@CMAKE_INSTALL_PREFIX@/" MXS_DEFAULT_LIB_SUBPATH; -static const char* default_cachedir = "@MAXSCALE_VARDIR@/" MXS_DEFAULT_CACHE_SUBPATH; -static const char* default_langdir = "@MAXSCALE_VARDIR@/" MXS_DEFAULT_LANG_SUBPATH; -static const char* default_execdir = "@CMAKE_INSTALL_PREFIX@/" MXS_DEFAULT_EXEC_SUBPATH; +static const char* default_piddir = "@DEFAULT_PIDDIR@"; +static const char* default_logdir = "@DEFAULT_LOGDIR@"; +static const char* default_datadir = "@DEFAULT_DATADIR@"; +static const char* default_libdir = "@DEFAULT_LIBDIR@"; +static const char* default_cachedir = "@DEFAULT_CACHEDIR@"; +static const char* default_langdir = "@DEFAULT_LANGDIR@"; +static const char* default_execdir = "@DEFAULT_EXECDIR@"; static char* configdir = NULL; static char* logdir = NULL; From e484aac6f03cc44694f4d6002def46270cdc2479 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 22 Sep 2016 10:09:25 +0300 Subject: [PATCH 36/36] Use CLOCK_MONOTONIC instead of CLOCK_MONOTONIC_RAW The CLOCK_MONOTONIC_RAW isn't supported on all of the platforms, namely CentOS 5. --- server/core/log_manager.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/core/log_manager.cc b/server/core/log_manager.cc index 769445fa2..64bab964a 100644 --- a/server/core/log_manager.cc +++ b/server/core/log_manager.cc @@ -215,10 +215,10 @@ static const int LM_MESSAGE_HASH_SIZE = 293; /** A prime, and roughly a quarter * * @return Current monotonic raw time in milliseconds. */ -static uint64_t time_monotonic_raw_ms() +static uint64_t time_monotonic_ms() { struct timespec now; - clock_gettime(CLOCK_MONOTONIC_RAW, &now); + clock_gettime(CLOCK_MONOTONIC, &now); return now.tv_sec * 1000 + now.tv_nsec / 1000000; } @@ -2815,7 +2815,7 @@ static message_suppression_t message_status(const char* file, int line) { LM_MESSAGE_STATS stats; spinlock_init(&stats.lock); - stats.first_ms = time_monotonic_raw_ms(); + stats.first_ms = time_monotonic_ms(); stats.last_ms = 0; stats.count = 0; @@ -2829,7 +2829,7 @@ static message_suppression_t message_status(const char* file, int line) if (value) { - uint64_t now_ms = time_monotonic_raw_ms(); + uint64_t now_ms = time_monotonic_ms(); spinlock_acquire(&value->lock);