From 87e66a0ea80f03d8a2451ab400779b6fe7c8b733 Mon Sep 17 00:00:00 2001 From: Mark Riddoch Date: Tue, 15 Jul 2014 17:53:39 +0100 Subject: [PATCH 01/60] Revert "Implementation of HTTPD protocol, gwbuf properties and a dmeo web application" This reverts commit 6fd5dff34902051f38932947493280e1e62dbeb1. --- server/core/buffer.c | 88 ----- server/core/monitor.c | 25 -- server/core/server.c | 25 -- server/core/service.c | 25 -- server/core/session.c | 25 -- server/include/buffer.h | 21 +- server/include/monitor.h | 1 - server/include/server.h | 1 - server/include/service.h | 1 - server/include/session.h | 1 - server/modules/include/httpd.h | 20 +- server/modules/protocol/httpd.c | 352 ++++++++++---------- server/modules/routing/Makefile | 8 +- server/modules/routing/webserver.c | 493 ----------------------------- 14 files changed, 178 insertions(+), 908 deletions(-) delete mode 100644 server/modules/routing/webserver.c diff --git a/server/core/buffer.c b/server/core/buffer.c index 4d28793fc..d3278a505 100644 --- a/server/core/buffer.c +++ b/server/core/buffer.c @@ -32,7 +32,6 @@ * 11/07/13 Mark Riddoch Add reference count mechanism * 16/07/2013 Massimiliano Pinto Added command type to gwbuf struct * 24/06/2014 Mark Riddoch Addition of gwbuf_trim - * 15/07/2014 Mark Riddoch Addition of properties * * @endverbatim */ @@ -78,13 +77,11 @@ SHARED_BUF *sbuf; free(sbuf); return NULL; } - spinlock_init(&rval->lock); rval->start = sbuf->data; rval->end = rval->start + size; sbuf->refcount = 1; rval->sbuf = sbuf; rval->next = NULL; - rval->properties = NULL; rval->gwbuf_type = GWBUF_TYPE_UNDEFINED; rval->command = 0; CHK_GWBUF(rval); @@ -99,22 +96,12 @@ SHARED_BUF *sbuf; void gwbuf_free(GWBUF *buf) { -BUF_PROPERTY *prop; - CHK_GWBUF(buf); if (atomic_add(&buf->sbuf->refcount, -1) == 1) { free(buf->sbuf->data); free(buf->sbuf); } - while (buf->properties) - { - prop = buf->properties; - buf->properties = prop->next; - free(prop->name); - free(prop->value); - free(prop); - } free(buf); } @@ -350,79 +337,4 @@ void gwbuf_set_type( } -/** - * Add a property to a buffer. - * - * @param buf The buffer to add the property to - * @param name The property name - * @param value The property value - * @return Non-zero on success - */ -int -gwbuf_add_property(GWBUF *buf, char *name, char *value) -{ -BUF_PROPERTY *prop; - if ((prop = malloc(sizeof(BUF_PROPERTY))) == NULL) - return 0; - - prop->name = strdup(name); - prop->value = strdup(value); - spinlock_acquire(&buf->lock); - prop->next = buf->properties; - buf->properties = prop; - spinlock_release(&buf->lock); - return 1; -} - -/** - * Return the value of a buffer property - * @param buf The buffer itself - * @param name The name of the property to return - * @return The property value or NULL if the property was not found. - */ -char * -gwbuf_get_property(GWBUF *buf, char *name) -{ -BUF_PROPERTY *prop; - - spinlock_acquire(&buf->lock); - prop = buf->properties; - while (prop && strcmp(prop->name, name) != 0) - prop = prop->next; - spinlock_release(&buf->lock); - if (prop) - return prop->value; - return NULL; -} - - -/** - * Convert a chain of GWBUF structures into a single GWBUF structure - * - * @param orig The chain to convert - * @return The contiguous buffer - */ -GWBUF * -gwbuf_make_contiguous(GWBUF *orig) -{ -GWBUF *newbuf; -char *ptr; -int len; - - if (orig->next == NULL) - return orig; - - if ((newbuf = gwbuf_alloc(gwbuf_length(orig))) != NULL) - { - ptr = GWBUF_DATA(newbuf); - while (orig) - { - len = GWBUF_LENGTH(orig); - memcpy(ptr, GWBUF_DATA(orig), len); - ptr += len; - orig = gwbuf_consume(orig, len); - } - } - return newbuf; -} diff --git a/server/core/monitor.c b/server/core/monitor.c index 8bb62bc2d..d7b53f5b7 100644 --- a/server/core/monitor.c +++ b/server/core/monitor.c @@ -312,28 +312,3 @@ monitorSetReplicationHeartbeat(MONITOR *mon, int replication_heartbeat) mon->module->replicationHeartbeat(mon->handle, replication_heartbeat); } } - -/** - * Iterate over the monitors, calling a function per call - * - * @param fcn The function to call - * @param data The data to pass to each call - */ -void -monitorIterate(void (*fcn)(MONITOR *, void *), void *data) -{ -MONITOR *monitor, *next; - - spinlock_acquire(&monLock); - monitor = allMonitors; - while (monitor) - { - next = monitor->next; - spinlock_release(&monLock); - (*fcn)(monitor, data); - spinlock_acquire(&monLock); - monitor = next; - } - spinlock_release(&monLock); - -} diff --git a/server/core/server.c b/server/core/server.c index cb3090679..bfe5e4d04 100644 --- a/server/core/server.c +++ b/server/core/server.c @@ -561,28 +561,3 @@ SERVER_PARAM *param = server->parameters; } return NULL; } - -/** - * Iterate over the servers, calling a function per call - * - * @param fcn The function to call - * @param data The data to pass to each call - */ -void -serverIterate(void (*fcn)(SERVER *, void *), void *data) -{ -SERVER *server, *next; - - spinlock_acquire(&server_spin); - server = allServers; - while (server) - { - next = server->next; - spinlock_release(&server_spin); - (*fcn)(server, data); - spinlock_acquire(&server_spin); - server = next; - } - spinlock_release(&server_spin); - -} diff --git a/server/core/service.c b/server/core/service.c index 592d90321..1102dabb4 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -1171,28 +1171,3 @@ serviceGetWeightingParameter(SERVICE *service) { return service->weightby; } - -/** - * Iterate over the services, calling a function per call - * - * @param fcn The function to call - * @param data The data to pass to each call - */ -void -serviceIterate(void (*fcn)(SERVICE *, void *), void *data) -{ -SERVICE *service, *next; - - spinlock_acquire(&service_spin); - service = allServices; - while (service) - { - next = service->next; - spinlock_release(&service_spin); - (*fcn)(service, data); - spinlock_acquire(&service_spin); - service = next; - } - spinlock_release(&service_spin); - -} diff --git a/server/core/session.c b/server/core/session.c index 9f1a55615..80767f9ff 100644 --- a/server/core/session.c +++ b/server/core/session.c @@ -756,28 +756,3 @@ session_getUser(SESSION *session) { return (session && session->client) ? session->client->user : NULL; } - -/** - * Iterate over the sessions, calling a function per call - * - * @param fcn The function to call - * @param data The data to pass to each call - */ -void -sessionIterate(void (*fcn)(SESSION *, void *), void *data) -{ -SESSION *session, *next; - - spinlock_acquire(&session_spin); - session = allSessions; - while (session) - { - next = session->next; - spinlock_release(&session_spin); - (*fcn)(session, data); - spinlock_acquire(&session_spin); - session = next; - } - spinlock_release(&session_spin); - -} diff --git a/server/include/buffer.h b/server/include/buffer.h index cf2f9e8dd..9729c538c 100644 --- a/server/include/buffer.h +++ b/server/include/buffer.h @@ -38,25 +38,12 @@ * 10/06/2013 Mark Riddoch Initial implementation * 11/07/2013 Mark Riddoch Addition of reference count in the gwbuf * 16/07/2013 Massimiliano Pinto Added command type for the queue - * 15/07/2014 Mark Riddoch Added buffer properties * * @endverbatim */ -#include #include -/** - * Buffer properties - used to store properties related to the buffer - * contents. This may be added at any point during the processing of the - * data, especially in the protocol stage of the processing. - */ -typedef struct buf_property { - char *name; - char *value; - struct buf_property *next; -} BUF_PROPERTY; - typedef enum { GWBUF_TYPE_UNDEFINED = 0x00, @@ -65,8 +52,7 @@ typedef enum GWBUF_TYPE_SINGLE_STMT = 0x04, GWBUF_TYPE_SESCMD_RESPONSE = 0x08, GWBUF_TYPE_RESPONSE_END = 0x10, - GWBUF_TYPE_SESCMD = 0x20, - GWBUF_TYPE_HTTP = 0x40 + GWBUF_TYPE_SESCMD = 0x20 } gwbuf_type_t; #define GWBUF_IS_TYPE_UNDEFINED(b) (b->gwbuf_type == 0) @@ -102,8 +88,6 @@ typedef struct gwbuf { SHARED_BUF *sbuf; /*< The shared buffer with the real data */ int command;/*< The command type for the queue */ gwbuf_type_t gwbuf_type; /*< buffer's data type information */ - SPINLOCK lock; - BUF_PROPERTY *properties; /*< Buffer properties */ } GWBUF; /*< @@ -137,7 +121,4 @@ extern unsigned int gwbuf_length(GWBUF *head); extern GWBUF *gwbuf_clone_portion(GWBUF *head, size_t offset, size_t len); extern GWBUF *gwbuf_clone_transform(GWBUF *head, gwbuf_type_t type); extern void gwbuf_set_type(GWBUF *head, gwbuf_type_t type); -extern int gwbuf_add_property(GWBUF *buf, char *name, char *value); -extern char *gwbuf_get_property(GWBUF *buf, char *name); -extern GWBUF *gwbuf_make_contiguous(GWBUF *); #endif diff --git a/server/include/monitor.h b/server/include/monitor.h index 4dee5f49c..d65fd075f 100644 --- a/server/include/monitor.h +++ b/server/include/monitor.h @@ -110,5 +110,4 @@ extern void monitorList(DCB *); extern void monitorSetId(MONITOR *, unsigned long); extern void monitorSetInterval (MONITOR *, unsigned long); extern void monitorSetReplicationHeartbeat(MONITOR *, int); -extern void monitorIterate(void (*fcn)(MONITOR *, void *), void *data); #endif diff --git a/server/include/server.h b/server/include/server.h index f622b5765..fdcc32e80 100644 --- a/server/include/server.h +++ b/server/include/server.h @@ -161,5 +161,4 @@ extern void serverAddParameter(SERVER *, char *, char *); extern char *serverGetParameter(SERVER *, char *); extern void server_update(SERVER *, char *, char *, char *); extern void server_set_unique_name(SERVER *, char *); -extern void serverIterate(void (*fcn)(SERVER *, void *), void *data); #endif diff --git a/server/include/service.h b/server/include/service.h index 0d8d32cda..cd13d411b 100644 --- a/server/include/service.h +++ b/server/include/service.h @@ -161,7 +161,6 @@ extern void serviceSetFilters(SERVICE *, char *); extern int serviceEnableRootUser(SERVICE *, int ); extern void serviceWeightBy(SERVICE *, char *); extern char *serviceGetWeightingParameter(SERVICE *); -extern void serviceIterate(void (*fcn)(SERVICE *, void *), void *data); extern void service_update(SERVICE *, char *, char *, char *); extern int service_refresh_users(SERVICE *); extern void printService(SERVICE *); diff --git a/server/include/session.h b/server/include/session.h index 0386a5eb6..cbd43fe40 100644 --- a/server/include/session.h +++ b/server/include/session.h @@ -157,7 +157,6 @@ void dprintAllSessions(struct dcb *); void dprintSession(struct dcb *, SESSION *); void dListSessions(struct dcb *); char *session_state(int); -void sessionIterate(void (*fcn)(SESSION *, void *), void *data); bool session_link_dcb(SESSION *, struct dcb *); SESSION* get_session_by_router_ses(void* rses); #endif diff --git a/server/modules/include/httpd.h b/server/modules/include/httpd.h index 920e65b2d..0d5e65604 100644 --- a/server/modules/include/httpd.h +++ b/server/modules/include/httpd.h @@ -48,20 +48,18 @@ #define HTTPD_FIELD_MAXLEN 8192 #define HTTPD_REQUESTLINE_MAXLEN 8192 -typedef enum { - METHOD_UNKNOWN = 0, - METHOD_POST, - METHOD_PUT, - METHOD_GET, - METHOD_HEAD -} HTTP_METHOD; /** * HTTPD session specific data * */ typedef struct httpd_session { - HTTP_METHOD method; - GWBUF *saved; - int request_len; - char *url; + char user[HTTPD_USER_MAXLEN]; /*< username for authentication*/ + char *cookies; /*< all input cookies */ + char hostname[HTTPD_HOSTNAME_MAXLEN]; /*< The hostname */ + char useragent[HTTPD_USERAGENT_MAXLEN]; /*< The useragent */ + char method[HTTPD_METHOD_MAXLEN]; /*< The HTTPD Method */ + char *url; /*< the URL in the request */ + char *path_info; /*< the Pathinfo, starts with /, is the extra path segments after the document name */ + char *query_string; /*< the Query string, starts with ?, after path_info and document name */ + int headers_received; /*< All the headers has been received, if 1 */ } HTTPD_session; diff --git a/server/modules/protocol/httpd.c b/server/modules/protocol/httpd.c index bdb06e560..7db1366ad 100644 --- a/server/modules/protocol/httpd.c +++ b/server/modules/protocol/httpd.c @@ -33,8 +33,6 @@ * Date Who Description * 08/07/2013 Massimiliano Pinto Initial version * 09/07/2013 Massimiliano Pinto Added /show?dcb|session for all dcbs|sessions - * 11/07/2014 Mark Riddoch Recoded as more generic protocol module - * removing hardcoded example * * @endverbatim */ @@ -42,10 +40,6 @@ #include #include #include -#include -#include - -extern int lm_enabled_logfiles_bitmask; MODULE_INFO info = { MODULE_API_PROTOCOL, @@ -54,6 +48,7 @@ MODULE_INFO info = { "An experimental HTTPD implementation for use in admnistration" }; +#define ISspace(x) isspace((int)(x)) #define HTTP_SERVER_STRING "Gateway(c) v.1.0.0" static char *version_str = "V1.0.1"; @@ -65,8 +60,8 @@ static int httpd_hangup(DCB *dcb); static int httpd_accept(DCB *dcb); static int httpd_close(DCB *dcb); static int httpd_listen(DCB *dcb, char *config); -static char *httpd_nextline(GWBUF *buf, char *ptr); -static void httpd_process_header(GWBUF *buf, char *sol, HTTPD_session *client_data); +static int httpd_get_line(int sock, char *buf, int size); +static void httpd_send_headers(DCB *dcb, int final); /** * The "module object" for the httpd protocol module. @@ -126,99 +121,132 @@ GetModuleObject() * @return */ static int -httpd_read_event(DCB *dcb) +httpd_read_event(DCB* dcb) { -SESSION *session = dcb->session; -GWBUF *buf = NULL; -char *ptr, *sol; -HTTPD_session *client_data = NULL; -int n; +//SESSION *session = dcb->session; +//ROUTER_OBJECT *router = session->service->router; +//ROUTER *router_instance = session->service->router_instance; +//void *rsession = session->router_session; - // Read all the available data - if ((n = dcb_read(dcb, &buf)) != -1) - { - client_data = dcb->data; +int numchars = 1; +char buf[HTTPD_REQUESTLINE_MAXLEN-1] = ""; +char *query_string = NULL; +char method[HTTPD_METHOD_MAXLEN-1] = ""; +char url[HTTPD_SMALL_BUFFER] = ""; +int cgi = 0; +size_t i, j; +int headers_read = 0; +HTTPD_session *client_data = NULL; - if (client_data->saved) - { - buf = gwbuf_append(client_data->saved, buf); - client_data->saved = NULL; - } - buf = gwbuf_make_contiguous(buf); - - ptr = GWBUF_DATA(buf); - - if (strncasecmp(ptr, "POST", 4)) - { - client_data->method = METHOD_POST; - gwbuf_add_property(buf, "Method", "POST"); - ptr = ptr + 4; - } - else if (strncasecmp(ptr, "PUT", 3)) - { - client_data->method = METHOD_PUT; - gwbuf_add_property(buf, "Method", "PUT"); - ptr = ptr + 3; - } - else if (strncasecmp(ptr, "GET", 3)) - { - client_data->method = METHOD_GET; - gwbuf_add_property(buf, "Method", "GET"); - ptr = ptr + 3; - } - else if (strncasecmp(ptr, "HEAD", 4)) - { - client_data->method = METHOD_HEAD; - gwbuf_add_property(buf, "Method", "HEAD"); - ptr = ptr + 4; - } - while (ptr < (char *)(buf->end) && isspace(*ptr)) - ptr++; - sol = ptr; - while (ptr < (char *)(buf->end) && isspace(*ptr) == 0) - ptr++; - client_data->url = strndup(sol, ptr - sol); - gwbuf_add_property(buf, "URL", client_data->url); - while ((sol = httpd_nextline(buf, ptr)) != NULL && - *sol != '\n' && *sol != '\r') - { - httpd_process_header(buf, sol, client_data); - ptr = sol; - } - - /* - * We have read all the headers, or run out of data to - * examine. - */ - if (sol == NULL) - { - client_data->saved = buf; - return 0; - } - else - { - if (((char *)(buf->end) - sol) - < client_data->request_len) - { - client_data->saved = buf; - } - else - { - LOGIF(LT, (skygw_log_write( - LOGFILE_TRACE, - "HTTPD: request %s.\n", client_data->url))); - SESSION_ROUTE_QUERY(session, buf); - if (client_data->url) - { - free(client_data->url); - client_data->url = NULL; - } - } - } + client_data = dcb->data; + /** + * get the request line + * METHOD URL HTTP_VER\r\n + */ + numchars = httpd_get_line(dcb->fd, buf, sizeof(buf)); + i = 0; j = 0; + while (!ISspace(buf[j]) && (i < sizeof(method) - 1)) { + method[i] = buf[j]; + i++; j++; } + method[i] = '\0'; + + strcpy(client_data->method, method); + + /* check allowed http methods */ + if (strcasecmp(method, "GET") && strcasecmp(method, "POST")) { + //httpd_unimplemented(dcb->fd); + return 0; + } + + if (strcasecmp(method, "POST") == 0) + cgi = 1; + + i = 0; + + while (ISspace(buf[j]) && (j < sizeof(buf))) { + j++; + } + + while (!ISspace(buf[j]) && (i < sizeof(url) - 1) && (j < sizeof(buf))) { + url[i] = buf[j]; + i++; j++; + } + + url[i] = '\0'; + + /** + * Get the query string if availble + */ + + if (strcasecmp(method, "GET") == 0) { + query_string = url; + while ((*query_string != '?') && (*query_string != '\0')) + query_string++; + if (*query_string == '?') { + cgi = 1; + *query_string = '\0'; + query_string++; + } + } + + /** + * Get the request headers + */ + + while ((numchars > 0) && strcmp("\n", buf)) { + char *value = NULL; + char *end = NULL; + numchars = httpd_get_line(dcb->fd, buf, sizeof(buf)); + if ( (value = strchr(buf, ':'))) { + *value = '\0'; + value++; + end = &value[strlen(value) -1]; + *end = '\0'; + + if (strncasecmp(buf, "Hostname", 6) == 0) { + strcpy(client_data->hostname, value); + } + if (strncasecmp(buf, "useragent", 9) == 0) { + strcpy(client_data->useragent, value); + } + } + } + + if (numchars) { + headers_read = 1; + memcpy(&client_data->headers_received, &headers_read, sizeof(int)); + } + + /** + * Now begins the server reply + */ + + /* send all the basic headers and close with \r\n */ + httpd_send_headers(dcb, 1); + + /** + * ToDO: launch proper content handling based on the requested URI, later REST interface + * + */ + + dcb_printf(dcb, "Welcome to HTTPD Gateway (c) %s\n\n", version_str); + + if (strcmp(url, "/show") == 0) { + if (strlen(query_string)) { + if (strcmp(query_string, "dcb") == 0) + dprintAllDCBs(dcb); + if (strcmp(query_string, "session") == 0) + dprintAllSessions(dcb); + } + } + + /* force the client connecton close */ + dcb_close(dcb); + return 0; } @@ -259,18 +287,6 @@ httpd_write(DCB *dcb, GWBUF *queue) static int httpd_error(DCB *dcb) { -HTTPD_session *client_data = NULL; - if (dcb->data) - { - client_data = dcb->data; - if (client_data->url) - { - free(client_data->url); - client_data->url = NULL; - } - free(dcb->data); - dcb->data = NULL; - } dcb_close(dcb); return 0; } @@ -302,7 +318,7 @@ int n_connect = 0; { int so = -1; struct sockaddr_in addr; - socklen_t addrlen = 0; + socklen_t addrlen; DCB *client = NULL; HTTPD_session *client_data = NULL; @@ -317,11 +333,10 @@ int n_connect = 0; memcpy(&client->func, &MyObject, sizeof(GWPROTOCOL)); /* we don't need the session */ - client->session = session_alloc(dcb->session->service, client); + client->session = NULL; /* create the session data for HTTPD */ client_data = (HTTPD_session *)calloc(1, sizeof(HTTPD_session)); - memset(client_data, 0, sizeof(HTTPD_session)); client->data = client_data; if (poll_add_dcb(client) == -1) @@ -410,84 +425,51 @@ int rc; } /** - * Return the start of the next line int the buffer. - * - * @param buf The GWBUF chain - * @param ptr Start point within the buffer - * - * @return the start of the next line or NULL if there are no more lines + * HTTPD get line from client */ -static char * -httpd_nextline(GWBUF *buf, char *ptr) -{ - while (ptr < (char *)(buf->end) && *ptr != '\n' && *ptr != '\r') - ptr++; - if (ptr >= (char *)(buf->end)) - return NULL; +static int httpd_get_line(int sock, char *buf, int size) { + int i = 0; + char c = '\0'; + int n; - /* Skip prcisely one CR/LF */ - if (*ptr == '\r') - ptr++; - if (*ptr == '\n') - ptr++; - return ptr; -} - -/** - * The headers to extract from the HTTP request and add as properties to the - * GWBUF structure. - */ -static char *headers[] = { - "Content-Type", - "User-Agent", - "From", - "Date", - NULL -}; - -/** - * Process a single header line - * - * @param buf The GWBUF that contains the request - * @param sol The current start of line - * @param client_data The client data structure for this request - */ -static void -httpd_process_header(GWBUF *buf, char *sol, HTTPD_session *client_data) -{ -char *ptr = sol; -char cbuf[300]; -int len, i; - - /* Find the end of the line */ - while (ptr < (char *)(buf->end) && *ptr != '\n' && *ptr != '\r') - ptr++; - - if (strncmp(sol, "Content-Length:", strlen("Content-Length:")) == 0) - { - char *p1 = sol + strlen("Content-Length:"); - while (isspace(*p1)) - p1++; - len = ptr - p1; - strncpy(cbuf, p1, len); - cbuf[len] = 0; - client_data->request_len = atoi(cbuf); - gwbuf_add_property(buf, "Content-Length", cbuf); - } - else - { - for (i = 0; headers[i]; i++) - { - if (strncmp(sol, headers[i], strlen(headers[i])) == 0) - { - char *p1 = sol + strlen(headers[i]) + 1; - while (isspace(*p1)) - p1++; - len = ptr - p1; - strncpy(cbuf, p1, len); - cbuf[len] = 0; - gwbuf_add_property(buf, headers[i], cbuf); + while ((i < size - 1) && (c != '\n')) { + n = recv(sock, &c, 1, 0); + /* DEBUG printf("%02X\n", c); */ + if (n > 0) { + if (c == '\r') { + n = recv(sock, &c, 1, MSG_PEEK); + /* DEBUG printf("%02X\n", c); */ + if ((n > 0) && (c == '\n')) + recv(sock, &c, 1, 0); + else + c = '\n'; } - } + buf[i] = c; + i++; + } else + c = '\n'; + } + + buf[i] = '\0'; + + return i; +} + +/** + * HTTPD send basic headers with 200 OK + */ +static void httpd_send_headers(DCB *dcb, int final) +{ + char date[64] = ""; + const char *fmt = "%a, %d %b %Y %H:%M:%S GMT"; + time_t httpd_current_time = time(NULL); + + strftime(date, sizeof(date), fmt, localtime(&httpd_current_time)); + + dcb_printf(dcb, "HTTP/1.1 200 OK\r\nDate: %s\r\nServer: %s\r\nConnection: close\r\nContent-Type: text/plain\r\n", date, HTTP_SERVER_STRING); + + /* close the headers */ + if (final) { + dcb_printf(dcb, "\r\n"); } } diff --git a/server/modules/routing/Makefile b/server/modules/routing/Makefile index a1095304c..4feac68fb 100644 --- a/server/modules/routing/Makefile +++ b/server/modules/routing/Makefile @@ -44,13 +44,10 @@ DEBUGCLISRCS=debugcli.c debugcmd.c DEBUGCLIOBJ=$(DEBUGCLISRCS:.c=.o) CLISRCS=cli.c debugcmd.c CLIOBJ=$(CLISRCS:.c=.o) -WEBSRCS=webserver.o -WEBOBJ=$(WEBSRCS:.c=.o) SRCS=$(TESTSRCS) $(READCONSRCS) $(DEBUGCLISRCS) cli.c OBJ=$(SRCS:.c=.o) LIBS=$(UTILSPATH)/skygw_utils.o -lssl -llog_manager -MODULES= libdebugcli.so libreadconnroute.so libtestroute.so libcli.so \ - libwebserver.so +MODULES= libdebugcli.so libreadconnroute.so libtestroute.so libcli.so all: $(MODULES) @@ -67,9 +64,6 @@ libdebugcli.so: $(DEBUGCLIOBJ) libcli.so: $(CLIOBJ) $(CC) $(LDFLAGS) $(CLIOBJ) $(LIBS) -o $@ -libwebserver.so: $(WEBOBJ) - $(CC) $(LDFLAGS) $(WEBOBJ) $(LIBS) -o $@ - libreadwritesplit.so: # (cd readwritesplit; touch depend.mk ; make; cp $@ ..) diff --git a/server/modules/routing/webserver.c b/server/modules/routing/webserver.c deleted file mode 100644 index ee8516108..000000000 --- a/server/modules/routing/webserver.c +++ /dev/null @@ -1,493 +0,0 @@ -/* - * This file is distributed as part of MaxScale. It is free - * software: you can redistribute it and/or modify it under the terms of the - * GNU General Public License as published by the Free Software Foundation, - * version 2. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more - * details. - * - * You should have received a copy of the GNU General Public License along with - * this program; if not, write to the Free Software Foundation, Inc., 51 - * Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Copyright SkySQL Ab 2014 - */ -#include -#include -#include -#include -#include -#include -#include -#include - -typedef struct { - SERVICE *service; -} WEB_INSTANCE; - -typedef struct { - SESSION *session; -} WEB_SESSION; - -static char *version_str = "V1.0.0"; - -MODULE_INFO info = { - MODULE_API_ROUTER, - MODULE_IN_DEVELOPMENT, - ROUTER_VERSION, - "A test router - not for use in real systems" -}; - -static ROUTER *createInstance(SERVICE *service, char **options); -static void *newSession(ROUTER *instance, SESSION *session); -static void closeSession(ROUTER *instance, void *session); -static void freeSession(ROUTER *instance, void *session); -static int routeQuery(ROUTER *instance, void *session, GWBUF *queue); -static void diagnostic(ROUTER *instance, DCB *dcb); -static uint8_t getCapabilities (ROUTER* inst, void* router_session); - - -static ROUTER_OBJECT MyObject = { - createInstance, - newSession, - closeSession, - freeSession, - routeQuery, - diagnostic, - NULL, - NULL, - getCapabilities -}; - - -static void send_index(WEB_SESSION *); -static void send_css(WEB_SESSION *); -static void send_menu(WEB_SESSION *); -static void send_blank(WEB_SESSION *); -static void send_title(WEB_SESSION *); -static void send_frame1(WEB_SESSION *); -static void send_services(WEB_SESSION *); -static void send_sessions(WEB_SESSION *); -static void send_servers(WEB_SESSION *); -static void send_monitors(WEB_SESSION *); -static void respond_error(WEB_SESSION *, int, char *); - -static struct { - char *page; - void (*fcn)(WEB_SESSION *); -} pages[] = { - { "index.html", send_index }, - { "services.html", send_services }, - { "menu.html", send_menu }, - { "sessions.html", send_sessions }, - { "blank.html", send_blank }, - { "title.html", send_title }, - { "frame1.html", send_frame1 }, - { "servers.html", send_servers }, - { "monitors.html", send_monitors }, - { "styles.css", send_css }, - { NULL, NULL } -}; - -/** - * Implementation of the mandatory version entry point - * - * @return version string of the module - */ -char * -version() -{ - return version_str; -} - -/** - * The module initialisation routine, called when the module - * is first loaded. - */ -void -ModuleInit() -{ - -} - -/** - * The module entry point routine. It is this routine that - * must populate the structure that is referred to as the - * "module object", this is a structure with the set of - * external entry points for this module. - * - * @return The module object - */ -ROUTER_OBJECT * -GetModuleObject() -{ - return &MyObject; -} - -/** - * Create an instance of the router for a particular service - * within the gateway. - * - * @param service The service this router is being create for - * @param options The options for this query router - * - * @return The instance data for this new instance - */ -static ROUTER * -createInstance(SERVICE *service, char **options) -{ -WEB_INSTANCE *inst; - - if ((inst = (WEB_INSTANCE *)malloc(sizeof(WEB_INSTANCE))) == NULL) - return NULL; - - inst->service = service; - return (ROUTER *)inst; -} - -/** - * Associate a new session with this instance of the router. - * - * @param instance The router instance data - * @param session The session itself - * @return Session specific data for this session - */ -static void * -newSession(ROUTER *instance, SESSION *session) -{ -WEB_SESSION *wsession; - - if ((wsession = (WEB_SESSION *)malloc(sizeof(WEB_SESSION))) == NULL) - return NULL; - - wsession->session = session; - return wsession; -} - -/** - * Close a session with the router, this is the mechanism - * by which a router may cleanup data structure etc. - * - * @param instance The router instance data - * @param session The session being closed - */ -static void -closeSession(ROUTER *instance, void *session) -{ - free(session); -} - -static void freeSession( - ROUTER* router_instance, - void* router_client_session) -{ - return; -} - -static int -routeQuery(ROUTER *instance, void *session, GWBUF *queue) -{ -WEB_SESSION *wsession = (WEB_SESSION *)session; -char *ptr; -int i, found = 0; -char *url; - - if ((url = gwbuf_get_property(queue, "URL")) == NULL) - { - respond_error(wsession, 404, "No URL available"); - } - - ptr = strrchr(url, '/'); - if (ptr) - ptr++; - else - ptr = url; - for (i = 0; pages[i].page; i++) - { - if (!strcmp(ptr, pages[i].page)) - { - (pages[i].fcn)(wsession); - found = 1; - } - } - if (!found) - respond_error(wsession, 404, "Unrecognised URL received"); - gwbuf_free(queue); - return 0; -} - -/** - * Diagnostics routine - * - * @param instance The router instance - * @param dcb The DCB for diagnostic output - */ -static void -diagnostic(ROUTER *instance, DCB *dcb) -{ -} - -/** - * Return the router capabilities bitmask - * - * @param inst The router instance - * @param router_session The router session - * @return Router capabilities bitmask - */ -static uint8_t -getCapabilities(ROUTER *inst, void *router_session) -{ - return 0; -} - -static char *index_page = -"" -"" -"MaxScale" -"" -"" -"" -"" -"" -""; - -static char *title_page = -"" -"" -"MaxScale" -"" -"

MaxScale - Status View

" -""; - -static char *frame1_page = -"" -"" -"" -"" -"" -""; - -static char *menu_page = -"" -"" -"" -"

Options

" -"

"; - -static char *blank_page = " "; - -static char *css = -"table, td, th { border: 1px solid blue; }\n" -"th { background-color: blue; color: white; padding: 5px }\n" -"td { padding: 5px; }\n" -"table { border-collapse: collapse; }\n" -"a:link { color: #0000FF; }\n" -"a:visted { color: #0000FF; }\n" -"a:hover { color: #FF0000; }\n" -"a:active { color: #0000FF; }\n" -"h1 { color: blue; font-family: serif }\n" -"h2 { color: blue; font-family: serif }\n" -"p { font-family: serif }\n" -"li { font-family: serif }\n"; - - -static void -send_html_header(DCB *dcb) -{ -char date[64] = ""; -const char *fmt = "%a, %d %b %Y %H:%M:%S GMT"; - - time_t httpd_current_time = time(NULL); - - strftime(date, sizeof(date), fmt, localtime(&httpd_current_time)); - - dcb_printf(dcb, "HTTP/1.1 200 OK\r\nDate: %s\r\nServer: %s\r\nConnection: close\r\nContent-Type: text/html\r\n", date, "MaxScale"); - - dcb_printf(dcb, "\r\n"); -} - -static void -send_static_html(DCB *dcb, char *html) -{ - dcb_printf(dcb, html); -} - -static void -send_index(WEB_SESSION *session) -{ -DCB *dcb = session->session->client; - - send_html_header(dcb); - send_static_html(dcb, index_page); - dcb_close(dcb); -} - -static void -send_css(WEB_SESSION *session) -{ -DCB *dcb = session->session->client; - - send_html_header(dcb); - send_static_html(dcb, css); - dcb_close(dcb); -} - -static void -send_title(WEB_SESSION *session) -{ -DCB *dcb = session->session->client; - - send_html_header(dcb); - send_static_html(dcb, title_page); - dcb_close(dcb); -} - -static void -send_frame1(WEB_SESSION *session) -{ -DCB *dcb = session->session->client; - - send_html_header(dcb); - send_static_html(dcb, frame1_page); - dcb_close(dcb); -} - -static void -send_menu(WEB_SESSION *session) -{ -DCB *dcb = session->session->client; - - send_html_header(dcb); - send_static_html(dcb, menu_page); - dcb_close(dcb); -} - -static void -send_blank(WEB_SESSION *session) -{ -DCB *dcb = session->session->client; - - send_html_header(dcb); - send_static_html(dcb, blank_page); - dcb_close(dcb); -} - -static void -service_row(SERVICE *service, DCB *dcb) -{ - dcb_printf(dcb, "%s%s%d%d\n", - service->name, service->routerModule, - service->stats.n_current, service->stats.n_sessions); -} - -static void -send_services(WEB_SESSION *session) -{ -DCB *dcb = session->session->client; - - send_html_header(dcb); - dcb_printf(dcb, ""); - dcb_printf(dcb, ""); - dcb_printf(dcb, "

Services

"); - dcb_printf(dcb, "\n"); - serviceIterate(service_row, dcb); - dcb_printf(dcb, "
NameRouter"); - dcb_printf(dcb, "Current SessionsTotal Sessions
\n"); - dcb_close(dcb); -} - -static void -session_row(SESSION *session, DCB *dcb) -{ - dcb_printf(dcb, "%-16p%s%s%s\n", - session, ((session->client && session->client->remote) - ? session->client->remote : ""), - (session->service && session->service->name - ? session->service->name : ""), - session_state(session->state)); -} - -static void -send_sessions(WEB_SESSION *session) -{ -DCB *dcb = session->session->client; - - send_html_header(dcb); - dcb_printf(dcb, ""); - dcb_printf(dcb, ""); - dcb_printf(dcb, "

Sessions

"); - dcb_printf(dcb, "\n"); - sessionIterate(session_row, dcb); - dcb_printf(dcb, "
SessionClient"); - dcb_printf(dcb, "ServiceState
\n"); - dcb_close(dcb); -} - -static void -server_row(SERVER *server, DCB *dcb) -{ - dcb_printf(dcb, "%s%s%d%s%d\n", - server->unique_name, server->name, server->port, - server_status(server), server->stats.n_current); -} - -static void -send_servers(WEB_SESSION *session) -{ -DCB *dcb = session->session->client; - - send_html_header(dcb); - dcb_printf(dcb, ""); - dcb_printf(dcb, ""); - dcb_printf(dcb, "

Servers

"); - dcb_printf(dcb, "\n"); - serverIterate(server_row, dcb); - dcb_printf(dcb, "
ServerAddress"); - dcb_printf(dcb, "PortStateConnections
\n"); - dcb_close(dcb); -} - -static void -monitor_row(MONITOR *monitor, DCB *dcb) -{ - dcb_printf(dcb, "%s%s\n", - monitor->name, monitor->state & MONITOR_STATE_RUNNING - ? "Running" : "Stopped"); -} - -static void -send_monitors(WEB_SESSION *session) -{ -DCB *dcb = session->session->client; - - send_html_header(dcb); - dcb_printf(dcb, ""); - dcb_printf(dcb, ""); - dcb_printf(dcb, "

Monitors

"); - dcb_printf(dcb, "\n"); - monitorIterate(monitor_row, dcb); - dcb_printf(dcb, "
MonitorState
\n"); - dcb_close(dcb); -} - -static void -respond_error(WEB_SESSION *session, int err, char *msg) -{ -DCB *dcb = session->session->client; - - dcb_printf(dcb, "HTTP/1.1 %d %s\n", err, msg); - dcb_printf(dcb, "Content-Type: text/html\n"); - dcb_printf(dcb, "\n"); - dcb_printf(dcb, "\n"); - dcb_printf(dcb, "MaxScale webserver plugin unable to satisfy request.\n"); - dcb_printf(dcb, "

Code: %d, %s\n", err, msg); - dcb_printf(dcb, ""); - dcb_close(dcb); -} From 4e720c74e870b6b2b3207fb510603c42d2ffcefa Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Tue, 29 Jul 2014 22:16:03 +0300 Subject: [PATCH 02/60] Bug 462, http://bugs.skysql.com/show_bug.cgi?id=462, fixed path MAXSCALE_HOME path expression in server/test/makefile --- server/test/makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/test/makefile b/server/test/makefile index d812494fc..7f9c36b84 100644 --- a/server/test/makefile +++ b/server/test/makefile @@ -10,7 +10,7 @@ include ../../test.inc TEST_ROOT := $(ROOT_PATH)/test PARENT_DIR := $(ROOT_PATH)/server CUR_DIR := $(PARENT_DIR)/test -export MAXSCALE_HOME=$(CUR_DIR)/MaxScale +export MAXSCALE_HOME=$(CUR_DIR) CC=cc TESTLOG := $(CUR_DIR)/testserver.log From 3585054b35b933a6cee37281a0a3920c6f1a9594 Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Tue, 29 Jul 2014 22:28:44 +0300 Subject: [PATCH 03/60] Bug 462, http://bugs.skysql.com/show_bug.cgi?id=462, fixed path MAXSCALE_HOME path expression in server/test/makefile --- server/test/makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/test/makefile b/server/test/makefile index d812494fc..7f9c36b84 100644 --- a/server/test/makefile +++ b/server/test/makefile @@ -10,7 +10,7 @@ include ../../test.inc TEST_ROOT := $(ROOT_PATH)/test PARENT_DIR := $(ROOT_PATH)/server CUR_DIR := $(PARENT_DIR)/test -export MAXSCALE_HOME=$(CUR_DIR)/MaxScale +export MAXSCALE_HOME=$(CUR_DIR) CC=cc TESTLOG := $(CUR_DIR)/testserver.log From 7a039e1a0bce37433cfab0e3ab3d0ba8a7a6b4ae Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Wed, 30 Jul 2014 20:40:08 +0300 Subject: [PATCH 04/60] Added check for session state before calling router->closeSession --- server/modules/protocol/mysql_client.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/server/modules/protocol/mysql_client.c b/server/modules/protocol/mysql_client.c index 1db23b4fe..db61084eb 100644 --- a/server/modules/protocol/mysql_client.c +++ b/server/modules/protocol/mysql_client.c @@ -1331,12 +1331,25 @@ gw_client_close(DCB *dcb) * session may be NULL if session_alloc failed. * In that case, router session wasn't created. */ - if (session != NULL) { + if (session != NULL) + { CHK_SESSION(session); spinlock_acquire(&session->ses_lock); - session->state = SESSION_STATE_STOPPING; - spinlock_release(&session->ses_lock); + if (session->state == SESSION_STATE_STOPPING) + { + /** + * Session is already getting closed so avoid + * redundant calls + */ + spinlock_release(&session->ses_lock); + return 1; + } + else + { + session->state = SESSION_STATE_STOPPING; + spinlock_release(&session->ses_lock); + } router = session->service->router; router_instance = session->service->router_instance; rsession = session->router_session; From 6b3c7041e3880fbdec8dff2de6d68806d123086d Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Tue, 5 Aug 2014 09:31:10 +0300 Subject: [PATCH 05/60] Bug #468, http://bugs.skysql.com/show_bug.cgi?id=468, Query classifier accessed freed thread context. If parsing fails thd doesn't need to be freed because it holds correct information about command type. session.c:session_setup_filters : fixed memory leak hintparser.c: added token_free for HINT_TOKENs and fixed a few memory leaks. mysql_client_server_protocol.h: added mysql_protocol_done which frees memory blocks pointed to by protocol members. Those can't be freed in dcb.c because dcb.c doesn't know about protocol's members. mysql_backend.c:gw_backend_close: fixed memory leak mysql_client.c: gw_client_close: fixed memory leak mysql_common.c: added implementation of mysql_protocol_done :protocol_archive_srv_command: tried to fix memory leak. Some memory is still leaking according to valgrind. Removed use of uninitialized local variable len. readwritesplit.c:execute_sescmd_in_backend: fixed a memory leak - visible only in DEBUG=Y build. --- query_classifier/query_classifier.cc | 16 +++++----- server/core/session.c | 1 + .../include/mysql_client_server_protocol.h | 1 + server/modules/protocol/mysql_backend.c | 2 ++ server/modules/protocol/mysql_client.c | 1 + server/modules/protocol/mysql_common.c | 31 ++++++++++++++++--- .../routing/readwritesplit/readwritesplit.c | 19 +++++++----- 7 files changed, 52 insertions(+), 19 deletions(-) diff --git a/query_classifier/query_classifier.cc b/query_classifier/query_classifier.cc index eccc35a31..889940e00 100644 --- a/query_classifier/query_classifier.cc +++ b/query_classifier/query_classifier.cc @@ -149,17 +149,17 @@ skygw_query_type_t skygw_query_classifier_get_type( thd = get_or_create_thd_for_parsing(mysql, query_str); if (thd == NULL) - { - skygw_query_classifier_free(mysql); - } - /** Create parse_tree inside thd */ - failp = create_parse_tree(thd); - - if (failp) { skygw_query_classifier_free(mysql); *p_mysql = NULL; + goto return_qtype; } + /** + * Create parse_tree inside thd. + * thd and even lex are readable even if parser failed so let it + * continue despite failure. + */ + failp = create_parse_tree(thd); qtype = resolve_query_type(thd); if (p_mysql == NULL) @@ -464,7 +464,7 @@ static skygw_query_type_t resolve_query_type( type |= QUERY_TYPE_DISABLE_AUTOCOMMIT; type |= QUERY_TYPE_BEGIN_TRX; } - /** + /** * REVOKE ALL, ASSIGN_TO_KEYCACHE, * PRELOAD_KEYS, FLUSH, RESET, CREATE|ALTER|DROP SERVER */ diff --git a/server/core/session.c b/server/core/session.c index 80767f9ff..62e1015e4 100644 --- a/server/core/session.c +++ b/server/core/session.c @@ -667,6 +667,7 @@ int i; session->filters[i].session = head->session; session->filters[i].instance = head->instance; session->head = *head; + free(head); } for (i = 0; i < service->n_filters; i++) diff --git a/server/modules/include/mysql_client_server_protocol.h b/server/modules/include/mysql_client_server_protocol.h index f7d6030c1..9e2147d05 100644 --- a/server/modules/include/mysql_client_server_protocol.h +++ b/server/modules/include/mysql_client_server_protocol.h @@ -299,6 +299,7 @@ typedef struct { void gw_mysql_close(MySQLProtocol **ptr); MySQLProtocol* mysql_protocol_init(DCB* dcb, int fd); +void mysql_protocol_done (DCB* dcb); MySQLProtocol *gw_mysql_init(MySQLProtocol *data); void gw_mysql_close(MySQLProtocol **ptr); int gw_receive_backend_auth(MySQLProtocol *protocol); diff --git a/server/modules/protocol/mysql_backend.c b/server/modules/protocol/mysql_backend.c index 43cd97bcd..5d8088d4e 100644 --- a/server/modules/protocol/mysql_backend.c +++ b/server/modules/protocol/mysql_backend.c @@ -996,6 +996,8 @@ gw_backend_close(DCB *dcb) /** Send COM_QUIT to the backend being closed */ mysql_send_com_quit(dcb, 0, quitbuf); + + mysql_protocol_done(dcb); if (session != NULL && session->state == SESSION_STATE_STOPPING) { diff --git a/server/modules/protocol/mysql_client.c b/server/modules/protocol/mysql_client.c index db61084eb..6cc9027de 100644 --- a/server/modules/protocol/mysql_client.c +++ b/server/modules/protocol/mysql_client.c @@ -1325,6 +1325,7 @@ gw_client_close(DCB *dcb) CHK_PROTOCOL(protocol); } #endif + mysql_protocol_done(dcb); session = dcb->session; /** diff --git a/server/modules/protocol/mysql_common.c b/server/modules/protocol/mysql_common.c index 4ce34452b..70050a13c 100644 --- a/server/modules/protocol/mysql_common.c +++ b/server/modules/protocol/mysql_common.c @@ -96,6 +96,31 @@ return_p: } +/** + * mysql_protocol_done + * + * free protocol allocations. + * + * @param dcb owner DCB + * + */ +void mysql_protocol_done ( + DCB* dcb) +{ + server_command_t* scmd = ((MySQLProtocol *)dcb->protocol)->protocol_cmd_history; + server_command_t* scmd2; + + while (scmd != NULL) + { + scmd2 = scmd->scom_next; + free(scmd); + scmd = scmd2; + } +} + + + + /** * gw_mysql_close * @@ -1601,7 +1626,7 @@ void protocol_archive_srv_command( { server_command_t* s1; server_command_t** s2; - int len; + int len = 0; spinlock_acquire(&p->protocol_lock); @@ -1617,9 +1642,7 @@ void protocol_archive_srv_command( s2 = &p->protocol_cmd_history; if (*s2 != NULL) - { - len = 0; - + { while ((*s2)->scom_next != NULL) { *s2 = (*s2)->scom_next; diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index 90132a9f9..7fcf22974 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -1132,6 +1132,8 @@ static int routeQuery( break; case MYSQL_COM_STMT_EXECUTE: + /** Parsing is not needed for this type of packet */ +#if defined(NOT_USED) plainsqlbuf = gwbuf_clone_transform(querybuf, GWBUF_TYPE_PLAINSQL); len = GWBUF_LENGTH(plainsqlbuf); @@ -1140,7 +1142,8 @@ static int routeQuery( memcpy(querystr, startpos, len); memset(&querystr[len], 0, 1); qtype = skygw_query_classifier_get_type(querystr, 0, &mysql); - qtype |= QUERY_TYPE_EXEC_STMT; +#endif + qtype = QUERY_TYPE_EXEC_STMT; break; case MYSQL_COM_SHUTDOWN: /**< 8 where should shutdown be routed ? */ @@ -1923,7 +1926,8 @@ static bool select_connect_backend_servers( } } } - } + } /*< log only */ + /** * Choose at least 1+min_nslaves (master and slave) and at most 1+max_nslaves * servers from the sorted list. First master found is selected. @@ -2666,8 +2670,9 @@ static bool execute_sescmd_in_backend( pthread_self(), dcb->fd, STRPACKETTYPE(cmd)))); + gwbuf_free(tmpbuf); } -#endif +#endif /*< SS_DEBUG */ switch (scur->scmd_cur_cmd->my_sescmd_packet_type) { case MYSQL_COM_CHANGE_USER: rc = dcb->func.auth( @@ -3065,7 +3070,7 @@ static bool router_option_configured( } return succp; } -#endif +#endif /*< NOT_USED */ static void rwsplit_process_router_options( ROUTER_INSTANCE* router, @@ -3346,7 +3351,7 @@ static void print_error_packet( { while ((buf = gwbuf_consume(buf, GWBUF_LENGTH(buf))) != NULL); } -#endif +#endif /*< SS_DEBUG */ } static int router_get_servercount( @@ -3568,10 +3573,10 @@ static prep_stmt_t* prep_stmt_init( if (pstmt != NULL) { - #if defined(SS_DEBUG) +#if defined(SS_DEBUG) pstmt->pstmt_chk_top = CHK_NUM_PREP_STMT; pstmt->pstmt_chk_tail = CHK_NUM_PREP_STMT; - #endif +#endif pstmt->pstmt_state = PREP_STMT_ALLOC; pstmt->pstmt_type = type; From 7558abb6faca935f2bed5a54f08125432e73e4ca Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Tue, 5 Aug 2014 10:42:13 +0300 Subject: [PATCH 06/60] Bug #468, http://bugs.skysql.com/show_bug.cgi?id=468, Query classifier accessed freed thread context. If parsing fails thd doesn't need to be freed because it holds correct information about command type. session.c:session_setup_filters : fixed memory leak hintparser.c: added token_free for HINT_TOKENs and fixed a few memory leaks. mysql_client_server_protocol.h: added mysql_protocol_done which frees memory blocks pointed to by protocol members. Those can't be freed in dcb.c because dcb.c doesn't know about protocol's members. mysql_backend.c:gw_backend_close: fixed memory leak mysql_client.c: gw_client_close: fixed memory leak mysql_common.c: added implementation of mysql_protocol_done :protocol_archive_srv_command: tried to fix memory leak. Some memory is still leaking according to valgrind. Removed use of uninitialized local variable len. readwritesplit.c:execute_sescmd_in_backend: fixed a memory leak - visible only in DEBUG=Y build. --- query_classifier/query_classifier.cc | 16 +++++----- server/core/session.c | 1 + .../include/mysql_client_server_protocol.h | 1 + server/modules/protocol/mysql_backend.c | 2 ++ server/modules/protocol/mysql_client.c | 1 + server/modules/protocol/mysql_common.c | 31 ++++++++++++++++--- .../routing/readwritesplit/readwritesplit.c | 19 +++++++----- 7 files changed, 52 insertions(+), 19 deletions(-) diff --git a/query_classifier/query_classifier.cc b/query_classifier/query_classifier.cc index eccc35a31..889940e00 100644 --- a/query_classifier/query_classifier.cc +++ b/query_classifier/query_classifier.cc @@ -149,17 +149,17 @@ skygw_query_type_t skygw_query_classifier_get_type( thd = get_or_create_thd_for_parsing(mysql, query_str); if (thd == NULL) - { - skygw_query_classifier_free(mysql); - } - /** Create parse_tree inside thd */ - failp = create_parse_tree(thd); - - if (failp) { skygw_query_classifier_free(mysql); *p_mysql = NULL; + goto return_qtype; } + /** + * Create parse_tree inside thd. + * thd and even lex are readable even if parser failed so let it + * continue despite failure. + */ + failp = create_parse_tree(thd); qtype = resolve_query_type(thd); if (p_mysql == NULL) @@ -464,7 +464,7 @@ static skygw_query_type_t resolve_query_type( type |= QUERY_TYPE_DISABLE_AUTOCOMMIT; type |= QUERY_TYPE_BEGIN_TRX; } - /** + /** * REVOKE ALL, ASSIGN_TO_KEYCACHE, * PRELOAD_KEYS, FLUSH, RESET, CREATE|ALTER|DROP SERVER */ diff --git a/server/core/session.c b/server/core/session.c index 80767f9ff..62e1015e4 100644 --- a/server/core/session.c +++ b/server/core/session.c @@ -667,6 +667,7 @@ int i; session->filters[i].session = head->session; session->filters[i].instance = head->instance; session->head = *head; + free(head); } for (i = 0; i < service->n_filters; i++) diff --git a/server/modules/include/mysql_client_server_protocol.h b/server/modules/include/mysql_client_server_protocol.h index f7d6030c1..9e2147d05 100644 --- a/server/modules/include/mysql_client_server_protocol.h +++ b/server/modules/include/mysql_client_server_protocol.h @@ -299,6 +299,7 @@ typedef struct { void gw_mysql_close(MySQLProtocol **ptr); MySQLProtocol* mysql_protocol_init(DCB* dcb, int fd); +void mysql_protocol_done (DCB* dcb); MySQLProtocol *gw_mysql_init(MySQLProtocol *data); void gw_mysql_close(MySQLProtocol **ptr); int gw_receive_backend_auth(MySQLProtocol *protocol); diff --git a/server/modules/protocol/mysql_backend.c b/server/modules/protocol/mysql_backend.c index 43cd97bcd..5d8088d4e 100644 --- a/server/modules/protocol/mysql_backend.c +++ b/server/modules/protocol/mysql_backend.c @@ -996,6 +996,8 @@ gw_backend_close(DCB *dcb) /** Send COM_QUIT to the backend being closed */ mysql_send_com_quit(dcb, 0, quitbuf); + + mysql_protocol_done(dcb); if (session != NULL && session->state == SESSION_STATE_STOPPING) { diff --git a/server/modules/protocol/mysql_client.c b/server/modules/protocol/mysql_client.c index db61084eb..6cc9027de 100644 --- a/server/modules/protocol/mysql_client.c +++ b/server/modules/protocol/mysql_client.c @@ -1325,6 +1325,7 @@ gw_client_close(DCB *dcb) CHK_PROTOCOL(protocol); } #endif + mysql_protocol_done(dcb); session = dcb->session; /** diff --git a/server/modules/protocol/mysql_common.c b/server/modules/protocol/mysql_common.c index 4ce34452b..70050a13c 100644 --- a/server/modules/protocol/mysql_common.c +++ b/server/modules/protocol/mysql_common.c @@ -96,6 +96,31 @@ return_p: } +/** + * mysql_protocol_done + * + * free protocol allocations. + * + * @param dcb owner DCB + * + */ +void mysql_protocol_done ( + DCB* dcb) +{ + server_command_t* scmd = ((MySQLProtocol *)dcb->protocol)->protocol_cmd_history; + server_command_t* scmd2; + + while (scmd != NULL) + { + scmd2 = scmd->scom_next; + free(scmd); + scmd = scmd2; + } +} + + + + /** * gw_mysql_close * @@ -1601,7 +1626,7 @@ void protocol_archive_srv_command( { server_command_t* s1; server_command_t** s2; - int len; + int len = 0; spinlock_acquire(&p->protocol_lock); @@ -1617,9 +1642,7 @@ void protocol_archive_srv_command( s2 = &p->protocol_cmd_history; if (*s2 != NULL) - { - len = 0; - + { while ((*s2)->scom_next != NULL) { *s2 = (*s2)->scom_next; diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index 90132a9f9..7fcf22974 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -1132,6 +1132,8 @@ static int routeQuery( break; case MYSQL_COM_STMT_EXECUTE: + /** Parsing is not needed for this type of packet */ +#if defined(NOT_USED) plainsqlbuf = gwbuf_clone_transform(querybuf, GWBUF_TYPE_PLAINSQL); len = GWBUF_LENGTH(plainsqlbuf); @@ -1140,7 +1142,8 @@ static int routeQuery( memcpy(querystr, startpos, len); memset(&querystr[len], 0, 1); qtype = skygw_query_classifier_get_type(querystr, 0, &mysql); - qtype |= QUERY_TYPE_EXEC_STMT; +#endif + qtype = QUERY_TYPE_EXEC_STMT; break; case MYSQL_COM_SHUTDOWN: /**< 8 where should shutdown be routed ? */ @@ -1923,7 +1926,8 @@ static bool select_connect_backend_servers( } } } - } + } /*< log only */ + /** * Choose at least 1+min_nslaves (master and slave) and at most 1+max_nslaves * servers from the sorted list. First master found is selected. @@ -2666,8 +2670,9 @@ static bool execute_sescmd_in_backend( pthread_self(), dcb->fd, STRPACKETTYPE(cmd)))); + gwbuf_free(tmpbuf); } -#endif +#endif /*< SS_DEBUG */ switch (scur->scmd_cur_cmd->my_sescmd_packet_type) { case MYSQL_COM_CHANGE_USER: rc = dcb->func.auth( @@ -3065,7 +3070,7 @@ static bool router_option_configured( } return succp; } -#endif +#endif /*< NOT_USED */ static void rwsplit_process_router_options( ROUTER_INSTANCE* router, @@ -3346,7 +3351,7 @@ static void print_error_packet( { while ((buf = gwbuf_consume(buf, GWBUF_LENGTH(buf))) != NULL); } -#endif +#endif /*< SS_DEBUG */ } static int router_get_servercount( @@ -3568,10 +3573,10 @@ static prep_stmt_t* prep_stmt_init( if (pstmt != NULL) { - #if defined(SS_DEBUG) +#if defined(SS_DEBUG) pstmt->pstmt_chk_top = CHK_NUM_PREP_STMT; pstmt->pstmt_chk_tail = CHK_NUM_PREP_STMT; - #endif +#endif pstmt->pstmt_state = PREP_STMT_ALLOC; pstmt->pstmt_type = type; From 4ef1df4e9d8ec6e4b120b2f059ecc2f3ab931b93 Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Tue, 5 Aug 2014 13:44:04 +0300 Subject: [PATCH 07/60] Bug #468, http://bugs.skysql.com/show_bug.cgi?id=468, Query classifier accessed freed thread context. If parsing fails thd doesn't need to be freed because it holds correct information about command type. --- query_classifier/query_classifier.cc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/query_classifier/query_classifier.cc b/query_classifier/query_classifier.cc index eccc35a31..889940e00 100644 --- a/query_classifier/query_classifier.cc +++ b/query_classifier/query_classifier.cc @@ -149,17 +149,17 @@ skygw_query_type_t skygw_query_classifier_get_type( thd = get_or_create_thd_for_parsing(mysql, query_str); if (thd == NULL) - { - skygw_query_classifier_free(mysql); - } - /** Create parse_tree inside thd */ - failp = create_parse_tree(thd); - - if (failp) { skygw_query_classifier_free(mysql); *p_mysql = NULL; + goto return_qtype; } + /** + * Create parse_tree inside thd. + * thd and even lex are readable even if parser failed so let it + * continue despite failure. + */ + failp = create_parse_tree(thd); qtype = resolve_query_type(thd); if (p_mysql == NULL) @@ -464,7 +464,7 @@ static skygw_query_type_t resolve_query_type( type |= QUERY_TYPE_DISABLE_AUTOCOMMIT; type |= QUERY_TYPE_BEGIN_TRX; } - /** + /** * REVOKE ALL, ASSIGN_TO_KEYCACHE, * PRELOAD_KEYS, FLUSH, RESET, CREATE|ALTER|DROP SERVER */ From dbfaa5a8eaa1d62def7dce5503ca24faefe46cd2 Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Tue, 5 Aug 2014 16:28:26 +0300 Subject: [PATCH 08/60] Fix to http://bugs.skysql.com/show_bug.cgi?id=469, connection counter leaks in master. Removed redundant counter increments. --- server/modules/routing/readwritesplit/readwritesplit.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index 7fcf22974..218972ab4 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -381,7 +381,7 @@ static void refreshInstance( * used in slave selection. */ if (!rlag_enabled) - { + { if (rlag_limited) { LOGIF(LE, (skygw_log_write_flush( @@ -1985,7 +1985,7 @@ static bool select_connect_backend_servers( backend_ref[i].bref_state = 0; bref_set_state(&backend_ref[i], BREF_IN_USE); - /** + /** * Increase backend connection counter. * Server's stats are _increased_ in * dcb.c:dcb_alloc ! @@ -2023,7 +2023,7 @@ static bool select_connect_backend_servers( session, b->backend_server->protocol); - if (backend_ref[i].bref_dcb != NULL) + if (backend_ref[i].bref_dcb != NULL) { master_connected = true; /** @@ -2040,8 +2040,6 @@ static bool select_connect_backend_servers( bref_set_state(&backend_ref[i], BREF_IN_USE); /** Increase backend connection counters */ - atomic_add(&b->backend_server->stats.n_current, 1); - atomic_add(&b->backend_server->stats.n_connections, 1); atomic_add(&b->backend_conn_count, 1); } else From 97ab902ede3e85ac353da354fd96d90e0844738d Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Tue, 5 Aug 2014 16:31:39 +0300 Subject: [PATCH 09/60] Fix to http://bugs.skysql.com/show_bug.cgi?id=469, connection counter leaks in master. Removed redundant counter increments. --- server/modules/routing/readwritesplit/readwritesplit.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index 7fcf22974..9d4b1b8ff 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -381,7 +381,7 @@ static void refreshInstance( * used in slave selection. */ if (!rlag_enabled) - { + { if (rlag_limited) { LOGIF(LE, (skygw_log_write_flush( @@ -1985,7 +1985,7 @@ static bool select_connect_backend_servers( backend_ref[i].bref_state = 0; bref_set_state(&backend_ref[i], BREF_IN_USE); - /** + /** * Increase backend connection counter. * Server's stats are _increased_ in * dcb.c:dcb_alloc ! @@ -2040,8 +2040,6 @@ static bool select_connect_backend_servers( bref_set_state(&backend_ref[i], BREF_IN_USE); /** Increase backend connection counters */ - atomic_add(&b->backend_server->stats.n_current, 1); - atomic_add(&b->backend_server->stats.n_connections, 1); atomic_add(&b->backend_conn_count, 1); } else From f738b2e0cbd272cf9eee6b1babf63533640234f9 Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Tue, 5 Aug 2014 16:38:00 +0300 Subject: [PATCH 10/60] Fix to http://bugs.skysql.com/show_bug.cgi?id=469, connection counter leaks in master. Removed redundant counter increments. --- .../routing/readwritesplit/readwritesplit.c | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index 90132a9f9..9d4b1b8ff 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -381,7 +381,7 @@ static void refreshInstance( * used in slave selection. */ if (!rlag_enabled) - { + { if (rlag_limited) { LOGIF(LE, (skygw_log_write_flush( @@ -1132,6 +1132,8 @@ static int routeQuery( break; case MYSQL_COM_STMT_EXECUTE: + /** Parsing is not needed for this type of packet */ +#if defined(NOT_USED) plainsqlbuf = gwbuf_clone_transform(querybuf, GWBUF_TYPE_PLAINSQL); len = GWBUF_LENGTH(plainsqlbuf); @@ -1140,7 +1142,8 @@ static int routeQuery( memcpy(querystr, startpos, len); memset(&querystr[len], 0, 1); qtype = skygw_query_classifier_get_type(querystr, 0, &mysql); - qtype |= QUERY_TYPE_EXEC_STMT; +#endif + qtype = QUERY_TYPE_EXEC_STMT; break; case MYSQL_COM_SHUTDOWN: /**< 8 where should shutdown be routed ? */ @@ -1923,7 +1926,8 @@ static bool select_connect_backend_servers( } } } - } + } /*< log only */ + /** * Choose at least 1+min_nslaves (master and slave) and at most 1+max_nslaves * servers from the sorted list. First master found is selected. @@ -1981,7 +1985,7 @@ static bool select_connect_backend_servers( backend_ref[i].bref_state = 0; bref_set_state(&backend_ref[i], BREF_IN_USE); - /** + /** * Increase backend connection counter. * Server's stats are _increased_ in * dcb.c:dcb_alloc ! @@ -2036,8 +2040,6 @@ static bool select_connect_backend_servers( bref_set_state(&backend_ref[i], BREF_IN_USE); /** Increase backend connection counters */ - atomic_add(&b->backend_server->stats.n_current, 1); - atomic_add(&b->backend_server->stats.n_connections, 1); atomic_add(&b->backend_conn_count, 1); } else @@ -2666,8 +2668,9 @@ static bool execute_sescmd_in_backend( pthread_self(), dcb->fd, STRPACKETTYPE(cmd)))); + gwbuf_free(tmpbuf); } -#endif +#endif /*< SS_DEBUG */ switch (scur->scmd_cur_cmd->my_sescmd_packet_type) { case MYSQL_COM_CHANGE_USER: rc = dcb->func.auth( @@ -3065,7 +3068,7 @@ static bool router_option_configured( } return succp; } -#endif +#endif /*< NOT_USED */ static void rwsplit_process_router_options( ROUTER_INSTANCE* router, @@ -3346,7 +3349,7 @@ static void print_error_packet( { while ((buf = gwbuf_consume(buf, GWBUF_LENGTH(buf))) != NULL); } -#endif +#endif /*< SS_DEBUG */ } static int router_get_servercount( @@ -3568,10 +3571,10 @@ static prep_stmt_t* prep_stmt_init( if (pstmt != NULL) { - #if defined(SS_DEBUG) +#if defined(SS_DEBUG) pstmt->pstmt_chk_top = CHK_NUM_PREP_STMT; pstmt->pstmt_chk_tail = CHK_NUM_PREP_STMT; - #endif +#endif pstmt->pstmt_state = PREP_STMT_ALLOC; pstmt->pstmt_type = type; From d3a79ce7c40167d90a0eade6404ea477a9b85828 Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Thu, 14 Aug 2014 13:05:05 +0300 Subject: [PATCH 11/60] Partial fix to #463, http://bugs.skysql.com/show_bug.cgi?id=463 log_manager.cc: fixed block buffer overflow. Queries are logged to trace log and long queries exceed the bufsize in length. Those were written beyond allocated memory areas. mysql_client_server_protocol.h: added mysql_protocol_state_t to indicate whether MySQL protocol object is allocated, usable or freed. Freed means that memory allocations made by the protocol are freed. That is, command history etc. mysql_backend.c: gw_backend_hangup and gw_error_backend_event used to call error handling function although session was already closing. Added check for session state. mysql_client.c: route_by_statement lost some packets in case where query was sent in multiple packets. mysql_common.c: gw_MySQL_get_next_packet failed in packet handling with route_by_statement. When multi-packet query was merged into one, packet type wasn't copied. protocol_archive_srv_command and mysql_protocol_done didn't have proper locking in place which lead to occasional crashes. --- log_manager/log_manager.cc | 26 +++--- server/include/dcb.h | 4 +- .../include/mysql_client_server_protocol.h | 7 ++ server/modules/protocol/mysql_backend.c | 79 +++++++++++++++---- server/modules/protocol/mysql_client.c | 20 ++--- server/modules/protocol/mysql_common.c | 56 +++++++++---- .../routing/readwritesplit/readwritesplit.c | 2 +- 7 files changed, 137 insertions(+), 57 deletions(-) diff --git a/log_manager/log_manager.cc b/log_manager/log_manager.cc index c8f96266e..733e9fffe 100644 --- a/log_manager/log_manager.cc +++ b/log_manager/log_manager.cc @@ -255,11 +255,7 @@ static int logmanager_write_log( static blockbuf_t* blockbuf_init(logfile_id_t id); static void blockbuf_node_done(void* bb_data); static char* blockbuf_get_writepos( -#if 0 - int** refcount, -#else blockbuf_t** p_bb, -#endif logfile_id_t id, size_t str_len, bool flush); @@ -653,7 +649,16 @@ static int logmanager_write_log( int safe_str_len; timestamp_len = get_timestamp_len(); - safe_str_len = MIN(timestamp_len-1+str_len, lf->lf_buf_size); + + /** Findout how much can be safely written with current block size */ + if (timestamp_len-1+str_len > lf->lf_buf_size) + { + safe_str_len = lf->lf_buf_size; + } + else + { + safe_str_len = timestamp_len-1+str_len; + } /** * Seek write position and register to block buffer. * Then print formatted string to write position. @@ -673,9 +678,9 @@ static int logmanager_write_log( * of the timestamp string. */ if (use_valist) { - vsnprintf(wp+timestamp_len, safe_str_len, str, valist); + vsnprintf(wp+timestamp_len, safe_str_len-timestamp_len, str, valist); } else { - snprintf(wp+timestamp_len, safe_str_len, "%s", str); + snprintf(wp+timestamp_len, safe_str_len-timestamp_len, "%s", str); } /** write to syslog */ @@ -694,12 +699,7 @@ static int logmanager_write_log( break; } } - - /** remove double line feed */ - if (wp[timestamp_len+str_len-2] == '\n') { - wp[timestamp_len+str_len-2]=' '; - } - wp[timestamp_len+str_len-1]='\n'; + wp[safe_str_len-1] = '\n'; blockbuf_unregister(bb); /** diff --git a/server/include/dcb.h b/server/include/dcb.h index 88964fc31..4f8be99d4 100644 --- a/server/include/dcb.h +++ b/server/include/dcb.h @@ -145,9 +145,9 @@ typedef enum { DCB_STATE_POLLING, /*< Waiting in the poll loop */ DCB_STATE_LISTENING, /*< The DCB is for a listening socket */ DCB_STATE_DISCONNECTED, /*< The socket is now closed */ - DCB_STATE_FREED, /*< Memory freed */ DCB_STATE_NOPOLLING, /*< Removed from poll mask */ - DCB_STATE_ZOMBIE /*< DCB is no longer active, waiting to free it */ + DCB_STATE_ZOMBIE, /*< DCB is no longer active, waiting to free it */ + DCB_STATE_FREED /*< Memory freed */ } dcb_state_t; typedef enum { diff --git a/server/modules/include/mysql_client_server_protocol.h b/server/modules/include/mysql_client_server_protocol.h index 9e2147d05..c875a1232 100644 --- a/server/modules/include/mysql_client_server_protocol.h +++ b/server/modules/include/mysql_client_server_protocol.h @@ -102,6 +102,12 @@ typedef enum { MYSQL_IDLE } mysql_auth_state_t; +typedef enum { + MYSQL_PROTOCOL_ALLOC, + MYSQL_PROTOCOL_ACTIVE, + MYSQL_PROTOCOL_DONE +} mysql_protocol_state_t; + /* * MySQL session specific data @@ -270,6 +276,7 @@ typedef struct { server_command_t protocol_command; /*< session command list */ server_command_t* protocol_cmd_history; /*< session command history */ mysql_auth_state_t protocol_auth_state; /*< Authentication status */ + mysql_protocol_state_t protocol_state; /*< Protocol struct status */ uint8_t scramble[MYSQL_SCRAMBLE_LEN]; /*< server scramble, * created or received */ uint32_t server_capabilities; /*< server capabilities, diff --git a/server/modules/protocol/mysql_backend.c b/server/modules/protocol/mysql_backend.c index 5d8088d4e..903c17022 100644 --- a/server/modules/protocol/mysql_backend.c +++ b/server/modules/protocol/mysql_backend.c @@ -762,12 +762,13 @@ return_rc: */ static int gw_error_backend_event(DCB *dcb) { - SESSION* session; - void* rsession; - ROUTER_OBJECT* router; - ROUTER* router_instance; - GWBUF* errbuf; - bool succp; + SESSION* session; + void* rsession; + ROUTER_OBJECT* router; + ROUTER* router_instance; + GWBUF* errbuf; + bool succp; + session_state_t ses_state; CHK_DCB(dcb); session = dcb->session; @@ -787,7 +788,7 @@ static int gw_error_backend_event(DCB *dcb) * closed by router and COM_QUIT sent or there was an error which * have already been handled. */ - if (dcb->session != DCB_STATE_POLLING) + if (dcb->state != DCB_STATE_POLLING) { return 1; } @@ -796,6 +797,29 @@ static int gw_error_backend_event(DCB *dcb) 0, "Lost connection to backend server."); + spinlock_acquire(&session->ses_lock); + ses_state = session->state; + spinlock_release(&session->ses_lock); + + /** + * Session might be initialized when DCB already is in the poll set. + * Thus hangup can occur in the middle of session initialization. + * Only complete and successfully initialized sessions allow for + * calling error handler. + */ + while (ses_state == SESSION_STATE_READY) + { + spinlock_acquire(&session->ses_lock); + ses_state = session->state; + spinlock_release(&session->ses_lock); + } + + if (ses_state != SESSION_STATE_ROUTER_READY) + { + gwbuf_free(errbuf); + goto retblock; + } + router->handleError(router_instance, rsession, errbuf, @@ -811,6 +835,7 @@ static int gw_error_backend_event(DCB *dcb) } dcb_close(dcb); +retblock: return 1; } @@ -924,12 +949,13 @@ return_fd: static int gw_backend_hangup(DCB *dcb) { - SESSION* session; - void* rsession; - ROUTER_OBJECT* router; - ROUTER* router_instance; - bool succp; - GWBUF* errbuf; + SESSION* session; + void* rsession; + ROUTER_OBJECT* router; + ROUTER* router_instance; + bool succp; + GWBUF* errbuf; + session_state_t ses_state; CHK_DCB(dcb); session = dcb->session; @@ -950,7 +976,29 @@ gw_backend_hangup(DCB *dcb) 1, 0, "Lost connection to backend server."); - + + spinlock_acquire(&session->ses_lock); + ses_state = session->state; + spinlock_release(&session->ses_lock); + + /** + * Session might be initialized when DCB already is in the poll set. + * Thus hangup can occur in the middle of session initialization. + * Only complete and successfully initialized sessions allow for + * calling error handler. + */ + while (ses_state == SESSION_STATE_READY) + { + spinlock_acquire(&session->ses_lock); + ses_state = session->state; + spinlock_release(&session->ses_lock); + } + + if (ses_state != SESSION_STATE_ROUTER_READY) + { + gwbuf_free(errbuf); + goto retblock; + } router->handleError(router_instance, rsession, errbuf, @@ -972,7 +1020,8 @@ gw_backend_hangup(DCB *dcb) } dcb_close(dcb); - return 1; +retblock: + return 1; } /** diff --git a/server/modules/protocol/mysql_client.c b/server/modules/protocol/mysql_client.c index 6cc9027de..fbe0589a5 100644 --- a/server/modules/protocol/mysql_client.c +++ b/server/modules/protocol/mysql_client.c @@ -65,7 +65,7 @@ static int gw_client_hangup_event(DCB *dcb); int mysql_send_ok(DCB *dcb, int packet_number, int in_affected_rows, const char* mysql_message); int MySQLSendHandshake(DCB* dcb); static int gw_mysql_do_authentication(DCB *dcb, GWBUF *queue); -static int route_by_statement(SESSION *, GWBUF *); +static int route_by_statement(SESSION *, GWBUF **); /* * The "module object" for the mysqld client protocol module. @@ -534,7 +534,7 @@ int gw_read_client_event( if (nbytes_read == 0) { goto return_rc; - } + } /** * if read queue existed appent read to it. * if length of read buffer is less than 3 or less than mysql packet @@ -783,7 +783,7 @@ int gw_read_client_event( * Feed each statement completely and separately * to router. */ - rc = route_by_statement(session, read_buffer); + rc = route_by_statement(session, &read_buffer); if (read_buffer != NULL) { @@ -1383,7 +1383,7 @@ gw_client_hangup_event(DCB *dcb) #if defined(SS_DEBUG) LOGIF(LE, (skygw_log_write_flush( LOGFILE_ERROR, - "Client hangup error handling."))); + "Client hangup error handling."))); #endif dcb_close(dcb); return 1; @@ -1399,7 +1399,9 @@ gw_client_hangup_event(DCB *dcb) * Return 1 in success. If the last packet is incomplete return success but * leave incomplete packet to readbuf. */ -static int route_by_statement(SESSION *session, GWBUF *readbuf) +static int route_by_statement( + SESSION* session, + GWBUF** p_readbuf) { int rc = -1; GWBUF* packetbuf; @@ -1407,7 +1409,7 @@ static int route_by_statement(SESSION *session, GWBUF *readbuf) gwbuf_type_t prevtype; GWBUF* tmpbuf; - tmpbuf = readbuf; + tmpbuf = *p_readbuf; while (tmpbuf != NULL) { ss_dassert(GWBUF_IS_TYPE_MYSQL(tmpbuf)); @@ -1416,9 +1418,9 @@ static int route_by_statement(SESSION *session, GWBUF *readbuf) #endif do { - ss_dassert(GWBUF_IS_TYPE_MYSQL(readbuf)); + ss_dassert(GWBUF_IS_TYPE_MYSQL((*p_readbuf))); - packetbuf = gw_MySQL_get_next_packet(&readbuf); + packetbuf = gw_MySQL_get_next_packet(p_readbuf); ss_dassert(GWBUF_IS_TYPE_MYSQL(packetbuf)); @@ -1447,7 +1449,7 @@ static int route_by_statement(SESSION *session, GWBUF *readbuf) goto return_rc; } } - while (readbuf != NULL); + while (*p_readbuf != NULL); return_rc: return rc; diff --git a/server/modules/protocol/mysql_common.c b/server/modules/protocol/mysql_common.c index 70050a13c..ef9991554 100644 --- a/server/modules/protocol/mysql_common.c +++ b/server/modules/protocol/mysql_common.c @@ -79,6 +79,7 @@ MySQLProtocol* mysql_protocol_init( strerror(eno)))); goto return_p; } + p->protocol_state = MYSQL_PROTOCOL_ALLOC; p->protocol_auth_state = MYSQL_ALLOC; p->protocol_command.scom_cmd = MYSQL_COM_UNDEFINED; p->protocol_command.scom_nresponse_packets = 0; @@ -90,6 +91,7 @@ MySQLProtocol* mysql_protocol_init( /*< Assign fd with protocol */ p->fd = fd; p->owner_dcb = dcb; + p->protocol_state = MYSQL_PROTOCOL_ACTIVE; CHK_PROTOCOL(p); return_p: return p; @@ -107,15 +109,25 @@ return_p: void mysql_protocol_done ( DCB* dcb) { - server_command_t* scmd = ((MySQLProtocol *)dcb->protocol)->protocol_cmd_history; + MySQLProtocol* p; + server_command_t* scmd; server_command_t* scmd2; + p = (MySQLProtocol *)dcb->protocol; + + spinlock_acquire(&p->protocol_lock); + + scmd = p->protocol_cmd_history; + while (scmd != NULL) { scmd2 = scmd->scom_next; free(scmd); scmd = scmd2; } + p->protocol_state = MYSQL_PROTOCOL_DONE; + + spinlock_release(&p->protocol_lock); } @@ -886,7 +898,7 @@ int mysql_send_com_quit( if (buf == NULL) { return 0; - } + } nbytes = dcb->func.write(dcb, buf); return nbytes; @@ -1542,16 +1554,18 @@ GWBUF* gw_MySQL_get_next_packet( packetbuf = gwbuf_alloc(packetlen); target = GWBUF_DATA(packetbuf); - + packetbuf->gwbuf_type = readbuf->gwbuf_type; /*< Copy the type too */ + while (nbytes_copied < packetlen) { uint8_t* src = GWBUF_DATA(readbuf); size_t buflen = GWBUF_LENGTH(readbuf); - + memcpy(target+nbytes_copied, src, buflen); - *p_readbuf = gwbuf_consume(readbuf, buflen); + readbuf = gwbuf_consume(readbuf, buflen); nbytes_copied += buflen; } + *p_readbuf = readbuf; ss_dassert(nbytes_copied == packetlen); } else @@ -1625,11 +1639,16 @@ void protocol_archive_srv_command( MySQLProtocol* p) { server_command_t* s1; - server_command_t** s2; + server_command_t* h1; int len = 0; spinlock_acquire(&p->protocol_lock); + if (p->protocol_state != MYSQL_PROTOCOL_ACTIVE) + { + goto retblock; + } + s1 = &p->protocol_command; LOGIF(LT, (skygw_log_write( @@ -1639,18 +1658,19 @@ void protocol_archive_srv_command( p->owner_dcb->fd))); /** Copy to history list */ - s2 = &p->protocol_cmd_history; - - if (*s2 != NULL) - { - while ((*s2)->scom_next != NULL) - { - *s2 = (*s2)->scom_next; - len += 1; - } + if ((h1 = p->protocol_cmd_history) == NULL) + { + p->protocol_cmd_history = server_command_copy(s1); } - *s2 = server_command_copy(s1); - + else + { + while (h1->scom_next != NULL) + { + h1 = h1->scom_next; + } + h1->scom_next = server_command_copy(s1); + } + /** Keep history limits, remove oldest */ if (len > MAX_CMD_HISTORY) { @@ -1669,6 +1689,8 @@ void protocol_archive_srv_command( p->protocol_command = *(s1->scom_next); free(s1->scom_next); } + +retblock: spinlock_release(&p->protocol_lock); } diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index 9d4b1b8ff..c10aca134 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -1549,7 +1549,7 @@ static void clientReply ( size_t len = MYSQL_GET_PACKET_LEN(buf); char* cmdstr = (char *)malloc(len+1); - snprintf(cmdstr, len+1, "%s", &buf[5]); + snprintf(cmdstr, len, "%s", &buf[5]); LOGIF(LE, (skygw_log_write_flush( LOGFILE_ERROR, From 01eb9822e03549ee8ce6c1bc9df4fb2fb08c1b11 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 14 Aug 2014 14:34:22 +0300 Subject: [PATCH 12/60] rwsplit router tests for session variables --- .../routing/readwritesplit/test/rwsplit.sh | 22 +++++++++++++++++++ .../readwritesplit/test/test_sescmd2.sql | 20 +++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 server/modules/routing/readwritesplit/test/test_sescmd2.sql diff --git a/server/modules/routing/readwritesplit/test/rwsplit.sh b/server/modules/routing/readwritesplit/test/rwsplit.sh index 36199e384..9d7afee57 100755 --- a/server/modules/routing/readwritesplit/test/rwsplit.sh +++ b/server/modules/routing/readwritesplit/test/rwsplit.sh @@ -230,3 +230,25 @@ if [ "$a" != "$TRETVAL" ]; then else echo "$TINPUT PASSED">>$TLOG ; fi + +echo "Session variables" >> $TLOG +echo "-----------------------------------" >> $TLOG + +RUNCMD=mysql\ --host=$THOST\ -P$TPORT\ -u$TUSER\ -p$TPWD\ --unbuffered=true\ --disable-reconnect\ -vv +TINPUT=test_sescmd2.sql +for ((i = 0;i<1000;i++)) +do + a=`$RUNCMD < ./$TINPUT 2>&1` + if [[ "`echo $a|grep -i 'error'`" != "" ]] + then + err=`echo "$a" | gawk ' /ERROR/{print a;print $0;exit 0}{a=$0}'` + echo "$err" >> $TLOG + echo "TEST FAILED" >> $TLOG + break + fi +done +if [[ "$err" == "" ]] +then + echo "TEST PASSED" >> $TLOG +fi +echo "-----------------------------------" >> $TLOG diff --git a/server/modules/routing/readwritesplit/test/test_sescmd2.sql b/server/modules/routing/readwritesplit/test/test_sescmd2.sql new file mode 100644 index 000000000..5cd179dfc --- /dev/null +++ b/server/modules/routing/readwritesplit/test/test_sescmd2.sql @@ -0,0 +1,20 @@ +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; +/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; From 77e5525436c180d70d417425f33317afb909891b Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Thu, 14 Aug 2014 15:15:22 +0300 Subject: [PATCH 13/60] mysql_client.c:gw_error_client_event & gw_client_hangup_event: added session state check, if session is already closing, don't start redundant call to dcb_close. mysql_common.c:mysql_protocol_done: added protocol state check. Used not to check it which caused double free of allocated memory. --- server/core/dcb.c | 7 +++--- server/modules/protocol/mysql_backend.c | 25 +++++++++---------- server/modules/protocol/mysql_client.c | 14 +++++++++++ server/modules/protocol/mysql_common.c | 5 ++++ .../routing/readwritesplit/readwritesplit.c | 2 +- 5 files changed, 36 insertions(+), 17 deletions(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index 1dfde0b58..090d57554 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -978,9 +978,10 @@ int below_water; } if (dolog) { - LOGIF(LE, (skygw_log_write_flush( - LOGFILE_ERROR, - "Error : Writing to %s socket failed due %d, %s.", + LOGIF(LD, (skygw_log_write( + LOGFILE_DEBUG, + "%lu [dcb_write] Writing to %s socket failed due %d, %s.", + pthread_self(), dcb_isclient(dcb) ? "client" : "backend server", saved_errno, strerror(saved_errno)))); diff --git a/server/modules/protocol/mysql_backend.c b/server/modules/protocol/mysql_backend.c index 903c17022..da00de083 100644 --- a/server/modules/protocol/mysql_backend.c +++ b/server/modules/protocol/mysql_backend.c @@ -777,11 +777,6 @@ static int gw_error_backend_event(DCB *dcb) router = session->service->router; router_instance = session->service->router_instance; -#if defined(SS_DEBUG) - LOGIF(LE, (skygw_log_write_flush( - LOGFILE_ERROR, - "Backend error event handling."))); -#endif /** * Avoid running redundant error handling procedure. * dcb_close is already called for the DCB. Thus, either connection is @@ -820,6 +815,11 @@ static int gw_error_backend_event(DCB *dcb) goto retblock; } +#if defined(SS_DEBUG) + LOGIF(LE, (skygw_log_write_flush( + LOGFILE_ERROR, + "Backend error event handling."))); +#endif router->handleError(router_instance, rsession, errbuf, @@ -963,14 +963,7 @@ gw_backend_hangup(DCB *dcb) rsession = session->router_session; router = session->service->router; - router_instance = session->service->router_instance; - -#if defined(SS_DEBUG) - LOGIF(LE, (skygw_log_write_flush( - LOGFILE_ERROR, - "Backend hangup error handling."))); -#endif - + router_instance = session->service->router_instance; errbuf = mysql_create_custom_error( 1, @@ -999,6 +992,12 @@ gw_backend_hangup(DCB *dcb) gwbuf_free(errbuf); goto retblock; } +#if defined(SS_DEBUG) + LOGIF(LE, (skygw_log_write_flush( + LOGFILE_ERROR, + "Backend hangup error handling."))); +#endif + router->handleError(router_instance, rsession, errbuf, diff --git a/server/modules/protocol/mysql_client.c b/server/modules/protocol/mysql_client.c index fbe0589a5..0f243fb1d 100644 --- a/server/modules/protocol/mysql_client.c +++ b/server/modules/protocol/mysql_client.c @@ -1300,12 +1300,19 @@ static int gw_error_client_event( STRDCBSTATE(dcb->state), (session != NULL ? session : NULL)))); + if (session != NULL && session->state == SESSION_STATE_STOPPING) + { + goto retblock; + } + #if defined(SS_DEBUG) LOGIF(LE, (skygw_log_write_flush( LOGFILE_ERROR, "Client error event handling."))); #endif dcb_close(dcb); + +retblock: return 1; } @@ -1380,12 +1387,19 @@ gw_client_hangup_event(DCB *dcb) { CHK_SESSION(session); } + + if (session != NULL && session->state == SESSION_STATE_STOPPING) + { + goto retblock; + } #if defined(SS_DEBUG) LOGIF(LE, (skygw_log_write_flush( LOGFILE_ERROR, "Client hangup error handling."))); #endif dcb_close(dcb); + +retblock: return 1; } diff --git a/server/modules/protocol/mysql_common.c b/server/modules/protocol/mysql_common.c index ef9991554..cd1e5c5f9 100644 --- a/server/modules/protocol/mysql_common.c +++ b/server/modules/protocol/mysql_common.c @@ -117,6 +117,10 @@ void mysql_protocol_done ( spinlock_acquire(&p->protocol_lock); + if (p->protocol_state != MYSQL_PROTOCOL_ACTIVE) + { + goto retblock; + } scmd = p->protocol_cmd_history; while (scmd != NULL) @@ -127,6 +131,7 @@ void mysql_protocol_done ( } p->protocol_state = MYSQL_PROTOCOL_DONE; +retblock: spinlock_release(&p->protocol_lock); } diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index c10aca134..f09b29ae5 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -1548,7 +1548,7 @@ static void clientReply ( (uint8_t *)GWBUF_DATA((scur->scmd_cur_cmd->my_sescmd_buf)); size_t len = MYSQL_GET_PACKET_LEN(buf); char* cmdstr = (char *)malloc(len+1); - + /** data+termination character == len */ snprintf(cmdstr, len, "%s", &buf[5]); LOGIF(LE, (skygw_log_write_flush( From 2393ac57e9dabb15da51061ab02ce28fc21fac02 Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Thu, 14 Aug 2014 17:23:46 +0300 Subject: [PATCH 14/60] mysql_common.c:protocol_add_srv_command didn't check that protocol status was MYSQL_PROTOCOL_ACTIVE and wrote to freed memory. --- server/modules/protocol/mysql_backend.c | 3 ++- server/modules/protocol/mysql_common.c | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/server/modules/protocol/mysql_backend.c b/server/modules/protocol/mysql_backend.c index da00de083..ce7b6ef97 100644 --- a/server/modules/protocol/mysql_backend.c +++ b/server/modules/protocol/mysql_backend.c @@ -1006,7 +1006,8 @@ gw_backend_hangup(DCB *dcb) &succp); /** There are not required backends available, close session. */ - if (!succp) { + if (!succp) + { #if defined(SS_DEBUG) LOGIF(LE, (skygw_log_write_flush( LOGFILE_ERROR, diff --git a/server/modules/protocol/mysql_common.c b/server/modules/protocol/mysql_common.c index cd1e5c5f9..4e06894d1 100644 --- a/server/modules/protocol/mysql_common.c +++ b/server/modules/protocol/mysql_common.c @@ -1712,6 +1712,10 @@ void protocol_add_srv_command( spinlock_acquire(&p->protocol_lock); + if (p->protocol_state != MYSQL_PROTOCOL_ACTIVE) + { + goto retblock; + } /** this is the only server command in protocol */ if (p->protocol_command.scom_cmd == MYSQL_COM_UNDEFINED) { @@ -1744,6 +1748,7 @@ void protocol_add_srv_command( c = c->scom_next; } #endif +retblock: spinlock_release(&p->protocol_lock); } From 902004c1ee4e6234e229d7eb890a75b5fac2eaa1 Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Thu, 14 Aug 2014 22:33:57 +0300 Subject: [PATCH 15/60] Fix to bug #463, http://bugs.skysql.com/show_bug.cgi?id=463 mysql_common.c:gw_MySQL_get_next_packet didn't handle case where an insert command followed by alter table in the same read buffer. It shouldn't been possible without multi-statement being set. --- server/core/dcb.c | 2 + server/modules/protocol/mysql_client.c | 3 +- server/modules/protocol/mysql_common.c | 52 +++++++------------ .../routing/readwritesplit/readwritesplit.c | 2 + 4 files changed, 24 insertions(+), 35 deletions(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index 090d57554..5a280c3b0 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -814,6 +814,8 @@ int below_water; } spinlock_acquire(&dcb->writeqlock); + + ss_dassert(dcb->state != DCB_STATE_ZOMBIE); if (dcb->writeq != NULL) { diff --git a/server/modules/protocol/mysql_client.c b/server/modules/protocol/mysql_client.c index 0f243fb1d..6ffe4e56f 100644 --- a/server/modules/protocol/mysql_client.c +++ b/server/modules/protocol/mysql_client.c @@ -1435,12 +1435,11 @@ static int route_by_statement( ss_dassert(GWBUF_IS_TYPE_MYSQL((*p_readbuf))); packetbuf = gw_MySQL_get_next_packet(p_readbuf); - - ss_dassert(GWBUF_IS_TYPE_MYSQL(packetbuf)); if (packetbuf != NULL) { CHK_GWBUF(packetbuf); + ss_dassert(GWBUF_IS_TYPE_MYSQL(packetbuf)); /** * This means that buffer includes exactly one MySQL * statement. diff --git a/server/modules/protocol/mysql_common.c b/server/modules/protocol/mysql_common.c index 4e06894d1..f9c0ebdea 100644 --- a/server/modules/protocol/mysql_common.c +++ b/server/modules/protocol/mysql_common.c @@ -1514,6 +1514,9 @@ GWBUF* gw_MySQL_get_next_packet( size_t packetlen; size_t totalbuflen; uint8_t* data; + size_t nbytes_copied = 0; + uint8_t* target; + readbuf = *p_readbuf; if (readbuf == NULL) @@ -1540,44 +1543,27 @@ GWBUF* gw_MySQL_get_next_packet( packetbuf = NULL; goto return_packetbuf; } - /** there is one complete packet in the buffer */ - if (packetlen == buflen) - { - packetbuf = gwbuf_clone_portion(readbuf, 0, packetlen); - *p_readbuf = gwbuf_consume(readbuf, packetlen); - goto return_packetbuf; - } + + packetbuf = gwbuf_alloc(packetlen); + target = GWBUF_DATA(packetbuf); + packetbuf->gwbuf_type = readbuf->gwbuf_type; /*< Copy the type too */ /** - * Packet spans multiple buffers. - * Allocate buffer for complete packet - * copy packet parts into it and consume copied bytes - */ - else if (packetlen > buflen) + * Copy first MySQL packet to packetbuf and leave posible other + * packets to read buffer. + */ + while (nbytes_copied < packetlen && totalbuflen > 0) { - size_t nbytes_copied = 0; - uint8_t* target; + uint8_t* src = GWBUF_DATA((*p_readbuf)); + size_t bytestocopy; - packetbuf = gwbuf_alloc(packetlen); - target = GWBUF_DATA(packetbuf); - packetbuf->gwbuf_type = readbuf->gwbuf_type; /*< Copy the type too */ + bytestocopy = MIN(buflen,packetlen-nbytes_copied); - while (nbytes_copied < packetlen) - { - uint8_t* src = GWBUF_DATA(readbuf); - size_t buflen = GWBUF_LENGTH(readbuf); - - memcpy(target+nbytes_copied, src, buflen); - readbuf = gwbuf_consume(readbuf, buflen); - nbytes_copied += buflen; - } - *p_readbuf = readbuf; - ss_dassert(nbytes_copied == packetlen); - } - else - { - packetbuf = gwbuf_clone_portion(readbuf, 0, packetlen); - *p_readbuf = gwbuf_consume(readbuf, packetlen); + memcpy(target+nbytes_copied, src, bytestocopy); + *p_readbuf = gwbuf_consume((*p_readbuf), bytestocopy); + totalbuflen = gwbuf_length((*p_readbuf)); + nbytes_copied += bytestocopy; } + ss_dassert(buflen == 0 || nbytes_copied == packetlen); return_packetbuf: return packetbuf; diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index f09b29ae5..a0c241920 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -1550,6 +1550,8 @@ static void clientReply ( char* cmdstr = (char *)malloc(len+1); /** data+termination character == len */ snprintf(cmdstr, len, "%s", &buf[5]); + + ss_dassert(len+4 == GWBUF_LENGTH(scur->scmd_cur_cmd->my_sescmd_buf)); LOGIF(LE, (skygw_log_write_flush( LOGFILE_ERROR, From 28cc98d33ae0116f399131974d6d5ad88dcd1a01 Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Thu, 14 Aug 2014 23:22:21 +0300 Subject: [PATCH 16/60] add gcov patch file --- gcov.diff | 138 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 gcov.diff diff --git a/gcov.diff b/gcov.diff new file mode 100644 index 000000000..c310d1b08 --- /dev/null +++ b/gcov.diff @@ -0,0 +1,138 @@ +diff --git a/makefile.inc b/makefile.inc +index f2d93bf..c7dbffa 100644 +--- a/makefile.inc ++++ b/makefile.inc +@@ -24,8 +24,8 @@ endif + + # -O2 -g -pipe -Wformat -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -fstack-protector --param=ssp-buffer-size=4 -fPIC + +-CFLAGS := $(CFLAGS) -Wall +-LDLIBS := $(LDLIBS) -pthread ++CFLAGS := $(CFLAGS) -Wall -fprofile-arcs -ftest-coverage ++LDLIBS := $(LDLIBS) -pthread -lgcov + LDMYSQL := -lmysqld + CPP_LDLIBS := -lstdc++ + +diff --git a/server/core/Makefile b/server/core/Makefile +index 9bf650c..9df75a7 100644 +--- a/server/core/Makefile ++++ b/server/core/Makefile +@@ -75,7 +75,7 @@ POBJS=maxpasswd.o secrets.o utils.o + LIBS=-L$(EMBEDDED_LIB) \ + -lmysqld \ + -lz -lm -lcrypt -lcrypto -ldl -laio -lrt -pthread -llog_manager \ +- -L../inih/extra -linih -lssl -lstdc++ ++ -L../inih/extra -linih -lssl -lstdc++ -lgcov + + all: maxscale maxkeys maxpasswd + +diff --git a/server/modules/filter/Makefile b/server/modules/filter/Makefile +index 931c35a..d5dcca9 100644 +--- a/server/modules/filter/Makefile ++++ b/server/modules/filter/Makefile +@@ -25,7 +25,7 @@ UTILSPATH := $(ROOT_PATH)/utils + + CC=cc + CFLAGS=-c -fPIC -I/usr/include -I../include -I../../include -I$(LOGPATH) \ +- -I$(UTILSPATH) -Wall -g ++ -I$(UTILSPATH) -Wall -g -fprofile-arcs -ftest-coverage + + include ../../../makefile.inc + +@@ -44,7 +44,7 @@ TEESRCS=tee.c + TEEOBJ=$(TEESRCS:.c=.o) + SRCS=$(TESTSRCS) $(QLASRCS) $(REGEXSRCS) $(TOPNSRCS) $(TEESRCS) + OBJ=$(SRCS:.c=.o) +-LIBS=$(UTILSPATH)/skygw_utils.o -lssl -llog_manager ++LIBS=$(UTILSPATH)/skygw_utils.o -lssl -llog_manager -lgcov + MODULES= libtestfilter.so libqlafilter.so libregexfilter.so libtopfilter.so libtee.so + + +diff --git a/server/modules/monitor/Makefile b/server/modules/monitor/Makefile +index 7fdbc58..bca01de 100644 +--- a/server/modules/monitor/Makefile ++++ b/server/modules/monitor/Makefile +@@ -28,7 +28,7 @@ CFLAGS=-c -fPIC -I. -I/usr/include -I../include -I../../include -I$(LOGPATH) \ + + LDFLAGS=-shared -L$(LOGPATH) -Wl,-rpath,$(DEST)/lib \ + -Wl,-rpath,$(LOGPATH) -Wl,-rpath,$(UTILSPATH) \ +- -Wl,-rpath,$(EMBEDDED_LIB) ++ -Wl,-rpath,$(EMBEDDED_LIB) -fprofile-arcs -ftest-coverage + + + +@@ -39,7 +39,7 @@ GALERAOBJ=$(GALERASRCS:.c=.o) + SRCS=$(MYSQLSRCS) + OBJ=$(SRCS:.c=.o) + LIBS=$(UTILSPATH)/skygw_utils.o -llog_manager \ +- -L$(EMBEDDED_LIB) -lmysqld ++ -L$(EMBEDDED_LIB) -lmysqld -lgcov + MODULES=libmysqlmon.so libgaleramon.so + + +diff --git a/server/modules/protocol/Makefile b/server/modules/protocol/Makefile +index 54a8f8c..c8913ab 100644 +--- a/server/modules/protocol/Makefile ++++ b/server/modules/protocol/Makefile +@@ -31,7 +31,7 @@ UTILSPATH := $(ROOT_PATH)/utils + + CC=cc + CFLAGS=-c -fPIC -I/usr/include -I../include -I../../include -I$(LOGPATH) \ +- -I$(UTILSPATH) -Wall -g ++ -I$(UTILSPATH) -Wall -g -fprofile-arcs -ftest-coverage + + include ../../../makefile.inc + +@@ -51,7 +51,7 @@ MAXSCALEDOBJ=$(MAXSCALEDSRCS:.c=.o) + SRCS=$(MYSQLCLIENTSRCS) $(MYSQLBACKENDSRCS) $(TELNETDSRCS) $(HTTPDSRCS) \ + $(MAXSCALEDSRCS) + OBJ=$(SRCS:.c=.o) +-LIBS=$(UTILSPATH)/skygw_utils.o ++LIBS=$(UTILSPATH)/skygw_utils.o -lgcov + MODULES=libMySQLClient.so libMySQLBackend.so libtelnetd.so libHTTPD.so \ + libmaxscaled.so + +diff --git a/server/modules/routing/Makefile b/server/modules/routing/Makefile +index 4feac68..afd1da7 100644 +--- a/server/modules/routing/Makefile ++++ b/server/modules/routing/Makefile +@@ -29,7 +29,7 @@ UTILSPATH := $(ROOT_PATH)/utils + + CC=cc + CFLAGS=-c -fPIC -I/usr/include -I../include -I../../include -I$(LOGPATH) \ +- -I$(UTILSPATH) -Wall -g ++ -I$(UTILSPATH) -Wall -g -fprofile-arcs -ftest-coverage + + include ../../../makefile.inc + +@@ -46,7 +46,7 @@ CLISRCS=cli.c debugcmd.c + CLIOBJ=$(CLISRCS:.c=.o) + SRCS=$(TESTSRCS) $(READCONSRCS) $(DEBUGCLISRCS) cli.c + OBJ=$(SRCS:.c=.o) +-LIBS=$(UTILSPATH)/skygw_utils.o -lssl -llog_manager ++LIBS=$(UTILSPATH)/skygw_utils.o -lssl -llog_manager -lgcov + MODULES= libdebugcli.so libreadconnroute.so libtestroute.so libcli.so + + +diff --git a/server/modules/routing/readwritesplit/Makefile b/server/modules/routing/readwritesplit/Makefile +index c60f2ff..a3a643e 100644 +--- a/server/modules/routing/readwritesplit/Makefile ++++ b/server/modules/routing/readwritesplit/Makefile +@@ -27,7 +27,7 @@ QCLASSPATH := $(ROOT_PATH)/query_classifier + CC=cc + CFLAGS=-c -fPIC -I/usr/include -I../../include -I../../../include \ + -I$(LOGPATH) -I$(UTILSPATH) -I$(QCLASSPATH) \ +- $(MYSQL_HEADERS) -Wall -g ++ $(MYSQL_HEADERS) -Wall -g -fprofile-arcs -ftest-coverage + + include ../../../../makefile.inc + +@@ -38,7 +38,7 @@ LDFLAGS=-shared -L$(LOGPATH) -L$(QCLASSPATH) -L$(EMBEDDED_LIB) \ + + SRCS=readwritesplit.c + OBJ=$(SRCS:.c=.o) +-LIBS=-lssl -pthread -llog_manager -lquery_classifier -lmysqld ++LIBS=-lssl -pthread -llog_manager -lquery_classifier -lmysqld -lgcov + MODULES=libreadwritesplit.so + + all: $(MODULES) From 7f18914d90c169c36a1715c4a27824c688cb528e Mon Sep 17 00:00:00 2001 From: Hartmut Holzgraefe Date: Fri, 15 Aug 2014 02:51:16 +0200 Subject: [PATCH 17/60] fixed wrong server port in README --- README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README b/README index 3750fb332..cdc9b35ac 100644 --- a/README +++ b/README @@ -182,7 +182,7 @@ on localhost: * a master on port 3000, with server_id=2 * a slave on port 3001, server_id doesn't matter -* a slave on port 2002, server_id doesn't matter +* a slave on port 3002, server_id doesn't matter On the master full privileges on the databases "test" and "FOO" are needed, on the saves SELECT permissions on test.* should From be52b3a71e65d45603b01c390d13490cb4f243c8 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 15 Aug 2014 13:41:56 +0300 Subject: [PATCH 18/60] more stress tests for rwsplit router --- .../routing/readwritesplit/test/rwsplit.sh | 38 +++++++++++++++---- .../readwritesplit/test/test_sescmd3.sql | 16 ++++++++ 2 files changed, 46 insertions(+), 8 deletions(-) create mode 100644 server/modules/routing/readwritesplit/test/test_sescmd3.sql diff --git a/server/modules/routing/readwritesplit/test/rwsplit.sh b/server/modules/routing/readwritesplit/test/rwsplit.sh index 9d7afee57..c2e403b16 100755 --- a/server/modules/routing/readwritesplit/test/rwsplit.sh +++ b/server/modules/routing/readwritesplit/test/rwsplit.sh @@ -230,25 +230,47 @@ if [ "$a" != "$TRETVAL" ]; then else echo "$TINPUT PASSED">>$TLOG ; fi - -echo "Session variables" >> $TLOG +echo "-----------------------------------" >> $TLOG +echo "Session variables: Stress Test 1" >> $TLOG echo "-----------------------------------" >> $TLOG -RUNCMD=mysql\ --host=$THOST\ -P$TPORT\ -u$TUSER\ -p$TPWD\ --unbuffered=true\ --disable-reconnect\ -vv +RUNCMD=mysql\ --host=$THOST\ -P$TPORT\ -u$TUSER\ -p$TPWD\ --unbuffered=true\ --disable-reconnect\ -q\ -r TINPUT=test_sescmd2.sql for ((i = 0;i<1000;i++)) do - a=`$RUNCMD < ./$TINPUT 2>&1` - if [[ "`echo $a|grep -i 'error'`" != "" ]] + a=`$RUNCMD < $TINPUT 2>&1` + if [[ "`echo "$a"|grep -i 'error'`" != "" ]] then - err=`echo "$a" | gawk ' /ERROR/{print a;print $0;exit 0}{a=$0}'` - echo "$err" >> $TLOG - echo "TEST FAILED" >> $TLOG + err=`echo "$a" | grep -i error` break fi done if [[ "$err" == "" ]] then echo "TEST PASSED" >> $TLOG +else + echo "$err" >> $TLOG + echo "Test FAILED at iteration $((i+1))" >> $TLOG fi echo "-----------------------------------" >> $TLOG +echo "Session variables: Stress Test 2" >> $TLOG +echo "-----------------------------------" >> $TLOG + +err="" +TINPUT=test_sescmd3.sql +for ((j = 0;j<1000;j++)) +do + b=`$RUNCMD < $TINPUT 2>&1` + if [[ "`echo "$b"|grep -i 'null'`" != "" ]] + then + err=`echo "$b" | grep -i null` + break + fi +done +if [[ "$err" == "" ]] +then + echo "TEST PASSED" >> $TLOG +else + echo "Test FAILED at iteration $((j+1))" >> $TLOG +fi +echo "" >> $TLOG diff --git a/server/modules/routing/readwritesplit/test/test_sescmd3.sql b/server/modules/routing/readwritesplit/test/test_sescmd3.sql new file mode 100644 index 000000000..ef5e9ed72 --- /dev/null +++ b/server/modules/routing/readwritesplit/test/test_sescmd3.sql @@ -0,0 +1,16 @@ +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; +select @OLD_SQL_NOTES; +select @OLD_CHARACTER_SET_CLIENT; +select @OLD_CHARACTER_SET_RESULTS; +select @OLD_COLLATION_CONNECTION; +select @OLD_TIME_ZONE; +select @OLD_UNIQUE_CHECKS; +select @OLD_FOREIGN_KEY_CHECKS; +select @OLD_SQL_MODE; From 85ebc6d543577f8321311090ebe093263fc3786c Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Fri, 15 Aug 2014 17:41:46 +0300 Subject: [PATCH 19/60] add gcov patch --- gcov.diff | 138 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 gcov.diff diff --git a/gcov.diff b/gcov.diff new file mode 100644 index 000000000..c310d1b08 --- /dev/null +++ b/gcov.diff @@ -0,0 +1,138 @@ +diff --git a/makefile.inc b/makefile.inc +index f2d93bf..c7dbffa 100644 +--- a/makefile.inc ++++ b/makefile.inc +@@ -24,8 +24,8 @@ endif + + # -O2 -g -pipe -Wformat -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -fstack-protector --param=ssp-buffer-size=4 -fPIC + +-CFLAGS := $(CFLAGS) -Wall +-LDLIBS := $(LDLIBS) -pthread ++CFLAGS := $(CFLAGS) -Wall -fprofile-arcs -ftest-coverage ++LDLIBS := $(LDLIBS) -pthread -lgcov + LDMYSQL := -lmysqld + CPP_LDLIBS := -lstdc++ + +diff --git a/server/core/Makefile b/server/core/Makefile +index 9bf650c..9df75a7 100644 +--- a/server/core/Makefile ++++ b/server/core/Makefile +@@ -75,7 +75,7 @@ POBJS=maxpasswd.o secrets.o utils.o + LIBS=-L$(EMBEDDED_LIB) \ + -lmysqld \ + -lz -lm -lcrypt -lcrypto -ldl -laio -lrt -pthread -llog_manager \ +- -L../inih/extra -linih -lssl -lstdc++ ++ -L../inih/extra -linih -lssl -lstdc++ -lgcov + + all: maxscale maxkeys maxpasswd + +diff --git a/server/modules/filter/Makefile b/server/modules/filter/Makefile +index 931c35a..d5dcca9 100644 +--- a/server/modules/filter/Makefile ++++ b/server/modules/filter/Makefile +@@ -25,7 +25,7 @@ UTILSPATH := $(ROOT_PATH)/utils + + CC=cc + CFLAGS=-c -fPIC -I/usr/include -I../include -I../../include -I$(LOGPATH) \ +- -I$(UTILSPATH) -Wall -g ++ -I$(UTILSPATH) -Wall -g -fprofile-arcs -ftest-coverage + + include ../../../makefile.inc + +@@ -44,7 +44,7 @@ TEESRCS=tee.c + TEEOBJ=$(TEESRCS:.c=.o) + SRCS=$(TESTSRCS) $(QLASRCS) $(REGEXSRCS) $(TOPNSRCS) $(TEESRCS) + OBJ=$(SRCS:.c=.o) +-LIBS=$(UTILSPATH)/skygw_utils.o -lssl -llog_manager ++LIBS=$(UTILSPATH)/skygw_utils.o -lssl -llog_manager -lgcov + MODULES= libtestfilter.so libqlafilter.so libregexfilter.so libtopfilter.so libtee.so + + +diff --git a/server/modules/monitor/Makefile b/server/modules/monitor/Makefile +index 7fdbc58..bca01de 100644 +--- a/server/modules/monitor/Makefile ++++ b/server/modules/monitor/Makefile +@@ -28,7 +28,7 @@ CFLAGS=-c -fPIC -I. -I/usr/include -I../include -I../../include -I$(LOGPATH) \ + + LDFLAGS=-shared -L$(LOGPATH) -Wl,-rpath,$(DEST)/lib \ + -Wl,-rpath,$(LOGPATH) -Wl,-rpath,$(UTILSPATH) \ +- -Wl,-rpath,$(EMBEDDED_LIB) ++ -Wl,-rpath,$(EMBEDDED_LIB) -fprofile-arcs -ftest-coverage + + + +@@ -39,7 +39,7 @@ GALERAOBJ=$(GALERASRCS:.c=.o) + SRCS=$(MYSQLSRCS) + OBJ=$(SRCS:.c=.o) + LIBS=$(UTILSPATH)/skygw_utils.o -llog_manager \ +- -L$(EMBEDDED_LIB) -lmysqld ++ -L$(EMBEDDED_LIB) -lmysqld -lgcov + MODULES=libmysqlmon.so libgaleramon.so + + +diff --git a/server/modules/protocol/Makefile b/server/modules/protocol/Makefile +index 54a8f8c..c8913ab 100644 +--- a/server/modules/protocol/Makefile ++++ b/server/modules/protocol/Makefile +@@ -31,7 +31,7 @@ UTILSPATH := $(ROOT_PATH)/utils + + CC=cc + CFLAGS=-c -fPIC -I/usr/include -I../include -I../../include -I$(LOGPATH) \ +- -I$(UTILSPATH) -Wall -g ++ -I$(UTILSPATH) -Wall -g -fprofile-arcs -ftest-coverage + + include ../../../makefile.inc + +@@ -51,7 +51,7 @@ MAXSCALEDOBJ=$(MAXSCALEDSRCS:.c=.o) + SRCS=$(MYSQLCLIENTSRCS) $(MYSQLBACKENDSRCS) $(TELNETDSRCS) $(HTTPDSRCS) \ + $(MAXSCALEDSRCS) + OBJ=$(SRCS:.c=.o) +-LIBS=$(UTILSPATH)/skygw_utils.o ++LIBS=$(UTILSPATH)/skygw_utils.o -lgcov + MODULES=libMySQLClient.so libMySQLBackend.so libtelnetd.so libHTTPD.so \ + libmaxscaled.so + +diff --git a/server/modules/routing/Makefile b/server/modules/routing/Makefile +index 4feac68..afd1da7 100644 +--- a/server/modules/routing/Makefile ++++ b/server/modules/routing/Makefile +@@ -29,7 +29,7 @@ UTILSPATH := $(ROOT_PATH)/utils + + CC=cc + CFLAGS=-c -fPIC -I/usr/include -I../include -I../../include -I$(LOGPATH) \ +- -I$(UTILSPATH) -Wall -g ++ -I$(UTILSPATH) -Wall -g -fprofile-arcs -ftest-coverage + + include ../../../makefile.inc + +@@ -46,7 +46,7 @@ CLISRCS=cli.c debugcmd.c + CLIOBJ=$(CLISRCS:.c=.o) + SRCS=$(TESTSRCS) $(READCONSRCS) $(DEBUGCLISRCS) cli.c + OBJ=$(SRCS:.c=.o) +-LIBS=$(UTILSPATH)/skygw_utils.o -lssl -llog_manager ++LIBS=$(UTILSPATH)/skygw_utils.o -lssl -llog_manager -lgcov + MODULES= libdebugcli.so libreadconnroute.so libtestroute.so libcli.so + + +diff --git a/server/modules/routing/readwritesplit/Makefile b/server/modules/routing/readwritesplit/Makefile +index c60f2ff..a3a643e 100644 +--- a/server/modules/routing/readwritesplit/Makefile ++++ b/server/modules/routing/readwritesplit/Makefile +@@ -27,7 +27,7 @@ QCLASSPATH := $(ROOT_PATH)/query_classifier + CC=cc + CFLAGS=-c -fPIC -I/usr/include -I../../include -I../../../include \ + -I$(LOGPATH) -I$(UTILSPATH) -I$(QCLASSPATH) \ +- $(MYSQL_HEADERS) -Wall -g ++ $(MYSQL_HEADERS) -Wall -g -fprofile-arcs -ftest-coverage + + include ../../../../makefile.inc + +@@ -38,7 +38,7 @@ LDFLAGS=-shared -L$(LOGPATH) -L$(QCLASSPATH) -L$(EMBEDDED_LIB) \ + + SRCS=readwritesplit.c + OBJ=$(SRCS:.c=.o) +-LIBS=-lssl -pthread -llog_manager -lquery_classifier -lmysqld ++LIBS=-lssl -pthread -llog_manager -lquery_classifier -lmysqld -lgcov + MODULES=libreadwritesplit.so + + all: $(MODULES) From 677a44f497efc5d99aa0015e61ecfcfbac1a499b Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Fri, 15 Aug 2014 18:00:39 +0300 Subject: [PATCH 20/60] Merge from release-1.0beta to Z2 --- log_manager/log_manager.cc | 26 ++--- server/core/dcb.c | 9 +- server/include/dcb.h | 4 +- .../include/mysql_client_server_protocol.h | 7 ++ server/modules/protocol/mysql_backend.c | 103 +++++++++++----- server/modules/protocol/mysql_client.c | 37 ++++-- server/modules/protocol/mysql_common.c | 110 ++++++++++-------- .../routing/readwritesplit/readwritesplit.c | 4 +- .../routing/readwritesplit/test/rwsplit.sh | 44 +++++++ .../readwritesplit/test/test_sescmd2.sql | 20 ++++ .../readwritesplit/test/test_sescmd3.sql | 16 +++ 11 files changed, 277 insertions(+), 103 deletions(-) create mode 100644 server/modules/routing/readwritesplit/test/test_sescmd2.sql create mode 100644 server/modules/routing/readwritesplit/test/test_sescmd3.sql diff --git a/log_manager/log_manager.cc b/log_manager/log_manager.cc index c8f96266e..733e9fffe 100644 --- a/log_manager/log_manager.cc +++ b/log_manager/log_manager.cc @@ -255,11 +255,7 @@ static int logmanager_write_log( static blockbuf_t* blockbuf_init(logfile_id_t id); static void blockbuf_node_done(void* bb_data); static char* blockbuf_get_writepos( -#if 0 - int** refcount, -#else blockbuf_t** p_bb, -#endif logfile_id_t id, size_t str_len, bool flush); @@ -653,7 +649,16 @@ static int logmanager_write_log( int safe_str_len; timestamp_len = get_timestamp_len(); - safe_str_len = MIN(timestamp_len-1+str_len, lf->lf_buf_size); + + /** Findout how much can be safely written with current block size */ + if (timestamp_len-1+str_len > lf->lf_buf_size) + { + safe_str_len = lf->lf_buf_size; + } + else + { + safe_str_len = timestamp_len-1+str_len; + } /** * Seek write position and register to block buffer. * Then print formatted string to write position. @@ -673,9 +678,9 @@ static int logmanager_write_log( * of the timestamp string. */ if (use_valist) { - vsnprintf(wp+timestamp_len, safe_str_len, str, valist); + vsnprintf(wp+timestamp_len, safe_str_len-timestamp_len, str, valist); } else { - snprintf(wp+timestamp_len, safe_str_len, "%s", str); + snprintf(wp+timestamp_len, safe_str_len-timestamp_len, "%s", str); } /** write to syslog */ @@ -694,12 +699,7 @@ static int logmanager_write_log( break; } } - - /** remove double line feed */ - if (wp[timestamp_len+str_len-2] == '\n') { - wp[timestamp_len+str_len-2]=' '; - } - wp[timestamp_len+str_len-1]='\n'; + wp[safe_str_len-1] = '\n'; blockbuf_unregister(bb); /** diff --git a/server/core/dcb.c b/server/core/dcb.c index 1dfde0b58..5a280c3b0 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -814,6 +814,8 @@ int below_water; } spinlock_acquire(&dcb->writeqlock); + + ss_dassert(dcb->state != DCB_STATE_ZOMBIE); if (dcb->writeq != NULL) { @@ -978,9 +980,10 @@ int below_water; } if (dolog) { - LOGIF(LE, (skygw_log_write_flush( - LOGFILE_ERROR, - "Error : Writing to %s socket failed due %d, %s.", + LOGIF(LD, (skygw_log_write( + LOGFILE_DEBUG, + "%lu [dcb_write] Writing to %s socket failed due %d, %s.", + pthread_self(), dcb_isclient(dcb) ? "client" : "backend server", saved_errno, strerror(saved_errno)))); diff --git a/server/include/dcb.h b/server/include/dcb.h index 88964fc31..4f8be99d4 100644 --- a/server/include/dcb.h +++ b/server/include/dcb.h @@ -145,9 +145,9 @@ typedef enum { DCB_STATE_POLLING, /*< Waiting in the poll loop */ DCB_STATE_LISTENING, /*< The DCB is for a listening socket */ DCB_STATE_DISCONNECTED, /*< The socket is now closed */ - DCB_STATE_FREED, /*< Memory freed */ DCB_STATE_NOPOLLING, /*< Removed from poll mask */ - DCB_STATE_ZOMBIE /*< DCB is no longer active, waiting to free it */ + DCB_STATE_ZOMBIE, /*< DCB is no longer active, waiting to free it */ + DCB_STATE_FREED /*< Memory freed */ } dcb_state_t; typedef enum { diff --git a/server/modules/include/mysql_client_server_protocol.h b/server/modules/include/mysql_client_server_protocol.h index 9e2147d05..c875a1232 100644 --- a/server/modules/include/mysql_client_server_protocol.h +++ b/server/modules/include/mysql_client_server_protocol.h @@ -102,6 +102,12 @@ typedef enum { MYSQL_IDLE } mysql_auth_state_t; +typedef enum { + MYSQL_PROTOCOL_ALLOC, + MYSQL_PROTOCOL_ACTIVE, + MYSQL_PROTOCOL_DONE +} mysql_protocol_state_t; + /* * MySQL session specific data @@ -270,6 +276,7 @@ typedef struct { server_command_t protocol_command; /*< session command list */ server_command_t* protocol_cmd_history; /*< session command history */ mysql_auth_state_t protocol_auth_state; /*< Authentication status */ + mysql_protocol_state_t protocol_state; /*< Protocol struct status */ uint8_t scramble[MYSQL_SCRAMBLE_LEN]; /*< server scramble, * created or received */ uint32_t server_capabilities; /*< server capabilities, diff --git a/server/modules/protocol/mysql_backend.c b/server/modules/protocol/mysql_backend.c index fee8d984b..d79d7a044 100644 --- a/server/modules/protocol/mysql_backend.c +++ b/server/modules/protocol/mysql_backend.c @@ -761,12 +761,13 @@ return_rc: */ static int gw_error_backend_event(DCB *dcb) { - SESSION* session; - void* rsession; - ROUTER_OBJECT* router; - ROUTER* router_instance; - GWBUF* errbuf; - bool succp; + SESSION* session; + void* rsession; + ROUTER_OBJECT* router; + ROUTER* router_instance; + GWBUF* errbuf; + bool succp; + session_state_t ses_state; CHK_DCB(dcb); session = dcb->session; @@ -775,11 +776,6 @@ static int gw_error_backend_event(DCB *dcb) router = session->service->router; router_instance = session->service->router_instance; -#if defined(SS_DEBUG) - LOGIF(LE, (skygw_log_write_flush( - LOGFILE_ERROR, - "Backend error event handling."))); -#endif /** * Avoid running redundant error handling procedure. * dcb_close is already called for the DCB. Thus, either connection is @@ -795,6 +791,34 @@ static int gw_error_backend_event(DCB *dcb) 0, "Lost connection to backend server."); + spinlock_acquire(&session->ses_lock); + ses_state = session->state; + spinlock_release(&session->ses_lock); + + /** + * Session might be initialized when DCB already is in the poll set. + * Thus hangup can occur in the middle of session initialization. + * Only complete and successfully initialized sessions allow for + * calling error handler. + */ + while (ses_state == SESSION_STATE_READY) + { + spinlock_acquire(&session->ses_lock); + ses_state = session->state; + spinlock_release(&session->ses_lock); + } + + if (ses_state != SESSION_STATE_ROUTER_READY) + { + gwbuf_free(errbuf); + goto retblock; + } + +#if defined(SS_DEBUG) + LOGIF(LE, (skygw_log_write_flush( + LOGFILE_ERROR, + "Backend error event handling."))); +#endif router->handleError(router_instance, rsession, errbuf, @@ -810,6 +834,7 @@ static int gw_error_backend_event(DCB *dcb) } dcb_close(dcb); +retblock: return 1; } @@ -923,12 +948,13 @@ return_fd: static int gw_backend_hangup(DCB *dcb) { - SESSION* session; - void* rsession; - ROUTER_OBJECT* router; - ROUTER* router_instance; - bool succp; - GWBUF* errbuf; + SESSION* session; + void* rsession; + ROUTER_OBJECT* router; + ROUTER* router_instance; + bool succp; + GWBUF* errbuf; + session_state_t ses_state; CHK_DCB(dcb); session = dcb->session; @@ -936,20 +962,41 @@ gw_backend_hangup(DCB *dcb) rsession = session->router_session; router = session->service->router; - router_instance = session->service->router_instance; - + router_instance = session->service->router_instance; + + errbuf = mysql_create_custom_error( + 1, + 0, + "Lost connection to backend server."); + + spinlock_acquire(&session->ses_lock); + ses_state = session->state; + spinlock_release(&session->ses_lock); + + /** + * Session might be initialized when DCB already is in the poll set. + * Thus hangup can occur in the middle of session initialization. + * Only complete and successfully initialized sessions allow for + * calling error handler. + */ + while (ses_state == SESSION_STATE_READY) + { + spinlock_acquire(&session->ses_lock); + ses_state = session->state; + spinlock_release(&session->ses_lock); + } + + if (ses_state != SESSION_STATE_ROUTER_READY) + { + gwbuf_free(errbuf); + goto retblock; + } #if defined(SS_DEBUG) LOGIF(LE, (skygw_log_write_flush( LOGFILE_ERROR, "Backend hangup error handling."))); #endif - - errbuf = mysql_create_custom_error( - 1, - 0, - "Lost connection to backend server."); - router->handleError(router_instance, rsession, errbuf, @@ -958,7 +1005,8 @@ gw_backend_hangup(DCB *dcb) &succp); /** There are not required backends available, close session. */ - if (!succp) { + if (!succp) + { #if defined(SS_DEBUG) LOGIF(LE, (skygw_log_write_flush( LOGFILE_ERROR, @@ -971,7 +1019,8 @@ gw_backend_hangup(DCB *dcb) } dcb_close(dcb); - return 1; +retblock: + return 1; } /** diff --git a/server/modules/protocol/mysql_client.c b/server/modules/protocol/mysql_client.c index b9d6e32e9..8454516e2 100644 --- a/server/modules/protocol/mysql_client.c +++ b/server/modules/protocol/mysql_client.c @@ -65,7 +65,7 @@ static int gw_client_hangup_event(DCB *dcb); int mysql_send_ok(DCB *dcb, int packet_number, int in_affected_rows, const char* mysql_message); int MySQLSendHandshake(DCB* dcb); static int gw_mysql_do_authentication(DCB *dcb, GWBUF *queue); -static int route_by_statement(SESSION *, GWBUF *); +static int route_by_statement(SESSION *, GWBUF **); /* * The "module object" for the mysqld client protocol module. @@ -534,7 +534,7 @@ int gw_read_client_event( if (nbytes_read == 0) { goto return_rc; - } + } /** * if read queue existed appent read to it. * if length of read buffer is less than 3 or less than mysql packet @@ -783,7 +783,7 @@ int gw_read_client_event( * Feed each statement completely and separately * to router. */ - rc = route_by_statement(session, read_buffer); + rc = route_by_statement(session, &read_buffer); if (read_buffer != NULL) { @@ -1300,12 +1300,19 @@ static int gw_error_client_event( STRDCBSTATE(dcb->state), (session != NULL ? session : NULL)))); + if (session != NULL && session->state == SESSION_STATE_STOPPING) + { + goto retblock; + } + #if defined(SS_DEBUG) LOGIF(LE, (skygw_log_write_flush( LOGFILE_ERROR, "Client error event handling."))); #endif dcb_close(dcb); + +retblock: return 1; } @@ -1380,12 +1387,19 @@ gw_client_hangup_event(DCB *dcb) { CHK_SESSION(session); } + + if (session != NULL && session->state == SESSION_STATE_STOPPING) + { + goto retblock; + } #if defined(SS_DEBUG) LOGIF(LE, (skygw_log_write_flush( LOGFILE_ERROR, - "Client hangup error handling."))); + "Client hangup error handling."))); #endif dcb_close(dcb); + +retblock: return 1; } @@ -1399,14 +1413,16 @@ gw_client_hangup_event(DCB *dcb) * Return 1 in success. If the last packet is incomplete return success but * leave incomplete packet to readbuf. */ -static int route_by_statement(SESSION *session, GWBUF *readbuf) +static int route_by_statement( + SESSION* session, + GWBUF** p_readbuf) { int rc = -1; GWBUF* packetbuf; #if defined(SS_DEBUG) GWBUF* tmpbuf; - tmpbuf = readbuf; + tmpbuf = *p_readbuf; while (tmpbuf != NULL) { ss_dassert(GWBUF_IS_TYPE_MYSQL(tmpbuf)); @@ -1415,15 +1431,14 @@ static int route_by_statement(SESSION *session, GWBUF *readbuf) #endif do { - ss_dassert(GWBUF_IS_TYPE_MYSQL(readbuf)); + ss_dassert(GWBUF_IS_TYPE_MYSQL((*p_readbuf))); - packetbuf = gw_MySQL_get_next_packet(&readbuf); - - ss_dassert(GWBUF_IS_TYPE_MYSQL(packetbuf)); + packetbuf = gw_MySQL_get_next_packet(p_readbuf); if (packetbuf != NULL) { CHK_GWBUF(packetbuf); + ss_dassert(GWBUF_IS_TYPE_MYSQL(packetbuf)); /** * This means that buffer includes exactly one MySQL * statement. @@ -1446,7 +1461,7 @@ static int route_by_statement(SESSION *session, GWBUF *readbuf) goto return_rc; } } - while (readbuf != NULL); + while (*p_readbuf != NULL); return_rc: return rc; diff --git a/server/modules/protocol/mysql_common.c b/server/modules/protocol/mysql_common.c index 70050a13c..f9c0ebdea 100644 --- a/server/modules/protocol/mysql_common.c +++ b/server/modules/protocol/mysql_common.c @@ -79,6 +79,7 @@ MySQLProtocol* mysql_protocol_init( strerror(eno)))); goto return_p; } + p->protocol_state = MYSQL_PROTOCOL_ALLOC; p->protocol_auth_state = MYSQL_ALLOC; p->protocol_command.scom_cmd = MYSQL_COM_UNDEFINED; p->protocol_command.scom_nresponse_packets = 0; @@ -90,6 +91,7 @@ MySQLProtocol* mysql_protocol_init( /*< Assign fd with protocol */ p->fd = fd; p->owner_dcb = dcb; + p->protocol_state = MYSQL_PROTOCOL_ACTIVE; CHK_PROTOCOL(p); return_p: return p; @@ -107,15 +109,30 @@ return_p: void mysql_protocol_done ( DCB* dcb) { - server_command_t* scmd = ((MySQLProtocol *)dcb->protocol)->protocol_cmd_history; + MySQLProtocol* p; + server_command_t* scmd; server_command_t* scmd2; + p = (MySQLProtocol *)dcb->protocol; + + spinlock_acquire(&p->protocol_lock); + + if (p->protocol_state != MYSQL_PROTOCOL_ACTIVE) + { + goto retblock; + } + scmd = p->protocol_cmd_history; + while (scmd != NULL) { scmd2 = scmd->scom_next; free(scmd); scmd = scmd2; } + p->protocol_state = MYSQL_PROTOCOL_DONE; + +retblock: + spinlock_release(&p->protocol_lock); } @@ -886,7 +903,7 @@ int mysql_send_com_quit( if (buf == NULL) { return 0; - } + } nbytes = dcb->func.write(dcb, buf); return nbytes; @@ -1497,6 +1514,9 @@ GWBUF* gw_MySQL_get_next_packet( size_t packetlen; size_t totalbuflen; uint8_t* data; + size_t nbytes_copied = 0; + uint8_t* target; + readbuf = *p_readbuf; if (readbuf == NULL) @@ -1523,42 +1543,27 @@ GWBUF* gw_MySQL_get_next_packet( packetbuf = NULL; goto return_packetbuf; } - /** there is one complete packet in the buffer */ - if (packetlen == buflen) - { - packetbuf = gwbuf_clone_portion(readbuf, 0, packetlen); - *p_readbuf = gwbuf_consume(readbuf, packetlen); - goto return_packetbuf; - } + + packetbuf = gwbuf_alloc(packetlen); + target = GWBUF_DATA(packetbuf); + packetbuf->gwbuf_type = readbuf->gwbuf_type; /*< Copy the type too */ /** - * Packet spans multiple buffers. - * Allocate buffer for complete packet - * copy packet parts into it and consume copied bytes - */ - else if (packetlen > buflen) + * Copy first MySQL packet to packetbuf and leave posible other + * packets to read buffer. + */ + while (nbytes_copied < packetlen && totalbuflen > 0) { - size_t nbytes_copied = 0; - uint8_t* target; + uint8_t* src = GWBUF_DATA((*p_readbuf)); + size_t bytestocopy; - packetbuf = gwbuf_alloc(packetlen); - target = GWBUF_DATA(packetbuf); - - while (nbytes_copied < packetlen) - { - uint8_t* src = GWBUF_DATA(readbuf); - size_t buflen = GWBUF_LENGTH(readbuf); - - memcpy(target+nbytes_copied, src, buflen); - *p_readbuf = gwbuf_consume(readbuf, buflen); - nbytes_copied += buflen; - } - ss_dassert(nbytes_copied == packetlen); - } - else - { - packetbuf = gwbuf_clone_portion(readbuf, 0, packetlen); - *p_readbuf = gwbuf_consume(readbuf, packetlen); + bytestocopy = MIN(buflen,packetlen-nbytes_copied); + + memcpy(target+nbytes_copied, src, bytestocopy); + *p_readbuf = gwbuf_consume((*p_readbuf), bytestocopy); + totalbuflen = gwbuf_length((*p_readbuf)); + nbytes_copied += bytestocopy; } + ss_dassert(buflen == 0 || nbytes_copied == packetlen); return_packetbuf: return packetbuf; @@ -1625,11 +1630,16 @@ void protocol_archive_srv_command( MySQLProtocol* p) { server_command_t* s1; - server_command_t** s2; + server_command_t* h1; int len = 0; spinlock_acquire(&p->protocol_lock); + if (p->protocol_state != MYSQL_PROTOCOL_ACTIVE) + { + goto retblock; + } + s1 = &p->protocol_command; LOGIF(LT, (skygw_log_write( @@ -1639,18 +1649,19 @@ void protocol_archive_srv_command( p->owner_dcb->fd))); /** Copy to history list */ - s2 = &p->protocol_cmd_history; - - if (*s2 != NULL) - { - while ((*s2)->scom_next != NULL) - { - *s2 = (*s2)->scom_next; - len += 1; - } + if ((h1 = p->protocol_cmd_history) == NULL) + { + p->protocol_cmd_history = server_command_copy(s1); } - *s2 = server_command_copy(s1); - + else + { + while (h1->scom_next != NULL) + { + h1 = h1->scom_next; + } + h1->scom_next = server_command_copy(s1); + } + /** Keep history limits, remove oldest */ if (len > MAX_CMD_HISTORY) { @@ -1669,6 +1680,8 @@ void protocol_archive_srv_command( p->protocol_command = *(s1->scom_next); free(s1->scom_next); } + +retblock: spinlock_release(&p->protocol_lock); } @@ -1685,6 +1698,10 @@ void protocol_add_srv_command( spinlock_acquire(&p->protocol_lock); + if (p->protocol_state != MYSQL_PROTOCOL_ACTIVE) + { + goto retblock; + } /** this is the only server command in protocol */ if (p->protocol_command.scom_cmd == MYSQL_COM_UNDEFINED) { @@ -1717,6 +1734,7 @@ void protocol_add_srv_command( c = c->scom_next; } #endif +retblock: spinlock_release(&p->protocol_lock); } diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index ffd6bf62c..46b6c5cf8 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -1725,8 +1725,10 @@ static void clientReply ( (uint8_t *)GWBUF_DATA((scur->scmd_cur_cmd->my_sescmd_buf)); size_t len = MYSQL_GET_PACKET_LEN(buf); char* cmdstr = (char *)malloc(len+1); + /** data+termination character == len */ + snprintf(cmdstr, len, "%s", &buf[5]); - snprintf(cmdstr, len+1, "%s", &buf[5]); + ss_dassert(len+4 == GWBUF_LENGTH(scur->scmd_cur_cmd->my_sescmd_buf)); LOGIF(LE, (skygw_log_write_flush( LOGFILE_ERROR, diff --git a/server/modules/routing/readwritesplit/test/rwsplit.sh b/server/modules/routing/readwritesplit/test/rwsplit.sh index 36199e384..c2e403b16 100755 --- a/server/modules/routing/readwritesplit/test/rwsplit.sh +++ b/server/modules/routing/readwritesplit/test/rwsplit.sh @@ -230,3 +230,47 @@ if [ "$a" != "$TRETVAL" ]; then else echo "$TINPUT PASSED">>$TLOG ; fi +echo "-----------------------------------" >> $TLOG +echo "Session variables: Stress Test 1" >> $TLOG +echo "-----------------------------------" >> $TLOG + +RUNCMD=mysql\ --host=$THOST\ -P$TPORT\ -u$TUSER\ -p$TPWD\ --unbuffered=true\ --disable-reconnect\ -q\ -r +TINPUT=test_sescmd2.sql +for ((i = 0;i<1000;i++)) +do + a=`$RUNCMD < $TINPUT 2>&1` + if [[ "`echo "$a"|grep -i 'error'`" != "" ]] + then + err=`echo "$a" | grep -i error` + break + fi +done +if [[ "$err" == "" ]] +then + echo "TEST PASSED" >> $TLOG +else + echo "$err" >> $TLOG + echo "Test FAILED at iteration $((i+1))" >> $TLOG +fi +echo "-----------------------------------" >> $TLOG +echo "Session variables: Stress Test 2" >> $TLOG +echo "-----------------------------------" >> $TLOG + +err="" +TINPUT=test_sescmd3.sql +for ((j = 0;j<1000;j++)) +do + b=`$RUNCMD < $TINPUT 2>&1` + if [[ "`echo "$b"|grep -i 'null'`" != "" ]] + then + err=`echo "$b" | grep -i null` + break + fi +done +if [[ "$err" == "" ]] +then + echo "TEST PASSED" >> $TLOG +else + echo "Test FAILED at iteration $((j+1))" >> $TLOG +fi +echo "" >> $TLOG diff --git a/server/modules/routing/readwritesplit/test/test_sescmd2.sql b/server/modules/routing/readwritesplit/test/test_sescmd2.sql new file mode 100644 index 000000000..5cd179dfc --- /dev/null +++ b/server/modules/routing/readwritesplit/test/test_sescmd2.sql @@ -0,0 +1,20 @@ +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; +/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; diff --git a/server/modules/routing/readwritesplit/test/test_sescmd3.sql b/server/modules/routing/readwritesplit/test/test_sescmd3.sql new file mode 100644 index 000000000..ef5e9ed72 --- /dev/null +++ b/server/modules/routing/readwritesplit/test/test_sescmd3.sql @@ -0,0 +1,16 @@ +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; +select @OLD_SQL_NOTES; +select @OLD_CHARACTER_SET_CLIENT; +select @OLD_CHARACTER_SET_RESULTS; +select @OLD_COLLATION_CONNECTION; +select @OLD_TIME_ZONE; +select @OLD_UNIQUE_CHECKS; +select @OLD_FOREIGN_KEY_CHECKS; +select @OLD_SQL_MODE; From fb87f683687aebfbf0a197f5d1fe7608867ae048 Mon Sep 17 00:00:00 2001 From: Hartmut Holzgraefe Date: Sun, 17 Aug 2014 21:02:24 +0000 Subject: [PATCH 21/60] consistently use DEL instead of rm in Makefiles --- client/Makefile | 9 ++++++--- log_manager/makefile | 2 +- query_classifier/makefile | 2 +- server/Makefile | 5 ++++- server/core/Makefile | 6 +++--- server/modules/filter/Makefile | 4 ++-- server/modules/monitor/Makefile | 6 ++++-- server/modules/protocol/Makefile | 4 ++-- server/modules/routing/Makefile | 4 ++-- server/modules/routing/readwritesplit/Makefile | 4 ++-- 10 files changed, 27 insertions(+), 19 deletions(-) diff --git a/client/Makefile b/client/Makefile index 216ef23ca..19f38785f 100644 --- a/client/Makefile +++ b/client/Makefile @@ -20,6 +20,9 @@ # client program # 18/06/14 Mark Riddoch Addition of conditional for histedit +include ../build_gateway.inc +include ../makefile.inc + ifeq ($(wildcard /usr/include/histedit.h), ) HISTLIB= HISTFLAG= @@ -63,14 +66,14 @@ maxadmin: $(OBJ) clean: - rm -f $(OBJ) maxadmin - - rm *.so + $(DEL) $(OBJ) maxadmin + $(DEL) *.so tags: ctags $(SRCS) $(HDRS) depend: - @rm -f depend.mk + @$(DEL) depend.mk cc -M $(CFLAGS) $(SRCS) > depend.mk install: maxadmin diff --git a/log_manager/makefile b/log_manager/makefile index b1386bbcc..a020c7932 100644 --- a/log_manager/makefile +++ b/log_manager/makefile @@ -44,7 +44,7 @@ install: liblink install liblog_manager.so.1.0.1 liblog_manager.so $(DEST)/lib depend: - @rm -f depend + @$(DEL) depend $(CPP) -M $(CFLAGS) \ $(MYSQL_HEADERS) \ -I$(UTILS_PATH) -I./ \ diff --git a/query_classifier/makefile b/query_classifier/makefile index efa456a01..b08f0a9bd 100644 --- a/query_classifier/makefile +++ b/query_classifier/makefile @@ -62,7 +62,7 @@ install: liblink install ./libquery_classifier.so.1.0.1 ./libquery_classifier.so $(DEST)/lib depend: - @rm -f depend + @$(DEL) depend $(CPP) -M $(CFLAGS) \ $(MYSQL_HEADERS) \ -I$(LOG_MANAGER_PATH) \ diff --git a/server/Makefile b/server/Makefile index 09120da07..fda9d14c1 100644 --- a/server/Makefile +++ b/server/Makefile @@ -24,6 +24,9 @@ # 08/07/13 Mark Riddoch Addition of monitor modules # 16/07/13 Mark Riddoch Renamed things to match the new naming +include ../build_gateway.inc +include ../makefile.inc + DEST=$(HOME)/usr/local/skysql all: @@ -45,7 +48,7 @@ testall: $(MAKE) -C test HAVE_SRV=$(HAVE_SRV) testall clean: - (cd Documentation; rm -rf html) + (cd Documentation; $(DEL) html) (cd core; touch depend.mk ; make clean) (cd modules/routing; touch depend.mk ; make clean) (cd modules/protocol; touch depend.mk ; make clean) diff --git a/server/core/Makefile b/server/core/Makefile index 9bf650c34..9a807a925 100644 --- a/server/core/Makefile +++ b/server/core/Makefile @@ -108,14 +108,14 @@ maxpasswd: $(POBJS) echo '#define MAXSCALE_VERSION "'`cat ../../VERSION`'"' > ../include/version.h clean: - rm -f $(OBJ) maxscale - - rm *.so + $(DEL) $(OBJ) maxscale + $(DEL) *.so tags: ctags $(SRCS) $(HDRS) depend: ../include/version.h - @rm -f depend.mk + @$(DEL) depend.mk cc -M $(CFLAGS) $(SRCS) > depend.mk install: maxscale maxkeys maxpasswd diff --git a/server/modules/filter/Makefile b/server/modules/filter/Makefile index 931c35ab1..8e6036e91 100644 --- a/server/modules/filter/Makefile +++ b/server/modules/filter/Makefile @@ -69,13 +69,13 @@ libtee.so: $(TEEOBJ) $(CC) $(CFLAGS) $< -o $@ clean: - rm -f $(OBJ) $(MODULES) + $(DEL) $(OBJ) $(MODULES) tags: ctags $(SRCS) $(HDRS) depend: - @rm -f depend.mk + @$(DEL) depend.mk cc -M $(CFLAGS) $(SRCS) > depend.mk install: $(MODULES) diff --git a/server/modules/monitor/Makefile b/server/modules/monitor/Makefile index 7fdbc5843..8b60824ea 100644 --- a/server/modules/monitor/Makefile +++ b/server/modules/monitor/Makefile @@ -19,6 +19,8 @@ # 08/07/13 Mark Riddoch Initial implementation include ../../../build_gateway.inc +include ../../../makefile.inc + LOGPATH := $(ROOT_PATH)/log_manager UTILSPATH := $(ROOT_PATH)/utils @@ -55,13 +57,13 @@ libgaleramon.so: $(GALERAOBJ) $(CC) $(CFLAGS) $< -o $@ clean: - rm -f $(OBJ) $(MODULES) + $(DEL) $(OBJ) $(MODULES) tags: ctags $(SRCS) $(HDRS) depend: - @rm -f depend.mk + @$(DEL) depend.mk cc -M $(CFLAGS) $(SRCS) > depend.mk install: $(MODULES) diff --git a/server/modules/protocol/Makefile b/server/modules/protocol/Makefile index 54a8f8c38..0b41d329e 100644 --- a/server/modules/protocol/Makefile +++ b/server/modules/protocol/Makefile @@ -76,7 +76,7 @@ libmaxscaled.so: $(MAXSCALEDOBJ) $(CC) $(CFLAGS) $< -o $@ clean: - rm -f $(OBJ) $(MODULES) + $(DEL) $(OBJ) $(MODULES) tags: ctags $(SRCS) $(HDRS) @@ -85,7 +85,7 @@ install: $(MODULES) install -D $(MODULES) $(DEST)/modules depend: - rm -f depend.mk + @$(DEL) depend.mk cc -M $(CFLAGS) $(SRCS) > depend.mk include depend.mk diff --git a/server/modules/routing/Makefile b/server/modules/routing/Makefile index 4feac68fb..8287bdaea 100644 --- a/server/modules/routing/Makefile +++ b/server/modules/routing/Makefile @@ -71,7 +71,7 @@ libreadwritesplit.so: $(CC) $(CFLAGS) $< -o $@ clean: - rm -f $(OBJ) $(MODULES) + $(DEL) $(OBJ) $(MODULES) (cd readwritesplit; touch depend.mk; make clean) tags: @@ -79,7 +79,7 @@ tags: (cd readwritesplit; make tags) depend: - @rm -f depend.mk + @$(DEL) depend.mk cc -M $(CFLAGS) $(SRCS) > depend.mk (cd readwritesplit; touch depend.mk ; make depend) diff --git a/server/modules/routing/readwritesplit/Makefile b/server/modules/routing/readwritesplit/Makefile index c60f2ffd1..80f2ec572 100644 --- a/server/modules/routing/readwritesplit/Makefile +++ b/server/modules/routing/readwritesplit/Makefile @@ -50,13 +50,13 @@ libreadwritesplit.so: $(OBJ) $(CC) $(CFLAGS) $< -o $@ clean: - rm -f $(OBJ) $(MODULES) + $(DEL) $(OBJ) $(MODULES) tags: ctags $(SRCS) $(HDRS) depend: - @rm -f depend.mk + @$(DEL) depend.mk cc -M $(CFLAGS) $(SRCS) > depend.mk install: $(MODULES) From 544e64a301a9d7ae1658135bb3403f0df50b704e Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Mon, 18 Aug 2014 14:19:46 +0300 Subject: [PATCH 22/60] query_classifier: implemented skygw_get_canonical which returns a copy of original string with given substrings being replaced with questionmarks. skygw_utils.cc: added string replacement function for use of skygw_get_canonical --- query_classifier/query_classifier.cc | 44 +++++++++++++++++- query_classifier/query_classifier.h | 2 + utils/skygw_utils.cc | 68 +++++++++++++++++++++++++++- utils/skygw_utils.h | 2 + 4 files changed, 114 insertions(+), 2 deletions(-) diff --git a/query_classifier/query_classifier.cc b/query_classifier/query_classifier.cc index 889940e00..184ed4f9d 100644 --- a/query_classifier/query_classifier.cc +++ b/query_classifier/query_classifier.cc @@ -412,7 +412,7 @@ static skygw_query_type_t resolve_query_type( ss_info_dassert(thd != NULL, ("thd is NULL\n")); - force_data_modify_op_replication = FALSE; + force_data_modify_op_replication = FALSE; lex = thd->lex; /** SELECT ..INTO variable|OUTFILE|DUMPFILE */ @@ -816,3 +816,45 @@ char* skygw_query_classifier_get_stmtname( return ((THD *)(mysql->thd))->lex->prepared_stmt_name.str; } + +char* skygw_get_canonical( + MYSQL* mysql, + char* querystr) +{ + THD* thd; + LEX* lex; + bool found = false; + char* newstr; + + thd = (THD *)mysql->thd; + lex = thd->lex; + Item* item; + + for (item=thd->free_list; item != NULL; item=item->next) + { + Item::Type itype; + + itype = item->type(); + + if (itype == Item::STRING_ITEM || itype == Item::INT_ITEM) + { + if (!found) + { + newstr = replace_str(querystr, + item->name, + "?"); + found = true; + } + else + { + char* prevstr = newstr; + + newstr = replace_str(prevstr, + item->name, + "?"); + free(prevstr); + } + } + } /*< for */ + return newstr; +} \ No newline at end of file diff --git a/query_classifier/query_classifier.h b/query_classifier/query_classifier.h index 31f9cf44e..fa7cefc78 100644 --- a/query_classifier/query_classifier.h +++ b/query_classifier/query_classifier.h @@ -60,6 +60,8 @@ skygw_query_type_t skygw_query_classifier_get_type( /** Free THD context and close MYSQL */ void skygw_query_classifier_free(MYSQL* mysql); char* skygw_query_classifier_get_stmtname(MYSQL* mysql); +char* skygw_get_canonical(MYSQL* mysql, char* querystr); + EXTERN_C_BLOCK_END diff --git a/utils/skygw_utils.cc b/utils/skygw_utils.cc index 1ed569962..c7175c76f 100644 --- a/utils/skygw_utils.cc +++ b/utils/skygw_utils.cc @@ -23,7 +23,7 @@ #include #include #include - +#include #include "skygw_debug.h" #include "skygw_types.h" #include "skygw_utils.h" @@ -1863,3 +1863,69 @@ void skygw_file_done( free(file); } } + +/** + * Replaces in the string str all the occurrences of the source string old with + * the destination string new. The lengths of the strings old and new may differ. + * The string new may be of any length, but the string "old" must be of non-zero + * length - the penalty for providing an empty string for the "old" parameter is + * an infinite loop. In addition, none of the three parameters may be NULL. + * + * @param str String to be modified + * @param old Substring to be replaced + * @param new Replacement + * @return String with replacements in new memory area or NULL if memory + * allocation failed. + * Dependencies: For this function to compile, you will need to also #include + * the following files: , and . + * + * Thanks, to Laird Shaw who implemented most of this function. + */ +char* replace_str ( + const char *str, + const char *old, + const char *replacement) +{ + char* ret; + char* r; + const char* p; + const char* q; + size_t oldlen; + size_t count; + size_t retlen; + size_t newlen; + + oldlen = strlen(old); + newlen = strlen(replacement); + + if (oldlen != newlen) + { + for (count = 0, p = str; (q = strstr(p, old)) != NULL; p = q + oldlen) + { + count++; + } + /* this is undefined if p - str > PTRDIFF_MAX */ + retlen = p - str + strlen(p) + count * (newlen - oldlen); + } + else + { + retlen = strlen(str); + } + if ((ret = (char *)malloc(retlen + 1)) == NULL) + { + return NULL; + } + + for (r = ret, p = str; (q = strstr(p, old)) != NULL; p = q + oldlen) + { + /* this is undefined if q - p > PTRDIFF_MAX */ + ptrdiff_t l = q - p; + memcpy(r, p, l); + r += l; + memcpy(r, replacement, newlen); + r += newlen; + } + strcpy(r, p); + + return ret; +} \ No newline at end of file diff --git a/utils/skygw_utils.h b/utils/skygw_utils.h index 992f169d2..854c82541 100644 --- a/utils/skygw_utils.h +++ b/utils/skygw_utils.h @@ -191,5 +191,7 @@ int skygw_rwlock_unlock(skygw_rwlock_t* rwlock); int skygw_rwlock_init(skygw_rwlock_t** rwlock); int atomic_add(int *variable, int value); +char* replace_str(const char* str, const char* old, const char* replacement); + #endif /* SKYGW_UTILS_H */ From 3dc09dfe434092e9b239f6058630650280b48ac4 Mon Sep 17 00:00:00 2001 From: Mark Riddoch Date: Mon, 18 Aug 2014 18:26:14 +0100 Subject: [PATCH 23/60] Addition of spinlock unit test --- server/core/test/makefile | 23 +++++- server/core/test/testspinlock.c | 131 ++++++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+), 4 deletions(-) create mode 100644 server/core/test/testspinlock.c diff --git a/server/core/test/makefile b/server/core/test/makefile index d3948575d..5f82b7fef 100644 --- a/server/core/test/makefile +++ b/server/core/test/makefile @@ -10,6 +10,8 @@ include ../../../test.inc CC=cc TESTLOG := $(shell pwd)/testhash.log +TESTS=testhash testspinlock + cleantests: - $(DEL) *.o - $(DEL) testhash @@ -20,11 +22,18 @@ testall: $(MAKE) DEBUG=Y buildtests $(MAKE) runtests -buildtests : +buildtests : $(TESTS) + +testhash: testhash.c $(CC) $(CFLAGS) \ -I$(ROOT_PATH)/server/include \ -I$(ROOT_PATH)/utils \ testhash.c ../hashtable.o ../atomic.o ../spinlock.o -o testhash +testspinlock: testspinlock.c + $(CC) $(CFLAGS) \ + -I$(ROOT_PATH)/server/include \ + -I$(ROOT_PATH)/utils \ + testspinlock.c ../spinlock.o ../atomic.o ../thread.o -o testspinlock runtests: @echo "" > $(TESTLOG) @@ -34,9 +43,15 @@ runtests: @echo "-------------------------------" >> $(TESTLOG) @ -./testhash 2>> $(TESTLOG) ifeq ($?,0) - @echo "MaxScale core PASSED" >> $(TESTLOG) + @echo "MaxScale hashtable PASSED" >> $(TESTLOG) else - @echo "MaxScale core FAILED" >> $(TESTLOG) + @echo "MaxScale hashtable FAILED" >> $(TESTLOG) +endif + @ -./testspinlock 2>> $(TESTLOG) +ifeq ($?,0) + @echo "MaxScale spinlock PASSED" >> $(TESTLOG) +else + @echo "MaxScale spinlock FAILED" >> $(TESTLOG) endif @echo "" >> $(TESTLOG) - @cat $(TESTLOG) >> $(TEST_MAXSCALE_LOG) \ No newline at end of file + @cat $(TESTLOG) >> $(TEST_MAXSCALE_LOG) diff --git a/server/core/test/testspinlock.c b/server/core/test/testspinlock.c new file mode 100644 index 000000000..5d03f0e47 --- /dev/null +++ b/server/core/test/testspinlock.c @@ -0,0 +1,131 @@ +/* + * This file is distributed as part of MaxScale. It is free + * software: you can redistribute it and/or modify it under the terms of the + * GNU General Public License as published by the Free Software Foundation, + * version 2. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 51 + * Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Copyright SkySQL Ab 2014 + */ + +/** + * + * @verbatim + * Revision History + * + * Date Who Description + * 18/08-2014 Mark Riddoch Initial implementation + * + * @endverbatim + */ + +#include +#include +#include + +#include +#include + + +/** + * test1 spinlock_acquire_nowait tests + * + * Test that spinlock_acquire_nowait returns false if the spinlock + * is already taken. + * + * Test that spinlock_acquire_nowait returns true if the spinlock + * is not taken. + * + * Test that spinlock_acquire_nowait does hold the spinlock. + */ +static int +test1() +{ +SPINLOCK lck; + + spinlock_init(&lck); + spinlock_acquire(&lck); + if (spinlock_acquire_nowait(&lck)) + { + fprintf(stderr, "spinlock_acquire_nowait: test 1 failed.\n"); + return 1; + } + spinlock_release(&lck); + if (!spinlock_acquire_nowait(&lck)) + { + fprintf(stderr, "spinlock_acquire_nowait: test 2 failed.\n"); + return 1; + } + if (spinlock_acquire_nowait(&lck)) + { + fprintf(stderr, "spinlock_acquire_nowait: test 3 failed.\n"); + return 1; + } + spinlock_release(&lck); + + return 0; +} + +static int acquire_time; + +static void +test2_helper(void *data) +{ +SPINLOCK *lck = (SPINLOCK *)data; +unsigned long t1 = time(0); + + spinlock_acquire(lck); + acquire_time = time(0) - t1; + spinlock_release(lck); + return; +} + +/** + * Check that spinlock correctly blocks another thread whilst the spinlock + * is held. + * + * Take out a lock. + * Start a second thread to take the same lock + * sleep for 10 seconds + * release lock + * verify that second thread took at least 8 seconds to obtain the lock + */ +static int +test2() +{ +SPINLOCK lck; +void *handle; + + acquire_time = 0; + spinlock_init(&lck); + spinlock_acquire(&lck); + handle = thread_start(test2_helper, (void *)&lck); + sleep(10); + spinlock_release(&lck); + thread_wait(handle); + + if (acquire_time < 8) + { + fprintf(stderr, "spinlock: test 1 failed.\n"); + return 1; + } + return 0; +} + +main(int argc, char **argv) +{ +int result = 0; + + result += test1(); + + exit(result); +} + From fb3a950a18bb08abef763f2883c5ad18ac720ecc Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Tue, 19 Aug 2014 09:07:18 +0300 Subject: [PATCH 24/60] Completed skygw_get_canonical by adding NULL checks and debug assertions. Replacable literal types are now INT_ITEM, STRING_ITEM, DECIMAL_ITEM, REAL_ITEM, VARBIN_ITEM and NULL_ITEM. --- query_classifier/query_classifier.cc | 46 +++++++++++++++++++++------- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/query_classifier/query_classifier.cc b/query_classifier/query_classifier.cc index 184ed4f9d..77b939819 100644 --- a/query_classifier/query_classifier.cc +++ b/query_classifier/query_classifier.cc @@ -817,6 +817,20 @@ char* skygw_query_classifier_get_stmtname( } +/** + * Replace user-provided literals with question marks. Return a copy of the + * querystr with replacements. + * + * @param mysql Database pointer + * @param querystr Query string + * + * @return Copy of querystr where literals are replaces with question marks or + * NULL if querystr is NULL, thread context or lex are NULL or if replacement + * function fails. + * + * Replaced literal types are STRING_ITEM,INT_ITEM,DECIMAL_ITEM,REAL_ITEM, + * VARBIN_ITEM,NULL_ITEM + */ char* skygw_get_canonical( MYSQL* mysql, char* querystr) @@ -824,11 +838,19 @@ char* skygw_get_canonical( THD* thd; LEX* lex; bool found = false; - char* newstr; + char* newstr = NULL; + Item* item; - thd = (THD *)mysql->thd; - lex = thd->lex; - Item* item; + ss_dassert(mysql != NULL && querystr != NULL); + + if (querystr == NULL || + mysql == NULL || + (thd = (THD *)mysql->thd) == NULL || + (lex = thd->lex) == NULL) + { + ss_dassert(thd != NULL && lex != NULL); + goto retblock; + } for (item=thd->free_list; item != NULL; item=item->next) { @@ -836,25 +858,27 @@ char* skygw_get_canonical( itype = item->type(); - if (itype == Item::STRING_ITEM || itype == Item::INT_ITEM) + if (itype == Item::STRING_ITEM || + itype == Item::INT_ITEM || + itype == Item::DECIMAL_ITEM || + itype == Item::REAL_ITEM || + itype == Item::VARBIN_ITEM || + itype == Item::NULL_ITEM) { if (!found) { - newstr = replace_str(querystr, - item->name, - "?"); + newstr = replace_str(querystr, item->name, "?"); found = true; } else { char* prevstr = newstr; - newstr = replace_str(prevstr, - item->name, - "?"); + newstr = replace_str(prevstr, item->name, "?"); free(prevstr); } } } /*< for */ +retblock: return newstr; } \ No newline at end of file From 13dfd34d5dcc00e13167280dfbb7852c0fc15a1d Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Tue, 19 Aug 2014 09:39:40 +0300 Subject: [PATCH 25/60] Test use for cacnonical query function. Effective in debug build only. --- .../routing/readwritesplit/readwritesplit.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index a0c241920..24262bc6d 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -1322,6 +1322,23 @@ static int routeQuery( } } return_ret: +#if defined(SS_DEBUG) + if (mysql != NULL && true) + { + char* canonical_query_str; + + canonical_query_str = skygw_get_canonical(mysql, querystr); + + if (canonical_query_str != NULL) + { + LOGIF(LT, (skygw_log_write( + LOGFILE_TRACE, + "Canonical version: %s", + canonical_query_str))); + free(canonical_query_str); + } + } +#endif if (plainsqlbuf != NULL) { gwbuf_free(plainsqlbuf); @@ -3647,4 +3664,3 @@ static BACKEND *get_root_master(backend_ref_t *servers, int router_nservers) { } return master_host; } - From d6a2ef699631a5ae5c45d9cdc4ccf593e2deec24 Mon Sep 17 00:00:00 2001 From: Mark Riddoch Date: Tue, 19 Aug 2014 13:22:40 +0100 Subject: [PATCH 26/60] Misc fixes to doxygen comments --- server/core/buffer.c | 8 +++++--- server/core/dcb.c | 15 ++++++++------- server/core/filter.c | 12 ++++++------ server/core/load_utils.c | 1 + server/core/maxpasswd.c | 2 +- server/core/monitor.c | 5 +++-- server/core/server.c | 3 +-- server/core/utils.c | 2 +- server/include/filter.h | 20 ++++++++++---------- server/modules/filter/qlafilter.c | 5 +++++ server/modules/filter/regexfilter.c | 5 ++++- server/modules/filter/tee.c | 3 +++ server/modules/filter/testfilter.c | 5 ++++- server/modules/filter/topfilter.c | 6 ++++++ 14 files changed, 58 insertions(+), 34 deletions(-) diff --git a/server/core/buffer.c b/server/core/buffer.c index d3278a505..26c2c1439 100644 --- a/server/core/buffer.c +++ b/server/core/buffer.c @@ -298,11 +298,13 @@ int rval = 0; } /** - * Trim bytes form the end of a GWBUF structure + * Trim bytes form the end of a GWBUF structure. If the + * buffer has n_bytes or less then it will be freed and + * NULL will be returned. * * @param buf The buffer to trim - * @param nbytes The number of bytes to trim off - * @return The buffer chain + * @param n_bytes The number of bytes to trim off + * @return The buffer chain or NULL if buffer has <= n_bytes */ GWBUF * gwbuf_trim(GWBUF *buf, unsigned int n_bytes) diff --git a/server/core/dcb.c b/server/core/dcb.c index 1dfde0b58..969702cf4 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -1703,14 +1703,15 @@ int gw_write( /** * Add a callback * - * Duplicate registrations are not allowed, therefore an error will be returned if - * the specific function, reason and userdata triple are already registered. + * Duplicate registrations are not allowed, therefore an error will be + * returned if the specific function, reason and userdata triple + * are already registered. * An error will also be returned if the is insufficient memeory available to * create the registration. * * @param dcb The DCB to add the callback to * @param reason The callback reason - * @param cb The callback function to call + * @param callback The callback function to call * @param userdata User data to send in the call * @return Non-zero (true) if the callback was added */ @@ -1766,7 +1767,7 @@ int rval = 1; * * @param dcb The DCB to add the callback to * @param reason The callback reason - * @param cb The callback function to call + * @param callback The callback function to call * @param userdata User data to send in the call * @return Non-zero (true) if the callback was removed */ @@ -1839,7 +1840,7 @@ DCB_CALLBACK *cb, *nextcb; /** * Check the passed DCB to ensure it is in the list of allDCBS * - * @param DCB The DCB to check + * @param dcb The DCB to check * @return 1 if the DCB is in the list, otherwise 0 */ int @@ -1936,8 +1937,8 @@ void dcb_call_foreach ( * Null protocol write routine used for cloned dcb's. It merely consumes * buffers written on the cloned DCB. * - * @params dcb The descriptor control block - * @params buf The buffer beign written + * @param dcb The descriptor control block + * @param buf The buffer being written * @return Always returns a good write operation result */ static int diff --git a/server/core/filter.c b/server/core/filter.c index baeea21d7..405a01470 100644 --- a/server/core/filter.c +++ b/server/core/filter.c @@ -39,8 +39,8 @@ extern int lm_enabled_logfiles_bitmask; -static SPINLOCK filter_spin = SPINLOCK_INIT; -static FILTER_DEF *allFilters = NULL; +static SPINLOCK filter_spin = SPINLOCK_INIT; /**< Protects the list of all filters */ +static FILTER_DEF *allFilters = NULL; /**< The list of all filters */ /** * Allocate a new filter within MaxScale @@ -79,7 +79,7 @@ FILTER_DEF *filter; /** * Deallocate the specified filter * - * @param server The service to deallocate + * @param filter The filter to deallocate * @return Returns true if the server was freed */ void @@ -243,8 +243,8 @@ int i; /** * Add a router option to a service * - * @param service The service to add the router option to - * @param option The option string + * @param filter The filter to add the option to + * @param option The option string */ void filterAddOption(FILTER_DEF *filter, char *option) @@ -273,7 +273,7 @@ int i; /** * Add a router parameter to a service * - * @param service The service to add the router option to + * @param filter The filter to add the parameter to * @param name The parameter name * @param value The parameter value */ diff --git a/server/core/load_utils.c b/server/core/load_utils.c index 93cdf95f6..cba0c6533 100644 --- a/server/core/load_utils.c +++ b/server/core/load_utils.c @@ -281,6 +281,7 @@ MODULES *mod = registered; * @param dlhandle The handle returned by dlopen * @param version The version string returned by the module * @param modobj The module object + * @param mod_info The module information */ static void register_module(const char *module, const char *type, void *dlhandle, char *version, void *modobj, MODULE_INFO *mod_info) diff --git a/server/core/maxpasswd.c b/server/core/maxpasswd.c index ac8b0a042..d11980ba3 100644 --- a/server/core/maxpasswd.c +++ b/server/core/maxpasswd.c @@ -34,7 +34,7 @@ * Encrypt a password for storing in the MaxScale.cnf file * * @param argc Argument count - * @param arv Argument vector + * @param argv Argument vector */ int main(int argc, char **argv) diff --git a/server/core/monitor.c b/server/core/monitor.c index d7b53f5b7..85ff878d7 100644 --- a/server/core/monitor.c +++ b/server/core/monitor.c @@ -207,7 +207,8 @@ MONITOR *ptr; /** * Show a single monitor * - * @param dcb DCB for printing output + * @param dcb DCB for printing output + * @param monitor The monitor to print information regarding */ void monitorShow(DCB *dcb, MONITOR *monitor) @@ -303,7 +304,7 @@ monitorSetInterval (MONITOR *mon, unsigned long interval) * Enable Replication Heartbeat support in monitor. * * @param mon The monitor instance - * @param interval The sampling interval in milliseconds + * @param replication_heartbeat The replication heartbeat */ void monitorSetReplicationHeartbeat(MONITOR *mon, int replication_heartbeat) diff --git a/server/core/server.c b/server/core/server.c index bfe5e4d04..5a75756ed 100644 --- a/server/core/server.c +++ b/server/core/server.c @@ -148,8 +148,7 @@ server_set_unique_name(SERVER *server, char *name) * Find an existing server using the unique section name in * configuration file * - * @param servname The Server name or address - * @param port The server port + * @param name The Server name defined in the header file * @return The server or NULL if not found */ SERVER * diff --git a/server/core/utils.c b/server/core/utils.c index c7e42dda1..1b8c6b5fe 100644 --- a/server/core/utils.c +++ b/server/core/utils.c @@ -204,7 +204,7 @@ void gw_sha1_2_str(const uint8_t *in, int in_len, const uint8_t *in2, int in2_le /** - * @node Gets errno corresponding to latest socket error + * node Gets errno corresponding to latest socket error * * Parameters: * @param fd - in, use diff --git a/server/include/filter.h b/server/include/filter.h index 568df291a..3076587e6 100644 --- a/server/include/filter.h +++ b/server/include/filter.h @@ -61,7 +61,7 @@ typedef struct { * filter pipline * routeQuery Called on each query that requires * routing - * clientReply + * clientReply Called for each reply packet * diagnostics Called to force the filter to print * diagnostic output * @@ -88,21 +88,21 @@ typedef struct filter_object { */ #define FILTER_VERSION {1, 1, 0} /** - * The definition of a filter form the configuration file. + * The definition of a filter from the configuration file. * This is basically the link between a plugin to load and the * optons to pass to that plugin. */ typedef struct filter_def { - char *name; /*< The Filter name */ - char *module; /*< The module to load */ - char **options; /*< The options set for this filter */ + char *name; /**< The Filter name */ + char *module; /**< The module to load */ + char **options; /**< The options set for this filter */ FILTER_PARAMETER - **parameters; /*< The filter parameters */ - FILTER filter; - FILTER_OBJECT *obj; - SPINLOCK spin; + **parameters; /**< The filter parameters */ + FILTER filter; /**< The runtime filter */ + FILTER_OBJECT *obj; /**< The "MODULE_OBJECT" for the filter */ + SPINLOCK spin; /**< Spinlock to protect the filter definition */ struct filter_def - *next; /*< Next filter in the chain of all filters */ + *next; /**< Next filter in the chain of all filters */ } FILTER_DEF; FILTER_DEF *filter_alloc(char *, char *); diff --git a/server/modules/filter/qlafilter.c b/server/modules/filter/qlafilter.c index da78713d2..b3accfbf8 100644 --- a/server/modules/filter/qlafilter.c +++ b/server/modules/filter/qlafilter.c @@ -17,6 +17,9 @@ */ /** + * @file qlafilter.c - Quary Log All Filter + * @verbatim + * * QLA Filter - Query Log All. A primitive query logging filter, simply * used to verify the filter mechanism for downstream filters. All queries * that are passed through the filter will be written to file. @@ -33,6 +36,7 @@ * 11/06/2014 Mark Riddoch Addition of source and match parameters * 19/06/2014 Mark Riddoch Addition of user parameter * + * @endverbatim */ #include #include @@ -154,6 +158,7 @@ GetModuleObject() * within MaxScale. * * @param options The options for this filter + * @param params The array of name/value pair parameters for the filter * * @return The instance data for this new instance */ diff --git a/server/modules/filter/regexfilter.c b/server/modules/filter/regexfilter.c index 5c4f1e7df..79cf70c09 100644 --- a/server/modules/filter/regexfilter.c +++ b/server/modules/filter/regexfilter.c @@ -27,7 +27,8 @@ extern int lm_enabled_logfiles_bitmask; /** - * regexfilter.c - a very simple regular expression rewrite filter. + * @file regexfilter.c - a very simple regular expression rewrite filter. + * @verbatim * * A simple regular expression query rewrite filter. * Two parameters should be defined in the filter configuration @@ -39,6 +40,7 @@ extern int lm_enabled_logfiles_bitmask; * * Date Who Description * 19/06/2014 Mark Riddoch Addition of source and user parameters + * @endverbatim */ MODULE_INFO info = { @@ -132,6 +134,7 @@ GetModuleObject() * within MaxScale. * * @param options The options for this filter + * @param params The array of name/value pair parameters for the filter * * @return The instance data for this new instance */ diff --git a/server/modules/filter/tee.c b/server/modules/filter/tee.c index 34efbb65b..f8e22c7eb 100644 --- a/server/modules/filter/tee.c +++ b/server/modules/filter/tee.c @@ -18,6 +18,7 @@ /** * @file tee.c A filter that splits the processing pipeline in two + * @verbatim * * Conditionally duplicate requests and send the duplicates to another service * within MaxScale. @@ -41,6 +42,7 @@ * 20/06/2014 Mark Riddoch Initial implementation * 24/06/2014 Mark Riddoch Addition of support for multi-packet queries * + * @endverbatim */ #include #include @@ -162,6 +164,7 @@ GetModuleObject() * within MaxScale. * * @param options The options for this filter + * @param params The array of name/value pair parameters for the filter * * @return The instance data for this new instance */ diff --git a/server/modules/filter/testfilter.c b/server/modules/filter/testfilter.c index f72471a36..289ccf4a2 100644 --- a/server/modules/filter/testfilter.c +++ b/server/modules/filter/testfilter.c @@ -21,13 +21,15 @@ #include /** - * testfilter.c - a very simple test filter. + * @file testfilter.c - a very simple test filter. + * @verbatim * * This filter is a very simple example used to test the filter API, * it merely counts the number of statements that flow through the * filter pipeline. * * Reporting is done via the diagnostics print routine. + * @endverbatim */ MODULE_INFO info = { @@ -114,6 +116,7 @@ GetModuleObject() * within MaxScale. * * @param options The options for this filter + * @param params The array of name/value pair parameters for the filter * * @return The instance data for this new instance */ diff --git a/server/modules/filter/topfilter.c b/server/modules/filter/topfilter.c index d4d594e6d..9ca281ef1 100644 --- a/server/modules/filter/topfilter.c +++ b/server/modules/filter/topfilter.c @@ -17,6 +17,9 @@ */ /** + * @file topfilter.c - Top N Longest Running Queries + * @verbatim + * * TOPN Filter - Query Log All. A primitive query logging filter, simply * used to verify the filter mechanism for downstream filters. All queries * that are passed through the filter will be written to file. @@ -30,6 +33,8 @@ * * Date Who Description * 18/06/2014 Mark Riddoch Addition of source and user filters + * + * @endverbatim */ #include #include @@ -172,6 +177,7 @@ GetModuleObject() * within MaxScale. * * @param options The options for this filter + * @param params The array of name/value pair parameters for the filter * * @return The instance data for this new instance */ From 24e16e97ed172909d30a6adf498e2c38dd7384dd Mon Sep 17 00:00:00 2001 From: Mark Riddoch Date: Wed, 20 Aug 2014 11:07:28 +0100 Subject: [PATCH 27/60] Updates for unit tests --- server/core/test/makefile | 36 +++++--- server/core/test/runtest.sh | 10 +++ server/core/test/testfilter.c | 153 ++++++++++++++++++++++++++++++++ server/core/test/testspinlock.c | 1 + 4 files changed, 186 insertions(+), 14 deletions(-) create mode 100755 server/core/test/runtest.sh create mode 100644 server/core/test/testfilter.c diff --git a/server/core/test/makefile b/server/core/test/makefile index 5f82b7fef..446027948 100644 --- a/server/core/test/makefile +++ b/server/core/test/makefile @@ -10,7 +10,18 @@ include ../../../test.inc CC=cc TESTLOG := $(shell pwd)/testhash.log -TESTS=testhash testspinlock +LOGPATH := $(ROOT_PATH)/log_manager +UTILSPATH := $(ROOT_PATH)/utils + +LDFLAGS=-rdynamic -L$(LOGPATH) \ + -Wl,-rpath,$(DEST)/lib \ + -Wl,-rpath,$(LOGPATH) -Wl,-rpath,$(UTILSPATH) \ + -Wl,-rpath,$(EMBEDDED_LIB) + +LIBS= -lz -lm -lcrypt -lcrypto -ldl -laio -lrt -pthread -llog_manager \ + -L../../inih/extra -linih -lssl -lstdc++ + +TESTS=testhash testspinlock testfilter cleantests: - $(DEL) *.o @@ -34,24 +45,21 @@ testspinlock: testspinlock.c -I$(ROOT_PATH)/server/include \ -I$(ROOT_PATH)/utils \ testspinlock.c ../spinlock.o ../atomic.o ../thread.o -o testspinlock +testfilter: testfilter.c libcore.a + $(CC) $(CFLAGS) $(LDFLAGS) \ + -I$(ROOT_PATH)/server/include \ + -I$(ROOT_PATH)/utils \ + testfilter.c libcore.a $(UTILSPATH)/skygw_utils.o $(LIBS) -o testfilter -runtests: +libcore.a: ../*.o + ar rv libcore.a ../*.o + +runtests: $(TESTS) @echo "" > $(TESTLOG) @echo "-------------------------------" >> $(TESTLOG) @echo $(shell date) >> $(TESTLOG) @echo "Test MaxScale core" >> $(TESTLOG) @echo "-------------------------------" >> $(TESTLOG) - @ -./testhash 2>> $(TESTLOG) -ifeq ($?,0) - @echo "MaxScale hashtable PASSED" >> $(TESTLOG) -else - @echo "MaxScale hashtable FAILED" >> $(TESTLOG) -endif - @ -./testspinlock 2>> $(TESTLOG) -ifeq ($?,0) - @echo "MaxScale spinlock PASSED" >> $(TESTLOG) -else - @echo "MaxScale spinlock FAILED" >> $(TESTLOG) -endif + $(foreach var,$(TESTS),./runtest.sh $(var) $(TESTLOG);) @echo "" >> $(TESTLOG) @cat $(TESTLOG) >> $(TEST_MAXSCALE_LOG) diff --git a/server/core/test/runtest.sh b/server/core/test/runtest.sh new file mode 100755 index 000000000..434d45701 --- /dev/null +++ b/server/core/test/runtest.sh @@ -0,0 +1,10 @@ +#!/bin/bash +test=$1 +log=$2 +echo Running test $test >> $log +./$test 2>> $log +if [ $? -ne 0 ]; then + echo $test " " FAILED >> $log +else + echo $test " " PASSED >> $log +fi diff --git a/server/core/test/testfilter.c b/server/core/test/testfilter.c new file mode 100644 index 000000000..eefc8ecfa --- /dev/null +++ b/server/core/test/testfilter.c @@ -0,0 +1,153 @@ +/* + * This file is distributed as part of MaxScale. It is free + * software: you can redistribute it and/or modify it under the terms of the + * GNU General Public License as published by the Free Software Foundation, + * version 2. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 51 + * Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Copyright SkySQL Ab 2014 + */ + +/** + * + * @verbatim + * Revision History + * + * Date Who Description + * 19-08-2014 Mark Riddoch Initial implementation + * + * @endverbatim + */ + +#include +#include +#include + +#include + + +/** + * test1 Filter creation, finding and deletion + * + */ +static int +test1() +{ +FILTER_DEF *f1, *f2; + + if ((f1 = filter_alloc("test1", "module")) == NULL) + { + fprintf(stderr, "filter_alloc: test 1 failed.\n"); + return 1; + } + if ((f2 = filter_find("test1")) == NULL) + { + fprintf(stderr, "filter_find: test 2 failed.\n"); + return 1; + } + filter_free(f1); + if ((f2 = filter_find("test1")) != NULL) + { + fprintf(stderr, "filter_find: test 3 failed delete.\n"); + return 1; + } + + return 0; +} + + +/** + * Passive tests for filter_add_option and filter_add_parameter + * + * These tests add options and parameters to a filter, the only failure + * is related hard crashes, such as SIGSEGV etc. as there are no good hooks + * to check the creation of parameters and options currently. + */ +static int +test2() +{ +FILTER_DEF *f1; + + if ((f1 = filter_alloc("test1", "module")) == NULL) + { + fprintf(stderr, "filter_alloc: test 1 failed.\n"); + return 1; + } + filterAddOption(f1, "option1"); + filterAddOption(f1, "option2"); + filterAddOption(f1, "option3"); + filterAddParameter(f1, "name1", "value1"); + filterAddParameter(f1, "name2", "value2"); + filterAddParameter(f1, "name3", "value3"); + return 0; +} + + +/** + * test3 Filter creation, finding and deletion soak test + * + */ +static int +test3() +{ +FILTER_DEF *f1; +char name[40]; +int i, n_filters = 1000; + + for (i = 0; i < n_filters; i++) + { + sprintf(name, "filter%d", i); + if ((f1 = filter_alloc(name, "module")) == NULL) + { + fprintf(stderr, + "filter_alloc: test 3 failed with %s.\n", name); + return 1; + } + } + for (i = 0; i < n_filters; i++) + { + sprintf(name, "filter%d", i); + if ((f1 = filter_find(name)) == NULL) + { + fprintf(stderr, "filter_find: test 3 failed.\n"); + return 1; + } + } + for (i = 0; i < n_filters; i++) + { + sprintf(name, "filter%d", i); + if ((f1 = filter_find(name)) == NULL) + { + fprintf(stderr, "filter_find: test 3 failed.\n"); + return 1; + } + filter_free(f1); + if ((f1 = filter_find(name)) != NULL) + { + fprintf(stderr, + "filter_find: test 3 failed - found deleted filter.\n"); + return 1; + } + } + + return 0; +} +main(int argc, char **argv) +{ +int result = 0; + + result += test1(); + result += test2(); + result += test3(); + + exit(result); +} + diff --git a/server/core/test/testspinlock.c b/server/core/test/testspinlock.c index 5d03f0e47..bcbfa2a3f 100644 --- a/server/core/test/testspinlock.c +++ b/server/core/test/testspinlock.c @@ -125,6 +125,7 @@ main(int argc, char **argv) int result = 0; result += test1(); + result += test2(); exit(result); } From e329d8cf13dc56312b0f15298e5da3c41d4b7883 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Wed, 20 Aug 2014 15:15:45 +0300 Subject: [PATCH 28/60] Fix to bug #479, http://bugs.skysql.com/show_bug.cgi?id=479 service.c was counting unfound filters towards the filter chain size. --- server/core/service.c | 1 + 1 file changed, 1 insertion(+) diff --git a/server/core/service.c b/server/core/service.c index 1102dabb4..080a65d8e 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -675,6 +675,7 @@ int n = 0; "Unable to find filter '%s' for service '%s'\n", trim(ptr), service->name ))); + n--; } flist[n] = NULL; ptr = strtok_r(NULL, "|", &brkt); From 65b25a825ad4449abe044c12133dea57dd52079d Mon Sep 17 00:00:00 2001 From: Mark Riddoch Date: Wed, 20 Aug 2014 14:50:44 +0100 Subject: [PATCH 29/60] Addition of adminusers unit test Fix to filters unit test --- server/core/test/makefile | 12 +- server/core/test/testadminusers.c | 278 ++++++++++++++++++++++++++++++ server/core/test/testfilter.c | 2 + server/include/adminusers.h | 2 + 4 files changed, 292 insertions(+), 2 deletions(-) create mode 100644 server/core/test/testadminusers.c diff --git a/server/core/test/makefile b/server/core/test/makefile index 446027948..7f6831082 100644 --- a/server/core/test/makefile +++ b/server/core/test/makefile @@ -8,7 +8,7 @@ include ../../../makefile.inc include ../../../test.inc CC=cc -TESTLOG := $(shell pwd)/testhash.log +TESTLOG := $(shell pwd)/testcore.log LOGPATH := $(ROOT_PATH)/log_manager UTILSPATH := $(ROOT_PATH)/utils @@ -21,7 +21,7 @@ LDFLAGS=-rdynamic -L$(LOGPATH) \ LIBS= -lz -lm -lcrypt -lcrypto -ldl -laio -lrt -pthread -llog_manager \ -L../../inih/extra -linih -lssl -lstdc++ -TESTS=testhash testspinlock testfilter +TESTS=testhash testspinlock testfilter testadminusers cleantests: - $(DEL) *.o @@ -40,17 +40,25 @@ testhash: testhash.c -I$(ROOT_PATH)/server/include \ -I$(ROOT_PATH)/utils \ testhash.c ../hashtable.o ../atomic.o ../spinlock.o -o testhash + testspinlock: testspinlock.c $(CC) $(CFLAGS) \ -I$(ROOT_PATH)/server/include \ -I$(ROOT_PATH)/utils \ testspinlock.c ../spinlock.o ../atomic.o ../thread.o -o testspinlock + testfilter: testfilter.c libcore.a $(CC) $(CFLAGS) $(LDFLAGS) \ -I$(ROOT_PATH)/server/include \ -I$(ROOT_PATH)/utils \ testfilter.c libcore.a $(UTILSPATH)/skygw_utils.o $(LIBS) -o testfilter +testadminusers: testadminusers.c libcore.a + $(CC) $(CFLAGS) $(LDFLAGS) \ + -I$(ROOT_PATH)/server/include \ + -I$(ROOT_PATH)/utils \ + testadminusers.c libcore.a $(UTILSPATH)/skygw_utils.o $(LIBS) -o testadminusers + libcore.a: ../*.o ar rv libcore.a ../*.o diff --git a/server/core/test/testadminusers.c b/server/core/test/testadminusers.c new file mode 100644 index 000000000..00ec3a452 --- /dev/null +++ b/server/core/test/testadminusers.c @@ -0,0 +1,278 @@ +/* + * This file is distributed as part of MaxScale. It is free + * software: you can redistribute it and/or modify it under the terms of the + * GNU General Public License as published by the Free Software Foundation, + * version 2. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 51 + * Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Copyright SkySQL Ab 2014 + */ + +/** + * + * @verbatim + * Revision History + * + * Date Who Description + * 20-08-2014 Mark Riddoch Initial implementation + * + * @endverbatim + */ + +#include +#include +#include + +#include + + +/** + * test1 default user + * + * Test that the username password admin/skysql is accepted if no users + * have been created and that no other users are accepted + * + * WARNING: $MAXSCALE_HOME/etc/passwd must be removed before this test is run + */ +static int +test1() +{ + if (admin_verify("admin", "skysql") == 0) + { + fprintf(stderr, "admin_verify: test 1.1 (default user) failed.\n"); + return 1; + } + if (admin_verify("bad", "user")) + { + fprintf(stderr, "admin_verify: test 1.2 (wrong user) failed.\n"); + return 1; + } + + return 0; +} + +/** + * test2 creating users + * + * Create a user + * Try to create a duplicate user - expects a failure + * Remove that user - expected to fail as one user must always remain + */ +static int +test2() +{ +char *err; + + if ((err = admin_add_user("user0", "passwd0")) != NULL) + { + fprintf(stderr, "admin_add_user: test 2.1 (add user) failed, %s.\n", err); + + return 1; + } + if (admin_add_user("user0", "passwd0") == NULL) + { + fprintf(stderr, "admin_add_user: test 2.2 (add user) failed, du;plicate.\n"); + + return 1; + } + + /* Deleting the last user is forbidden so we expect this to fail */ + if ((err = admin_remove_user("user0", "passwd0")) == NULL) + { + fprintf(stderr, "admin_remove_user: test 2.3 (add user) failed, %s.\n", err); + + return 1; + } + return 0; +} + +/** + * test3 search/verify users + * + * Create a user + * Search for that user + * Search for a non-existant user + * Remove the user + * Search for the user that was removed + */ +static int +test3() +{ +char *err; + + if ((err = admin_add_user("user1", "passwd1")) != NULL) + { + fprintf(stderr, "admin_add_user: test 3.1 (add user) failed, %s.\n", err); + + return 1; + } + + if (admin_search_user("user1") == 0) + { + fprintf(stderr, "admin_search_user: test 3.2 (search user) failed.\n"); + + return 1; + } + if (admin_search_user("user2") != 0) + { + fprintf(stderr, "admin_search_user: test 3.3 (search user) failed, unexpeted user found.\n"); + + return 1; + } + + if ((err = admin_remove_user("user1", "passwd1")) != NULL) + { + fprintf(stderr, "admin_remove_user: test 3.4 (add user) failed, %s.\n", err); + + return 1; + } + + if (admin_search_user("user1")) + { + fprintf(stderr, "admin_search_user: test 3.5 (search user) failed - user was deleted.\n"); + + return 1; + } + return 0; +} + +/** + * test4 verify users + * + * Create a numebr of users + * search for each user in turn + * verify each user in turn (password verification) + * Verify each user in turn with incorrect password + * Randomly verify each user + * Remove each user + */ +static int +test4() +{ +char *err, user[40], passwd[40]; +int i, n_users = 50; + + for (i = 1; i < n_users; i++) + { + sprintf(user, "user%d", i); + sprintf(passwd, "passwd%d", i); + if ((err = admin_add_user(user, passwd)) != NULL) + { + fprintf(stderr, "admin_add_user: test 4.1 (add user) failed, %s.\n", err); + + return 1; + } + } + + for (i = 1; i < n_users; i++) + { + sprintf(user, "user%d", i); + if (admin_search_user(user) == 0) + { + fprintf(stderr, "admin_search_user: test 4.2 (search user) failed.\n"); + + return 1; + } + } + for (i = 1; i < n_users; i++) + { + sprintf(user, "user%d", i); + sprintf(passwd, "passwd%d", i); + if (admin_verify(user, passwd) == 0) + { + fprintf(stderr, "admin_verify: test 4.3 (search user) failed.\n"); + + return 1; + } + } + + for (i = 1; i < n_users; i++) + { + sprintf(user, "user%d", i); + sprintf(passwd, "badpasswd%d", i); + if (admin_verify(user, passwd) != 0) + { + fprintf(stderr, "admin_verify: test 4.4 (search user) failed.\n"); + + return 1; + } + } + srand(time(0)); + for (i = 1; i < 1000; i++) + { + int j; + j = rand() % n_users; + if (j == 0) + j = 1; + sprintf(user, "user%d", j); + sprintf(passwd, "passwd%d", j); + if (admin_verify(user, passwd) == 0) + { + fprintf(stderr, "admin_verify: test 4.5 (random) failed.\n"); + + return 1; + } + } + + for (i = 1; i < n_users; i++) + { + sprintf(user, "user%d", i); + sprintf(passwd, "passwd%d", i); + if ((err = admin_remove_user(user, passwd)) != NULL) + { + fprintf(stderr, "admin_remove_user: test 4.6 (add user) failed, %s.\n", err); + + return 1; + } + } + return 0; +} + +/** + * test5 remove first user + * + * Create a user so that user0 may be removed + * Remove the first user created (user0) + */ +static int +test5() +{ +char *err; + + if ((err = admin_add_user("user", "passwd")) != NULL) + { + fprintf(stderr, "admin_add_user: test 5.1 (add user) failed, %s.\n", err); + + return 1; + } + if ((err = admin_remove_user("user0", "passwd0")) != NULL) + { + fprintf(stderr, "admin_remove_user: test 5.2 (add user) failed, %s.\n", err); + + return 1; + } + return 0; +} + +int +main(int argc, char **argv) +{ +int result = 0; + + result += test1(); + result += test2(); + result += test3(); + result += test4(); + result += test5(); + + exit(result); +} + diff --git a/server/core/test/testfilter.c b/server/core/test/testfilter.c index eefc8ecfa..55f7fadf3 100644 --- a/server/core/test/testfilter.c +++ b/server/core/test/testfilter.c @@ -140,6 +140,8 @@ int i, n_filters = 1000; return 0; } + +int main(int argc, char **argv) { int result = 0; diff --git a/server/include/adminusers.h b/server/include/adminusers.h index d46570e38..07afa5390 100644 --- a/server/include/adminusers.h +++ b/server/include/adminusers.h @@ -29,6 +29,8 @@ * * @endverbatim */ +#include + #define ADMIN_SALT "MS" extern int admin_verify(char *, char *); From a853b72baf2e12dcf21c88c724d83025e62484f5 Mon Sep 17 00:00:00 2001 From: counterpoint Date: Wed, 20 Aug 2014 17:22:32 +0100 Subject: [PATCH 30/60] Modify build_gateway.inc so that variables are used, thus avoiding a need for editing. Please review the file to see the variables that are used - they should be obvious. --- build_gateway.inc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/build_gateway.inc b/build_gateway.inc index 83ffaf762..0e1d6b75f 100644 --- a/build_gateway.inc +++ b/build_gateway.inc @@ -12,7 +12,7 @@ # # Set debug flags # -DEBUG := +DEBUG := ${MAXSCALE_DEBUG} # # Set build env @@ -22,7 +22,7 @@ UNIX := Y # # Set MaxScale branch directory # -ROOT_PATH := $(HOME)/src/bazaar/tmp/maxscale +ROOT_PATH := $(HOME)/${MAXSCALE_SOURCE} INC_PATH := $(HOME)/usr/include # @@ -38,7 +38,7 @@ MYSQL_HEADERS := -I$(INC_PATH) -I$(MYSQL_ROOT)/ -I$(MYSQL_ROOT)/private/ -I$(MYS # # Set DYNLIB=Y if you want to link MaxScale with dynamic embedded lib # -DYNLIB := +DYNLIB := ${MAXSCALE_DYNLIB} # # Set path to Embedded MySQL Server @@ -51,3 +51,4 @@ endif # Set path to MySQL errors file # ERRMSG := $(HOME)/usr/share/mysql + From c501d4d4e1f49086c60a2af8e3afabc1944a737f Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Wed, 20 Aug 2014 22:10:36 +0300 Subject: [PATCH 31/60] Changes related to canonical query format implementation. query_classifier.cc: Now query can be parsed outside query_classifier_get_type by calling function parse_query. It creates parsing_info_t struct which is then added to the GWBUF which also includes the query. Parsing information follows the buffered query and it is freed at the same time with query buffer, in gwbuf_free. buffer.c: additions of parsing information to gwbuf struct. modutil.c: added function which returns query from GWBUF in plain text string. readwritesplit.c:routeQuery now only calls query_classifier_get_type to get the query type instead of extracting plain text query from the GWBUF buffer. --- query_classifier/makefile | 8 + query_classifier/query_classifier.cc | 365 ++++++++++++------ query_classifier/query_classifier.h | 17 +- server/core/buffer.c | 16 +- server/core/modutil.c | 55 +++ server/include/buffer.h | 23 +- server/include/modutil.h | 2 + .../routing/readwritesplit/readwritesplit.c | 72 +--- utils/skygw_debug.h | 10 +- 9 files changed, 391 insertions(+), 177 deletions(-) diff --git a/query_classifier/makefile b/query_classifier/makefile index b08f0a9bd..35ef36d2a 100644 --- a/query_classifier/makefile +++ b/query_classifier/makefile @@ -8,6 +8,8 @@ SRCS := query_classifier.cc UTILS_PATH := $(ROOT_PATH)/utils QUERY_CLASSIFIER_PATH := $(ROOT_PATH)/query_classifier LOG_MANAGER_PATH := $(ROOT_PATH)/log_manager +SERVER_INC_PATH := $(ROOT_PATH)/server/include +MODULE_INC_PATH := $(ROOT_PATH)/server/modules/include makeall: clean all @@ -43,6 +45,9 @@ libcomp: $(CPP) -c $(CFLAGS) \ $(MYSQL_HEADERS) \ -I$(LOG_MANAGER_PATH) \ + -I$(SERVER_INC_PATH) \ + -I$(MODULE_INC_PATH) \ + -I$(UTILS_PATH) \ -I./ \ -fPIC ./query_classifier.cc -o query_classifier.o @@ -66,6 +71,9 @@ depend: $(CPP) -M $(CFLAGS) \ $(MYSQL_HEADERS) \ -I$(LOG_MANAGER_PATH) \ + -I$(SERVER_INC_PATH) \ + -I$(MODULE_INC_PATH) \ + -I$(UTILS_PATH) \ -I./ \ $(SRCS) > depend diff --git a/query_classifier/query_classifier.cc b/query_classifier/query_classifier.cc index 77b939819..7354d9639 100644 --- a/query_classifier/query_classifier.cc +++ b/query_classifier/query_classifier.cc @@ -34,6 +34,7 @@ #include "../utils/skygw_types.h" #include "../utils/skygw_debug.h" #include +#include #include #include @@ -83,122 +84,146 @@ static bool skygw_stmt_causes_implicit_commit( static int is_autocommit_stmt( LEX* lex); -/** - * @node (write brief function description here) - * - * Parameters: - * @param query_str - - * - * - * @param client_flag - - * - * - * @return - * +static bool query_is_parsed( + GWBUF* buf); + +static void parsing_info_set_plain_str(void* ptr, + char* str); + +/** + * Calls parser for the query includede in the buffer. Creates and adds parsing + * information to buffer if it doesn't exist already. Resolves the query type. * - * @details (write detailed description here) - * + * @param querybuf buffer including the query and possibly the parsing information + * + * @return query type */ -skygw_query_type_t skygw_query_classifier_get_type( - const char* query, - unsigned long client_flags, - MYSQL** p_mysql) +skygw_query_type_t query_classifier_get_type( + GWBUF* querybuf) { - MYSQL* mysql; - char* query_str; - const char* user = "skygw"; - const char* db = "skygw"; - THD* thd; + MYSQL* mysql; skygw_query_type_t qtype = QUERY_TYPE_UNKNOWN; - bool failp = FALSE; - - ss_info_dassert(query != NULL, ("query_str is NULL")); + bool succp; - query_str = const_cast(query); - LOGIF(LT, (skygw_log_write( - LOGFILE_TRACE, - "Query : \"%s\"", query_str))); + ss_info_dassert(querybuf != NULL, ("querybuf is NULL")); - /** Get server handle */ - mysql = mysql_init(NULL); - - if (mysql == NULL) { - LOGIF(LE, (skygw_log_write_flush( - LOGFILE_ERROR, - "Error : call to mysql_real_connect failed due %d, %s.", - mysql_errno(mysql), - mysql_error(mysql)))); - - mysql_library_end(); - goto return_qtype; - } - - if (p_mysql != NULL) + /** Create parsing info for the query and store it to buffer */ + if (!query_is_parsed(querybuf)) { - *p_mysql = mysql; + succp = parse_query(querybuf); } - /** Set methods and authentication to mysql */ - mysql_options(mysql, MYSQL_READ_DEFAULT_GROUP, "libmysqld_skygw"); - mysql_options(mysql, MYSQL_OPT_USE_EMBEDDED_CONNECTION, NULL); - mysql->methods = &embedded_methods; - mysql->user = my_strdup(user, MYF(0)); - mysql->db = my_strdup(db, MYF(0)); - mysql->passwd = NULL; - - /** Get one or create new THD object to be use in parsing */ - thd = get_or_create_thd_for_parsing(mysql, query_str); - - if (thd == NULL) + /** Read thd pointer and resolve the query type with it. */ + if (succp) { - skygw_query_classifier_free(mysql); - *p_mysql = NULL; - goto return_qtype; - } - /** - * Create parse_tree inside thd. - * thd and even lex are readable even if parser failed so let it - * continue despite failure. - */ - failp = create_parse_tree(thd); - qtype = resolve_query_type(thd); + parsing_info_t* pi = (parsing_info_t*)gwbuf_get_parsing_info(querybuf); + mysql = (MYSQL *)pi->pi_handle; - if (p_mysql == NULL) - { - skygw_query_classifier_free(mysql); + /** Find out the query type */ + if (mysql != NULL) + { + qtype = resolve_query_type((THD *)mysql->thd); + } } -return_qtype: return qtype; } - -void skygw_query_classifier_free( - MYSQL* mysql) +/** + * Create parsing info and try to parse the query included in the query buffer. + * Store pointer to created parse_tree_t object to buffer. + * + * @param querybuf buffer including the query and possibly the parsing information + * + * @return true if succeed, false otherwise + */ +bool parse_query ( + GWBUF* querybuf) { - if (mysql->thd != NULL) + bool succp; + THD* thd; + uint8_t* data; + size_t len; + char* query_str; + parsing_info_t* pi; + + CHK_GWBUF(querybuf); + ss_dassert(!query_is_parsed(querybuf)); + + if (querybuf->gwbuf_parsing_info == NULL) { - (*mysql->methods->free_embedded_thd)(mysql); - mysql->thd = NULL; + /** Create parsing info */ + querybuf->gwbuf_parsing_info = parsing_info_init(parsing_info_done); } - mysql_close(mysql); - mysql_thread_end(); -} + + if (querybuf->gwbuf_parsing_info == NULL) + { + succp = false; + goto retblock; + } + /** Extract query and copy it to different buffer */ + data = (uint8_t*)GWBUF_DATA(querybuf); + len = MYSQL_GET_PACKET_LEN(data)-1; /*< distract 1 for packet type byte */ + query_str = (char *)malloc(len+1); + + if (query_str == NULL) + { + succp = false; + goto retblock; + } + memcpy(query_str, &data[5], len); + memset(&query_str[len], 0, 1); + parsing_info_set_plain_str(querybuf->gwbuf_parsing_info, query_str); + + /** Get one or create new THD object to be use in parsing */ + pi = (parsing_info_t *)querybuf->gwbuf_parsing_info; + thd = get_or_create_thd_for_parsing((MYSQL *)pi->pi_handle, query_str); + + if (thd == NULL) + { + parsing_info_done(querybuf->gwbuf_parsing_info); + querybuf->gwbuf_parsing_info = NULL; + succp = false; + goto retblock; + } + /** + * Create parse_tree inside thd. + * thd and lex are readable even if creating parse tree fails. + */ + create_parse_tree(thd); + succp = true; +retblock: + return succp; +} +/** + * If buffer has non-NULL gwbuf_parsing_info it is parsed and it has parsing + * information included. + * + * @param buf buffer being examined + * + * @return true or false + */ +static bool query_is_parsed( + GWBUF* buf) +{ + if (buf->gwbuf_parsing_info != NULL) + { + return true; + } + return false; +} -/** - * @node (write brief function description here) + +/** + * Create a thread context, thd, init embedded server, connect to it, and allocate + * query to thd. * * Parameters: - * @param mysql - - * - * - * @param query_str - - * - * - * @return - * + * @param mysql Database handle * - * @details (write detailed description here) + * @param query_str Query in plain txt string + * + * @return Thread context pointer * */ static THD* get_or_create_thd_for_parsing( @@ -821,8 +846,7 @@ char* skygw_query_classifier_get_stmtname( * Replace user-provided literals with question marks. Return a copy of the * querystr with replacements. * - * @param mysql Database pointer - * @param querystr Query string + * @param querybuf GWBUF buffer including necessary parsing info * * @return Copy of querystr where literals are replaces with question marks or * NULL if querystr is NULL, thread context or lex are NULL or if replacement @@ -832,23 +856,32 @@ char* skygw_query_classifier_get_stmtname( * VARBIN_ITEM,NULL_ITEM */ char* skygw_get_canonical( - MYSQL* mysql, - char* querystr) + GWBUF* querybuf) { - THD* thd; - LEX* lex; - bool found = false; - char* newstr = NULL; - Item* item; + parsing_info_t* pi; + MYSQL* mysql; + THD* thd; + LEX* lex; + bool found = false; + char* newstr = NULL; + Item* item; + char* querystr; - ss_dassert(mysql != NULL && querystr != NULL); - - if (querystr == NULL || - mysql == NULL || + if (querybuf->gwbuf_parsing_info == NULL) + { + goto retblock; + } + pi = (parsing_info_t*)querybuf->gwbuf_parsing_info; + + if ((querystr = pi->pi_query_plain_str) == NULL || + (mysql = (MYSQL *)pi->pi_handle) == NULL || (thd = (THD *)mysql->thd) == NULL || (lex = thd->lex) == NULL) { - ss_dassert(thd != NULL && lex != NULL); + ss_dassert(querystr != NULL && + mysql != NULL && + thd != NULL && + lex != NULL); goto retblock; } @@ -858,12 +891,13 @@ char* skygw_get_canonical( itype = item->type(); - if (itype == Item::STRING_ITEM || + if (item->name != NULL && + (itype == Item::STRING_ITEM || itype == Item::INT_ITEM || itype == Item::DECIMAL_ITEM || itype == Item::REAL_ITEM || itype == Item::VARBIN_ITEM || - itype == Item::NULL_ITEM) + itype == Item::NULL_ITEM)) { if (!found) { @@ -881,4 +915,117 @@ char* skygw_get_canonical( } /*< for */ retblock: return newstr; +} + + +/** + * Create parsing information; initialize mysql handle, allocate parsing info + * struct and set handle and free function pointer to it. + * + * @param donefun pointer to free function + * + * @return pointer to parsing information + */ +parsing_info_t* parsing_info_init( + void (*donefun)(void *)) +{ + parsing_info_t* pi = NULL; + MYSQL* mysql; + const char* user = "skygw"; + const char* db = "skygw"; + + ss_dassert(donefun != NULL); + + /** Get server handle */ + mysql = mysql_init(NULL); + ss_dassert(mysql != NULL); + + if (mysql == NULL) { + LOGIF(LE, (skygw_log_write_flush( + LOGFILE_ERROR, + "Error : call to mysql_real_connect failed due %d, %s.", + mysql_errno(mysql), + mysql_error(mysql)))); + + mysql_library_end(); + goto retblock; + } + /** Set methods and authentication to mysql */ + mysql_options(mysql, MYSQL_READ_DEFAULT_GROUP, "libmysqld_skygw"); + mysql_options(mysql, MYSQL_OPT_USE_EMBEDDED_CONNECTION, NULL); + mysql->methods = &embedded_methods; + mysql->user = my_strdup(user, MYF(0)); + mysql->db = my_strdup(db, MYF(0)); + mysql->passwd = NULL; + + pi = (parsing_info_t*)calloc(1, sizeof(parsing_info_t)); + + if (pi == NULL) + { + mysql_close(mysql); + mysql_thread_end(); + goto retblock; + } +#if defined(SS_DEBUG) + pi->pi_chk_top = CHK_NUM_PINFO; + pi->pi_chk_tail = CHK_NUM_PINFO; +#endif + /** Set handle and free function to parsing info struct */ + pi->pi_handle = mysql; + pi->pi_done_fp = donefun; + +retblock: + return pi; +} + +/** + * Free function for parsing info. Called by gwbuf_free or in case initialization + * of parsing information fails. + * + * @param ptr Pointer to parsing information, cast required + * + * @return void + * + */ +void parsing_info_done( + void* ptr) +{ + parsing_info_t* pi = (parsing_info_t *)ptr; + + if (pi->pi_handle != NULL) + { + MYSQL* mysql = (MYSQL *)pi->pi_handle; + + if (mysql->thd != NULL) + { + (*mysql->methods->free_embedded_thd)(mysql); + mysql->thd = NULL; + } + mysql_close(mysql); + mysql_thread_end(); + } + /** Free plain text query string */ + if (pi->pi_query_plain_str != NULL) + { + free(pi->pi_query_plain_str); + } + free(pi); +} + +/** + * Add plain text query string to parsing info. + * + * @param ptr Pointer to parsing info struct, cast required + * @param str String to be added + * + * @return void + */ +static void parsing_info_set_plain_str( + void* ptr, + char* str) +{ + parsing_info_t* pi = (parsing_info_t *)ptr; + CHK_PARSING_INFO(pi); + + pi->pi_query_plain_str = str; } \ No newline at end of file diff --git a/query_classifier/query_classifier.h b/query_classifier/query_classifier.h index fa7cefc78..beef84c2c 100644 --- a/query_classifier/query_classifier.h +++ b/query_classifier/query_classifier.h @@ -20,7 +20,8 @@ Copyright SkySQL Ab /** getpid */ #include #include -#include "../utils/skygw_utils.h" +#include +#include EXTERN_C_BLOCK_BEGIN @@ -52,15 +53,15 @@ typedef enum { * Create THD and use it for creating parse tree. Examine parse tree and * classify the query. */ -skygw_query_type_t skygw_query_classifier_get_type( - const char* query_str, - unsigned long client_flags, - MYSQL** mysql); +skygw_query_type_t query_classifier_get_type(GWBUF* querybuf); /** Free THD context and close MYSQL */ -void skygw_query_classifier_free(MYSQL* mysql); -char* skygw_query_classifier_get_stmtname(MYSQL* mysql); -char* skygw_get_canonical(MYSQL* mysql, char* querystr); +char* skygw_query_classifier_get_stmtname(MYSQL* mysql); +char* skygw_get_canonical(GWBUF* querybuf); +bool parse_query (GWBUF* querybuf); +parsing_info_t* parsing_info_init(void (*donefun)(void *)); +void parsing_info_done(void* ptr); + EXTERN_C_BLOCK_END diff --git a/server/core/buffer.c b/server/core/buffer.c index d3278a505..47e9a22eb 100644 --- a/server/core/buffer.c +++ b/server/core/buffer.c @@ -83,7 +83,7 @@ SHARED_BUF *sbuf; rval->sbuf = sbuf; rval->next = NULL; rval->gwbuf_type = GWBUF_TYPE_UNDEFINED; - rval->command = 0; + rval->gwbuf_parsing_info = NULL; CHK_GWBUF(rval); return rval; } @@ -102,6 +102,11 @@ gwbuf_free(GWBUF *buf) free(buf->sbuf->data); free(buf->sbuf); } + if (buf->gwbuf_parsing_info != NULL) + { + parsing_info_t* pi = (parsing_info_t *)buf->gwbuf_parsing_info; + pi->pi_done_fp(pi); + } free(buf); } @@ -131,6 +136,7 @@ GWBUF *rval; rval->end = buf->end; rval->gwbuf_type = buf->gwbuf_type; rval->next = NULL; + rval->gwbuf_parsing_info = NULL; CHK_GWBUF(rval); return rval; } @@ -157,6 +163,7 @@ GWBUF *gwbuf_clone_portion( clonebuf->end = (void *)((char *)clonebuf->start)+length; clonebuf->gwbuf_type = buf->gwbuf_type; /*< clone the type for now */ clonebuf->next = NULL; + clonebuf->gwbuf_parsing_info = NULL; CHK_GWBUF(clonebuf); return clonebuf; @@ -336,5 +343,10 @@ void gwbuf_set_type( } } - +void* gwbuf_get_parsing_info( + GWBUF* buf) +{ + CHK_GWBUF(buf); + return buf->gwbuf_parsing_info; +} diff --git a/server/core/modutil.c b/server/core/modutil.c index 78f389ebf..781c55763 100644 --- a/server/core/modutil.c +++ b/server/core/modutil.c @@ -29,6 +29,7 @@ */ #include #include +#include /** * Check if a GWBUF structure is a MySQL COM_QUERY packet @@ -171,3 +172,57 @@ GWBUF *addition; return orig; } + +/** + * Copy query string from GWBUF buffer to separate memory area. + * + * @param buf GWBUF buffer including the query + * + * @return Plaint text query if the packet type is COM_QUERY. Otherwise return + * a string including the packet type. + */ +char* modutil_get_query( + GWBUF* buf) +{ + uint8_t* packet; + mysql_server_cmd_t packet_type; + size_t len; + char* query_str; + + packet = GWBUF_DATA(buf); + packet_type = packet[4]; + + switch (packet_type) { + case MYSQL_COM_QUIT: + len = strlen("[Quit msg]")+1; + if ((query_str = (char *)malloc(len+1)) == NULL) + { + goto retblock; + } + memcpy(query_str, "[Quit msg]", len); + memset(&query_str[len], 0, 1); + break; + + case MYSQL_COM_QUERY: + len = MYSQL_GET_PACKET_LEN(packet)-1; /*< distract 1 for packet type byte */ + if ((query_str = (char *)malloc(len+1)) == NULL) + { + goto retblock; + } + memcpy(query_str, &packet[5], len); + memset(&query_str[len], 0, 1); + break; + + default: + len = strlen(STRPACKETTYPE(packet_type))+1; + if ((query_str = (char *)malloc(len+1)) == NULL) + { + goto retblock; + } + memcpy(query_str, STRPACKETTYPE(packet_type), len); + memset(&query_str[len], 0, 1); + break; + } /*< switch */ +retblock: + return query_str; +} \ No newline at end of file diff --git a/server/include/buffer.h b/server/include/buffer.h index 9729c538c..57e20dc2e 100644 --- a/server/include/buffer.h +++ b/server/include/buffer.h @@ -43,6 +43,7 @@ */ #include +EXTERN_C_BLOCK_BEGIN typedef enum { @@ -73,6 +74,20 @@ typedef struct { int refcount; /*< Reference count on the buffer */ } SHARED_BUF; + +typedef struct parsing_info_st { +#if defined(SS_DEBUG) + skygw_chk_t pi_chk_top; +#endif + void* pi_handle; /*< parsing info object pointer */ + char* pi_query_plain_str; /*< query as plain string */ + void (*pi_done_fp)(void *); /*< clean-up function for parsing info */ +#if defined(SS_DEBUG) + skygw_chk_t pi_chk_tail; +#endif +} parsing_info_t; + + /** * The buffer structure used by the descriptor control blocks. * @@ -86,7 +101,7 @@ typedef struct gwbuf { void *start; /*< Start of the valid data */ void *end; /*< First byte after the valid data */ SHARED_BUF *sbuf; /*< The shared buffer with the real data */ - int command;/*< The command type for the queue */ + void *gwbuf_parsing_info; /*< parsing info object pointer */ gwbuf_type_t gwbuf_type; /*< buffer's data type information */ } GWBUF; @@ -121,4 +136,10 @@ extern unsigned int gwbuf_length(GWBUF *head); extern GWBUF *gwbuf_clone_portion(GWBUF *head, size_t offset, size_t len); extern GWBUF *gwbuf_clone_transform(GWBUF *head, gwbuf_type_t type); extern void gwbuf_set_type(GWBUF *head, gwbuf_type_t type); +void* gwbuf_get_parsing_info(GWBUF* buf); + + +EXTERN_C_BLOCK_END + + #endif diff --git a/server/include/modutil.h b/server/include/modutil.h index 00336f937..a0624752a 100644 --- a/server/include/modutil.h +++ b/server/include/modutil.h @@ -36,4 +36,6 @@ extern int modutil_is_SQL(GWBUF *); extern int modutil_extract_SQL(GWBUF *, char **, int *); extern int modutil_MySQL_Query(GWBUF *, char **, int *, int *); extern GWBUF *modutil_replace_SQL(GWBUF *, char *); +char* modutil_get_query(GWBUF* buf); + #endif diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index 24262bc6d..f89f68562 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -31,6 +31,7 @@ #include #include #include +#include #include MODULE_INFO info = { @@ -703,7 +704,7 @@ static void* newSession( backend_ref[i].bref_sescmd_cur.scmd_cur_ptr_property = &client_rses->rses_properties[RSES_PROP_TYPE_SESCMD]; backend_ref[i].bref_sescmd_cur.scmd_cur_cmd = NULL; - } + } max_nslaves = rses_get_max_slavecount(client_rses, router_nservers); max_slave_rlag = rses_get_max_replication_lag(client_rses); @@ -1031,9 +1032,6 @@ static int routeQuery( GWBUF* querybuf) { skygw_query_type_t qtype = QUERY_TYPE_UNKNOWN; - GWBUF* plainsqlbuf = NULL; - char* querystr = NULL; - char* startpos; mysql_server_cmd_t packet_type; uint8_t* packet; int ret = 0; @@ -1042,8 +1040,6 @@ static int routeQuery( ROUTER_INSTANCE* inst = (ROUTER_INSTANCE *)instance; ROUTER_CLIENT_SES* router_cli_ses = (ROUTER_CLIENT_SES *)router_session; bool rses_is_closed = false; - size_t len; - MYSQL* mysql = NULL; CHK_CLIENT_RSES(router_cli_ses); @@ -1066,6 +1062,8 @@ static int routeQuery( */ if (packet_type != MYSQL_COM_QUIT) { + char* query_str = modutil_get_query(querybuf); + LOGIF(LE, (skygw_log_write_flush( LOGFILE_ERROR, @@ -1073,15 +1071,15 @@ static int routeQuery( "backend server. %s.", STRPACKETTYPE(packet_type), STRQTYPE(qtype), - (querystr == NULL ? "(empty)" : querystr), + (query_str == NULL ? "(empty)" : query_str), (rses_is_closed ? "Router was closed" : "Router has no backend servers where to " "route to")))); + free(querybuf); } goto return_ret; } inst->stats.n_queries++; - startpos = (char *)&packet[5]; master_dcb = router_cli_ses->rses_master_ref->bref_dcb; CHK_DCB(master_dcb); @@ -1105,44 +1103,16 @@ static int routeQuery( break; case MYSQL_COM_QUERY: - plainsqlbuf = gwbuf_clone_transform(querybuf, - GWBUF_TYPE_PLAINSQL); - len = GWBUF_LENGTH(plainsqlbuf); - /** unnecessary if buffer includes additional terminating null */ - querystr = (char *)malloc(len+1); - memcpy(querystr, startpos, len); - memset(&querystr[len], 0, 1); - /** - * Use mysql handle to query information from parse tree. - * call skygw_query_classifier_free before exit! - */ - qtype = skygw_query_classifier_get_type(querystr, 0, &mysql); + qtype = query_classifier_get_type(querybuf); break; case MYSQL_COM_STMT_PREPARE: - plainsqlbuf = gwbuf_clone_transform(querybuf, - GWBUF_TYPE_PLAINSQL); - len = GWBUF_LENGTH(plainsqlbuf); - /** unnecessary if buffer includes additional terminating null */ - querystr = (char *)malloc(len+1); - memcpy(querystr, startpos, len); - memset(&querystr[len], 0, 1); - qtype = skygw_query_classifier_get_type(querystr, 0, &mysql); + qtype = query_classifier_get_type(querybuf); qtype |= QUERY_TYPE_PREPARE_STMT; break; case MYSQL_COM_STMT_EXECUTE: /** Parsing is not needed for this type of packet */ -#if defined(NOT_USED) - plainsqlbuf = gwbuf_clone_transform(querybuf, - GWBUF_TYPE_PLAINSQL); - len = GWBUF_LENGTH(plainsqlbuf); - /** unnecessary if buffer includes additional terminating null */ - querystr = (char *)malloc(len+1); - memcpy(querystr, startpos, len); - memset(&querystr[len], 0, 1); - qtype = skygw_query_classifier_get_type(querystr, 0, &mysql); -#endif qtype = QUERY_TYPE_EXEC_STMT; break; @@ -1207,7 +1177,7 @@ static int routeQuery( */ bool succp = route_session_write( router_cli_ses, - querybuf, + gwbuf_clone(querybuf), inst, packet_type, qtype); @@ -1238,7 +1208,7 @@ static int routeQuery( if (succp) { - if ((ret = slave_dcb->func.write(slave_dcb, querybuf)) == 1) + if ((ret = slave_dcb->func.write(slave_dcb, gwbuf_clone(querybuf))) == 1) { backend_ref_t* bref; @@ -1252,10 +1222,12 @@ static int routeQuery( } else { + char* query_str = modutil_get_query(querybuf); LOGIF(LE, (skygw_log_write_flush( LOGFILE_ERROR, "Error : Routing query \"%s\" failed.", - querystr))); + (query_str == NULL ? "not available" : query_str)))); + free(query_str); } } rses_end_locked_router_action(router_cli_ses); @@ -1296,7 +1268,7 @@ static int routeQuery( if (succp) { - if ((ret = master_dcb->func.write(master_dcb, querybuf)) == 1) + if ((ret = master_dcb->func.write(master_dcb, gwbuf_clone(querybuf))) == 1) { backend_ref_t* bref; @@ -1323,11 +1295,10 @@ static int routeQuery( } return_ret: #if defined(SS_DEBUG) - if (mysql != NULL && true) { char* canonical_query_str; - canonical_query_str = skygw_get_canonical(mysql, querystr); + canonical_query_str = skygw_get_canonical(querybuf); if (canonical_query_str != NULL) { @@ -1339,18 +1310,7 @@ return_ret: } } #endif - if (plainsqlbuf != NULL) - { - gwbuf_free(plainsqlbuf); - } - if (querystr != NULL) - { - free(querystr); - } - if (mysql != NULL) - { - skygw_query_classifier_free(mysql); - } + gwbuf_free(querybuf); return ret; } diff --git a/utils/skygw_debug.h b/utils/skygw_debug.h index 43a609a40..9ffc5e7e2 100644 --- a/utils/skygw_debug.h +++ b/utils/skygw_debug.h @@ -123,7 +123,8 @@ typedef enum skygw_chk_t { CHK_NUM_SESCMD_CUR, CHK_NUM_BACKEND, CHK_NUM_BACKEND_REF, - CHK_NUM_PREP_STMT + CHK_NUM_PREP_STMT, + CHK_NUM_PINFO } skygw_chk_t; # define STRBOOL(b) ((b) ? "true" : "false") @@ -486,6 +487,13 @@ typedef enum skygw_chk_t { "Prepared statement struct has invalid check fields"); \ } +#define CHK_PARSING_INFO(p) { \ + ss_info_dassert((p)->pi_chk_top == CHK_NUM_PINFO && \ + (p)->pi_chk_tail == CHK_NUM_PINFO, \ + "Parsing info struct has invalid check fields"); \ +} + + #if defined(SS_DEBUG) bool conn_open[10240]; From fa2189373dce93e7938612e192653870cb7cbcbc Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Thu, 21 Aug 2014 22:28:23 +0300 Subject: [PATCH 32/60] query_classifier.cc: query_is_parsed is now global function and can be used to check whether parsing information already exists in the buffer. skygw_utils.cc: removed replace_str and implemented replace_literal which replaces user-provided literals in query with predefined string "?". Replacing is done one by one, so it is suboptimal ipmlementation since all literals could be passed to replacement function in one call and processed all before returning. --- query_classifier/query_classifier.cc | 19 ++-- query_classifier/query_classifier.h | 2 + utils/makefile | 3 +- utils/skygw_types.h | 2 + utils/skygw_utils.cc | 124 +++++++++++++++------------ utils/skygw_utils.h | 8 +- 6 files changed, 94 insertions(+), 64 deletions(-) diff --git a/query_classifier/query_classifier.cc b/query_classifier/query_classifier.cc index 7354d9639..9c2ab146a 100644 --- a/query_classifier/query_classifier.cc +++ b/query_classifier/query_classifier.cc @@ -84,9 +84,6 @@ static bool skygw_stmt_causes_implicit_commit( static int is_autocommit_stmt( LEX* lex); -static bool query_is_parsed( - GWBUF* buf); - static void parsing_info_set_plain_str(void* ptr, char* str); @@ -203,7 +200,7 @@ retblock: * * @return true or false */ -static bool query_is_parsed( +bool query_is_parsed( GWBUF* buf) { if (buf->gwbuf_parsing_info != NULL) @@ -873,7 +870,7 @@ char* skygw_get_canonical( } pi = (parsing_info_t*)querybuf->gwbuf_parsing_info; - if ((querystr = pi->pi_query_plain_str) == NULL || + if (pi->pi_query_plain_str == NULL || (mysql = (MYSQL *)pi->pi_handle) == NULL || (thd = (THD *)mysql->thd) == NULL || (lex = thd->lex) == NULL) @@ -885,6 +882,8 @@ char* skygw_get_canonical( goto retblock; } + querystr = strdup(pi->pi_query_plain_str); + for (item=thd->free_list; item != NULL; item=item->next) { Item::Type itype; @@ -901,14 +900,18 @@ char* skygw_get_canonical( { if (!found) { - newstr = replace_str(querystr, item->name, "?"); - found = true; + newstr = replace_literal(querystr, item->name, "?"); + if (newstr != NULL) + { + free(querystr); + found = true; + } } else { char* prevstr = newstr; - newstr = replace_str(prevstr, item->name, "?"); + newstr = replace_literal(prevstr, item->name, "?"); free(prevstr); } } diff --git a/query_classifier/query_classifier.h b/query_classifier/query_classifier.h index beef84c2c..dd5ddc5cc 100644 --- a/query_classifier/query_classifier.h +++ b/query_classifier/query_classifier.h @@ -61,6 +61,8 @@ char* skygw_get_canonical(GWBUF* querybuf); bool parse_query (GWBUF* querybuf); parsing_info_t* parsing_info_init(void (*donefun)(void *)); void parsing_info_done(void* ptr); +bool query_is_parsed(GWBUF* buf); + diff --git a/utils/makefile b/utils/makefile index a8df09a7f..0faa27144 100644 --- a/utils/makefile +++ b/utils/makefile @@ -3,6 +3,7 @@ include ../makefile.inc CC = gcc CPP = g++ +UTILS_PATH := $(ROOT_PATH)/utils makeall: clean all @@ -13,7 +14,7 @@ clean: - $(DEL) *~ all: - $(CPP) -c $(CFLAGS) \ + $(CPP) -c $(CFLAGS) -I$(UTILS_PATH) \ -fPIC skygw_utils.cc -o skygw_utils.o cleantests: diff --git a/utils/skygw_types.h b/utils/skygw_types.h index d497fbfd7..cccf55f2b 100644 --- a/utils/skygw_types.h +++ b/utils/skygw_types.h @@ -44,4 +44,6 @@ # endif #endif +#define MAX_ERROR_MSG PATH_MAX + #endif /* SKYGW_TYPES_H */ diff --git a/utils/skygw_utils.cc b/utils/skygw_utils.cc index c7175c76f..2b2204814 100644 --- a/utils/skygw_utils.cc +++ b/utils/skygw_utils.cc @@ -24,8 +24,9 @@ #include #include #include +#include #include "skygw_debug.h" -#include "skygw_types.h" +#include #include "skygw_utils.h" const char* timestamp_formatstr = "%04d %02d/%02d %02d:%02d:%02d "; @@ -1864,68 +1865,83 @@ void skygw_file_done( } } + /** - * Replaces in the string str all the occurrences of the source string old with - * the destination string new. The lengths of the strings old and new may differ. - * The string new may be of any length, but the string "old" must be of non-zero - * length - the penalty for providing an empty string for the "old" parameter is - * an infinite loop. In addition, none of the three parameters may be NULL. + * Find the given needle - user-provided literal - and replace it with + * replacement string. Separate user-provided literals from matching table names + * etc. by searching only substrings preceded by non-letter and non-number. * - * @param str String to be modified - * @param old Substring to be replaced - * @param new Replacement - * @return String with replacements in new memory area or NULL if memory - * allocation failed. - * Dependencies: For this function to compile, you will need to also #include - * the following files: , and . + * @param haystack Plain text query string, not to be freed + * @param needle Substring to be searched, not to be freed + * @param replacement Replacement text, not to be freed * - * Thanks, to Laird Shaw who implemented most of this function. - */ -char* replace_str ( - const char *str, - const char *old, - const char *replacement) + * @return newly allocated string where needle is replaced + */ +char* replace_literal( + char* haystack, + const char* needle, + const char* replacement) { - char* ret; - char* r; - const char* p; - const char* q; - size_t oldlen; - size_t count; - size_t retlen; - size_t newlen; + const char* prefix = "[ =',\\(]"; /*< ' ','=','(',''',',' are allowed before needle */ + const char* suffix = "[$^[:alnum:]]?"; /*< alpha-num chars aren't allowed after the needle */ + char* search_re; + char* newstr; + regex_t re; + regmatch_t match; + int rc; + size_t rlen = strlen(replacement); + size_t nlen = strlen(needle); + size_t hlen = strlen(haystack); - oldlen = strlen(old); - newlen = strlen(replacement); + search_re = (char *)malloc(strlen(prefix)+nlen+strlen(suffix)+1); + sprintf(search_re, "%s%s%s", prefix, needle, suffix); + /** + * +2 because there may be up to 2 extra characters which + * aren't replaced + */ + newstr = (char *)malloc(hlen-nlen+rlen); - if (oldlen != newlen) + rc = regcomp(&re, search_re, REG_EXTENDED); + ss_dassert(rc == 0); + + if (rc != 0) { - for (count = 0, p = str; (q = strstr(p, old)) != NULL; p = q + oldlen) - { - count++; - } - /* this is undefined if p - str > PTRDIFF_MAX */ - retlen = p - str + strlen(p) + count * (newlen - oldlen); - } - else - { - retlen = strlen(str); - } - if ((ret = (char *)malloc(retlen + 1)) == NULL) - { - return NULL; + char error_message[MAX_ERROR_MSG]; + regerror (rc, &re, error_message, MAX_ERROR_MSG); + fprintf(stderr, + "Regex error compiling '%s': %s\n", + search_re, + error_message); + free(search_re); + newstr = NULL; + goto retblock; } + rc = regexec(&re, haystack, 1, &match, 0); - for (r = ret, p = str; (q = strstr(p, old)) != NULL; p = q + oldlen) + if (rc != 0) { - /* this is undefined if q - p > PTRDIFF_MAX */ - ptrdiff_t l = q - p; - memcpy(r, p, l); - r += l; - memcpy(r, replacement, newlen); - r += newlen; + free(search_re); + newstr = NULL; + goto retblock; } - strcpy(r, p); + memcpy(newstr, haystack, match.rm_so+1); + memcpy(newstr+match.rm_so+1, replacement, rlen); + /** +1 is terminating byte */ + memcpy(newstr+match.rm_so+1+rlen, haystack+match.rm_so+1+nlen, hlen-(match.rm_so+1)-nlen+1); - return ret; -} \ No newline at end of file + regfree(&re); + +retblock: + return newstr; +} + + + + + + + + + + + diff --git a/utils/skygw_utils.h b/utils/skygw_utils.h index 854c82541..a50dd08b6 100644 --- a/utils/skygw_utils.h +++ b/utils/skygw_utils.h @@ -191,7 +191,13 @@ int skygw_rwlock_unlock(skygw_rwlock_t* rwlock); int skygw_rwlock_init(skygw_rwlock_t** rwlock); int atomic_add(int *variable, int value); -char* replace_str(const char* str, const char* old, const char* replacement); +EXTERN_C_BLOCK_BEGIN + +char* replace_literal(char* haystack, + const char* needle, + const char* replacement); + +EXTERN_C_BLOCK_END #endif /* SKYGW_UTILS_H */ From 8d1eae6fde3c416a8537f31bce8413ed29fad6b0 Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Thu, 21 Aug 2014 22:34:50 +0300 Subject: [PATCH 33/60] Fixed comment. --- utils/skygw_utils.cc | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/utils/skygw_utils.cc b/utils/skygw_utils.cc index 2b2204814..c9c3fb77f 100644 --- a/utils/skygw_utils.cc +++ b/utils/skygw_utils.cc @@ -1895,10 +1895,7 @@ char* replace_literal( search_re = (char *)malloc(strlen(prefix)+nlen+strlen(suffix)+1); sprintf(search_re, "%s%s%s", prefix, needle, suffix); - /** - * +2 because there may be up to 2 extra characters which - * aren't replaced - */ + /** Allocate memory for new string */ newstr = (char *)malloc(hlen-nlen+rlen); rc = regcomp(&re, search_re, REG_EXTENDED); From 3a5b8ef64c074e33670de570cc19f26152f1d59f Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Thu, 21 Aug 2014 23:08:21 +0300 Subject: [PATCH 34/60] query_classifier.cc: cleaned up and simplified skygw_get_canonical skygw_util.cc:fixed memory allocation issue where terminating byte wasn't counted. Added some error checks. --- query_classifier/query_classifier.cc | 20 ++--------------- utils/skygw_utils.cc | 32 ++++++++++++++++++++++------ 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/query_classifier/query_classifier.cc b/query_classifier/query_classifier.cc index 9c2ab146a..2f29ddba1 100644 --- a/query_classifier/query_classifier.cc +++ b/query_classifier/query_classifier.cc @@ -860,7 +860,6 @@ char* skygw_get_canonical( THD* thd; LEX* lex; bool found = false; - char* newstr = NULL; Item* item; char* querystr; @@ -898,26 +897,11 @@ char* skygw_get_canonical( itype == Item::VARBIN_ITEM || itype == Item::NULL_ITEM)) { - if (!found) - { - newstr = replace_literal(querystr, item->name, "?"); - if (newstr != NULL) - { - free(querystr); - found = true; - } - } - else - { - char* prevstr = newstr; - - newstr = replace_literal(prevstr, item->name, "?"); - free(prevstr); - } + querystr = replace_literal(querystr, item->name, "?"); } } /*< for */ retblock: - return newstr; + return querystr; } diff --git a/utils/skygw_utils.cc b/utils/skygw_utils.cc index c9c3fb77f..6ffafd7c3 100644 --- a/utils/skygw_utils.cc +++ b/utils/skygw_utils.cc @@ -1882,7 +1882,7 @@ char* replace_literal( const char* needle, const char* replacement) { - const char* prefix = "[ =',\\(]"; /*< ' ','=','(',''',',' are allowed before needle */ + const char* prefix = "[ ='\",\\(]"; /*< ' ','=','(',''',''"',',' are allowed before needle */ const char* suffix = "[$^[:alnum:]]?"; /*< alpha-num chars aren't allowed after the needle */ char* search_re; char* newstr; @@ -1894,9 +1894,27 @@ char* replace_literal( size_t hlen = strlen(haystack); search_re = (char *)malloc(strlen(prefix)+nlen+strlen(suffix)+1); + + if (search_re == NULL) + { + fprintf(stderr, "Regex memory allocation failed : %s\n", + strerror(errno)); + newstr = haystack; + goto retblock; + } + sprintf(search_re, "%s%s%s", prefix, needle, suffix); - /** Allocate memory for new string */ - newstr = (char *)malloc(hlen-nlen+rlen); + /** Allocate memory for new string +1 for terminating byte */ + newstr = (char *)malloc(hlen-nlen+rlen+1); + + if (newstr == NULL) + { + fprintf(stderr, "Regex memory allocation failed : %s\n", + strerror(errno)); + free(search_re); + newstr = haystack; + goto retblock; + } rc = regcomp(&re, search_re, REG_EXTENDED); ss_dassert(rc == 0); @@ -1910,7 +1928,7 @@ char* replace_literal( search_re, error_message); free(search_re); - newstr = NULL; + newstr = haystack; goto retblock; } rc = regexec(&re, haystack, 1, &match, 0); @@ -1918,7 +1936,7 @@ char* replace_literal( if (rc != 0) { free(search_re); - newstr = NULL; + newstr = haystack; goto retblock; } memcpy(newstr, haystack, match.rm_so+1); @@ -1927,8 +1945,8 @@ char* replace_literal( memcpy(newstr+match.rm_so+1+rlen, haystack+match.rm_so+1+nlen, hlen-(match.rm_so+1)-nlen+1); regfree(&re); - -retblock: + free(haystack); +retblock: return newstr; } From 3bc88e4eb97b06bc9fcb019bd91c0d50a107f3d4 Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Thu, 21 Aug 2014 23:31:23 +0300 Subject: [PATCH 35/60] skygw_get_canonical didn't return NULL pointerin cases where parsing info didn't exist or something went wrong. --- query_classifier/query_classifier.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/query_classifier/query_classifier.cc b/query_classifier/query_classifier.cc index 2f29ddba1..8379c1b89 100644 --- a/query_classifier/query_classifier.cc +++ b/query_classifier/query_classifier.cc @@ -865,6 +865,7 @@ char* skygw_get_canonical( if (querybuf->gwbuf_parsing_info == NULL) { + querystr = NULL; goto retblock; } pi = (parsing_info_t*)querybuf->gwbuf_parsing_info; @@ -878,6 +879,7 @@ char* skygw_get_canonical( mysql != NULL && thd != NULL && lex != NULL); + querystr = NULL; goto retblock; } From 3476558f526d2e788b42445169ba8d747241838a Mon Sep 17 00:00:00 2001 From: Mark Riddoch Date: Fri, 22 Aug 2014 14:25:27 +0100 Subject: [PATCH 36/60] Fixed soem errors from a cppcheck run mbrampton@martin-office:~/Dropbox/development/skygit/MaxScale/server$ cppcheck -q core/*.c [core/adminusers.c:302]: (error) Resource leak: fp_tmp [core/filter.c:382]: (error) Uninitialized variable: me [core/service.c:1071]: (error) Uninitialized variable: succp --- server/core/adminusers.c | 2 ++ server/core/filter.c | 2 +- server/core/service.c | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/server/core/adminusers.c b/server/core/adminusers.c index 6ed70ed3c..61cd7c077 100644 --- a/server/core/adminusers.c +++ b/server/core/adminusers.c @@ -298,6 +298,7 @@ char* admin_remove_user( fname, err))); fclose(fp); + fclose(fp_tmp); unlink(fname_tmp); return ADMIN_ERR_PWDFILEACCESS; } @@ -325,6 +326,7 @@ char* admin_remove_user( fname, err))); fclose(fp); + fclose(fp_tmp); unlink(fname_tmp); return ADMIN_ERR_PWDFILEACCESS; } diff --git a/server/core/filter.c b/server/core/filter.c index 405a01470..99525f7f6 100644 --- a/server/core/filter.c +++ b/server/core/filter.c @@ -359,7 +359,7 @@ DOWNSTREAM *me; UPSTREAM * filterUpstream(FILTER_DEF *filter, void *fsession, UPSTREAM *upstream) { -UPSTREAM *me; +UPSTREAM *me = NULL; /* * The the filter has no setUpstream entry point then is does diff --git a/server/core/service.c b/server/core/service.c index 1102dabb4..1b79db346 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -1008,7 +1008,7 @@ bool service_set_param_value ( { char* p; int valint; - bool succp; + bool succp = true; /** * Find out whether the value is numeric and ends with '%' or '\0' From c133c6ef4affe1075cb4922a12c34e5f7f022795 Mon Sep 17 00:00:00 2001 From: Mark Riddoch Date: Fri, 22 Aug 2014 14:33:14 +0100 Subject: [PATCH 37/60] Fix for bug 479 - Undefined filter reference in MaxScale.cnf causes a crash --- server/core/filter.c | 3 +++ server/core/session.c | 14 ++++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/server/core/filter.c b/server/core/filter.c index 99525f7f6..3b3ef4dd6 100644 --- a/server/core/filter.c +++ b/server/core/filter.c @@ -318,6 +318,9 @@ filterApply(FILTER_DEF *filter, SESSION *session, DOWNSTREAM *downstream) { DOWNSTREAM *me; + if (filter == NULL) + return NULL; + if (filter->obj == NULL) { /* Filter not yet loaded */ diff --git a/server/core/session.c b/server/core/session.c index 62e1015e4..db59f9e6e 100644 --- a/server/core/session.c +++ b/server/core/session.c @@ -333,13 +333,15 @@ bool session_free( { for (i = 0; i < session->n_filters; i++) { - session->filters[i].filter->obj->closeSession( + if (session->filters[i].filter) + session->filters[i].filter->obj->closeSession( session->filters[i].instance, session->filters[i].session); } for (i = 0; i < session->n_filters; i++) { - session->filters[i].filter->obj->freeSession( + if (session->filters[i].filter) + session->filters[i].filter->obj->freeSession( session->filters[i].instance, session->filters[i].session); } @@ -653,6 +655,14 @@ int i; session->n_filters = service->n_filters; for (i = service->n_filters - 1; i >= 0; i--) { + if (service->filters[i] == NULL) + { + LOGIF(LE, (skygw_log_write_flush( + LOGFILE_ERROR, + "Service '%s' contians an unresolved filter.\n", + service->name))); + return 0; + } if ((head = filterApply(service->filters[i], session, &session->head)) == NULL) { From 493feb49ba2dda1c53c9fc1055cae75e9118fe97 Mon Sep 17 00:00:00 2001 From: Mark Riddoch Date: Fri, 22 Aug 2014 14:46:26 +0100 Subject: [PATCH 38/60] Fix for bug 410 - MaxScale.cnf server option is not parsed for spaces --- server/core/config.c | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/server/core/config.c b/server/core/config.c index f2c9185dc..1a8689bfb 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -65,6 +65,29 @@ static char *config_file = NULL; static GATEWAY_CONF gateway; char *version_string = NULL; + +/** + * Trim whitespace from the front and rear of a string + * + * @param str String to trim + * @return Trimmed string, changes are done in situ + */ +static char * +trim(char *str) +{ +char *ptr; + + while (isspace(*str)) + str++; + + /* Point to last character of the string */ + ptr = str + strlen(str) - 1; + while (ptr > str && isspace(*ptr)) + *ptr-- = 0; + + return str; +} + /** * Config item handler for the ini file reader * @@ -508,7 +531,7 @@ int error_count = 0; CONFIG_CONTEXT *obj1 = context; while (obj1) { - if (strcmp(s, obj1->object) == 0 && + if (strcmp(trim(s), obj1->object) == 0 && obj->element && obj1->element) { serviceAddBackend( From ee52ac64a9c9955d309753bc8b203e2b6bc473e0 Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Fri, 22 Aug 2014 19:01:56 +0300 Subject: [PATCH 39/60] query_classifier.cc:skygw_get_canonical: if the item to be replaced is an empty string "", it is processed differently from the other cases due to difficulties to get wanted result by adding special rule for that to regex. query_classifier.h: added parsing information structure to query classifier away from buffer.h. buffer.c:introduced a buffer object which includes object pointer and a clean-up call-back function. Buffer objects form a list which is cleaned up by the last referrer of the buffer, as a part of gwbuf_free. Buffer object list is protected by a spinlock gwbuf_lock. Also added identifier type, bufobj_id_t which is enumerated type and currently includes one value only, GWBUF_PARSING_INFO. Added also a bitfield for information about the buffer. It currently has one type only, GWBUF_INFO_PARSED indicating that buffer content is parsed and there is buffer object of type GWBUF_PARSING_INFO. skygw_utils.cc:replace_literal:changed regexec matching to case insensitive because user-defined literals are sometimes converted to upper-case ones. --- query_classifier/query_classifier.cc | 82 +++++++++++++------- query_classifier/query_classifier.h | 18 ++++- server/core/buffer.c | 111 ++++++++++++++++++++++++--- server/include/buffer.h | 49 ++++++++---- utils/skygw_utils.cc | 3 +- 5 files changed, 208 insertions(+), 55 deletions(-) diff --git a/query_classifier/query_classifier.cc b/query_classifier/query_classifier.cc index 8379c1b89..009cbe94f 100644 --- a/query_classifier/query_classifier.cc +++ b/query_classifier/query_classifier.cc @@ -105,20 +105,29 @@ skygw_query_type_t query_classifier_get_type( ss_info_dassert(querybuf != NULL, ("querybuf is NULL")); /** Create parsing info for the query and store it to buffer */ - if (!query_is_parsed(querybuf)) + succp = query_is_parsed(querybuf); + + if (!succp) { succp = parse_query(querybuf); } /** Read thd pointer and resolve the query type with it. */ if (succp) { - parsing_info_t* pi = (parsing_info_t*)gwbuf_get_parsing_info(querybuf); - mysql = (MYSQL *)pi->pi_handle; - - /** Find out the query type */ - if (mysql != NULL) + parsing_info_t* pi; + + pi = (parsing_info_t*)gwbuf_get_buffer_object_data(querybuf, + GWBUF_PARSING_INFO); + + if (pi != NULL) { - qtype = resolve_query_type((THD *)mysql->thd); + mysql = (MYSQL *)pi->pi_handle; + + /** Find out the query type */ + if (mysql != NULL) + { + qtype = resolve_query_type((THD *)mysql->thd); + } } } return qtype; @@ -143,19 +152,21 @@ bool parse_query ( parsing_info_t* pi; CHK_GWBUF(querybuf); + /** Do not parse without releasing previous parse info first */ ss_dassert(!query_is_parsed(querybuf)); - if (querybuf->gwbuf_parsing_info == NULL) + if (query_is_parsed(querybuf)) { - /** Create parsing info */ - querybuf->gwbuf_parsing_info = parsing_info_init(parsing_info_done); + return false; } + /** Create parsing info */ + pi = parsing_info_init(parsing_info_done); - if (querybuf->gwbuf_parsing_info == NULL) + if (pi == NULL) { succp = false; goto retblock; - } + } /** Extract query and copy it to different buffer */ data = (uint8_t*)GWBUF_DATA(querybuf); len = MYSQL_GET_PACKET_LEN(data)-1; /*< distract 1 for packet type byte */ @@ -163,21 +174,22 @@ bool parse_query ( if (query_str == NULL) { + /** Free parsing info data */ + parsing_info_done(pi); succp = false; goto retblock; } memcpy(query_str, &data[5], len); memset(&query_str[len], 0, 1); - parsing_info_set_plain_str(querybuf->gwbuf_parsing_info, query_str); + parsing_info_set_plain_str(pi, query_str); /** Get one or create new THD object to be use in parsing */ - pi = (parsing_info_t *)querybuf->gwbuf_parsing_info; thd = get_or_create_thd_for_parsing((MYSQL *)pi->pi_handle, query_str); if (thd == NULL) { - parsing_info_done(querybuf->gwbuf_parsing_info); - querybuf->gwbuf_parsing_info = NULL; + /** Free parsing info data */ + parsing_info_done(pi); succp = false; goto retblock; } @@ -186,6 +198,12 @@ bool parse_query ( * thd and lex are readable even if creating parse tree fails. */ create_parse_tree(thd); + /** Add complete parsing info struct to the query buffer */ + gwbuf_add_buffer_object(querybuf, + GWBUF_PARSING_INFO, + (void *)pi, + parsing_info_done); + succp = true; retblock: return succp; @@ -203,11 +221,8 @@ retblock: bool query_is_parsed( GWBUF* buf) { - if (buf->gwbuf_parsing_info != NULL) - { - return true; - } - return false; + CHK_GWBUF(buf); + return GWBUF_IS_PARSED(buf); } @@ -859,23 +874,29 @@ char* skygw_get_canonical( MYSQL* mysql; THD* thd; LEX* lex; - bool found = false; Item* item; char* querystr; - if (querybuf->gwbuf_parsing_info == NULL) + if (!GWBUF_IS_PARSED(querybuf)) { querystr = NULL; goto retblock; - } - pi = (parsing_info_t*)querybuf->gwbuf_parsing_info; + } + pi = (parsing_info_t *)gwbuf_get_buffer_object_data(querybuf, + GWBUF_PARSING_INFO); + if (pi == NULL) + { + querystr = NULL; + goto retblock; + } + if (pi->pi_query_plain_str == NULL || (mysql = (MYSQL *)pi->pi_handle) == NULL || (thd = (THD *)mysql->thd) == NULL || (lex = thd->lex) == NULL) { - ss_dassert(querystr != NULL && + ss_dassert(pi->pi_query_plain_str != NULL && mysql != NULL && thd != NULL && lex != NULL); @@ -899,7 +920,14 @@ char* skygw_get_canonical( itype == Item::VARBIN_ITEM || itype == Item::NULL_ITEM)) { - querystr = replace_literal(querystr, item->name, "?"); + if (itype == Item::STRING_ITEM && strlen(item->name) == 0) + { + querystr = replace_literal(querystr, "\"\"", "\"?\""); + } + else + { + querystr = replace_literal(querystr, item->name, "?"); + } } } /*< for */ retblock: diff --git a/query_classifier/query_classifier.h b/query_classifier/query_classifier.h index dd5ddc5cc..325087910 100644 --- a/query_classifier/query_classifier.h +++ b/query_classifier/query_classifier.h @@ -47,6 +47,20 @@ typedef enum { QUERY_TYPE_EXEC_STMT = 0x1000 /*< Execute prepared statement */ } skygw_query_type_t; + +typedef struct parsing_info_st { +#if defined(SS_DEBUG) + skygw_chk_t pi_chk_top; +#endif + void* pi_handle; /*< parsing info object pointer */ + char* pi_query_plain_str; /*< query as plain string */ + void (*pi_done_fp)(void *); /*< clean-up function for parsing info */ +#if defined(SS_DEBUG) + skygw_chk_t pi_chk_tail; +#endif +} parsing_info_t; + + #define QUERY_IS_TYPE(mask,type) ((mask & type) == type) /** @@ -61,9 +75,7 @@ char* skygw_get_canonical(GWBUF* querybuf); bool parse_query (GWBUF* querybuf); parsing_info_t* parsing_info_init(void (*donefun)(void *)); void parsing_info_done(void* ptr); -bool query_is_parsed(GWBUF* buf); - - +bool query_is_parsed(GWBUF* buf); EXTERN_C_BLOCK_END diff --git a/server/core/buffer.c b/server/core/buffer.c index 47e9a22eb..3b8a8c8e6 100644 --- a/server/core/buffer.c +++ b/server/core/buffer.c @@ -40,6 +40,11 @@ #include #include +static buffer_object_t* gwbuf_remove_buffer_object( + GWBUF* buf, + buffer_object_t* bufobj); + + /** * Allocate a new gateway buffer structure of size bytes. * @@ -77,13 +82,15 @@ SHARED_BUF *sbuf; free(sbuf); return NULL; } + spinlock_init(&rval->gwbuf_lock); rval->start = sbuf->data; rval->end = rval->start + size; sbuf->refcount = 1; rval->sbuf = sbuf; rval->next = NULL; rval->gwbuf_type = GWBUF_TYPE_UNDEFINED; - rval->gwbuf_parsing_info = NULL; + rval->gwbuf_info = GWBUF_INFO_NONE; + rval->gwbuf_bufobj = NULL; CHK_GWBUF(rval); return rval; } @@ -96,16 +103,19 @@ SHARED_BUF *sbuf; void gwbuf_free(GWBUF *buf) { + buffer_object_t* bo; + CHK_GWBUF(buf); if (atomic_add(&buf->sbuf->refcount, -1) == 1) { free(buf->sbuf->data); free(buf->sbuf); - } - if (buf->gwbuf_parsing_info != NULL) - { - parsing_info_t* pi = (parsing_info_t *)buf->gwbuf_parsing_info; - pi->pi_done_fp(pi); + bo = buf->gwbuf_bufobj; + + while (bo != NULL) + { + bo = gwbuf_remove_buffer_object(buf, bo); + } } free(buf); } @@ -135,8 +145,9 @@ GWBUF *rval; rval->start = buf->start; rval->end = buf->end; rval->gwbuf_type = buf->gwbuf_type; + rval->gwbuf_info = buf->gwbuf_info; + rval->gwbuf_bufobj = buf->gwbuf_bufobj; rval->next = NULL; - rval->gwbuf_parsing_info = NULL; CHK_GWBUF(rval); return rval; } @@ -162,8 +173,9 @@ GWBUF *gwbuf_clone_portion( clonebuf->start = (void *)((char*)buf->start)+start_offset; clonebuf->end = (void *)((char *)clonebuf->start)+length; clonebuf->gwbuf_type = buf->gwbuf_type; /*< clone the type for now */ + clonebuf->gwbuf_info = buf->gwbuf_info; + clonebuf->gwbuf_bufobj = buf->gwbuf_bufobj; clonebuf->next = NULL; - clonebuf->gwbuf_parsing_info = NULL; CHK_GWBUF(clonebuf); return clonebuf; @@ -343,10 +355,87 @@ void gwbuf_set_type( } } -void* gwbuf_get_parsing_info( - GWBUF* buf) +/** + * Add a buffer object to GWBUF buffer. + * + * @param buf GWBUF where object is added + * @param id Type identifier for object + * @param data Object data + * @param donefun_dp Clean-up function to be executed before buffer is freed. + */ +void gwbuf_add_buffer_object( + GWBUF* buf, + bufobj_id_t id, + void* data, + void (*donefun_fp)(void *)) { + buffer_object_t** p_b; + buffer_object_t* newb; + CHK_GWBUF(buf); - return buf->gwbuf_parsing_info; + newb = (buffer_object_t *)malloc(sizeof(buffer_object_t)); + newb->bo_id = id; + newb->bo_data = data; + newb->bo_donefun_fp = donefun_fp; + newb->bo_next = NULL; + /** Lock */ + spinlock_acquire(&buf->gwbuf_lock); + p_b = &buf->gwbuf_bufobj; + /** Search the end of the list and add there */ + while (*p_b != NULL) + { + p_b = &(*p_b)->bo_next; + } + *p_b = newb; + /** Set flag */ + buf->gwbuf_info |= GWBUF_INFO_PARSED; + /** Unlock */ + spinlock_release(&buf->gwbuf_lock); +} + +/** + * Search buffer object which matches with the id. + * + * @param buf GWBUF to be searched + * @param id Identifier for the object + * + * @return Searched buffer object or NULL if not found + */ +void* gwbuf_get_buffer_object_data( + GWBUF* buf, + bufobj_id_t id) +{ + buffer_object_t* bo; + + CHK_GWBUF(buf); + /** Lock */ + spinlock_acquire(&buf->gwbuf_lock); + bo = buf->gwbuf_bufobj; + + while (bo != NULL && bo->bo_id != id) + { + bo = bo->bo_next; + } + /** Unlock */ + spinlock_release(&buf->gwbuf_lock); + + return bo->bo_data; +} + + +/** + * @return pointer to next buffer object or NULL + */ +static buffer_object_t* gwbuf_remove_buffer_object( + GWBUF* buf, + buffer_object_t* bufobj) +{ + buffer_object_t* next; + + next = bufobj->bo_next; + /** Call corresponding clean-up function to clean buffer object's data */ + bufobj->bo_donefun_fp(bufobj->bo_data); + free(bufobj); + return next; } diff --git a/server/include/buffer.h b/server/include/buffer.h index 57e20dc2e..8f6dcf864 100644 --- a/server/include/buffer.h +++ b/server/include/buffer.h @@ -42,6 +42,8 @@ * @endverbatim */ #include +#include + EXTERN_C_BLOCK_BEGIN @@ -74,18 +76,33 @@ typedef struct { int refcount; /*< Reference count on the buffer */ } SHARED_BUF; +typedef enum +{ + GWBUF_INFO_NONE = 0x0, + GWBUF_INFO_PARSED = 0x1 +} gwbuf_info_t; -typedef struct parsing_info_st { -#if defined(SS_DEBUG) - skygw_chk_t pi_chk_top; -#endif - void* pi_handle; /*< parsing info object pointer */ - char* pi_query_plain_str; /*< query as plain string */ - void (*pi_done_fp)(void *); /*< clean-up function for parsing info */ -#if defined(SS_DEBUG) - skygw_chk_t pi_chk_tail; -#endif -} parsing_info_t; +#define GWBUF_IS_PARSED(b) (b->gwbuf_info & GWBUF_INFO_PARSED) + +/** + * A structure for cleaning up memory allocations of structures which are + * referred to by GWBUF and deallocated in gwbuf_free but GWBUF doesn't + * know what they are. + * All functions on the list are executed before freeing memory of GWBUF struct. + */ +typedef enum +{ + GWBUF_PARSING_INFO +} bufobj_id_t; + +typedef struct buffer_object_st buffer_object_t; + +struct buffer_object_st { + bufobj_id_t bo_id; + void* bo_data; + void (*bo_donefun_fp)(void *); + buffer_object_t* bo_next; +}; /** @@ -97,11 +114,13 @@ typedef struct parsing_info_st { * be copied within the gateway. */ typedef struct gwbuf { + SPINLOCK gwbuf_lock; struct gwbuf *next; /*< Next buffer in a linked chain of buffers */ void *start; /*< Start of the valid data */ void *end; /*< First byte after the valid data */ SHARED_BUF *sbuf; /*< The shared buffer with the real data */ - void *gwbuf_parsing_info; /*< parsing info object pointer */ + buffer_object_t *gwbuf_bufobj; /*< List of objects referred to by GWBUF */ + gwbuf_info_t gwbuf_info; /*< Info bits */ gwbuf_type_t gwbuf_type; /*< buffer's data type information */ } GWBUF; @@ -136,8 +155,12 @@ extern unsigned int gwbuf_length(GWBUF *head); extern GWBUF *gwbuf_clone_portion(GWBUF *head, size_t offset, size_t len); extern GWBUF *gwbuf_clone_transform(GWBUF *head, gwbuf_type_t type); extern void gwbuf_set_type(GWBUF *head, gwbuf_type_t type); -void* gwbuf_get_parsing_info(GWBUF* buf); +void gwbuf_add_buffer_object(GWBUF* buf, + bufobj_id_t id, + void* data, + void (*donefun_fp)(void *)); +void* gwbuf_get_buffer_object_data(GWBUF* buf, bufobj_id_t id); EXTERN_C_BLOCK_END diff --git a/utils/skygw_utils.cc b/utils/skygw_utils.cc index 6ffafd7c3..7bfc42949 100644 --- a/utils/skygw_utils.cc +++ b/utils/skygw_utils.cc @@ -1916,7 +1916,7 @@ char* replace_literal( goto retblock; } - rc = regcomp(&re, search_re, REG_EXTENDED); + rc = regcomp(&re, search_re, REG_EXTENDED|REG_ICASE); ss_dassert(rc == 0); if (rc != 0) @@ -1936,6 +1936,7 @@ char* replace_literal( if (rc != 0) { free(search_re); + regfree(&re); newstr = haystack; goto retblock; } From 73707cc9da989d523023cd3ac2411a2863d461fa Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 22 Aug 2014 20:44:26 +0300 Subject: [PATCH 40/60] canonical query tests for query classifier --- .../test/canonical_tests/Makefile | 63 +++++++++++ .../test/canonical_tests/canonizer.c | 101 ++++++++++++++++++ .../test/canonical_tests/canontest.sh | 21 ++++ .../test/canonical_tests/errmsg.sys | Bin 0 -> 47264 bytes .../test/canonical_tests/expected.sql | 6 ++ .../test/canonical_tests/input.sql | 6 ++ query_classifier/test/makefile | 4 +- 7 files changed, 199 insertions(+), 2 deletions(-) create mode 100644 query_classifier/test/canonical_tests/Makefile create mode 100644 query_classifier/test/canonical_tests/canonizer.c create mode 100755 query_classifier/test/canonical_tests/canontest.sh create mode 100644 query_classifier/test/canonical_tests/errmsg.sys create mode 100755 query_classifier/test/canonical_tests/expected.sql create mode 100755 query_classifier/test/canonical_tests/input.sql diff --git a/query_classifier/test/canonical_tests/Makefile b/query_classifier/test/canonical_tests/Makefile new file mode 100644 index 000000000..a416e1aa9 --- /dev/null +++ b/query_classifier/test/canonical_tests/Makefile @@ -0,0 +1,63 @@ +# cleantests - clean local and subdirectories' tests +# buildtests - build all local and subdirectories' tests +# runtests - run all local tests +# testall - clean, build and run local and subdirectories' tests + +include ../../../build_gateway.inc +include ../../../makefile.inc +include ../../../test.inc + +CC = gcc +CPP = g++ + +TESTPATH := $(shell pwd) +TESTLOG := $(TESTPATH)/testqclass.log +QUERY_CLASSIFIER_PATH := $(ROOT_PATH)/query_classifier +LOG_MANAGER_PATH := $(ROOT_PATH)/log_manager +UTILS_PATH := $(ROOT_PATH)/utils +CORE_PATH := $(ROOT_PATH)/server/core +TESTAPP = $(TESTPATH)/canonizer + +LDFLAGS=-L$(QUERY_CLASSIFIER_PATH) \ + -L$(LOG_MANAGER_PATH) \ + -L$(EMBEDDED_LIB) \ + -Wl,-rpath,$(DEST)/lib \ + -Wl,-rpath,$(EMBEDDED_LIB) \ + -Wl,-rpath,$(LOG_MANAGER_PATH) \ + -Wl,-rpath,$(QUERY_CLASSIFIER_PATH) + +LIBS=-lpthread -lquery_classifier -lz -ldl -lssl -laio -lcrypt -lcrypto -lrt \ + -llog_manager $(UTILS_PATH)/skygw_utils.o $(CORE_PATH)/buffer.o $(CORE_PATH)/atomic.o $(CORE_PATH)/spinlock.o + +CFLAGS=$ -g $(MYSQL_HEADERS) \ + -I$(QUERY_CLASSIFIER_PATH) \ + $(MYSQL_HEADERS) \ + -I$(ROOT_PATH)/server/include \ + -I$(UTILS_PATH) + +EMBFLAGS=$(shell mysql_config --cflags --libmysqld-libs) + + +testall: + $(MAKE) cleantests + $(MAKE) buildtests + $(MAKE) runtests + +cleantests: + - $(DEL) *.o + - $(DEL) *~ + - $(DEL) canonizer + - $(DEL) aria_log* + - $(DEL) ib* + +buildtests: $(OBJS) + $(CC) $(CFLAGS) $(LDFLAGS) $(EMBFLAGS) $(LIBS) canonizer.c -o $(TESTAPP) + +runtests: + @echo "" > $(TESTLOG) + @echo "-------------------------------" >> $(TESTLOG) + @echo $(shell date) >> $(TESTLOG) + @echo "Canonical Query Tests" >> $(TESTLOG) + @echo "-------------------------------" >> $(TESTLOG) + @echo "" >> $(TESTLOG) + ./canontest.sh $(TESTLOG) input.sql output.sql expected.sql diff --git a/query_classifier/test/canonical_tests/canonizer.c b/query_classifier/test/canonical_tests/canonizer.c new file mode 100644 index 000000000..a4c9df90f --- /dev/null +++ b/query_classifier/test/canonical_tests/canonizer.c @@ -0,0 +1,101 @@ +#include +#include +#include +#include +#include +#include +#include + +static char* server_options[] = { + "SkySQL Gateway", + "--datadir=./", + "--language=./", + "--skip-innodb", + "--default-storage-engine=myisam", + NULL +}; + +const int num_elements = (sizeof(server_options) / sizeof(char *)) - 1; + +static char* server_groups[] = { + "embedded", + "server", + "server", + NULL +}; + +int main(int argc, char** argv) +{ + + int fdin,fdout,i=0,fnamelen,fsz,lines = 0; + unsigned int psize; + GWBUF** qbuff; + char *qin, *outnm, *buffer, *tok; + + if(argc != 3){ + printf("Usage: canonizer \n"); + return 1; + } + + + + bool failed = mysql_library_init(num_elements, server_options, server_groups); + + if(failed){ + printf("Embedded server init failed.\n"); + return 1; + } + + fnamelen = strlen(argv[1]) + 16; + fdin = open(argv[1],O_RDONLY); + fsz = lseek(fdin,0,SEEK_END); + lseek(fdin,0,SEEK_SET); + + if(!(buffer = malloc(sizeof(char)*fsz))){ + printf("Error: Failed to allocate memory."); + return 1; + } + + read(fdin,buffer,fsz); + tok = strpbrk(buffer,"\n"); + lines = 1; + + while((tok = strpbrk(tok + 1,"\n"))){ + lines++; + } + + qbuff = malloc(sizeof(GWBUF*)*lines); + + i = 0; + tok = strtok(buffer,"\n"); + + while(tok){ + qin = strdup(tok); + psize = strlen(qin); + qbuff[i] = gwbuf_alloc(psize + 6); + *(qbuff[i]->sbuf->data + 0) = (unsigned char)psize; + *(qbuff[i]->sbuf->data + 1) = (unsigned char)(psize>>8); + *(qbuff[i]->sbuf->data + 2) = (unsigned char)(psize>>16); + *(qbuff[i]->sbuf->data + 4) = 0x03; + memcpy(qbuff[i]->sbuf->data + 5,qin,psize); + *(qbuff[i]->sbuf->data + 5 + psize) = 0x00; + tok = strtok(NULL,"\n"); + free(qin); + i++; + } + + fdout = open(argv[2],O_TRUNC|O_CREAT|O_WRONLY,S_IRWXU|S_IXGRP|S_IXOTH); + + for(i = 0;i " + exit 0 +fi +TESTLOG=$1 +INPUT=$2 +OUTPUT=$3 +EXPECTED=$4 +DIFFLOG=diff.out +$PWD/canonizer $INPUT $OUTPUT +diff $OUTPUT $EXPECTED > $DIFFLOG +if [ $? -eq 0 ] +then + echo "PASSED" >> $TESTLOG +else + echo "FAILED" >> $TESTLOG + echo "Diff output: " >> $TESTLOG + cat $DIFFLOG >> $TESTLOG +fi diff --git a/query_classifier/test/canonical_tests/errmsg.sys b/query_classifier/test/canonical_tests/errmsg.sys new file mode 100644 index 0000000000000000000000000000000000000000..70ec93c1dba02ce8ec5c73bf88e40abb59ab8c45 GIT binary patch literal 47264 zcmezOkBO0y!EEDFW(F*PgMop8gF%2nm_eMuo*{!_3d2!`-wfuAMU1l<_cA_b_ws+> zmla45C>NM6uuR~R0EeKbV5Q)C!Ha?)1e=8>3au5oC&VSJBkU`jExbecj_?m*36UU? zWRXuIX`;(T<;85pGR1nu){DIm;}Z;bP#-SdjzDWJ6dbGwwjk_8=n$DV)n)@}GwA{54wfeMfXh~}aXcuX3*8ZU_ zqvNcTqtmH#T!%x~Q#V(4y)LVszupYJ<9ZDGy84m&)%t7nf9Ts8BpFOKIBf9TfWy$v zFw?NZaIN8G!~cewMp;G|js6=M8G9RN8Q(PiXdGmcYtm=3+l1XT)U?@jt?3n0Q8O2_ zcC&M4zs>y3v&{u8rdhnRFtvPVscq$Dm2I`g>Ymjvt4!-w>+9CsHnBFFY@XW)+xFP* zvi)x>XXkELV7JKbxE;5>xqXg(qy04d)%J(%AJ~7f=XWr3C~}zYu+PEA@q?p)ldn^% zQ;XA5r{hj9oc=nAIIBC`J109&cV6#&%=w=4XJ<|qHJ31#IW8w%Ub_gpI=Gg*Zgl(%RZ$m@ld znzx5{o%dqzBi`S%Ve^&n_3|zEUF7@Bm(x$)FVL^tZ=&CQKV|=5 z|04fR{{#Ng0bv1~1D*!h1QrG^2|OG4AW$_ZF=%4Y@t}V}+QEInyMzA+?+s}R6%FeT z+aAUk?jPO}ekhzFA|PU6#Kj1M$o|N$k+xCQQIn(OqFtktqNhjiihdlOAG1FuHFjz2 zky!V*>2bH>#Nt!p*Tp}JU!L$L!9Vd};-f^fq_m{1Ne`36lf9EGk{2c4N`8|3E?GY% zGi6)K?Ue5+d8zAD`O?DEHm3bZ(@#%HUy}YjT{ojIV}Ax?rcvgk%zc@kGcB_UvsP!_ z&C<=z&0dksm1CHboU=OTMvg>oVs3lxp4{)bL3w-fzUP_dN98x?@6NxIFIW&+aJqoE z(7mv!aAD!$!qP7i<63H6^oacl@ynBm#i+?Tk@hrpwzcCqqL*+ zQK@X1XIW8MU)h?nH)YP{P36nWMJtw6uv7+AE~>mx$zG*bWnWcT)m3$?%DFnedS3OZ z>gUzcH8wR3H4kc7Yt3sjYgy|&>e}j#)G^oV)u+`zu1{!~-f+A@x6!sSqOqj0qj72D zxyHYZ#!badi<%xZ$v1~Jw>9r+e$;H=lF>4~WoOIJ7N^$S)_JYRTkp3rwVAdRw>@m* zXfJKw*#5L#vcspNzT-{D*N)#EterxgGMy@&KAlaSi#m^XzVDRia_-9O>hC(&^|ed7 z+oXF!_w8<$9-E%Dp5~slJ)e3UdfR&s^m_Mw>YLl|J>koQf{A}7R!;giY0+edDRZVU zOii2mdg`C4k<%tmJ2&meG?VGs(|f1CneI5FYsS$TKV}%u44K(I^V&?0S&g#}&w4sb zX149@=Go_Gi_Zz3^L9?}T!DEt^RCVlm>)d9WB#7`m*&5lzkk7!g$0Wo7mF-8y~JSY z#HI6>&0YR^x$Fwl6*E@MS}}XYoE39d%v&*k#ex+JS1ekwc*T+xOIIvgv3$jf6)RV) zTCsY?niXqTtXr{u#fB9dS8Q6bdBv6$TUTsbv3|3#a#eo$E zR~%Y#c*T(wM^_wMaeT#z6(?7mT5)>CnH6VOoLg~z#f23YS6o_gdBv3#S65tHaec*& z6*pJhT5)^DofUUi+*@&f#e)?OS3Fwrc*T_A&Xrs%xmWV6SE{U3U8%NGeWk`q&6Qd!wO8t_)Lp5!Qh%kvO2d^#D~(s0tTbI|w$gm1 z#Y)SSRx7Pn+N`u)X}8jTrNc_cl};<2SGufpU0Jho)5O#3@WkEJ;jCEQT0rXlaDcP)xPf3UIO1ip=7YVyG)pit-B(QXuz&jYd+Q zlAl@(@)uYegKJSyevtx5S3zc8Izp#LGT8GesX3`7NKVvIK=L;My+yexnML3r13LkW z*8DsLs0I0H2!&X}wkS0*MWHA)Ilm}HAu~^*xU#q;HCF)~)KGt?ry7Aru_ z!WB;-mF4+G*IU5q|m~ow7keUaUvjRICmsM$*c_~CF1N#mX zR#2CNgA0p`k~0$X(o+>cDogUg7GtUmf&@}VVzB~5d5S`DW?pitLQZ0Fi2~TB%sfcQ zAxvO!$t=!RNGr|BQP5B=*0feAPs}U%#u`Vg_P2QoXliUZpluq1Vt1$hLG%mq?r6Xh2)(4;#62nVfq514&+*} zD{yB$h{F6lg`(8F#9X+EMuwm~3JoL(2RRZUxag7b??6cUs2i%L>c7{W4B%MspHK*?$fpumJ1$l#t|0?JGIMGC4Z3TZ|8xeCQ4 z`9+E8sS2ri>6v+{3?blx1XU2#47m9jiDjt@B^jB;3i$;knfZCpSOF(LM6N;D?U|RH zUj$0|3YmFn`9-;jAWfk70u^#dNtywn5EP*xS+ENb(F#ua)(Ry>m7w%dlv8l~VED@ZIZF3&GYu~JYi)@1O@M^s(KsX5>nV+hU5&dV>) zQ%KIw%}vZpVQ|jRDb3A8%Ad)Jd3pII3Q4I7d8IiyaP?SR1<8zXS1KgtfKq6s0=N<_ zhU-LR3`0;W0n^6JJOx!REv5lEe~)oXosb1=SRW zz|z#BN`>;oVujS)f|5!Gzx)z~(!9*V(o{&js-KvXnOF=dMnN&H&fu9>mY9>7qL7lB zmROoo0u3RM#RwaHOLIyx3vyBw3W_pw6N@Ur#b-)tT4r8q3In_($xf{-R)D!Z#abaZ zu>w@ufrY_298Ix8L1Iw}HXRUgn3iB@Xg~^Wkk?>w1F-@WiaDuy=_MJUcvDSL02i+e z-l?Fn4J}ooR(i0;27{B2zmo!bI4eL}nMtXj;tJff1K9-iUoxaxD$gv*P$> z2UKZsDmZ$bgIpa$TopY1TwEhSl{wT%Q1pVsoWU(KH77-(IJF?LD6u5JNFlK(y%bzP zWfp_#Ife3!#1c>%gjLJ18WdzgZem_(Vh%%OekmxT)Js6#P)N(HNKH{F$}fke6L4|@ zMSroiLP1U{r~_6C>V&0%EG||kNiE9F%u57S_(_%O3?UgvJwHPui(++n1cOp`Mk=D( zNrCqz@{2%;1QeR!nmVZxRCP1Bp|)btY7j`fz1Rv8{x0Bd7pQIn#YS*;W8>o+ewwMd~PBQXyw4a%q>y*i+l11R*-TB@*Opt!UEl!9R;B*GoVND&3?1%g^Q zki1*00FEa|pAgp|g%C$4A6H1ebqVqhfbikD7Sc-xOTbH0M5;(Ef#y$rkb|MsHQqo| zP(Te~L~??ZOO*;GiAA7>5IDgiwI)F2d1*lksQZ(dr{EUk@2ilUlUP~|YDGdCBG4?B z369EAu>TW_Qo-puzdSFs2-MJps!`1;1vP5Gc7WS|NR;0k{+#@DT~Lb^T!%sjGxRhJbuBF|HMubIGpHhf_Ikk4k*E;l>gWRQ z2tk?$;4uPF=?(UF3RVL^Ei7b1eEglgAyF6%t`zjaWiTY#kwOzvGk{yOiKrDPQe_Tr zse%0ij($+%0!alp0Fl}gU};dU4tDi%bq-M|&o9kMQAn*w%mvl%;8rgR zwk9OlzyS(wNq*xu%>;fB5Y^?xu0H~J)ZUlflshR?D z2Dq7=sF0PPnTOB`N!P^;LHXt29;E`Post78K=c$q$uKvuA~Ux%7nH&v@}N=#ls>>U zCfHsDP-{mATuOi&)u37%T!HC;Qjcm*ikagC%X1 z!F@SMI;d1gR7lE92UQhdNlF;Z(84q)KQRT~JwliPmCek{EXhpF z$*fAnqQ)B3>9a5>29@Xqi6t4}zI=W;v|C)9kyw0XMm`{eUB%W4rFme*kP04>q7qYbGxHSEa!QLcbV0)l#p;+g zfT|H_cLh{x4OO)ORb4R7S-=T+wBmx8NR$PiK@icqlv ztX4+05R$E6!w!(v3%Icl&UghynPr(dsp+ZU%m!@+fU_^SSODc{aOOd@7a?g6+(Ja; zTDb8FpmG453}GYt;AjJlCV_j|phBQ1zZ{g{L1u%8R>0PQN}6H?uo_riR&WNp9o*_j zL2uqb5)-r?o~V$QTCM=e+|U7N1=SQyYfvr*jSj)WB_A> z6`T{mo`>{VP&?hAB04_}lEJ}46G%g^pjr%^guw%_MXBkj6$K2wnH8YW%};|4pg@ao z1r1+MKMhSCU&jayO&w?dP`?ljO&vWwJxxeb&dWzrpINMsnUj;6o|pp)KG1kxDk!VK z)j37N+J_+-sYNik;?m>{h4i9C(69=m_YV#^a0G*#3+j-825~_71fB&MKlNLU~=gVP^44TIbUG0zj~Y4;#UzYzT(*D!x? zSD4G-eJ(5sq$FRVI5kzFJR>tX12hT(%TJ*61ZrazgT^*8QWd~HhZNOF{lgMa@H_g2 zfPxrY$ASiyATDl3S@9~7#!%MP+XZ;l2`$2j3VW)5>U?=Jg#3{ke`8_reSP>5`x#J=7>d*lz&`3!lcoYU2vfz9V3I%9= zl9`{UP>@-W3NalP{$N#U$$2F?8W4LSrWPwe%fbAjl++>yXISu*=7EM z2_4FUXhSs{G)w?0Za~QdU1c#?X-Q^oD##+jD&YwRHnm0AQ2Z)`Qj0;QIHbb~O2MGv zTG%K6o^XaIen_qWHDKXII;_10YQiG-{XmUUj6psOU5=q4{_&oE&Oxreu6`lV7F{ue zr(dvZP>6zytB+%(s|$4a2U=)Aif4#RU`2tt0<5Ky2_8-YrQy^Rh)U!RDq;WyTJoWF zG~pga8~uRvULgZvP@QnA8A9{G8Kop&0bFyH=B1<-M!@PlJsEfQsLeqQuO+ z5*<*UhuDj7A$r^(75LyXEi(@^k^~!whNf;%i3Uoi;NeaM$jA|cV@?iq3>w^{2nh1@ zbqtDB@OF)a^xBeA6=2P8cq;?cUx3utnR$@GQyow#pb+Zk85rscANYYLPLNkJ!39x% z8f4lETnd1OXN&U78A3oEM~MHx$q-a-Wfp;^(?E3$Y@{8c3|az%TCcD|6WZ_vW$4Tt zSPBVp^mI{xren0GW^rO#ssgn6mz$rG3LYZ>HJwU|z+L2gm}DZP(+Ey-kWhn+DS(S{ zNUsn~9C0P!GP0kl&BPF`RsP#uRh!3-&~GoX_ea3i7KQc#7o zaA7SzXn6`sEU6W#$)zQrvE58Wivt`}B}IvO#fjkgBXG$D9?SsEPo$KB#uXKu{e68s zL$FE(`TO`dIXZh|m2~q74fcrl@pli#D(&py>g*lh@97u9KxU-^YEnSz9B^w8UdMp4 zH(JgHx1dl$4cs$_1zS>P9%%9g9Ow!urMU$RPMLW*`RR}q4=FmJq6yRt2y%st4F!j| z1~IrLg40$}YEo%t4tQ1>R3$-5kNh+RusNyFQ3ga-gs$BIS5CQ!#U-iGY7*oH2ESC; z@&Z`3fus&e7F0rjOCHcvHOLHT(gF82+(LbPz~crGr+^#dV8t1r`91K!V@_sq2~-Or zU8E{NI^m$|2Q(rHD{Ddd0Nh&wrIE5!@IWHWU&Wv@4AeVIF3AL~WFC6etmVM`d&z_G^=oSzFC%FWY7$_0rz(4hxtty`Cj3Ei;FVXPR0XP-6r4RA{oGv@d>uhK zP9ek}(p&@QU8J!aNIn8DivdlBlqQ#y7NuH40v=MD<)#)FCxW{9&~6E2NeFZ@2v%!A zl;-4vdbObOWl)ZXcCjH|gtUcWt$1j{E=n!VFD*(=EoKM>t(1T?bm45xCPvVg0(bZbslOpkqREL0;PD+ zI-<1vqSVZENOKXK@W72uP>lgo0qV&^l0l+EazC4d!d&=gw9!b+?{@Y;|3JO%8^z$?Qb^&lvN z!0JEH)IX?OY-E8PyQQFV1XNb5f`%niKrJp%E&_)(ban_{9~48@btNLY1W5S|(nJJ@ ziF%0w#ClM9pQ-?HE@t%$>U5{&WF~_bd4d`U;4FwV@`Y6WB<6q%5I6gO?vFfRa0?#h#)7o)Q8zP~kp-lx?8;7F2bE*BwDh2v9>H12oZ9tgZkn z2^BPwON+qK2_Crv&yH)NE)lZ^uY3b{88kqPbW%aE5CAFG0;YcOQQzM;}}!L1a-4LNrK< zQ%k^o0R_AU!LlBt6$R=PA{G>3u@~-6km)7);P3#4Bq-T|;sCsU6RE_36yynYN~KWK^#)IGz}?}B->611)x zx`Y6ftiTC^ap$C>6AP8{AL>WikbXSV}&qAqj4kGx&pAZHN|ba(-S( zB52YUG`jcfZ zX{hRH6>CD<_a*tDLdzpH2WiUzxKCJAR9a91YI=d95nLmKd#Vi1mB~4o$w*$v&r>Ld zxGED=Z{+8dr51q;B}kbDDM`RR7gbnY8Uzm$PlR0_M_!^diVAWLU#98EiiKs7r8wV~D3CXm%8|Fbf} zfI0+F3E1K$P@6R=H8l@3^Z;7I#1IM}8U=-LVo4%o(i=Qr51JH*EKY(=ii6sSAm^y2 zC}}28f-44jyrif!548It#abZ~wCo971f)W{jbPp2ybhU( zDhE$FfgK3RYw)EV(7FO;v8pv>JPwpxbD-l`uwgss7%U{KBF)Z07w#7&g7&OHDjksH zA(mmNb^)aYxCg++Ar6(H;N{=Y{w;K~1gNY984f9y!6_6r3zeH$0ZJ;#`8he@JX{Pa zA9PfUHNmYK4b@^W8>AW0@dX>;3|Slw+8PC3!kAwKDKGQWz*!yCorTPQKw9mP0oS6` zvi$5+a5EigMi^{*UMi?S1oz-@7(vj;oDP zP6V%80?!4ZcFaNJjj&o6)^0_aC4=;zLCGCuOcpeR3Njm%Li3ACz~x9Ks7ncIlz<(Z zlL%VilbHrz!~-=6w7eLkGqV`9fegHj2f7+Rvlv#afr#v=i|Z22sYsl>Uw}wgSvsBp_arF z=u+7HBBbgPR`aM9gIcwgM#XT3Iw+AtsviYq)f8w=R-B&$s{S(b6cAEKZ4tx*vV71Q zvRqIr37jFDC0-&l6vJ?Y6u92FeP@E6( zA#^hpWHkshO(F#zWK0y4;vqw_;FX1lA%J918#O;2ed!@&^8k3r4&0LkHBLbD-X)NV zP&Fr~)Ec}91JY81D1_I8Ii;{=60nq@4oc%7KP07sq71xO3bj8D&yJu4un_aWn!)ac zwHpwLACzpNF$nUTE~qP(QFK`Xt%N+RrV>V1x2agY7AULgX0@CSelxf4^7L73TdUFaXE#eRPgdH z_%bh0>Qe<3D#azB_57gTG|F0CP^K|30Phe5T2p(tzyCf|&5o`x| z8V=IEg?Jy_)r5r`XuuxYZv*Qp$%k~*GfTi8&df^(n+Q`5oq|UySU`aXE(la%oA*HJ z4^(|asvkse9uh^6?K&u_02Gnnel&dhC1`*bvp}(I(Xn+f7PXKhSG;FU7x&*kU19=G2@`f}HAY;t1VHRl3 z1+4rmMsLIbvg@i6BI^auW5rG0}AAyIiLA^C_-K>Drn1Y3hDtKEjY_TeM zwLZvP$Z}QaglK-6f@-mbCb$X!uQY|O+Q(cLp`f9flcK4hs{n6!F*xVv!3Ty?Qj>E) z+b}=_#2{OWLA&-q9Z_{v5EC?rhu9wv?(c$w3>5F6=|JczN>E-0@4zT60c~qlEe18# zU>yW-x3Czr-Ac6>+~5WG<-nq_S+SCQ1@HhbSR;}kbjTJWtO^~$&I3(jl_Y0?YMPwH zq|_X6S2`#al6>>h6~Ka^UaMl*NoB&ZPwZfJmdqA+(mdHRv&Mo1+LDO$lzk3>+VQ36^84eB={Rq}|* z4A{sRWZMNe)qxf(LpKMjmg*>^=a(p`mNJCqffHM5ih{GhpNl6bCc%l&FJA!~>uKO! z#GpMQ(3l3LCRnV45)-631Ummn<0A?680fL18y zJn*u}OwbrxK4>u=gKuIEs2c)Wq7I%yt1L)W$Vg3uY@0{ZhrCNZ zH7^C!69Mfp0=S}nw`K1j0!RCdCw1<#d%d<&}cLDc|g)DaSf@ZH*oC8^MirV1a1 z0qKMo3Y7wvs-VgTe9VM;F*MtQh66!ky`YJ8=vXS`?2U|6luc-$vnjA@0ykWttuLI4 zU^xwxOyS{)<`D46G$;uvBq|^pOt3K-Xyl>k0W~MeGE>Xdixt2-Fcm-xN+4}$`06)U zgA%*|ycleNLT+kNI%u3FJ-;ZkBqJ9Th3X~nzCL(8H7NDx<(DgHBo-H!=BB17fFcjP zpIRX?2dYmKVp>uvsKEdZV9-cnek!C}04_N}!wjHpv)~nP;1zUe%k?tzKqp>+LJpp0 z5zz_`6VM1gsD^YD!1j<&RkOC)71@Nj} za9D#nJtY~bxzOeqWW1_KAu$J3wh-GYbN3JN2i4V}vID8H0ooA?o)7^S%HWM?&|-%n z1X9YumnVSRY@j{{xbQ=7%RyDE7sG=Hyh;b`pZp^9)B;%-0E!ZDTNYGXU~!x(Xjv*~ zrxB>l0bjtaS`1O12dYS6MKNSoF(OBS#@|76C*TN1-(KjFS(XXjiI!BUV1U#Xz|mC# zcT6DN5e0QsJxG>CZn{I4!>c0u9a6tQN@?gs1GEHW0Iz0K@DBhr96$@pK@+dwlnKiE z3g8ffc?6!!P&I*O$Wx&m6icIGgep+~m)KnbMfu?NIwHPcP49xzqI7W7fM)7JOS?f0 zNzi`dVuk?dnH-?rXa>?^t3(CW6y21}^vn{4jMR$6l+@(RT=+^6h7i!ee+kGGa3Kr{ zl*~K@$nqx0PA&#deSc`z3pC#VvLD>m1@ANmRW`5_G+=9qAoUw4p23C|fVOp)z>EZW zAr8fdJ5Y=J0uQ~=Z9SOAHFws+)#X1R4BZDLT?0jq$I zazOf&7zRO`9`J4j$TLN-fCO!j0nZ46TPdKe0H6_3klB!}Oo+BSC=x&^9~?vADh9GO z0F+aTQo$n)Dd4nOP+C-w4>~|5Gp{7I2y{vSbS4& zbTQhCuxY*uwg>{Gz1H6v$dTP;i1fT#z+B5YK^H*pLzW+{6NK z^98)Z1Jn*dI+u(gyeK~}T_HCyJu?|pVMA76F$7>vheGFMKm{ksWG$g{Vjeiz=H{nlrd2}b z9l&!ppt(lyp>g0!M4=?32t2I?*>#GuaDbYYnFrp@k(pPL58kDULp?YGKphWogVL zJY68do_=ADK2{2_ArC~yO9QmbClB5r2hB@>&h^oRXbbXnbM*AFQUHzmf!dm&mIY!3 zBc!tjTEGZ8ya~Gg3*4myEt$*D1LYd%8Y%@<&~OT9GN2?CVzhr~NU*1itCa$HjVEX% z6Qn!^6dC!YC7`)j1xynmT7!IDgMzFSkV}^QWbmRE@Oc=Zgo^Bnq@u*Ut)0;+}~Wg@7P4L$6nC>6Bc7b(plovs8a>5HKKPs9K!cvT^I*Diy9 zD0tY`(+|XeWlhi$e^BOz#ucdL1?{sW!ovX6aqx2uS12e>EltVSh4&{i^Awz1-TZ@G z6=01u&~!agAYmI*0~bG_wk@b!0+pwr$)EJp{9Lf-@{_Va1J__h;6Vq_^iVOVqX@0| z++F>BT|8u0w1$c}Fbc_cD zzEIErg{W38Xy>4oLOEzHF=(W;7_#Uv2XtZ`cppqoetLXTW?ptpkvNJXLuDS zC+4Jr7Rf-)lmIO_0TnN>ZjZX2f-m&gjLc%tpP=j~-!-|6ZBJh%2@H$M;%o4cU0je?( z=N`aT5`v33$j$*PNZSR}EhtI_PXK|^Y<^NsYA&SxUJM?J1LZEr{Bl0zFefZV=_!C~ zT!^`lJ`uRdY-R!~a#BGxFnDz~lI@`83i#**#2Rz>@CVv}09MmM^%%%_NJAZ5N~9{J zrKW&Z=z`|SL16$I=!YD-r2usrZ22~5DIs`53>?$o4iU5xg0|~m6PloL%|y`l6zZ6R z-?eVg_3w~iDoTchB+ReSfe}zeQV#BX6enk-<|g7gpapa~8~E%!P?G^X(h51H1tbP4 zQkgq&`Rk+9fo=(*@G1kMMLsA8v9NMa2ho>f{iAEj;R_Jiv@uI_g6UDfre|NRbMgwgfLI zN~#2fIAlT|G}fJ*p#T}GOM!HB(F#A9k&yBosg(v=4F*cu;B*V=EEW`{mZj!_7OBEl zYe7bqKy7d|*F((1QZvE(sPIudSo#U`lqg~kMf3uqe*=+s__i$JGUS1J^iCgy;S@kBp*Sq~AppsG19AF_Y}q#fzR zV~zYA=*B!yah97{qN%6g4Bc=5TGa>Y<$ zXrdLf4B(SrKvfQ;Sb;260BM7yGw2D~5Dv7ogvJs0Xm3#M4DIoQ`reS-16tw+-jhMZ z78?Z(NV)}|Jp?ihv?mDJD57`#AWXvCr~w{sN&;+UU6w!T4pkMF*st1347v#+O7wmG*Sn*20_hD_|ihi z+BoQLlA=^ya7u$T$HCJ(kS-Bu69A%d20j}a)OFJUZJY~*B1kJP zKQBHdwV(uatS@Bg17u+de9sOj0Ab|{B(9Nq_u$?#To%$mf(9wL!~<_f)dP(Pg0dQ@ za|1qp5>!){7LJD1E z1kE{!kttB1pcVhH?OD*60j~oF59xt|8&-P97nkIg#DnL#5Oqs2mR(S#nhYQ(f--J` z9-M`k3IrE0@EBz9%g@sVkHdnB0`RaNB5Xh*4=&^&yA;e!ilGCsu<{31M1totVGVz% zUlmZc7No-Gf52NHz}>l|ROk&FDWJLwy3-Y0g%vaSq(XKtfNv9k^aqNd%lyFg0W=be z)sd1EXdDNW+46NWLC3d)yHH5uF`!UKIz%5_ii3<{0B!LHFUDjDMzj*mK}S0yHDW;3 z0BH0=1DcaSCxb)IsD>SS0!nS55k}(H6+%W_K}lT~H0laXhM@8nR9b^V9zKdr`Vk+X z-cm+pdWJ$yYFTOysL07nEe3C%0VhB3rWOW2(6QZc3qf^ZQfe}2r@Fd2gBxg@HfS6H zx}FTw5CKgZd%F6#1S>$s4je(7Z9x-ldEm7?@Dt17YnegA_n;}NeDKykq|Jih!?6|I zAcnd6Is5nrySji*BL@W^tiu5ro!3!NEryg}pg}T3?*W=uV68656eFa)gK7XcVyV^r zfSe|PnvlTR1srFf3;<5pup9s?=b;XTgaK$45TukL7<^O{ti=zROi&1L3j7enuVVyQA!t?tzJm`|7J$z?08L_m)go&PE=_`qLozye5nEC!s9wqdFGB|F zfpqtiz(<)g_=3*G2Df@sL0i6%49L%e?3aMK5WJ#0KM%a=0)>5wF8x;AvI{=R3NDEmd@@lq zW)?$I1GFg!>YRcO?17sF83BNEA;%>^d)aA4x#0OVP~ir$3FJ!9@jQ9ZWzygSyg|t{ z2{I~D0zRx0Ixq{<46b@Wv(=VHNMoBt&~1yL90%zvgQ_G3(C7^`bHSZdtbm$h;A1w> zSY`mH`(Onh*Wh4<5D!Pt{$9|EX;|2TO1xyyu}k1eCJ{Vp0xDHeO@)~PsefR4L8CjM zQWxYn(AgAlN5P{2a@q&zuz#>~z?lV9euKMTpa26!D=5^#i(%mUL959FV7Ua8gh0Uv z3S-ztWKgA_6~@8SlTgcRf8<@=aBbHTw4 ziUGLh)D+k*EJ!CgFBNq02Wb2oG>rppaf35BB#yv?eBiS|;RX~dXk_Lk=ahm^zCp>9 znvfxJa44mwK)0NL3mf=3k|p^HAm1Tn6S%pcAOiOVJRE~P^ifhUND366$hLuk7c%h; z+Q|Yf(xD@e$?y%Zpa~xaNANMakm&*?@WcYTh9)r`y7L-zrWPo! z;jV;@t-;KJg)Jzm!DD>jog?6*%wbB9wq--6iIVg4vY>8+mcB>{6?%vQQV9jO8$9*^ z@d{{+2IMYC6A+wa5XvAYfPoi+gO7uUp91FW@95(i?CgrDSTSp}{5%EBBv8x{sWfnmXWx!!HC@q5K)#2;sP%BSx z#S5woOH1-~;r75H2XtC3C?$YAg79lG=;TjOlO2x-@=+=gxWS;dI_xMA$CMOAdIK## z136UzQpO^xT;$AB%-~o8J!%_N+Q3bPByZ5F=1$+n+_UpOa-+@(o>PL zJbVldWCgUb3dyS|B|E4T1Peeb2P6}~1A(9n0Wu*e719I*&8C1Rz9HK$^dVFH;4BHQ zX&_k{URr|7YN+MK3dTkXNuW9+uOu}+wFq3|LF!?KfSl5FeB+e7HS$v>sfzL%O>Rj*u;X@X`Q0 zUy}**X-*|#*iT0RIuQ-ai$$QmG^i2=&1EPU8u)_4z8F+ML6v}3e}INjKojAhU8?1& z3XtO>z}^Pe{*VQEpyUXu?~*`U{E9$BMu~X}pm@s30o^2rs1OrD3rfMO(!z@%w*!JQ zKIjZ;XydIYzg!ozHVJw98#<#7?f8Q0acFA?c7zUix&pjY6ESuPZp4GFf}GF+JADUS zD(ULxMN!yfORRT7V9ZM7Pdo2ryyglASH-#2|dIbK~y`C20Fmu4K)dsPg`~`M23J3qU{KD2rlw5jwN&8yYCs)o z$dPNoP>(~Vq(Q^gpzaMwAGC>IkXT%zrvM2ZP>us{7D|Dg0fYz^(54Z{=36~**nmf1 zVG}lp{t;+qvMdu+TR<;pgN%QHPV)raiKhqIrU|Z$^b}lS>;6?!bQB=H7|@+ZpfLq- z-U2xjv@;lVRD=R31AtG0fHf^Z#R()nAR~{U$ReZ_;UDNoI4IwN{ex^0SOD72fv)%j zEga8;96A7M+=Er97b}3K(HMe39R$$q18AuUXwNrT7F=^dRyu)-W%$f8__936I26n$ z;Mj+3L53#|D+SQ(1Eh?{b$k}2pnwcafJcTgw~N6>xgqT^@G>>nN>NaFL9bK*x?Rro0hT-3p1|MZMs7(Nh4ASwKebAU6YM7K0Kg_%wL%)Coj8$Xw7oIdoDS zR6BzWDAr>LEdZ^r1l7i%#EN;U7^KGwRt*}10cRK;1+-h?binKQK|AEZM?vO*Zk8>{ zS4c#!9b3OZ60);$DOjo?-@tZoEb28vIR6QRq!q1J#~ zWT^^iIhEjb#n8hzpq(-_$D?&PK?$qz;0&kW%t<6G4}@ zfX<@AA`Knh2iKgS@fpw+J)qvXi;E8^fuqlCf>JQ}IuS_2BQp;+a04G~Kw962uns&* zjl7OCvp64GdO!wvA^8I|o(}GDfbJm$ZOuwVJ_`|4{DDUMit-_MMQ8`o$&r1>kL3&~yyfh+2P^GPve}?!!fDhk`pwAYa0| znD9OltaOA;(1D!{8jA(T6DV0i2Asi7El_et+P{y;N3hlbsQu%Tnw$?Fm&i|305xvR zOhD&?g3>Hxmj&nowo>rzeV{VlH4JnpIVfF#+s>c>1`n-3)~BbW=0Q%v0965?m;xQY znVJVaKNsW}&^%abY6^VK0m@_vXeJoEgBMco27`7^f`b{nXb@5prX=Pjrh|^Uh2M`0 z4sFPAF=*Bd-0Fs=W6;nPw6O^4Q9!oRqc&5}!yFXthyoc_XoH*%u1b)Kd(fdAuz~}z zNDmf0kQrI<%3VDLa9<0i7P?Flln_DrL7^mHA3PHTw-wUgf*Md#o)29wWd&~0B2Qb@bJVO3ns)*Z;DLt1K&Muu=79#^q3eqwtwvBk0(3?+xH}EGGMkuICAc%- z?*}?4Jix~lyaNh!%v1pAwCtcTS5TfQPA!4VAVXSApneXhEmQz%_T=WKrhryl=2XJA z)`Du{%=|pPfwde|@MPvCXQqJmzk&|>0PO%sO@VddAV(MKDfmK;76vb20`*n#^l8Dj zdx7V}Kt&O_q5-WE1h2Y<7d_xrhoJFwXa^0kcvbJ@~5uUpUqt%FC}$^h=71;Mv2fg5+AVzIO+6-^4n|PJ@(77&>9QEsXdfX0dq@w#X11(r8}AVIzuLp}7dungVZE%1?v#OEKL5 z-W3Pl=L#$AAPPCvL2uaq6brqn92G7wXs1_%{TLhpL#NbQhLD#mz+R-Vg#mPmP z;2CZObYt`woZ)ea)N+7^5V(9wErDD}4%^ZM?bv}A-+}f9!!NY~wJLr59bFVan+q76 zz`K&cdkRxP4H!^{0GS6$`rry5vI_`wij03~2r-+0tU%WfW`hXG#oORrL@ut5EFg zTf0+W2!zPy+&#V?ZUKE@;eC zPa(886@0h~*{a};apF@W}I zf!h4J;GqiWI8k0oQao}d0+&w6#XI=&C+Ii}C`O^jMu4{c=42+9z$e@=^?*h#Q$ejg z(9$(LWTXo)m({R=t;ONqfbA9S`IWEdXo6hylSdDRf|qGe<);CXn$T9A)^ zfu{m+lGcNkcAy5NI{act@S2_gsLw$W9+X;~pHm9jZK)0lU=RaTl7Mu9&oTm+F}{%7 zu|cc)K^+;$kSt=fA9VH!yfexW0z2=DAt)8pXhgbw7~Ho4we9l2+Zgjoko%vYgI@A< z9fO@cJryu7@rEq%0_Awf>RougXGlp+DoqDnLYAyhke>-!E0CC30=|JD1+-cmQ$8cJ zL;*%fS$u2bt?g0rjbhbit$2pq-(RDg{)_ zK~9gsr4Z}^@EJaEn{X+Gr!^!~gIt3>-TfSWAbmlIQ;NXjB9NeEfJp1Z)q$q@L4^v) zxeAGhV};-oGT?D>*q%qo_#ym6FK`BiyHf!X3ZR4luL*Ffg6+=D$p`Jp0qv&*=RnBK z=!mic+O5t^0j(!Z&H>%H0v*5uB|^yXKDaMhS^z7Zb#-AGJyjRfInjlfp_`e)06Cut zymJ6l%)o}0t-*6};EgNLOHaVncS*j2zaMB7j*F|Wqn`^ni-ULDfsW;HDFy9|0^NZP zPu$RHdr-0l=S}c9A-MaOR}5aMk_NuU)2h>sI&iTG zK4TC*mjm9#gIM?gE*8KYLqsJ99pD0u9l=*+C4zRyfmWp;hLAk{V9g}(I%4G6s#M6N z7&s=u1Bc+i0Ue+Ljxoff7|7M2>xdB}@L;zIX6j3f(LZ9T`Ga1c^NGpbBVK1AM+6EJ$Eo zPk1$HrI48m>mz`Y1*8`OnhJs+C!hf7@+9S#WPr{M0WB&`EC4q^Qo+MckTq;@6S2vI zjvmP?1D~`CAEu?iDUkMMJiPZ6La!90q11w+)RfHRlGGI9El0iH z60hZ;x!Ph#(*o=O(CSjiGIY>gtB^1xzrZqObX$f9`A@EEw|P^`xg3@bZ8k&G7P;1wmH zBeFn|1l=qHY7SsdQGxfkK>`C*I zC}=^^Nsb-xF=n_OP;XFZOCoqy7<@4&Xg>j#0}`Nvx)6n+aZi+~FW4~(;MqXvkUcn5 zkxbG9_1fWS3Eby}t~pReEYE>A3gHK5A!XiV$Q31^p$gC{C~&I=)QU4U0#DL|TY6vv zKmwqK3F3rYh*t2rZg8Fe)j`l6D98j*GZbzkNGB+TKo0Op2n@_zMzCG7#&Bz&1fnwL$8(f;N;Dg9kOyPH2UVd>}0Vz&LW7!4J~qhxi2Ke$X@< zc;g(10Um7yr7-Za4bYSsWXJ}7^f8u#0^H!%gY-+m9W_1hunK6L34Ej@XrpN+=q?<{ zi5qF)1GFF~_dw3?0|yr9uz;jWP$pQSs2{)1C3ULF7!!MfF5&_nG8zspkp~9{SK6|3dF(?aAY7wI?4v?a8M^3wBi(U z)&MA^6O-~mS0E^)l!C`f5)}$k^T6|;sTH8ve9&wfXmtiCA3;hIP+K3om=dx85j5@q ziZ%vEq?2DDeStjC9w^YlKk!6q3M8v!<|!c8tFSA`kn3o0T@ES|Py`|MIVe8BYY`Hm z`>-=%H8OI^2`MqahdSmVzYn1I01;MneYB zNz4I`!NKAFK`!8IpO>!y8tlwW%S;Bf1Hgq3NDp}F0sNMuymW=K%tVC($dD|9dyu1F z2)O80@N{wY3-NUGbahb(_YCm>)%cmvGZYawf`D%dF9NOV)IwZB1RgVkFJ*xZfq+gU z1xGV{ydAuz2|VnGSl$lW)dRkp5V6r1z9Swq=QFbOHFX91Bn;9DwCn&i22RXaq+nx*>O@YL2ej2zsMLLi(FJA#Q^idQtc!H1B zgw!jLMauB87D$4GjqD>UfgHu706K}JSPwQ~i6~k?E6)=1Ae$jUl?12~2d#sF3@L&R z1yqMGiiI`eiO~abD`+JmXx0xpsR}yj9JF3CT>*4IO;HLc!a!a~%1qZ|fOe)l-TnN7 zT=n$yVCN`cyDzd$31v}^-S`w9b47NE6RaubWP zK??#QXXd0-<|XEWIu1FN3h-%dP>rRh09wBdY50Lgjq?j2gA$M%*Fk6d!6v~$$JoJU z%Ry~GP|1qh66nl1QC2|Jf-Qh|(Tl-rm_Ush&|nRu2@Yu`Cgy=xvVsQWiy#{tp_LKj zQU%DB5+IG>^@FJE{y<}5XoEy}zQt9iVT1yrkqFwN6zb<080xA3 zss!Po03Brjt@i*gvVkorM%h>ZT4)40%n#aV%}XsyEm9~>O$E&?LfoYQI@bf75;Nh) zkw7NOAngU{+Hp|wN&_Fa0cuZzniC)q&<>IO;?xw-A(yZQCn!Tf(xx7R62lMzj-YjW z1*M?L9%xQPRFR+s2jJz7iRg+L;Hw(Ii$_2!qQN6%&~SsbutA*&sHvb*hrt~@4h^yk z(sG5!=b~5-Qiyfr4zwH+G;jh^z<|`&M^X%16Hriz8lDJ0!nA-j zfQKs}2X3P4h0bR|l!H3Q@IEMb8Xt7S7i6n7*dTP>h*mvxXF6y+0DO@I7SDr5K|y6N zr1=kRc0v|$n8M>+r)WHIcXn?2ni$Ftq(B3Fa z0u=DDl?X5~hM-i)Vgb++NyzfzWKbCZI+(pIwFo?*0bW}JS(OSJ%+EzTaN z643GBdQ>n9vYH;W#u_}Zq+S9)c@I={K&}%5rC?B4fQ~^2jb7#Ffv(B{-&hAb`4BP{ z1Q~kse@+3A=_#76v7fg>kMEcCZL%q_+5dJ3!?iN(J2<4jZwdOgD6xJ3mbU+H`>}KLDN2k(bWk0%_2|L_vq(g4+V%g`a5a zgJ9RxzL&r{dK zwK7IQqbL=;`xkUUSY~c!P9o@-bI9U*&?-6Tb|6h_(2c90C4b;W7@(sUL8B3%OVx_d zjfNa(4?2|!y0H`7>ja(q4B3YS*{K1#{}Vhu4a(bKL!lEDkjo9gsRne!CU{~1JmR4M zx$D6x5;~I#TcwHI=?1N+D2818ilH2O7!+vLE$kR~@Y>yMP>M@I>_Y> $(TESTLOG) endif - @cat $(TESTLOG) >> $(TEST_MAXSCALE_LOG) \ No newline at end of file + @cat $(TESTLOG) >> $(TEST_MAXSCALE_LOG) From 662b4b00ee37eba78c5a5294b8098266f8148658 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 22 Aug 2014 20:50:54 +0300 Subject: [PATCH 41/60] the errmsg.sys file is now copied during test building --- query_classifier/test/canonical_tests/Makefile | 1 + .../test/canonical_tests/errmsg.sys | Bin 47264 -> 0 bytes 2 files changed, 1 insertion(+) delete mode 100644 query_classifier/test/canonical_tests/errmsg.sys diff --git a/query_classifier/test/canonical_tests/Makefile b/query_classifier/test/canonical_tests/Makefile index a416e1aa9..6db90ac17 100644 --- a/query_classifier/test/canonical_tests/Makefile +++ b/query_classifier/test/canonical_tests/Makefile @@ -51,6 +51,7 @@ cleantests: - $(DEL) ib* buildtests: $(OBJS) + cp $(ERRMSG)/errmsg.sys . $(CC) $(CFLAGS) $(LDFLAGS) $(EMBFLAGS) $(LIBS) canonizer.c -o $(TESTAPP) runtests: diff --git a/query_classifier/test/canonical_tests/errmsg.sys b/query_classifier/test/canonical_tests/errmsg.sys deleted file mode 100644 index 70ec93c1dba02ce8ec5c73bf88e40abb59ab8c45..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 47264 zcmezOkBO0y!EEDFW(F*PgMop8gF%2nm_eMuo*{!_3d2!`-wfuAMU1l<_cA_b_ws+> zmla45C>NM6uuR~R0EeKbV5Q)C!Ha?)1e=8>3au5oC&VSJBkU`jExbecj_?m*36UU? zWRXuIX`;(T<;85pGR1nu){DIm;}Z;bP#-SdjzDWJ6dbGwwjk_8=n$DV)n)@}GwA{54wfeMfXh~}aXcuX3*8ZU_ zqvNcTqtmH#T!%x~Q#V(4y)LVszupYJ<9ZDGy84m&)%t7nf9Ts8BpFOKIBf9TfWy$v zFw?NZaIN8G!~cewMp;G|js6=M8G9RN8Q(PiXdGmcYtm=3+l1XT)U?@jt?3n0Q8O2_ zcC&M4zs>y3v&{u8rdhnRFtvPVscq$Dm2I`g>Ymjvt4!-w>+9CsHnBFFY@XW)+xFP* zvi)x>XXkELV7JKbxE;5>xqXg(qy04d)%J(%AJ~7f=XWr3C~}zYu+PEA@q?p)ldn^% zQ;XA5r{hj9oc=nAIIBC`J109&cV6#&%=w=4XJ<|qHJ31#IW8w%Ub_gpI=Gg*Zgl(%RZ$m@ld znzx5{o%dqzBi`S%Ve^&n_3|zEUF7@Bm(x$)FVL^tZ=&CQKV|=5 z|04fR{{#Ng0bv1~1D*!h1QrG^2|OG4AW$_ZF=%4Y@t}V}+QEInyMzA+?+s}R6%FeT z+aAUk?jPO}ekhzFA|PU6#Kj1M$o|N$k+xCQQIn(OqFtktqNhjiihdlOAG1FuHFjz2 zky!V*>2bH>#Nt!p*Tp}JU!L$L!9Vd};-f^fq_m{1Ne`36lf9EGk{2c4N`8|3E?GY% zGi6)K?Ue5+d8zAD`O?DEHm3bZ(@#%HUy}YjT{ojIV}Ax?rcvgk%zc@kGcB_UvsP!_ z&C<=z&0dksm1CHboU=OTMvg>oVs3lxp4{)bL3w-fzUP_dN98x?@6NxIFIW&+aJqoE z(7mv!aAD!$!qP7i<63H6^oacl@ynBm#i+?Tk@hrpwzcCqqL*+ zQK@X1XIW8MU)h?nH)YP{P36nWMJtw6uv7+AE~>mx$zG*bWnWcT)m3$?%DFnedS3OZ z>gUzcH8wR3H4kc7Yt3sjYgy|&>e}j#)G^oV)u+`zu1{!~-f+A@x6!sSqOqj0qj72D zxyHYZ#!badi<%xZ$v1~Jw>9r+e$;H=lF>4~WoOIJ7N^$S)_JYRTkp3rwVAdRw>@m* zXfJKw*#5L#vcspNzT-{D*N)#EterxgGMy@&KAlaSi#m^XzVDRia_-9O>hC(&^|ed7 z+oXF!_w8<$9-E%Dp5~slJ)e3UdfR&s^m_Mw>YLl|J>koQf{A}7R!;giY0+edDRZVU zOii2mdg`C4k<%tmJ2&meG?VGs(|f1CneI5FYsS$TKV}%u44K(I^V&?0S&g#}&w4sb zX149@=Go_Gi_Zz3^L9?}T!DEt^RCVlm>)d9WB#7`m*&5lzkk7!g$0Wo7mF-8y~JSY z#HI6>&0YR^x$Fwl6*E@MS}}XYoE39d%v&*k#ex+JS1ekwc*T+xOIIvgv3$jf6)RV) zTCsY?niXqTtXr{u#fB9dS8Q6bdBv6$TUTsbv3|3#a#eo$E zR~%Y#c*T(wM^_wMaeT#z6(?7mT5)>CnH6VOoLg~z#f23YS6o_gdBv3#S65tHaec*& z6*pJhT5)^DofUUi+*@&f#e)?OS3Fwrc*T_A&Xrs%xmWV6SE{U3U8%NGeWk`q&6Qd!wO8t_)Lp5!Qh%kvO2d^#D~(s0tTbI|w$gm1 z#Y)SSRx7Pn+N`u)X}8jTrNc_cl};<2SGufpU0Jho)5O#3@WkEJ;jCEQT0rXlaDcP)xPf3UIO1ip=7YVyG)pit-B(QXuz&jYd+Q zlAl@(@)uYegKJSyevtx5S3zc8Izp#LGT8GesX3`7NKVvIK=L;My+yexnML3r13LkW z*8DsLs0I0H2!&X}wkS0*MWHA)Ilm}HAu~^*xU#q;HCF)~)KGt?ry7Aru_ z!WB;-mF4+G*IU5q|m~ow7keUaUvjRICmsM$*c_~CF1N#mX zR#2CNgA0p`k~0$X(o+>cDogUg7GtUmf&@}VVzB~5d5S`DW?pitLQZ0Fi2~TB%sfcQ zAxvO!$t=!RNGr|BQP5B=*0feAPs}U%#u`Vg_P2QoXliUZpluq1Vt1$hLG%mq?r6Xh2)(4;#62nVfq514&+*} zD{yB$h{F6lg`(8F#9X+EMuwm~3JoL(2RRZUxag7b??6cUs2i%L>c7{W4B%MspHK*?$fpumJ1$l#t|0?JGIMGC4Z3TZ|8xeCQ4 z`9+E8sS2ri>6v+{3?blx1XU2#47m9jiDjt@B^jB;3i$;knfZCpSOF(LM6N;D?U|RH zUj$0|3YmFn`9-;jAWfk70u^#dNtywn5EP*xS+ENb(F#ua)(Ry>m7w%dlv8l~VED@ZIZF3&GYu~JYi)@1O@M^s(KsX5>nV+hU5&dV>) zQ%KIw%}vZpVQ|jRDb3A8%Ad)Jd3pII3Q4I7d8IiyaP?SR1<8zXS1KgtfKq6s0=N<_ zhU-LR3`0;W0n^6JJOx!REv5lEe~)oXosb1=SRW zz|z#BN`>;oVujS)f|5!Gzx)z~(!9*V(o{&js-KvXnOF=dMnN&H&fu9>mY9>7qL7lB zmROoo0u3RM#RwaHOLIyx3vyBw3W_pw6N@Ur#b-)tT4r8q3In_($xf{-R)D!Z#abaZ zu>w@ufrY_298Ix8L1Iw}HXRUgn3iB@Xg~^Wkk?>w1F-@WiaDuy=_MJUcvDSL02i+e z-l?Fn4J}ooR(i0;27{B2zmo!bI4eL}nMtXj;tJff1K9-iUoxaxD$gv*P$> z2UKZsDmZ$bgIpa$TopY1TwEhSl{wT%Q1pVsoWU(KH77-(IJF?LD6u5JNFlK(y%bzP zWfp_#Ife3!#1c>%gjLJ18WdzgZem_(Vh%%OekmxT)Js6#P)N(HNKH{F$}fke6L4|@ zMSroiLP1U{r~_6C>V&0%EG||kNiE9F%u57S_(_%O3?UgvJwHPui(++n1cOp`Mk=D( zNrCqz@{2%;1QeR!nmVZxRCP1Bp|)btY7j`fz1Rv8{x0Bd7pQIn#YS*;W8>o+ewwMd~PBQXyw4a%q>y*i+l11R*-TB@*Opt!UEl!9R;B*GoVND&3?1%g^Q zki1*00FEa|pAgp|g%C$4A6H1ebqVqhfbikD7Sc-xOTbH0M5;(Ef#y$rkb|MsHQqo| zP(Te~L~??ZOO*;GiAA7>5IDgiwI)F2d1*lksQZ(dr{EUk@2ilUlUP~|YDGdCBG4?B z369EAu>TW_Qo-puzdSFs2-MJps!`1;1vP5Gc7WS|NR;0k{+#@DT~Lb^T!%sjGxRhJbuBF|HMubIGpHhf_Ikk4k*E;l>gWRQ z2tk?$;4uPF=?(UF3RVL^Ei7b1eEglgAyF6%t`zjaWiTY#kwOzvGk{yOiKrDPQe_Tr zse%0ij($+%0!alp0Fl}gU};dU4tDi%bq-M|&o9kMQAn*w%mvl%;8rgR zwk9OlzyS(wNq*xu%>;fB5Y^?xu0H~J)ZUlflshR?D z2Dq7=sF0PPnTOB`N!P^;LHXt29;E`Post78K=c$q$uKvuA~Ux%7nH&v@}N=#ls>>U zCfHsDP-{mATuOi&)u37%T!HC;Qjcm*ikagC%X1 z!F@SMI;d1gR7lE92UQhdNlF;Z(84q)KQRT~JwliPmCek{EXhpF z$*fAnqQ)B3>9a5>29@Xqi6t4}zI=W;v|C)9kyw0XMm`{eUB%W4rFme*kP04>q7qYbGxHSEa!QLcbV0)l#p;+g zfT|H_cLh{x4OO)ORb4R7S-=T+wBmx8NR$PiK@icqlv ztX4+05R$E6!w!(v3%Icl&UghynPr(dsp+ZU%m!@+fU_^SSODc{aOOd@7a?g6+(Ja; zTDb8FpmG453}GYt;AjJlCV_j|phBQ1zZ{g{L1u%8R>0PQN}6H?uo_riR&WNp9o*_j zL2uqb5)-r?o~V$QTCM=e+|U7N1=SQyYfvr*jSj)WB_A> z6`T{mo`>{VP&?hAB04_}lEJ}46G%g^pjr%^guw%_MXBkj6$K2wnH8YW%};|4pg@ao z1r1+MKMhSCU&jayO&w?dP`?ljO&vWwJxxeb&dWzrpINMsnUj;6o|pp)KG1kxDk!VK z)j37N+J_+-sYNik;?m>{h4i9C(69=m_YV#^a0G*#3+j-825~_71fB&MKlNLU~=gVP^44TIbUG0zj~Y4;#UzYzT(*D!x? zSD4G-eJ(5sq$FRVI5kzFJR>tX12hT(%TJ*61ZrazgT^*8QWd~HhZNOF{lgMa@H_g2 zfPxrY$ASiyATDl3S@9~7#!%MP+XZ;l2`$2j3VW)5>U?=Jg#3{ke`8_reSP>5`x#J=7>d*lz&`3!lcoYU2vfz9V3I%9= zl9`{UP>@-W3NalP{$N#U$$2F?8W4LSrWPwe%fbAjl++>yXISu*=7EM z2_4FUXhSs{G)w?0Za~QdU1c#?X-Q^oD##+jD&YwRHnm0AQ2Z)`Qj0;QIHbb~O2MGv zTG%K6o^XaIen_qWHDKXII;_10YQiG-{XmUUj6psOU5=q4{_&oE&Oxreu6`lV7F{ue zr(dvZP>6zytB+%(s|$4a2U=)Aif4#RU`2tt0<5Ky2_8-YrQy^Rh)U!RDq;WyTJoWF zG~pga8~uRvULgZvP@QnA8A9{G8Kop&0bFyH=B1<-M!@PlJsEfQsLeqQuO+ z5*<*UhuDj7A$r^(75LyXEi(@^k^~!whNf;%i3Uoi;NeaM$jA|cV@?iq3>w^{2nh1@ zbqtDB@OF)a^xBeA6=2P8cq;?cUx3utnR$@GQyow#pb+Zk85rscANYYLPLNkJ!39x% z8f4lETnd1OXN&U78A3oEM~MHx$q-a-Wfp;^(?E3$Y@{8c3|az%TCcD|6WZ_vW$4Tt zSPBVp^mI{xren0GW^rO#ssgn6mz$rG3LYZ>HJwU|z+L2gm}DZP(+Ey-kWhn+DS(S{ zNUsn~9C0P!GP0kl&BPF`RsP#uRh!3-&~GoX_ea3i7KQc#7o zaA7SzXn6`sEU6W#$)zQrvE58Wivt`}B}IvO#fjkgBXG$D9?SsEPo$KB#uXKu{e68s zL$FE(`TO`dIXZh|m2~q74fcrl@pli#D(&py>g*lh@97u9KxU-^YEnSz9B^w8UdMp4 zH(JgHx1dl$4cs$_1zS>P9%%9g9Ow!urMU$RPMLW*`RR}q4=FmJq6yRt2y%st4F!j| z1~IrLg40$}YEo%t4tQ1>R3$-5kNh+RusNyFQ3ga-gs$BIS5CQ!#U-iGY7*oH2ESC; z@&Z`3fus&e7F0rjOCHcvHOLHT(gF82+(LbPz~crGr+^#dV8t1r`91K!V@_sq2~-Or zU8E{NI^m$|2Q(rHD{Ddd0Nh&wrIE5!@IWHWU&Wv@4AeVIF3AL~WFC6etmVM`d&z_G^=oSzFC%FWY7$_0rz(4hxtty`Cj3Ei;FVXPR0XP-6r4RA{oGv@d>uhK zP9ek}(p&@QU8J!aNIn8DivdlBlqQ#y7NuH40v=MD<)#)FCxW{9&~6E2NeFZ@2v%!A zl;-4vdbObOWl)ZXcCjH|gtUcWt$1j{E=n!VFD*(=EoKM>t(1T?bm45xCPvVg0(bZbslOpkqREL0;PD+ zI-<1vqSVZENOKXK@W72uP>lgo0qV&^l0l+EazC4d!d&=gw9!b+?{@Y;|3JO%8^z$?Qb^&lvN z!0JEH)IX?OY-E8PyQQFV1XNb5f`%niKrJp%E&_)(ban_{9~48@btNLY1W5S|(nJJ@ ziF%0w#ClM9pQ-?HE@t%$>U5{&WF~_bd4d`U;4FwV@`Y6WB<6q%5I6gO?vFfRa0?#h#)7o)Q8zP~kp-lx?8;7F2bE*BwDh2v9>H12oZ9tgZkn z2^BPwON+qK2_Crv&yH)NE)lZ^uY3b{88kqPbW%aE5CAFG0;YcOQQzM;}}!L1a-4LNrK< zQ%k^o0R_AU!LlBt6$R=PA{G>3u@~-6km)7);P3#4Bq-T|;sCsU6RE_36yynYN~KWK^#)IGz}?}B->611)x zx`Y6ftiTC^ap$C>6AP8{AL>WikbXSV}&qAqj4kGx&pAZHN|ba(-S( zB52YUG`jcfZ zX{hRH6>CD<_a*tDLdzpH2WiUzxKCJAR9a91YI=d95nLmKd#Vi1mB~4o$w*$v&r>Ld zxGED=Z{+8dr51q;B}kbDDM`RR7gbnY8Uzm$PlR0_M_!^diVAWLU#98EiiKs7r8wV~D3CXm%8|Fbf} zfI0+F3E1K$P@6R=H8l@3^Z;7I#1IM}8U=-LVo4%o(i=Qr51JH*EKY(=ii6sSAm^y2 zC}}28f-44jyrif!548It#abZ~wCo971f)W{jbPp2ybhU( zDhE$FfgK3RYw)EV(7FO;v8pv>JPwpxbD-l`uwgss7%U{KBF)Z07w#7&g7&OHDjksH zA(mmNb^)aYxCg++Ar6(H;N{=Y{w;K~1gNY984f9y!6_6r3zeH$0ZJ;#`8he@JX{Pa zA9PfUHNmYK4b@^W8>AW0@dX>;3|Slw+8PC3!kAwKDKGQWz*!yCorTPQKw9mP0oS6` zvi$5+a5EigMi^{*UMi?S1oz-@7(vj;oDP zP6V%80?!4ZcFaNJjj&o6)^0_aC4=;zLCGCuOcpeR3Njm%Li3ACz~x9Ks7ncIlz<(Z zlL%VilbHrz!~-=6w7eLkGqV`9fegHj2f7+Rvlv#afr#v=i|Z22sYsl>Uw}wgSvsBp_arF z=u+7HBBbgPR`aM9gIcwgM#XT3Iw+AtsviYq)f8w=R-B&$s{S(b6cAEKZ4tx*vV71Q zvRqIr37jFDC0-&l6vJ?Y6u92FeP@E6( zA#^hpWHkshO(F#zWK0y4;vqw_;FX1lA%J918#O;2ed!@&^8k3r4&0LkHBLbD-X)NV zP&Fr~)Ec}91JY81D1_I8Ii;{=60nq@4oc%7KP07sq71xO3bj8D&yJu4un_aWn!)ac zwHpwLACzpNF$nUTE~qP(QFK`Xt%N+RrV>V1x2agY7AULgX0@CSelxf4^7L73TdUFaXE#eRPgdH z_%bh0>Qe<3D#azB_57gTG|F0CP^K|30Phe5T2p(tzyCf|&5o`x| z8V=IEg?Jy_)r5r`XuuxYZv*Qp$%k~*GfTi8&df^(n+Q`5oq|UySU`aXE(la%oA*HJ z4^(|asvkse9uh^6?K&u_02Gnnel&dhC1`*bvp}(I(Xn+f7PXKhSG;FU7x&*kU19=G2@`f}HAY;t1VHRl3 z1+4rmMsLIbvg@i6BI^auW5rG0}AAyIiLA^C_-K>Drn1Y3hDtKEjY_TeM zwLZvP$Z}QaglK-6f@-mbCb$X!uQY|O+Q(cLp`f9flcK4hs{n6!F*xVv!3Ty?Qj>E) z+b}=_#2{OWLA&-q9Z_{v5EC?rhu9wv?(c$w3>5F6=|JczN>E-0@4zT60c~qlEe18# zU>yW-x3Czr-Ac6>+~5WG<-nq_S+SCQ1@HhbSR;}kbjTJWtO^~$&I3(jl_Y0?YMPwH zq|_X6S2`#al6>>h6~Ka^UaMl*NoB&ZPwZfJmdqA+(mdHRv&Mo1+LDO$lzk3>+VQ36^84eB={Rq}|* z4A{sRWZMNe)qxf(LpKMjmg*>^=a(p`mNJCqffHM5ih{GhpNl6bCc%l&FJA!~>uKO! z#GpMQ(3l3LCRnV45)-631Ummn<0A?680fL18y zJn*u}OwbrxK4>u=gKuIEs2c)Wq7I%yt1L)W$Vg3uY@0{ZhrCNZ zH7^C!69Mfp0=S}nw`K1j0!RCdCw1<#d%d<&}cLDc|g)DaSf@ZH*oC8^MirV1a1 z0qKMo3Y7wvs-VgTe9VM;F*MtQh66!ky`YJ8=vXS`?2U|6luc-$vnjA@0ykWttuLI4 zU^xwxOyS{)<`D46G$;uvBq|^pOt3K-Xyl>k0W~MeGE>Xdixt2-Fcm-xN+4}$`06)U zgA%*|ycleNLT+kNI%u3FJ-;ZkBqJ9Th3X~nzCL(8H7NDx<(DgHBo-H!=BB17fFcjP zpIRX?2dYmKVp>uvsKEdZV9-cnek!C}04_N}!wjHpv)~nP;1zUe%k?tzKqp>+LJpp0 z5zz_`6VM1gsD^YD!1j<&RkOC)71@Nj} za9D#nJtY~bxzOeqWW1_KAu$J3wh-GYbN3JN2i4V}vID8H0ooA?o)7^S%HWM?&|-%n z1X9YumnVSRY@j{{xbQ=7%RyDE7sG=Hyh;b`pZp^9)B;%-0E!ZDTNYGXU~!x(Xjv*~ zrxB>l0bjtaS`1O12dYS6MKNSoF(OBS#@|76C*TN1-(KjFS(XXjiI!BUV1U#Xz|mC# zcT6DN5e0QsJxG>CZn{I4!>c0u9a6tQN@?gs1GEHW0Iz0K@DBhr96$@pK@+dwlnKiE z3g8ffc?6!!P&I*O$Wx&m6icIGgep+~m)KnbMfu?NIwHPcP49xzqI7W7fM)7JOS?f0 zNzi`dVuk?dnH-?rXa>?^t3(CW6y21}^vn{4jMR$6l+@(RT=+^6h7i!ee+kGGa3Kr{ zl*~K@$nqx0PA&#deSc`z3pC#VvLD>m1@ANmRW`5_G+=9qAoUw4p23C|fVOp)z>EZW zAr8fdJ5Y=J0uQ~=Z9SOAHFws+)#X1R4BZDLT?0jq$I zazOf&7zRO`9`J4j$TLN-fCO!j0nZ46TPdKe0H6_3klB!}Oo+BSC=x&^9~?vADh9GO z0F+aTQo$n)Dd4nOP+C-w4>~|5Gp{7I2y{vSbS4& zbTQhCuxY*uwg>{Gz1H6v$dTP;i1fT#z+B5YK^H*pLzW+{6NK z^98)Z1Jn*dI+u(gyeK~}T_HCyJu?|pVMA76F$7>vheGFMKm{ksWG$g{Vjeiz=H{nlrd2}b z9l&!ppt(lyp>g0!M4=?32t2I?*>#GuaDbYYnFrp@k(pPL58kDULp?YGKphWogVL zJY68do_=ADK2{2_ArC~yO9QmbClB5r2hB@>&h^oRXbbXnbM*AFQUHzmf!dm&mIY!3 zBc!tjTEGZ8ya~Gg3*4myEt$*D1LYd%8Y%@<&~OT9GN2?CVzhr~NU*1itCa$HjVEX% z6Qn!^6dC!YC7`)j1xynmT7!IDgMzFSkV}^QWbmRE@Oc=Zgo^Bnq@u*Ut)0;+}~Wg@7P4L$6nC>6Bc7b(plovs8a>5HKKPs9K!cvT^I*Diy9 zD0tY`(+|XeWlhi$e^BOz#ucdL1?{sW!ovX6aqx2uS12e>EltVSh4&{i^Awz1-TZ@G z6=01u&~!agAYmI*0~bG_wk@b!0+pwr$)EJp{9Lf-@{_Va1J__h;6Vq_^iVOVqX@0| z++F>BT|8u0w1$c}Fbc_cD zzEIErg{W38Xy>4oLOEzHF=(W;7_#Uv2XtZ`cppqoetLXTW?ptpkvNJXLuDS zC+4Jr7Rf-)lmIO_0TnN>ZjZX2f-m&gjLc%tpP=j~-!-|6ZBJh%2@H$M;%o4cU0je?( z=N`aT5`v33$j$*PNZSR}EhtI_PXK|^Y<^NsYA&SxUJM?J1LZEr{Bl0zFefZV=_!C~ zT!^`lJ`uRdY-R!~a#BGxFnDz~lI@`83i#**#2Rz>@CVv}09MmM^%%%_NJAZ5N~9{J zrKW&Z=z`|SL16$I=!YD-r2usrZ22~5DIs`53>?$o4iU5xg0|~m6PloL%|y`l6zZ6R z-?eVg_3w~iDoTchB+ReSfe}zeQV#BX6enk-<|g7gpapa~8~E%!P?G^X(h51H1tbP4 zQkgq&`Rk+9fo=(*@G1kMMLsA8v9NMa2ho>f{iAEj;R_Jiv@uI_g6UDfre|NRbMgwgfLI zN~#2fIAlT|G}fJ*p#T}GOM!HB(F#A9k&yBosg(v=4F*cu;B*V=EEW`{mZj!_7OBEl zYe7bqKy7d|*F((1QZvE(sPIudSo#U`lqg~kMf3uqe*=+s__i$JGUS1J^iCgy;S@kBp*Sq~AppsG19AF_Y}q#fzR zV~zYA=*B!yah97{qN%6g4Bc=5TGa>Y<$ zXrdLf4B(SrKvfQ;Sb;260BM7yGw2D~5Dv7ogvJs0Xm3#M4DIoQ`reS-16tw+-jhMZ z78?Z(NV)}|Jp?ihv?mDJD57`#AWXvCr~w{sN&;+UU6w!T4pkMF*st1347v#+O7wmG*Sn*20_hD_|ihi z+BoQLlA=^ya7u$T$HCJ(kS-Bu69A%d20j}a)OFJUZJY~*B1kJP zKQBHdwV(uatS@Bg17u+de9sOj0Ab|{B(9Nq_u$?#To%$mf(9wL!~<_f)dP(Pg0dQ@ za|1qp5>!){7LJD1E z1kE{!kttB1pcVhH?OD*60j~oF59xt|8&-P97nkIg#DnL#5Oqs2mR(S#nhYQ(f--J` z9-M`k3IrE0@EBz9%g@sVkHdnB0`RaNB5Xh*4=&^&yA;e!ilGCsu<{31M1totVGVz% zUlmZc7No-Gf52NHz}>l|ROk&FDWJLwy3-Y0g%vaSq(XKtfNv9k^aqNd%lyFg0W=be z)sd1EXdDNW+46NWLC3d)yHH5uF`!UKIz%5_ii3<{0B!LHFUDjDMzj*mK}S0yHDW;3 z0BH0=1DcaSCxb)IsD>SS0!nS55k}(H6+%W_K}lT~H0laXhM@8nR9b^V9zKdr`Vk+X z-cm+pdWJ$yYFTOysL07nEe3C%0VhB3rWOW2(6QZc3qf^ZQfe}2r@Fd2gBxg@HfS6H zx}FTw5CKgZd%F6#1S>$s4je(7Z9x-ldEm7?@Dt17YnegA_n;}NeDKykq|Jih!?6|I zAcnd6Is5nrySji*BL@W^tiu5ro!3!NEryg}pg}T3?*W=uV68656eFa)gK7XcVyV^r zfSe|PnvlTR1srFf3;<5pup9s?=b;XTgaK$45TukL7<^O{ti=zROi&1L3j7enuVVyQA!t?tzJm`|7J$z?08L_m)go&PE=_`qLozye5nEC!s9wqdFGB|F zfpqtiz(<)g_=3*G2Df@sL0i6%49L%e?3aMK5WJ#0KM%a=0)>5wF8x;AvI{=R3NDEmd@@lq zW)?$I1GFg!>YRcO?17sF83BNEA;%>^d)aA4x#0OVP~ir$3FJ!9@jQ9ZWzygSyg|t{ z2{I~D0zRx0Ixq{<46b@Wv(=VHNMoBt&~1yL90%zvgQ_G3(C7^`bHSZdtbm$h;A1w> zSY`mH`(Onh*Wh4<5D!Pt{$9|EX;|2TO1xyyu}k1eCJ{Vp0xDHeO@)~PsefR4L8CjM zQWxYn(AgAlN5P{2a@q&zuz#>~z?lV9euKMTpa26!D=5^#i(%mUL959FV7Ua8gh0Uv z3S-ztWKgA_6~@8SlTgcRf8<@=aBbHTw4 ziUGLh)D+k*EJ!CgFBNq02Wb2oG>rppaf35BB#yv?eBiS|;RX~dXk_Lk=ahm^zCp>9 znvfxJa44mwK)0NL3mf=3k|p^HAm1Tn6S%pcAOiOVJRE~P^ifhUND366$hLuk7c%h; z+Q|Yf(xD@e$?y%Zpa~xaNANMakm&*?@WcYTh9)r`y7L-zrWPo! z;jV;@t-;KJg)Jzm!DD>jog?6*%wbB9wq--6iIVg4vY>8+mcB>{6?%vQQV9jO8$9*^ z@d{{+2IMYC6A+wa5XvAYfPoi+gO7uUp91FW@95(i?CgrDSTSp}{5%EBBv8x{sWfnmXWx!!HC@q5K)#2;sP%BSx z#S5woOH1-~;r75H2XtC3C?$YAg79lG=;TjOlO2x-@=+=gxWS;dI_xMA$CMOAdIK## z136UzQpO^xT;$AB%-~o8J!%_N+Q3bPByZ5F=1$+n+_UpOa-+@(o>PL zJbVldWCgUb3dyS|B|E4T1Peeb2P6}~1A(9n0Wu*e719I*&8C1Rz9HK$^dVFH;4BHQ zX&_k{URr|7YN+MK3dTkXNuW9+uOu}+wFq3|LF!?KfSl5FeB+e7HS$v>sfzL%O>Rj*u;X@X`Q0 zUy}**X-*|#*iT0RIuQ-ai$$QmG^i2=&1EPU8u)_4z8F+ML6v}3e}INjKojAhU8?1& z3XtO>z}^Pe{*VQEpyUXu?~*`U{E9$BMu~X}pm@s30o^2rs1OrD3rfMO(!z@%w*!JQ zKIjZ;XydIYzg!ozHVJw98#<#7?f8Q0acFA?c7zUix&pjY6ESuPZp4GFf}GF+JADUS zD(ULxMN!yfORRT7V9ZM7Pdo2ryyglASH-#2|dIbK~y`C20Fmu4K)dsPg`~`M23J3qU{KD2rlw5jwN&8yYCs)o z$dPNoP>(~Vq(Q^gpzaMwAGC>IkXT%zrvM2ZP>us{7D|Dg0fYz^(54Z{=36~**nmf1 zVG}lp{t;+qvMdu+TR<;pgN%QHPV)raiKhqIrU|Z$^b}lS>;6?!bQB=H7|@+ZpfLq- z-U2xjv@;lVRD=R31AtG0fHf^Z#R()nAR~{U$ReZ_;UDNoI4IwN{ex^0SOD72fv)%j zEga8;96A7M+=Er97b}3K(HMe39R$$q18AuUXwNrT7F=^dRyu)-W%$f8__936I26n$ z;Mj+3L53#|D+SQ(1Eh?{b$k}2pnwcafJcTgw~N6>xgqT^@G>>nN>NaFL9bK*x?Rro0hT-3p1|MZMs7(Nh4ASwKebAU6YM7K0Kg_%wL%)Coj8$Xw7oIdoDS zR6BzWDAr>LEdZ^r1l7i%#EN;U7^KGwRt*}10cRK;1+-h?binKQK|AEZM?vO*Zk8>{ zS4c#!9b3OZ60);$DOjo?-@tZoEb28vIR6QRq!q1J#~ zWT^^iIhEjb#n8hzpq(-_$D?&PK?$qz;0&kW%t<6G4}@ zfX<@AA`Knh2iKgS@fpw+J)qvXi;E8^fuqlCf>JQ}IuS_2BQp;+a04G~Kw962uns&* zjl7OCvp64GdO!wvA^8I|o(}GDfbJm$ZOuwVJ_`|4{DDUMit-_MMQ8`o$&r1>kL3&~yyfh+2P^GPve}?!!fDhk`pwAYa0| znD9OltaOA;(1D!{8jA(T6DV0i2Asi7El_et+P{y;N3hlbsQu%Tnw$?Fm&i|305xvR zOhD&?g3>Hxmj&nowo>rzeV{VlH4JnpIVfF#+s>c>1`n-3)~BbW=0Q%v0965?m;xQY znVJVaKNsW}&^%abY6^VK0m@_vXeJoEgBMco27`7^f`b{nXb@5prX=Pjrh|^Uh2M`0 z4sFPAF=*Bd-0Fs=W6;nPw6O^4Q9!oRqc&5}!yFXthyoc_XoH*%u1b)Kd(fdAuz~}z zNDmf0kQrI<%3VDLa9<0i7P?Flln_DrL7^mHA3PHTw-wUgf*Md#o)29wWd&~0B2Qb@bJVO3ns)*Z;DLt1K&Muu=79#^q3eqwtwvBk0(3?+xH}EGGMkuICAc%- z?*}?4Jix~lyaNh!%v1pAwCtcTS5TfQPA!4VAVXSApneXhEmQz%_T=WKrhryl=2XJA z)`Du{%=|pPfwde|@MPvCXQqJmzk&|>0PO%sO@VddAV(MKDfmK;76vb20`*n#^l8Dj zdx7V}Kt&O_q5-WE1h2Y<7d_xrhoJFwXa^0kcvbJ@~5uUpUqt%FC}$^h=71;Mv2fg5+AVzIO+6-^4n|PJ@(77&>9QEsXdfX0dq@w#X11(r8}AVIzuLp}7dungVZE%1?v#OEKL5 z-W3Pl=L#$AAPPCvL2uaq6brqn92G7wXs1_%{TLhpL#NbQhLD#mz+R-Vg#mPmP z;2CZObYt`woZ)ea)N+7^5V(9wErDD}4%^ZM?bv}A-+}f9!!NY~wJLr59bFVan+q76 zz`K&cdkRxP4H!^{0GS6$`rry5vI_`wij03~2r-+0tU%WfW`hXG#oORrL@ut5EFg zTf0+W2!zPy+&#V?ZUKE@;eC zPa(886@0h~*{a};apF@W}I zf!h4J;GqiWI8k0oQao}d0+&w6#XI=&C+Ii}C`O^jMu4{c=42+9z$e@=^?*h#Q$ejg z(9$(LWTXo)m({R=t;ONqfbA9S`IWEdXo6hylSdDRf|qGe<);CXn$T9A)^ zfu{m+lGcNkcAy5NI{act@S2_gsLw$W9+X;~pHm9jZK)0lU=RaTl7Mu9&oTm+F}{%7 zu|cc)K^+;$kSt=fA9VH!yfexW0z2=DAt)8pXhgbw7~Ho4we9l2+Zgjoko%vYgI@A< z9fO@cJryu7@rEq%0_Awf>RougXGlp+DoqDnLYAyhke>-!E0CC30=|JD1+-cmQ$8cJ zL;*%fS$u2bt?g0rjbhbit$2pq-(RDg{)_ zK~9gsr4Z}^@EJaEn{X+Gr!^!~gIt3>-TfSWAbmlIQ;NXjB9NeEfJp1Z)q$q@L4^v) zxeAGhV};-oGT?D>*q%qo_#ym6FK`BiyHf!X3ZR4luL*Ffg6+=D$p`Jp0qv&*=RnBK z=!mic+O5t^0j(!Z&H>%H0v*5uB|^yXKDaMhS^z7Zb#-AGJyjRfInjlfp_`e)06Cut zymJ6l%)o}0t-*6};EgNLOHaVncS*j2zaMB7j*F|Wqn`^ni-ULDfsW;HDFy9|0^NZP zPu$RHdr-0l=S}c9A-MaOR}5aMk_NuU)2h>sI&iTG zK4TC*mjm9#gIM?gE*8KYLqsJ99pD0u9l=*+C4zRyfmWp;hLAk{V9g}(I%4G6s#M6N z7&s=u1Bc+i0Ue+Ljxoff7|7M2>xdB}@L;zIX6j3f(LZ9T`Ga1c^NGpbBVK1AM+6EJ$Eo zPk1$HrI48m>mz`Y1*8`OnhJs+C!hf7@+9S#WPr{M0WB&`EC4q^Qo+MckTq;@6S2vI zjvmP?1D~`CAEu?iDUkMMJiPZ6La!90q11w+)RfHRlGGI9El0iH z60hZ;x!Ph#(*o=O(CSjiGIY>gtB^1xzrZqObX$f9`A@EEw|P^`xg3@bZ8k&G7P;1wmH zBeFn|1l=qHY7SsdQGxfkK>`C*I zC}=^^Nsb-xF=n_OP;XFZOCoqy7<@4&Xg>j#0}`Nvx)6n+aZi+~FW4~(;MqXvkUcn5 zkxbG9_1fWS3Eby}t~pReEYE>A3gHK5A!XiV$Q31^p$gC{C~&I=)QU4U0#DL|TY6vv zKmwqK3F3rYh*t2rZg8Fe)j`l6D98j*GZbzkNGB+TKo0Op2n@_zMzCG7#&Bz&1fnwL$8(f;N;Dg9kOyPH2UVd>}0Vz&LW7!4J~qhxi2Ke$X@< zc;g(10Um7yr7-Za4bYSsWXJ}7^f8u#0^H!%gY-+m9W_1hunK6L34Ej@XrpN+=q?<{ zi5qF)1GFF~_dw3?0|yr9uz;jWP$pQSs2{)1C3ULF7!!MfF5&_nG8zspkp~9{SK6|3dF(?aAY7wI?4v?a8M^3wBi(U z)&MA^6O-~mS0E^)l!C`f5)}$k^T6|;sTH8ve9&wfXmtiCA3;hIP+K3om=dx85j5@q ziZ%vEq?2DDeStjC9w^YlKk!6q3M8v!<|!c8tFSA`kn3o0T@ES|Py`|MIVe8BYY`Hm z`>-=%H8OI^2`MqahdSmVzYn1I01;MneYB zNz4I`!NKAFK`!8IpO>!y8tlwW%S;Bf1Hgq3NDp}F0sNMuymW=K%tVC($dD|9dyu1F z2)O80@N{wY3-NUGbahb(_YCm>)%cmvGZYawf`D%dF9NOV)IwZB1RgVkFJ*xZfq+gU z1xGV{ydAuz2|VnGSl$lW)dRkp5V6r1z9Swq=QFbOHFX91Bn;9DwCn&i22RXaq+nx*>O@YL2ej2zsMLLi(FJA#Q^idQtc!H1B zgw!jLMauB87D$4GjqD>UfgHu706K}JSPwQ~i6~k?E6)=1Ae$jUl?12~2d#sF3@L&R z1yqMGiiI`eiO~abD`+JmXx0xpsR}yj9JF3CT>*4IO;HLc!a!a~%1qZ|fOe)l-TnN7 zT=n$yVCN`cyDzd$31v}^-S`w9b47NE6RaubWP zK??#QXXd0-<|XEWIu1FN3h-%dP>rRh09wBdY50Lgjq?j2gA$M%*Fk6d!6v~$$JoJU z%Ry~GP|1qh66nl1QC2|Jf-Qh|(Tl-rm_Ush&|nRu2@Yu`Cgy=xvVsQWiy#{tp_LKj zQU%DB5+IG>^@FJE{y<}5XoEy}zQt9iVT1yrkqFwN6zb<080xA3 zss!Po03Brjt@i*gvVkorM%h>ZT4)40%n#aV%}XsyEm9~>O$E&?LfoYQI@bf75;Nh) zkw7NOAngU{+Hp|wN&_Fa0cuZzniC)q&<>IO;?xw-A(yZQCn!Tf(xx7R62lMzj-YjW z1*M?L9%xQPRFR+s2jJz7iRg+L;Hw(Ii$_2!qQN6%&~SsbutA*&sHvb*hrt~@4h^yk z(sG5!=b~5-Qiyfr4zwH+G;jh^z<|`&M^X%16Hriz8lDJ0!nA-j zfQKs}2X3P4h0bR|l!H3Q@IEMb8Xt7S7i6n7*dTP>h*mvxXF6y+0DO@I7SDr5K|y6N zr1=kRc0v|$n8M>+r)WHIcXn?2ni$Ftq(B3Fa z0u=DDl?X5~hM-i)Vgb++NyzfzWKbCZI+(pIwFo?*0bW}JS(OSJ%+EzTaN z643GBdQ>n9vYH;W#u_}Zq+S9)c@I={K&}%5rC?B4fQ~^2jb7#Ffv(B{-&hAb`4BP{ z1Q~kse@+3A=_#76v7fg>kMEcCZL%q_+5dJ3!?iN(J2<4jZwdOgD6xJ3mbU+H`>}KLDN2k(bWk0%_2|L_vq(g4+V%g`a5a zgJ9RxzL&r{dK zwK7IQqbL=;`xkUUSY~c!P9o@-bI9U*&?-6Tb|6h_(2c90C4b;W7@(sUL8B3%OVx_d zjfNa(4?2|!y0H`7>ja(q4B3YS*{K1#{}Vhu4a(bKL!lEDkjo9gsRne!CU{~1JmR4M zx$D6x5;~I#TcwHI=?1N+D2818ilH2O7!+vLE$kR~@Y>yMP>M@I>_Y Date: Mon, 25 Aug 2014 15:09:33 +0300 Subject: [PATCH 42/60] more test cases for canonical queries and makefile typo fixes --- query_classifier/test/canonical_tests/Makefile | 4 ++-- query_classifier/test/canonical_tests/expected.sql | 7 +++++++ query_classifier/test/canonical_tests/input.sql | 7 +++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/query_classifier/test/canonical_tests/Makefile b/query_classifier/test/canonical_tests/Makefile index 6db90ac17..3d69507ec 100644 --- a/query_classifier/test/canonical_tests/Makefile +++ b/query_classifier/test/canonical_tests/Makefile @@ -26,10 +26,10 @@ LDFLAGS=-L$(QUERY_CLASSIFIER_PATH) \ -Wl,-rpath,$(LOG_MANAGER_PATH) \ -Wl,-rpath,$(QUERY_CLASSIFIER_PATH) -LIBS=-lpthread -lquery_classifier -lz -ldl -lssl -laio -lcrypt -lcrypto -lrt \ +LIBS=-lstdc++ -lpthread -lquery_classifier -lz -ldl -lssl -laio -lcrypt -lcrypto -lrt \ -llog_manager $(UTILS_PATH)/skygw_utils.o $(CORE_PATH)/buffer.o $(CORE_PATH)/atomic.o $(CORE_PATH)/spinlock.o -CFLAGS=$ -g $(MYSQL_HEADERS) \ +CFLAGS=-g $(MYSQL_HEADERS) \ -I$(QUERY_CLASSIFIER_PATH) \ $(MYSQL_HEADERS) \ -I$(ROOT_PATH)/server/include \ diff --git a/query_classifier/test/canonical_tests/expected.sql b/query_classifier/test/canonical_tests/expected.sql index e1db906af..9ba3aa925 100755 --- a/query_classifier/test/canonical_tests/expected.sql +++ b/query_classifier/test/canonical_tests/expected.sql @@ -4,3 +4,10 @@ select ?,?,?,?,?,? from tst select * from tst where fname like '?' select * from tst where lname like '?' order by fname insert into tst values ("?","?"),("?",?),("?","?") +drop table if exists tst +create table tst(fname varchar(30), lname varchar(30)) +update tst set lname="?" where fname like '?' or lname like '?' +delete from tst where lname like '?' and fname like '?' +select ? from tst where fname='?' or lname like '?' +select ?,?,?,? from tst where name='?' or name='?' or name='?' +select count(?),count(?),count(?),count(?),count (?),count(?) from tst diff --git a/query_classifier/test/canonical_tests/input.sql b/query_classifier/test/canonical_tests/input.sql index eabf0af32..192db761c 100755 --- a/query_classifier/test/canonical_tests/input.sql +++ b/query_classifier/test/canonical_tests/input.sql @@ -4,3 +4,10 @@ select 1,2,3,4,5,6 from tst; select * from tst where fname like '%a%'; select * from tst where lname like '%e%' order by fname; insert into tst values ("John","Doe"),("Plato",null),("Nietzsche",""); +drop table if exists tst; +create table tst(fname varchar(30), lname varchar(30)); +update tst set lname="Human" where fname like '%a%' or lname like '%a%'; +delete from tst where lname like '%man%' and fname like '%ard%'; +select 100 from tst where fname='10' or lname like '%100%'; +select 1,20,300,4000 from tst where name='1000' or name='200' or name='30' or name='4'; +select count(1),count(10),count(100),count(2),count (20),count(200) from tst; From c5fbb1f29594adbc403e2d09feec8cd4c1e880be Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Mon, 25 Aug 2014 22:17:21 +0300 Subject: [PATCH 43/60] query_classifier.cc:parsing_info_done: calling mysql_thread_end caused segfauls when same thread tried next time call free_embedded_thd because thread's sysvar was set to NULL in mysql_thread_end. Added a few lines to canonical query test's input script which are not handled correctly. --- log_manager/test/makefile | 2 +- query_classifier/query_classifier.cc | 4 +--- query_classifier/test/canonical_tests/input.sql | 4 ++++ 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/log_manager/test/makefile b/log_manager/test/makefile index 75df90c60..71fd074f1 100644 --- a/log_manager/test/makefile +++ b/log_manager/test/makefile @@ -39,7 +39,7 @@ buildtests: -o testlog \ -I$(MARIADB_SRC_PATH)/include \ -I$(LOG_MANAGER_PATH) -I$(UTILS_PATH) testlog.c \ - -llog_manager $(LDLIBS) \ + -lstdc++ -llog_manager $(LDLIBS) \ $(UTILS_PATH)/skygw_utils.o diff --git a/query_classifier/query_classifier.cc b/query_classifier/query_classifier.cc index 009cbe94f..8cef07a10 100644 --- a/query_classifier/query_classifier.cc +++ b/query_classifier/query_classifier.cc @@ -964,9 +964,8 @@ parsing_info_t* parsing_info_init( mysql_errno(mysql), mysql_error(mysql)))); - mysql_library_end(); goto retblock; - } + } /** Set methods and authentication to mysql */ mysql_options(mysql, MYSQL_READ_DEFAULT_GROUP, "libmysqld_skygw"); mysql_options(mysql, MYSQL_OPT_USE_EMBEDDED_CONNECTION, NULL); @@ -1019,7 +1018,6 @@ void parsing_info_done( mysql->thd = NULL; } mysql_close(mysql); - mysql_thread_end(); } /** Free plain text query string */ if (pi->pi_query_plain_str != NULL) diff --git a/query_classifier/test/canonical_tests/input.sql b/query_classifier/test/canonical_tests/input.sql index eabf0af32..5e0410761 100755 --- a/query_classifier/test/canonical_tests/input.sql +++ b/query_classifier/test/canonical_tests/input.sql @@ -4,3 +4,7 @@ select 1,2,3,4,5,6 from tst; select * from tst where fname like '%a%'; select * from tst where lname like '%e%' order by fname; insert into tst values ("John","Doe"),("Plato",null),("Nietzsche",""); +select md5("200000foo") =10, sleep(2), rand(100); +select * from my1 where md5("110") =10; +select md5("100foo") =10; +select * from my1 where md5("100") =10; From 69104d7dee584261e4fcfa2ae3833b45a009dfea Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Tue, 26 Aug 2014 11:08:05 +0300 Subject: [PATCH 44/60] skygw_utils.cc:replace_literal: Fixed regular expression which, for example, accepted "200" with needle "2" query_classifier.cc: fixed invalid argument list in logging command. input.sql: added a few previously failed cases for canonical query test --- query_classifier/query_classifier.cc | 1 - query_classifier/test/canonical_tests/input.sql | 4 ++++ utils/skygw_utils.cc | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/query_classifier/query_classifier.cc b/query_classifier/query_classifier.cc index 8cef07a10..7a4bcba40 100644 --- a/query_classifier/query_classifier.cc +++ b/query_classifier/query_classifier.cc @@ -672,7 +672,6 @@ static skygw_query_type_t resolve_query_type( "%lu [resolve_query_type] " "functype FUNC_SP, stored proc " "or unknown function.", - "%s:%s", pthread_self()))); break; case Item_func::UDF_FUNC: diff --git a/query_classifier/test/canonical_tests/input.sql b/query_classifier/test/canonical_tests/input.sql index 192db761c..59e9a2496 100755 --- a/query_classifier/test/canonical_tests/input.sql +++ b/query_classifier/test/canonical_tests/input.sql @@ -1,3 +1,7 @@ +select md5("200000foo") =10, sleep(2), rand(100); +select * from my1 where md5("110") =10; +select md5("100foo") =10; +select * from my1 where md5("100") =10; select sleep(2); select * from tst where lname='Doe'; select 1,2,3,4,5,6 from tst; diff --git a/utils/skygw_utils.cc b/utils/skygw_utils.cc index 7bfc42949..8297ca5ba 100644 --- a/utils/skygw_utils.cc +++ b/utils/skygw_utils.cc @@ -1883,7 +1883,7 @@ char* replace_literal( const char* replacement) { const char* prefix = "[ ='\",\\(]"; /*< ' ','=','(',''',''"',',' are allowed before needle */ - const char* suffix = "[$^[:alnum:]]?"; /*< alpha-num chars aren't allowed after the needle */ + const char* suffix = "[^[:alnum:]]"; /*< alpha-num chars aren't allowed after the needle */ char* search_re; char* newstr; regex_t re; From cf5821d4ef6f392a91460f8be8767465fcd1608b Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Tue, 26 Aug 2014 11:14:51 +0300 Subject: [PATCH 45/60] Added expected results for missing queries. --- query_classifier/test/canonical_tests/expected.sql | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/query_classifier/test/canonical_tests/expected.sql b/query_classifier/test/canonical_tests/expected.sql index 9ba3aa925..fabb27dc7 100755 --- a/query_classifier/test/canonical_tests/expected.sql +++ b/query_classifier/test/canonical_tests/expected.sql @@ -1,3 +1,7 @@ +select md5(?) =?, sleep(?), rand(?); +select * from my1 where md5(?) =?; +select md5(?) =?; +select * from my1 where md5(?) =?; select sleep(?) select * from tst where lname='?' select ?,?,?,?,?,? from tst From 531d8d7b47cefc1abe9df60a5da044a232a6a2a4 Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Fri, 29 Aug 2014 10:08:48 +0300 Subject: [PATCH 46/60] query_classifier.cc: added detection for CREATE TEMPORARY TABLE and setting a new query type QUERY_TYPE_CREATE_TMP_TABLE for it. query_classifier.h: added QUERY_TYPE_CREATE_TMP_TABLE and QUERY_TYPE_READ_TMP_TABLE for use of temporary table support. hashtable.c:Added variant of hashtable which is 'flat', that is, stored to existing memory instead of allocating memory as a part of the call. Existing function declarations don't change but added hashtable_alloc_flat for the purpose. Both hashtable_alloc and hashtable_alloc_flat now call the real allocation function, hashtable_alloc_real. hashtable_free only frees memory which is allocated in hashtable_alloc_real. hashtable.h: added a flag to HASHTABLE struct to indicate whether hashtable owns its memory or not. readwritesplit.h: Added RSES_PROP_TYPE_TMPTABLES property type to be used for keeping the hashtable for tablenames. readwritesplit.c: Added comments about temporary table support implementation. --- query_classifier/query_classifier.cc | 42 +++++++- query_classifier/query_classifier.h | 6 +- server/core/hashtable.c | 102 +++++++++++++++++- server/include/hashtable.h | 5 + server/modules/include/readwritesplit.h | 4 +- .../routing/readwritesplit/readwritesplit.c | 24 ++++- 6 files changed, 175 insertions(+), 8 deletions(-) diff --git a/query_classifier/query_classifier.cc b/query_classifier/query_classifier.cc index 889940e00..fe712008c 100644 --- a/query_classifier/query_classifier.cc +++ b/query_classifier/query_classifier.cc @@ -494,8 +494,15 @@ static skygw_query_type_t resolve_query_type( force_data_modify_op_replication) { type |= QUERY_TYPE_SESSION_WRITE; - } else { + } + else + { type |= QUERY_TYPE_WRITE; + + if (lex->create_info.options & HA_LEX_CREATE_TMP_TABLE) + { + type |= QUERY_TYPE_CREATE_TMP_TABLE; + } } goto return_qtype; @@ -692,6 +699,17 @@ static skygw_query_type_t resolve_query_type( break; } } /**< for */ +#if defined(TEMPORARY_TABLES) + if ((skygw_query_type_t)type == QUERY_TYPE_READ) + { + /** + * Find out the database name and all tables the query + * uses. Create a hashvalue from each and if any of the + * values can be found from property's hashtable, set + * query type to QUERY_TYPE_READ_TMP_TABLE. + */ + } +#endif } /**< if */ return_qtype: qtype = (skygw_query_type_t)type; @@ -816,3 +834,25 @@ char* skygw_query_classifier_get_stmtname( return ((THD *)(mysql->thd))->lex->prepared_stmt_name.str; } + +/** + * Finds the head of the list of tables affected by the current select statement. + * @param thd Pointer to a valid thread descriptor structure + * @return Head of the TABLE_LIST chain or NULL in case of an error + */ +void* skygw_get_affected_tables(void* thdp) +{ + THD* thd = (THD*)thdp; + + if(thd == NULL || + thd->lex == NULL || + thd->lex->current_select == NULL) + { + ss_dassert(thd != NULL && + thd->lex != NULL && + thd->lex->current_select != NULL); + return NULL; + } + + return (void*)thd->lex->current_select->table_list.first; +} \ No newline at end of file diff --git a/query_classifier/query_classifier.h b/query_classifier/query_classifier.h index 31f9cf44e..52bfea4f2 100644 --- a/query_classifier/query_classifier.h +++ b/query_classifier/query_classifier.h @@ -43,7 +43,9 @@ typedef enum { QUERY_TYPE_COMMIT = 0x0200, /*< COMMIT */ QUERY_TYPE_PREPARE_NAMED_STMT = 0x0400, /*< Prepared stmt with name from user */ QUERY_TYPE_PREPARE_STMT = 0x0800, /*< Prepared stmt with id provided by server */ - QUERY_TYPE_EXEC_STMT = 0x1000 /*< Execute prepared statement */ + QUERY_TYPE_EXEC_STMT = 0x1000, /*< Execute prepared statement */ + QUERY_TYPE_CREATE_TMP_TABLE = 0x2000, /*< Create temporary table */ + QUERY_TYPE_READ_TMP_TABLE = 0x4000 /*< Read temporary table */ } skygw_query_type_t; #define QUERY_IS_TYPE(mask,type) ((mask & type) == type) @@ -60,6 +62,8 @@ skygw_query_type_t skygw_query_classifier_get_type( /** Free THD context and close MYSQL */ void skygw_query_classifier_free(MYSQL* mysql); char* skygw_query_classifier_get_stmtname(MYSQL* mysql); +void* skygw_get_affected_tables(void* thdp); + EXTERN_C_BLOCK_END diff --git a/server/core/hashtable.c b/server/core/hashtable.c index 50857bfec..b56f5d169 100644 --- a/server/core/hashtable.c +++ b/server/core/hashtable.c @@ -63,6 +63,10 @@ static void hashtable_read_lock(HASHTABLE *table); static void hashtable_read_unlock(HASHTABLE *table); static void hashtable_write_lock(HASHTABLE *table); static void hashtable_write_unlock(HASHTABLE *table); +static HASHTABLE *hashtable_alloc_real(HASHTABLE* target, + int size, + int (*hashfn)(), + int (*cmpfn)()); /** * Special null function used as default memory allfunctions in the hashtable @@ -81,11 +85,75 @@ nullfn(void *data) /** * Allocate a new hash table * + * @param target The address where hashtable is to be initialized, if NULL then allocate * @param size The size of the hash table * @param hashfn The user supplied hash function * @param cmpfn The user supplied key comparison function * @return The hashtable table */ +#if 1 +HASHTABLE * +hashtable_alloc(int size, int (*hashfn)(), int (*cmpfn)()) +{ + return hashtable_alloc_real(NULL, size, hashfn, cmpfn); +} + +HASHTABLE* hashtable_alloc_flat( + HASHTABLE* target, + int size, + int (*hashfn)(), + int (*cmpfn)()) +{ + return hashtable_alloc_real(target, size, hashfn, cmpfn); +} + +static HASHTABLE * +hashtable_alloc_real( + HASHTABLE* target, + int size, + int (*hashfn)(), + int (*cmpfn)()) +{ + HASHTABLE *rval; + + if (target == NULL) + { + if ((rval = malloc(sizeof(HASHTABLE))) == NULL) + return NULL; + rval->ht_isflat = false; + } + else + { + rval = target; + rval->ht_isflat = true; + } + +#if defined(SS_DEBUG) + rval->ht_chk_top = CHK_NUM_HASHTABLE; + rval->ht_chk_tail = CHK_NUM_HASHTABLE; +#endif + rval->hashsize = size; + rval->hashfn = hashfn; + rval->cmpfn = cmpfn; + rval->kcopyfn = nullfn; + rval->vcopyfn = nullfn; + rval->kfreefn = nullfn; + rval->vfreefn = nullfn; + rval->n_readers = 0; + rval->writelock = 0; + spinlock_init(&rval->spin); + if ((rval->entries = (HASHENTRIES **)calloc(size, sizeof(HASHENTRIES *))) == NULL) + { + free(rval); + return NULL; + } + memset(rval->entries, 0, size * sizeof(HASHENTRIES *)); + + return rval; +} + +#else + HASHTABLE * hashtable_alloc(int size, int (*hashfn)(), int (*cmpfn)()) { @@ -117,18 +185,49 @@ HASHTABLE *rval; return rval; } - +#endif /** * Delete an entire hash table * * @param table The hash table to delete */ +#if 1 + void hashtable_free(HASHTABLE *table) { int i; HASHENTRIES *entry, *ptr; + hashtable_write_lock(table); + for (i = 0; i < table->hashsize; i++) + { + entry = table->entries[i]; + while (entry) + { + ptr = entry->next; + table->kfreefn(entry->key); + table->vfreefn(entry->value); + free(entry); + entry = ptr; + } + } + free(table->entries); + + if (!table->ht_isflat) + { + free(table); + } +} + +#else + +void +hashtable_free(HASHTABLE *table) +{ + int i; + HASHENTRIES *entry, *ptr; + hashtable_write_lock(table); for (i = 0; i < table->hashsize; i++) { @@ -146,6 +245,7 @@ HASHENTRIES *entry, *ptr; free(table); } +#endif /** * Provide memory management functions to the hash table. This allows * function pointers to be registered that can make copies of the diff --git a/server/include/hashtable.h b/server/include/hashtable.h index 92bc71b8e..175bd5d32 100644 --- a/server/include/hashtable.h +++ b/server/include/hashtable.h @@ -84,12 +84,17 @@ typedef struct hashtable { SPINLOCK spin; /**< Internal spinlock for the hashtable */ int n_readers; /**< Number of clients reading the table */ int writelock; /**< The table is locked by a writer */ + bool ht_isflat; /**< Indicates whether hashtable is in stack or heap */ #if defined(SS_DEBUG) skygw_chk_t ht_chk_tail; #endif } HASHTABLE; extern HASHTABLE *hashtable_alloc(int, int (*hashfn)(), int (*cmpfn)()); +HASHTABLE *hashtable_alloc_flat(HASHTABLE* target, + int size, + int (*hashfn)(), + int (*cmpfn)()); /**< Allocate a hashtable */ extern void hashtable_memory_fns(HASHTABLE *, HASHMEMORYFN, HASHMEMORYFN, HASHMEMORYFN, HASHMEMORYFN); /**< Provide an interface to control key/value memory diff --git a/server/modules/include/readwritesplit.h b/server/modules/include/readwritesplit.h index 6fc639005..3530dc086 100644 --- a/server/modules/include/readwritesplit.h +++ b/server/modules/include/readwritesplit.h @@ -78,7 +78,8 @@ typedef enum rses_property_type_t { RSES_PROP_TYPE_UNDEFINED=-1, RSES_PROP_TYPE_SESCMD=0, RSES_PROP_TYPE_FIRST = RSES_PROP_TYPE_SESCMD, - RSES_PROP_TYPE_LAST=RSES_PROP_TYPE_SESCMD, + RSES_PROP_TYPE_TMPTABLES, + RSES_PROP_TYPE_LAST=RSES_PROP_TYPE_TMPTABLES, RSES_PROP_TYPE_COUNT=RSES_PROP_TYPE_LAST+1 } rses_property_type_t; @@ -143,6 +144,7 @@ struct rses_property_st { rses_property_type_t rses_prop_type; union rses_prop_data { mysql_sescmd_t sescmd; + HASHTABLE tmp_table_hash; void* placeholder; /*< to be removed due new type */ } rses_prop_data; rses_property_t* rses_prop_next; /*< next property of same type */ diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index 1c22d4979..7fdd5d8ef 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -1263,6 +1263,9 @@ static int routeQuery( ss_dassert(succp); goto return_ret; } + /** + * qtype is QUERY_TYPE_WRITE or QUERY_TYPE_READ_TMP_TABLE + */ else { bool succp = true; @@ -1287,7 +1290,15 @@ static int routeQuery( if (!rses_begin_locked_router_action(router_cli_ses)) { goto return_ret; - } + } +#if defined(TEMPORARY_TABLES) + /** + * If query is of type QUERY_TYPE_CREATE_TMP_TABLE then find out + * the database and table name, create a hashvalue and + * add it to the router client session's property. If property + * doesn't exist then create it first. + */ +#endif if (master_dcb == NULL) { @@ -2682,8 +2693,13 @@ static bool execute_sescmd_in_backend( sescmd_cursor_clone_querybuf(scur)); break; - case MYSQL_COM_QUERY: - case MYSQL_COM_INIT_DB: + case MYSQL_COM_INIT_DB: +#if defined(TEMPORARY_TABLES) + /** + * Record database name and store to session. + */ +#endif + case MYSQL_COM_QUERY: default: /** * Mark session command buffer, it triggers writing @@ -2969,7 +2985,7 @@ static bool route_session_write( rses_property_done(prop); succp = false; goto return_succp; - } + } /** * Additional reference is created to querybuf to * prevent it from being released before properties From ed94f08b4eb5b799c4137467ab9e7b4254693b2f Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Fri, 29 Aug 2014 11:37:41 +0300 Subject: [PATCH 47/60] Added rule version.h: $(ROOT_PATH)/VERSION --- query_classifier/makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/query_classifier/makefile b/query_classifier/makefile index 35ef36d2a..b248fe7f8 100644 --- a/query_classifier/makefile +++ b/query_classifier/makefile @@ -36,6 +36,9 @@ testall: $(MAKE) -C test testall +version.h: $(ROOT_PATH)/VERSION + echo '#define MAXSCALE_VERSION "'`cat $(ROOT_PATH)/VERSION`'"' > $(ROOT_PATH)/include/version.h + utils: $(MAKE) -C $(UTILS_PATH) clean all From 7629c455a63c56e428b581d2e9cb944f502204eb Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Sat, 30 Aug 2014 08:27:05 +0300 Subject: [PATCH 48/60] partial implementation --- query_classifier/query_classifier.cc | 110 +++++++++- query_classifier/query_classifier.h | 2 + server/modules/include/readwritesplit.h | 2 +- .../routing/readwritesplit/readwritesplit.c | 202 ++++++++++++++++-- 4 files changed, 292 insertions(+), 24 deletions(-) diff --git a/query_classifier/query_classifier.cc b/query_classifier/query_classifier.cc index 3f2c18eb4..14fa7ec44 100644 --- a/query_classifier/query_classifier.cc +++ b/query_classifier/query_classifier.cc @@ -736,15 +736,17 @@ static skygw_query_type_t resolve_query_type( } } /**< for */ #if defined(TEMPORARY_TABLES) - if ((skygw_query_type_t)type == QUERY_TYPE_READ) - { - /** - * Find out the database name and all tables the query - * uses. Create a hashvalue from each and if any of the - * values can be found from property's hashtable, set - * query type to QUERY_TYPE_READ_TMP_TABLE. - */ - } + if ((skygw_query_type_t)type == QUERY_TYPE_READ) + { + /** + * Find out the database name and all tables the query + * uses. Create a hashvalue from each and if any of the + * values can be found from property's hashtable, set + * query type to QUERY_TYPE_READ_TMP_TABLE. + */ + + + } #endif } /**< if */ return_qtype: @@ -874,8 +876,8 @@ char* skygw_query_classifier_get_stmtname( /** * Finds the head of the list of tables affected by the current select statement. - * @param thd Pointer to a valid thread descriptor structure - * @return Head of the TABLE_LIST chain or NULL in case of an error + * @param thd Pointer to a valid THD + * @return Pointer to the head of the TABLE_LIST chain or NULL in case of an error */ void* skygw_get_affected_tables(void* thdp) { @@ -893,6 +895,92 @@ void* skygw_get_affected_tables(void* thdp) return (void*)thd->lex->current_select->table_list.first; } + + +/** + * Reads the parsetree and lists all the affected tables and views in the query. + * In the case of an error, the size of the table is set to zero and no memory is allocated. + * The caller must free the allocated memory. + * + * @param querybuf GWBUF where the table names are extracted from + * @param tblsize Pointer where the number of tables is written + * @return Array of null-terminated strings with the table names + */ +char** skygw_get_table_names(GWBUF* querybuf,int* tblsize) +{ + parsing_info_t* pi; + MYSQL* mysql; + THD* thd; + TABLE_LIST* tbl; + SELECT_LEX*slx; + int i = 0, currtblsz = 0; + char**tables,**tmp; + + if (!GWBUF_IS_PARSED(querybuf)) + { + tables = NULL; + goto retblock; + } + pi = (parsing_info_t *)gwbuf_get_buffer_object_data(querybuf, + GWBUF_PARSING_INFO); + + if (pi == NULL) + { + tables = NULL; + goto retblock; + } + + if (pi->pi_query_plain_str == NULL || + (mysql = (MYSQL *)pi->pi_handle) == NULL || + (thd = (THD *)mysql->thd) == NULL) + { + ss_dassert(pi->pi_query_plain_str != NULL && + mysql != NULL && + thd != NULL); + tables = NULL; + goto retblock; + } + + thd->lex->current_select = thd->lex->all_selects_list; + + while(thd->lex->current_select){ + + tbl = (TABLE_LIST*)skygw_get_affected_tables(thd); + + while (tbl) + { + if(i >= currtblsz){ + + tmp = (char**)malloc(sizeof(char*)*(currtblsz*2+1)); + + if(tmp){ + if(currtblsz > 0){ + int x; + for(x = 0;xalias); + tbl=tbl->next_local; + } + thd->lex->current_select = thd->lex->current_select->next_select_in_list(); + } + + retblock: + *tblsize = i; + return tables; +} + + /* * Replace user-provided literals with question marks. Return a copy of the * querystr with replacements. diff --git a/query_classifier/query_classifier.h b/query_classifier/query_classifier.h index 6f5ecd803..595ba502b 100644 --- a/query_classifier/query_classifier.h +++ b/query_classifier/query_classifier.h @@ -73,6 +73,8 @@ skygw_query_type_t query_classifier_get_type(GWBUF* querybuf); /** Free THD context and close MYSQL */ char* skygw_query_classifier_get_stmtname(MYSQL* mysql); +void* skygw_get_affected_tables(void* thdp); +char** skygw_get_table_names(GWBUF* querybuf,int* tblsize); char* skygw_get_canonical(GWBUF* querybuf); bool parse_query (GWBUF* querybuf); parsing_info_t* parsing_info_init(void (*donefun)(void *)); diff --git a/server/modules/include/readwritesplit.h b/server/modules/include/readwritesplit.h index 3530dc086..f8024fde6 100644 --- a/server/modules/include/readwritesplit.h +++ b/server/modules/include/readwritesplit.h @@ -145,7 +145,7 @@ struct rses_property_st { union rses_prop_data { mysql_sescmd_t sescmd; HASHTABLE tmp_table_hash; - void* placeholder; /*< to be removed due new type */ + HASHTABLE* temp_tables; } rses_prop_data; rses_property_t* rses_prop_next; /*< next property of same type */ #if defined(SS_DEBUG) diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index aa118f005..dbc99d801 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -273,6 +273,33 @@ static bool have_enough_servers( static SPINLOCK instlock; static ROUTER_INSTANCE* instances; +static int hashkeyfun(void* key); +static int hashcmpfun (void *, void *); + +static int hashkeyfun( + void* key) +{ + unsigned int hash = 0,c = 0; + char* ptr = (char*)key; + while((c = *ptr++)){ + hash = c + (hash << 6) + (hash << 16) - hash; + } + return *(int *)key; +} + +static int hashcmpfun( + void* v1, + void* v2) +{ + int i1; + int i2; + + i1 = *(int *)v1; + i2 = *(int *)v2; + + return (i1 < i2 ? -1 : (i1 > i2 ? 1 : 0)); +} + /** * Implementation of the mandatory version entry point * @@ -1034,12 +1061,21 @@ static int routeQuery( skygw_query_type_t qtype = QUERY_TYPE_UNKNOWN; mysql_server_cmd_t packet_type; uint8_t* packet; - int ret = 0; + int ret = 0, + tsize = 0, + klen = 0, + i = 0; DCB* master_dcb = NULL; DCB* slave_dcb = NULL; ROUTER_INSTANCE* inst = (ROUTER_INSTANCE *)instance; ROUTER_CLIENT_SES* router_cli_ses = (ROUTER_CLIENT_SES *)router_session; + rses_property_t* rses_prop; bool rses_is_closed = false; + bool target_tmp_table = false; + char *dbname,*hkey; + char** tbl; + HASHTABLE* h; + MYSQL_session* data; CHK_CLIENT_RSES(router_cli_ses); @@ -1053,6 +1089,7 @@ static int routeQuery( packet = GWBUF_DATA(querybuf); packet_type = packet[4]; + rses_prop = *router_cli_ses->rses_properties; if (rses_is_closed) { @@ -1083,6 +1120,9 @@ static int routeQuery( master_dcb = router_cli_ses->rses_master_ref->bref_dcb; CHK_DCB(master_dcb); + + data = (MYSQL_session*)master_dcb->session->data; + dbname = data->db; switch(packet_type) { case MYSQL_COM_QUIT: /*< 1 QUIT will close all sessions */ @@ -1128,6 +1168,51 @@ static int routeQuery( break; } /**< switch by packet type */ + + /** + *Check if the query targets a temporary table + */ + if(QUERY_IS_TYPE(qtype, QUERY_TYPE_READ)){ + + + tbl = skygw_get_table_names(querybuf,&tsize); + + if(tsize > 0) + { /**Query targets at least one table*/ + for(i = 0;irses_prop_data.temp_tables){ + if((target_tmp_table = hashtable_fetch(rses_prop->rses_prop_data.temp_tables,(void *)hkey))) + { + /**Query target is a temporary table*/ + + LOGIF(LT, (skygw_log_write( + LOGFILE_TRACE, + "Query targets a temporary table: %s",hkey))); + } + } + + + + } + + for(i = 0;irses_transaction_active) + !router_cli_ses->rses_transaction_active && + !target_tmp_table) { bool succp; @@ -1263,19 +1349,98 @@ static int routeQuery( { goto return_ret; } -#if defined(TEMPORARY_TABLES) + + //#if defined(TEMPORARY_TABLES) + + if(!query_is_parsed(querybuf) && !parse_query(querybuf)){ + LOGIF(LE, (skygw_log_write( + LOGFILE_ERROR, + "Error: Unable to parse query."))); + } + qtype = query_classifier_get_type(querybuf); + + if(QUERY_IS_TYPE(qtype, QUERY_TYPE_CREATE_TMP_TABLE)){ + + + + bool is_temp = true; + + + if(rses_prop == NULL){ + if((rses_prop = + (rses_property_t*)calloc(1,sizeof(rses_property_t)))){ + rses_prop->rses_prop_rsession = router_cli_ses; + rses_prop->rses_prop_refcount = 1; + rses_prop->rses_prop_next = NULL; + rses_prop->rses_prop_type = RSES_PROP_TYPE_SESCMD; + *router_cli_ses->rses_properties = rses_prop; + } + + } + + if( rses_prop->rses_prop_data.temp_tables == NULL){ + + h = hashtable_alloc(10000, hashkeyfun, hashcmpfun); + + if(h){ + rses_prop->rses_prop_data.temp_tables = h; + } + + } + + tbl = skygw_get_table_names(querybuf,&tsize); + + if(tsize == 1 && tbl[0]) + { /**One valid table created*/ + + klen = strlen(dbname) + strlen(tbl[0]) + 2; + hkey = calloc(klen,sizeof(char)); + strcpy(hkey,dbname); + strcat(hkey,"."); + strcat(hkey,tbl[0]); + + if( + hashtable_add( + rses_prop->rses_prop_data.temp_tables, + (void *)hkey, + (void *)&is_temp + ) + == 0) /**Conflict in hash table*/ + { + LOGIF(LT, (skygw_log_write( + LOGFILE_TRACE, + "Temporary table conflict in hashtable: %s",hkey))); + } + char* retkey = hashtable_fetch( + rses_prop->rses_prop_data.temp_tables, + hkey); + if(retkey){ +LOGIF(LT, (skygw_log_write( + LOGFILE_TRACE, + "Temporary table added: %s",retkey))); + } + } + + if(tsize > 0){ + for(i = 0;isession->data; + tmpbuf = scur->scmd_cur_cmd->my_sescmd_buf; + qlen = MYSQL_GET_PACKET_LEN((unsigned char*)tmpbuf->start); + strncpy(data->db,tmpbuf->start+5,qlen - 1); + + } + //#endif case MYSQL_COM_QUERY: default: /** From 45faa388774175d46d05285d51fd36832c45b553 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Sat, 30 Aug 2014 19:51:59 +0300 Subject: [PATCH 49/60] added temporary table detection for reads --- .../routing/readwritesplit/readwritesplit.c | 163 +++++++++--------- 1 file changed, 85 insertions(+), 78 deletions(-) diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index dbc99d801..05afbf587 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -291,15 +291,26 @@ static int hashcmpfun( void* v1, void* v2) { - int i1; - int i2; + char* i1 = (char*) v1; + char* i2 = (char*) v2; - i1 = *(int *)v1; - i2 = *(int *)v2; - - return (i1 < i2 ? -1 : (i1 > i2 ? 1 : 0)); + return strcmp(i1,i2); } +static void* hstrdup(void* fval) +{ + char* str = (char*)fval; + return strdup(str); +} + + +static void* hfree(void* fval) +{ + free (fval); + return NULL; +} + + /** * Implementation of the mandatory version entry point * @@ -1069,7 +1080,7 @@ static int routeQuery( DCB* slave_dcb = NULL; ROUTER_INSTANCE* inst = (ROUTER_INSTANCE *)instance; ROUTER_CLIENT_SES* router_cli_ses = (ROUTER_CLIENT_SES *)router_session; - rses_property_t* rses_prop; + rses_property_t* rses_prop_tmp; bool rses_is_closed = false; bool target_tmp_table = false; char *dbname,*hkey; @@ -1089,7 +1100,7 @@ static int routeQuery( packet = GWBUF_DATA(querybuf); packet_type = packet[4]; - rses_prop = *router_cli_ses->rses_properties; + rses_prop_tmp = router_cli_ses->rses_properties[RSES_PROP_TYPE_TMPTABLES]; if (rses_is_closed) { @@ -1172,9 +1183,9 @@ static int routeQuery( /** *Check if the query targets a temporary table */ + if(QUERY_IS_TYPE(qtype, QUERY_TYPE_READ)){ - tbl = skygw_get_table_names(querybuf,&tsize); if(tsize > 0) @@ -1185,18 +1196,18 @@ static int routeQuery( strcpy(hkey,dbname); strcat(hkey,"."); strcat(hkey,tbl[0]); - if(rses_prop && rses_prop->rses_prop_data.temp_tables){ - if((target_tmp_table = hashtable_fetch(rses_prop->rses_prop_data.temp_tables,(void *)hkey))) + if(rses_prop_tmp && rses_prop_tmp->rses_prop_data.temp_tables){ + if((target_tmp_table = (bool)hashtable_fetch(rses_prop_tmp->rses_prop_data.temp_tables,(void *)hkey))) { + /**Query target is a temporary table*/ - - LOGIF(LT, (skygw_log_write( - LOGFILE_TRACE, + qtype = QUERY_TYPE_READ_TMP_TABLE; + LOGIF(LT, (skygw_log_write(LOGFILE_TRACE, "Query targets a temporary table: %s",hkey))); } } - + free(hkey); } @@ -1207,10 +1218,6 @@ static int routeQuery( } - - - - } /** @@ -1274,8 +1281,7 @@ static int routeQuery( goto return_ret; } else if (QUERY_IS_TYPE(qtype, QUERY_TYPE_READ) && - !router_cli_ses->rses_transaction_active && - !target_tmp_table) + !router_cli_ses->rses_transaction_active) { bool succp; @@ -1350,40 +1356,42 @@ static int routeQuery( goto return_ret; } - //#if defined(TEMPORARY_TABLES) - - if(!query_is_parsed(querybuf) && !parse_query(querybuf)){ - LOGIF(LE, (skygw_log_write( - LOGFILE_ERROR, - "Error: Unable to parse query."))); - } - qtype = query_classifier_get_type(querybuf); - if(QUERY_IS_TYPE(qtype, QUERY_TYPE_CREATE_TMP_TABLE)){ - - - - bool is_temp = true; - + /** + * If query is of type QUERY_TYPE_CREATE_TMP_TABLE then find out + * the database and table name, create a hashvalue and + * add it to the router client session's property. If property + * doesn't exist then create it first. + */ - if(rses_prop == NULL){ - if((rses_prop = + if(QUERY_IS_TYPE(qtype, QUERY_TYPE_CREATE_TMP_TABLE)){ + + bool is_temp = true; + + if(rses_prop_tmp == NULL){ + if((rses_prop_tmp = (rses_property_t*)calloc(1,sizeof(rses_property_t)))){ - rses_prop->rses_prop_rsession = router_cli_ses; - rses_prop->rses_prop_refcount = 1; - rses_prop->rses_prop_next = NULL; - rses_prop->rses_prop_type = RSES_PROP_TYPE_SESCMD; - *router_cli_ses->rses_properties = rses_prop; + +#if defined(SS_DEBUG) + rses_prop_tmp->rses_prop_chk_top = CHK_NUM_ROUTER_PROPERTY; + rses_prop_tmp->rses_prop_chk_tail = CHK_NUM_ROUTER_PROPERTY; +#endif + + rses_prop_tmp->rses_prop_rsession = router_cli_ses; + rses_prop_tmp->rses_prop_refcount = 1; + rses_prop_tmp->rses_prop_next = NULL; + rses_prop_tmp->rses_prop_type = RSES_PROP_TYPE_TMPTABLES; + router_cli_ses->rses_properties[RSES_PROP_TYPE_TMPTABLES] = rses_prop_tmp; } } - if( rses_prop->rses_prop_data.temp_tables == NULL){ + if( rses_prop_tmp->rses_prop_data.temp_tables == NULL){ - h = hashtable_alloc(10000, hashkeyfun, hashcmpfun); - + h = hashtable_alloc(7, hashkeyfun, hashcmpfun); + hashtable_memory_fns(h,hstrdup,NULL,hfree,NULL); if(h){ - rses_prop->rses_prop_data.temp_tables = h; + rses_prop_tmp->rses_prop_data.temp_tables = h; } } @@ -1393,32 +1401,34 @@ static int routeQuery( if(tsize == 1 && tbl[0]) { /**One valid table created*/ - klen = strlen(dbname) + strlen(tbl[0]) + 2; - hkey = calloc(klen,sizeof(char)); - strcpy(hkey,dbname); - strcat(hkey,"."); - strcat(hkey,tbl[0]); + klen = strlen(dbname) + strlen(tbl[0]) + 2; + hkey = calloc(klen,sizeof(char)); + strcpy(hkey,dbname); + strcat(hkey,"."); + strcat(hkey,tbl[0]); - if( - hashtable_add( - rses_prop->rses_prop_data.temp_tables, - (void *)hkey, - (void *)&is_temp - ) - == 0) /**Conflict in hash table*/ - { + if( + hashtable_add( + rses_prop_tmp->rses_prop_data.temp_tables, + (void *)hkey, + (void *)is_temp + ) + == 0) /**Conflict in hash table*/ + { + LOGIF(LT, (skygw_log_write( + LOGFILE_TRACE, + "Temporary table conflict in hashtable: %s",hkey))); + } +#if defined(SS_DEBUG) + bool retkey = hashtable_fetch( + rses_prop_tmp->rses_prop_data.temp_tables, + hkey); + if(retkey){ LOGIF(LT, (skygw_log_write( LOGFILE_TRACE, - "Temporary table conflict in hashtable: %s",hkey))); + "Temporary table added: %s",hkey))); } - char* retkey = hashtable_fetch( - rses_prop->rses_prop_data.temp_tables, - hkey); - if(retkey){ -LOGIF(LT, (skygw_log_write( - LOGFILE_TRACE, - "Temporary table added: %s",retkey))); - } +#endif } if(tsize > 0){ @@ -1426,16 +1436,9 @@ LOGIF(LT, (skygw_log_write( free(tbl[i]); } free(tbl); + free(hkey); } } - /** - * If query is of type QUERY_TYPE_CREATE_TMP_TABLE then find out - * the database and table name, create a hashvalue and - * add it to the router client session's property. If property - * doesn't exist then create it first. - */ - - //#endif if (master_dcb == NULL) { @@ -2436,6 +2439,9 @@ static void rses_property_done( case RSES_PROP_TYPE_SESCMD: mysql_sescmd_done(&prop->rses_prop_data.sescmd); break; + case RSES_PROP_TYPE_TMPTABLES: + hashtable_free(prop->rses_prop_data.temp_tables); + break; default: LOGIF(LD, (skygw_log_write( LOGFILE_DEBUG, @@ -2836,7 +2842,7 @@ static bool execute_sescmd_in_backend( break; case MYSQL_COM_INIT_DB: - //#if defined(TEMPORARY_TABLES) + /** * Record database name and store to session. */ @@ -2850,10 +2856,11 @@ static bool execute_sescmd_in_backend( data = dcb->session->data; tmpbuf = scur->scmd_cur_cmd->my_sescmd_buf; qlen = MYSQL_GET_PACKET_LEN((unsigned char*)tmpbuf->start); + memset(data->db,0,MYSQL_DATABASE_MAXLEN+1); strncpy(data->db,tmpbuf->start+5,qlen - 1); } - //#endif + case MYSQL_COM_QUERY: default: /** From ecc89a823b917b4a160abb05d8dacdf87eec8078 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Sun, 31 Aug 2014 19:30:00 +0300 Subject: [PATCH 50/60] added tests for temporary tables --- .../routing/readwritesplit/test/rwsplit.sh | 26 ++++++++++++++++--- .../test/test_temporary_table.sql | 5 ++++ 2 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 server/modules/routing/readwritesplit/test/test_temporary_table.sql diff --git a/server/modules/routing/readwritesplit/test/rwsplit.sh b/server/modules/routing/readwritesplit/test/rwsplit.sh index c2e403b16..f55960009 100755 --- a/server/modules/routing/readwritesplit/test/rwsplit.sh +++ b/server/modules/routing/readwritesplit/test/rwsplit.sh @@ -230,6 +230,16 @@ if [ "$a" != "$TRETVAL" ]; then else echo "$TINPUT PASSED">>$TLOG ; fi + +TINPUT=test_temporary_table.sql +a=`$RUNCMD < ./$TINPUT` +TRETVAL=1 +if [ "$a" != "$TRETVAL" ]; then + echo "$TINPUT FAILED, return value $a when $TRETVAL was expected">>$TLOG; +else + echo "$TINPUT PASSED">>$TLOG ; +fi + echo "-----------------------------------" >> $TLOG echo "Session variables: Stress Test 1" >> $TLOG echo "-----------------------------------" >> $TLOG @@ -238,6 +248,10 @@ RUNCMD=mysql\ --host=$THOST\ -P$TPORT\ -u$TUSER\ -p$TPWD\ --unbuffered=true\ --d TINPUT=test_sescmd2.sql for ((i = 0;i<1000;i++)) do + if [[ $(( i % 50 )) -eq 0 ]] + then + printf "." + fi a=`$RUNCMD < $TINPUT 2>&1` if [[ "`echo "$a"|grep -i 'error'`" != "" ]] then @@ -255,15 +269,19 @@ fi echo "-----------------------------------" >> $TLOG echo "Session variables: Stress Test 2" >> $TLOG echo "-----------------------------------" >> $TLOG - +echo "" err="" TINPUT=test_sescmd3.sql for ((j = 0;j<1000;j++)) do - b=`$RUNCMD < $TINPUT 2>&1` - if [[ "`echo "$b"|grep -i 'null'`" != "" ]] + if [[ $(( j % 50 )) -eq 0 ]] then - err=`echo "$b" | grep -i null` + printf "." + fi + b=`$RUNCMD < $TINPUT 2>&1` + if [[ "`echo "$b"|grep -i 'null|error'`" != "" ]] + then + err=`echo "$b" | grep -i null|error` break fi done diff --git a/server/modules/routing/readwritesplit/test/test_temporary_table.sql b/server/modules/routing/readwritesplit/test/test_temporary_table.sql new file mode 100644 index 000000000..374510389 --- /dev/null +++ b/server/modules/routing/readwritesplit/test/test_temporary_table.sql @@ -0,0 +1,5 @@ +use test; +drop table if exists t1; +create temporary table t1 (id integer); +insert into t1 values(1); +select id from t1; From 58e8c05c8a035fb1d73660adef0a83dace4671e2 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Sun, 31 Aug 2014 20:19:47 +0300 Subject: [PATCH 51/60] added detection of drop table targeting a temporary table --- query_classifier/query_classifier.cc | 6 +++ query_classifier/query_classifier.h | 3 +- .../routing/readwritesplit/readwritesplit.c | 50 ++++++++++++------- 3 files changed, 40 insertions(+), 19 deletions(-) diff --git a/query_classifier/query_classifier.cc b/query_classifier/query_classifier.cc index 14fa7ec44..92931e7ab 100644 --- a/query_classifier/query_classifier.cc +++ b/query_classifier/query_classifier.cc @@ -540,7 +540,13 @@ static skygw_query_type_t resolve_query_type( { type |= QUERY_TYPE_CREATE_TMP_TABLE; } + } + + if(lex->sql_command == SQLCOM_DROP_TABLE) + { + type |= QUERY_TYPE_DROP_TABLE; + } goto return_qtype; } diff --git a/query_classifier/query_classifier.h b/query_classifier/query_classifier.h index 595ba502b..86d5033d4 100644 --- a/query_classifier/query_classifier.h +++ b/query_classifier/query_classifier.h @@ -46,7 +46,8 @@ typedef enum { QUERY_TYPE_PREPARE_STMT = 0x0800, /*< Prepared stmt with id provided by server */ QUERY_TYPE_EXEC_STMT = 0x1000, /*< Execute prepared statement */ QUERY_TYPE_CREATE_TMP_TABLE = 0x2000, /*< Create temporary table */ - QUERY_TYPE_READ_TMP_TABLE = 0x4000 /*< Read temporary table */ + QUERY_TYPE_READ_TMP_TABLE = 0x4000, /*< Read temporary table */ + QUERY_TYPE_DROP_TABLE = 0x8000 } skygw_query_type_t; diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index 05afbf587..72fe0c03a 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -1364,6 +1364,31 @@ static int routeQuery( * doesn't exist then create it first. */ + if(QUERY_IS_TYPE(qtype, QUERY_TYPE_CREATE_TMP_TABLE) || + QUERY_IS_TYPE(qtype, QUERY_TYPE_DROP_TABLE)){ + + tbl = skygw_get_table_names(querybuf,&tsize); + + if(tsize == 1 && tbl[0]) + { /**One valid table created*/ + + klen = strlen(dbname) + strlen(tbl[0]) + 2; + hkey = calloc(klen,sizeof(char)); + strcpy(hkey,dbname); + strcat(hkey,"."); + strcat(hkey,tbl[0]); + } + + + if(tsize > 0){ + for(i = 0;irses_prop_data.temp_tables, @@ -1431,15 +1445,15 @@ static int routeQuery( #endif } - if(tsize > 0){ - for(i = 0;irses_prop_data.temp_tables){ + hashtable_delete(rses_prop_tmp->rses_prop_data.temp_tables, (void *)hkey); } } - + + free(hkey); + if (master_dcb == NULL) { succp = get_dcb(&master_dcb, router_cli_ses, BE_MASTER); From 40f85f9cadd5b745e68055d45812462cfef0f4a4 Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Sun, 31 Aug 2014 22:56:30 +0300 Subject: [PATCH 52/60] Fix to bug http://bugs.skysql.com/show_bug.cgi?id=488. SHOW VARIABLES was treated as if it was session write command. Instead it is a read-only query. Changed to route that to master. --- query_classifier/query_classifier.cc | 15 ++++++++++++--- query_classifier/query_classifier.h | 3 ++- .../routing/readwritesplit/readwritesplit.c | 2 +- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/query_classifier/query_classifier.cc b/query_classifier/query_classifier.cc index 7a4bcba40..b249206d0 100644 --- a/query_classifier/query_classifier.cc +++ b/query_classifier/query_classifier.cc @@ -512,7 +512,15 @@ static skygw_query_type_t resolve_query_type( } else if (lex->option_type == OPT_SESSION) { - type |= QUERY_TYPE_SESSION_WRITE; + /** SHOW commands are all reads to one backend */ + if (lex->sql_command == SQLCOM_SHOW_VARIABLES) + { + type |= QUERY_TYPE_SESSION_READ; + } + else + { + type |= QUERY_TYPE_SESSION_WRITE; + } goto return_qtype; } /** @@ -531,10 +539,11 @@ static skygw_query_type_t resolve_query_type( force_data_modify_op_replication) { type |= QUERY_TYPE_SESSION_WRITE; - } else { + } + else + { type |= QUERY_TYPE_WRITE; } - goto return_qtype; } diff --git a/query_classifier/query_classifier.h b/query_classifier/query_classifier.h index 325087910..8284650f4 100644 --- a/query_classifier/query_classifier.h +++ b/query_classifier/query_classifier.h @@ -44,7 +44,8 @@ typedef enum { QUERY_TYPE_COMMIT = 0x0200, /*< COMMIT */ QUERY_TYPE_PREPARE_NAMED_STMT = 0x0400, /*< Prepared stmt with name from user */ QUERY_TYPE_PREPARE_STMT = 0x0800, /*< Prepared stmt with id provided by server */ - QUERY_TYPE_EXEC_STMT = 0x1000 /*< Execute prepared statement */ + QUERY_TYPE_EXEC_STMT = 0x1000, /*< Execute prepared statement */ + QUERY_TYPE_SESSION_READ = 0x2000 /*< Read session data (from master 31.8.14) */ } skygw_query_type_t; diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index 210feaac2..5f9f4715f 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -1451,7 +1451,7 @@ static int routeQuery( if (succp) /*< Have DCB of the target backend */ { - if ((ret = target_dcb->func.write(target_dcb, querybuf)) == 1) + if ((ret = target_dcb->func.write(target_dcb, gwbuf_clone(querybuf))) == 1) { backend_ref_t* bref; From bc939501e9cf567d93b0b40dab3b7897129cb14a Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Mon, 1 Sep 2014 10:11:04 +0300 Subject: [PATCH 53/60] minor bugfix to memory allocations --- .../routing/readwritesplit/readwritesplit.c | 118 +++++++++--------- 1 file changed, 62 insertions(+), 56 deletions(-) diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index 72fe0c03a..ba4f70489 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -1377,83 +1377,89 @@ static int routeQuery( strcpy(hkey,dbname); strcat(hkey,"."); strcat(hkey,tbl[0]); - } + } - if(tsize > 0){ - for(i = 0;irses_prop_chk_top = CHK_NUM_ROUTER_PROPERTY; - rses_prop_tmp->rses_prop_chk_tail = CHK_NUM_ROUTER_PROPERTY; + rses_prop_tmp->rses_prop_chk_top = CHK_NUM_ROUTER_PROPERTY; + rses_prop_tmp->rses_prop_chk_tail = CHK_NUM_ROUTER_PROPERTY; #endif - rses_prop_tmp->rses_prop_rsession = router_cli_ses; - rses_prop_tmp->rses_prop_refcount = 1; - rses_prop_tmp->rses_prop_next = NULL; - rses_prop_tmp->rses_prop_type = RSES_PROP_TYPE_TMPTABLES; - router_cli_ses->rses_properties[RSES_PROP_TYPE_TMPTABLES] = rses_prop_tmp; + rses_prop_tmp->rses_prop_rsession = router_cli_ses; + rses_prop_tmp->rses_prop_refcount = 1; + rses_prop_tmp->rses_prop_next = NULL; + rses_prop_tmp->rses_prop_type = RSES_PROP_TYPE_TMPTABLES; + router_cli_ses->rses_properties[RSES_PROP_TYPE_TMPTABLES] = rses_prop_tmp; + } + } - } + if( rses_prop_tmp->rses_prop_data.temp_tables == NULL){ - if( rses_prop_tmp->rses_prop_data.temp_tables == NULL){ - - h = hashtable_alloc(7, hashkeyfun, hashcmpfun); - hashtable_memory_fns(h,hstrdup,NULL,hfree,NULL); - if(h){ - rses_prop_tmp->rses_prop_data.temp_tables = h; + h = hashtable_alloc(7, hashkeyfun, hashcmpfun); + hashtable_memory_fns(h,hstrdup,NULL,hfree,NULL); + if(h){ + rses_prop_tmp->rses_prop_data.temp_tables = h; + } + } - } - - if( - hashtable_add( - rses_prop_tmp->rses_prop_data.temp_tables, - (void *)hkey, - (void *)is_temp - ) - == 0) /**Conflict in hash table*/ - { - LOGIF(LT, (skygw_log_write( - LOGFILE_TRACE, - "Temporary table conflict in hashtable: %s",hkey))); - } -#if defined(SS_DEBUG) - bool retkey = hashtable_fetch( - rses_prop_tmp->rses_prop_data.temp_tables, - hkey); - if(retkey){ + if( + hashtable_add( + rses_prop_tmp->rses_prop_data.temp_tables, + (void *)hkey, + (void *)is_temp + ) + == 0) /**Conflict in hash table*/ + { LOGIF(LT, (skygw_log_write( LOGFILE_TRACE, - "Temporary table added: %s",hkey))); + "Temporary table conflict in hashtable: %s",hkey))); } + +#if defined(SS_DEBUG) + bool retkey = hashtable_fetch( + rses_prop_tmp->rses_prop_data.temp_tables, + hkey); + if(retkey){ + LOGIF(LT, (skygw_log_write( + LOGFILE_TRACE, + "Temporary table added: %s",hkey))); + } #endif + + } + + /**Check if DROP TABLE... targets a temporary table*/ + if(QUERY_IS_TYPE(qtype, QUERY_TYPE_DROP_TABLE)) + { + if(rses_prop_tmp && rses_prop_tmp->rses_prop_data.temp_tables) + { + hashtable_delete(rses_prop_tmp->rses_prop_data.temp_tables, (void *)hkey); + } + } + + free(hkey); + + if(tsize > 0) + { + for(i = 0;irses_prop_data.temp_tables){ - hashtable_delete(rses_prop_tmp->rses_prop_data.temp_tables, (void *)hkey); - } } - free(hkey); - if (master_dcb == NULL) { succp = get_dcb(&master_dcb, router_cli_ses, BE_MASTER); From 067ce3c8866003f1889a717aa6e6d15e51c7119b Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Mon, 1 Sep 2014 10:35:38 +0300 Subject: [PATCH 54/60] removed unneeded QUERY_TYPE_DROP_TABLE type from query_classifier.h --- query_classifier/query_classifier.cc | 17 ----------------- query_classifier/query_classifier.h | 1 - .../routing/readwritesplit/readwritesplit.c | 4 ++-- 3 files changed, 2 insertions(+), 20 deletions(-) diff --git a/query_classifier/query_classifier.cc b/query_classifier/query_classifier.cc index 92931e7ab..1e8916208 100644 --- a/query_classifier/query_classifier.cc +++ b/query_classifier/query_classifier.cc @@ -542,11 +542,6 @@ static skygw_query_type_t resolve_query_type( } } - - if(lex->sql_command == SQLCOM_DROP_TABLE) - { - type |= QUERY_TYPE_DROP_TABLE; - } goto return_qtype; } @@ -741,19 +736,7 @@ static skygw_query_type_t resolve_query_type( break; } } /**< for */ -#if defined(TEMPORARY_TABLES) - if ((skygw_query_type_t)type == QUERY_TYPE_READ) - { - /** - * Find out the database name and all tables the query - * uses. Create a hashvalue from each and if any of the - * values can be found from property's hashtable, set - * query type to QUERY_TYPE_READ_TMP_TABLE. - */ - - } -#endif } /**< if */ return_qtype: qtype = (skygw_query_type_t)type; diff --git a/query_classifier/query_classifier.h b/query_classifier/query_classifier.h index 86d5033d4..a0dccf988 100644 --- a/query_classifier/query_classifier.h +++ b/query_classifier/query_classifier.h @@ -47,7 +47,6 @@ typedef enum { QUERY_TYPE_EXEC_STMT = 0x1000, /*< Execute prepared statement */ QUERY_TYPE_CREATE_TMP_TABLE = 0x2000, /*< Create temporary table */ QUERY_TYPE_READ_TMP_TABLE = 0x4000, /*< Read temporary table */ - QUERY_TYPE_DROP_TABLE = 0x8000 } skygw_query_type_t; diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index ba4f70489..10d738d6c 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -1365,7 +1365,7 @@ static int routeQuery( */ if(QUERY_IS_TYPE(qtype, QUERY_TYPE_CREATE_TMP_TABLE) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_DROP_TABLE)){ + packet_type == MYSQL_COM_DROP_DB){ tbl = skygw_get_table_names(querybuf,&tsize); @@ -1439,7 +1439,7 @@ static int routeQuery( } /**Check if DROP TABLE... targets a temporary table*/ - if(QUERY_IS_TYPE(qtype, QUERY_TYPE_DROP_TABLE)) + if(packet_type == MYSQL_COM_DROP_DB) { if(rses_prop_tmp && rses_prop_tmp->rses_prop_data.temp_tables) { From 20abbbdf572faf085340fae5c3d33f1e99fa609e Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Mon, 1 Sep 2014 13:23:04 +0300 Subject: [PATCH 55/60] query_classifier.cc:skygw_get_canonical: Fixed bug in how strings were passed to replace_literal function. Changed to use item->str_ptr which stores all strings in the same way unlike item->name in which some string values have double quotation. skygw_utils.cc:Fixed regexp so that it detects literals at the end of line too. --- query_classifier/query_classifier.cc | 26 ++++++++++++++++++-------- utils/skygw_utils.cc | 2 +- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/query_classifier/query_classifier.cc b/query_classifier/query_classifier.cc index b249206d0..3d1513b48 100644 --- a/query_classifier/query_classifier.cc +++ b/query_classifier/query_classifier.cc @@ -892,7 +892,8 @@ char* skygw_get_canonical( } pi = (parsing_info_t *)gwbuf_get_buffer_object_data(querybuf, GWBUF_PARSING_INFO); - + CHK_PARSING_INFO(pi); + if (pi == NULL) { querystr = NULL; @@ -911,10 +912,9 @@ char* skygw_get_canonical( querystr = NULL; goto retblock; } - querystr = strdup(pi->pi_query_plain_str); - - for (item=thd->free_list; item != NULL; item=item->next) + + for (item=thd->free_list; item != NULL; item=item->next) { Item::Type itype; @@ -928,10 +928,20 @@ char* skygw_get_canonical( itype == Item::VARBIN_ITEM || itype == Item::NULL_ITEM)) { - if (itype == Item::STRING_ITEM && strlen(item->name) == 0) - { - querystr = replace_literal(querystr, "\"\"", "\"?\""); - } + if (itype == Item::STRING_ITEM) + { + String tokenstr; + String* res = item->val_str_ascii(&tokenstr); + + if (res->is_empty()) /*< empty string */ + { + querystr = replace_literal(querystr, "\"\"", "\"?\""); + } + else + { + querystr = replace_literal(querystr, res->ptr(), "?"); + } + } else { querystr = replace_literal(querystr, item->name, "?"); diff --git a/utils/skygw_utils.cc b/utils/skygw_utils.cc index 8297ca5ba..d1209b7ca 100644 --- a/utils/skygw_utils.cc +++ b/utils/skygw_utils.cc @@ -1883,7 +1883,7 @@ char* replace_literal( const char* replacement) { const char* prefix = "[ ='\",\\(]"; /*< ' ','=','(',''',''"',',' are allowed before needle */ - const char* suffix = "[^[:alnum:]]"; /*< alpha-num chars aren't allowed after the needle */ + const char* suffix = "([^[:alnum:]]|$)"; /*< alpha-num chars aren't allowed after the needle */ char* search_re; char* newstr; regex_t re; From 164d8b1e32fed38031eb9ffd85d45b70425c15e0 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Mon, 1 Sep 2014 13:40:52 +0300 Subject: [PATCH 56/60] Fixed various memory leaks dbuser.c: key.user value was never freed skygw_utils.cc: replace_literal values were not always freed --- server/core/dbusers.c | 2 +- utils/skygw_utils.cc | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/server/core/dbusers.c b/server/core/dbusers.c index dd36d683c..2e4dcb18b 100644 --- a/server/core/dbusers.c +++ b/server/core/dbusers.c @@ -388,7 +388,7 @@ getUsers(SERVICE *service, struct users *users) memcpy(users->cksum, hash, SHA_DIGEST_LENGTH); free(users_data); - + free(key.user); mysql_free_result(result); mysql_close(con); mysql_thread_end(); diff --git a/utils/skygw_utils.cc b/utils/skygw_utils.cc index 8297ca5ba..dab614f25 100644 --- a/utils/skygw_utils.cc +++ b/utils/skygw_utils.cc @@ -1912,6 +1912,7 @@ char* replace_literal( fprintf(stderr, "Regex memory allocation failed : %s\n", strerror(errno)); free(search_re); + free(newstr); newstr = haystack; goto retblock; } @@ -1928,6 +1929,7 @@ char* replace_literal( search_re, error_message); free(search_re); + free(newstr); newstr = haystack; goto retblock; } @@ -1936,6 +1938,7 @@ char* replace_literal( if (rc != 0) { free(search_re); + free(newstr); regfree(&re); newstr = haystack; goto retblock; @@ -1947,6 +1950,7 @@ char* replace_literal( regfree(&re); free(haystack); + free(search_re); retblock: return newstr; } From 3c1abf4b649d907eb27f52c93bc61515d7d4b9c1 Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Mon, 1 Sep 2014 13:57:31 +0300 Subject: [PATCH 57/60] Cleanup --- query_classifier/query_classifier.cc | 53 ++++++++++++++-------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/query_classifier/query_classifier.cc b/query_classifier/query_classifier.cc index 3d1513b48..54e4cdaf0 100644 --- a/query_classifier/query_classifier.cc +++ b/query_classifier/query_classifier.cc @@ -918,35 +918,34 @@ char* skygw_get_canonical( { Item::Type itype; - itype = item->type(); - - if (item->name != NULL && - (itype == Item::STRING_ITEM || - itype == Item::INT_ITEM || - itype == Item::DECIMAL_ITEM || - itype == Item::REAL_ITEM || - itype == Item::VARBIN_ITEM || - itype == Item::NULL_ITEM)) - { - if (itype == Item::STRING_ITEM) - { - String tokenstr; - String* res = item->val_str_ascii(&tokenstr); + if (item->name == NULL) + { + continue; + } + itype = item->type(); - if (res->is_empty()) /*< empty string */ - { - querystr = replace_literal(querystr, "\"\"", "\"?\""); - } - else - { - querystr = replace_literal(querystr, res->ptr(), "?"); - } + if (itype == Item::STRING_ITEM) + { + String tokenstr; + String* res = item->val_str_ascii(&tokenstr); + + if (res->is_empty()) /*< empty string */ + { + querystr = replace_literal(querystr, "\"\"", "\"?\""); } - else - { - querystr = replace_literal(querystr, item->name, "?"); - } - } + else + { + querystr = replace_literal(querystr, res->ptr(), "?"); + } + } + else if (itype == Item::INT_ITEM || + itype == Item::DECIMAL_ITEM || + itype == Item::REAL_ITEM || + itype == Item::VARBIN_ITEM || + itype == Item::NULL_ITEM) + { + querystr = replace_literal(querystr, item->name, "?"); + } } /*< for */ retblock: return querystr; From 0fed5c2c5b24cdcf44e37b3e2ca5203bd5f19d0a Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Mon, 1 Sep 2014 19:37:31 +0300 Subject: [PATCH 58/60] Fix to bug #510, http://bugs.skysql.com/show_bug.cgi?id=510, made every MaxScale thread to call mysql_thread_init() before entering poll_waitevents. Also main thread does this before starting services. Removed all calls to mysql_thread_init() and to mysql_thread_end() from elsewhere than from poll.c:poll_waitevents and from gateway.c:main skygw_utils.cc: replace_literal: fixed memory leak --- query_classifier/query_classifier.cc | 3 +-- server/core/dbusers.c | 9 --------- server/core/gateway.c | 17 +++++++++++++---- server/core/poll.c | 8 ++++++-- utils/skygw_utils.cc | 1 + 5 files changed, 21 insertions(+), 17 deletions(-) diff --git a/query_classifier/query_classifier.cc b/query_classifier/query_classifier.cc index 54e4cdaf0..a3d169a00 100644 --- a/query_classifier/query_classifier.cc +++ b/query_classifier/query_classifier.cc @@ -979,7 +979,7 @@ parsing_info_t* parsing_info_init( LOGFILE_ERROR, "Error : call to mysql_real_connect failed due %d, %s.", mysql_errno(mysql), - mysql_error(mysql)))); + mysql_error(mysql)))); goto retblock; } @@ -996,7 +996,6 @@ parsing_info_t* parsing_info_init( if (pi == NULL) { mysql_close(mysql); - mysql_thread_end(); goto retblock; } #if defined(SS_DEBUG) diff --git a/server/core/dbusers.c b/server/core/dbusers.c index dd36d683c..145f30caa 100644 --- a/server/core/dbusers.c +++ b/server/core/dbusers.c @@ -187,14 +187,6 @@ getUsers(SERVICE *service, struct users *users) if (service_user == NULL || service_passwd == NULL) return -1; - /** multi-thread environment requires that thread init succeeds. */ - if (mysql_thread_init()) { - LOGIF(LE, (skygw_log_write_flush( - LOGFILE_ERROR, - "Error : mysql_thread_init failed."))); - return -1; - } - con = mysql_init(NULL); if (con == NULL) { @@ -391,7 +383,6 @@ getUsers(SERVICE *service, struct users *users) mysql_free_result(result); mysql_close(con); - mysql_thread_end(); return total_users; } diff --git a/server/core/gateway.c b/server/core/gateway.c index fda19fff8..7c3fe96aa 100644 --- a/server/core/gateway.c +++ b/server/core/gateway.c @@ -1339,11 +1339,16 @@ int main(int argc, char **argv) /* Init MaxScale poll system */ poll_init(); - /*< - * Start the services that were created above - */ + /** + * Init mysql thread context for main thread as well. Needed when users + * are queried from backends. + */ + mysql_thread_init(); + + /** Start the services that were created above */ n_services = serviceStartAll(); - if (n_services == 0) + + if (n_services == 0) { char* logerr = "Failed to start any MaxScale services. Exiting."; print_log_n_stderr(true, !daemon_mode, logerr, logerr, 0); @@ -1396,9 +1401,13 @@ int main(int argc, char **argv) /*< Stop all the monitors */ monitorStopAll(); + LOGIF(LM, (skygw_log_write( LOGFILE_MESSAGE, "MaxScale is shutting down."))); + /** Release mysql thread context*/ + mysql_thread_end(); + datadir_cleanup(); LOGIF(LM, (skygw_log_write( LOGFILE_MESSAGE, diff --git a/server/core/poll.c b/server/core/poll.c index 87d3640f0..5e37c1cf5 100644 --- a/server/core/poll.c +++ b/server/core/poll.c @@ -252,9 +252,11 @@ poll_waitevents(void *arg) static bool process_zombies_only = false; /*< flag for all threads */ DCB *zombies = NULL; - /* Add this thread to the bitmask of running polling threads */ + /** Add this thread to the bitmask of running polling threads */ bitmask_set(&poll_mask, thread_id); - + /** Init mysql thread context for use with a mysql handle and a parser */ + mysql_thread_init(); + while (1) { #if BLOCKINGPOLL @@ -495,6 +497,8 @@ poll_waitevents(void *arg) return; } } /*< while(1) */ + /** Release mysql thread context */ + mysql_thread_end(); } /** diff --git a/utils/skygw_utils.cc b/utils/skygw_utils.cc index d1209b7ca..d8b768199 100644 --- a/utils/skygw_utils.cc +++ b/utils/skygw_utils.cc @@ -1947,6 +1947,7 @@ char* replace_literal( regfree(&re); free(haystack); + free(search_re); retblock: return newstr; } From 52f3adbf20780978dd3e6da1c0c7782e830bebbf Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Mon, 1 Sep 2014 19:50:25 +0300 Subject: [PATCH 59/60] fixed temporary tables looking for database drops instead of table drops --- query_classifier/query_classifier.cc | 76 ++++++++++++++- query_classifier/query_classifier.h | 2 + .../routing/readwritesplit/readwritesplit.c | 96 ++++++++++--------- 3 files changed, 127 insertions(+), 47 deletions(-) diff --git a/query_classifier/query_classifier.cc b/query_classifier/query_classifier.cc index d722c2b16..4e3befaa3 100644 --- a/query_classifier/query_classifier.cc +++ b/query_classifier/query_classifier.cc @@ -544,7 +544,8 @@ static skygw_query_type_t resolve_query_type( { type |= QUERY_TYPE_WRITE; - if (lex->create_info.options & HA_LEX_CREATE_TMP_TABLE) + if (lex->create_info.options & HA_LEX_CREATE_TMP_TABLE && + lex->sql_command == SQLCOM_CREATE_TABLE) { type |= QUERY_TYPE_CREATE_TMP_TABLE; } @@ -870,7 +871,6 @@ char* skygw_query_classifier_get_stmtname( } /** - * Finds the head of the list of tables affected by the current select statement. * @param thd Pointer to a valid THD * @return Pointer to the head of the TABLE_LIST chain or NULL in case of an error @@ -908,7 +908,6 @@ char** skygw_get_table_names(GWBUF* querybuf,int* tblsize) MYSQL* mysql; THD* thd; TABLE_LIST* tbl; - SELECT_LEX*slx; int i = 0, currtblsz = 0; char**tables,**tmp; @@ -975,7 +974,78 @@ char** skygw_get_table_names(GWBUF* querybuf,int* tblsize) *tblsize = i; return tables; } +/** + * Extract the name of the created table. + * @param querybuf Buffer to use. + * @return A pointer to the name if a table was created, otherwise NULL + */ +char* skygw_get_created_table_name(GWBUF* querybuf) +{ + parsing_info_t* pi; + MYSQL* mysql; + THD* thd; + if (!GWBUF_IS_PARSED(querybuf)) + { + return NULL; + } + pi = (parsing_info_t *)gwbuf_get_buffer_object_data(querybuf, + GWBUF_PARSING_INFO); + + if (pi == NULL) + { + return NULL; + } + + if ((mysql = (MYSQL *)pi->pi_handle) == NULL || + (thd = (THD *)mysql->thd) == NULL) + { + ss_dassert(mysql != NULL && + thd != NULL); + return NULL; + } + + if(thd->lex->create_last_non_select_table && + thd->lex->create_last_non_select_table->table_name){ + char* name = strdup(thd->lex->create_last_non_select_table->table_name); + return name; + }else{ + return NULL; + } + +} +/** + * Checks whether the buffer contains a DROP TABLE... query. + * @param querybuf Buffer to inspect + * @return true if it contains the query otherwise false + */ +bool is_drop_table_query(GWBUF* querybuf) +{ + parsing_info_t* pi; + MYSQL* mysql; + THD* thd; + + if (!GWBUF_IS_PARSED(querybuf)) + { + return false; + } + pi = (parsing_info_t *)gwbuf_get_buffer_object_data(querybuf, + GWBUF_PARSING_INFO); + if (pi == NULL) + { + return false; + } + + if ((mysql = (MYSQL *)pi->pi_handle) == NULL || + (thd = (THD *)mysql->thd) == NULL) + { + ss_dassert(mysql != NULL && + thd != NULL); + return false; + } + + return thd->lex->sql_command == SQLCOM_DROP_TABLE; +} /* * Replace user-provided literals with question marks. Return a copy of the diff --git a/query_classifier/query_classifier.h b/query_classifier/query_classifier.h index 10098ff14..ccf08a6ea 100644 --- a/query_classifier/query_classifier.h +++ b/query_classifier/query_classifier.h @@ -74,6 +74,8 @@ skygw_query_type_t query_classifier_get_type(GWBUF* querybuf); /** Free THD context and close MYSQL */ char* skygw_query_classifier_get_stmtname(MYSQL* mysql); +char* skygw_get_created_table_name(GWBUF* querybuf); +bool is_drop_table_query(GWBUF* querybuf); void* skygw_get_affected_tables(void* thdp); char** skygw_get_table_names(GWBUF* querybuf,int* tblsize); char* skygw_get_canonical(GWBUF* querybuf); diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index aca14218d..93fd3fd36 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -287,6 +287,9 @@ static int hashcmpfun (void *, void *); static int hashkeyfun( void* key) { + if(key == NULL){ + return 0; + } unsigned int hash = 0,c = 0; char* ptr = (char*)key; while((c = *ptr++)){ @@ -1246,7 +1249,6 @@ static int routeQuery( HASHTABLE* h; MYSQL_session* data; size_t len; - MYSQL* mysql = NULL; route_target_t route_target; @@ -1541,40 +1543,35 @@ static int routeQuery( } target_dcb = master_dcb; } - /** Lock router session */ - /*if (!rses_begin_locked_router_action(router_cli_ses)) - { - goto return_ret; - } */ + /** * If query is of type QUERY_TYPE_CREATE_TMP_TABLE then find out * the database and table name, create a hashvalue and * add it to the router client session's property. If property - * doesn't exist then create it first. + * doesn't exist then create it first. If the query is DROP TABLE... + * then see if it targets a temporary table and remove it from the hashtable + * if it does. */ - - if(QUERY_IS_TYPE(qtype, QUERY_TYPE_CREATE_TMP_TABLE) || - packet_type == MYSQL_COM_DROP_DB){ - - tbl = skygw_get_table_names(querybuf,&tsize); - - if(tsize == 1 && tbl[0]) - { /**One valid table created*/ - - klen = strlen(dbname) + strlen(tbl[0]) + 2; + + if(QUERY_IS_TYPE(qtype, QUERY_TYPE_CREATE_TMP_TABLE)){ + + bool is_temp = true; + char* tblname = NULL; + + tblname = skygw_get_created_table_name(querybuf); + if(tblname && strlen(tblname) > 0){ + klen = strlen(dbname) + strlen(tblname) + 2; hkey = calloc(klen,sizeof(char)); strcpy(hkey,dbname); strcat(hkey,"."); - strcat(hkey,tbl[0]); + strcat(hkey,tblname); + }else{ + hkey = NULL; + } - } - if(QUERY_IS_TYPE(qtype, QUERY_TYPE_CREATE_TMP_TABLE)){ - - bool is_temp = true; - if(rses_prop_tmp == NULL){ if((rses_prop_tmp = (rses_property_t*)calloc(1,sizeof(rses_property_t)))){ @@ -1603,7 +1600,7 @@ static int routeQuery( } - if( + if( hkey && hashtable_add( rses_prop_tmp->rses_prop_data.temp_tables, (void *)hkey, @@ -1616,40 +1613,51 @@ static int routeQuery( "Temporary table conflict in hashtable: %s",hkey))); } + #if defined(SS_DEBUG) - bool retkey = hashtable_fetch( - rses_prop_tmp->rses_prop_data.temp_tables, + bool retkey = hashtable_fetch(rses_prop_tmp->rses_prop_data.temp_tables, hkey); - if(retkey){ - LOGIF(LT, (skygw_log_write( - LOGFILE_TRACE, - "Temporary table added: %s",hkey))); - } + if(retkey) + { + LOGIF(LT, (skygw_log_write(LOGFILE_TRACE, + "Temporary table added: %s",hkey))); + } #endif - + free(hkey); } /**Check if DROP TABLE... targets a temporary table*/ - if(packet_type == MYSQL_COM_DROP_DB) + if(is_drop_table_query(querybuf)) { - if(rses_prop_tmp && rses_prop_tmp->rses_prop_data.temp_tables) - { - hashtable_delete(rses_prop_tmp->rses_prop_data.temp_tables, (void *)hkey); - } - } - - free(hkey); - if(tsize > 0) - { + tbl = skygw_get_table_names(querybuf,&tsize); + for(i = 0;irses_prop_data.temp_tables) + { + if(hashtable_delete(rses_prop_tmp->rses_prop_data.temp_tables, (void *)hkey)) + { + LOGIF(LT, (skygw_log_write(LOGFILE_TRACE, + "Temporary table dropped: %s",hkey))); + } + } + free(tbl[i]); + free(hkey); } free(tbl); - } - } + } + + if (master_dcb == NULL) { From cda39a62fbf0fb11d89a95ab7f0f79b2e4952a5a Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Wed, 3 Sep 2014 22:09:50 +0300 Subject: [PATCH 60/60] Fixes to Includes imprvements to hints processing. If hint can't be followed query is routed possibly to slave, and eventually to master if other attempts fail. --- Makefile | 1 + query_classifier/makefile | 4 - query_classifier/query_classifier.cc | 25 +- server/core/dcb.c | 2 - .../routing/readwritesplit/readwritesplit.c | 474 ++++++++++-------- 5 files changed, 285 insertions(+), 221 deletions(-) diff --git a/Makefile b/Makefile index 328fa9b7a..b8a085468 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,7 @@ clean: (cd client; touch depend.mk; make clean) depend: + echo '#define MAXSCALE_VERSION "'`cat $(ROOT_PATH)/VERSION`'"' > $(ROOT_PATH)/server/include/version.h (cd log_manager; make depend) (cd query_classifier; make depend) (cd server; make depend) diff --git a/query_classifier/makefile b/query_classifier/makefile index b248fe7f8..4f8cf34c8 100644 --- a/query_classifier/makefile +++ b/query_classifier/makefile @@ -34,10 +34,6 @@ runtests: testall: $(MAKE) -C test testall - - -version.h: $(ROOT_PATH)/VERSION - echo '#define MAXSCALE_VERSION "'`cat $(ROOT_PATH)/VERSION`'"' > $(ROOT_PATH)/include/version.h utils: $(MAKE) -C $(UTILS_PATH) clean all diff --git a/query_classifier/query_classifier.cc b/query_classifier/query_classifier.cc index a3d169a00..477420e1a 100644 --- a/query_classifier/query_classifier.cc +++ b/query_classifier/query_classifier.cc @@ -693,7 +693,6 @@ static skygw_query_type_t resolve_query_type( pthread_self()))); break; case Item_func::NOW_FUNC: - case Item_func::GSYSVAR_FUNC: func_qtype |= QUERY_TYPE_LOCAL_READ; LOGIF(LD, (skygw_log_write( LOGFILE_DEBUG, @@ -702,8 +701,30 @@ static skygw_query_type_t resolve_query_type( "executed in MaxScale.", pthread_self()))); break; + /** System session variable */ + case Item_func::GSYSVAR_FUNC: + /** User-defined variable read */ + case Item_func::GUSERVAR_FUNC: + /** User-defined variable modification */ + case Item_func::SUSERVAR_FUNC: + func_qtype |= QUERY_TYPE_SESSION_READ; + LOGIF(LD, (skygw_log_write( + LOGFILE_DEBUG, + "%lu [resolve_query_type] " + "functype SUSERVAR_FUNC, could be " + "executed in MaxScale.", + pthread_self()))); + break; case Item_func::UNKNOWN_FUNC: - func_qtype |= QUERY_TYPE_READ; + if (item->name != NULL && + strcmp(item->name, "last_insert_id()") == 0) + { + func_qtype |= QUERY_TYPE_SESSION_READ; + } + else + { + func_qtype |= QUERY_TYPE_READ; + } /** * Many built-in functions are of this * type, for example, rand(), soundex(), diff --git a/server/core/dcb.c b/server/core/dcb.c index 382940626..577a89210 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -815,8 +815,6 @@ int below_water; spinlock_acquire(&dcb->writeqlock); - ss_dassert(dcb->state != DCB_STATE_ZOMBIE); - if (dcb->writeq != NULL) { /* diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index 5f9f4715f..cfdc121eb 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -957,76 +957,74 @@ static bool get_dcb( /** get root master from available servers */ master_host = get_root_master(backend_ref, rses->rses_nbackends); + if (name != NULL) /*< Choose backend by name from a hint */ + { + ss_dassert(btype != BE_MASTER); /*< Master dominates and no name should be passed with it */ + + for (i=0; irses_nbackends; i++) + { + BACKEND* b = backend_ref[i].bref_backend; + + /** + * To become chosen: + * backend must be in use, name must match, + * root master node must be found, + * backend's role must be either slave, relay + * server, or master. + */ + if (BREF_IS_IN_USE((&backend_ref[i])) && + (strncasecmp( + name, + b->backend_server->unique_name, + PATH_MAX) == 0) && + master_host != NULL && + (SERVER_IS_SLAVE(b->backend_server) || + SERVER_IS_RELAY_SERVER(b->backend_server) || + SERVER_IS_MASTER(b->backend_server))) + { + *p_dcb = backend_ref[i].bref_dcb; + succp = true; + ss_dassert(backend_ref[i].bref_dcb->state != DCB_STATE_ZOMBIE); + break; + } + } + if (succp) + { + goto return_succp; + } + } + if (btype == BE_SLAVE) { - if (name != NULL) /*< Choose backend by name (hint) */ - { - for (i=0; irses_nbackends; i++) - { - BACKEND* b = backend_ref[i].bref_backend; - - /** - * To become chosen: - * backend must be in use, name must match, - * root master node must be found, - * backend's role must be either slave, relay - * server, or master. - */ - if (BREF_IS_IN_USE((&backend_ref[i])) && - (strncasecmp( - name, - b->backend_server->unique_name, - MIN(strlen(b->backend_server->unique_name), PATH_MAX)) == 0) && - master_host != NULL && -#if 0 - (max_rlag == MAX_RLAG_UNDEFINED || - (b->backend_server->rlag != MAX_RLAG_NOT_AVAILABLE && - b->backend_server->rlag <= max_rlag)) && -#endif - (SERVER_IS_SLAVE(b->backend_server) || - SERVER_IS_RELAY_SERVER(b->backend_server) || - SERVER_IS_MASTER(b->backend_server))) - { - *p_dcb = backend_ref[i].bref_dcb; - succp = true; - ss_dassert(backend_ref[i].bref_dcb->state != DCB_STATE_ZOMBIE); - break; - } - } - } - - if (!succp) /*< No hints or finding named backend failed */ - { - for (i=0; irses_nbackends; i++) - { - BACKEND* b = backend_ref[i].bref_backend; - /** - * To become chosen: - * backend must be in use, - * root master node must be found, - * backend is not allowed to be the master, - * backend's role can be either slave or relay - * server and it must have least connections - * at the moment. - */ - if (BREF_IS_IN_USE((&backend_ref[i])) && - master_host != NULL && - b->backend_server != master_host->backend_server && - (max_rlag == MAX_RLAG_UNDEFINED || - (b->backend_server->rlag != MAX_RLAG_NOT_AVAILABLE && - b->backend_server->rlag <= max_rlag)) && - (SERVER_IS_SLAVE(b->backend_server) || - SERVER_IS_RELAY_SERVER(b->backend_server)) && - (smallest_nconn == -1 || - b->backend_conn_count < smallest_nconn)) - { - *p_dcb = backend_ref[i].bref_dcb; - smallest_nconn = b->backend_conn_count; - succp = true; - ss_dassert(backend_ref[i].bref_dcb->state != DCB_STATE_ZOMBIE); - } - } - } + for (i=0; irses_nbackends; i++) + { + BACKEND* b = backend_ref[i].bref_backend; + /** + * To become chosen: + * backend must be in use, + * root master node must be found, + * backend is not allowed to be the master, + * backend's role can be either slave or relay + * server and it must have least connections + * at the moment. + */ + if (BREF_IS_IN_USE((&backend_ref[i])) && + master_host != NULL && + b->backend_server != master_host->backend_server && + (max_rlag == MAX_RLAG_UNDEFINED || + (b->backend_server->rlag != MAX_RLAG_NOT_AVAILABLE && + b->backend_server->rlag <= max_rlag)) && + (SERVER_IS_SLAVE(b->backend_server) || + SERVER_IS_RELAY_SERVER(b->backend_server)) && + (smallest_nconn == -1 || + b->backend_conn_count < smallest_nconn)) + { + *p_dcb = backend_ref[i].bref_dcb; + smallest_nconn = b->backend_conn_count; + succp = true; + ss_dassert(backend_ref[i].bref_dcb->state != DCB_STATE_ZOMBIE); + } + } if (!succp) /*< No valid slave was found, search master next */ { @@ -1066,7 +1064,7 @@ return_succp: * Examine the query type, transaction state and routing hints. Find out the * target for query routing. * - * @param qtype Type of query + * @param qtype Type of query * @param trx_active Is transacation active or not * @param hint Pointer to list of hints attached to the query buffer * @@ -1087,10 +1085,22 @@ static route_target_t get_route_target ( /** hints don't affect on routing */ target = TARGET_ALL; } - else if (QUERY_IS_TYPE(qtype, QUERY_TYPE_READ) && !trx_active) + /** + * Read-only statements to slave or to master can be re-routed after + * the hints + */ + else if ((QUERY_IS_TYPE(qtype, QUERY_TYPE_READ) || + QUERY_IS_TYPE(qtype, QUERY_TYPE_SESSION_READ)) && + !trx_active) { - target = TARGET_SLAVE; - + if (QUERY_IS_TYPE(qtype, QUERY_TYPE_READ)) + { + target = TARGET_SLAVE; + } + else + { + target = TARGET_MASTER; + } /** process routing hints */ while (hint != NULL) { @@ -1104,8 +1114,15 @@ static route_target_t get_route_target ( } else if (hint->type == HINT_ROUTE_TO_NAMED_SERVER) { - target |= TARGET_NAMED_SERVER; /*< add */ - } + /** + * Searching for a named server. If it can't be + * found, the oroginal target is chosen. + */ + target |= TARGET_NAMED_SERVER; + LOGIF(LT, (skygw_log_write( + LOGFILE_TRACE, + "Hint: route to named server : "))); + } else if (hint->type == HINT_ROUTE_TO_UPTODATE_SERVER) { /** not implemented */ @@ -1141,6 +1158,7 @@ static route_target_t get_route_target ( } else if (hint->type == HINT_ROUTE_TO_SLAVE) { + target = TARGET_SLAVE; LOGIF(LT, (skygw_log_write( LOGFILE_TRACE, "Hint: route to slave."))); @@ -1199,8 +1217,10 @@ static int routeQuery( ROUTER_CLIENT_SES* router_cli_ses = (ROUTER_CLIENT_SES *)router_session; bool rses_is_closed = false; size_t len; - MYSQL* mysql = NULL; route_target_t route_target; + bool succp = false; + int rlag_max = MAX_RLAG_UNDEFINED; + backend_type_t btype; /*< target backend type */ CHK_CLIENT_RSES(router_cli_ses); @@ -1238,7 +1258,7 @@ static int routeQuery( "route to")))); free(querybuf); } - goto return_ret; + goto retblock; } inst->stats.n_queries++; @@ -1332,148 +1352,176 @@ static int routeQuery( * If query would otherwise be routed to slave then the hint determines * actual target server if it exists. * - * route_target is a bitfield and may include multiple values. + * route_target is a bitfield and may include : + * TARGET_ALL + * - route to all connected backend servers + * TARGET_SLAVE[|TARGET_NAMED_SERVER|TARGET_RLAG_MAX] + * - route primarily according to hints, then to slave and if those + * failed, eventually to master + * TARGET_MASTER[|TARGET_NAMED_SERVER|TARGET_RLAG_MAX] + * - route primarily according to the hints and if they failed, + * eventually to master */ route_target = get_route_target(qtype, router_cli_ses->rses_transaction_active, querybuf->hint); - - if (TARGET_IS_ALL(route_target)) - { - /** - * It is not sure if the session command in question requires - * response. Statement is examined in route_session_write. - */ - bool succp = route_session_write( - router_cli_ses, - gwbuf_clone(querybuf), - inst, - packet_type, - qtype); - if (succp) - { - ret = 1; - } - goto return_ret; - } - /** - * Handle routing to master and to slave - */ - else - { - bool succp = true; - HINT* hint; - char* named_server = NULL; - int rlag_max = MAX_RLAG_UNDEFINED; - - if (router_cli_ses->rses_transaction_active) /*< all to master */ - { - route_target = TARGET_MASTER; /*< override old value */ - - LOGIF(LT, (skygw_log_write( - LOGFILE_TRACE, - "Transaction is active, routing to Master."))); - } - LOGIF(LT, (skygw_log_write(LOGFILE_TRACE, "%s", STRQTYPE(qtype)))); - - /** Lock router session */ - if (!rses_begin_locked_router_action(router_cli_ses)) - { - goto return_ret; - } - - if (TARGET_IS_SLAVE(route_target)) - { - if (TARGET_IS_NAMED_SERVER(route_target) || - TARGET_IS_RLAG_MAX(route_target)) - { - hint = querybuf->hint; - - while (hint != NULL) - { - if (hint->type == HINT_ROUTE_TO_NAMED_SERVER) - { - named_server = hint->data; - LOGIF(LT, (skygw_log_write( - LOGFILE_TRACE, - "Hint: route to server " - "'%s'", - named_server))); - - } - else if (hint->type == HINT_PARAMETER && - (strncasecmp( - (char *)hint->data, - "max_slave_replication_lag", - strlen("max_slave_replication_lag")) == 0)) - { - int val = (int) strtol((char *)hint->value, - (char **)NULL, 10); - - if (val != 0 || errno == 0) - { - rlag_max = val; - LOGIF(LT, (skygw_log_write( - LOGFILE_TRACE, - "Hint: " - "max_slave_replication_lag=%d", - rlag_max))); - } - } - hint = hint->next; - } - } - - if (rlag_max == MAX_RLAG_UNDEFINED) /*< no rlag max hint, use config */ - { - rlag_max = rses_get_max_replication_lag(router_cli_ses); - } - - succp = get_dcb(&target_dcb, - router_cli_ses, - BE_SLAVE, - named_server, - rlag_max); - } - else if (TARGET_IS_MASTER(route_target)) - { - if (master_dcb == NULL) - { - succp = get_dcb(&master_dcb, - router_cli_ses, - BE_MASTER, - NULL, - MAX_RLAG_UNDEFINED); - } - target_dcb = master_dcb; - } - - if (succp) /*< Have DCB of the target backend */ - { - if ((ret = target_dcb->func.write(target_dcb, gwbuf_clone(querybuf))) == 1) - { - backend_ref_t* bref; - - atomic_add(&inst->stats.n_slave, 1); - /** - * Add one query response waiter to backend reference - */ - bref = get_bref_from_dcb(router_cli_ses, target_dcb); - bref_set_state(bref, BREF_QUERY_ACTIVE); - bref_set_state(bref, BREF_WAITING_RESULT); - } - else - { - LOGIF(LE, (skygw_log_write_flush( - LOGFILE_ERROR, - "Error : Routing query \"%s\" failed.", - querystr))); - } - } - rses_end_locked_router_action(router_cli_ses); - } -return_ret: + if (TARGET_IS_ALL(route_target)) + { + /** + * It is not sure if the session command in question requires + * response. Statement is examined in route_session_write. + * Router locking is done inside the function. + */ + succp = route_session_write(router_cli_ses, + gwbuf_clone(querybuf), + inst, + packet_type, + qtype); + + if (succp) + { + ret = 1; + } + goto retblock; + } + + /** Lock router session */ + if (!rses_begin_locked_router_action(router_cli_ses)) + { + goto retblock; + } + /** + * There is a hint which either names the target backend or + * hint which sets maximum allowed replication lag for the + * backend. + */ + if (TARGET_IS_NAMED_SERVER(route_target) || + TARGET_IS_RLAG_MAX(route_target)) + { + HINT* hint; + char* named_server = NULL; + + hint = querybuf->hint; + + while (hint != NULL) + { + if (hint->type == HINT_ROUTE_TO_NAMED_SERVER) + { + /** + * Set the name of searched + * backend server. + */ + named_server = hint->data; + LOGIF(LT, (skygw_log_write( + LOGFILE_TRACE, + "Hint: route to server " + "'%s'", + named_server))); + } + else if (hint->type == HINT_PARAMETER && + (strncasecmp((char *)hint->data, + "max_slave_replication_lag", + strlen("max_slave_replication_lag")) == 0)) + { + int val = (int) strtol((char *)hint->value, + (char **)NULL, 10); + + if (val != 0 || errno == 0) + { + /** + * Set max. acceptable + * replication lag + * value for backend srv + */ + rlag_max = val; + LOGIF(LT, (skygw_log_write( + LOGFILE_TRACE, + "Hint: " + "max_slave_replication_lag=%d", + rlag_max))); + } + } + hint = hint->next; + } /*< while */ + + if (rlag_max == MAX_RLAG_UNDEFINED) /*< no rlag max hint, use config */ + { + rlag_max = rses_get_max_replication_lag(router_cli_ses); + } + btype = BE_UNDEFINED; /*< target may be master or slave */ + /** + * Search backend server by name or replication lag. + * If it fails, then try to find valid slave or master. + */ + succp = get_dcb(&target_dcb, + router_cli_ses, + btype, + named_server, + rlag_max); + } + + if (!succp && TARGET_IS_SLAVE(route_target)) + { + btype = BE_SLAVE; + + if (rlag_max == MAX_RLAG_UNDEFINED) /*< no rlag max hint, use config */ + { + rlag_max = rses_get_max_replication_lag(router_cli_ses); + } + /** + * Search suitable backend server, get DCB in target_dcb + */ + succp = get_dcb(&target_dcb, + router_cli_ses, + BE_SLAVE, + NULL, + rlag_max); + } + + if (!succp && TARGET_IS_MASTER(route_target)) + { + if (master_dcb == NULL) + { + succp = get_dcb(&master_dcb, + router_cli_ses, + BE_MASTER, + NULL, + MAX_RLAG_UNDEFINED); + } + else + { + succp = true; + } + target_dcb = master_dcb; + } + ss_dassert(succp); + + + if (succp) /*< Have DCB of the target backend */ + { + if ((ret = target_dcb->func.write(target_dcb, gwbuf_clone(querybuf))) == 1) + { + backend_ref_t* bref; + + atomic_add(&inst->stats.n_slave, 1); + /** + * Add one query response waiter to backend reference + */ + bref = get_bref_from_dcb(router_cli_ses, target_dcb); + bref_set_state(bref, BREF_QUERY_ACTIVE); + bref_set_state(bref, BREF_WAITING_RESULT); + } + else + { + LOGIF(LE, (skygw_log_write_flush( + LOGFILE_ERROR, + "Error : Routing query \"%s\" failed.", + querystr))); + } + } + rses_end_locked_router_action(router_cli_ses); +retblock: #if defined(SS_DEBUG) { char* canonical_query_str;