From 3e111534a195fe928b260068504649b1e89c6280 Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Wed, 26 Feb 2014 20:07:09 +0200 Subject: [PATCH 1/6] Support for mysql session variable commands, for example, SET AUTOCOMMIT=0 Session commands are identified by query clasisfier, and added to the session command property list in router client session object. Session commands are then executed in existing backend servers but only one of them will reply to client. --- server/core/dcb.c | 1 + server/include/spinlock.h | 2 + server/modules/include/readwritesplit.h | 90 +- server/modules/routing/readconnroute.c | 16 +- .../routing/readwritesplit/readwritesplit.c | 883 ++++++++++++++---- utils/skygw_debug.h | 18 +- 6 files changed, 826 insertions(+), 184 deletions(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index c8051e811..306a3b103 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -1335,3 +1335,4 @@ int gw_write( return w; } + diff --git a/server/include/spinlock.h b/server/include/spinlock.h index c29b84605..42f7b5c2e 100644 --- a/server/include/spinlock.h +++ b/server/include/spinlock.h @@ -53,6 +53,8 @@ typedef struct spinlock { #define SPINLOCK_INIT { 0 } #endif +#define SPINLOCK_IS_LOCKED(l) ((l)->lock != 0 ? true : false) + extern void spinlock_init(SPINLOCK *lock); extern void spinlock_acquire(SPINLOCK *lock); extern int spinlock_acquire_nowait(SPINLOCK *lock); diff --git a/server/modules/include/readwritesplit.h b/server/modules/include/readwritesplit.h index 1c0ca39f6..6abec7563 100644 --- a/server/modules/include/readwritesplit.h +++ b/server/modules/include/readwritesplit.h @@ -24,7 +24,7 @@ * @verbatim * Revision History * - * bazaar.. + * See GitHub https://github.com/skysql/MaxScale * * @endverbatim */ @@ -41,25 +41,88 @@ typedef struct backend { int backend_conn_count; /*< Number of connections to the server */ } BACKEND; +typedef struct rses_property_st rses_property_t; +typedef struct router_client_session ROUTER_CLIENT_SES; + +typedef enum rses_property_type_t { + RSES_PROP_TYPE_UNDEFINED=0, + RSES_PROP_TYPE_FIRST, + RSES_PROP_TYPE_SESCMD=RSES_PROP_TYPE_FIRST, + RSES_PROP_TYPE_LAST=RSES_PROP_TYPE_SESCMD, + RSES_PROP_TYPE_COUNT=RSES_PROP_TYPE_LAST+1 +} rses_property_type_t; + +/** + * Session variable command + */ +typedef struct mysql_sescmd_st { +#if defined(SS_DEBUG) + skygw_chk_t my_sescmd_chk_top; +#endif + ROUTER_CLIENT_SES* my_sescmd_rsession; /*< parent router session */ + rses_property_t* my_sescmd_prop; /*< parent property */ + GWBUF* my_sescmd_buf; /*< client query reference */ + bool my_sescmd_is_replied; /*< is cmd replied to client */ +#if defined(SS_DEBUG) + skygw_chk_t my_sescmd_chk_tail; +#endif +} mysql_sescmd_t; + + +/** + * Property structure + */ +struct rses_property_st { +#if defined(SS_DEBUG) + skygw_chk_t rses_prop_chk_top; +#endif + SPINLOCK rses_prop_lock; /*< protect property content */ + int rses_prop_refcount; + rses_property_type_t rses_prop_type; + union rses_prop_data { + mysql_sescmd_t sescmd; + void* placeholder; /*< to be removed due new type */ + } rses_prop_data; + rses_property_t* rses_prop_next; /*< next property of same type */ +#if defined(SS_DEBUG) + skygw_chk_t rses_prop_chk_tail; +#endif +}; + +typedef struct sescmd_cursor_st { + rses_property_t** scmd_cur_ptr_property; /*< address of pointer to owner property */ + mysql_sescmd_t* scmd_cur_cmd; /*< pointer to current session command */ + bool scmd_cur_active; /*< true if command is being executed */ +} sescmd_cursor_t; + +typedef enum backend_type_t { + BE_UNDEFINED=-1, + BE_MASTER, + BE_SLAVE, + BE_COUNT +} backend_type_t; + /** * The client session structure used within this router. */ -typedef struct router_client_session { +struct router_client_session { #if defined(SS_DEBUG) - skygw_chk_t rses_chk_top; + skygw_chk_t rses_chk_top; #endif - SPINLOCK rses_lock; /*< protects rses_deleted */ - int rses_versno; /*< even = no active update, else odd */ - bool rses_closed; /*< true when closeSession is called */ - BACKEND* be_slave; /*< Slave backend used by client session */ - BACKEND* be_master; /*< Master backend used by client session */ - DCB* slave_dcb; /*< Slave connection */ - DCB* master_dcb; /*< Master connection */ + SPINLOCK rses_lock; /*< protects rses_deleted */ + int rses_versno; /*< even = no active update, else odd */ + bool rses_closed; /*< true when closeSession is called */ + /** Properties listed by their type */ + rses_property_t* rses_properties[RSES_PROP_TYPE_COUNT]; + BACKEND* rses_backend[BE_COUNT];/*< Backends used by client session */ + DCB* rses_dcb[BE_COUNT]; + /*< cursor is pointer and status variable to current session command */ + sescmd_cursor_t rses_cursor[BE_COUNT]; struct router_client_session* next; #if defined(SS_DEBUG) - skygw_chk_t rses_chk_tail; + skygw_chk_t rses_chk_tail; #endif -} ROUTER_CLIENT_SES; +}; /** * The statistics for this router instance @@ -88,5 +151,4 @@ typedef struct router_instance { struct router_instance* next; /*< Next router on the list */ } ROUTER_INSTANCE; - -#endif +#endif /*< _RWSPLITROUTER_H */ diff --git a/server/modules/routing/readconnroute.c b/server/modules/routing/readconnroute.c index c4f982b41..a17e833b8 100644 --- a/server/modules/routing/readconnroute.c +++ b/server/modules/routing/readconnroute.c @@ -119,10 +119,10 @@ static ROUTER_OBJECT MyObject = { errorReply }; -static bool rses_begin_router_action( +static bool rses_begin_locked_router_action( ROUTER_CLIENT_SES* rses); -static void rses_exit_router_action( +static void rses_end_locked_router_action( ROUTER_CLIENT_SES* rses); static SPINLOCK instlock; @@ -498,13 +498,13 @@ DCB* backend_dcb; /** * Lock router client session for secure read and update. */ - if (rses_begin_router_action(router_cli_ses)) + if (rses_begin_locked_router_action(router_cli_ses)) { backend_dcb = router_cli_ses->backend_dcb; router_cli_ses->backend_dcb = NULL; router_cli_ses->rses_closed = true; /** Unlock */ - rses_exit_router_action(router_cli_ses); + rses_end_locked_router_action(router_cli_ses); /** * Close the backend server connection @@ -550,14 +550,14 @@ routeQuery(ROUTER *instance, void *router_session, GWBUF *queue) /** * Lock router client session for secure read of DCBs */ - rses_is_closed = !(rses_begin_router_action(router_cli_ses)); + rses_is_closed = !(rses_begin_locked_router_action(router_cli_ses)); } if (!rses_is_closed) { backend_dcb = router_cli_ses->backend_dcb; /** unlock */ - rses_exit_router_action(router_cli_ses); + rses_end_locked_router_action(router_cli_ses); } if (rses_is_closed || backend_dcb == NULL) @@ -696,7 +696,7 @@ errorReply( * @details (write detailed description here) * */ -static bool rses_begin_router_action( +static bool rses_begin_locked_router_action( ROUTER_CLIENT_SES* rses) { bool succp = false; @@ -731,7 +731,7 @@ return_succp: * @details (write detailed description here) * */ -static void rses_exit_router_action( +static void rses_end_locked_router_action( ROUTER_CLIENT_SES* rses) { CHK_CLIENT_RSES(rses); diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index b6ed199ff..538c9502f 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -82,12 +82,62 @@ static ROUTER_OBJECT MyObject = { clientReply, NULL }; -static bool rses_begin_router_action( +static bool rses_begin_locked_router_action( ROUTER_CLIENT_SES* rses); -static void rses_exit_router_action( +static void rses_end_locked_router_action( ROUTER_CLIENT_SES* rses); +static void mysql_sescmd_done( + mysql_sescmd_t* sescmd); + +static mysql_sescmd_t* mysql_sescmd_init ( + rses_property_t* rses_prop, + GWBUF* sescmd_buf, + ROUTER_CLIENT_SES* rses); + +static rses_property_t* mysql_sescmd_get_property( + mysql_sescmd_t* scmd); + +static rses_property_t* rses_property_init( + rses_property_type_t prop_type); + +static void rses_property_add( + ROUTER_CLIENT_SES* rses, + rses_property_t* prop); + +static void rses_property_done( + rses_property_t* prop); + +static sescmd_cursor_t* rses_get_sescmd_cursor( + ROUTER_CLIENT_SES* rses, + backend_type_t be_type); + + +static bool execute_sescmd_in_backend( + ROUTER_CLIENT_SES* rses, + backend_type_t be_type); + +static bool sescmd_cursor_is_active( + sescmd_cursor_t* sescmd_cursor); + +static GWBUF* sescmd_cursor_get_querybuf( + sescmd_cursor_t* scur); + +static mysql_sescmd_t* sescmd_cursor_get_command( + sescmd_cursor_t* scur); + +static bool sescmd_cursor_next( + sescmd_cursor_t* scur); + +static void sescmd_reply_to_client( + DCB* client_dcb, + mysql_sescmd_t* scmd); + +static bool cont_exec_sescmd_in_backend( + ROUTER_CLIENT_SES* rses, + backend_type_t be_type); + static SPINLOCK instlock; static ROUTER_INSTANCE* instances; @@ -110,8 +160,8 @@ void ModuleInit() { LOGIF(LM, (skygw_log_write_flush( - LOGFILE_MESSAGE, - "Initializing statemend-based read/write split router module."))); + LOGFILE_MESSAGE, + "Initializing statemend-based read/write split router module."))); spinlock_init(&instlock); instances = NULL; } @@ -148,7 +198,7 @@ static ROUTER* createInstance( int i; if ((router = calloc(1, sizeof(ROUTER_INSTANCE))) == NULL) { - return NULL; + return NULL; } router->service = service; spinlock_init(&router->lock); @@ -157,23 +207,23 @@ static ROUTER* createInstance( server = service->databases; for (n=0; server != NULL; server=server->nextdb) { - n++; + n++; } router->servers = (BACKEND **)calloc(n + 1, sizeof(BACKEND *)); if (router->servers == NULL) { - free(router); - return NULL; + free(router); + return NULL; } if (options != NULL) { LOGIF(LM, (skygw_log_write_flush( - LOGFILE_MESSAGE, - "Router options supplied to read/write statement router " - "module but none are supported. The options will be " - "ignored."))); + LOGFILE_MESSAGE, + "Router options supplied to read/write statement router " + "module but none are supported. The options will be " + "ignored."))); } /** @@ -233,10 +283,10 @@ static ROUTER* createInstance( else { LOGIF(LE, (skygw_log_write_flush( - LOGFILE_ERROR, - "Warning : Unsupported router option %s " - "for readwritesplitrouter.", - options[i]))); + LOGFILE_ERROR, + "Warning : Unsupported router option %s " + "for readwritesplitrouter.", + options[i]))); } } } @@ -267,11 +317,10 @@ static void* newSession( ROUTER* router_inst, SESSION* session) { - BACKEND* be_slave = NULL; - BACKEND* be_master = NULL; - ROUTER_CLIENT_SES* client_rses; - ROUTER_INSTANCE* router = (ROUTER_INSTANCE *)router_inst; - bool succp; + BACKEND* local_backend[BE_COUNT]; + ROUTER_CLIENT_SES* client_rses; + ROUTER_INSTANCE* router = (ROUTER_INSTANCE *)router_inst; + bool succp; client_rses = (ROUTER_CLIENT_SES *)calloc(1, sizeof(ROUTER_CLIENT_SES)); @@ -281,16 +330,31 @@ static void* newSession( ss_dassert(false); return NULL; } + memset(local_backend, 0, BE_COUNT*sizeof(void*)); + spinlock_init(&client_rses->rses_lock); #if defined(SS_DEBUG) client_rses->rses_chk_top = CHK_NUM_ROUTER_SES; client_rses->rses_chk_tail = CHK_NUM_ROUTER_SES; #endif - /** + /** store pointers to sescmd list to both cursors */ + client_rses->rses_cursor[BE_MASTER].scmd_cur_active = false; + client_rses->rses_cursor[BE_MASTER].scmd_cur_cmd = NULL; + client_rses->rses_cursor[BE_MASTER].scmd_cur_ptr_property = + &client_rses->rses_properties[RSES_PROP_TYPE_SESCMD]; + + client_rses->rses_cursor[BE_SLAVE].scmd_cur_active = false; + client_rses->rses_cursor[BE_SLAVE].scmd_cur_cmd = NULL; + client_rses->rses_cursor[BE_SLAVE].scmd_cur_ptr_property = + &client_rses->rses_properties[RSES_PROP_TYPE_SESCMD]; + + /** * Find a backend server to connect to. This is the extent of the * load balancing algorithm we need to implement for this simple * connection router. */ - succp = search_backend_servers(&be_master, &be_slave, router); + succp = search_backend_servers(&local_backend[BE_MASTER], + &local_backend[BE_SLAVE], + router); /** Both Master and Slave must be found */ if (!succp) { @@ -300,11 +364,12 @@ static void* newSession( /** * Open the slave connection. */ - client_rses->slave_dcb = dcb_connect(be_slave->backend_server, - session, - be_slave->backend_server->protocol); + client_rses->rses_dcb[BE_SLAVE] = dcb_connect( + local_backend[BE_SLAVE]->backend_server, + session, + local_backend[BE_SLAVE]->backend_server->protocol); - if (client_rses->slave_dcb == NULL) { + if (client_rses->rses_dcb[BE_SLAVE] == NULL) { ss_dassert(session->refcount == 1); free(client_rses); return NULL; @@ -312,13 +377,14 @@ static void* newSession( /** * Open the master connection. */ - client_rses->master_dcb = dcb_connect(be_master->backend_server, - session, - be_master->backend_server->protocol); - if (client_rses->master_dcb == NULL) + client_rses->rses_dcb[BE_MASTER] = dcb_connect( + local_backend[BE_MASTER]->backend_server, + session, + local_backend[BE_MASTER]->backend_server->protocol); + if (client_rses->rses_dcb[BE_MASTER] == NULL) { /** Close slave connection first. */ - client_rses->slave_dcb->func.close(client_rses->slave_dcb); + client_rses->rses_dcb[BE_SLAVE]->func.close(client_rses->rses_dcb[BE_SLAVE]); free(client_rses); return NULL; } @@ -326,11 +392,11 @@ static void* newSession( * We now have a master and a slave server with the least connections. * Bump the connection counts for these servers. */ - atomic_add(&be_slave->backend_conn_count, 1); - atomic_add(&be_master->backend_conn_count, 1); + atomic_add(&local_backend[BE_SLAVE]->backend_conn_count, 1); + atomic_add(&local_backend[BE_MASTER]->backend_conn_count, 1); - client_rses->be_slave = be_slave; - client_rses->be_master = be_master; + client_rses->rses_backend[BE_SLAVE] = local_backend[BE_SLAVE]; + client_rses->rses_backend[BE_MASTER] = local_backend[BE_MASTER]; router->stats.n_sessions += 1; /** @@ -371,16 +437,16 @@ static void closeSession( /** * Lock router client session for secure read and update. */ - if (rses_begin_router_action(router_cli_ses)) + if (rses_begin_locked_router_action(router_cli_ses)) { - slave_dcb = router_cli_ses->slave_dcb; - router_cli_ses->slave_dcb = NULL; - master_dcb = router_cli_ses->master_dcb; - router_cli_ses->master_dcb = NULL; + slave_dcb = router_cli_ses->rses_dcb[BE_SLAVE]; + router_cli_ses->rses_dcb[BE_SLAVE] = NULL; + master_dcb = router_cli_ses->rses_dcb[BE_MASTER]; + router_cli_ses->rses_dcb[BE_MASTER] = NULL; router_cli_ses->rses_closed = true; /** Unlock */ - rses_exit_router_action(router_cli_ses); + rses_end_locked_router_action(router_cli_ses); /** * Close the backend server connections @@ -403,32 +469,49 @@ static void freeSession( { ROUTER_CLIENT_SES* router_cli_ses; ROUTER_INSTANCE* router; + int i; router_cli_ses = (ROUTER_CLIENT_SES *)router_client_session; router = (ROUTER_INSTANCE *)router_instance; - atomic_add(&router_cli_ses->be_slave->backend_conn_count, -1); - atomic_add(&router_cli_ses->be_master->backend_conn_count, -1); - atomic_add(&router_cli_ses->be_slave->backend_server->stats.n_current, -1); - atomic_add(&router_cli_ses->be_master->backend_server->stats.n_current, -1); + atomic_add(&router_cli_ses->rses_backend[BE_SLAVE]->backend_conn_count, -1); + atomic_add(&router_cli_ses->rses_backend[BE_MASTER]->backend_conn_count, -1); + atomic_add(&router_cli_ses->rses_backend[BE_SLAVE]->backend_server->stats.n_current, -1); + atomic_add(&router_cli_ses->rses_backend[BE_MASTER]->backend_server->stats.n_current, -1); spinlock_acquire(&router->lock); if (router->connections == router_cli_ses) { - router->connections = router_cli_ses->next; + router->connections = router_cli_ses->next; } else { - ROUTER_CLIENT_SES* ptr = router->connections; + ROUTER_CLIENT_SES* ptr = router->connections; - while (ptr && ptr->next != router_cli_ses) { - ptr = ptr->next; - } + while (ptr && ptr->next != router_cli_ses) { + ptr = ptr->next; + } - if (ptr) { - ptr->next = router_cli_ses->next; - } + if (ptr) { + ptr->next = router_cli_ses->next; + } } spinlock_release(&router->lock); + /** + * For each property type, walk through the list, finalize properties + * and free the allocated memory. + */ + for (i=RSES_PROP_TYPE_FIRST; irses_properties[i]; + rses_property_t* q = p; + + while (p != NULL) + { + q = p->rses_prop_next; + rses_property_done(p); + p = q; + } + } /* * We are no longer in the linked list, free * all the memory and other resources associated @@ -473,6 +556,7 @@ static int routeQuery( ROUTER_INSTANCE* inst = (ROUTER_INSTANCE *)instance; ROUTER_CLIENT_SES* router_cli_ses = (ROUTER_CLIENT_SES *)router_session; bool rses_is_closed; + rses_property_t* prop; CHK_CLIENT_RSES(router_cli_ses); @@ -529,28 +613,28 @@ static int routeQuery( /** * Lock router client session for secure read of DCBs */ - rses_is_closed = !(rses_begin_router_action(router_cli_ses)); + rses_is_closed = !(rses_begin_locked_router_action(router_cli_ses)); } if (!rses_is_closed) { - master_dcb = router_cli_ses->master_dcb; - slave_dcb = router_cli_ses->slave_dcb; + master_dcb = router_cli_ses->rses_dcb[BE_MASTER]; + slave_dcb = router_cli_ses->rses_dcb[BE_SLAVE]; /** unlock */ - rses_exit_router_action(router_cli_ses); + rses_end_locked_router_action(router_cli_ses); } if (rses_is_closed || (master_dcb == NULL && slave_dcb == NULL)) { LOGIF(LE, (skygw_log_write_flush( - LOGFILE_ERROR, - "Error: Failed to route %s:%s:\"%s\" to backend server. " - "%s.", - STRPACKETTYPE(packet_type), - STRQTYPE(qtype), - (querystr == NULL ? "(empty)" : querystr), - (rses_is_closed ? "Router was closed" : - "Router has no backend servers where to route to")))); + LOGFILE_ERROR, + "Error: Failed to route %s:%s:\"%s\" to backend server. " + "%s.", + STRPACKETTYPE(packet_type), + STRQTYPE(qtype), + (querystr == NULL ? "(empty)" : querystr), + (rses_is_closed ? "Router was closed" : + "Router has no backend servers where to route to")))); goto return_ret; } @@ -559,19 +643,18 @@ static int routeQuery( "String\t\"%s\"", querystr == NULL ? "(empty)" : querystr))); LOGIF(LT, (skygw_log_write(LOGFILE_TRACE, - "Packet type\t%s", + "Packet type\t%s", STRPACKETTYPE(packet_type)))); switch (qtype) { case QUERY_TYPE_WRITE: LOGIF(LT, (skygw_log_write( LOGFILE_TRACE, - "%lu [routeQuery:rwsplit] Query type\t%s, routing to " - "Master.", + "%lu [routeQuery:rwsplit] Query type\t%s, " + "routing to Master.", pthread_self(), STRQTYPE(qtype)))); - - ret = master_dcb->func.write(master_dcb, querybuf); + ret = master_dcb->func.write(master_dcb, querybuf); atomic_add(&inst->stats.n_master, 1); goto return_ret; @@ -584,14 +667,41 @@ static int routeQuery( "routing to Slave.", pthread_self(), STRQTYPE(qtype)))); - - ret = slave_dcb->func.write(slave_dcb, querybuf); - atomic_add(&inst->stats.n_slave, 1); - + /** Lock router session */ + if (!rses_begin_locked_router_action(router_cli_ses)) + { + /** Log error to debug */ + goto return_ret; + } + /** If session command is being executed, route to master */ + if (sescmd_cursor_is_active(rses_get_sescmd_cursor( + router_cli_ses, + BE_MASTER))) + { + ret = master_dcb->func.write(master_dcb, querybuf); + atomic_add(&inst->stats.n_master, 1); + } + else{ + ret = slave_dcb->func.write(slave_dcb, querybuf); + atomic_add(&inst->stats.n_slave, 1); + } + rses_end_locked_router_action(router_cli_ses); goto return_ret; break; case QUERY_TYPE_SESSION_WRITE: + /** + * Execute in backends used by current router session. + * Save session variable commands to router session property + * struct. Thus, they + * can be replayed in backends which are started and joined later. + * + * Suppress OK packets sent to MaxScale by slaves. + * + * DOES THIS ALL APPLY TO COM_QUIT AS WELL?? + * + */ + /** * Update connections which are used in this session. * @@ -638,7 +748,7 @@ static int routeQuery( ret = master_dcb->func.write(master_dcb, querybuf); slave_dcb->func.write(slave_dcb, bufcopy); break; - + case COM_CHANGE_USER: master_dcb->func.auth( master_dcb, @@ -653,12 +763,56 @@ static int routeQuery( break; case COM_QUERY: - ret = master_dcb->func.session(master_dcb, (void *)querybuf); - slave_dcb->func.session(slave_dcb, (void *)bufcopy); + /** + * 1. Create new property of type RSES_PROP_TYPE_SESCMD. + * 2. Add property to the ROUTER_CLIENT_SES struct of + * this router session. + * 3. For each backend, and for each non-executed + * sescmd: + * call execution of current sescmd in + * all backends as long as both have executed + * them all. + * Execution call is dcb->func.session. + * All sescmds are executed when its return value is + * NULL, otherwise it is a pointer to next property. + */ + prop = rses_property_init(RSES_PROP_TYPE_SESCMD); + /** + * Additional reference is created to querybuf to + * prevent it from being released before properties + * are cleaned up as a part of router sessionclean-up. + */ + mysql_sescmd_init(prop, + gwbuf_clone(querybuf), + router_cli_ses); + + /** Lock router session */ + if (!rses_begin_locked_router_action(router_cli_ses)) + { + rses_property_done(prop); + goto return_ret; + } + /** Add sescmd property to router client session */ + rses_property_add(router_cli_ses, prop); + + /** Execute session command in master */ + if (!execute_sescmd_in_backend(router_cli_ses, BE_MASTER)) + { + /** Log error */ + } + /** Execute session command in slave */ + if (!execute_sescmd_in_backend(router_cli_ses, BE_SLAVE)) + { + /** Log error */ + } + + /** Unlock router session */ + rses_end_locked_router_action(router_cli_ses); break; default: - ret = master_dcb->func.session(master_dcb, (void *)querybuf); + ret = master_dcb->func.session(master_dcb, + (void *)querybuf); slave_dcb->func.session(slave_dcb, (void *)bufcopy); break; } /**< switch by packet type */ @@ -707,7 +861,7 @@ return_ret: * @details (write detailed description here) * */ -static bool rses_begin_router_action( +static bool rses_begin_locked_router_action( ROUTER_CLIENT_SES* rses) { bool succp = false; @@ -725,6 +879,10 @@ static bool rses_begin_router_action( succp = true; return_succp: + if (!succp) + { + /** log that router session was closed */ + } return succp; } @@ -742,7 +900,7 @@ return_succp: * @details (write detailed description here) * */ -static void rses_exit_router_action( +static void rses_end_locked_router_action( ROUTER_CLIENT_SES* rses) { CHK_CLIENT_RSES(rses); @@ -761,9 +919,9 @@ static void rses_exit_router_action( static void diagnostic(ROUTER *instance, DCB *dcb) { -ROUTER_CLIENT_SES *router_cli_ses; -ROUTER_INSTANCE *router = (ROUTER_INSTANCE *)instance; -int i = 0; + ROUTER_CLIENT_SES *router_cli_ses; + ROUTER_INSTANCE *router = (ROUTER_INSTANCE *)instance; + int i = 0; spinlock_acquire(&router->lock); router_cli_ses = router->connections; @@ -811,8 +969,11 @@ static void clientReply( DCB* backend_dcb) { DCB* master_dcb; + DCB* slave_dcb; DCB* client_dcb; ROUTER_CLIENT_SES* router_cli_ses; + sescmd_cursor_t* scur = NULL; + backend_type_t be_type = BE_UNDEFINED; router_cli_ses = (ROUTER_CLIENT_SES *)router_session; CHK_CLIENT_RSES(router_cli_ses); @@ -821,34 +982,80 @@ static void clientReply( * Lock router client session for secure read of router session members. * Note that this could be done without lock by using version # */ - if (rses_begin_router_action(router_cli_ses)) + if (!rses_begin_locked_router_action(router_cli_ses)) { - master_dcb = router_cli_ses->master_dcb; + goto lock_failed; + } + master_dcb = router_cli_ses->rses_dcb[BE_MASTER]; + slave_dcb = router_cli_ses->rses_dcb[BE_SLAVE]; - /** Unlock */ - rses_exit_router_action(router_cli_ses); + /** Unlock */ + rses_end_locked_router_action(router_cli_ses); - client_dcb = backend_dcb->session->client; - - if (backend_dcb != NULL && - backend_dcb->command == ROUTER_CHANGE_SESSION) - { - /* if backend_dcb is master we can reply to client */ - if (client_dcb != NULL && - backend_dcb == master_dcb) - { - client_dcb->func.write(client_dcb, writebuf); - } else { - /* consume the gwbuf without writing to client */ - gwbuf_consume(writebuf, gwbuf_length(writebuf)); - } - } - else if (client_dcb != NULL) - { - /* normal flow */ - client_dcb->func.write(client_dcb, writebuf); - } - } + client_dcb = backend_dcb->session->client; + + /** + * 1. Check if backend received reply to sescmd. + * 2. Check sescmd's state whether OK_PACKET has been + * sent to client already and if not, lock property cursor, + * reply to client, and move property cursor forward. Finally + * release the lock. + * 3. If reply for this sescmd is sent, lock property cursor + * and + */ + if (client_dcb == NULL) + { + gwbuf_consume(writebuf, gwbuf_length(writebuf)); + /** Log that client was closed before reply */ + return; + } + + if (backend_dcb == master_dcb) + { + be_type = BE_MASTER; + } + else if (backend_dcb == slave_dcb) + { + be_type = BE_SLAVE; + } + scur = rses_get_sescmd_cursor(router_cli_ses, be_type); + ss_dassert(writebuf == sescmd_cursor_get_querybuf(scur)); + + /** + * Active cursor means that reply is from session command + * execution. + */ + if (sescmd_cursor_is_active(scur)) + { + mysql_sescmd_t* scmd = sescmd_cursor_get_command(scur); + + sescmd_reply_to_client(client_dcb, scmd); + + /** + * If there is a pending sescmd, start its execution. + */ + if (!rses_begin_locked_router_action(router_cli_ses)) + { + goto lock_failed; + } + /** Start execution of all pending ses commands. */ + while (sescmd_cursor_next(scur)) + { + if (!cont_exec_sescmd_in_backend(router_cli_ses, be_type)) + { + /** Log error */ + } + else + { + /** Log execution of pending sescmd */ + } + } + rses_end_locked_router_action(router_cli_ses); + } + return; /*< succeed */ +lock_failed: + /** log that router session couldn't be locked */ + return; } /** @@ -878,8 +1085,7 @@ static bool search_backend_servers( BACKEND** p_slave, ROUTER_INSTANCE* router) { - BACKEND* be_master = NULL; - BACKEND* be_slave = NULL; + BACKEND* local_backend[BE_COUNT] = {NULL,NULL}; int i; bool succp = true; @@ -902,17 +1108,17 @@ static bool search_backend_servers( BACKEND* be = router->servers[i]; if (be != NULL) { - LOGIF(LT, (skygw_log_write( - LOGFILE_TRACE, - "%lu [search_backend_servers] Examine server " - "%s:%d with %d connections. Status is %d, " - "router->bitvalue is %d", - pthread_self(), - be->backend_server->name, - be->backend_server->port, - be->backend_conn_count, - be->backend_server->status, - router->bitmask))); + LOGIF(LT, (skygw_log_write( + LOGFILE_TRACE, + "%lu [search_backend_servers] Examine server " + "%s:%d with %d connections. Status is %d, " + "router->bitvalue is %d", + pthread_self(), + be->backend_server->name, + be->backend_server->port, + be->backend_conn_count, + be->backend_server->status, + router->bitmask))); } if (be != NULL && @@ -927,24 +1133,24 @@ static bool search_backend_servers( * If no candidate set, set first running * server as an initial candidate server. */ - if (be_slave == NULL) + if (local_backend[BE_SLAVE] == NULL) { - be_slave = be; + local_backend[BE_SLAVE] = be; } else if (be->backend_conn_count < - be_slave->backend_conn_count) + local_backend[BE_SLAVE]->backend_conn_count) { /** * This running server has fewer * connections, set it as a new * candidate. */ - be_slave = be; + local_backend[BE_SLAVE] = be; } else if (be->backend_conn_count == - be_slave->backend_conn_count && + local_backend[BE_SLAVE]->backend_conn_count && be->backend_server->stats.n_connections < - be_slave->backend_server->stats.n_connections) + local_backend[BE_SLAVE]->backend_server->stats.n_connections) { /** * This running server has the same @@ -954,58 +1160,415 @@ static bool search_backend_servers( * than candidate, set this server * to candidate. */ - be_slave = be; + local_backend[BE_SLAVE] = be; } } else if (p_master != NULL && - be_master == NULL && + local_backend[BE_MASTER] == NULL && SERVER_IS_MASTER(be->backend_server)) { - be_master = be; + local_backend[BE_MASTER] = be; } } } - if (p_slave != NULL && be_slave == NULL) { + if (p_slave != NULL && local_backend[BE_SLAVE] == NULL) { succp = false; LOGIF(LE, (skygw_log_write_flush( - LOGFILE_ERROR, - "Error : Couldn't find suitable Slave from %d " - "candidates.", - i))); + LOGFILE_ERROR, + "Error : Couldn't find suitable Slave from %d " + "candidates.", + i))); } - if (p_master != NULL && be_master == NULL) { + if (p_master != NULL && local_backend[BE_MASTER] == NULL) { succp = false; LOGIF(LE, (skygw_log_write_flush( - LOGFILE_ERROR, - "Error : Couldn't find suitable Master from %d " - "candidates.", - i))); + LOGFILE_ERROR, + "Error : Couldn't find suitable Master from %d " + "candidates.", + i))); } - if (be_slave != NULL) { - *p_slave = be_slave; + if (local_backend[BE_SLAVE] != NULL) { + *p_slave = local_backend[BE_SLAVE]; LOGIF(LT, (skygw_log_write( - LOGFILE_TRACE, - "%lu [readwritesplit:search_backend_servers] Selected " - "Slave %s:%d from %d candidates.", - pthread_self(), - be_slave->backend_server->name, - be_slave->backend_server->port, - i))); + LOGFILE_TRACE, + "%lu [readwritesplit:search_backend_servers] Selected " + "Slave %s:%d from %d candidates.", + pthread_self(), + local_backend[BE_SLAVE]->backend_server->name, + local_backend[BE_SLAVE]->backend_server->port, + i))); } - if (be_master != NULL) { - *p_master = be_master; + if (local_backend[BE_MASTER] != NULL) { + *p_master = local_backend[BE_MASTER]; LOGIF(LT, (skygw_log_write( - LOGFILE_TRACE, - "%lu [readwritesplit:search_backend_servers] Selected " - "Master %s:%d " - "from %d candidates.", - pthread_self(), - be_master->backend_server->name, - be_master->backend_server->port, - i))); + LOGFILE_TRACE, + "%lu [readwritesplit:search_backend_servers] Selected " + "Master %s:%d " + "from %d candidates.", + pthread_self(), + local_backend[BE_MASTER]->backend_server->name, + local_backend[BE_MASTER]->backend_server->port, + i))); } return succp; } + + +/** + * Create a generic router session property strcture. + */ +static rses_property_t* rses_property_init( + rses_property_type_t prop_type) +{ + rses_property_t* prop; + + prop = (rses_property_t*)calloc(1, sizeof(rses_property_t)); + if (prop == NULL) + { + goto return_prop; + } + spinlock_init(&prop->rses_prop_lock); + prop->rses_prop_type = prop_type; +#if defined(SS_DEBUG) + prop->rses_prop_chk_top = CHK_NUM_ROUTER_PROPERTY; + prop->rses_prop_chk_tail = CHK_NUM_ROUTER_PROPERTY; +#endif + +return_prop: + CHK_RSES_PROP(prop); + return prop; +} + +/** + * Create session command property. + */ +static mysql_sescmd_t* mysql_sescmd_init ( + rses_property_t* rses_prop, + GWBUF* sescmd_buf, + ROUTER_CLIENT_SES* rses) +{ + mysql_sescmd_t* sescmd; + + CHK_RSES_PROP(rses_prop); + sescmd = &rses_prop->rses_prop_data.sescmd; + sescmd->my_sescmd_prop = rses_prop; /*< reference to owning property */ +#if defined(SS_DEBUG) + sescmd->my_sescmd_chk_top = CHK_NUM_MY_SESCMD; + sescmd->my_sescmd_chk_tail = CHK_NUM_MY_SESCMD; +#endif + sescmd->my_sescmd_buf = sescmd_buf; /*< session command query */ + ss_dassert(sescmd_buf->sbuf->refcount > 0); + + return sescmd; +} + +/** + * Property is freed at the end of router client session. + */ +static void rses_property_done( + rses_property_t* prop) +{ + CHK_RSES_PROP(prop); + + switch (prop->rses_prop_type) { + case RSES_PROP_TYPE_SESCMD: + mysql_sescmd_done(&prop->rses_prop_data.sescmd); + break; + default: + LOGIF(LD, (skygw_log_write_flush( + LOGFILE_DEBUG, + "%lu [rses_property_done] Unknown property type %d " + "in property %p", + pthread_self(), + prop->rses_prop_type, + prop))); + + ss_dassert(false); + break; + } + free(prop); +} + + +static void mysql_sescmd_done( + mysql_sescmd_t* sescmd) +{ + CHK_RSES_PROP(sescmd->my_sescmd_prop); + gwbuf_free(sescmd->my_sescmd_buf); + memset(sescmd, 0, sizeof(mysql_sescmd_t)); +} + +/** + * Add property to the router_client_ses structure's rses_properties + * array. The slot is determined by the type of property. + * In each slot there is a list of properties of similar type. + * + * Router client session must be locked. + */ +static void rses_property_add( + ROUTER_CLIENT_SES* rses, + rses_property_t* prop) +{ + rses_property_t* p; + + CHK_CLIENT_RSES(rses); + CHK_RSES_PROP(prop); + ss_dassert(rses->rses_lock.lock != 0); + p = rses->rses_properties[prop->rses_prop_type]; + + if (p == NULL) + { + rses->rses_properties[prop->rses_prop_type] = prop; + } + else + { + while (p->rses_prop_next != NULL) + { + p = p->rses_prop_next; + } + p->rses_prop_next = prop; + } +} + +static void rses_begin_locked_property_action( + rses_property_t* prop) +{ + CHK_RSES_PROP(prop); + spinlock_acquire(&prop->rses_prop_lock); +} + +static void rses_end_locked_property_action( + rses_property_t* prop) +{ + CHK_RSES_PROP(prop); + spinlock_release(&prop->rses_prop_lock); +} + + +/** router must be locked */ +static void sescmd_cursor_set_active( + sescmd_cursor_t* sescmd_cursor, + bool value) +{ + ss_dassert(SPINLOCK_IS_LOCKED(&(*sescmd_cursor->scmd_cur_ptr_property)->rses_prop_lock)); + /** avoid calling unnecessarily */ + ss_dassert(sescmd_cursor->scmd_cur_active != value); + sescmd_cursor->scmd_cur_active = value; +} + + +static void sescmd_reply_to_client( + DCB* client_dcb, + mysql_sescmd_t* scmd) +{ + rses_property_t* prop; + + CHK_DCB(client_dcb); + CHK_MYSQL_SESCMD(scmd); + CHK_GWBUF(scmd->my_sescmd_buf); + + prop = mysql_sescmd_get_property(scmd); + + rses_begin_locked_property_action(prop); + + if (!scmd->my_sescmd_is_replied) + { + CHK_GWBUF(scmd->my_sescmd_buf); + client_dcb->func.write(client_dcb, scmd->my_sescmd_buf); + scmd->my_sescmd_is_replied = true; + } + rses_end_locked_property_action(prop); +} + +static mysql_sescmd_t* sescmd_cursor_get_command( + sescmd_cursor_t* scur) +{ + mysql_sescmd_t* scmd = scur->scmd_cur_cmd; + CHK_MYSQL_SESCMD(scmd); + + return scmd; +} + +/** router must be locked */ +static sescmd_cursor_t* rses_get_sescmd_cursor( + ROUTER_CLIENT_SES* rses, + backend_type_t be_type) +{ + CHK_CLIENT_RSES(rses); + return &rses->rses_cursor[be_type]; +} + +/** router must be locked */ +static bool sescmd_cursor_is_active( + sescmd_cursor_t* sescmd_cursor) +{ + bool succp = sescmd_cursor->scmd_cur_active; + return succp; +} + +/** Router session must be locked */ +static GWBUF* sescmd_cursor_get_querybuf( + sescmd_cursor_t* scur) +{ + GWBUF* buf; + ss_dassert(scur->scmd_cur_cmd != NULL); + + buf = scur->scmd_cur_cmd->my_sescmd_buf; + + CHK_GWBUF(buf); + return buf; +} + +/** + * If session command cursor is passive, sends the command to backend for + * execution. + * + * Returns true if command was sent or added successfully to the queue. + * Returns false if command sending failed or if there are no pending session + * commands. + * + * Router session must be locked. + */ +static bool execute_sescmd_in_backend( + ROUTER_CLIENT_SES* rses, + backend_type_t be_type) +{ + DCB* dcb; + bool succp = true; + int rc = 0; + sescmd_cursor_t* scur; + + dcb = rses->rses_dcb[be_type]; + + CHK_DCB(dcb); + CHK_CLIENT_RSES(rses); + ss_dassert(SPINLOCK_IS_LOCKED(&rses->rses_lock)); + + scur = rses_get_sescmd_cursor(rses, be_type); + + /** Return if there are no pending ses commands */ + if (scur->scmd_cur_cmd == NULL) + { + succp = false; + } + + if (!sescmd_cursor_is_active(scur)) + { + sescmd_cursor_set_active(scur, true); + rc = dcb->func.session(dcb, sescmd_cursor_get_querybuf(scur)); + + if (rc != 0) + { + succp = false; + } + } + return succp; +} + +/** + * Execute session commands when cursor is already active. + * + * Router session must be locked + * + * Return true if there was pending sescmd and sending command to + * backend server succeed. Otherwise false. + */ +static bool cont_exec_sescmd_in_backend( + ROUTER_CLIENT_SES* rses, + backend_type_t be_type) +{ + bool succp = true; + DCB* dcb; + sescmd_cursor_t* scur; + int rc; + + CHK_CLIENT_RSES(rses); + + dcb = rses->rses_dcb[be_type]; + CHK_DCB(dcb); + + ss_dassert(SPINLOCK_IS_LOCKED(&rses->rses_lock)); + + scur = rses_get_sescmd_cursor(rses, be_type); + + ss_dassert(sescmd_cursor_is_active(scur)); + + if (scur->scmd_cur_cmd == NULL) + { + succp = false; + } + rc = dcb->func.session(dcb, sescmd_cursor_get_querybuf(scur)); + + if (rc != 0) + { + succp = false; + } + + return succp; +} + +/** + * Moves cursor to next property and copied address of its sescmd to cursor. + * Current propery must be non-null. + * + * Router session must be locked + */ +static bool sescmd_cursor_next( + sescmd_cursor_t* scur) +{ + bool succp = false; + rses_property_t* prop_curr; + rses_property_t* prop_next; + + ss_dassert(SPINLOCK_IS_LOCKED(&(*scur->scmd_cur_ptr_property)->rses_prop_lock)); + + if (scur == NULL || + *(scur->scmd_cur_ptr_property) == NULL || + scur->scmd_cur_cmd == NULL) + { + /** Log error to debug */ + goto return_succp; + } +#if defined(SS_DEBUG) + prop_curr = *(scur->scmd_cur_ptr_property); + prop_next = prop_curr->rses_prop_next; +#endif + CHK_RSES_PROP(prop_curr); + CHK_MYSQL_SESCMD(scur->scmd_cur_cmd); + CHK_RSES_PROP(scur->scmd_cur_cmd->my_sescmd_prop); + ss_dassert(scur->scmd_cur_cmd->my_sescmd_prop == prop_curr); + + /** If there is a next property move forward */ + if ((*scur->scmd_cur_ptr_property)->rses_prop_next != NULL) + { + scur->scmd_cur_ptr_property = + &((*scur->scmd_cur_ptr_property)->rses_prop_next); + scur->scmd_cur_cmd = + &(*scur->scmd_cur_ptr_property)->rses_prop_data.sescmd; + } + else + { + /** No more properties, can't proceed. */ + goto return_succp; + } + CHK_RSES_PROP((*(scur->scmd_cur_ptr_property))); + CHK_MYSQL_SESCMD(scur->scmd_cur_cmd); + ss_dassert(prop_next == *(scur->scmd_cur_ptr_property)); + + if (scur->scmd_cur_cmd != NULL) + { + succp = true; + } +return_succp: + return succp; +} + +static rses_property_t* mysql_sescmd_get_property( + mysql_sescmd_t* scmd) +{ + CHK_MYSQL_SESCMD(scmd); + return scmd->my_sescmd_prop; +} diff --git a/utils/skygw_debug.h b/utils/skygw_debug.h index ca2ef07dd..508e0f28d 100644 --- a/utils/skygw_debug.h +++ b/utils/skygw_debug.h @@ -117,7 +117,9 @@ typedef enum skygw_chk_t { CHK_NUM_DCB, CHK_NUM_PROTOCOL, CHK_NUM_SESSION, - CHK_NUM_ROUTER_SES + CHK_NUM_ROUTER_SES, + CHK_NUM_MY_SESCMD, + CHK_NUM_ROUTER_PROPERTY } skygw_chk_t; # define STRBOOL(b) ((b) ? "true" : "false") @@ -428,7 +430,19 @@ typedef enum skygw_chk_t { "Router client session has invalid check fields"); \ } - +#define CHK_RSES_PROP(p) { \ + ss_info_dassert((p)->rses_prop_chk_top == CHK_NUM_ROUTER_PROPERTY && \ + (p)->rses_prop_chk_tail == CHK_NUM_ROUTER_PROPERTY, \ + "Router property has invalid check fields"); \ + } + +#define CHK_MYSQL_SESCMD(s) { \ + ss_info_dassert((s)->my_sescmd_chk_top == CHK_NUM_MY_SESCMD && \ + (s)->my_sescmd_chk_tail == CHK_NUM_MY_SESCMD, \ + "Session command has invalid check fields"); \ + } + + #if defined(SS_DEBUG) bool conn_open[10240]; #endif From c28892323a1bfc1908c38ad5c386d8505ddce70d Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Fri, 7 Mar 2014 20:53:33 +0200 Subject: [PATCH 2/6] Added support for session commands to readwrite split router. Added support for multi-statement packets. This is an intermediate commit to save work. Code is not cleaned and there are debug prints and prototypes to be removed. --- server/core/buffer.c | 29 +- server/core/dcb.c | 84 +- server/include/buffer.h | 2 +- server/include/dcb.h | 11 +- server/include/gw.h | 8 +- server/modules/include/readwritesplit.h | 33 +- server/modules/protocol/mysql_backend.c | 16 +- server/modules/protocol/mysql_client.c | 66 +- .../routing/readwritesplit/readwritesplit.c | 1137 +++++++++++------ utils/skygw_debug.h | 6 +- 10 files changed, 921 insertions(+), 471 deletions(-) diff --git a/server/core/buffer.c b/server/core/buffer.c index d46ad475e..e3c649f69 100644 --- a/server/core/buffer.c +++ b/server/core/buffer.c @@ -128,10 +128,37 @@ GWBUF *rval; rval->start = buf->start; rval->end = buf->end; rval->next = NULL; - rval->command = buf->command; +// rval->command = buf->command; CHK_GWBUF(rval); return rval; } + + +GWBUF *gwbuf_clone_portion( + GWBUF *buf, + size_t start_offset, + size_t length) +{ + GWBUF* clonebuf; + + CHK_GWBUF(buf); + ss_dassert(start_offset+length <= GWBUF_LENGTH(buf)); + + if ((clonebuf = (GWBUF *)malloc(sizeof(GWBUF))) == NULL) + { + return NULL; + } + atomic_add(&buf->sbuf->refcount, 1); + clonebuf->sbuf = buf->sbuf; + clonebuf->start = (void *)((char*)buf->start)+start_offset; + clonebuf->end = (void *)((char *)clonebuf->start)+length; + clonebuf->next = NULL; + CHK_GWBUF(clonebuf); + return clonebuf; + +} + + /** * Append a buffer onto a linked list of buffer structures. * diff --git a/server/core/dcb.c b/server/core/dcb.c index 306a3b103..fcf2e0cf2 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -302,6 +302,8 @@ dcb_final_free(DCB *dcb) if (dcb->remote) free(dcb->remote); bitmask_free(&dcb->memdata.bitmask); + simple_mutex_done(&dcb->dcb_read_lock); + simple_mutex_done(&dcb->dcb_write_lock); free(dcb); } @@ -520,6 +522,8 @@ int rc; * Successfully connected to backend. Assign file descriptor to dcb */ dcb->fd = fd; + /** Copy status field to DCB */ + dcb->dcb_server_status = server->status; /*< * backend_dcb is connected to backend server, and once backend_dcb @@ -683,6 +687,15 @@ dcb_write(DCB *dcb, GWBUF *queue) dcb->state != DCB_STATE_LISTENING && dcb->state != DCB_STATE_NOPOLLING)) { + LOGIF(LD, (skygw_log_write( + LOGFILE_DEBUG, + "%lu [dcb_write] Write aborted to dcb %p because " + "it is in state %s", + pthread_self(), + dcb->stats.n_buffered, + dcb, + STRDCBSTATE(dcb->state), + dcb->fd))); return 0; } @@ -743,7 +756,11 @@ dcb_write(DCB *dcb, GWBUF *queue) #endif /* SS_DEBUG */ len = GWBUF_LENGTH(queue); GW_NOINTR_CALL( - w = gw_write(dcb->fd, GWBUF_DATA(queue), len); + w = gw_write( +#if defined(SS_DEBUG) + dcb, +#endif + dcb->fd, GWBUF_DATA(queue), len); dcb->stats.n_writes++; ); @@ -855,9 +872,13 @@ int saved_errno = 0; while (dcb->writeq != NULL) { len = GWBUF_LENGTH(dcb->writeq); - GW_NOINTR_CALL(w = gw_write(dcb->fd, - GWBUF_DATA(dcb->writeq), - len);); + GW_NOINTR_CALL(w = gw_write( +#if defined(SS_DEBUG) + dcb, +#endif + dcb->fd, + GWBUF_DATA(dcb->writeq), + len);); saved_errno = errno; errno = 0; @@ -1312,12 +1333,15 @@ static bool dcb_set_state_nomutex( } int gw_write( +#if defined(SS_DEBUG) + DCB* dcb, +#endif int fd, const void* buf, size_t nbytes) { int w; -#if defined(SS_DEBUG) +#if defined(SS_DEBUG) if (dcb_fake_write_errno[fd] != 0) { ss_dassert(dcb_fake_write_ev[fd] != 0); w = write(fd, buf, nbytes/2); /*< leave peer to read missing bytes */ @@ -1332,6 +1356,56 @@ int gw_write( #else w = write(fd, buf, nbytes); #endif /* SS_DEBUG && SS_TEST */ + +#if defined(SS_DEBUG) + { + size_t len; + unsigned char* packet = (unsigned char *)buf; + char* str; + + /** Print only MySQL packets */ + if (w > 5) + { + str = (char *)&packet[5]; + len = packet[0]; + len += 255*packet[1]; + len += 255*255*packet[2]; + + if (strncmp(str, "insert", 6) == 0 || + strncmp(str, "create", 6) == 0 || + strncmp(str, "drop", 4) == 0) + { + ss_dassert((dcb->dcb_server_status & (SERVER_RUNNING|SERVER_MASTER|SERVER_SLAVE))==(SERVER_RUNNING|SERVER_MASTER)); + } + + if (strncmp(str, "set autocommit", 14) == 0 && nbytes > 17) + { + char* s = (char *)calloc(1, nbytes+1); + + if (nbytes-5 > len) + { + size_t len2 = packet[4+len]; + len2 += 255*packet[4+len+1]; + len2 += 255*255*packet[4+len+2]; + + char* str2 = (char *)&packet[4+len+5]; + snprintf(s, 5+len+len2, "long %s %s", (char *)str, (char *)str2); + } + else + { + snprintf(s, len, "%s", (char *)str); + } + LOGIF(LT, (skygw_log_write( + LOGFILE_TRACE, + "%lu [gw_write] Wrote %d bytes : %s ", + pthread_self(), + w, + s))); + free(s); + } + } + } +#endif return w; } diff --git a/server/include/buffer.h b/server/include/buffer.h index ca9669ecd..de5a13148 100644 --- a/server/include/buffer.h +++ b/server/include/buffer.h @@ -92,6 +92,6 @@ extern GWBUF *gwbuf_clone(GWBUF *buf); extern GWBUF *gwbuf_append(GWBUF *head, GWBUF *tail); extern GWBUF *gwbuf_consume(GWBUF *head, unsigned int length); extern unsigned int gwbuf_length(GWBUF *head); - +extern GWBUF *gwbuf_clone_portion(GWBUF *head, size_t offset, size_t len); #endif diff --git a/server/include/dcb.h b/server/include/dcb.h index e46784b27..cacf6527e 100644 --- a/server/include/dcb.h +++ b/server/include/dcb.h @@ -87,6 +87,7 @@ typedef struct gw_protocol { int (*listen)(struct dcb *, char *); int (*auth)(struct dcb *, struct server *, struct session *, GWBUF *); int (*session)(struct dcb *, void *); + void* (*getstmt)(void* buf); } GWPROTOCOL; /** @@ -176,7 +177,7 @@ typedef struct dcb { SPINLOCK authlock; /**< Generic Authorization spinlock */ DCBSTATS stats; /**< DCB related statistics */ - + unsigned int dcb_server_status; /*< the server role indicator from SERVER */ struct dcb *next; /**< Next DCB in the chain of allocated DCB's */ struct service *service; /**< The related service */ void *data; /**< Specific client data */ @@ -202,7 +203,13 @@ int fail_accept_errno; #define DCB_ISZOMBIE(x) ((x)->state == DCB_STATE_ZOMBIE) DCB *dcb_get_zombies(void); -int gw_write(int fd, const void* buf, size_t nbytes); +int gw_write( +#if defined(SS_DEBUG) + DCB* dcb, +#endif + int fd, + const void* buf, + size_t nbytes); int dcb_write(DCB *, GWBUF *); DCB *dcb_alloc(dcb_role_t); void dcb_free(DCB *); diff --git a/server/include/gw.h b/server/include/gw.h index 98e48c60a..17911d13a 100644 --- a/server/include/gw.h +++ b/server/include/gw.h @@ -56,6 +56,12 @@ int MySQLWrite(DCB *dcb, GWBUF *queue); int gw_write_backend_event(DCB *dcb); int gw_read_backend_event(DCB *dcb); int setnonblocking(int fd); -int gw_write(int fd, const void* buf, size_t nbytes); +int gw_write( +#if defined(SS_DEBUG) + DCB* dcb, +#endif + int fd, + const void* buf, + size_t nbytes); int gw_getsockerrno(int fd); int parse_bindconfig(char *, unsigned short, struct sockaddr_in *); diff --git a/server/modules/include/readwritesplit.h b/server/modules/include/readwritesplit.h index 6abec7563..95b414958 100644 --- a/server/modules/include/readwritesplit.h +++ b/server/modules/include/readwritesplit.h @@ -46,12 +46,19 @@ typedef struct router_client_session ROUTER_CLIENT_SES; typedef enum rses_property_type_t { RSES_PROP_TYPE_UNDEFINED=0, - RSES_PROP_TYPE_FIRST, - RSES_PROP_TYPE_SESCMD=RSES_PROP_TYPE_FIRST, + RSES_PROP_TYPE_SESCMD, + RSES_PROP_TYPE_FIRST = RSES_PROP_TYPE_SESCMD, RSES_PROP_TYPE_LAST=RSES_PROP_TYPE_SESCMD, RSES_PROP_TYPE_COUNT=RSES_PROP_TYPE_LAST+1 } rses_property_type_t; +typedef enum backend_type_t { + BE_UNDEFINED=-1, + BE_MASTER, + BE_SLAVE, + BE_COUNT +} backend_type_t; + /** * Session variable command */ @@ -59,9 +66,9 @@ typedef struct mysql_sescmd_st { #if defined(SS_DEBUG) skygw_chk_t my_sescmd_chk_top; #endif - ROUTER_CLIENT_SES* my_sescmd_rsession; /*< parent router session */ +// ROUTER_CLIENT_SES* my_sescmd_rsession; /*< parent router session */ rses_property_t* my_sescmd_prop; /*< parent property */ - GWBUF* my_sescmd_buf; /*< client query reference */ + GWBUF* my_sescmd_buf; /*< query buffer */ bool my_sescmd_is_replied; /*< is cmd replied to client */ #if defined(SS_DEBUG) skygw_chk_t my_sescmd_chk_tail; @@ -76,7 +83,8 @@ struct rses_property_st { #if defined(SS_DEBUG) skygw_chk_t rses_prop_chk_top; #endif - SPINLOCK rses_prop_lock; /*< protect property content */ + ROUTER_CLIENT_SES* rses_prop_rsession; /*< parent router session */ +// SPINLOCK rses_prop_lock; /*< protect property content */ int rses_prop_refcount; rses_property_type_t rses_prop_type; union rses_prop_data { @@ -90,18 +98,13 @@ struct rses_property_st { }; typedef struct sescmd_cursor_st { - rses_property_t** scmd_cur_ptr_property; /*< address of pointer to owner property */ - mysql_sescmd_t* scmd_cur_cmd; /*< pointer to current session command */ - bool scmd_cur_active; /*< true if command is being executed */ + ROUTER_CLIENT_SES* scmd_cur_rses; /*< pointer to owning router session */ + rses_property_t** scmd_cur_ptr_property; /*< address of pointer to owner property */ + mysql_sescmd_t* scmd_cur_cmd; /*< pointer to current session command */ + bool scmd_cur_active; /*< true if command is being executed */ + backend_type_t scmd_cur_be_type; /*< BE_MASTER or BE_SLAVE */ } sescmd_cursor_t; -typedef enum backend_type_t { - BE_UNDEFINED=-1, - BE_MASTER, - BE_SLAVE, - BE_COUNT -} backend_type_t; - /** * The client session structure used within this router. */ diff --git a/server/modules/protocol/mysql_backend.c b/server/modules/protocol/mysql_backend.c index a031648c7..06d6e3a85 100644 --- a/server/modules/protocol/mysql_backend.c +++ b/server/modules/protocol/mysql_backend.c @@ -521,7 +521,7 @@ gw_MySQLWrite_backend(DCB *dcb, GWBUF *queue) /*< * Don't write to backend if backend_dcb is not in poll set anymore. */ - spinlock_acquire(&dcb->authlock); + spinlock_acquire(&dcb->dcb_initlock); if (dcb->state != DCB_STATE_POLLING) { /*< vraa : errorHandle */ /*< Free buffer memory */ @@ -536,10 +536,12 @@ gw_MySQLWrite_backend(DCB *dcb, GWBUF *queue) dcb->fd, STRDCBSTATE(dcb->state)))); - spinlock_release(&dcb->authlock); + spinlock_release(&dcb->dcb_initlock); return 0; } - + spinlock_release(&dcb->dcb_initlock); + + spinlock_acquire(&dcb->authlock); /*< * Now put the incoming data to the delay queue unless backend is * connected with auth ok @@ -553,7 +555,6 @@ gw_MySQLWrite_backend(DCB *dcb, GWBUF *queue) dcb, dcb->fd, STRPROTOCOLSTATE(backend_protocol->state)))); - backend_set_delayqueue(dcb, queue); spinlock_release(&dcb->authlock); return 1; @@ -562,9 +563,9 @@ gw_MySQLWrite_backend(DCB *dcb, GWBUF *queue) /*< * Now we set the last command received, from the current queue */ - memcpy(&dcb->command, &queue->command, sizeof(dcb->command)); - +// memcpy(&dcb->command, &queue->command, sizeof(dcb->command)); spinlock_release(&dcb->authlock); +// LOGIF(LD, debuglog_statements(dcb, gwbuf_clone(queue))); rc = dcb_write(dcb, queue); return rc; } @@ -805,7 +806,7 @@ static int backend_write_delayqueue(DCB *dcb) * Now we set the last command received, from the delayed queue */ - memcpy(&dcb->command, &localq->command, sizeof(dcb->command)); +// memcpy(&dcb->command, &localq->command, sizeof(dcb->command)); spinlock_release(&dcb->delayqlock); rc = dcb_write(dcb, localq); @@ -911,7 +912,6 @@ static int gw_change_user(DCB *backend, SERVER *server, SESSION *in_session, GWB strcpy(current_session->user, username); strcpy(current_session->db, database); memcpy(current_session->client_sha1, client_sha1, sizeof(current_session->client_sha1)); - } // consume all the data received from client diff --git a/server/modules/protocol/mysql_client.c b/server/modules/protocol/mysql_client.c index a8cde7337..c2f50a76a 100644 --- a/server/modules/protocol/mysql_client.c +++ b/server/modules/protocol/mysql_client.c @@ -48,6 +48,7 @@ static int gw_MySQLWrite_client(DCB *dcb, GWBUF *queue); static int gw_error_client_event(DCB *dcb); static int gw_client_close(DCB *dcb); static int gw_client_hangup_event(DCB *dcb); +static void* gw_MySQL_get_next_stmt(void* buffer); int mysql_send_ok(DCB *dcb, int packet_number, int in_affected_rows, const char* mysql_message); int MySQLSendHandshake(DCB* dcb); @@ -67,8 +68,9 @@ static GWPROTOCOL MyObject = { gw_client_close, /* Close */ gw_MySQLListener, /* Listen */ NULL, /* Authentication */ - NULL /* Session */ - }; + NULL, /* Session */ + gw_MySQL_get_next_stmt /* get single stmt from read buf */ +}; /** * Implementation of the mandatory version entry point @@ -607,8 +609,7 @@ int gw_read_client_event(DCB* dcb) { */ { int len = -1; - GWBUF *queue = NULL; - GWBUF *gw_buffer = NULL; + GWBUF *read_buffer = NULL; uint8_t *ptr_buff = NULL; int mysql_command = -1; @@ -626,16 +627,15 @@ int gw_read_client_event(DCB* dcb) { ////////////////////////////////////////////////////// // read and handle errors & close, or return if busy ////////////////////////////////////////////////////// - rc = gw_read_gwbuff(dcb, &gw_buffer, b); + rc = gw_read_gwbuff(dcb, &read_buffer, b); if (rc != 0) { goto return_rc; } /* Now, we are assuming in the first buffer there is * the information form mysql command */ - queue = gw_buffer; - len = GWBUF_LENGTH(queue); - ptr_buff = GWBUF_DATA(queue); + len = GWBUF_LENGTH(read_buffer); + ptr_buff = GWBUF_DATA(read_buffer); /* get mysql commang at fifth byte */ if (ptr_buff) { @@ -669,12 +669,12 @@ int gw_read_client_event(DCB* dcb) { } rc = 1; /** Free buffer */ - queue = gwbuf_consume(queue, len); + read_buffer = gwbuf_consume(read_buffer, len); goto return_rc; } /** Route COM_QUIT to backend */ if (mysql_command == '\x01') { - router->routeQuery(router_instance, rsession, queue); + router->routeQuery(router_instance, rsession, read_buffer); LOGIF(LD, (skygw_log_write_flush( LOGFILE_DEBUG, "%lu [gw_read_client_event] Routed COM_QUIT to " @@ -693,7 +693,7 @@ int gw_read_client_event(DCB* dcb) { /** Route other commands to backend */ rc = router->routeQuery(router_instance, rsession, - queue); + read_buffer); /** succeed */ if (rc == 1) { rc = 0; /**< here '0' means success */ @@ -1203,3 +1203,47 @@ gw_client_hangup_event(DCB *dcb) return_rc: return rc; } + +/** + * Remove the first mysql statement from buffer. Return pointer to the removed + * statement or NULL if buffer is empty. + * + * Clone buf, calculate the length of included mysql stmt, and point the + * statement with cloned buffer. Move the start pointer of buf accordingly + * so that it only cover the remaining buffer. + * + */ +static void* gw_MySQL_get_next_stmt( + void* buffer) +{ + GWBUF* readbuf = (GWBUF *)buffer; + GWBUF* stmtbuf; + unsigned char* packet; + size_t len; + + CHK_GWBUF(readbuf); + + if (GWBUF_EMPTY(readbuf)) + { + stmtbuf = NULL; + goto return_stmtbuf; + } + packet = GWBUF_DATA(readbuf); + len = packet[0]; + len += 255*packet[1]; + len += 255*255*packet[2]; + + /** vraa :Multi-packet stmt is not supported as of 7.3.14 */ + if (len+4 > GWBUF_LENGTH(readbuf)) + { + stmtbuf = NULL; + goto return_stmtbuf; + } + stmtbuf = gwbuf_clone_portion(readbuf, 0, 4+len); + gwbuf_consume(readbuf, 4+len); + +return_stmtbuf: + return (void *)stmtbuf; +} + + diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index 538c9502f..8f38c79c3 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -109,19 +109,25 @@ static void rses_property_add( static void rses_property_done( rses_property_t* prop); +static mysql_sescmd_t* rses_property_get_sescmd( + rses_property_t* prop); + static sescmd_cursor_t* rses_get_sescmd_cursor( ROUTER_CLIENT_SES* rses, backend_type_t be_type); - static bool execute_sescmd_in_backend( ROUTER_CLIENT_SES* rses, backend_type_t be_type); +static void sescmd_cursor_set_active( + sescmd_cursor_t* sescmd_cursor, + bool value); + static bool sescmd_cursor_is_active( sescmd_cursor_t* sescmd_cursor); -static GWBUF* sescmd_cursor_get_querybuf( +static GWBUF* sescmd_cursor_clone_querybuf( sescmd_cursor_t* scur); static mysql_sescmd_t* sescmd_cursor_get_command( @@ -130,13 +136,20 @@ static mysql_sescmd_t* sescmd_cursor_get_command( static bool sescmd_cursor_next( sescmd_cursor_t* scur); -static void sescmd_reply_to_client( +static bool sescmd_reply_to_client( DCB* client_dcb, - mysql_sescmd_t* scmd); + mysql_sescmd_t* scmd, + GWBUF* writebuf); static bool cont_exec_sescmd_in_backend( - ROUTER_CLIENT_SES* rses, - backend_type_t be_type); + ROUTER_CLIENT_SES* rses, + backend_type_t be_type); + +static void tracelog_routed_query( + ROUTER_CLIENT_SES* rses, + char* funcname, + DCB* dcb, + GWBUF* buf); static SPINLOCK instlock; static ROUTER_INSTANCE* instances; @@ -337,16 +350,20 @@ static void* newSession( client_rses->rses_chk_tail = CHK_NUM_ROUTER_SES; #endif /** store pointers to sescmd list to both cursors */ + client_rses->rses_cursor[BE_MASTER].scmd_cur_rses = client_rses; client_rses->rses_cursor[BE_MASTER].scmd_cur_active = false; - client_rses->rses_cursor[BE_MASTER].scmd_cur_cmd = NULL; client_rses->rses_cursor[BE_MASTER].scmd_cur_ptr_property = &client_rses->rses_properties[RSES_PROP_TYPE_SESCMD]; - - client_rses->rses_cursor[BE_SLAVE].scmd_cur_active = false; - client_rses->rses_cursor[BE_SLAVE].scmd_cur_cmd = NULL; + client_rses->rses_cursor[BE_MASTER].scmd_cur_cmd = NULL; + client_rses->rses_cursor[BE_MASTER].scmd_cur_be_type = BE_MASTER; + + client_rses->rses_cursor[BE_SLAVE].scmd_cur_rses = client_rses; + client_rses->rses_cursor[BE_SLAVE].scmd_cur_active = false; client_rses->rses_cursor[BE_SLAVE].scmd_cur_ptr_property = &client_rses->rses_properties[RSES_PROP_TYPE_SESCMD]; - + client_rses->rses_cursor[BE_SLAVE].scmd_cur_cmd = NULL; + client_rses->rses_cursor[BE_SLAVE].scmd_cur_be_type = BE_SLAVE; + /** * Find a backend server to connect to. This is the extent of the * load balancing algorithm we need to implement for this simple @@ -544,6 +561,7 @@ static int routeQuery( GWBUF* querybuf) { skygw_query_type_t qtype = QUERY_TYPE_UNKNOWN; + GWBUF* stmtbuf; char* querystr = NULL; char* startpos; size_t len; @@ -552,7 +570,7 @@ static int routeQuery( int ret = 0; DCB* master_dcb = NULL; DCB* slave_dcb = NULL; - GWBUF* bufcopy = NULL; +// GWBUF* bufcopy = NULL; ROUTER_INSTANCE* inst = (ROUTER_INSTANCE *)instance; ROUTER_CLIENT_SES* router_cli_ses = (ROUTER_CLIENT_SES *)router_session; bool rses_is_closed; @@ -561,284 +579,364 @@ static int routeQuery( CHK_CLIENT_RSES(router_cli_ses); inst->stats.n_queries++; - - packet = GWBUF_DATA(querybuf); - packet_type = packet[4]; - startpos = (char *)&packet[5]; - len = packet[0]; - len += 255*packet[1]; - len += 255*255*packet[2]; - - switch(packet_type) { - case COM_QUIT: /**< 1 QUIT will close all sessions */ - case COM_INIT_DB: /**< 2 DDL must go to the master */ - case COM_REFRESH: /**< 7 - I guess this is session but not sure */ - case COM_DEBUG: /**< 0d all servers dump debug info to stdout */ - case COM_PING: /**< 0e all servers are pinged */ - case COM_CHANGE_USER: /**< 11 all servers change it accordingly */ - qtype = QUERY_TYPE_SESSION_WRITE; - break; - - case COM_CREATE_DB: /**< 5 DDL must go to the master */ - case COM_DROP_DB: /**< 6 DDL must go to the master */ - qtype = QUERY_TYPE_WRITE; - break; - - case COM_QUERY: - querystr = (char *)malloc(len); - memcpy(querystr, startpos, len-1); - memset(&querystr[len-1], 0, 1); - qtype = skygw_query_classifier_get_type(querystr, 0); - break; - - case COM_SHUTDOWN: /**< 8 where should shutdown be routed ? */ - case COM_STATISTICS: /**< 9 ? */ - case COM_PROCESS_INFO: /**< 0a ? */ - case COM_CONNECT: /**< 0b ? */ - case COM_PROCESS_KILL: /**< 0c ? */ - case COM_TIME: /**< 0f should this be run in gateway ? */ - case COM_DELAYED_INSERT: /**< 10 ? */ - case COM_DAEMON: /**< 1d ? */ - default: - break; - } /**< switch by packet type */ - - /** Dirty read for quick check if router is closed. */ - if (router_cli_ses->rses_closed) + /** Lock router session */ + if (!rses_begin_locked_router_action(router_cli_ses)) { - rses_is_closed = true; - } - else - { - /** - * Lock router client session for secure read of DCBs - */ - rses_is_closed = !(rses_begin_locked_router_action(router_cli_ses)); - } - - if (!rses_is_closed) - { - master_dcb = router_cli_ses->rses_dcb[BE_MASTER]; - slave_dcb = router_cli_ses->rses_dcb[BE_SLAVE]; - /** unlock */ - rses_end_locked_router_action(router_cli_ses); - } - - if (rses_is_closed || (master_dcb == NULL && slave_dcb == NULL)) - { - LOGIF(LE, (skygw_log_write_flush( - LOGFILE_ERROR, - "Error: Failed to route %s:%s:\"%s\" to backend server. " - "%s.", - STRPACKETTYPE(packet_type), - STRQTYPE(qtype), - (querystr == NULL ? "(empty)" : querystr), - (rses_is_closed ? "Router was closed" : - "Router has no backend servers where to route to")))); - goto return_ret; } + master_dcb = router_cli_ses->rses_dcb[BE_MASTER]; - LOGIF(LT, (skygw_log_write(LOGFILE_TRACE, - "String\t\"%s\"", - querystr == NULL ? "(empty)" : querystr))); - LOGIF(LT, (skygw_log_write(LOGFILE_TRACE, - "Packet type\t%s", - STRPACKETTYPE(packet_type)))); + /** stmtbuf is clone of querybuf, and only covers one stmt */ + stmtbuf = (GWBUF *)master_dcb->session->client->func.getstmt((void *)querybuf); + /** Unlock router session */ + rses_end_locked_router_action(router_cli_ses); - switch (qtype) { - case QUERY_TYPE_WRITE: - LOGIF(LT, (skygw_log_write( - LOGFILE_TRACE, - "%lu [routeQuery:rwsplit] Query type\t%s, " - "routing to Master.", - pthread_self(), - STRQTYPE(qtype)))); - ret = master_dcb->func.write(master_dcb, querybuf); - atomic_add(&inst->stats.n_master, 1); - - goto return_ret; - break; - - case QUERY_TYPE_READ: - LOGIF(LT, (skygw_log_write( - LOGFILE_TRACE, - "%lu [routeQuery:rwsplit] Query type\t%s, " - "routing to Slave.", - pthread_self(), - STRQTYPE(qtype)))); - /** Lock router session */ - if (!rses_begin_locked_router_action(router_cli_ses)) - { - /** Log error to debug */ - goto return_ret; - } - /** If session command is being executed, route to master */ - if (sescmd_cursor_is_active(rses_get_sescmd_cursor( - router_cli_ses, - BE_MASTER))) - { - ret = master_dcb->func.write(master_dcb, querybuf); - atomic_add(&inst->stats.n_master, 1); - } - else{ - ret = slave_dcb->func.write(slave_dcb, querybuf); - atomic_add(&inst->stats.n_slave, 1); - } - rses_end_locked_router_action(router_cli_ses); - goto return_ret; - break; - - case QUERY_TYPE_SESSION_WRITE: - /** - * Execute in backends used by current router session. - * Save session variable commands to router session property - * struct. Thus, they - * can be replayed in backends which are started and joined later. - * - * Suppress OK packets sent to MaxScale by slaves. - * - * DOES THIS ALL APPLY TO COM_QUIT AS WELL?? - * - */ - - /** - * Update connections which are used in this session. - * - * For each connection updated, add a flag which indicates that - * OK Packet must arrive for this command before server - * in question is allowed to be used by router. That is, - * maintain a queue of pending OK packets and remove item - * from queue by FIFO. - * - * Return when the master responds OK Packet. Send that - * OK packet back to client. - * - * Suppress OK packets sent to MaxScale by slaves. - * - * Open questions: - * How to handle interleaving session write - * and queries? It would be simple if OK must be received - * from all/both servers before continuing query execution. - * How to maintain the order of operations? Execution queue - * would solve the problem. In the queue some things must be - * executed in serialized manner while some could be executed - * in parallel. Queries mostly. - * - * Instead of waiting for the OK packet from the master, the - * first OK packet could also be sent to client. TBD. - * vraa 9.12.13 - * - */ - LOGIF(LT, (skygw_log_write( - LOGFILE_TRACE, - "%lu [routeQuery:rwsplit] DCB M:%p s:%p, " - "Query type\t%s, " - "packet type %s, routing to all servers.", - pthread_self(), - master_dcb, - slave_dcb, - STRQTYPE(qtype), - STRPACKETTYPE(packet_type)))); - - bufcopy = gwbuf_clone(querybuf); + while (stmtbuf != NULL) + { + packet = GWBUF_DATA(stmtbuf); + packet_type = packet[4]; + startpos = (char *)&packet[5]; + len = packet[0]; + len += 255*packet[1]; + len += 255*255*packet[2]; switch(packet_type) { - case COM_QUIT: - ret = master_dcb->func.write(master_dcb, querybuf); - slave_dcb->func.write(slave_dcb, bufcopy); + case COM_QUIT: /**< 1 QUIT will close all sessions */ + case COM_INIT_DB: /**< 2 DDL must go to the master */ + case COM_REFRESH: /**< 7 - I guess this is session but not sure */ + case COM_DEBUG: /**< 0d all servers dump debug info to stdout */ + case COM_PING: /**< 0e all servers are pinged */ + case COM_CHANGE_USER: /**< 11 all servers change it accordingly */ + qtype = QUERY_TYPE_SESSION_WRITE; + break; + + case COM_CREATE_DB: /**< 5 DDL must go to the master */ + case COM_DROP_DB: /**< 6 DDL must go to the master */ + qtype = QUERY_TYPE_WRITE; + break; + + case COM_QUERY: + querystr = (char *)malloc(len); + memcpy(querystr, startpos, len-1); + memset(&querystr[len-1], 0, 1); + qtype = skygw_query_classifier_get_type(querystr, 0); + break; + + case COM_SHUTDOWN: /**< 8 where should shutdown be routed ? */ + case COM_STATISTICS: /**< 9 ? */ + case COM_PROCESS_INFO: /**< 0a ? */ + case COM_CONNECT: /**< 0b ? */ + case COM_PROCESS_KILL: /**< 0c ? */ + case COM_TIME: /**< 0f should this be run in gateway ? */ + case COM_DELAYED_INSERT: /**< 10 ? */ + case COM_DAEMON: /**< 1d ? */ + default: + break; + } /**< switch by packet type */ + + /** Dirty read for quick check if router is closed. */ + if (router_cli_ses->rses_closed) + { + rses_is_closed = true; + } + else + { + /** + * Lock router client session for secure read of DCBs + */ + rses_is_closed = + !(rses_begin_locked_router_action(router_cli_ses)); + } + + if (!rses_is_closed) + { + master_dcb = router_cli_ses->rses_dcb[BE_MASTER]; + slave_dcb = router_cli_ses->rses_dcb[BE_SLAVE]; + /** unlock */ + rses_end_locked_router_action(router_cli_ses); + } + + if (rses_is_closed || (master_dcb == NULL && slave_dcb == NULL)) + { + LOGIF(LE, (skygw_log_write_flush( + LOGFILE_ERROR, + "Error: Failed to route %s:%s:\"%s\" to " + "backend server. %s.", + STRPACKETTYPE(packet_type), + STRQTYPE(qtype), + (querystr == NULL ? "(empty)" : querystr), + (rses_is_closed ? "Router was closed" : + "Router has no backend servers where to " + "route to")))); + goto return_ret; + } + + LOGIF(LT, (skygw_log_write(LOGFILE_TRACE, + "String\t\"%s\"", + querystr == NULL ? "(empty)" : querystr))); + LOGIF(LT, (skygw_log_write(LOGFILE_TRACE, + "Packet type\t%s", + STRPACKETTYPE(packet_type)))); + + switch (qtype) { + case QUERY_TYPE_WRITE: + LOGIF(LT, (skygw_log_write( + LOGFILE_TRACE, + "%lu [routeQuery:rwsplit] Query type\t%s, " + "routing to Master.", + pthread_self(), + STRQTYPE(qtype)))); + + LOGIF(LT, tracelog_routed_query(router_cli_ses, + "routeQuery", + master_dcb, + gwbuf_clone(stmtbuf))); + + ret = master_dcb->func.write(master_dcb, stmtbuf); + atomic_add(&inst->stats.n_master, 1); + + goto return_ret; + break; + + case QUERY_TYPE_READ: + LOGIF(LT, (skygw_log_write( + LOGFILE_TRACE, + "%lu [routeQuery:rwsplit] Query type\t%s, " + "routing to Slave.", + pthread_self(), + STRQTYPE(qtype)))); + /** Lock router session */ + if (!rses_begin_locked_router_action(router_cli_ses)) + { + /** Log error to debug */ + goto return_ret; + } + /** + * If session command is being executed in slave + * route to master + */ + if (sescmd_cursor_is_active(rses_get_sescmd_cursor( + router_cli_ses, + BE_SLAVE))) + { + LOGIF(LT, tracelog_routed_query(router_cli_ses, + "routeQuery", + master_dcb, + gwbuf_clone(stmtbuf))); + + ret = master_dcb->func.write(master_dcb, stmtbuf); + atomic_add(&inst->stats.n_master, 1); + } + else + { + LOGIF(LT, tracelog_routed_query(router_cli_ses, + "routeQuery", + slave_dcb, + gwbuf_clone(stmtbuf))); + + ret = slave_dcb->func.write(slave_dcb, stmtbuf); + atomic_add(&inst->stats.n_slave, 1); + } + rses_end_locked_router_action(router_cli_ses); + goto return_ret; break; - case COM_CHANGE_USER: - master_dcb->func.auth( - master_dcb, - NULL, - master_dcb->session, - querybuf); - slave_dcb->func.auth( - slave_dcb, - NULL, - master_dcb->session, - bufcopy); - break; + case QUERY_TYPE_SESSION_WRITE: + /** + * Execute in backends used by current router session. + * Save session variable commands to router session property + * struct. Thus, they + * can be replayed in backends which are started and joined later. + * + * Suppress OK packets sent to MaxScale by slaves. + * + * DOES THIS ALL APPLY TO COM_QUIT AS WELL?? + * + */ - case COM_QUERY: - /** - * 1. Create new property of type RSES_PROP_TYPE_SESCMD. - * 2. Add property to the ROUTER_CLIENT_SES struct of - * this router session. - * 3. For each backend, and for each non-executed - * sescmd: - * call execution of current sescmd in - * all backends as long as both have executed - * them all. - * Execution call is dcb->func.session. - * All sescmds are executed when its return value is - * NULL, otherwise it is a pointer to next property. - */ - prop = rses_property_init(RSES_PROP_TYPE_SESCMD); - /** - * Additional reference is created to querybuf to - * prevent it from being released before properties - * are cleaned up as a part of router sessionclean-up. - */ - mysql_sescmd_init(prop, - gwbuf_clone(querybuf), - router_cli_ses); - - /** Lock router session */ - if (!rses_begin_locked_router_action(router_cli_ses)) - { - rses_property_done(prop); - goto return_ret; - } - /** Add sescmd property to router client session */ - rses_property_add(router_cli_ses, prop); - - /** Execute session command in master */ - if (!execute_sescmd_in_backend(router_cli_ses, BE_MASTER)) - { - /** Log error */ - } - /** Execute session command in slave */ - if (!execute_sescmd_in_backend(router_cli_ses, BE_SLAVE)) - { - /** Log error */ - } - - /** Unlock router session */ - rses_end_locked_router_action(router_cli_ses); + /** + * Update connections which are used in this session. + * + * For each connection updated, add a flag which indicates that + * OK Packet must arrive for this command before server + * in question is allowed to be used by router. That is, + * maintain a queue of pending OK packets and remove item + * from queue by FIFO. + * + * Return when the master responds OK Packet. Send that + * OK packet back to client. + * + * Suppress OK packets sent to MaxScale by slaves. + * + * Open questions: + * How to handle interleaving session write + * and queries? It would be simple if OK must be received + * from all/both servers before continuing query execution. + * How to maintain the order of operations? Execution queue + * would solve the problem. In the queue some things must be + * executed in serialized manner while some could be executed + * in parallel. Queries mostly. + * + * Instead of waiting for the OK packet from the master, the + * first OK packet could also be sent to client. TBD. + * vraa 9.12.13 + * + */ + LOGIF(LT, (skygw_log_write( + LOGFILE_TRACE, + "%lu [routeQuery:rwsplit] DCB M:%p s:%p, " + "Query type\t%s, " + "packet type %s, routing to all servers.", + pthread_self(), + master_dcb, + slave_dcb, + STRQTYPE(qtype), + STRPACKETTYPE(packet_type)))); + + switch(packet_type) { + /** + case COM_QUIT: + ret = master_dcb->func.write(master_dcb, gwbuf_clone(querybuf)); + slave_dcb->func.write(slave_dcb, querybuf); + break; + */ + case COM_CHANGE_USER: + + LOGIF(LT, tracelog_routed_query( + router_cli_ses, + "routeQuery", + master_dcb, + gwbuf_clone(stmtbuf))); + + master_dcb->func.auth( + master_dcb, + NULL, + master_dcb->session, + gwbuf_clone(stmtbuf)); + + LOGIF(LT, tracelog_routed_query( + router_cli_ses, + "routeQuery", + slave_dcb, + gwbuf_clone(stmtbuf))); + + slave_dcb->func.auth( + slave_dcb, + NULL, + master_dcb->session, + stmtbuf); + break; + + case COM_QUIT: + case COM_QUERY: + /** + * 1. Create new property of type RSES_PROP_TYPE_SESCMD. + * 2. Add property to the ROUTER_CLIENT_SES struct of + * this router session. + * 3. For each backend, and for each non-executed + * sescmd: + * call execution of current sescmd in + * all backends as long as both have executed + * them all. + * Execution call is dcb->func.session. + * All sescmds are executed when its return value is + * NULL, otherwise it is a pointer to next property. + */ + prop = rses_property_init(RSES_PROP_TYPE_SESCMD); + /** + * Additional reference is created to querybuf to + * prevent it from being released before properties + * are cleaned up as a part of router sessionclean-up. + */ + mysql_sescmd_init(prop, stmtbuf, router_cli_ses); + + /** Lock router session */ + if (!rses_begin_locked_router_action(router_cli_ses)) + { + rses_property_done(prop); + goto return_ret; + } + /** Add sescmd property to router client session */ + rses_property_add(router_cli_ses, prop); + + /** Execute session command in master */ + if (execute_sescmd_in_backend(router_cli_ses, BE_MASTER)) + { + ret = 1; + } + else + { + /** Log error */ + } + /** Execute session command in slave */ + if (execute_sescmd_in_backend(router_cli_ses, BE_SLAVE)) + { + ret = 1; + } + else + { + /** Log error */ + } + + /** Unlock router session */ + rses_end_locked_router_action(router_cli_ses); + break; + + default: + LOGIF(LT, tracelog_routed_query(router_cli_ses, + "routeQuery", + master_dcb, + gwbuf_clone(stmtbuf))); + ret = master_dcb->func.write(master_dcb, + (void *)gwbuf_clone(stmtbuf)); + + LOGIF(LT, tracelog_routed_query(router_cli_ses, + "routeQuery", + slave_dcb, + gwbuf_clone(stmtbuf))); + + slave_dcb->func.write(slave_dcb, (void *)stmtbuf); + break; + } /**< switch by packet type */ + + atomic_add(&inst->stats.n_all, 1); + goto return_ret; break; default: - ret = master_dcb->func.session(master_dcb, - (void *)querybuf); - slave_dcb->func.session(slave_dcb, (void *)bufcopy); + LOGIF(LT, (skygw_log_write( + LOGFILE_TRACE, + "%lu [routeQuery:rwsplit] Query type\t%s, " + "routing to Master by default.", + pthread_self(), + STRQTYPE(qtype)))); + + /** + * Is this really ok? + * What is not known is routed to master. + */ + LOGIF(LT, tracelog_routed_query(router_cli_ses, + "routeQuery", + master_dcb, + gwbuf_clone(stmtbuf))); + + ret = master_dcb->func.write(master_dcb, stmtbuf); + atomic_add(&inst->stats.n_master, 1); + goto return_ret; break; - } /**< switch by packet type */ + } /*< switch by query type */ - atomic_add(&inst->stats.n_all, 1); - goto return_ret; - break; - - default: - LOGIF(LT, (skygw_log_write( - LOGFILE_TRACE, - "%lu [routeQuery:rwsplit] Query type\t%s, " - "routing to Master by default.", - pthread_self(), - STRQTYPE(qtype)))); + /** get next stmt */ + if (!rses_begin_locked_router_action(router_cli_ses)) + { + goto return_ret; + } + master_dcb = router_cli_ses->rses_dcb[BE_MASTER]; - /** - * Is this really ok? - * What is not known is routed to master. - */ - ret = master_dcb->func.write(master_dcb, querybuf); - atomic_add(&inst->stats.n_master, 1); - goto return_ret; - break; - } /**< switch by query type */ - + /** stmtbuf is clone of querybuf, and only covers one stmt */ + stmtbuf = (GWBUF *)master_dcb->session->client->func.getstmt((void *)querybuf); + rses_end_locked_router_action(router_cli_ses); + } /* while (stmtbuf != NULL) */ return_ret: free(querystr); return ret; @@ -984,16 +1082,20 @@ static void clientReply( */ if (!rses_begin_locked_router_action(router_cli_ses)) { - goto lock_failed; + /** is this needed ??*/ + gwbuf_consume(writebuf, gwbuf_length(writebuf)); + goto lock_failed; } master_dcb = router_cli_ses->rses_dcb[BE_MASTER]; slave_dcb = router_cli_ses->rses_dcb[BE_SLAVE]; - - /** Unlock */ - rses_end_locked_router_action(router_cli_ses); - + + /** Holding lock ensures that router session remains open */ + ss_dassert(backend_dcb->session != NULL); client_dcb = backend_dcb->session->client; + /** Unlock */ + rses_end_locked_router_action(router_cli_ses); + /** * 1. Check if backend received reply to sescmd. * 2. Check sescmd's state whether OK_PACKET has been @@ -1018,27 +1120,23 @@ static void clientReply( { be_type = BE_SLAVE; } - scur = rses_get_sescmd_cursor(router_cli_ses, be_type); - ss_dassert(writebuf == sescmd_cursor_get_querybuf(scur)); - + /** Lock router session */ + if (!rses_begin_locked_router_action(router_cli_ses)) + { + /** Log to debug that router was closed */ + goto lock_failed; + } + scur = rses_get_sescmd_cursor(router_cli_ses, be_type); /** * Active cursor means that reply is from session command * execution. */ if (sescmd_cursor_is_active(scur)) { - mysql_sescmd_t* scmd = sescmd_cursor_get_command(scur); - - sescmd_reply_to_client(client_dcb, scmd); - - /** - * If there is a pending sescmd, start its execution. - */ - if (!rses_begin_locked_router_action(router_cli_ses)) - { - goto lock_failed; - } - /** Start execution of all pending ses commands. */ + mysql_sescmd_t* scmd = sescmd_cursor_get_command(scur); + sescmd_reply_to_client(client_dcb, scmd, writebuf); + + /** Read next sescmd property */ while (sescmd_cursor_next(scur)) { if (!cont_exec_sescmd_in_backend(router_cli_ses, be_type)) @@ -1050,8 +1148,26 @@ static void clientReply( /** Log execution of pending sescmd */ } } + /** Set cursor passive. */ + sescmd_cursor_set_active(scur, false); + + /** Unlock router session */ rses_end_locked_router_action(router_cli_ses); } + else if (client_dcb != NULL) + { + /** Write reply to client DCB */ + client_dcb->func.write(client_dcb, writebuf); + /** Unlock router session */ + rses_end_locked_router_action(router_cli_ses); + LOGIF(LT, (skygw_log_write_flush( + LOGFILE_TRACE, + "%lu [clientReply:rwsplit] client dcb %p, " + "backend dcb %p. End of normal reply.", + pthread_self(), + client_dcb, + backend_dcb))); + } return; /*< succeed */ lock_failed: /** log that router session couldn't be locked */ @@ -1230,7 +1346,7 @@ static rses_property_t* rses_property_init( { goto return_prop; } - spinlock_init(&prop->rses_prop_lock); +// spinlock_init(&prop->rses_prop_lock); prop->rses_prop_type = prop_type; #if defined(SS_DEBUG) prop->rses_prop_chk_top = CHK_NUM_ROUTER_PROPERTY; @@ -1242,29 +1358,6 @@ return_prop: return prop; } -/** - * Create session command property. - */ -static mysql_sescmd_t* mysql_sescmd_init ( - rses_property_t* rses_prop, - GWBUF* sescmd_buf, - ROUTER_CLIENT_SES* rses) -{ - mysql_sescmd_t* sescmd; - - CHK_RSES_PROP(rses_prop); - sescmd = &rses_prop->rses_prop_data.sescmd; - sescmd->my_sescmd_prop = rses_prop; /*< reference to owning property */ -#if defined(SS_DEBUG) - sescmd->my_sescmd_chk_top = CHK_NUM_MY_SESCMD; - sescmd->my_sescmd_chk_tail = CHK_NUM_MY_SESCMD; -#endif - sescmd->my_sescmd_buf = sescmd_buf; /*< session command query */ - ss_dassert(sescmd_buf->sbuf->refcount > 0); - - return sescmd; -} - /** * Property is freed at the end of router client session. */ @@ -1275,7 +1368,7 @@ static void rses_property_done( switch (prop->rses_prop_type) { case RSES_PROP_TYPE_SESCMD: - mysql_sescmd_done(&prop->rses_prop_data.sescmd); + mysql_sescmd_done(&prop->rses_prop_data.sescmd); break; default: LOGIF(LD, (skygw_log_write_flush( @@ -1292,15 +1385,6 @@ static void rses_property_done( free(prop); } - -static void mysql_sescmd_done( - mysql_sescmd_t* sescmd) -{ - CHK_RSES_PROP(sescmd->my_sescmd_prop); - gwbuf_free(sescmd->my_sescmd_buf); - memset(sescmd, 0, sizeof(mysql_sescmd_t)); -} - /** * Add property to the router_client_ses structure's rses_properties * array. The slot is determined by the type of property. @@ -1309,86 +1393,165 @@ static void mysql_sescmd_done( * Router client session must be locked. */ static void rses_property_add( - ROUTER_CLIENT_SES* rses, - rses_property_t* prop) + ROUTER_CLIENT_SES* rses, + rses_property_t* prop) { - rses_property_t* p; - - CHK_CLIENT_RSES(rses); - CHK_RSES_PROP(prop); - ss_dassert(rses->rses_lock.lock != 0); - p = rses->rses_properties[prop->rses_prop_type]; - - if (p == NULL) - { - rses->rses_properties[prop->rses_prop_type] = prop; - } - else - { - while (p->rses_prop_next != NULL) - { - p = p->rses_prop_next; - } - p->rses_prop_next = prop; - } + rses_property_t* p; + + CHK_CLIENT_RSES(rses); + CHK_RSES_PROP(prop); + ss_dassert(SPINLOCK_IS_LOCKED(&rses->rses_lock)); + + prop->rses_prop_rsession = rses; + p = rses->rses_properties[prop->rses_prop_type]; + + if (p == NULL) + { + rses->rses_properties[prop->rses_prop_type] = prop; + } + else + { + while (p->rses_prop_next != NULL) + { + p = p->rses_prop_next; + } + p->rses_prop_next = prop; + } } -static void rses_begin_locked_property_action( - rses_property_t* prop) +/** Router sessiosn must be locked */ +static mysql_sescmd_t* rses_property_get_sescmd( + rses_property_t* prop) { - CHK_RSES_PROP(prop); - spinlock_acquire(&prop->rses_prop_lock); + mysql_sescmd_t* sescmd; + + CHK_RSES_PROP(prop); + ss_dassert(prop->rses_prop_rsession == NULL || + SPINLOCK_IS_LOCKED(&prop->rses_prop_rsession->rses_lock)); + + sescmd = &prop->rses_prop_data.sescmd; + + if (sescmd != NULL) + { + CHK_MYSQL_SESCMD(sescmd); + } + return sescmd; +} + +// static rses_property_t* rses_property_get_ptr_next( + +/** +static void rses_begin_locked_property_action( + rses_property_t* prop) +{ + CHK_RSES_PROP(prop); + spinlock_acquire(&prop->rses_prop_lock); } static void rses_end_locked_property_action( - rses_property_t* prop) + rses_property_t* prop) { - CHK_RSES_PROP(prop); - spinlock_release(&prop->rses_prop_lock); + CHK_RSES_PROP(prop); + spinlock_release(&prop->rses_prop_lock); +} +*/ + +/** + * Create session command property. + */ +static mysql_sescmd_t* mysql_sescmd_init ( + rses_property_t* rses_prop, + GWBUF* sescmd_buf, + ROUTER_CLIENT_SES* rses) +{ + mysql_sescmd_t* sescmd; + + CHK_RSES_PROP(rses_prop); + /** Can't call rses_property_get_sescmd with uninitialized sescmd */ + sescmd = &rses_prop->rses_prop_data.sescmd; + sescmd->my_sescmd_prop = rses_prop; /*< reference to owning property */ +// sescmd->my_sescmd_rsession = rses; +#if defined(SS_DEBUG) + sescmd->my_sescmd_chk_top = CHK_NUM_MY_SESCMD; + sescmd->my_sescmd_chk_tail = CHK_NUM_MY_SESCMD; +#endif + /** Set session command buffer */ + sescmd->my_sescmd_buf = sescmd_buf; + + return sescmd; } -/** router must be locked */ -static void sescmd_cursor_set_active( - sescmd_cursor_t* sescmd_cursor, - bool value) +static void mysql_sescmd_done( + mysql_sescmd_t* sescmd) { - ss_dassert(SPINLOCK_IS_LOCKED(&(*sescmd_cursor->scmd_cur_ptr_property)->rses_prop_lock)); - /** avoid calling unnecessarily */ - ss_dassert(sescmd_cursor->scmd_cur_active != value); - sescmd_cursor->scmd_cur_active = value; + CHK_RSES_PROP(sescmd->my_sescmd_prop); + gwbuf_free(sescmd->my_sescmd_buf); + memset(sescmd, 0, sizeof(mysql_sescmd_t)); } -static void sescmd_reply_to_client( + +/** + * Write session command reply from backend to client if command haven't yet + * been replied. + * Return true if succeed, false if command was already replied. + * + * Router session must be locked */ +static bool sescmd_reply_to_client( DCB* client_dcb, - mysql_sescmd_t* scmd) + mysql_sescmd_t* scmd, + GWBUF* writebuf) { - rses_property_t* prop; + bool succp = false; + + // rses_property_t* prop; CHK_DCB(client_dcb); CHK_MYSQL_SESCMD(scmd); - CHK_GWBUF(scmd->my_sescmd_buf); + CHK_GWBUF(writebuf); + ss_dassert(SPINLOCK_IS_LOCKED( + &scmd->my_sescmd_prop->rses_prop_rsession->rses_lock)); - prop = mysql_sescmd_get_property(scmd); +// prop = mysql_sescmd_get_property(scmd); - rses_begin_locked_property_action(prop); +// rses_begin_locked_property_action(prop); if (!scmd->my_sescmd_is_replied) { - CHK_GWBUF(scmd->my_sescmd_buf); - client_dcb->func.write(client_dcb, scmd->my_sescmd_buf); + client_dcb->func.write(client_dcb, writebuf); scmd->my_sescmd_is_replied = true; + succp = true; + LOGIF(LT, (skygw_log_write( + LOGFILE_TRACE, + "%lu [sescmd_reply_to_client] Replied to client dcb %p.", + pthread_self(), + client_dcb))); } - rses_end_locked_property_action(prop); + else + { + } +// rses_end_locked_property_action(prop); + return succp; } +/** + * Get the address of current session command. + * + * Router session must be locked */ static mysql_sescmd_t* sescmd_cursor_get_command( sescmd_cursor_t* scur) { - mysql_sescmd_t* scmd = scur->scmd_cur_cmd; - CHK_MYSQL_SESCMD(scmd); - + mysql_sescmd_t* scmd; + + ss_dassert(SPINLOCK_IS_LOCKED(&scur->scmd_cur_rses->rses_lock)); + + scur->scmd_cur_cmd = rses_property_get_sescmd(*scur->scmd_cur_ptr_property); + + CHK_MYSQL_SESCMD(scur->scmd_cur_cmd); + + scmd = scur->scmd_cur_cmd; + return scmd; } @@ -1398,6 +1561,8 @@ static sescmd_cursor_t* rses_get_sescmd_cursor( backend_type_t be_type) { CHK_CLIENT_RSES(rses); + ss_dassert(SPINLOCK_IS_LOCKED(&rses->rses_lock)); + return &rses->rses_cursor[be_type]; } @@ -1405,18 +1570,35 @@ static sescmd_cursor_t* rses_get_sescmd_cursor( static bool sescmd_cursor_is_active( sescmd_cursor_t* sescmd_cursor) { - bool succp = sescmd_cursor->scmd_cur_active; + bool succp; + ss_dassert(SPINLOCK_IS_LOCKED(&sescmd_cursor->scmd_cur_rses->rses_lock)); + + succp = sescmd_cursor->scmd_cur_active; return succp; } -/** Router session must be locked */ -static GWBUF* sescmd_cursor_get_querybuf( +/** router must be locked */ +static void sescmd_cursor_set_active( + sescmd_cursor_t* sescmd_cursor, + bool value) +{ + ss_dassert(SPINLOCK_IS_LOCKED(&sescmd_cursor->scmd_cur_rses->rses_lock)); + /** avoid calling unnecessarily */ + ss_dassert(sescmd_cursor->scmd_cur_active != value); + sescmd_cursor->scmd_cur_active = value; +} + +/** + * Clone session command's command buffer. + * Router session must be locked + */ +static GWBUF* sescmd_cursor_clone_querybuf( sescmd_cursor_t* scur) { GWBUF* buf; ss_dassert(scur->scmd_cur_cmd != NULL); - buf = scur->scmd_cur_cmd->my_sescmd_buf; + buf = gwbuf_clone(scur->scmd_cur_cmd->my_sescmd_buf); CHK_GWBUF(buf); return buf; @@ -1447,24 +1629,48 @@ static bool execute_sescmd_in_backend( CHK_CLIENT_RSES(rses); ss_dassert(SPINLOCK_IS_LOCKED(&rses->rses_lock)); + /** + * Get cursor pointer and copy of command buffer to cursor. + */ scur = rses_get_sescmd_cursor(rses, be_type); /** Return if there are no pending ses commands */ - if (scur->scmd_cur_cmd == NULL) + if (sescmd_cursor_get_command(scur) == NULL) { succp = false; + goto return_succp; } if (!sescmd_cursor_is_active(scur)) { + /** Cursor is left active when function returns. */ sescmd_cursor_set_active(scur, true); - rc = dcb->func.session(dcb, sescmd_cursor_get_querybuf(scur)); + + LOGIF(LT, tracelog_routed_query(rses, + "execute_sescmd_in_backend", + dcb, + sescmd_cursor_clone_querybuf(scur))); + rc = dcb->func.write(dcb, sescmd_cursor_clone_querybuf(scur)); +// rc = dcb->func.session(dcb, sescmd_cursor_clone_querybuf(scur)); - if (rc != 0) + if (rc != 1) { succp = false; } } + else + { + LOGIF(LD, (skygw_log_write_flush( + LOGFILE_DEBUG, + "%lu [routeQuery] Couldn't directly send SESSION " + "WRITER command to dcb %p because session command " + "cursor was executing previous command. Added " + "command to the queue.", + pthread_self(), + dcb))); + } + +return_succp: return succp; } @@ -1477,36 +1683,42 @@ static bool execute_sescmd_in_backend( * backend server succeed. Otherwise false. */ static bool cont_exec_sescmd_in_backend( - ROUTER_CLIENT_SES* rses, - backend_type_t be_type) + ROUTER_CLIENT_SES* rses, + backend_type_t be_type) { - bool succp = true; - DCB* dcb; - sescmd_cursor_t* scur; - int rc; - - CHK_CLIENT_RSES(rses); - - dcb = rses->rses_dcb[be_type]; - CHK_DCB(dcb); - - ss_dassert(SPINLOCK_IS_LOCKED(&rses->rses_lock)); - - scur = rses_get_sescmd_cursor(rses, be_type); - - ss_dassert(sescmd_cursor_is_active(scur)); + DCB* dcb; + bool succp = true; + int rc = 0; + sescmd_cursor_t* scur; + + dcb = rses->rses_dcb[be_type]; + + CHK_DCB(dcb); + CHK_CLIENT_RSES(rses); + ss_dassert(SPINLOCK_IS_LOCKED(&rses->rses_lock)); + scur = rses_get_sescmd_cursor(rses, be_type); + ss_dassert(sescmd_cursor_is_active(scur)); + + /** Return if there are no pending ses commands */ if (scur->scmd_cur_cmd == NULL) { succp = false; + goto return_succp; } - rc = dcb->func.session(dcb, sescmd_cursor_get_querybuf(scur)); - - if (rc != 0) + + LOGIF(LT, tracelog_routed_query(rses, + "cont_exec_sescmd_in_backend", + dcb, + sescmd_cursor_clone_querybuf(scur))); + +// rc = dcb->func.session(dcb, sescmd_cursor_clone_querybuf(scur)); + rc = dcb->func.write(dcb, sescmd_cursor_clone_querybuf(scur)); + if (rc != 1) { succp = false; } - +return_succp: return succp; } @@ -1522,46 +1734,59 @@ static bool sescmd_cursor_next( bool succp = false; rses_property_t* prop_curr; rses_property_t* prop_next; - - ss_dassert(SPINLOCK_IS_LOCKED(&(*scur->scmd_cur_ptr_property)->rses_prop_lock)); - + + ss_dassert(scur != NULL); + ss_dassert(*(scur->scmd_cur_ptr_property) != NULL); + ss_dassert(SPINLOCK_IS_LOCKED( + &(*(scur->scmd_cur_ptr_property))->rses_prop_rsession->rses_lock)); + + /** Illegal situation */ if (scur == NULL || - *(scur->scmd_cur_ptr_property) == NULL || + *scur->scmd_cur_ptr_property == NULL || scur->scmd_cur_cmd == NULL) { - /** Log error to debug */ + /** Log error */ goto return_succp; } -#if defined(SS_DEBUG) prop_curr = *(scur->scmd_cur_ptr_property); - prop_next = prop_curr->rses_prop_next; -#endif - CHK_RSES_PROP(prop_curr); - CHK_MYSQL_SESCMD(scur->scmd_cur_cmd); - CHK_RSES_PROP(scur->scmd_cur_cmd->my_sescmd_prop); - ss_dassert(scur->scmd_cur_cmd->my_sescmd_prop == prop_curr); + CHK_MYSQL_SESCMD(scur->scmd_cur_cmd); + ss_dassert(prop_curr == mysql_sescmd_get_property(scur->scmd_cur_cmd)); + CHK_RSES_PROP(prop_curr); + + /** Copy address of pointer to next property */ + scur->scmd_cur_ptr_property = &(prop_curr->rses_prop_next); + prop_next = *scur->scmd_cur_ptr_property; + ss_dassert(prop_next == *(scur->scmd_cur_ptr_property)); + + /** If there is a next property move forward */ - if ((*scur->scmd_cur_ptr_property)->rses_prop_next != NULL) + if (prop_next != NULL) { - scur->scmd_cur_ptr_property = - &((*scur->scmd_cur_ptr_property)->rses_prop_next); - scur->scmd_cur_cmd = - &(*scur->scmd_cur_ptr_property)->rses_prop_data.sescmd; + CHK_RSES_PROP(prop_next); + CHK_RSES_PROP((*(scur->scmd_cur_ptr_property))); + + /** Get pointer to next property's sescmd */ + scur->scmd_cur_cmd = rses_property_get_sescmd(prop_next); + + ss_dassert(prop_next == scur->scmd_cur_cmd->my_sescmd_prop); + CHK_MYSQL_SESCMD(scur->scmd_cur_cmd); + CHK_RSES_PROP(scur->scmd_cur_cmd->my_sescmd_prop); } else { /** No more properties, can't proceed. */ goto return_succp; } - CHK_RSES_PROP((*(scur->scmd_cur_ptr_property))); - CHK_MYSQL_SESCMD(scur->scmd_cur_cmd); - ss_dassert(prop_next == *(scur->scmd_cur_ptr_property)); if (scur->scmd_cur_cmd != NULL) { - succp = true; - } + succp = true; + } + else + { + /** Log error, sescmd shouldn't be NULL */ + } return_succp: return succp; } @@ -1572,3 +1797,63 @@ static rses_property_t* mysql_sescmd_get_property( CHK_MYSQL_SESCMD(scmd); return scmd->my_sescmd_prop; } + + +static void tracelog_routed_query( + ROUTER_CLIENT_SES* rses, + char* funcname, + DCB* dcb, + GWBUF* buf) +{ + unsigned char* packet = GWBUF_DATA(buf); + unsigned char packet_type = packet[4]; + size_t len; + size_t buflen = GWBUF_LENGTH(buf); + char* querystr; + char* startpos = (char *)&packet[5]; + backend_type_t be_type; + + len = packet[0]; + len += 255*packet[1]; + len += 255*255*packet[2]; + + if (rses->rses_dcb[BE_MASTER] == dcb) + { + be_type = BE_MASTER; + } + else if (rses->rses_dcb[BE_SLAVE] == dcb) + { + be_type = BE_SLAVE; + } + else + { + be_type = BE_UNDEFINED; + } + + if (packet_type == '\x03') + { + querystr = (char *)malloc(len); + memcpy(querystr, startpos, len-1); + querystr[len-1] = '\0'; + LOGIF(LT, (skygw_log_write_flush( + LOGFILE_TRACE, + "%lu [%s] %d bytes long buf, \"%s\" -> %s:%d %s dcb %p", + pthread_self(), + funcname, + buflen, + querystr, + (be_type == BE_MASTER ? + rses->rses_backend[BE_MASTER]->backend_server->name : + (be_type == BE_SLAVE ? + rses->rses_backend[BE_SLAVE]->backend_server->name : + "Target DCB is neither of the backends. This is error")), + (be_type == BE_MASTER ? + rses->rses_backend[BE_MASTER]->backend_server->port : + (be_type == BE_SLAVE ? + rses->rses_backend[BE_SLAVE]->backend_server->port : + -1)), + STRBETYPE(be_type), + dcb))); + } + gwbuf_free(buf); +} \ No newline at end of file diff --git a/utils/skygw_debug.h b/utils/skygw_debug.h index 508e0f28d..8c11d9782 100644 --- a/utils/skygw_debug.h +++ b/utils/skygw_debug.h @@ -216,7 +216,11 @@ typedef enum skygw_chk_t { ((r) == DCB_ROLE_REQUEST_HANDLER ? "DCB_ROLE_REQUEST_HANDLER" : \ "UNKNOWN DCB ROLE")) - +#define STRBETYPE(t) ((t) == BE_MASTER ? "BE_MASTER" : \ + ((t) == BE_SLAVE ? "BE_SLAVE" : \ + ((t) == BE_UNDEFINED ? "BE_UNDEFINED" : \ + "Unknown backend tpe"))) + #define CHK_MLIST(l) { \ ss_info_dassert((l->mlist_chk_top == CHK_NUM_MLIST && \ l->mlist_chk_tail == CHK_NUM_MLIST), \ From cb6a976555509236f3776d5ddd85324f0a7cd9ef Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Tue, 11 Mar 2014 23:12:11 +0200 Subject: [PATCH 3/6] Router has now capability value which currently tells whether router session expects stream or individual, complete statements. With read con nection router stream is used and with read/write split router individual statements are passed to router. Added new function to ROUTER_OBJECT : uint8_t (*getCapabilities)(ROUTER *instance, void* router_session); which is implemented in every route r. Added support for multi-statement packets in rwsplit router. In other words, if network packet includes multiple mysql statements, they are separated and passed to router one by one. Multi-packet statements (those which exceeds network packet boundaries) are _not_ supported yet. --- server/core/buffer.c | 82 ++ server/core/dcb.c | 16 +- server/include/buffer.h | 18 +- server/include/dcb.h | 1 - server/include/router.h | 10 +- server/modules/include/readconnection.h | 4 +- server/modules/include/readwritesplit.h | 3 +- server/modules/protocol/mysql_backend.c | 4 +- server/modules/protocol/mysql_client.c | 201 ++++- server/modules/routing/debugcli.c | 11 +- server/modules/routing/readconnroute.c | 17 +- .../routing/readwritesplit/readwritesplit.c | 745 +++++++++--------- server/modules/routing/testroute.c | 11 +- 13 files changed, 701 insertions(+), 422 deletions(-) diff --git a/server/core/buffer.c b/server/core/buffer.c index e3c649f69..57228b253 100644 --- a/server/core/buffer.c +++ b/server/core/buffer.c @@ -81,6 +81,7 @@ SHARED_BUF *sbuf; sbuf->refcount = 1; rval->sbuf = sbuf; rval->next = NULL; + rval->gwbuf_type = GWBUF_TYPE_UNDEFINED; rval->command = 0; CHK_GWBUF(rval); return rval; @@ -127,6 +128,7 @@ GWBUF *rval; rval->sbuf = buf->sbuf; rval->start = buf->start; rval->end = buf->end; + rval->gwbuf_type = buf->gwbuf_type; rval->next = NULL; // rval->command = buf->command; CHK_GWBUF(rval); @@ -152,12 +154,67 @@ GWBUF *gwbuf_clone_portion( clonebuf->sbuf = buf->sbuf; clonebuf->start = (void *)((char*)buf->start)+start_offset; clonebuf->end = (void *)((char *)clonebuf->start)+length; + clonebuf->gwbuf_type = buf->gwbuf_type; /*< clone the type for now */ clonebuf->next = NULL; CHK_GWBUF(clonebuf); return clonebuf; } +/** + * Returns pointer to GWBUF of a requested type. + * As of 10.3.14 only MySQL to plain text conversion is supported. + * Return NULL if conversion between types is not supported or due lacking + * type information. + */ +GWBUF *gwbuf_clone_transform( + GWBUF * head, + gwbuf_type_t targettype) +{ + gwbuf_type_t src_type; + GWBUF* clonebuf; + + CHK_GWBUF(head); + src_type = head->gwbuf_type; + + if (targettype == GWBUF_TYPE_UNDEFINED || + src_type == GWBUF_TYPE_UNDEFINED || + src_type == GWBUF_TYPE_PLAINSQL || + targettype == src_type) + { + clonebuf = NULL; + goto return_clonebuf; + } + + switch (src_type) + { + case GWBUF_TYPE_MYSQL: + if (targettype == GWBUF_TYPE_PLAINSQL) + { + /** Crete reference to string part of buffer */ + clonebuf = gwbuf_clone_portion( + head, + 5, + GWBUF_LENGTH(head)-5); + ss_dassert(clonebuf != NULL); + /** Overwrite the type with new format */ + clonebuf->gwbuf_type = targettype; + } + else + { + clonebuf = NULL; + } + break; + + default: + clonebuf = NULL; + break; + } /*< switch (src_type) */ + +return_clonebuf: + return clonebuf; +} + /** * Append a buffer onto a linked list of buffer structures. @@ -234,3 +291,28 @@ int rval = 0; } return rval; } + +bool gwbuf_set_type( + GWBUF* buf, + gwbuf_type_t type) +{ + bool succp; + CHK_GWBUF(buf); + + switch (type) { + case GWBUF_TYPE_MYSQL: + case GWBUF_TYPE_PLAINSQL: + case GWBUF_TYPE_UNDEFINED: + buf->gwbuf_type = type; + succp = true; + break; + default: + succp = false; + break; + } + ss_dassert(succp); + return succp; +} + + + diff --git a/server/core/dcb.c b/server/core/dcb.c index fcf2e0cf2..db9e96b60 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -1357,19 +1357,19 @@ int gw_write( w = write(fd, buf, nbytes); #endif /* SS_DEBUG && SS_TEST */ -#if defined(SS_DEBUG) +#if defined(SS_DEBUG_MYSQL) { - size_t len; - unsigned char* packet = (unsigned char *)buf; - char* str; + size_t len; + uint8_t* packet = (uint8_t *)buf; + char* str; /** Print only MySQL packets */ if (w > 5) { str = (char *)&packet[5]; len = packet[0]; - len += 255*packet[1]; - len += 255*255*packet[2]; + len += 256*packet[1]; + len += 256*256*packet[2]; if (strncmp(str, "insert", 6) == 0 || strncmp(str, "create", 6) == 0 || @@ -1385,8 +1385,8 @@ int gw_write( if (nbytes-5 > len) { size_t len2 = packet[4+len]; - len2 += 255*packet[4+len+1]; - len2 += 255*255*packet[4+len+2]; + len2 += 256*packet[4+len+1]; + len2 += 256*256*packet[4+len+2]; char* str2 = (char *)&packet[4+len+5]; snprintf(s, 5+len+len2, "long %s %s", (char *)str, (char *)str2); diff --git a/server/include/buffer.h b/server/include/buffer.h index de5a13148..53c81a4a6 100644 --- a/server/include/buffer.h +++ b/server/include/buffer.h @@ -41,9 +41,18 @@ * * @endverbatim */ +#include + + +typedef enum +{ + GWBUF_TYPE_UNDEFINED = 0x0, + GWBUF_TYPE_PLAINSQL = 0x1, + GWBUF_TYPE_MYSQL = 0x2 +} gwbuf_type_t; /** - * A structure to encapsualte the data in a form that the data itself can be + * A structure to encapsulate the data in a form that the data itself can be * shared between multiple GWBUF's without the need to make multiple copies * but still maintain separate data pointers. */ @@ -64,8 +73,9 @@ typedef struct gwbuf { struct gwbuf *next; /*< Next buffer in a linked chain of buffers */ void *start; /*< Start of the valid data */ void *end; /*< First byte after the valid data */ - SHARED_BUF *sbuf; /*< The shared buffer with the real data */ + SHARED_BUF *sbuf; /*< The shared buffer with the real data */ int command;/*< The command type for the queue */ + gwbuf_type_t gwbuf_type; /*< buffer's data type information */ } GWBUF; /*< @@ -83,6 +93,7 @@ typedef struct gwbuf { /*< Consume a number of bytes in the buffer */ #define GWBUF_CONSUME(b, bytes) (b)->start += bytes +#define GWBUF_TYPE(b) (b)->gwbuf_type /*< * Function prototypes for the API to maniplate the buffers */ @@ -93,5 +104,6 @@ extern GWBUF *gwbuf_append(GWBUF *head, GWBUF *tail); extern GWBUF *gwbuf_consume(GWBUF *head, unsigned int length); extern unsigned int gwbuf_length(GWBUF *head); extern GWBUF *gwbuf_clone_portion(GWBUF *head, size_t offset, size_t len); - +extern GWBUF *gwbuf_clone_transform(GWBUF *head, gwbuf_type_t type); +extern bool gwbuf_set_type(GWBUF *head, gwbuf_type_t type); #endif diff --git a/server/include/dcb.h b/server/include/dcb.h index cacf6527e..29ab643be 100644 --- a/server/include/dcb.h +++ b/server/include/dcb.h @@ -87,7 +87,6 @@ typedef struct gw_protocol { int (*listen)(struct dcb *, char *); int (*auth)(struct dcb *, struct server *, struct session *, GWBUF *); int (*session)(struct dcb *, void *); - void* (*getstmt)(void* buf); } GWPROTOCOL; /** diff --git a/server/include/router.h b/server/include/router.h index 9c93f3271..004f91d51 100644 --- a/server/include/router.h +++ b/server/include/router.h @@ -35,6 +35,7 @@ #include #include #include +#include /** * The ROUTER handle points to module specific data, so the best we can do @@ -74,10 +75,13 @@ typedef struct router_object { void (*diagnostics)(ROUTER *instance, DCB *dcb); void (*clientReply)(ROUTER* instance, void* router_session, GWBUF* queue, DCB *backend_dcb); void (*errorReply)(ROUTER* instance, void* router_session, char* message, DCB *backend_dcb, int action); + uint8_t (*getCapabilities)(ROUTER *instance, void* router_session); } ROUTER_OBJECT; -/* Router commands */ -#define ROUTER_DEFAULT 0 /**< Standard routing */ -#define ROUTER_CHANGE_SESSION 1 /**< Route a change session */ +typedef enum router_capability_t { + RCAP_TYPE_UNDEFINED = 0, + RCAP_TYPE_STMT_INPUT = (1 << 0), + RCAP_TYPE_PACKET_INPUT = (1 << 1) +} router_capability_t; #endif diff --git a/server/modules/include/readconnection.h b/server/modules/include/readconnection.h index 70f6de9cd..c6f19995c 100644 --- a/server/modules/include/readconnection.h +++ b/server/modules/include/readconnection.h @@ -53,8 +53,8 @@ typedef struct router_client_session { bool rses_closed; /*< true when closeSession is called */ BACKEND *backend; /*< Backend used by the client session */ DCB *backend_dcb; /*< DCB Connection to the backend */ - struct router_client_session - *next; + struct router_client_session *next; + int rses_capabilities; /*< input type, for example */ #if defined(SS_DEBUG) skygw_chk_t rses_chk_tail; #endif diff --git a/server/modules/include/readwritesplit.h b/server/modules/include/readwritesplit.h index 95b414958..c5dcbdaaa 100644 --- a/server/modules/include/readwritesplit.h +++ b/server/modules/include/readwritesplit.h @@ -66,7 +66,6 @@ typedef struct mysql_sescmd_st { #if defined(SS_DEBUG) skygw_chk_t my_sescmd_chk_top; #endif -// ROUTER_CLIENT_SES* my_sescmd_rsession; /*< parent router session */ rses_property_t* my_sescmd_prop; /*< parent property */ GWBUF* my_sescmd_buf; /*< query buffer */ bool my_sescmd_is_replied; /*< is cmd replied to client */ @@ -84,7 +83,6 @@ struct rses_property_st { skygw_chk_t rses_prop_chk_top; #endif ROUTER_CLIENT_SES* rses_prop_rsession; /*< parent router session */ -// SPINLOCK rses_prop_lock; /*< protect property content */ int rses_prop_refcount; rses_property_type_t rses_prop_type; union rses_prop_data { @@ -121,6 +119,7 @@ struct router_client_session { DCB* rses_dcb[BE_COUNT]; /*< cursor is pointer and status variable to current session command */ sescmd_cursor_t rses_cursor[BE_COUNT]; + int rses_capabilities; /*< input type, for example */ struct router_client_session* next; #if defined(SS_DEBUG) skygw_chk_t rses_chk_tail; diff --git a/server/modules/protocol/mysql_backend.c b/server/modules/protocol/mysql_backend.c index 06d6e3a85..03e433062 100644 --- a/server/modules/protocol/mysql_backend.c +++ b/server/modules/protocol/mysql_backend.c @@ -856,7 +856,7 @@ static int gw_change_user(DCB *backend, SERVER *server, SESSION *in_session, GWB backend_protocol = backend->protocol; client_protocol = in_session->client->protocol; - queue->command = ROUTER_CHANGE_SESSION; +// queue->command = ROUTER_CHANGE_SESSION; // now get the user, after 4 bytes header and 1 byte command client_auth_packet += 5; @@ -939,7 +939,7 @@ static int gw_session(DCB *backend_dcb, void *data) { GWBUF *queue = NULL; queue = (GWBUF *) data; - queue->command = ROUTER_CHANGE_SESSION; +// queue->command = ROUTER_CHANGE_SESSION; backend_dcb->func.write(backend_dcb, queue); return 1; diff --git a/server/modules/protocol/mysql_client.c b/server/modules/protocol/mysql_client.c index c2f50a76a..255ede201 100644 --- a/server/modules/protocol/mysql_client.c +++ b/server/modules/protocol/mysql_client.c @@ -48,11 +48,16 @@ static int gw_MySQLWrite_client(DCB *dcb, GWBUF *queue); static int gw_error_client_event(DCB *dcb); static int gw_client_close(DCB *dcb); static int gw_client_hangup_event(DCB *dcb); -static void* gw_MySQL_get_next_stmt(void* buffer); int mysql_send_ok(DCB *dcb, int packet_number, int in_affected_rows, const char* mysql_message); int MySQLSendHandshake(DCB* dcb); static int gw_mysql_do_authentication(DCB *dcb, GWBUF *queue); +static int route_by_statement( + ROUTER* router_instance, + ROUTER_OBJECT* router, + void* rsession, + GWBUF* read_buf); +static GWBUF* gw_MySQL_get_next_stmt(GWBUF** buffer); /* * The "module object" for the mysqld client protocol module. @@ -68,8 +73,7 @@ static GWPROTOCOL MyObject = { gw_client_close, /* Close */ gw_MySQLListener, /* Listen */ NULL, /* Authentication */ - NULL, /* Session */ - gw_MySQL_get_next_stmt /* get single stmt from read buf */ + NULL /* Session */ }; /** @@ -609,9 +613,11 @@ int gw_read_client_event(DCB* dcb) { */ { int len = -1; + uint8_t cap = 0; GWBUF *read_buffer = NULL; uint8_t *ptr_buff = NULL; int mysql_command = -1; + bool stmt_input; /*< router input type */ session = dcb->session; @@ -640,7 +646,7 @@ int gw_read_client_event(DCB* dcb) { /* get mysql commang at fifth byte */ if (ptr_buff) { mysql_command = ptr_buff[4]; - } + } /** * Without rsession there is no access to backend. * COM_QUIT : close client dcb @@ -672,6 +678,32 @@ int gw_read_client_event(DCB* dcb) { read_buffer = gwbuf_consume(read_buffer, len); goto return_rc; } + /** Ask what type of input the router expects */ + cap = router->getCapabilities(router_instance, rsession); + + if (cap == 0 || (cap == RCAP_TYPE_PACKET_INPUT)) + { + stmt_input = false; + } + else if (cap == RCAP_TYPE_STMT_INPUT) + { + stmt_input = true; + /** Mark buffer to as MySQL type */ + gwbuf_set_type(read_buffer, GWBUF_TYPE_MYSQL); + } + else + { + mysql_send_custom_error(dcb, + 1, + 0, + "Reading router capabilities " + "failed. Router session is " + "closed."); + rc = 1; + goto return_rc; + } + + /** Route COM_QUIT to backend */ if (mysql_command == '\x01') { router->routeQuery(router_instance, rsession, read_buffer); @@ -690,10 +722,25 @@ int gw_read_client_event(DCB* dcb) { } else { - /** Route other commands to backend */ - rc = router->routeQuery(router_instance, + if (stmt_input) + { + /** + * Feed each statement completely and separately + * to router. + */ + rc = route_by_statement(router_instance, + router, + rsession, + read_buffer); + } + else + { + /** Feed whole packet to router */ + rc = router->routeQuery(router_instance, rsession, read_buffer); + } + /** succeed */ if (rc == 1) { rc = 0; /**< here '0' means success */ @@ -1213,37 +1260,143 @@ return_rc: * so that it only cover the remaining buffer. * */ -static void* gw_MySQL_get_next_stmt( - void* buffer) +static GWBUF* gw_MySQL_get_next_stmt( + GWBUF** p_readbuf) { - GWBUF* readbuf = (GWBUF *)buffer; GWBUF* stmtbuf; - unsigned char* packet; - size_t len; + size_t buflen; + size_t strlen; + uint8_t* packet; - CHK_GWBUF(readbuf); - - if (GWBUF_EMPTY(readbuf)) + if (*p_readbuf == NULL) { stmtbuf = NULL; goto return_stmtbuf; - } - packet = GWBUF_DATA(readbuf); - len = packet[0]; - len += 255*packet[1]; - len += 255*255*packet[2]; + } + CHK_GWBUF(*p_readbuf); - /** vraa :Multi-packet stmt is not supported as of 7.3.14 */ - if (len+4 > GWBUF_LENGTH(readbuf)) + if (GWBUF_EMPTY(*p_readbuf)) { stmtbuf = NULL; goto return_stmtbuf; } - stmtbuf = gwbuf_clone_portion(readbuf, 0, 4+len); - gwbuf_consume(readbuf, 4+len); + buflen = GWBUF_LENGTH((*p_readbuf)); + packet = GWBUF_DATA((*p_readbuf)); + strlen = MYSQL_GET_PACKET_LEN(packet); + + /** vraa :Multi-packet stmt is not supported as of 7.3.14 */ + if (strlen-1 > buflen-5) + { + stmtbuf = NULL; + goto return_stmtbuf; + } + stmtbuf = gwbuf_clone_portion(*p_readbuf, 0, strlen+4); + *p_readbuf = gwbuf_consume(*p_readbuf, strlen+4); return_stmtbuf: - return (void *)stmtbuf; + return stmtbuf; +} + +/** + * Detect if buffer includes partial mysql packet or multiple packets. + * Store partial packet to pendingqueue. Send complete packets one by one + * to router. + */ +static int route_by_statement( + ROUTER* router_instance, + ROUTER_OBJECT* router, + void* rsession, + GWBUF* readbuf) +{ + int rc = -1; + DCB* master_dcb; + GWBUF* stmtbuf; + uint8_t* payload; + static size_t len; + +#if defined(SS_DEBUG) + uint8_t router_capabilities; + + router_capabilities = router->getCapabilities(router_instance, rsession); + + ss_dassert(router_capabilities == RCAP_TYPE_STMT_INPUT); +#endif + do + { + stmtbuf = gw_MySQL_get_next_stmt(&readbuf); + ss_dassert(stmtbuf != NULL); + CHK_GWBUF(stmtbuf); + + payload = (uint8_t *)GWBUF_DATA(stmtbuf); + len += MYSQL_GET_PACKET_LEN(payload); + /** + * If message is longer than read data, suspend routing and + * add statement buffer to wait queue. + */ + rc = router->routeQuery(router_instance, rsession, stmtbuf); + len = 0; /*< if routed, reset the length indicator */ + } + while (readbuf != NULL); + + return rc; } +/** + * Create a character array including the query string. + * GWBUF given as input includes either one complete or partial query. + * Length of buffer is at most the query length+4 (length of packet header). + */ +#if defined(NOT_USED) +static char* gw_get_or_create_querystr ( + void* data, + bool* new_allocation) +{ + GWBUF* buf = (GWBUF *)data; + size_t buflen; /*< first gw buffer data length */ + size_t packetlen; /*< length of mysql packet */ + size_t querylen; /*< total buffer length- */ + size_t nbytes_copied; + char* startpos; /*< first byte of query in gw buffer */ + char* str; /*< resulting query string */ + + CHK_GWBUF(buf); + packetlen = MYSQL_GET_PACKET_LEN((uint8_t *)GWBUF_DATA(buf)); + str = (char *)malloc(packetlen); /*< leave space for terminating null */ + + if (str == NULL) + { + goto return_str; + } + *new_allocation = true; + /** + * First buffer includes 4 bytes header and a type indicator byte. + */ + buflen = GWBUF_LENGTH(buf); + querylen = packetlen-1; + ss_dassert(buflen<=querylen+5); /*< 5 == header+type indicator */ + startpos = (char *)GWBUF_DATA(buf)+5; + nbytes_copied = MIN(querylen, buflen-5); + memcpy(str, startpos, nbytes_copied); + memset(&str[querylen-1], 0, 1); + buf = gwbuf_consume(buf, querylen-1); + + /** + * In case of multi-packet statement whole buffer consists of query + * string. + */ + while (buf != NULL) + { + buflen = GWBUF_LENGTH(buf); + memcpy(str+nbytes_copied, GWBUF_DATA(buf), buflen); + nbytes_copied += buflen; + buf = gwbuf_consume(buf, buflen); + } + ss_dassert(str[querylen-1] == 0); + +return_str: + return str; +} +#endif + + diff --git a/server/modules/routing/debugcli.c b/server/modules/routing/debugcli.c index 9c21c66f2..66c02c98b 100644 --- a/server/modules/routing/debugcli.c +++ b/server/modules/routing/debugcli.c @@ -54,6 +54,7 @@ static void closeSession(ROUTER *instance, void *router_session); static void freeSession(ROUTER *instance, void *router_session); static int execute(ROUTER *instance, void *router_session, GWBUF *queue); static void diagnostics(ROUTER *instance, DCB *dcb); +static uint8_t getCapabilities (ROUTER* inst, void* router_session); /** The module object definition */ static ROUTER_OBJECT MyObject = { @@ -64,7 +65,8 @@ static ROUTER_OBJECT MyObject = { execute, diagnostics, NULL, - NULL + NULL, + getCapabilities }; extern int execute_cmd(CLI_SESSION *cli); @@ -273,3 +275,10 @@ diagnostics(ROUTER *instance, DCB *dcb) { return; /* Nothing to do currently */ } + +static uint8_t getCapabilities( + ROUTER* inst, + void* router_session) +{ + return 0; +} \ No newline at end of file diff --git a/server/modules/routing/readconnroute.c b/server/modules/routing/readconnroute.c index a17e833b8..9bfa669e9 100644 --- a/server/modules/routing/readconnroute.c +++ b/server/modules/routing/readconnroute.c @@ -106,6 +106,8 @@ static void errorReply( char *message, DCB *backend_dcb, int action); +static uint8_t getCapabilities (ROUTER* inst, void* router_session); + /** The module object definition */ static ROUTER_OBJECT MyObject = { @@ -116,7 +118,8 @@ static ROUTER_OBJECT MyObject = { routeQuery, diagnostics, clientReply, - errorReply + errorReply, + getCapabilities }; static bool rses_begin_locked_router_action( @@ -379,6 +382,8 @@ int i; return NULL; } + client_rses->rses_capabilities = RCAP_TYPE_PACKET_INPUT; + /* * We now have the server with the least connections. * Bump the connection count for this server @@ -668,7 +673,7 @@ static void errorReply( ROUTER *instance, void *router_session, - char *message, + char *message, DCB *backend_dcb, int action) { @@ -737,3 +742,11 @@ static void rses_end_locked_router_action( CHK_CLIENT_RSES(rses); spinlock_release(&rses->rses_lock); } + + +static uint8_t getCapabilities( + ROUTER* inst, + void* router_session) +{ + return 0; +} \ No newline at end of file diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index 8f38c79c3..a70b7c8d9 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -66,6 +67,8 @@ static void clientReply( void* router_session, GWBUF* queue, DCB* backend_dcb); +static uint8_t getCapabilities (ROUTER* inst, void* router_session); + static bool search_backend_servers( BACKEND** p_master, @@ -80,7 +83,8 @@ static ROUTER_OBJECT MyObject = { routeQuery, diagnostic, clientReply, - NULL + NULL, + getCapabilities }; static bool rses_begin_locked_router_action( ROUTER_CLIENT_SES* rses); @@ -416,6 +420,7 @@ static void* newSession( client_rses->rses_backend[BE_MASTER] = local_backend[BE_MASTER]; router->stats.n_sessions += 1; + client_rses->rses_capabilities = RCAP_TYPE_STMT_INPUT; /** * Version is bigger than zero once initialized. */ @@ -561,384 +566,363 @@ static int routeQuery( GWBUF* querybuf) { skygw_query_type_t qtype = QUERY_TYPE_UNKNOWN; - GWBUF* stmtbuf; + GWBUF* plainsqlbuf = NULL; char* querystr = NULL; char* startpos; - size_t len; unsigned char packet_type; - unsigned char* packet; + uint8_t* packet; int ret = 0; DCB* master_dcb = NULL; DCB* slave_dcb = NULL; -// GWBUF* bufcopy = NULL; ROUTER_INSTANCE* inst = (ROUTER_INSTANCE *)instance; ROUTER_CLIENT_SES* router_cli_ses = (ROUTER_CLIENT_SES *)router_session; bool rses_is_closed; rses_property_t* prop; + size_t len; CHK_CLIENT_RSES(router_cli_ses); - - inst->stats.n_queries++; - /** Lock router session */ - if (!rses_begin_locked_router_action(router_cli_ses)) + + + /** Dirty read for quick check if router is closed. */ + if (router_cli_ses->rses_closed) { + rses_is_closed = true; + } + else + { + /** + * Lock router client session for secure read of DCBs + */ + rses_is_closed = + !(rses_begin_locked_router_action(router_cli_ses)); + } + + if (!rses_is_closed) + { + master_dcb = router_cli_ses->rses_dcb[BE_MASTER]; + slave_dcb = router_cli_ses->rses_dcb[BE_SLAVE]; + /** unlock */ + rses_end_locked_router_action(router_cli_ses); + } + + packet = GWBUF_DATA(querybuf); + packet_type = packet[4]; + + if (rses_is_closed || (master_dcb == NULL && slave_dcb == NULL)) + { + LOGIF(LE, (skygw_log_write_flush( + LOGFILE_ERROR, + "Error: Failed to route %s:%s:\"%s\" to " + "backend server. %s.", + STRPACKETTYPE(packet_type), + STRQTYPE(qtype), + (querystr == NULL ? "(empty)" : querystr), + (rses_is_closed ? "Router was closed" : + "Router has no backend servers where to " + "route to")))); goto return_ret; } - master_dcb = router_cli_ses->rses_dcb[BE_MASTER]; + inst->stats.n_queries++; + startpos = (char *)&packet[5]; - /** stmtbuf is clone of querybuf, and only covers one stmt */ - stmtbuf = (GWBUF *)master_dcb->session->client->func.getstmt((void *)querybuf); - /** Unlock router session */ - rses_end_locked_router_action(router_cli_ses); + switch(packet_type) { + case COM_QUIT: /**< 1 QUIT will close all sessions */ + case COM_INIT_DB: /**< 2 DDL must go to the master */ + case COM_REFRESH: /**< 7 - I guess this is session but not sure */ + case COM_DEBUG: /**< 0d all servers dump debug info to stdout */ + case COM_PING: /**< 0e all servers are pinged */ + case COM_CHANGE_USER: /**< 11 all servers change it accordingly */ + qtype = QUERY_TYPE_SESSION_WRITE; + break; + + case COM_CREATE_DB: /**< 5 DDL must go to the master */ + case COM_DROP_DB: /**< 6 DDL must go to the master */ + qtype = QUERY_TYPE_WRITE; + break; + + case COM_QUERY: + plainsqlbuf = gwbuf_clone_transform(querybuf, + GWBUF_TYPE_PLAINSQL); + len = GWBUF_LENGTH(plainsqlbuf); + /** unnecessary if buffer includes additional terminating null */ + querystr = (char *)malloc(len+1); + memcpy(querystr, startpos, len); + memset(&querystr[len], 0, 1); + // querystr = (char *)GWBUF_DATA(plainsqlbuf); + /* + querystr = master_dcb->func.getquerystr( + (void *) gwbuf_clone(querybuf), + &querystr_is_copy); + */ + qtype = skygw_query_classifier_get_type(querystr, 0); + break; + + case COM_SHUTDOWN: /**< 8 where should shutdown be routed ? */ + case COM_STATISTICS: /**< 9 ? */ + case COM_PROCESS_INFO: /**< 0a ? */ + case COM_CONNECT: /**< 0b ? */ + case COM_PROCESS_KILL: /**< 0c ? */ + case COM_TIME: /**< 0f should this be run in gateway ? */ + case COM_DELAYED_INSERT: /**< 10 ? */ + case COM_DAEMON: /**< 1d ? */ + default: + break; + } /**< switch by packet type */ - while (stmtbuf != NULL) - { - packet = GWBUF_DATA(stmtbuf); - packet_type = packet[4]; - startpos = (char *)&packet[5]; - len = packet[0]; - len += 255*packet[1]; - len += 255*255*packet[2]; - - switch(packet_type) { - case COM_QUIT: /**< 1 QUIT will close all sessions */ - case COM_INIT_DB: /**< 2 DDL must go to the master */ - case COM_REFRESH: /**< 7 - I guess this is session but not sure */ - case COM_DEBUG: /**< 0d all servers dump debug info to stdout */ - case COM_PING: /**< 0e all servers are pinged */ - case COM_CHANGE_USER: /**< 11 all servers change it accordingly */ - qtype = QUERY_TYPE_SESSION_WRITE; - break; - - case COM_CREATE_DB: /**< 5 DDL must go to the master */ - case COM_DROP_DB: /**< 6 DDL must go to the master */ - qtype = QUERY_TYPE_WRITE; - break; - - case COM_QUERY: - querystr = (char *)malloc(len); - memcpy(querystr, startpos, len-1); - memset(&querystr[len-1], 0, 1); - qtype = skygw_query_classifier_get_type(querystr, 0); - break; - - case COM_SHUTDOWN: /**< 8 where should shutdown be routed ? */ - case COM_STATISTICS: /**< 9 ? */ - case COM_PROCESS_INFO: /**< 0a ? */ - case COM_CONNECT: /**< 0b ? */ - case COM_PROCESS_KILL: /**< 0c ? */ - case COM_TIME: /**< 0f should this be run in gateway ? */ - case COM_DELAYED_INSERT: /**< 10 ? */ - case COM_DAEMON: /**< 1d ? */ - default: - break; - } /**< switch by packet type */ - - /** Dirty read for quick check if router is closed. */ - if (router_cli_ses->rses_closed) + LOGIF(LT, (skygw_log_write(LOGFILE_TRACE, + "String\t\"%s\"", + querystr == NULL ? "(empty)" : querystr))); + LOGIF(LT, (skygw_log_write(LOGFILE_TRACE, + "Packet type\t%s", + STRPACKETTYPE(packet_type)))); + + switch (qtype) { + case QUERY_TYPE_WRITE: + LOGIF(LT, (skygw_log_write( + LOGFILE_TRACE, + "%lu [routeQuery:rwsplit] Query type\t%s, " + "routing to Master.", + pthread_self(), + STRQTYPE(qtype)))); + + LOGIF(LT, tracelog_routed_query(router_cli_ses, + "routeQuery", + master_dcb, + gwbuf_clone(querybuf))); + + ret = master_dcb->func.write(master_dcb, querybuf); + atomic_add(&inst->stats.n_master, 1); + + goto return_ret; + break; + + case QUERY_TYPE_READ: + LOGIF(LT, (skygw_log_write( + LOGFILE_TRACE, + "%lu [routeQuery:rwsplit] Query type\t%s, " + "routing to Slave.", + pthread_self(), + STRQTYPE(qtype)))); + /** Lock router session */ + if (!rses_begin_locked_router_action(router_cli_ses)) { - rses_is_closed = true; + /** Log error to debug */ + goto return_ret; + } + /** + * If session command is being executed in slave + * route to master + */ + if (sescmd_cursor_is_active(rses_get_sescmd_cursor( + router_cli_ses, + BE_SLAVE))) + { + LOGIF(LT, tracelog_routed_query(router_cli_ses, + "routeQuery", + master_dcb, + gwbuf_clone(querybuf))); + + ret = master_dcb->func.write(master_dcb, querybuf); + atomic_add(&inst->stats.n_master, 1); } else { - /** - * Lock router client session for secure read of DCBs - */ - rses_is_closed = - !(rses_begin_locked_router_action(router_cli_ses)); - } - - if (!rses_is_closed) - { - master_dcb = router_cli_ses->rses_dcb[BE_MASTER]; - slave_dcb = router_cli_ses->rses_dcb[BE_SLAVE]; - /** unlock */ - rses_end_locked_router_action(router_cli_ses); - } - - if (rses_is_closed || (master_dcb == NULL && slave_dcb == NULL)) - { - LOGIF(LE, (skygw_log_write_flush( - LOGFILE_ERROR, - "Error: Failed to route %s:%s:\"%s\" to " - "backend server. %s.", - STRPACKETTYPE(packet_type), - STRQTYPE(qtype), - (querystr == NULL ? "(empty)" : querystr), - (rses_is_closed ? "Router was closed" : - "Router has no backend servers where to " - "route to")))); - goto return_ret; - } - - LOGIF(LT, (skygw_log_write(LOGFILE_TRACE, - "String\t\"%s\"", - querystr == NULL ? "(empty)" : querystr))); - LOGIF(LT, (skygw_log_write(LOGFILE_TRACE, - "Packet type\t%s", - STRPACKETTYPE(packet_type)))); - - switch (qtype) { - case QUERY_TYPE_WRITE: - LOGIF(LT, (skygw_log_write( - LOGFILE_TRACE, - "%lu [routeQuery:rwsplit] Query type\t%s, " - "routing to Master.", - pthread_self(), - STRQTYPE(qtype)))); - LOGIF(LT, tracelog_routed_query(router_cli_ses, "routeQuery", - master_dcb, - gwbuf_clone(stmtbuf))); + slave_dcb, + gwbuf_clone(querybuf))); - ret = master_dcb->func.write(master_dcb, stmtbuf); - atomic_add(&inst->stats.n_master, 1); + ret = slave_dcb->func.write(slave_dcb, querybuf); + atomic_add(&inst->stats.n_slave, 1); + } + rses_end_locked_router_action(router_cli_ses); + goto return_ret; + break; + + case QUERY_TYPE_SESSION_WRITE: + /** + * Execute in backends used by current router session. + * Save session variable commands to router session property + * struct. Thus, they + * can be replayed in backends which are started and joined later. + * + * Suppress OK packets sent to MaxScale by slaves. + * + * DOES THIS ALL APPLY TO COM_QUIT AS WELL?? + * + */ + + /** + * Update connections which are used in this session. + * + * For each connection updated, add a flag which indicates that + * OK Packet must arrive for this command before server + * in question is allowed to be used by router. That is, + * maintain a queue of pending OK packets and remove item + * from queue by FIFO. + * + * Return when the master responds OK Packet. Send that + * OK packet back to client. + * + * Suppress OK packets sent to MaxScale by slaves. + * + * Open questions: + * How to handle interleaving session write + * and queries? It would be simple if OK must be received + * from all/both servers before continuing query execution. + * How to maintain the order of operations? Execution queue + * would solve the problem. In the queue some things must be + * executed in serialized manner while some could be executed + * in parallel. Queries mostly. + * + * Instead of waiting for the OK packet from the master, the + * first OK packet could also be sent to client. TBD. + * vraa 9.12.13 + * + */ + LOGIF(LT, (skygw_log_write( + LOGFILE_TRACE, + "%lu [routeQuery:rwsplit] DCB M:%p s:%p, " + "Query type\t%s, " + "packet type %s, routing to all servers.", + pthread_self(), + master_dcb, + slave_dcb, + STRQTYPE(qtype), + STRPACKETTYPE(packet_type)))); + + switch(packet_type) { + case COM_CHANGE_USER: - goto return_ret; + LOGIF(LT, tracelog_routed_query( + router_cli_ses, + "routeQuery", + master_dcb, + gwbuf_clone(querybuf))); + + master_dcb->func.auth( + master_dcb, + NULL, + master_dcb->session, + gwbuf_clone(querybuf)); + + LOGIF(LT, tracelog_routed_query( + router_cli_ses, + "routeQuery", + slave_dcb, + gwbuf_clone(querybuf))); + + slave_dcb->func.auth( + slave_dcb, + NULL, + master_dcb->session, + querybuf); break; + + case COM_QUIT: + case COM_QUERY: + /** + * 1. Create new property of type RSES_PROP_TYPE_SESCMD. + * 2. Add property to the ROUTER_CLIENT_SES struct of + * this router session. + * 3. For each backend, and for each non-executed + * sescmd: + * call execution of current sescmd in + * all backends as long as both have executed + * them all. + * Execution call is dcb->func.session. + * All sescmds are executed when its return value is + * NULL, otherwise it is a pointer to next property. + */ + prop = rses_property_init(RSES_PROP_TYPE_SESCMD); + /** + * Additional reference is created to querybuf to + * prevent it from being released before properties + * are cleaned up as a part of router sessionclean-up. + */ + mysql_sescmd_init(prop, querybuf, router_cli_ses); - case QUERY_TYPE_READ: - LOGIF(LT, (skygw_log_write( - LOGFILE_TRACE, - "%lu [routeQuery:rwsplit] Query type\t%s, " - "routing to Slave.", - pthread_self(), - STRQTYPE(qtype)))); /** Lock router session */ if (!rses_begin_locked_router_action(router_cli_ses)) { - /** Log error to debug */ + rses_property_done(prop); goto return_ret; } - /** - * If session command is being executed in slave - * route to master - */ - if (sescmd_cursor_is_active(rses_get_sescmd_cursor( - router_cli_ses, - BE_SLAVE))) + /** Add sescmd property to router client session */ + rses_property_add(router_cli_ses, prop); + + /** Execute session command in master */ + if (execute_sescmd_in_backend(router_cli_ses, BE_MASTER)) { - LOGIF(LT, tracelog_routed_query(router_cli_ses, - "routeQuery", - master_dcb, - gwbuf_clone(stmtbuf))); - - ret = master_dcb->func.write(master_dcb, stmtbuf); - atomic_add(&inst->stats.n_master, 1); + ret = 1; } else { - LOGIF(LT, tracelog_routed_query(router_cli_ses, - "routeQuery", - slave_dcb, - gwbuf_clone(stmtbuf))); - - ret = slave_dcb->func.write(slave_dcb, stmtbuf); - atomic_add(&inst->stats.n_slave, 1); + /** Log error */ } + /** Execute session command in slave */ + if (execute_sescmd_in_backend(router_cli_ses, BE_SLAVE)) + { + ret = 1; + } + else + { + /** Log error */ + } + + /** Unlock router session */ rses_end_locked_router_action(router_cli_ses); - goto return_ret; - break; - - case QUERY_TYPE_SESSION_WRITE: - /** - * Execute in backends used by current router session. - * Save session variable commands to router session property - * struct. Thus, they - * can be replayed in backends which are started and joined later. - * - * Suppress OK packets sent to MaxScale by slaves. - * - * DOES THIS ALL APPLY TO COM_QUIT AS WELL?? - * - */ - - /** - * Update connections which are used in this session. - * - * For each connection updated, add a flag which indicates that - * OK Packet must arrive for this command before server - * in question is allowed to be used by router. That is, - * maintain a queue of pending OK packets and remove item - * from queue by FIFO. - * - * Return when the master responds OK Packet. Send that - * OK packet back to client. - * - * Suppress OK packets sent to MaxScale by slaves. - * - * Open questions: - * How to handle interleaving session write - * and queries? It would be simple if OK must be received - * from all/both servers before continuing query execution. - * How to maintain the order of operations? Execution queue - * would solve the problem. In the queue some things must be - * executed in serialized manner while some could be executed - * in parallel. Queries mostly. - * - * Instead of waiting for the OK packet from the master, the - * first OK packet could also be sent to client. TBD. - * vraa 9.12.13 - * - */ - LOGIF(LT, (skygw_log_write( - LOGFILE_TRACE, - "%lu [routeQuery:rwsplit] DCB M:%p s:%p, " - "Query type\t%s, " - "packet type %s, routing to all servers.", - pthread_self(), - master_dcb, - slave_dcb, - STRQTYPE(qtype), - STRPACKETTYPE(packet_type)))); - - switch(packet_type) { - /** - case COM_QUIT: - ret = master_dcb->func.write(master_dcb, gwbuf_clone(querybuf)); - slave_dcb->func.write(slave_dcb, querybuf); - break; - */ - case COM_CHANGE_USER: - - LOGIF(LT, tracelog_routed_query( - router_cli_ses, - "routeQuery", - master_dcb, - gwbuf_clone(stmtbuf))); - - master_dcb->func.auth( - master_dcb, - NULL, - master_dcb->session, - gwbuf_clone(stmtbuf)); - - LOGIF(LT, tracelog_routed_query( - router_cli_ses, - "routeQuery", - slave_dcb, - gwbuf_clone(stmtbuf))); - - slave_dcb->func.auth( - slave_dcb, - NULL, - master_dcb->session, - stmtbuf); - break; - - case COM_QUIT: - case COM_QUERY: - /** - * 1. Create new property of type RSES_PROP_TYPE_SESCMD. - * 2. Add property to the ROUTER_CLIENT_SES struct of - * this router session. - * 3. For each backend, and for each non-executed - * sescmd: - * call execution of current sescmd in - * all backends as long as both have executed - * them all. - * Execution call is dcb->func.session. - * All sescmds are executed when its return value is - * NULL, otherwise it is a pointer to next property. - */ - prop = rses_property_init(RSES_PROP_TYPE_SESCMD); - /** - * Additional reference is created to querybuf to - * prevent it from being released before properties - * are cleaned up as a part of router sessionclean-up. - */ - mysql_sescmd_init(prop, stmtbuf, router_cli_ses); - - /** Lock router session */ - if (!rses_begin_locked_router_action(router_cli_ses)) - { - rses_property_done(prop); - goto return_ret; - } - /** Add sescmd property to router client session */ - rses_property_add(router_cli_ses, prop); - - /** Execute session command in master */ - if (execute_sescmd_in_backend(router_cli_ses, BE_MASTER)) - { - ret = 1; - } - else - { - /** Log error */ - } - /** Execute session command in slave */ - if (execute_sescmd_in_backend(router_cli_ses, BE_SLAVE)) - { - ret = 1; - } - else - { - /** Log error */ - } - - /** Unlock router session */ - rses_end_locked_router_action(router_cli_ses); - break; - - default: - LOGIF(LT, tracelog_routed_query(router_cli_ses, - "routeQuery", - master_dcb, - gwbuf_clone(stmtbuf))); - ret = master_dcb->func.write(master_dcb, - (void *)gwbuf_clone(stmtbuf)); - - LOGIF(LT, tracelog_routed_query(router_cli_ses, - "routeQuery", - slave_dcb, - gwbuf_clone(stmtbuf))); - - slave_dcb->func.write(slave_dcb, (void *)stmtbuf); - break; - } /**< switch by packet type */ - - atomic_add(&inst->stats.n_all, 1); - goto return_ret; break; default: - LOGIF(LT, (skygw_log_write( - LOGFILE_TRACE, - "%lu [routeQuery:rwsplit] Query type\t%s, " - "routing to Master by default.", - pthread_self(), - STRQTYPE(qtype)))); - - /** - * Is this really ok? - * What is not known is routed to master. - */ LOGIF(LT, tracelog_routed_query(router_cli_ses, "routeQuery", master_dcb, - gwbuf_clone(stmtbuf))); + gwbuf_clone(querybuf))); + ret = master_dcb->func.write(master_dcb, + (void *)gwbuf_clone(querybuf)); - ret = master_dcb->func.write(master_dcb, stmtbuf); - atomic_add(&inst->stats.n_master, 1); - goto return_ret; + LOGIF(LT, tracelog_routed_query(router_cli_ses, + "routeQuery", + slave_dcb, + gwbuf_clone(querybuf))); + + slave_dcb->func.write(slave_dcb, (void *)querybuf); break; - } /*< switch by query type */ + } /**< switch by packet type */ - /** get next stmt */ - if (!rses_begin_locked_router_action(router_cli_ses)) - { - goto return_ret; - } - master_dcb = router_cli_ses->rses_dcb[BE_MASTER]; + atomic_add(&inst->stats.n_all, 1); + goto return_ret; + break; + + default: + LOGIF(LT, (skygw_log_write( + LOGFILE_TRACE, + "%lu [routeQuery:rwsplit] Query type\t%s, " + "routing to Master by default.", + pthread_self(), + STRQTYPE(qtype)))); - /** stmtbuf is clone of querybuf, and only covers one stmt */ - stmtbuf = (GWBUF *)master_dcb->session->client->func.getstmt((void *)querybuf); - rses_end_locked_router_action(router_cli_ses); - } /* while (stmtbuf != NULL) */ + /** + * Is this really ok? + * What is not known is routed to master. + */ + LOGIF(LT, tracelog_routed_query(router_cli_ses, + "routeQuery", + master_dcb, + gwbuf_clone(querybuf))); + + ret = master_dcb->func.write(master_dcb, querybuf); + atomic_add(&inst->stats.n_master, 1); + goto return_ret; + break; + } /*< switch by query type */ + return_ret: - free(querystr); + if (plainsqlbuf != NULL) + { + gwbuf_free(plainsqlbuf); + } return ret; } @@ -1346,7 +1330,6 @@ static rses_property_t* rses_property_init( { goto return_prop; } -// spinlock_init(&prop->rses_prop_lock); prop->rses_prop_type = prop_type; #if defined(SS_DEBUG) prop->rses_prop_chk_top = CHK_NUM_ROUTER_PROPERTY; @@ -1419,7 +1402,7 @@ static void rses_property_add( } } -/** Router sessiosn must be locked */ +/** Router session must be locked */ static mysql_sescmd_t* rses_property_get_sescmd( rses_property_t* prop) { @@ -1437,9 +1420,7 @@ static mysql_sescmd_t* rses_property_get_sescmd( } return sescmd; } - -// static rses_property_t* rses_property_get_ptr_next( - + /** static void rses_begin_locked_property_action( rses_property_t* prop) @@ -1505,18 +1486,12 @@ static bool sescmd_reply_to_client( { bool succp = false; - // rses_property_t* prop; - CHK_DCB(client_dcb); CHK_MYSQL_SESCMD(scmd); CHK_GWBUF(writebuf); ss_dassert(SPINLOCK_IS_LOCKED( &scmd->my_sescmd_prop->rses_prop_rsession->rses_lock)); -// prop = mysql_sescmd_get_property(scmd); - -// rses_begin_locked_property_action(prop); - if (!scmd->my_sescmd_is_replied) { client_dcb->func.write(client_dcb, writebuf); @@ -1531,7 +1506,6 @@ static bool sescmd_reply_to_client( else { } -// rses_end_locked_property_action(prop); return succp; } @@ -1651,7 +1625,6 @@ static bool execute_sescmd_in_backend( dcb, sescmd_cursor_clone_querybuf(scur))); rc = dcb->func.write(dcb, sescmd_cursor_clone_querybuf(scur)); -// rc = dcb->func.session(dcb, sescmd_cursor_clone_querybuf(scur)); if (rc != 1) { @@ -1805,18 +1778,14 @@ static void tracelog_routed_query( DCB* dcb, GWBUF* buf) { - unsigned char* packet = GWBUF_DATA(buf); + uint8_t* packet = GWBUF_DATA(buf); unsigned char packet_type = packet[4]; size_t len; size_t buflen = GWBUF_LENGTH(buf); char* querystr; char* startpos = (char *)&packet[5]; backend_type_t be_type; - - len = packet[0]; - len += 255*packet[1]; - len += 255*255*packet[2]; - + if (rses->rses_dcb[BE_MASTER] == dcb) { be_type = BE_MASTER; @@ -1829,31 +1798,61 @@ static void tracelog_routed_query( { be_type = BE_UNDEFINED; } - - if (packet_type == '\x03') + if (GWBUF_TYPE(buf) == GWBUF_TYPE_MYSQL) { - querystr = (char *)malloc(len); - memcpy(querystr, startpos, len-1); - querystr[len-1] = '\0'; - LOGIF(LT, (skygw_log_write_flush( - LOGFILE_TRACE, - "%lu [%s] %d bytes long buf, \"%s\" -> %s:%d %s dcb %p", - pthread_self(), - funcname, - buflen, - querystr, - (be_type == BE_MASTER ? - rses->rses_backend[BE_MASTER]->backend_server->name : - (be_type == BE_SLAVE ? - rses->rses_backend[BE_SLAVE]->backend_server->name : - "Target DCB is neither of the backends. This is error")), - (be_type == BE_MASTER ? - rses->rses_backend[BE_MASTER]->backend_server->port : - (be_type == BE_SLAVE ? - rses->rses_backend[BE_SLAVE]->backend_server->port : - -1)), - STRBETYPE(be_type), - dcb))); + len = packet[0]; + len += 256*packet[1]; + len += 256*256*packet[2]; + + if (packet_type == '\x03') + { + querystr = (char *)malloc(len); + memcpy(querystr, startpos, len-1); + querystr[len-1] = '\0'; + LOGIF(LT, (skygw_log_write_flush( + LOGFILE_TRACE, + "%lu [%s] %d bytes long buf, \"%s\" -> %s:%d %s dcb %p", + pthread_self(), + funcname, + buflen, + querystr, + (be_type == BE_MASTER ? + rses->rses_backend[BE_MASTER]->backend_server->name : + (be_type == BE_SLAVE ? + rses->rses_backend[BE_SLAVE]->backend_server->name : + "Target DCB is neither of the backends. This is error")), + (be_type == BE_MASTER ? + rses->rses_backend[BE_MASTER]->backend_server->port : + (be_type == BE_SLAVE ? + rses->rses_backend[BE_SLAVE]->backend_server->port : + -1)), + STRBETYPE(be_type), + dcb))); + } } gwbuf_free(buf); +} + +/** + * Return rc, rc < 0 if router session is closed. rc == 0 if there are no + * capabilities specified, rc > 0 when there are capabilities. + */ +static uint8_t getCapabilities ( + ROUTER* inst, + void* router_session) +{ + ROUTER_CLIENT_SES* rses = (ROUTER_CLIENT_SES *)router_session; + uint8_t rc; + + if (!rses_begin_locked_router_action(rses)) + { + rc = 0xff; + goto return_rc; + } + rc = rses->rses_capabilities; + + rses_end_locked_router_action(rses); + +return_rc: + return rc; } \ No newline at end of file diff --git a/server/modules/routing/testroute.c b/server/modules/routing/testroute.c index 2e1615f25..c0bd73fee 100644 --- a/server/modules/routing/testroute.c +++ b/server/modules/routing/testroute.c @@ -26,6 +26,7 @@ static void closeSession(ROUTER *instance, void *session); static void freeSession(ROUTER *instance, void *session); static int routeQuery(ROUTER *instance, void *session, GWBUF *queue); static void diagnostic(ROUTER *instance, DCB *dcb); +static uint8_t getCapabilities (ROUTER* inst, void* router_session); static ROUTER_OBJECT MyObject = { @@ -36,7 +37,8 @@ static ROUTER_OBJECT MyObject = { routeQuery, diagnostic, NULL, - NULL + NULL, + getCapabilities }; /** @@ -137,3 +139,10 @@ static void diagnostic(ROUTER *instance, DCB *dcb) { } + +static uint8_t getCapabilities( + ROUTER* inst, + void* router_session) +{ + return 0; +} \ No newline at end of file From a3f7eebdc9b8034b371f7d143b96dca7931d6cb5 Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Fri, 14 Mar 2014 13:25:37 +0200 Subject: [PATCH 4/6] Extended session command support to cover COM_CHANGE_USER, and COM_INIT_DB. This implementation doesn't guarantee execution order between session commands and queries if other backend server lags behind in session command execution. In poll.c : moved processing of EPOLLERR and EPOLLHUP after processing of EPOLLIN and EPOLLOUT. This ensures that COM_QUIT messages are read and routed forward before signals arrive (from local client/backend). --- server/core/buffer.c | 5 +- server/core/dcb.c | 8 +- server/core/poll.c | 103 +++-- server/modules/include/readwritesplit.h | 1 + server/modules/protocol/mysql_backend.c | 38 +- server/modules/protocol/mysql_client.c | 14 +- server/modules/protocol/mysql_common.c | 4 +- .../routing/readwritesplit/readwritesplit.c | 353 ++++++++---------- 8 files changed, 233 insertions(+), 293 deletions(-) diff --git a/server/core/buffer.c b/server/core/buffer.c index 57228b253..b21cf216c 100644 --- a/server/core/buffer.c +++ b/server/core/buffer.c @@ -130,7 +130,6 @@ GWBUF *rval; rval->end = buf->end; rval->gwbuf_type = buf->gwbuf_type; rval->next = NULL; -// rval->command = buf->command; CHK_GWBUF(rval); return rval; } @@ -234,6 +233,7 @@ GWBUF *ptr = head; if (!head) return tail; CHK_GWBUF(head); + CHK_GWBUF(tail); while (ptr->next) { ptr = ptr->next; @@ -262,9 +262,10 @@ GWBUF * gwbuf_consume(GWBUF *head, unsigned int length) { GWBUF *rval = head; - CHK_GWBUF(head); GWBUF_CONSUME(head, length); + CHK_GWBUF(head); + if (GWBUF_EMPTY(head)) { rval = head->next; diff --git a/server/core/dcb.c b/server/core/dcb.c index db9e96b60..d1b5a4ffb 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -698,8 +698,7 @@ dcb_write(DCB *dcb, GWBUF *queue) dcb->fd))); return 0; } - - + spinlock_acquire(&dcb->writeqlock); if (dcb->writeq != NULL) @@ -809,9 +808,9 @@ dcb_write(DCB *dcb, GWBUF *queue) pthread_self(), w, dcb, - STRDCBSTATE(dcb->state), + STRDCBSTATE(dcb->state), dcb->fd))); - } + } /*< while (queue != NULL) */ /*< * What wasn't successfully written is stored to write queue * for suspended write. @@ -829,7 +828,6 @@ dcb_write(DCB *dcb, GWBUF *queue) saved_errno != EAGAIN && saved_errno != EWOULDBLOCK) { - queue = gwbuf_consume(queue, gwbuf_length(queue)); LOGIF(LE, (skygw_log_write_flush( LOGFILE_ERROR, "Error : Writing to %s socket failed due %d, %s.", diff --git a/server/core/poll.c b/server/core/poll.c index 16c8807a9..7cd4ae992 100644 --- a/server/core/poll.c +++ b/server/core/poll.c @@ -357,60 +357,12 @@ poll_waitevents(void *arg) dcb, STRDCBROLE(dcb->dcb_role)))); - if (ev & EPOLLERR) - { - int eno = gw_getsockerrno(dcb->fd); -#if defined(SS_DEBUG) - if (eno == 0) { - eno = dcb_fake_write_errno[dcb->fd]; - LOGIF(LD, (skygw_log_write( - LOGFILE_DEBUG, - "%lu [poll_waitevents] " - "Added fake errno %d. " - "%s", - pthread_self(), - eno, - strerror(eno)))); - } - dcb_fake_write_errno[dcb->fd] = 0; -#endif - if (eno != 0) { - LOGIF(LD, (skygw_log_write( - LOGFILE_DEBUG, - "%lu [poll_waitevents] " - "EPOLLERR due %d, %s.", - pthread_self(), - eno, - strerror(eno)))); - } - atomic_add(&pollStats.n_error, 1); - dcb->func.error(dcb); - } - if (ev & EPOLLHUP) - { - int eno = 0; - eno = gw_getsockerrno(dcb->fd); - - LOGIF(LD, (skygw_log_write( - LOGFILE_DEBUG, - "%lu [poll_waitevents] " - "EPOLLHUP on dcb %p, fd %d. " - "Errno %d, %s.", - pthread_self(), - dcb, - dcb->fd, - eno, - strerror(eno)))); - atomic_add(&pollStats.n_hup, 1); - dcb->func.hangup(dcb); - } if (ev & EPOLLOUT) { int eno = 0; eno = gw_getsockerrno(dcb->fd); if (eno == 0) { -#if 1 simple_mutex_lock( &dcb->dcb_write_lock, true); @@ -418,16 +370,13 @@ poll_waitevents(void *arg) !dcb->dcb_write_active, "Write already active"); dcb->dcb_write_active = TRUE; -#endif atomic_add( &pollStats.n_write, 1); dcb->func.write_ready(dcb); -#if 1 dcb->dcb_write_active = FALSE; simple_mutex_unlock( &dcb->dcb_write_lock); -#endif } else { LOGIF(LD, (skygw_log_write( LOGFILE_DEBUG, @@ -443,13 +392,12 @@ poll_waitevents(void *arg) } if (ev & EPOLLIN) { -#if 1 simple_mutex_lock(&dcb->dcb_read_lock, true); ss_info_dassert(!dcb->dcb_read_active, "Read already active"); dcb->dcb_read_active = TRUE; -#endif + if (dcb->state == DCB_STATE_LISTENING) { LOGIF(LD, (skygw_log_write( @@ -474,12 +422,57 @@ poll_waitevents(void *arg) atomic_add(&pollStats.n_read, 1); dcb->func.read(dcb); } -#if 1 dcb->dcb_read_active = FALSE; simple_mutex_unlock( &dcb->dcb_read_lock); -#endif } + if (ev & EPOLLERR) + { + int eno = gw_getsockerrno(dcb->fd); + #if defined(SS_DEBUG) + if (eno == 0) { + eno = dcb_fake_write_errno[dcb->fd]; + LOGIF(LD, (skygw_log_write( + LOGFILE_DEBUG, + "%lu [poll_waitevents] " + "Added fake errno %d. " + "%s", + pthread_self(), + eno, + strerror(eno)))); + } + dcb_fake_write_errno[dcb->fd] = 0; + #endif + if (eno != 0) { + LOGIF(LD, (skygw_log_write( + LOGFILE_DEBUG, + "%lu [poll_waitevents] " + "EPOLLERR due %d, %s.", + pthread_self(), + eno, + strerror(eno)))); + } + atomic_add(&pollStats.n_error, 1); + dcb->func.error(dcb); + } + if (ev & EPOLLHUP) + { + int eno = 0; + eno = gw_getsockerrno(dcb->fd); + + LOGIF(LD, (skygw_log_write( + LOGFILE_DEBUG, + "%lu [poll_waitevents] " + "EPOLLHUP on dcb %p, fd %d. " + "Errno %d, %s.", + pthread_self(), + dcb, + dcb->fd, + eno, + strerror(eno)))); + atomic_add(&pollStats.n_hup, 1); + dcb->func.hangup(dcb); + } } /*< for */ no_op = FALSE; } diff --git a/server/modules/include/readwritesplit.h b/server/modules/include/readwritesplit.h index c5dcbdaaa..477096b76 100644 --- a/server/modules/include/readwritesplit.h +++ b/server/modules/include/readwritesplit.h @@ -68,6 +68,7 @@ typedef struct mysql_sescmd_st { #endif rses_property_t* my_sescmd_prop; /*< parent property */ GWBUF* my_sescmd_buf; /*< query buffer */ + unsigned char my_sescmd_packet_type;/*< packet type */ bool my_sescmd_is_replied; /*< is cmd replied to client */ #if defined(SS_DEBUG) skygw_chk_t my_sescmd_chk_tail; diff --git a/server/modules/protocol/mysql_backend.c b/server/modules/protocol/mysql_backend.c index 03e433062..78216d62f 100644 --- a/server/modules/protocol/mysql_backend.c +++ b/server/modules/protocol/mysql_backend.c @@ -283,6 +283,8 @@ static int gw_read_backend_event(DCB *dcb) { } if (backend_protocol->state == MYSQL_AUTH_FAILED) { + + spinlock_acquire(&dcb->delayqlock); /*< * vraa : errorHandle * check the delayq before the reply @@ -295,10 +297,12 @@ static int gw_read_backend_event(DCB *dcb) { 0, "Connection to backend lost."); // consume all the delay queue - dcb->delayq = gwbuf_consume( + while ((dcb->delayq = gwbuf_consume( dcb->delayq, - gwbuf_length(dcb->delayq)); + GWBUF_LENGTH(dcb->delayq))) != NULL); } + spinlock_release(&dcb->delayqlock); + while (session->state != SESSION_STATE_ROUTER_READY) { @@ -347,7 +351,7 @@ static int gw_read_backend_event(DCB *dcb) { pthread_self(), dcb->fd, current_session->user))); - + /* check the delay queue and flush the data */ if (dcb->delayq) { @@ -802,12 +806,6 @@ static int backend_write_delayqueue(DCB *dcb) localq = dcb->delayq; dcb->delayq = NULL; - /*< - * Now we set the last command received, from the delayed queue - */ - -// memcpy(&dcb->command, &localq->command, sizeof(dcb->command)); - spinlock_release(&dcb->delayqlock); rc = dcb_write(dcb, localq); @@ -856,8 +854,6 @@ static int gw_change_user(DCB *backend, SERVER *server, SESSION *in_session, GWB backend_protocol = backend->protocol; client_protocol = in_session->client->protocol; -// queue->command = ROUTER_CHANGE_SESSION; - // now get the user, after 4 bytes header and 1 byte command client_auth_packet += 5; strcpy(username, (char *)client_auth_packet); @@ -899,30 +895,14 @@ static int gw_change_user(DCB *backend, SERVER *server, SESSION *in_session, GWB rv = gw_send_change_user_to_backend(database, username, client_sha1, backend_protocol); - /*< - * The current queue was not handled by func.write() in gw_send_change_user_to_backend() - * We wrote a new gwbuf - * Set backend command here! - */ - memcpy(&backend->command, &queue->command, sizeof(backend->command)); - /*< * Now copy new data into user session */ strcpy(current_session->user, username); strcpy(current_session->db, database); memcpy(current_session->client_sha1, client_sha1, sizeof(current_session->client_sha1)); - } - - // consume all the data received from client - - spinlock_acquire(&backend->writeqlock); - - len = gwbuf_length(queue); - queue = gwbuf_consume(queue, len); - - spinlock_release(&backend->writeqlock); - + } + gwbuf_free(queue); return rv; } diff --git a/server/modules/protocol/mysql_client.c b/server/modules/protocol/mysql_client.c index 255ede201..ecea34972 100644 --- a/server/modules/protocol/mysql_client.c +++ b/server/modules/protocol/mysql_client.c @@ -1157,7 +1157,6 @@ static int gw_error_client_event(DCB *dcb) { router = session->service->router; router_instance = session->service->router_instance; rsession = session->router_session; - router->closeSession(router_instance, rsession); } dcb_close(dcb); @@ -1215,7 +1214,8 @@ gw_client_hangup_event(DCB *dcb) void* router_instance; void* rsession; int rc = 1; -#if defined(SS_DEBUG) + + #if defined(SS_DEBUG) MySQLProtocol* protocol = (MySQLProtocol *)dcb->protocol; if (dcb->state == DCB_STATE_POLLING || dcb->state == DCB_STATE_NOPOLLING || @@ -1224,8 +1224,6 @@ gw_client_hangup_event(DCB *dcb) CHK_PROTOCOL(protocol); } #endif - - CHK_DCB(dcb); if (dcb->state != DCB_STATE_POLLING) { @@ -1242,7 +1240,6 @@ gw_client_hangup_event(DCB *dcb) router = session->service->router; router_instance = session->service->router_instance; rsession = session->router_session; - router->closeSession(router_instance, rsession); } @@ -1314,13 +1311,6 @@ static int route_by_statement( uint8_t* payload; static size_t len; -#if defined(SS_DEBUG) - uint8_t router_capabilities; - - router_capabilities = router->getCapabilities(router_instance, rsession); - - ss_dassert(router_capabilities == RCAP_TYPE_STMT_INPUT); -#endif do { stmtbuf = gw_MySQL_get_next_stmt(&readbuf); diff --git a/server/modules/protocol/mysql_common.c b/server/modules/protocol/mysql_common.c index 01fd75b27..7bfb3666c 100644 --- a/server/modules/protocol/mysql_common.c +++ b/server/modules/protocol/mysql_common.c @@ -181,7 +181,7 @@ int gw_read_backend_handshake(MySQLProtocol *conn) { conn->state = MYSQL_AUTH_SENT; // consume all the data here - head = gwbuf_consume(head, gwbuf_length(head)); + head = gwbuf_consume(head, GWBUF_LENGTH(head)); return 0; } @@ -337,7 +337,7 @@ int gw_receive_backend_auth( /*< * Remove data from buffer. */ - head = gwbuf_consume(head, gwbuf_length(head)); + head = gwbuf_consume(head, GWBUF_LENGTH(head)); } else if (n == 0) { diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index a70b7c8d9..1548eac57 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -98,6 +98,7 @@ static void mysql_sescmd_done( static mysql_sescmd_t* mysql_sescmd_init ( rses_property_t* rses_prop, GWBUF* sescmd_buf, + unsigned char packet_type, ROUTER_CLIENT_SES* rses); static rses_property_t* mysql_sescmd_get_property( @@ -590,9 +591,7 @@ static int routeQuery( } else { - /** - * Lock router client session for secure read of DCBs - */ + /*< Lock router client session for secure read of DCBs */ rses_is_closed = !(rses_begin_locked_router_action(router_cli_ses)); } @@ -678,6 +677,43 @@ static int routeQuery( switch (qtype) { case QUERY_TYPE_WRITE: +#if 0 + /** + * Running this block cause deadlock because read mutex is + * on hold. This doesn't serialize subsequent session commands + * and queries if there are multiple session commands and other + * backend starts to lag behind. vraa : 14.3.13 + */ + /** + * Wait until master has executed all its session commands. + * TODO: if master fails it needs to be detected in the loop. + */ + if (!rses_begin_locked_router_action(router_cli_ses)) + { + goto return_ret; + } + + while (sescmd_cursor_is_active(rses_get_sescmd_cursor( + router_cli_ses, + BE_MASTER))) + { + rses_end_locked_router_action(router_cli_ses); + + LOGIF(LD, (skygw_log_write( + LOGFILE_DEBUG, + "%lu [routeQuery:rwsplit] Session command is " + "active in MASTER. Waiting in loop.", + pthread_self()))); + + usleep(10); + + if (!rses_begin_locked_router_action(router_cli_ses)) + { + goto return_ret; + } + } + rses_end_locked_router_action(router_cli_ses); +#endif LOGIF(LT, (skygw_log_write( LOGFILE_TRACE, "%lu [routeQuery:rwsplit] Query type\t%s, " @@ -702,84 +738,65 @@ static int routeQuery( "%lu [routeQuery:rwsplit] Query type\t%s, " "routing to Slave.", pthread_self(), - STRQTYPE(qtype)))); - /** Lock router session */ - if (!rses_begin_locked_router_action(router_cli_ses)) + STRQTYPE(qtype)))); + while (true) { - /** Log error to debug */ - goto return_ret; - } - /** - * If session command is being executed in slave - * route to master - */ - if (sescmd_cursor_is_active(rses_get_sescmd_cursor( - router_cli_ses, - BE_SLAVE))) - { - LOGIF(LT, tracelog_routed_query(router_cli_ses, - "routeQuery", - master_dcb, - gwbuf_clone(querybuf))); - - ret = master_dcb->func.write(master_dcb, querybuf); - atomic_add(&inst->stats.n_master, 1); - } - else - { - LOGIF(LT, tracelog_routed_query(router_cli_ses, - "routeQuery", - slave_dcb, - gwbuf_clone(querybuf))); - - ret = slave_dcb->func.write(slave_dcb, querybuf); - atomic_add(&inst->stats.n_slave, 1); - } - rses_end_locked_router_action(router_cli_ses); + if (!rses_begin_locked_router_action(router_cli_ses)) + { + goto return_ret; + } + /** + * If session command is being executed in slave + * route to master. + */ + if (!sescmd_cursor_is_active(rses_get_sescmd_cursor( + router_cli_ses, + BE_SLAVE))) + { + rses_end_locked_router_action(router_cli_ses); + LOGIF(LT, tracelog_routed_query(router_cli_ses, + "routeQuery", + slave_dcb, + gwbuf_clone(querybuf))); + + ret = slave_dcb->func.write(slave_dcb, querybuf); + atomic_add(&inst->stats.n_slave, 1); + break; + } + else if (!sescmd_cursor_is_active(rses_get_sescmd_cursor( + router_cli_ses, + BE_MASTER))) + + { + rses_end_locked_router_action(router_cli_ses); + LOGIF(LT, tracelog_routed_query(router_cli_ses, + "routeQuery", + master_dcb, + gwbuf_clone(querybuf))); + + ret = slave_dcb->func.write(master_dcb, querybuf); + atomic_add(&inst->stats.n_master, 1); + break; + } + rses_end_locked_router_action(router_cli_ses); + } /*< while (true) */ goto return_ret; break; case QUERY_TYPE_SESSION_WRITE: /** - * Execute in backends used by current router session. - * Save session variable commands to router session property - * struct. Thus, they - * can be replayed in backends which are started and joined later. - * - * Suppress OK packets sent to MaxScale by slaves. - * - * DOES THIS ALL APPLY TO COM_QUIT AS WELL?? - * - */ - - /** - * Update connections which are used in this session. - * - * For each connection updated, add a flag which indicates that - * OK Packet must arrive for this command before server - * in question is allowed to be used by router. That is, - * maintain a queue of pending OK packets and remove item - * from queue by FIFO. - * - * Return when the master responds OK Packet. Send that - * OK packet back to client. - * - * Suppress OK packets sent to MaxScale by slaves. - * - * Open questions: - * How to handle interleaving session write - * and queries? It would be simple if OK must be received - * from all/both servers before continuing query execution. - * How to maintain the order of operations? Execution queue - * would solve the problem. In the queue some things must be - * executed in serialized manner while some could be executed - * in parallel. Queries mostly. - * - * Instead of waiting for the OK packet from the master, the - * first OK packet could also be sent to client. TBD. - * vraa 9.12.13 - * - */ + * Execute in backends used by current router session. + * Save session variable commands to router session property + * struct. Thus, they can be replayed in backends which are + * started and joined later. + * + * Suppress redundant OK packets sent by backends. + * + * DOES THIS ALL APPLY TO COM_QUIT AS WELL?? + * + * The first OK packet is replied to the client. + * + */ LOGIF(LT, (skygw_log_write( LOGFILE_TRACE, "%lu [routeQuery:rwsplit] DCB M:%p s:%p, " @@ -791,106 +808,45 @@ static int routeQuery( STRQTYPE(qtype), STRPACKETTYPE(packet_type)))); - switch(packet_type) { - case COM_CHANGE_USER: - - LOGIF(LT, tracelog_routed_query( - router_cli_ses, - "routeQuery", - master_dcb, - gwbuf_clone(querybuf))); - - master_dcb->func.auth( - master_dcb, - NULL, - master_dcb->session, - gwbuf_clone(querybuf)); - - LOGIF(LT, tracelog_routed_query( - router_cli_ses, - "routeQuery", - slave_dcb, - gwbuf_clone(querybuf))); - - slave_dcb->func.auth( - slave_dcb, - NULL, - master_dcb->session, - querybuf); - break; - - case COM_QUIT: - case COM_QUERY: - /** - * 1. Create new property of type RSES_PROP_TYPE_SESCMD. - * 2. Add property to the ROUTER_CLIENT_SES struct of - * this router session. - * 3. For each backend, and for each non-executed - * sescmd: - * call execution of current sescmd in - * all backends as long as both have executed - * them all. - * Execution call is dcb->func.session. - * All sescmds are executed when its return value is - * NULL, otherwise it is a pointer to next property. - */ - prop = rses_property_init(RSES_PROP_TYPE_SESCMD); - /** - * Additional reference is created to querybuf to - * prevent it from being released before properties - * are cleaned up as a part of router sessionclean-up. - */ - mysql_sescmd_init(prop, querybuf, router_cli_ses); - - /** Lock router session */ - if (!rses_begin_locked_router_action(router_cli_ses)) - { - rses_property_done(prop); - goto return_ret; - } - /** Add sescmd property to router client session */ - rses_property_add(router_cli_ses, prop); - - /** Execute session command in master */ - if (execute_sescmd_in_backend(router_cli_ses, BE_MASTER)) - { - ret = 1; - } - else - { - /** Log error */ - } - /** Execute session command in slave */ - if (execute_sescmd_in_backend(router_cli_ses, BE_SLAVE)) - { - ret = 1; - } - else - { - /** Log error */ - } - - /** Unlock router session */ - rses_end_locked_router_action(router_cli_ses); - break; - - default: - LOGIF(LT, tracelog_routed_query(router_cli_ses, - "routeQuery", - master_dcb, - gwbuf_clone(querybuf))); - ret = master_dcb->func.write(master_dcb, - (void *)gwbuf_clone(querybuf)); - - LOGIF(LT, tracelog_routed_query(router_cli_ses, - "routeQuery", - slave_dcb, - gwbuf_clone(querybuf))); - - slave_dcb->func.write(slave_dcb, (void *)querybuf); - break; - } /**< switch by packet type */ - + prop = rses_property_init(RSES_PROP_TYPE_SESCMD); + /** + * Additional reference is created to querybuf to + * prevent it from being released before properties + * are cleaned up as a part of router sessionclean-up. + */ + mysql_sescmd_init(prop, querybuf, packet_type, router_cli_ses); + + /** Lock router session */ + if (!rses_begin_locked_router_action(router_cli_ses)) + { + rses_property_done(prop); + goto return_ret; + } + /** Add sescmd property to router client session */ + rses_property_add(router_cli_ses, prop); + + /** Execute session command in master */ + if (execute_sescmd_in_backend(router_cli_ses, BE_MASTER)) + { + ret = 1; + } + else + { + /** Log error */ + } + /** Execute session command in slave */ + if (execute_sescmd_in_backend(router_cli_ses, BE_SLAVE)) + { + ret = 1; + } + else + { + /** Log error */ + } + + /** Unlock router session */ + rses_end_locked_router_action(router_cli_ses); + atomic_add(&inst->stats.n_all, 1); goto return_ret; break; @@ -1066,8 +1022,9 @@ static void clientReply( */ if (!rses_begin_locked_router_action(router_cli_ses)) { - /** is this needed ??*/ - gwbuf_consume(writebuf, gwbuf_length(writebuf)); + while ((writebuf = gwbuf_consume( + writebuf, + GWBUF_LENGTH(writebuf))) != NULL); goto lock_failed; } master_dcb = router_cli_ses->rses_dcb[BE_MASTER]; @@ -1091,7 +1048,9 @@ static void clientReply( */ if (client_dcb == NULL) { - gwbuf_consume(writebuf, gwbuf_length(writebuf)); + while ((writebuf = gwbuf_consume( + writebuf, + GWBUF_LENGTH(writebuf))) != NULL); /** Log that client was closed before reply */ return; } @@ -1111,7 +1070,7 @@ static void clientReply( goto lock_failed; } scur = rses_get_sescmd_cursor(router_cli_ses, be_type); - /** + /** * Active cursor means that reply is from session command * execution. */ @@ -1134,6 +1093,7 @@ static void clientReply( } /** Set cursor passive. */ sescmd_cursor_set_active(scur, false); + ss_dassert(!sescmd_cursor_is_active(scur)); /** Unlock router session */ rses_end_locked_router_action(router_cli_ses); @@ -1142,6 +1102,7 @@ static void clientReply( { /** Write reply to client DCB */ client_dcb->func.write(client_dcb, writebuf); + ss_dassert(!sescmd_cursor_is_active(scur)); /** Unlock router session */ rses_end_locked_router_action(router_cli_ses); LOGIF(LT, (skygw_log_write_flush( @@ -1150,12 +1111,10 @@ static void clientReply( "backend dcb %p. End of normal reply.", pthread_self(), client_dcb, - backend_dcb))); + backend_dcb))); } - return; /*< succeed */ lock_failed: - /** log that router session couldn't be locked */ - return; + return; } /** @@ -1443,6 +1402,7 @@ static void rses_end_locked_property_action( static mysql_sescmd_t* mysql_sescmd_init ( rses_property_t* rses_prop, GWBUF* sescmd_buf, + unsigned char packet_type, ROUTER_CLIENT_SES* rses) { mysql_sescmd_t* sescmd; @@ -1453,11 +1413,12 @@ static mysql_sescmd_t* mysql_sescmd_init ( sescmd->my_sescmd_prop = rses_prop; /*< reference to owning property */ // sescmd->my_sescmd_rsession = rses; #if defined(SS_DEBUG) - sescmd->my_sescmd_chk_top = CHK_NUM_MY_SESCMD; + sescmd->my_sescmd_chk_top = CHK_NUM_MY_SESCMD; sescmd->my_sescmd_chk_tail = CHK_NUM_MY_SESCMD; #endif /** Set session command buffer */ - sescmd->my_sescmd_buf = sescmd_buf; + sescmd->my_sescmd_buf = sescmd_buf; + sescmd->my_sescmd_packet_type = packet_type; return sescmd; } @@ -1624,8 +1585,25 @@ static bool execute_sescmd_in_backend( "execute_sescmd_in_backend", dcb, sescmd_cursor_clone_querybuf(scur))); - rc = dcb->func.write(dcb, sescmd_cursor_clone_querybuf(scur)); - + switch (scur->scmd_cur_cmd->my_sescmd_packet_type) { + case COM_CHANGE_USER: + rc = dcb->func.auth( + dcb, + NULL, + dcb->session, + sescmd_cursor_clone_querybuf(scur)); + break; + + case COM_QUIT: + case COM_QUERY: + case COM_INIT_DB: + default: + rc = dcb->func.write( + dcb, + sescmd_cursor_clone_querybuf(scur)); + break; + } + if (rc != 1) { succp = false; @@ -1685,7 +1663,6 @@ static bool cont_exec_sescmd_in_backend( dcb, sescmd_cursor_clone_querybuf(scur))); -// rc = dcb->func.session(dcb, sescmd_cursor_clone_querybuf(scur)); rc = dcb->func.write(dcb, sescmd_cursor_clone_querybuf(scur)); if (rc != 1) { From 90f701be8e980bc41b7b4633d7486a7c406e01c7 Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Sun, 16 Mar 2014 19:43:49 +0200 Subject: [PATCH 5/6] readwritesplit.c : router is changed so that it guarantees to keep the execution order of session commands and queries when they are routed to backend servers. In the same way it maintains the order of response packets and discards duplicate responses. For each session command a sescmd property is created and added to the end of list of session commands. List is owned by router client session and it includes all session commands from the beginning of router session. Router maintains an individual session command cursor for each backend. A cursor refers to the first session command which the corresponding backend server haven't yet responded yet. When response message arrives at any time from backend, first it is checked whether backend's cursor is active. Cursor is active if a session command is routed to backend and the backend haven't responded to it yet. If cursor is active, next it is checked whether the current session command property has been responded by other backend. If both are true, then response message is sent to client as is. If session command response is routed to client already, the arrived response is discarded. --- server/core/dcb.c | 10 +- .../include/mysql_client_server_protocol.h | 1 + server/modules/protocol/mysql_backend.c | 4 - server/modules/protocol/mysql_client.c | 48 --- server/modules/protocol/mysql_common.c | 54 ++++ .../routing/readwritesplit/readwritesplit.c | 276 ++++++++---------- 6 files changed, 182 insertions(+), 211 deletions(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index d1b5a4ffb..7c648ccd6 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -768,6 +768,8 @@ dcb_write(DCB *dcb, GWBUF *queue) saved_errno = errno; errno = 0; + if (LOG_IS_ENABLED(LOGFILE_DEBUG)) + { if (saved_errno == EPIPE) { LOGIF(LD, (skygw_log_write( LOGFILE_DEBUG, @@ -780,7 +782,12 @@ dcb_write(DCB *dcb, GWBUF *queue) dcb->fd, saved_errno, strerror(saved_errno)))); - } else if (saved_errno != EAGAIN && + } + } + if (LOG_IS_ENABLED(LOGFILE_ERROR)) + { + if (saved_errno != EPIPE && + saved_errno != EAGAIN && saved_errno != EWOULDBLOCK) { LOGIF(LE, (skygw_log_write_flush( @@ -794,6 +801,7 @@ dcb_write(DCB *dcb, GWBUF *queue) saved_errno, strerror(saved_errno)))); } + } break; } /* diff --git a/server/modules/include/mysql_client_server_protocol.h b/server/modules/include/mysql_client_server_protocol.h index e23519da3..7dd6517e0 100644 --- a/server/modules/include/mysql_client_server_protocol.h +++ b/server/modules/include/mysql_client_server_protocol.h @@ -296,3 +296,4 @@ char *gw_strend(register const char *s); int setnonblocking(int fd); void setipaddress(struct in_addr *a, char *p); int gw_read_gwbuff(DCB *dcb, GWBUF **head, int b); +GWBUF* gw_MySQL_get_next_stmt(GWBUF** p_readbuf); diff --git a/server/modules/protocol/mysql_backend.c b/server/modules/protocol/mysql_backend.c index 78216d62f..45457e665 100644 --- a/server/modules/protocol/mysql_backend.c +++ b/server/modules/protocol/mysql_backend.c @@ -567,9 +567,7 @@ gw_MySQLWrite_backend(DCB *dcb, GWBUF *queue) /*< * Now we set the last command received, from the current queue */ -// memcpy(&dcb->command, &queue->command, sizeof(dcb->command)); spinlock_release(&dcb->authlock); -// LOGIF(LD, debuglog_statements(dcb, gwbuf_clone(queue))); rc = dcb_write(dcb, queue); return rc; } @@ -847,7 +845,6 @@ static int gw_change_user(DCB *backend, SERVER *server, SESSION *in_session, GWB unsigned int auth_token_len = 0; uint8_t *auth_token = NULL; int rv = -1; - int len = 0; int auth_ret = 1; current_session = (MYSQL_session *)in_session->client->data; @@ -919,7 +916,6 @@ static int gw_session(DCB *backend_dcb, void *data) { GWBUF *queue = NULL; queue = (GWBUF *) data; -// queue->command = ROUTER_CHANGE_SESSION; backend_dcb->func.write(backend_dcb, queue); return 1; diff --git a/server/modules/protocol/mysql_client.c b/server/modules/protocol/mysql_client.c index ecea34972..fbac8853b 100644 --- a/server/modules/protocol/mysql_client.c +++ b/server/modules/protocol/mysql_client.c @@ -57,7 +57,6 @@ static int route_by_statement( ROUTER_OBJECT* router, void* rsession, GWBUF* read_buf); -static GWBUF* gw_MySQL_get_next_stmt(GWBUF** buffer); /* * The "module object" for the mysqld client protocol module. @@ -1248,51 +1247,6 @@ return_rc: return rc; } -/** - * Remove the first mysql statement from buffer. Return pointer to the removed - * statement or NULL if buffer is empty. - * - * Clone buf, calculate the length of included mysql stmt, and point the - * statement with cloned buffer. Move the start pointer of buf accordingly - * so that it only cover the remaining buffer. - * - */ -static GWBUF* gw_MySQL_get_next_stmt( - GWBUF** p_readbuf) -{ - GWBUF* stmtbuf; - size_t buflen; - size_t strlen; - uint8_t* packet; - - if (*p_readbuf == NULL) - { - stmtbuf = NULL; - goto return_stmtbuf; - } - CHK_GWBUF(*p_readbuf); - - if (GWBUF_EMPTY(*p_readbuf)) - { - stmtbuf = NULL; - goto return_stmtbuf; - } - buflen = GWBUF_LENGTH((*p_readbuf)); - packet = GWBUF_DATA((*p_readbuf)); - strlen = MYSQL_GET_PACKET_LEN(packet); - - /** vraa :Multi-packet stmt is not supported as of 7.3.14 */ - if (strlen-1 > buflen-5) - { - stmtbuf = NULL; - goto return_stmtbuf; - } - stmtbuf = gwbuf_clone_portion(*p_readbuf, 0, strlen+4); - *p_readbuf = gwbuf_consume(*p_readbuf, strlen+4); - -return_stmtbuf: - return stmtbuf; -} /** * Detect if buffer includes partial mysql packet or multiple packets. @@ -1318,13 +1272,11 @@ static int route_by_statement( CHK_GWBUF(stmtbuf); payload = (uint8_t *)GWBUF_DATA(stmtbuf); - len += MYSQL_GET_PACKET_LEN(payload); /** * If message is longer than read data, suspend routing and * add statement buffer to wait queue. */ rc = router->routeQuery(router_instance, rsession, stmtbuf); - len = 0; /*< if routed, reset the length indicator */ } while (readbuf != NULL); diff --git a/server/modules/protocol/mysql_common.c b/server/modules/protocol/mysql_common.c index 7bfb3666c..b208da9c7 100644 --- a/server/modules/protocol/mysql_common.c +++ b/server/modules/protocol/mysql_common.c @@ -1209,3 +1209,57 @@ mysql_send_auth_error (DCB *dcb, int packet_number, int in_affected_rows, const return sizeof(mysql_packet_header) + mysql_payload_size; } + + +/** + * Remove the first mysql statement from buffer. Return pointer to the removed + * statement or NULL if buffer is empty. + * + * Clone buf, calculate the length of included mysql stmt, and point the + * statement with cloned buffer. Move the start pointer of buf accordingly + * so that it only cover the remaining buffer. + * + */ +GWBUF* gw_MySQL_get_next_stmt( + GWBUF** p_readbuf) +{ + GWBUF* stmtbuf; + size_t buflen; + size_t strlen; + uint8_t* packet; + + if (*p_readbuf == NULL) + { + stmtbuf = NULL; + goto return_stmtbuf; + } + CHK_GWBUF(*p_readbuf); + + if (GWBUF_EMPTY(*p_readbuf)) + { + stmtbuf = NULL; + goto return_stmtbuf; + } + buflen = GWBUF_LENGTH((*p_readbuf)); + packet = GWBUF_DATA((*p_readbuf)); + strlen = MYSQL_GET_PACKET_LEN(packet); + + if (strlen+4 == buflen) + { + stmtbuf = *p_readbuf; + *p_readbuf = NULL; + goto return_stmtbuf; + } + /** vraa :Multi-packet stmt is not supported as of 7.3.14 */ + if (strlen-1 > buflen-5) + { + stmtbuf = NULL; + goto return_stmtbuf; + } + stmtbuf = gwbuf_clone_portion(*p_readbuf, 0, strlen+4); + *p_readbuf = gwbuf_consume(*p_readbuf, strlen+4); + +return_stmtbuf: + return stmtbuf; +} + diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index 1548eac57..42de6a8fa 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -677,43 +677,6 @@ static int routeQuery( switch (qtype) { case QUERY_TYPE_WRITE: -#if 0 - /** - * Running this block cause deadlock because read mutex is - * on hold. This doesn't serialize subsequent session commands - * and queries if there are multiple session commands and other - * backend starts to lag behind. vraa : 14.3.13 - */ - /** - * Wait until master has executed all its session commands. - * TODO: if master fails it needs to be detected in the loop. - */ - if (!rses_begin_locked_router_action(router_cli_ses)) - { - goto return_ret; - } - - while (sescmd_cursor_is_active(rses_get_sescmd_cursor( - router_cli_ses, - BE_MASTER))) - { - rses_end_locked_router_action(router_cli_ses); - - LOGIF(LD, (skygw_log_write( - LOGFILE_DEBUG, - "%lu [routeQuery:rwsplit] Session command is " - "active in MASTER. Waiting in loop.", - pthread_self()))); - - usleep(10); - - if (!rses_begin_locked_router_action(router_cli_ses)) - { - goto return_ret; - } - } - rses_end_locked_router_action(router_cli_ses); -#endif LOGIF(LT, (skygw_log_write( LOGFILE_TRACE, "%lu [routeQuery:rwsplit] Query type\t%s, " @@ -733,53 +696,24 @@ static int routeQuery( break; case QUERY_TYPE_READ: - LOGIF(LT, (skygw_log_write( + LOGIF(LT, (skygw_log_write_flush( LOGFILE_TRACE, "%lu [routeQuery:rwsplit] Query type\t%s, " "routing to Slave.", pthread_self(), STRQTYPE(qtype)))); - while (true) - { - if (!rses_begin_locked_router_action(router_cli_ses)) - { - goto return_ret; - } - /** - * If session command is being executed in slave - * route to master. - */ - if (!sescmd_cursor_is_active(rses_get_sescmd_cursor( - router_cli_ses, - BE_SLAVE))) - { - rses_end_locked_router_action(router_cli_ses); - LOGIF(LT, tracelog_routed_query(router_cli_ses, - "routeQuery", - slave_dcb, - gwbuf_clone(querybuf))); - - ret = slave_dcb->func.write(slave_dcb, querybuf); - atomic_add(&inst->stats.n_slave, 1); - break; - } - else if (!sescmd_cursor_is_active(rses_get_sescmd_cursor( - router_cli_ses, - BE_MASTER))) - - { - rses_end_locked_router_action(router_cli_ses); - LOGIF(LT, tracelog_routed_query(router_cli_ses, - "routeQuery", - master_dcb, - gwbuf_clone(querybuf))); - - ret = slave_dcb->func.write(master_dcb, querybuf); - atomic_add(&inst->stats.n_master, 1); - break; - } - rses_end_locked_router_action(router_cli_ses); - } /*< while (true) */ + LOGIF(LT, tracelog_routed_query(router_cli_ses, + "routeQuery", + slave_dcb, + gwbuf_clone(querybuf))); + ret = slave_dcb->func.write(slave_dcb, querybuf); + LOGIF(LT, (skygw_log_write_flush( + LOGFILE_TRACE, + "%lu [routeQuery:rwsplit] Routed.", + pthread_self()))); + + + atomic_add(&inst->stats.n_slave, 1); goto return_ret; break; @@ -807,7 +741,26 @@ static int routeQuery( slave_dcb, STRQTYPE(qtype), STRPACKETTYPE(packet_type)))); + /** + * COM_QUIT is one-way message. Server doesn't respond to that. + * Therefore reply processing is unnecessary and session + * command property is not needed. It is just routed to both + * backends. + */ + if (packet_type == COM_QUIT) + { + int rc; + int rc2; + rc = master_dcb->func.write(master_dcb, gwbuf_clone(querybuf)); + rc2 = slave_dcb->func.write(slave_dcb, gwbuf_clone(querybuf)); + + if (rc == 1 && rc == rc2) + { + ret = 1; + } + goto return_ret; + } prop = rses_property_init(RSES_PROP_TYPE_SESCMD); /** * Additional reference is created to querybuf to @@ -1063,6 +1016,11 @@ static void clientReply( { be_type = BE_SLAVE; } + LOGIF(LT, tracelog_routed_query(router_cli_ses, + "reply_by_statement", + backend_dcb, + gwbuf_clone(writebuf))); + /** Lock router session */ if (!rses_begin_locked_router_action(router_cli_ses)) { @@ -1076,27 +1034,15 @@ static void clientReply( */ if (sescmd_cursor_is_active(scur)) { - mysql_sescmd_t* scmd = sescmd_cursor_get_command(scur); - sescmd_reply_to_client(client_dcb, scmd, writebuf); - - /** Read next sescmd property */ - while (sescmd_cursor_next(scur)) - { - if (!cont_exec_sescmd_in_backend(router_cli_ses, be_type)) - { - /** Log error */ - } - else - { - /** Log execution of pending sescmd */ - } - } - /** Set cursor passive. */ - sescmd_cursor_set_active(scur, false); - ss_dassert(!sescmd_cursor_is_active(scur)); - - /** Unlock router session */ - rses_end_locked_router_action(router_cli_ses); + mysql_sescmd_t* scmd = sescmd_cursor_get_command(scur); + sescmd_reply_to_client(client_dcb, scmd, writebuf); + /** When sescmd list is empty set cursor passive. */ + if (!sescmd_cursor_next(scur)) + { + sescmd_cursor_set_active(scur, false); + } + /** Unlock router session */ + rses_end_locked_router_action(router_cli_ses); } else if (client_dcb != NULL) { @@ -1411,7 +1357,6 @@ static mysql_sescmd_t* mysql_sescmd_init ( /** Can't call rses_property_get_sescmd with uninitialized sescmd */ sescmd = &rses_prop->rses_prop_data.sescmd; sescmd->my_sescmd_prop = rses_prop; /*< reference to owning property */ -// sescmd->my_sescmd_rsession = rses; #if defined(SS_DEBUG) sescmd->my_sescmd_chk_top = CHK_NUM_MY_SESCMD; sescmd->my_sescmd_chk_tail = CHK_NUM_MY_SESCMD; @@ -1452,21 +1397,43 @@ static bool sescmd_reply_to_client( CHK_GWBUF(writebuf); ss_dassert(SPINLOCK_IS_LOCKED( &scmd->my_sescmd_prop->rses_prop_rsession->rses_lock)); - - if (!scmd->my_sescmd_is_replied) - { - client_dcb->func.write(client_dcb, writebuf); - scmd->my_sescmd_is_replied = true; - succp = true; - LOGIF(LT, (skygw_log_write( - LOGFILE_TRACE, - "%lu [sescmd_reply_to_client] Replied to client dcb %p.", - pthread_self(), - client_dcb))); - } - else + /** + * This depends on MySQL protoocl and doesn't work with others. + * TODO: write a function to MySQL protocol module and add + * a new protocol function 'discard n first messages'. + */ + if (scmd->my_sescmd_is_replied) { + size_t len; + uint8_t* packet; + /** + * Skip reply message because it is duplicate of alredy + * replied message. + */ + packet = (uint8_t*)writebuf->start; + len = packet[0]; + len += packet[1]*256; + len += packet[2]*256*256; + writebuf = gwbuf_consume(writebuf, len+4); + + if (writebuf == NULL || GWBUF_EMPTY(writebuf)) + { + succp = true; + goto return_succp; + } } + + client_dcb->func.write(client_dcb, writebuf); + scmd->my_sescmd_is_replied = true; + succp = true; + LOGIF(LT, (skygw_log_write_flush( + LOGFILE_TRACE, + "%lu [sescmd_reply_to_client] Replied cmd %p to client dcb %p.", + pthread_self(), + scmd, + client_dcb))); + +return_succp: return succp; } @@ -1575,52 +1542,44 @@ static bool execute_sescmd_in_backend( succp = false; goto return_succp; } - - if (!sescmd_cursor_is_active(scur)) - { - /** Cursor is left active when function returns. */ - sescmd_cursor_set_active(scur, true); - - LOGIF(LT, tracelog_routed_query(rses, - "execute_sescmd_in_backend", - dcb, - sescmd_cursor_clone_querybuf(scur))); - switch (scur->scmd_cur_cmd->my_sescmd_packet_type) { - case COM_CHANGE_USER: - rc = dcb->func.auth( - dcb, - NULL, - dcb->session, - sescmd_cursor_clone_querybuf(scur)); - break; - - case COM_QUIT: - case COM_QUERY: - case COM_INIT_DB: - default: - rc = dcb->func.write( - dcb, - sescmd_cursor_clone_querybuf(scur)); - break; - } - - if (rc != 1) - { - succp = false; - } - } - else + if (!sescmd_cursor_is_active(scur)) { - LOGIF(LD, (skygw_log_write_flush( - LOGFILE_DEBUG, - "%lu [routeQuery] Couldn't directly send SESSION " - "WRITER command to dcb %p because session command " - "cursor was executing previous command. Added " - "command to the queue.", - pthread_self(), - dcb))); + /** Cursor is left active when function returns. */ + sescmd_cursor_set_active(scur, true); } + LOGIF(LT, tracelog_routed_query(rses, + "execute_sescmd_in_backend", + dcb, + sescmd_cursor_clone_querybuf(scur))); + switch (scur->scmd_cur_cmd->my_sescmd_packet_type) { + case COM_CHANGE_USER: + rc = dcb->func.auth( + dcb, + NULL, + dcb->session, + sescmd_cursor_clone_querybuf(scur)); + break; + case COM_QUIT: + case COM_QUERY: + case COM_INIT_DB: + default: + rc = dcb->func.write( + dcb, + sescmd_cursor_clone_querybuf(scur)); + break; + } + LOGIF(LT, (skygw_log_write_flush( + LOGFILE_TRACE, + "%lu [execute_sescmd_in_backend] Routed %s cmd %p.", + pthread_self(), + STRPACKETTYPE(scur->scmd_cur_cmd->my_sescmd_packet_type), + scur->scmd_cur_cmd))); + + if (rc != 1) + { + succp = false; + } return_succp: return succp; } @@ -1664,7 +1623,8 @@ static bool cont_exec_sescmd_in_backend( sescmd_cursor_clone_querybuf(scur))); rc = dcb->func.write(dcb, sescmd_cursor_clone_querybuf(scur)); - if (rc != 1) + + if (rc != 1) { succp = false; } From 67d9b3afb94559f13b44780c9c54760b667a068a Mon Sep 17 00:00:00 2001 From: VilhoRaatikka Date: Mon, 17 Mar 2014 22:26:32 +0200 Subject: [PATCH 6/6] Fixed bug in how response messages from backends were handled. Now messages are examined if session command cursor is active . That means that (at least) the first packet in the message is response to session command. Then if the response is alread y forwarded to client it is discarded. Otherwise it is routed to client and the command is marked as responded so that the o ther backend knows to discard its duplicate response. --- .../routing/readwritesplit/readwritesplit.c | 234 ++++++++---------- 1 file changed, 106 insertions(+), 128 deletions(-) diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index 42de6a8fa..ecd0a4508 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -141,10 +141,10 @@ static mysql_sescmd_t* sescmd_cursor_get_command( static bool sescmd_cursor_next( sescmd_cursor_t* scur); -static bool sescmd_reply_to_client( - DCB* client_dcb, - mysql_sescmd_t* scmd, - GWBUF* writebuf); +static GWBUF* sescmd_cursor_process_replies( + DCB* client_dcb, + GWBUF* replybuf, + sescmd_cursor_t* scur); static bool cont_exec_sescmd_in_backend( ROUTER_CLIENT_SES* rses, @@ -1020,37 +1020,34 @@ static void clientReply( "reply_by_statement", backend_dcb, gwbuf_clone(writebuf))); - /** Lock router session */ if (!rses_begin_locked_router_action(router_cli_ses)) { /** Log to debug that router was closed */ goto lock_failed; } + scur = rses_get_sescmd_cursor(router_cli_ses, be_type); /** * Active cursor means that reply is from session command - * execution. + * execution. Majority of the time there are no session commands + * being executed. */ if (sescmd_cursor_is_active(scur)) { - mysql_sescmd_t* scmd = sescmd_cursor_get_command(scur); - sescmd_reply_to_client(client_dcb, scmd, writebuf); - /** When sescmd list is empty set cursor passive. */ - if (!sescmd_cursor_next(scur)) - { - sescmd_cursor_set_active(scur, false); - } - /** Unlock router session */ - rses_end_locked_router_action(router_cli_ses); + writebuf = sescmd_cursor_process_replies(client_dcb, + writebuf, + scur); + } - else if (client_dcb != NULL) + /** Unlock router session */ + rses_end_locked_router_action(router_cli_ses); + + if (writebuf != NULL && client_dcb != NULL) { /** Write reply to client DCB */ client_dcb->func.write(client_dcb, writebuf); - ss_dassert(!sescmd_cursor_is_active(scur)); - /** Unlock router session */ - rses_end_locked_router_action(router_cli_ses); + LOGIF(LT, (skygw_log_write_flush( LOGFILE_TRACE, "%lu [clientReply:rwsplit] client dcb %p, " @@ -1059,6 +1056,7 @@ static void clientReply( client_dcb, backend_dcb))); } + lock_failed: return; } @@ -1307,7 +1305,10 @@ static void rses_property_add( } } -/** Router session must be locked */ +/** + * Router session must be locked. + * Return session command pointer if succeed, NULL if failed. + */ static mysql_sescmd_t* rses_property_get_sescmd( rses_property_t* prop) { @@ -1377,66 +1378,89 @@ static void mysql_sescmd_done( memset(sescmd, 0, sizeof(mysql_sescmd_t)); } - - -/** - * Write session command reply from backend to client if command haven't yet - * been replied. - * Return true if succeed, false if command was already replied. - * - * Router session must be locked */ -static bool sescmd_reply_to_client( - DCB* client_dcb, - mysql_sescmd_t* scmd, - GWBUF* writebuf) +/** + * All cases where backend message starts at least with one response to session + * command are handled here. + * Read session commands from property list. If command is already replied, + * discard packet. Else send reply to client. In both cases move cursor forward + * until all session command replies are handled. + */ +static GWBUF* sescmd_cursor_process_replies( + DCB* client_dcb, + GWBUF* replybuf, + sescmd_cursor_t* scur) { - bool succp = false; + const size_t headerlen = 4; /*< mysql packet header */ + uint8_t* packet; + size_t packetlen; + mysql_sescmd_t* scmd; - CHK_DCB(client_dcb); - CHK_MYSQL_SESCMD(scmd); - CHK_GWBUF(writebuf); - ss_dassert(SPINLOCK_IS_LOCKED( - &scmd->my_sescmd_prop->rses_prop_rsession->rses_lock)); - /** - * This depends on MySQL protoocl and doesn't work with others. - * TODO: write a function to MySQL protocol module and add - * a new protocol function 'discard n first messages'. + ss_dassert(SPINLOCK_IS_LOCKED(&(scur->scmd_cur_rses->rses_lock))); + + scmd = sescmd_cursor_get_command(scur); + + CHK_DCB(client_dcb); + CHK_GWBUF(replybuf); + + /** + * Walk through packets in the message and the list of session + *commands. */ - if (scmd->my_sescmd_is_replied) - { - size_t len; - uint8_t* packet; - /** - * Skip reply message because it is duplicate of alredy - * replied message. - */ - packet = (uint8_t*)writebuf->start; - len = packet[0]; - len += packet[1]*256; - len += packet[2]*256*256; - writebuf = gwbuf_consume(writebuf, len+4); - - if (writebuf == NULL || GWBUF_EMPTY(writebuf)) + while (scmd != NULL && replybuf != NULL) + { + if (scmd->my_sescmd_is_replied) { - succp = true; - goto return_succp; + /** + * Discard heading packets if their related command is + * already replied. + */ + CHK_GWBUF(replybuf); + packet = (uint8_t *)GWBUF_DATA(replybuf); + packetlen = packet[0]+packet[1]*256+packet[2]*256*256; + replybuf = gwbuf_consume(replybuf, packetlen+headerlen); + + LOGIF(LT, (skygw_log_write_flush( + LOGFILE_TRACE, + "%lu [sescmd_cursor_process_replies] cmd %p " + "is already replied. Discarded %d bytes from " + "the %s replybuffer.", + pthread_self(), + scmd, + packetlen+headerlen, + STRBETYPE(scur->scmd_cur_be_type)))); } - } - - client_dcb->func.write(client_dcb, writebuf); - scmd->my_sescmd_is_replied = true; - succp = true; - LOGIF(LT, (skygw_log_write_flush( - LOGFILE_TRACE, - "%lu [sescmd_reply_to_client] Replied cmd %p to client dcb %p.", - pthread_self(), - scmd, - client_dcb))); - -return_succp: - return succp; + else + { + /** Mark the rest session commands as replied */ + scmd->my_sescmd_is_replied = true; + LOGIF(LT, (skygw_log_write_flush( + LOGFILE_TRACE, + "%lu [sescmd_cursor_process_replies] Marked " + "cmd %p to as replied. Left message to %s's " + "buffer for reply.", + pthread_self(), + scmd, + STRBETYPE(scur->scmd_cur_be_type)))); + } + + if (sescmd_cursor_next(scur)) + { + scmd = sescmd_cursor_get_command(scur); + } + else + { + scmd = NULL; + /** All session commands are replied */ + scur->scmd_cur_active = false; + } + } + ss_dassert(replybuf == NULL || *scur->scmd_cur_ptr_property == NULL); + + return replybuf; } + + /** * Get the address of current session command. * @@ -1445,9 +1469,8 @@ static mysql_sescmd_t* sescmd_cursor_get_command( sescmd_cursor_t* scur) { mysql_sescmd_t* scmd; - - ss_dassert(SPINLOCK_IS_LOCKED(&scur->scmd_cur_rses->rses_lock)); + ss_dassert(SPINLOCK_IS_LOCKED(&(scur->scmd_cur_rses->rses_lock))); scur->scmd_cur_cmd = rses_property_get_sescmd(*scur->scmd_cur_ptr_property); CHK_MYSQL_SESCMD(scur->scmd_cur_cmd); @@ -1524,7 +1547,7 @@ static bool execute_sescmd_in_backend( bool succp = true; int rc = 0; sescmd_cursor_t* scur; - + dcb = rses->rses_dcb[be_type]; CHK_DCB(dcb); @@ -1535,14 +1558,15 @@ static bool execute_sescmd_in_backend( * Get cursor pointer and copy of command buffer to cursor. */ scur = rses_get_sescmd_cursor(rses, be_type); - - /** Return if there are no pending ses commands */ + + /** Return if there are no pending ses commands */ if (sescmd_cursor_get_command(scur) == NULL) { succp = false; goto return_succp; } - if (!sescmd_cursor_is_active(scur)) + + if (!sescmd_cursor_is_active(scur)) { /** Cursor is left active when function returns. */ sescmd_cursor_set_active(scur, true); @@ -1551,6 +1575,7 @@ static bool execute_sescmd_in_backend( "execute_sescmd_in_backend", dcb, sescmd_cursor_clone_querybuf(scur))); + switch (scur->scmd_cur_cmd->my_sescmd_packet_type) { case COM_CHANGE_USER: rc = dcb->func.auth( @@ -1584,57 +1609,10 @@ return_succp: return succp; } -/** - * Execute session commands when cursor is already active. - * - * Router session must be locked - * - * Return true if there was pending sescmd and sending command to - * backend server succeed. Otherwise false. - */ -static bool cont_exec_sescmd_in_backend( - ROUTER_CLIENT_SES* rses, - backend_type_t be_type) -{ - DCB* dcb; - bool succp = true; - int rc = 0; - sescmd_cursor_t* scur; - - dcb = rses->rses_dcb[be_type]; - - CHK_DCB(dcb); - CHK_CLIENT_RSES(rses); - ss_dassert(SPINLOCK_IS_LOCKED(&rses->rses_lock)); - - scur = rses_get_sescmd_cursor(rses, be_type); - ss_dassert(sescmd_cursor_is_active(scur)); - - /** Return if there are no pending ses commands */ - if (scur->scmd_cur_cmd == NULL) - { - succp = false; - goto return_succp; - } - - LOGIF(LT, tracelog_routed_query(rses, - "cont_exec_sescmd_in_backend", - dcb, - sescmd_cursor_clone_querybuf(scur))); - - rc = dcb->func.write(dcb, sescmd_cursor_clone_querybuf(scur)); - - if (rc != 1) - { - succp = false; - } -return_succp: - return succp; -} - /** * Moves cursor to next property and copied address of its sescmd to cursor. * Current propery must be non-null. + * If current property is the last on the list, *scur->scmd_ptr_property == NULL * * Router session must be locked */ @@ -1695,7 +1673,7 @@ static bool sescmd_cursor_next( } else { - /** Log error, sescmd shouldn't be NULL */ + ss_dassert(false); /*< Log error, sescmd shouldn't be NULL */ } return_succp: return succp;