From 8bf73ea154b2f04e80765287fed9eb5de25012fe Mon Sep 17 00:00:00 2001 From: vraatikka Date: Tue, 17 Sep 2013 00:07:56 +0300 Subject: [PATCH] Replaced write system function with wrapper gw_write. It allows for generating failures by using telnet commands, fail backendfd, fail clientfd, which are available in debug build only. --- server/core/dcb.c | 60 +++++++++++++++++++++---- server/core/gateway.c | 8 +++- server/core/utils.c | 32 +++++++++++++ server/include/dcb.h | 19 +++++--- server/include/gw.h | 2 + server/modules/protocol/httpd.c | 4 +- server/modules/protocol/mysql_backend.c | 18 ++++---- server/modules/protocol/mysql_client.c | 41 ++++++++++++----- server/modules/protocol/mysql_common.c | 2 +- server/modules/protocol/telnetd.c | 4 +- server/modules/routing/debugcmd.c | 49 +++++++++++++++++--- 11 files changed, 197 insertions(+), 42 deletions(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index 49342299c..f614b4682 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -450,7 +450,6 @@ dcb_connect(SERVER *server, SESSION *session, const char *protocol) { DCB *dcb; GWPROTOCOL *funcs; -int val; int fd; if ((dcb = dcb_alloc(DCB_ROLE_REQUEST_HANDLER)) == NULL) @@ -506,7 +505,7 @@ int fd; session->client, session->client->fd); } - ss_dassert(dcb->fd = -1); + ss_dassert(dcb->fd == -1); /** * Successfully connected to backend. Assign file descriptor to dcb */ @@ -670,14 +669,32 @@ int w, saved_errno = 0; */ while (queue != NULL) { +#if defined(SS_DEBUG) + if (dcb->session) { + if (dcb_isclient(dcb)) { + if (fail_next_client_fd) { + dcb_fake_write_errno[dcb->fd] = 32; + dcb_fake_write_ev[dcb->fd] = 29; + fail_next_client_fd = false; + } + } else { + if (fail_next_backend_fd) { + dcb_fake_write_errno[dcb->fd] = 32; + dcb_fake_write_ev[dcb->fd] = 29; + fail_next_backend_fd = false; + } + } + } +#endif /* SS_DEBUG */ len = GWBUF_LENGTH(queue); - GW_NOINTR_CALL(w = write(dcb->fd, GWBUF_DATA(queue), len); dcb->stats.n_writes++); + GW_NOINTR_CALL(w = gw_write(dcb->fd, GWBUF_DATA(queue), len); + dcb->stats.n_writes++); saved_errno = errno; errno = 0; if (w < 0) { - skygw_log_write( + skygw_log_write_flush( LOGFILE_ERROR, "%lu [dcb_write] Write to dcb %p fd %d " "failed due errno %d, %s", @@ -724,7 +741,7 @@ int w, saved_errno = 0; } /** - * Drain the write queue of a DCB. THis is called as part of the EPOLLOUT handling + * Drain the write queue of a DCB. This is called as part of the EPOLLOUT handling * of a socket and will try to send any buffered data from the write queue * up until the point the write would block. * @@ -751,17 +768,20 @@ int saved_errno = 0; while (dcb->writeq != NULL) { len = GWBUF_LENGTH(dcb->writeq); - GW_NOINTR_CALL(w = write(dcb->fd, GWBUF_DATA(dcb->writeq), len);); + GW_NOINTR_CALL(w = gw_write(dcb->fd, GWBUF_DATA(dcb->writeq), len);); saved_errno = errno; + errno = 0; + if (w < 0) { skygw_log_write( LOGFILE_ERROR, "%lu [dcb_drain_writeq] Write to fd %d " - "failed due errno %d", + "failed due errno %d, %s", pthread_self(), dcb->fd, - saved_errno); + saved_errno, + strerror(saved_errno)); break; } @@ -1169,3 +1189,27 @@ static bool dcb_set_state_nomutex( return succp; } +int gw_write( + int fd, + const void* buf, + size_t nbytes) +{ + int w; +#if defined(SS_DEBUG) + if (dcb_fake_write_errno[fd] != 0) { + ss_dassert(dcb_fake_write_ev[fd] != 0); + w = write(fd, buf, nbytes/2); /**< leave peer to read missing bytes */ + + if (w > 0) { + w = -1; + errno = dcb_fake_write_errno[fd]; + } + } else { + w = write(fd, buf, nbytes); + } +#else + w = write(fd, buf, nbytes); +#endif /* SS_DEBUG && SS_TEST */ + return w; +} + diff --git a/server/core/gateway.c b/server/core/gateway.c index ad9330ecd..04a6f54bc 100644 --- a/server/core/gateway.c +++ b/server/core/gateway.c @@ -109,7 +109,7 @@ static void sighup_handler (int i) } static void sigterm_handler (int i) { -extern void shutdown_gateway(); + extern void shutdown_gateway(); skygw_log_write( LOGFILE_ERROR, "Signal SIGTERM %i received ...Exiting!\n", i); shutdown_gateway(); @@ -212,6 +212,10 @@ int l; #if defined(SS_DEBUG) memset(conn_open, 0, sizeof(bool)*1024); +memset(dcb_fake_write_errno, 0, sizeof(unsigned char)*1024); +memset(dcb_fake_write_ev, 0, sizeof(__int32_t)*1024); +fail_next_backend_fd = false; +fail_next_client_fd = false; #endif l = atexit(skygw_logmanager_exit); @@ -412,7 +416,7 @@ memset(conn_open, 0, sizeof(bool)*1024); /* * Start the services that were created above */ - n_services = serviceStartAll(); + n_services = serviceStartAll(); skygw_log_write(LOGFILE_MESSAGE, "Started modules succesfully."); /** diff --git a/server/core/utils.c b/server/core/utils.c index ea8ce837d..49e7c1c42 100644 --- a/server/core/utils.c +++ b/server/core/utils.c @@ -73,6 +73,8 @@ int setnonblocking(int fd) { return 0; } + + char *gw_strend(register const char *s) { while (*s++); return (char*) (s-1); @@ -188,3 +190,33 @@ void gw_sha1_2_str(const uint8_t *in, int in_len, const uint8_t *in2, int in2_le memcpy(out, hash, SHA_DIGEST_LENGTH); } + + +/** + * @node Gets errno corresponding to latest socket error + * + * Parameters: + * @param fd - in, use + * socket to examine + * + * @return errno + * + * + * @details (write detailed description here) + * + */ +int gw_getsockerrno( + int fd) +{ + int eno = 0; + socklen_t elen = sizeof(eno); + + if (fd <= 0) { + goto return_eno; + } + + getsockopt(fd, SOL_SOCKET, SO_ERROR, (void *)&eno, &elen); + +return_eno: + return eno; +} diff --git a/server/include/dcb.h b/server/include/dcb.h index 6845cf12a..8e73d3a7e 100644 --- a/server/include/dcb.h +++ b/server/include/dcb.h @@ -185,19 +185,26 @@ typedef struct dcb { #endif } DCB; +#if defined(SS_DEBUG) +unsigned char dcb_fake_write_errno[1024]; +__int32_t dcb_fake_write_ev[1024]; +bool fail_next_backend_fd; +bool fail_next_client_fd; +#endif /* A few useful macros */ #define DCB_SESSION(x) (x)->session #define DCB_PROTOCOL(x, type) (type *)((x)->protocol) #define DCB_ISZOMBIE(x) ((x)->state == DCB_STATE_ZOMBIE) +int gw_write(int fd, const void* buf, size_t nbytes); +int dcb_write(DCB *, GWBUF *); DCB *dcb_alloc(dcb_role_t); -void dcb_free(DCB *); /* Free a DCB */ -DCB *dcb_connect(struct server *, struct session *, const char *); /* prepare Backend connection */ -int dcb_read(DCB *, GWBUF **); /* Generic read routine */ -int dcb_write(DCB *, GWBUF *); /* Generic write routine */ -int dcb_drain_writeq(DCB *); /* Generic write routine */ -void dcb_close(DCB *); /* Generic close functionality */ +void dcb_free(DCB *); +DCB *dcb_connect(struct server *, struct session *, const char *); +int dcb_read(DCB *, GWBUF **); +int dcb_drain_writeq(DCB *); +void dcb_close(DCB *); void dcb_process_zombies(int); /* Process Zombies */ void printAllDCBs(); /* Debug to print all DCB in the system */ void printDCB(DCB *); /* Debug print routine */ diff --git a/server/include/gw.h b/server/include/gw.h index 4115ff293..b03a924f7 100644 --- a/server/include/gw.h +++ b/server/include/gw.h @@ -59,3 +59,5 @@ int MySQLWrite(DCB *dcb, GWBUF *queue); int gw_write_backend_event(DCB *dcb); int gw_read_backend_event(DCB *dcb); int setnonblocking(int fd); +int gw_write(int fd, const void* buf, size_t nbytes); +int gw_getsockerrno(int fd); diff --git a/server/modules/protocol/httpd.c b/server/modules/protocol/httpd.c index 0e6f7d379..143d39da7 100644 --- a/server/modules/protocol/httpd.c +++ b/server/modules/protocol/httpd.c @@ -267,7 +267,9 @@ httpd_write_event(DCB *dcb) static int httpd_write(DCB *dcb, GWBUF *queue) { - return dcb_write(dcb, queue); + int rc; + rc = dcb_write(dcb, queue); + return rc; } /** diff --git a/server/modules/protocol/mysql_backend.c b/server/modules/protocol/mysql_backend.c index 33e8664aa..228fff913 100644 --- a/server/modules/protocol/mysql_backend.c +++ b/server/modules/protocol/mysql_backend.c @@ -149,11 +149,10 @@ static int gw_read_backend_event(DCB *dcb) { CHK_DCB(dcb); CHK_SESSION(dcb->session); - ss_info_dassert(dcb->session != NULL, - "Backend dcb doesn't have session"); backend_protocol = (MySQLProtocol *) dcb->protocol; - + CHK_PROTOCOL(backend_protocol); + /** return only with complete session */ current_session = gw_get_shared_session_auth_info(dcb); ss_dassert(current_session != NULL); @@ -180,7 +179,8 @@ static int gw_read_backend_event(DCB *dcb) { current_session->db, current_session->user, current_session->client_sha1, - backend_protocol) != 0) { + backend_protocol) != 0) + { backend_protocol->state = MYSQL_AUTH_FAILED; rc = 1; } else { @@ -411,6 +411,7 @@ static int gw_MySQLWrite_backend(DCB *dcb, GWBUF *queue) { MySQLProtocol *backend_protocol = dcb->protocol; + int rc; /** * Don't write to backend if backend_dcb is not in poll set anymore. @@ -444,8 +445,8 @@ gw_MySQLWrite_backend(DCB *dcb, GWBUF *queue) memcpy(&dcb->command, &queue->command, sizeof(dcb->command)); spinlock_release(&dcb->authlock); - - return dcb_write(dcb, queue); + rc = dcb_write(dcb, queue); + return rc; } /** @@ -631,6 +632,7 @@ static void backend_set_delayqueue(DCB *dcb, GWBUF *queue) { static int backend_write_delayqueue(DCB *dcb) { GWBUF *localq = NULL; + int rc; spinlock_acquire(&dcb->delayqlock); @@ -644,8 +646,8 @@ static int backend_write_delayqueue(DCB *dcb) memcpy(&dcb->command, &localq->command, sizeof(dcb->command)); spinlock_release(&dcb->delayqlock); - - return dcb_write(dcb, localq); + rc = dcb_write(dcb, localq); + return rc; } diff --git a/server/modules/protocol/mysql_client.c b/server/modules/protocol/mysql_client.c index 369f7e190..884d3fca3 100644 --- a/server/modules/protocol/mysql_client.c +++ b/server/modules/protocol/mysql_client.c @@ -461,7 +461,7 @@ int w, saved_errno = 0; { len = GWBUF_LENGTH(queue); GW_NOINTR_CALL( - w = write(dcb->fd,GWBUF_DATA(queue), len); + w = gw_write(dcb->fd,GWBUF_DATA(queue), len); dcb->stats.n_writes++); saved_errno = errno; if (w < 0) @@ -642,12 +642,17 @@ int gw_read_client_event(DCB* dcb) { /* len = GWBUF_LENGTH(queue); */ ptr_buff = GWBUF_DATA(queue); - /* get mysql commang at fourth byte */ + /* get mysql commang at fifth byte */ if (ptr_buff) { mysql_command = ptr_buff[4]; } if (mysql_command == '\x03') { + /** + * SQL Trace here. + * Length can be calculated and it must be passed as + * argument. + */ /* this is a standard MySQL query !!!! */ } /** @@ -716,27 +721,43 @@ return_rc: /////////////////////////////////////////////// // client write event to Client triggered by EPOLLOUT ////////////////////////////////////////////// +/** + * @node Client's fd became writable, and EPOLLOUT event + * arrived. As a consequence, client input buffer (writeq) is flushed. + * + * Parameters: + * @param dcb - in, use + * client dcb + * + * @return constantly 1 + * + * + * @details (write detailed description here) + * + */ int gw_write_client_event(DCB *dcb) { MySQLProtocol *protocol = NULL; CHK_DCB(dcb); + ss_dassert(dcb->state != DCB_STATE_DISCONNECTED); + if (dcb == NULL) { fprintf(stderr, "DCB is NULL, return\n"); - return 1; + goto return_1; } - ss_dassert(dcb->state != DCB_STATE_DISCONNECTED); if (dcb->state == DCB_STATE_DISCONNECTED) { - return 1; + goto return_1; } - if (dcb->protocol) { - protocol = DCB_PROTOCOL(dcb, MySQLProtocol); - } else { - goto return_1; + + if (dcb->protocol == NULL) { + goto return_1; } - + protocol = (MySQLProtocol *)dcb->protocol; + CHK_PROTOCOL(protocol); + if (protocol->state == MYSQL_IDLE || protocol->state == MYSQL_WAITING_RESULT) { diff --git a/server/modules/protocol/mysql_common.c b/server/modules/protocol/mysql_common.c index 39d3553f1..ced8674f8 100644 --- a/server/modules/protocol/mysql_common.c +++ b/server/modules/protocol/mysql_common.c @@ -504,7 +504,7 @@ int gw_send_authentication_to_backend(char *dbname, char *user, uint8_t *passwd, // write to backend dcb // ToDO: handle the EAGAIN | EWOULDBLOCK - rv = write(dcb->fd, GWBUF_DATA(buffer), bytes); + rv = gw_write(dcb->fd, GWBUF_DATA(buffer), bytes); gwbuf_consume(buffer, bytes); diff --git a/server/modules/protocol/telnetd.c b/server/modules/protocol/telnetd.c index f31f7f4c6..8a9c77e2f 100644 --- a/server/modules/protocol/telnetd.c +++ b/server/modules/protocol/telnetd.c @@ -224,7 +224,9 @@ telnetd_write_event(DCB *dcb) static int telnetd_write(DCB *dcb, GWBUF *queue) { - return dcb_write(dcb, queue); + int rc; + rc = dcb_write(dcb, queue); + return rc; } /** diff --git a/server/modules/routing/debugcmd.c b/server/modules/routing/debugcmd.c index 2e7ea717b..44b841ec1 100644 --- a/server/modules/routing/debugcmd.c +++ b/server/modules/routing/debugcmd.c @@ -200,10 +200,40 @@ struct subcommand enableoptions[] = { * * The subcommands of the disable command * */ struct subcommand disableoptions[] = { - { "log", 1, disable_log_action, "Disable Log for MaxScale, Options: trace | error | message E.g. disable log trace", - {ARG_TYPE_STRING, 0, 0} }, - { NULL, 0, NULL, NULL, - {0, 0, 0} } + { "log", 1, disable_log_action, "Disable Log for MaxScale, Options: trace | error | message E.g. disable log trace", + {ARG_TYPE_STRING, 0, 0} }, + { NULL, 0, NULL, NULL, + {0, 0, 0} } +}; + +static void fail_backendfd(void); +static void fail_clientfd(void); + +/** + * * The subcommands of the fail command + * */ +struct subcommand failoptions[] = { + { + "backendfd", + 0, + fail_backendfd, + "Fail backend socket for next operation.", + {ARG_TYPE_STRING, 0, 0} + }, + { + "clientfd", + 0, + fail_clientfd, + "Fail client socket for next operation.", + {ARG_TYPE_STRING, 0, 0} + }, + { + NULL, + 0, + NULL, + NULL, + {0, 0, 0} + } }; @@ -254,6 +284,7 @@ static struct { { "reload", reloadoptions }, { "enable", enableoptions }, { "disable", disableoptions }, + { "fail", failoptions }, { NULL, NULL } }; @@ -697,4 +728,12 @@ static void disable_log_action(DCB *dcb, char *arg1) { skygw_log_disable(type); } -//// +static void fail_backendfd(void) +{ + fail_next_backend_fd = TRUE; +} + +static void fail_clientfd(void) +{ + fail_next_client_fd = TRUE; +}