Replaced write system function with wrapper gw_write. It allows for generating failures by using telnet commands, fail backendfd, fail clientfd, which are available in debug build only.

server/core/dcb.c

@@ -450,7 +450,6 @@ dcb_connect(SERVER *server, SESSION *session, const char *protocol)
 {
 DCB		*dcb;
 GWPROTOCOL	*funcs;
-int             val;
 int             fd;
 
 	if ((dcb = dcb_alloc(DCB_ROLE_REQUEST_HANDLER)) == NULL)
@@ -506,7 +505,7 @@ int             fd;
                         session->client,
                         session->client->fd);
         }
-        ss_dassert(dcb->fd = -1);
+        ss_dassert(dcb->fd == -1);
         /**
          * Successfully connected to backend. Assign file descriptor to dcb
          */
@@ -670,14 +669,32 @@ int	w, saved_errno = 0;
 		 */
 		while (queue != NULL)
 		{
+#if defined(SS_DEBUG)
+                        if (dcb->session) {
+                                if (dcb_isclient(dcb)) {
+                                        if (fail_next_client_fd) {
+                                                dcb_fake_write_errno[dcb->fd] = 32;
+                                                dcb_fake_write_ev[dcb->fd] = 29;
+                                                fail_next_client_fd = false;
+                                        }
+                                } else {
+                                        if (fail_next_backend_fd) {
+                                                dcb_fake_write_errno[dcb->fd] = 32;
+                                                dcb_fake_write_ev[dcb->fd] = 29;
+                                                fail_next_backend_fd = false;
+                                        }
+                                }
+                        }
+#endif /* SS_DEBUG */
 			len = GWBUF_LENGTH(queue);
-			GW_NOINTR_CALL(w = write(dcb->fd, GWBUF_DATA(queue), len); dcb->stats.n_writes++);
+			GW_NOINTR_CALL(w = gw_write(dcb->fd, GWBUF_DATA(queue), len);
+                                       dcb->stats.n_writes++);
 			saved_errno = errno;
                         errno = 0;
                         
 			if (w < 0)
 			{
-                                skygw_log_write(
+                                skygw_log_write_flush(
                                         LOGFILE_ERROR,
                                         "%lu [dcb_write] Write to dcb %p fd %d "
                                         "failed due errno %d, %s",
@@ -724,7 +741,7 @@ int	w, saved_errno = 0;
 }
 
 /**
- * Drain the write queue of a DCB. THis is called as part of the EPOLLOUT handling
+ * Drain the write queue of a DCB. This is called as part of the EPOLLOUT handling
  * of a socket and will try to send any buffered data from the write queue
  * up until the point the write would block.
  *
@@ -751,17 +768,20 @@ int saved_errno = 0;
 		while (dcb->writeq != NULL)
 		{
 			len = GWBUF_LENGTH(dcb->writeq);
-			GW_NOINTR_CALL(w = write(dcb->fd, GWBUF_DATA(dcb->writeq), len););
+			GW_NOINTR_CALL(w = gw_write(dcb->fd, GWBUF_DATA(dcb->writeq), len););
 			saved_errno = errno;
+                        errno = 0;
+                        
 			if (w < 0)
 			{
                                 skygw_log_write(
                                         LOGFILE_ERROR,
                                         "%lu [dcb_drain_writeq] Write to fd %d "
-                                        "failed due errno %d",
+                                        "failed due errno %d, %s",
                                         pthread_self(),
                                         dcb->fd,
-                                        saved_errno);
+                                        saved_errno,
+                                        strerror(saved_errno));
                                 break;
 			}
                         
@@ -1169,3 +1189,27 @@ static bool dcb_set_state_nomutex(
         return succp;
 }
 
+int gw_write(
+        int fd,
+        const void* buf,
+        size_t nbytes)
+{
+        int w;
+#if defined(SS_DEBUG)
+        if (dcb_fake_write_errno[fd] != 0) {
+                ss_dassert(dcb_fake_write_ev[fd] != 0);
+                w = write(fd, buf, nbytes/2); /**< leave peer to read missing bytes */
+
+                if (w > 0) {
+                        w = -1;
+                        errno = dcb_fake_write_errno[fd];
+                }
+        } else {
+                w = write(fd, buf, nbytes);
+        }
+#else
+        w = write(fd, buf, nbytes);           
+#endif /* SS_DEBUG && SS_TEST */
+        return w;
+}
+

server/core/gateway.c

@@ -109,7 +109,7 @@ static void sighup_handler (int i)
 }
 
 static void sigterm_handler (int i) {
-extern void shutdown_gateway();
+        extern void shutdown_gateway();
 
 	skygw_log_write( LOGFILE_ERROR, "Signal SIGTERM %i received ...Exiting!\n", i);
 	shutdown_gateway();
@@ -212,6 +212,10 @@ int 	        l;
 
 #if defined(SS_DEBUG)
 memset(conn_open, 0, sizeof(bool)*1024);
+memset(dcb_fake_write_errno, 0, sizeof(unsigned char)*1024);
+memset(dcb_fake_write_ev, 0, sizeof(__int32_t)*1024);
+fail_next_backend_fd = false;
+fail_next_client_fd = false;
 #endif
         l = atexit(skygw_logmanager_exit);
 
@@ -412,7 +416,7 @@ memset(conn_open, 0, sizeof(bool)*1024);
 	/*
 	 * Start the services that were created above
 	 */
-	n_services = serviceStartAll();
+        n_services = serviceStartAll();
 	skygw_log_write(LOGFILE_MESSAGE, "Started modules succesfully.");
 
         /**

server/core/utils.c

@@ -73,6 +73,8 @@ int setnonblocking(int fd) {
 	return 0;
 }
 
+        
+
 char *gw_strend(register const char *s) {
 	while (*s++);
 	return (char*) (s-1);
@@ -188,3 +190,33 @@ void gw_sha1_2_str(const uint8_t *in, int in_len, const uint8_t *in2, int in2_le
 
 	memcpy(out, hash, SHA_DIGEST_LENGTH);
 }
+
+
+/** 
+ * @node Gets errno corresponding to latest socket error 
+ *
+ * Parameters:
+ * @param fd - in, use
+ *          socket to examine
+ *
+ * @return errno
+ *
+ * 
+ * @details (write detailed description here)
+ *
+ */
+int gw_getsockerrno(
+        int fd)
+{
+        int eno = 0;
+        socklen_t elen = sizeof(eno);
+        
+        if (fd <= 0) {
+                goto return_eno;
+        }
+        
+        getsockopt(fd, SOL_SOCKET, SO_ERROR, (void *)&eno, &elen);
+
+return_eno:
+        return eno;
+}

server/include/dcb.h

@@ -185,19 +185,26 @@ typedef struct dcb {
 #endif
 } DCB;
 
+#if defined(SS_DEBUG)
+unsigned char dcb_fake_write_errno[1024];
+__int32_t     dcb_fake_write_ev[1024];
+bool          fail_next_backend_fd;
+bool          fail_next_client_fd;
+#endif
 
 /* A few useful macros */
 #define	DCB_SESSION(x)			(x)->session
 #define DCB_PROTOCOL(x, type)		(type *)((x)->protocol)
 #define	DCB_ISZOMBIE(x)			((x)->state == DCB_STATE_ZOMBIE)
 
+int             gw_write(int fd, const void* buf, size_t nbytes);
+int             dcb_write(DCB *, GWBUF *);
 DCB             *dcb_alloc(dcb_role_t);
-void		dcb_free(DCB *);			/* Free a DCB */
+void            dcb_free(DCB *);
-DCB		*dcb_connect(struct server *, struct session *, const char *);	/* prepare Backend connection */
+DCB             *dcb_connect(struct server *, struct session *, const char *);	
-int		dcb_read(DCB *, GWBUF **);		/* Generic read routine */
+int             dcb_read(DCB *, GWBUF **);
-int		dcb_write(DCB *, GWBUF *);		/* Generic write routine */
+int             dcb_drain_writeq(DCB *);
-int		dcb_drain_writeq(DCB *);		/* Generic write routine */
+void            dcb_close(DCB *);
-void		dcb_close(DCB *);			/* Generic close functionality */
 void		dcb_process_zombies(int);		/* Process Zombies */
 void		printAllDCBs();				/* Debug to print all DCB in the system */
 void		printDCB(DCB *);			/* Debug print routine */

server/include/gw.h

@@ -59,3 +59,5 @@ int  MySQLWrite(DCB *dcb, GWBUF *queue);
 int  gw_write_backend_event(DCB *dcb);
 int  gw_read_backend_event(DCB *dcb);
 int  setnonblocking(int fd);
+int  gw_write(int fd, const void* buf, size_t nbytes);
+int  gw_getsockerrno(int fd);

server/modules/protocol/httpd.c

@@ -267,7 +267,9 @@ httpd_write_event(DCB *dcb)
 static int
 httpd_write(DCB *dcb, GWBUF *queue)
 {
-	return dcb_write(dcb, queue);
+        int rc;
+        rc = dcb_write(dcb, queue);
+	return rc;
 }
 
 /**

server/modules/protocol/mysql_backend.c

@@ -149,10 +149,9 @@ static int gw_read_backend_event(DCB *dcb) {
 
         CHK_DCB(dcb);        
 	CHK_SESSION(dcb->session);
-        ss_info_dassert(dcb->session != NULL,
-                        "Backend dcb doesn't have session");
         
         backend_protocol = (MySQLProtocol *) dcb->protocol;
+        CHK_PROTOCOL(backend_protocol);
         
         /** return only with complete session */
         current_session = gw_get_shared_session_auth_info(dcb);
@@ -180,7 +179,8 @@ static int gw_read_backend_event(DCB *dcb) {
                         	current_session->db,
                         	current_session->user,
                         	current_session->client_sha1,
-                        	backend_protocol) != 0) {
+                        	backend_protocol) != 0)
+                        {
 				backend_protocol->state = MYSQL_AUTH_FAILED;
                 		rc = 1;
 			} else {
@@ -411,6 +411,7 @@ static int
 gw_MySQLWrite_backend(DCB *dcb, GWBUF *queue)
 {
 	MySQLProtocol *backend_protocol = dcb->protocol;
+        int rc;
 
         /**
          * Don't write to backend if backend_dcb is not in poll set anymore.
@@ -444,8 +445,8 @@ gw_MySQLWrite_backend(DCB *dcb, GWBUF *queue)
         memcpy(&dcb->command, &queue->command, sizeof(dcb->command));
 
 	spinlock_release(&dcb->authlock);
-
+        rc = dcb_write(dcb, queue);
-	return dcb_write(dcb, queue);
+	return rc;
 }
 
 /**
@@ -631,6 +632,7 @@ static void backend_set_delayqueue(DCB *dcb, GWBUF *queue) {
 static int backend_write_delayqueue(DCB *dcb)
 {
 	GWBUF *localq = NULL;
+        int   rc;
 
 	spinlock_acquire(&dcb->delayqlock);
 
@@ -644,8 +646,8 @@ static int backend_write_delayqueue(DCB *dcb)
         memcpy(&dcb->command, &localq->command, sizeof(dcb->command));
 
 	spinlock_release(&dcb->delayqlock);
-
+        rc = dcb_write(dcb, localq);
-	return dcb_write(dcb, localq);
+	return rc;
 }
 
 

server/modules/protocol/mysql_client.c

@@ -461,7 +461,7 @@ int	w, saved_errno = 0;
 		{
 			len = GWBUF_LENGTH(queue);
 			GW_NOINTR_CALL(
-                                w = write(dcb->fd,GWBUF_DATA(queue), len);
+                                w = gw_write(dcb->fd,GWBUF_DATA(queue), len);
                                 dcb->stats.n_writes++);
 			saved_errno = errno;
 			if (w < 0)
@@ -642,12 +642,17 @@ int gw_read_client_event(DCB* dcb) {
                  /* len = GWBUF_LENGTH(queue); */
                 ptr_buff = GWBUF_DATA(queue);
                 
-                /* get mysql commang at fourth byte */
+                /* get mysql commang at fifth byte */
                 if (ptr_buff) {
                         mysql_command = ptr_buff[4];
                 }
 
                 if (mysql_command  == '\x03') {
+                        /**
+                         * SQL Trace here.
+                         * Length can be calculated and it must be passed as
+                         * argument.
+                         */
                         /* this is a standard MySQL query !!!! */
                 }
                 /**
@@ -716,26 +721,42 @@ return_rc:
 ///////////////////////////////////////////////
 // client write event to Client triggered by EPOLLOUT
 //////////////////////////////////////////////
+/** 
+ * @node Client's fd became writable, and EPOLLOUT event
+ * arrived. As a consequence, client input buffer (writeq) is flushed. 
+ *
+ * Parameters:
+ * @param dcb - in, use
+ *          client dcb
+ *
+ * @return constantly 1
+ *
+ * 
+ * @details (write detailed description here)
+ *
+ */
 int gw_write_client_event(DCB *dcb)
 {
 	MySQLProtocol *protocol = NULL;
 
         CHK_DCB(dcb);
 
-	if (dcb == NULL) {
-		fprintf(stderr, "DCB is NULL, return\n");
-		return 1;
-	}
         ss_dassert(dcb->state != DCB_STATE_DISCONNECTED);
         
+	if (dcb == NULL) {
+		fprintf(stderr, "DCB is NULL, return\n");
+		goto return_1;
+	}
+
 	if (dcb->state == DCB_STATE_DISCONNECTED) {
-		return 1;
+		goto return_1;
 	}
-	if (dcb->protocol) {
+        
-		protocol = DCB_PROTOCOL(dcb, MySQLProtocol);
+	if (dcb->protocol == NULL) {
-	} else {
+	        goto return_1;
-                goto return_1;
 	}
+        protocol = (MySQLProtocol *)dcb->protocol;
+        CHK_PROTOCOL(protocol);
         
 	if (protocol->state == MYSQL_IDLE ||
             protocol->state == MYSQL_WAITING_RESULT)

server/modules/protocol/mysql_common.c

@@ -504,7 +504,7 @@ int gw_send_authentication_to_backend(char *dbname, char *user, uint8_t *passwd,
 
 	// write to backend dcb
 	// ToDO: handle the EAGAIN | EWOULDBLOCK
-	rv = write(dcb->fd, GWBUF_DATA(buffer), bytes);
+	rv = gw_write(dcb->fd, GWBUF_DATA(buffer), bytes);
 
 	gwbuf_consume(buffer, bytes);
 

server/modules/protocol/telnetd.c

@@ -224,7 +224,9 @@ telnetd_write_event(DCB *dcb)
 static int
 telnetd_write(DCB *dcb, GWBUF *queue)
 {
-	return dcb_write(dcb, queue);
+        int rc;
+        rc = dcb_write(dcb, queue);
+	return rc;
 }
 
 /**

server/modules/routing/debugcmd.c

@@ -200,10 +200,40 @@ struct subcommand enableoptions[] = {
  *  * The subcommands of the disable command
  *   */
 struct subcommand disableoptions[] = {
-        { "log",        1, disable_log_action,  "Disable Log for MaxScale, Options: trace | error | message E.g. disable log trace",
+    { "log",        1, disable_log_action,  "Disable Log for MaxScale, Options: trace | error | message E.g. disable log trace",
-                                {ARG_TYPE_STRING, 0, 0} },
+      {ARG_TYPE_STRING, 0, 0} },
-        { NULL,         0, NULL,                NULL,
+    { NULL,         0, NULL,                NULL,
-                                {0, 0, 0} }
+      {0, 0, 0} }
+};
+
+static void fail_backendfd(void);
+static void fail_clientfd(void);
+
+/**
+ *  * The subcommands of the fail command
+ *   */
+struct subcommand failoptions[] = {
+    {
+        "backendfd",
+        0,
+        fail_backendfd,
+        "Fail backend socket for next operation.",
+        {ARG_TYPE_STRING, 0, 0}
+    },
+    {
+        "clientfd",
+        0,
+        fail_clientfd,
+        "Fail client socket for next operation.",
+        {ARG_TYPE_STRING, 0, 0}
+    },
+    {
+        NULL,
+        0,
+        NULL,
+        NULL,
+        {0, 0, 0}
+    }
 };
 
 
@@ -254,6 +284,7 @@ static struct {
 	{ "reload",	reloadoptions },
 	{ "enable",     enableoptions },
 	{ "disable",    disableoptions },
+        { "fail",       failoptions },
 	{ NULL,		NULL	}
 };
 
@@ -697,4 +728,12 @@ static void disable_log_action(DCB *dcb, char *arg1) {
         skygw_log_disable(type);
 }
 
-////
+static void fail_backendfd(void)
+{ 
+        fail_next_backend_fd = TRUE;
+}
+
+static void fail_clientfd(void)
+{ 
+        fail_next_client_fd = TRUE;
+}