SSL handshake now successfully completes when a client connects with SSL enabled.

Documentation/Reference/MaxScale-and-SSL.md

@@ -0,0 +1,13 @@
+# MaxScale and SSL
+
+MaxScale supports client side SSL connections. Enabling is done on a per service basis and each service has its own set of certificates.
+
+## SSL Options
+
+Here are the options which relate to SSL and certificates.
+Parameter|Values|Description
+----------------------------
+ssl         | disabled, enabled, required |`disable` disables SSL, `enabled` enables SSL for client connections but still allows non-SSL connections and `required` requires SSL from all client connections. With the `required` option, client connections that do not use SSL will be rejected.
+ssl_cert    | <path to file>              |Path to server certificate
+ssl_key     | <path to file>              |Path to server private key
+ssl_ca_cert | <path to file>              |Path to Certificate Authority file

server/core/dcb.c

@@ -941,18 +941,12 @@ int dcb_read_SSL(
                         /** Handle closed client socket */
                         if (dcb_isclient(dcb))
                         {
-                                char c;
+                                char c = 0;
-                                int l_errno = 0;
                                 int r = -1;
 
                                 /* try to read 1 byte, without consuming the socket buffer */
-                                r = recv(dcb->fd, &c, sizeof(char), MSG_PEEK);
+                                r = SSL_peek(ssl, &c, sizeof(char));
-                                l_errno = errno;
+                                if (r <= 0)
-
-                                if (r <= 0 &&
-                                        l_errno != EAGAIN &&
-                                        l_errno != EWOULDBLOCK &&
-					l_errno != 0)
                                 {
                                         n = -1;
                                         goto return_n;
@@ -989,13 +983,15 @@ int dcb_read_SSL(
                         n = -1;
                         goto return_n;
                 }
-                GW_NOINTR_CALL(n = SSL_read(ssl, GWBUF_DATA(buffer), bufsize);
+                n = SSL_read(ssl, GWBUF_DATA(buffer), bufsize);
-                dcb->stats.n_reads++);
+                dcb->stats.n_reads++;
 
+		int ssl_errno = 0;
                 if (n <= 0)
                 {
-		    int ssl_errno = ERR_get_error();
+		    ssl_errno = ERR_get_error();
-		    if(ssl_errno != SSL_ERROR_WANT_READ)
+
+		    if(ssl_errno != SSL_ERROR_WANT_READ && ssl_errno != SSL_ERROR_NONE)
 		    {
 		    LOGIF(LE, (skygw_log_write_flush(
 			    LOGFILE_ERROR,
@@ -1023,6 +1019,8 @@ int dcb_read_SSL(
                         dcb->fd)));
                 /*< Append read data to the gwbuf */
                 *head = gwbuf_append(*head, buffer);
+		if(ssl_errno == SSL_ERROR_WANT_READ || ssl_errno == SSL_ERROR_NONE)
+		    break;
         } /*< while (true) */
 return_n:
         return n;

server/core/gateway.c

@@ -196,7 +196,9 @@ static bool resolve_maxscale_conf_fname(
 
 static char* check_dir_access(char* dirname,bool,bool);
 static int set_user();
-
+static void maxscale_ssl_lock(int mode,int n,const char* file, int line);
+static unsigned long maxscale_ssl_id();
+static SPINLOCK* ssl_locks;
 /**
  * Handler for SIGHUP signal. Reload the configuration for the
  * gateway.
@@ -1371,6 +1373,22 @@ int main(int argc, char **argv)
                 goto return_main;
         }
 	
+	/** OpenSSL initialization */
+
+	SSL_library_init();
+	SSL_load_error_strings();
+	int n_locks = CRYPTO_num_locks();
+	if((ssl_locks = malloc(n_locks*sizeof(SPINLOCK))) == NULL)
+	{
+	    rc = MAXSCALE_INTERNALERROR;
+	    goto return_main;
+	}
+
+	for(i = 0;i<n_locks;i++)
+	    spinlock_init(&ssl_locks[i]);
+
+	CRYPTO_set_locking_callback(maxscale_ssl_lock);
+	CRYPTO_set_id_callback(maxscale_ssl_id);
 	/* register exit function for embedded MySQL library */
         l = atexit(libmysqld_done);
 
@@ -2002,3 +2020,20 @@ static int set_user(char* user)
 
     return rval;
 }
+
+static void maxscale_ssl_lock(int mode,int n,const char* file, int line)
+{
+    if(mode & CRYPTO_LOCK)
+    {
+	spinlock_acquire(&ssl_locks[n]);
+    }
+    else
+    {
+	spinlock_release(&ssl_locks[n]);
+    }
+}
+
+static unsigned long maxscale_ssl_id()
+{
+    return (unsigned long)pthread_self();
+}

server/core/service.c

@@ -1843,7 +1843,7 @@ int serviceInitSSL(SERVICE* service)
 	SSL_CTX_set_verify(service->ctx,SSL_VERIFY_PEER,NULL);
 
 	/* Set the verification depth to 1 */
-	SSL_CTX_set_verify_depth(service->ctx,10);
+	SSL_CTX_set_verify_depth(service->ctx,1);
 	service->ssl_init_done = true;
     }
     return 0;

server/modules/include/mysql_client_server_protocol.h

@@ -99,10 +99,11 @@ typedef enum {
         MYSQL_AUTH_RECV,
         MYSQL_AUTH_FAILED,
         MYSQL_HANDSHAKE_FAILED,
-        MYSQL_AUTH_SSL_REQ, /*< client requested SSL */
+        MYSQL_AUTH_SSL_REQ, /*< client requested SSL but SSL_accept hasn't beed called */
-        MYSQL_AUTH_SSL_EXCHANGE_DONE, /*< SSL handshake done */
+        MYSQL_AUTH_SSL_HANDSHAKE_DONE, /*< SSL handshake has been fully completed */
-        MYSQL_AUTH_SSL_EXCHANGE_ERR, /*< SSL handshake failure */
+        MYSQL_AUTH_SSL_HANDSHAKE_FAILED, /*< SSL handshake failed for any reason */
-        MYSQL_AUTH_SSL_RECV, /*<  */
+        MYSQL_AUTH_SSL_HANDSHAKE_ONGOING, /*< SSL_accept has been called but the
+                                           * SSL handshake hasn't been completed */
         MYSQL_IDLE
 } mysql_auth_state_t;
 

server/modules/protocol/mysql_client.c

@@ -117,8 +117,6 @@ version()
 void
 ModuleInit()
 {
-    SSL_library_init();
-    SSL_load_error_strings();
 }
 
 /**
@@ -466,7 +464,7 @@ static int gw_mysql_do_authentication(DCB *dcb, GWBUF *queue) {
                         &protocol->client_capabilities);
         */
 
-	if(protocol->protocol_auth_state == MYSQL_AUTH_SSL_EXCHANGE_DONE)
+	if(protocol->protocol_auth_state == MYSQL_AUTH_SSL_HANDSHAKE_DONE)
 	    goto ssl_hs_done;
 	
 	ssl = protocol->client_capabilities & GW_MYSQL_CAPABILITIES_SSL;
@@ -637,48 +635,6 @@ gw_MySQLWrite_client_SSL(DCB *dcb, GWBUF *queue)
     return dcb_write_SSL(dcb, protocol->ssl, queue);
 }
 
-
-int gw_read_client_event_SSL(
-DCB* dcb)
-{
-    SESSION        *session = NULL;
-    ROUTER_OBJECT  *router = NULL;
-    ROUTER         *router_instance = NULL;
-    void           *rsession = NULL;
-    MySQLProtocol  *protocol = NULL;
-    GWBUF          *read_buffer = NULL;
-    int             rc = 0;
-    int             nbytes_read = 0;
-    uint8_t         cap = 0;
-    bool            stmt_input = false; /*< router input type */
-
-    CHK_DCB(dcb);
-    protocol = DCB_PROTOCOL(dcb, MySQLProtocol);
-    CHK_PROTOCOL(protocol);
-
-
-    if(protocol->protocol_auth_state == MYSQL_AUTH_SSL_REQ)
-    {
-	if(do_ssl_accept(protocol) == 1)
-	{
-	    spinlock_acquire(&protocol->protocol_lock);
-	    protocol->protocol_auth_state = MYSQL_AUTH_SSL_EXCHANGE_DONE;
-	    spinlock_release(&protocol->protocol_lock);
-
-	    spinlock_acquire(&dcb->authlock);
-	    dcb->func.read = gw_read_client_event_SSL;
-	    dcb->func.write = gw_MySQLWrite_client_SSL;
-	    dcb->func.write_ready = gw_write_client_event;
-	    spinlock_release(&dcb->authlock);
-	}
-	goto return_rc;
-    }
-
-    return_rc:
-
-    return rc;
-}
-
 /**
  * Client read event triggered by EPOLLIN
  *
@@ -703,26 +659,26 @@ int gw_read_client_event(
         protocol = DCB_PROTOCOL(dcb, MySQLProtocol);
         CHK_PROTOCOL(protocol);
 
-	/** Let the OpenSSL API do the reading from the socket */
+	if(protocol->protocol_auth_state == MYSQL_AUTH_SSL_HANDSHAKE_ONGOING ||
-	if(protocol->protocol_auth_state == MYSQL_AUTH_SSL_REQ)
+	 protocol->protocol_auth_state == MYSQL_AUTH_SSL_REQ)
 	{
-	    if(do_ssl_accept(protocol) == 1)
+	    switch(do_ssl_accept(protocol))
 	    {
-		spinlock_acquire(&protocol->protocol_lock);
+	    case 0:
-		protocol->protocol_auth_state = MYSQL_AUTH_SSL_EXCHANGE_DONE;
+		return 0;
-		protocol->use_ssl = true;
+		break;
-		spinlock_release(&protocol->protocol_lock);
+	    case 1:
-
+		return 0;
-		spinlock_acquire(&dcb->authlock);
+		break;
-		//dcb->func.read = gw_read_client_event_SSL;
+	    case -1:
-		//dcb->func.write = gw_MySQLWrite_client_SSL;
+		return 1;
-		//dcb->func.write_ready = gw_write_client_event_SSL;
+		break;
-		spinlock_release(&dcb->authlock);
+	    default:
+		return 1;
+		break;
 	    }
-	    goto return_rc;
 	}
 
-
 	if(protocol->use_ssl)
 	{
 	    rc = dcb_read_SSL(dcb,protocol->ssl, &read_buffer);
@@ -880,8 +836,13 @@ int gw_read_client_event(
 		
                 auth_val = gw_mysql_do_authentication(dcb, read_buffer);
 
-		if(protocol->protocol_auth_state == MYSQL_AUTH_SSL_REQ)
+		if(protocol->protocol_auth_state == MYSQL_AUTH_SSL_REQ ||
+		 protocol->protocol_auth_state == MYSQL_AUTH_SSL_HANDSHAKE_ONGOING ||
+		 protocol->protocol_auth_state == MYSQL_AUTH_SSL_HANDSHAKE_DONE ||
+		 protocol->protocol_auth_state == MYSQL_AUTH_SSL_HANDSHAKE_FAILED)
+		{
 		    break;
+		}
 		
 		if (auth_val == 0)
 		{
@@ -976,9 +937,103 @@ int gw_read_client_event(
 	}
         break;
         
-	case MYSQL_AUTH_SSL_EXCHANGE_DONE:
+	case MYSQL_AUTH_SSL_HANDSHAKE_DONE:
 	{
-	    protocol->protocol_auth_state = MYSQL_AUTH_SSL_EXCHANGE_DONE;
+	    int auth_val;
+
+	    auth_val = gw_mysql_do_authentication(dcb, read_buffer);
+
+
+	    if (auth_val == 0)
+	    {
+		SESSION *session;
+
+		protocol->protocol_auth_state = MYSQL_AUTH_RECV;
+		/**
+		 * Create session, and a router session for it.
+		 * If successful, there will be backend connection(s)
+		 * after this point.
+		 */
+		session = session_alloc(dcb->service, dcb);
+
+		if (session != NULL)
+		{
+		    CHK_SESSION(session);
+		    ss_dassert(session->state != SESSION_STATE_ALLOC);
+
+		    protocol->protocol_auth_state = MYSQL_IDLE;
+		    /**
+		     * Send an AUTH_OK packet to the client,
+		     * packet sequence is # 2
+		     */
+		    mysql_send_ok(dcb, 2, 0, NULL);
+		}
+		else
+		{
+		    protocol->protocol_auth_state = MYSQL_AUTH_FAILED;
+		    LOGIF(LD, (skygw_log_write(
+			    LOGFILE_DEBUG,
+					     "%lu [gw_read_client_event] session "
+			    "creation failed. fd %d, "
+			    "state = MYSQL_AUTH_FAILED.",
+					     pthread_self(),
+					     protocol->owner_dcb->fd)));
+
+		    /** Send ERR 1045 to client */
+		    mysql_send_auth_error(
+			    dcb,
+				     2,
+				     0,
+				     "failed to create new session");
+
+		    dcb_close(dcb);
+		}
+	    }
+	    else
+	    {
+		char* fail_str = NULL;
+
+		protocol->protocol_auth_state = MYSQL_AUTH_FAILED;
+
+		if (auth_val == 2) {
+		    /** Send error 1049 to client */
+		    int message_len = 25 + MYSQL_DATABASE_MAXLEN;
+
+		    fail_str = calloc(1, message_len+1);
+		    snprintf(fail_str, message_len, "Unknown database '%s'",
+			     (char*)((MYSQL_session *)dcb->data)->db);
+
+		    modutil_send_mysql_err_packet(dcb, 2, 0, 1049, "42000", fail_str);
+		} else {
+		    /** Send error 1045 to client */
+		    fail_str = create_auth_fail_str((char *)((MYSQL_session *)dcb->data)->user,
+					     dcb->remote,
+					     (char*)((MYSQL_session *)dcb->data)->client_sha1,
+					     (char*)((MYSQL_session *)dcb->data)->db);
+		    modutil_send_mysql_err_packet(dcb, 2, 0, 1045, "28000", fail_str);
+		}
+		if (fail_str)
+		    free(fail_str);
+
+		LOGIF(LD, (skygw_log_write(
+			LOGFILE_DEBUG,
+					 "%lu [gw_read_client_event] after "
+			"gw_mysql_do_authentication, fd %d, "
+			"state = MYSQL_AUTH_FAILED.",
+					 protocol->owner_dcb->fd,
+					 pthread_self())));
+		/**
+		 * Release MYSQL_session since it is not used anymore.
+		 */
+		if (!DCB_IS_CLONE(dcb))
+		{
+		    free(dcb->data);
+		}
+		dcb->data = NULL;
+
+		dcb_close(dcb);
+	    }
+	    read_buffer = gwbuf_consume(read_buffer, nbytes_read);
 	}
 	break;
 
@@ -1821,110 +1876,82 @@ return_rc:
         return rc;
 }
 
-
 /**
- * Create a character array including the query string. 
+ * Do the SSL authentication handshake.
- * GWBUF given as input includes either one complete or partial query.
+ * This functions
- * Length of buffer is at most the query length+4 (length of packet header).
+ * @param protocol
+ * @return
  */
-#if defined(NOT_USED)
-static char* gw_get_or_create_querystr (
-        void*  data,
-        bool*  new_allocation)
-{
-        GWBUF* buf = (GWBUF *)data;
-        size_t buflen;    /*< first gw buffer data length */
-        size_t packetlen; /*< length of mysql packet */
-        size_t querylen;  /*< total buffer length-<length of type indicator> */
-        size_t nbytes_copied;
-        char*  startpos;  /*< first byte of query in gw buffer */
-        char*  str;       /*< resulting query string */
-        
-        CHK_GWBUF(buf);
-        packetlen = MYSQL_GET_PACKET_LEN((uint8_t *)GWBUF_DATA(buf)); 
-        str = (char *)malloc(packetlen); /*< leave space for terminating null */
-        
-        if (str == NULL)
-        {
-                goto return_str;
-        }
-        *new_allocation = true;
-        /** 
-         * First buffer includes 4 bytes header and a type indicator byte.
-         */
-        buflen    = GWBUF_LENGTH(buf);
-        querylen  = packetlen-1;        
-        ss_dassert(buflen<=querylen+5); /*< 5 == header+type indicator */
-        startpos = (char *)GWBUF_DATA(buf)+5;
-        nbytes_copied = MIN(querylen, buflen-5);
-        memcpy(str, startpos, nbytes_copied);
-        memset(&str[querylen-1], 0, 1);
-        buf = gwbuf_consume(buf, querylen-1);
-        
-        /** 
-         * In case of multi-packet statement whole buffer consists of query
-         * string.
-         */
-        while (buf != NULL)
-        {
-                buflen = GWBUF_LENGTH(buf);
-                memcpy(str+nbytes_copied, GWBUF_DATA(buf), buflen);
-                nbytes_copied += buflen;
-                buf = gwbuf_consume(buf, buflen);
-        }
-        ss_dassert(str[querylen-1] == 0);
-        
-return_str:
-        return str;
-}
-#endif
-
 int do_ssl_accept(MySQLProtocol* protocol)
 {
     int rval,errnum;
     char errbuf[2014];
+    DCB* dcb;
 
-    switch((rval = SSL_accept(protocol->ssl)))
+    rval = SSL_accept(protocol->ssl);
+
+    switch(rval)
     {
     case 0:
 	errnum = SSL_get_error(protocol->ssl,rval);
-		skygw_log_write_flush(LT,"SSL_accept ongoing for %s@%s",
+	skygw_log_write_flush(LT,"SSL_accept shutdown for %s@%s",
 			 protocol->owner_dcb->user,
 			 protocol->owner_dcb->remote);
+	return -1;
 	break;
     case 1:
-		protocol->protocol_auth_state = MYSQL_AUTH_SSL_EXCHANGE_DONE;
+	spinlock_acquire(&protocol->protocol_lock);
-		rval = 1;
+	dcb = protocol->owner_dcb;
+	protocol->protocol_auth_state = MYSQL_AUTH_SSL_HANDSHAKE_DONE;
 	protocol->use_ssl = true;
+	spinlock_release(&protocol->protocol_lock);
+
+	spinlock_acquire(&dcb->authlock);
+	dcb->func.write = gw_MySQLWrite_client_SSL;
+	dcb->func.write_ready = gw_write_client_event_SSL;
+	spinlock_release(&dcb->authlock);
+
+	rval = 1;
+
 	skygw_log_write_flush(LT,"SSL_accept done for %s@%s",
 			 protocol->owner_dcb->user,
 			 protocol->owner_dcb->remote);
 	break;
 
-	    default:
+    case -1:
 	errnum = SSL_get_error(protocol->ssl,rval);
-		if(errnum == SSL_ERROR_WANT_READ)
+
+	if(errnum == SSL_ERROR_WANT_READ || errnum == SSL_ERROR_WANT_WRITE ||
+	 errnum == SSL_ERROR_WANT_X509_LOOKUP)
 	{
 	    /** Not all of the data has been read. Go back to the poll
 	     queue and wait for more.*/
+
 	    rval = 0;
-		    protocol->protocol_auth_state = MYSQL_AUTH_SSL_REQ;
+	    skygw_log_write_flush(LT,"SSL_accept ongoing for %s@%s",
-		    skygw_log_write_flush(LT,"SSL_accept partially done for %s@%s",
 			     protocol->owner_dcb->user,
 			     protocol->owner_dcb->remote);
 	}
 	else
 	{
+	    spinlock_acquire(&protocol->protocol_lock);
+	    protocol->protocol_auth_state = MYSQL_AUTH_SSL_HANDSHAKE_FAILED;
+	    spinlock_release(&protocol->protocol_lock);
+	    rval = -1;
+	    
 	    skygw_log_write_flush(LE,
 			     "Error: Fatal error in SSL_accept for %s@%s: %s",
 			     protocol->owner_dcb->user,
 			     protocol->owner_dcb->remote,
 			     ERR_error_string(errnum,NULL));
-		    protocol->protocol_auth_state = MYSQL_AUTH_SSL_EXCHANGE_ERR;
-		    rval = -1;
 	}
 	break;
 
+    default:
+	skygw_log_write_flush(LE,
+			 "Error: Fatal error in SSL_accept, returned value was %d.",
+			 rval);
+	break;
     }
 
     return rval;