Add support for 10.2 server capabilities

Added support for 10.2 server capabilities and cleaned up the server
capability flags.

include/maxscale/protocol/mysql.h

@@ -84,11 +84,10 @@ MXS_BEGIN_DECLS
 #define MYSQL_CHARSET_OFFSET    12
 #define MYSQL_CLIENT_CAP_OFFSET 4
 #define MYSQL_CLIENT_CAP_SIZE   4
+#define MARIADB_CAP_OFFSET MYSQL_CHARSET_OFFSET + 19
 
 #define GW_MYSQL_PROTOCOL_VERSION 10 // version is 10
 #define GW_MYSQL_HANDSHAKE_FILLER 0x00
-#define GW_MYSQL_SERVER_CAPABILITIES_BYTE1 0xff
-#define GW_MYSQL_SERVER_CAPABILITIES_BYTE2 0xf7
 #define GW_MYSQL_SERVER_LANGUAGE 0x08
 #define GW_MYSQL_MAX_PACKET_LEN 0xffffffL;
 #define GW_MYSQL_SCRAMBLE_SIZE 20
@@ -190,7 +189,8 @@ typedef struct mysql_session
 typedef enum
 {
     GW_MYSQL_CAPABILITIES_NONE =                   0,
-    GW_MYSQL_CAPABILITIES_LONG_PASSWORD =          (1 << 0),
+    /** This is sent by pre-10.2 clients */
+    GW_MYSQL_CAPABILITIES_CLIENT_MYSQL =           (1 << 0),
     GW_MYSQL_CAPABILITIES_FOUND_ROWS =             (1 << 1),
     GW_MYSQL_CAPABILITIES_LONG_FLAG =              (1 << 2),
     GW_MYSQL_CAPABILITIES_CONNECT_WITH_DB =        (1 << 3),
@@ -210,9 +210,15 @@ typedef enum
     GW_MYSQL_CAPABILITIES_MULTI_RESULTS =          (1 << 17),
     GW_MYSQL_CAPABILITIES_PS_MULTI_RESULTS =       (1 << 18),
     GW_MYSQL_CAPABILITIES_PLUGIN_AUTH =            (1 << 19),
+    GW_MYSQL_CAPABILITIES_CONNECT_ATTRS =          (1 << 20),
+    GW_MYSQL_CAPABILITIES_AUTH_LENENC_DATA =       (1 << 21),
+    GW_MYSQL_CAPABILITIES_EXPIRE_PASSWORD =        (1 << 22),
+    GW_MYSQL_CAPABILITIES_SESSION_TRACK =          (1 << 23),
+    GW_MYSQL_CAPABILITIES_DEPRECATE_EOF =          (1 << 24),
     GW_MYSQL_CAPABILITIES_SSL_VERIFY_SERVER_CERT = (1 << 30),
     GW_MYSQL_CAPABILITIES_REMEMBER_OPTIONS =       (1 << 31),
-    GW_MYSQL_CAPABILITIES_CLIENT = (GW_MYSQL_CAPABILITIES_LONG_PASSWORD |
+    GW_MYSQL_CAPABILITIES_CLIENT = (
+        GW_MYSQL_CAPABILITIES_CLIENT_MYSQL |
         GW_MYSQL_CAPABILITIES_FOUND_ROWS |
         GW_MYSQL_CAPABILITIES_LONG_FLAG |
         GW_MYSQL_CAPABILITIES_CONNECT_WITH_DB |
@@ -224,21 +230,40 @@ typedef enum
         GW_MYSQL_CAPABILITIES_MULTI_RESULTS |
         GW_MYSQL_CAPABILITIES_PS_MULTI_RESULTS |
         GW_MYSQL_CAPABILITIES_SECURE_CONNECTION),
-    GW_MYSQL_CAPABILITIES_CLIENT_COMPRESS = (GW_MYSQL_CAPABILITIES_LONG_PASSWORD |
+    GW_MYSQL_CAPABILITIES_SERVER = (
+        GW_MYSQL_CAPABILITIES_CLIENT_MYSQL |
         GW_MYSQL_CAPABILITIES_FOUND_ROWS |
         GW_MYSQL_CAPABILITIES_LONG_FLAG |
         GW_MYSQL_CAPABILITIES_CONNECT_WITH_DB |
+        GW_MYSQL_CAPABILITIES_NO_SCHEMA |
+        GW_MYSQL_CAPABILITIES_ODBC |
         GW_MYSQL_CAPABILITIES_LOCAL_FILES |
-                                             GW_MYSQL_CAPABILITIES_PLUGIN_AUTH |
+        GW_MYSQL_CAPABILITIES_IGNORE_SPACE |
-                                             GW_MYSQL_CAPABILITIES_TRANSACTIONS |
         GW_MYSQL_CAPABILITIES_PROTOCOL_41 |
+        GW_MYSQL_CAPABILITIES_INTERACTIVE |
+        GW_MYSQL_CAPABILITIES_IGNORE_SIGPIPE |
+        GW_MYSQL_CAPABILITIES_TRANSACTIONS |
+        GW_MYSQL_CAPABILITIES_RESERVED |
+        GW_MYSQL_CAPABILITIES_SECURE_CONNECTION |
         GW_MYSQL_CAPABILITIES_MULTI_STATEMENTS |
         GW_MYSQL_CAPABILITIES_MULTI_RESULTS |
         GW_MYSQL_CAPABILITIES_PS_MULTI_RESULTS |
-                                             GW_MYSQL_CAPABILITIES_COMPRESS
+        GW_MYSQL_CAPABILITIES_PLUGIN_AUTH),
-                                            ),
 } gw_mysql_capabilities_t;
 
+/**
+ * Capabilities supported by MariaDB 10.2 and later, stored in the last 4 bytes
+ * of the 10 byte filler of the initial handshake packet.
+ *
+ * The actual capability bytes use by the server are left shifted by an extra 32
+ * bits to get one 64 bit capability that combines the old and new capabilities.
+ * Since we only use these in the non-shifted form, the definitions declared here
+ * are right shifted by 32 bytes and can be directly copied into the extra capabilities.
+ */
+#define MXS_MARIA_CAP_PROGRESS             (1 << 0)
+#define MXS_MARIA_CAP_COM_MULTI            (1 << 1)
+#define MXS_MARIA_CAP_STMT_BULK_OPERATIONS (1 << 2)
+
 typedef enum enum_server_command mysql_server_cmd_t;
 
 static const mysql_server_cmd_t MYSQL_COM_UNDEFINED = (mysql_server_cmd_t) - 1;
@@ -277,6 +302,7 @@ typedef struct
     uint8_t                scramble[MYSQL_SCRAMBLE_LEN]; /*< server scramble, created or received */
     uint32_t               server_capabilities;          /*< server capabilities, created or received */
     uint32_t               client_capabilities;          /*< client capabilities, created or received */
+    uint32_t               extra_capabilities;           /*< MariaDB 10.2 capabilities */
     unsigned long          tid;                          /*< MySQL Thread ID, in handshake */
     unsigned int           charset;                      /*< MySQL character set at connect time */
     bool                   ignore_reply;                 /*< If the reply should be discarded */

server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c

@@ -190,12 +190,10 @@ static int gw_create_backend_connection(DCB *backend_dcb,
     /** Copy client flags to backend protocol */
     if (backend_dcb->session->client_dcb->protocol)
     {
-        /** Copy client flags to backend protocol */
+        MySQLProtocol *client = (MySQLProtocol*)backend_dcb->session->client_dcb->protocol;
-        protocol->client_capabilities =
+        protocol->client_capabilities = client->client_capabilities;
-            ((MySQLProtocol *)(backend_dcb->session->client_dcb->protocol))->client_capabilities;
+        protocol->charset = client->charset;
-        /** Copy client charset to backend protocol */
+        protocol->extra_capabilities = client->extra_capabilities;
-        protocol->charset =
-            ((MySQLProtocol *)(backend_dcb->session->client_dcb->protocol))->charset;
     }
     else
     {

server/modules/protocol/MySQL/MySQLClient/mysql_client.c

@@ -225,7 +225,7 @@ int MySQLSendHandshake(DCB* dcb)
     uint8_t mysql_server_language = 8;
     uint8_t mysql_server_status[2];
     uint8_t mysql_scramble_len = 21;
-    uint8_t mysql_filler_ten[10];
+    uint8_t mysql_filler_ten[10] = {};
     /* uint8_t mysql_last_byte = 0x00; not needed */
     char server_scramble[GW_MYSQL_SCRAMBLE_SIZE + 1] = "";
     char *version_string;
@@ -239,12 +239,14 @@ int MySQLSendHandshake(DCB* dcb)
 
     MySQLProtocol *protocol = DCB_PROTOCOL(dcb, MySQLProtocol);
     GWBUF *buf;
+    bool is_maria = false;
 
     /* get the version string from service property if available*/
     if (dcb->service->version_string != NULL)
     {
         version_string = dcb->service->version_string;
         len_version_string = strlen(version_string);
+        is_maria = strstr(version_string, "10.2.");
     }
     else
     {
@@ -257,9 +259,15 @@ int MySQLSendHandshake(DCB* dcb)
     // copy back to the caller
     memcpy(protocol->scramble, server_scramble, GW_MYSQL_SCRAMBLE_SIZE);
 
-    // fill the handshake packet
+    if (is_maria)
-
+    {
-    memset(mysql_filler_ten, 0x00, sizeof(mysql_filler_ten));
+        /**
+         * The new 10.2 capability flags are stored in the last 4 bytes of the
+         * 10 byte filler block.
+         */
+        uint32_t new_flags = MXS_MARIA_CAP_STMT_BULK_OPERATIONS;
+        memcpy(mysql_filler_ten + 6, &new_flags, sizeof(new_flags));
+    }
 
     // thread id, now put thePID
     id_num = getpid() + dcb->fd;
@@ -325,11 +333,19 @@ int MySQLSendHandshake(DCB* dcb)
     mysql_handshake_payload++;
 
     // write server capabilities part one
-    mysql_server_capabilities_one[0] = GW_MYSQL_SERVER_CAPABILITIES_BYTE1;
+    mysql_server_capabilities_one[0] = (uint8_t)GW_MYSQL_CAPABILITIES_SERVER;
-    mysql_server_capabilities_one[1] = GW_MYSQL_SERVER_CAPABILITIES_BYTE2;
+    mysql_server_capabilities_one[1] = (uint8_t)(GW_MYSQL_CAPABILITIES_SERVER >> 8);
 
+    // Check that we match the old values
+    ss_dassert(mysql_server_capabilities_one[0] = 0xff);
+    ss_dassert(mysql_server_capabilities_one[1] = 0xf7);
 
-    mysql_server_capabilities_one[0] &= ~(int)GW_MYSQL_CAPABILITIES_COMPRESS;
+    if (is_maria)
+    {
+        /** A MariaDB 10.2 server doesn't send the CLIENT_MYSQL capability
+         * to signal that it supports extended capabilities */
+        mysql_server_capabilities_one[0] &= ~(uint8_t)GW_MYSQL_CAPABILITIES_CLIENT_MYSQL;
+    }
 
     if (ssl_required_by_dcb(dcb))
     {
@@ -350,8 +366,13 @@ int MySQLSendHandshake(DCB* dcb)
     mysql_handshake_payload = mysql_handshake_payload + sizeof(mysql_server_status);
 
     //write server capabilities part two
-    mysql_server_capabilities_two[0] = 15;
+    mysql_server_capabilities_two[0] = (uint8_t)(GW_MYSQL_CAPABILITIES_SERVER >> 16);
-    mysql_server_capabilities_two[1] = 128;
+    mysql_server_capabilities_two[1] = (uint8_t)(GW_MYSQL_CAPABILITIES_SERVER >> 24);
+
+    // Check that we match the old values
+    ss_dassert(mysql_server_capabilities_two[0] == 15);
+    /** NOTE: pre-2.1 versions sent the fourth byte of the capabilities as
+     the value 128 even though there's no such capability. */
 
     memcpy(mysql_handshake_payload, mysql_server_capabilities_two, sizeof(mysql_server_capabilities_two));
     mysql_handshake_payload = mysql_handshake_payload + sizeof(mysql_server_capabilities_two);
@@ -532,6 +553,13 @@ static void store_client_information(DCB *dcb, GWBUF *buffer)
     proto->client_capabilities = gw_mysql_get_byte4(data + MYSQL_CLIENT_CAP_OFFSET);
     proto->charset = data[MYSQL_CHARSET_OFFSET];
 
+    /** MariaDB 10.2 compatible clients don't set the first bit to signal that
+     * there are extra capabilities stored in the last 4 bytes of the 23 byte filler. */
+    if ((proto->client_capabilities & GW_MYSQL_CAPABILITIES_CLIENT_MYSQL) == 0)
+    {
+        proto->extra_capabilities = gw_mysql_get_byte4(data + MARIADB_CAP_OFFSET);
+    }
+
     if (len > MYSQL_AUTH_PACKET_BASE_SIZE)
     {
         strcpy(ses->user, (char*)data + MYSQL_AUTH_PACKET_BASE_SIZE);

server/modules/protocol/MySQL/mysql_common.c

@@ -103,6 +103,7 @@ MySQLProtocol* mysql_protocol_init(DCB* dcb, int fd)
     p->protocol_command.scom_nresponse_packets = 0;
     p->protocol_command.scom_nbytes_to_read = 0;
     p->stored_query = NULL;
+    p->extra_capabilities = 0;
 #if defined(SS_DEBUG)
     p->protocol_chk_top = CHK_NUM_PROTOCOL;
     p->protocol_chk_tail = CHK_NUM_PROTOCOL;
@@ -1364,8 +1365,12 @@ mxs_auth_state_t gw_send_backend_auth(DCB *dcb)
 
     payload++;
 
-    // 23 bytes of 0
+    // 19 filler bytes of 0
-    payload += 23;
+    payload += 19;
+
+    // Either MariaDB 10.2 extra capabilities or 4 bytes filler
+    memcpy(payload, &conn->extra_capabilities, sizeof(conn->extra_capabilities));
+    payload += 4;
 
     if (dcb->server->server_ssl && dcb->ssl_state != SSL_ESTABLISHED)
     {