MXS-1209: Master GTID Registration, part1

New option ‘mariadb10_master_gtid’ in use.
Only MariaDB 10 Slaves with GTID request can register if the option is
set.
When receiving a Master reply to a GTID request and the GTID_LIST is
seen, an IGNORABLE event could be written at the end of file if
GTID_LIST next_pos is beyond that EOF

server/modules/routing/binlogrouter/blr.c

@@ -538,6 +538,10 @@ createInstance(SERVICE *service, char **options)
                 {
                     inst->mariadb10_gtid = config_truth_value(value);
                 }
+                else if (strcmp(options[i], "mariadb10_master_gtid") == 0)
+                {
+                    inst->mariadb10_master_gtid = config_truth_value(value);
+                }
                 else if (strcmp(options[i], "encryption_algorithm") == 0)
                 {
                     int ret = blr_check_encryption_algorithm(value);
@@ -2429,7 +2433,8 @@ blr_ping(ROUTER_INSTANCE *router, ROUTER_SLAVE *slave, GWBUF *queue)
  *
  */
 int
-blr_send_custom_error(DCB *dcb, int packet_number,
+blr_send_custom_error(DCB *dcb,
+                      int packet_number,
                       int affected_rows,
                       char *msg,
                       char *statemsg,

server/modules/routing/binlogrouter/blr.h

@@ -189,6 +189,13 @@ enum blr_aes_mode
  */
 #define BLR_REQUEST_ANNOTATE_ROWS_EVENT         2
 
+/** MaxScale generated events */
+typedef enum
+{
+    BLRM_IGNORABLE,          /*< Ignorable event */
+    BLRM_START_ENCRYPTION    /*< Start Encryption event */
+} generated_event_t;
+
 /**
  * How often to call the binlog status function (seconds)
  */

server/modules/routing/binlogrouter/blr_file.c

@@ -188,7 +188,7 @@ static uint8_t *blr_create_ignorable_event(uint32_t event_size,
                                            REP_HEADER *hdr,
                                            uint32_t event_pos,
                                            bool do_checksum);
-static int blr_write_special_event(ROUTER_INSTANCE *router,
+int blr_write_special_event(ROUTER_INSTANCE *router,
                                    uint32_t file_offset,
                                    uint32_t hole_size,
                                    REP_HEADER *hdr,
@@ -221,13 +221,6 @@ static int blr_binlog_event_check(ROUTER_INSTANCE *router,
 
 static void blr_report_checksum(REP_HEADER hdr, const uint8_t *buffer, char *output);
 
-/** MaxScale generated events */
-typedef enum
-{
-    BLRM_IGNORABLE, /*< Ignorable event */
-    BLRM_START_ENCRYPTION /*< Start Encryption event */
-} generated_event_t;
-
 /**
  * MariaDB 10.1.7 Start Encryption event content
  *
@@ -2069,7 +2062,9 @@ blr_read_events_all_events(ROUTER_INSTANCE *router,
                     else
                     {
                         char mariadb_gtid[GTID_MAX_LEN + 1];
-                        snprintf(mariadb_gtid, GTID_MAX_LEN, "%u-%u-%lu",
+                        snprintf(mariadb_gtid,
+                                 GTID_MAX_LEN,
+                                 "%u-%u-%lu",
                                  domainid, hdr.serverid,
                                  n_sequence);
 
@@ -2097,6 +2092,46 @@ blr_read_events_all_events(ROUTER_INSTANCE *router,
             }
         }
 
+        /**
+         * Check for GTID_LIST_EVENT
+         */
+        if (router->mariadb10_compat)
+        {
+            if (hdr.event_type == MARIADB10_GTID_GTID_LIST_EVENT)
+            {
+                uint32_t n_gtids; /* The lower 28 bits are the number of GTIDs */
+                uint32_t domainid;  /* 4 bytes */
+                uint32_t serverid;  /* 4 bytes */
+                uint64_t n_sequence;/* 8 bytes */
+                uint8_t flags; /* 1 byte, 4 bits */
+                char mariadb_gtid[GTID_MAX_LEN + 1];
+
+                n_gtids = extract_field(ptr, 32);
+                n_gtids &= 0x01111111;
+
+                domainid = extract_field(ptr + 4, 32);
+                serverid = extract_field(ptr + 4 + 4, 32);
+                n_sequence = extract_field(ptr + 4 + 4 + 4, 64);
+
+                snprintf(mariadb_gtid,
+                         GTID_MAX_LEN,
+                         "%u-%u-%lu",
+                         domainid,
+                         serverid,
+                         n_sequence);
+
+                MXS_DEBUG("GTID List has %lu GTIDs, first is %s",
+                           (unsigned long)n_gtids,
+                           mariadb_gtid);
+
+                /* Set MariaDB GTID */
+                if (router->mariadb10_gtid)
+                {
+                    strcpy(router->last_mariadb_gtid, mariadb_gtid);
+                }
+            }
+        }
+
         /**
          * Check QUERY_EVENT
          *
@@ -2666,7 +2701,7 @@ blr_create_ignorable_event(uint32_t event_size,
  * @param type          Type of special event to create and write
  * @return              1 on success, 0 on error
  */
-static int
+int
 blr_write_special_event(ROUTER_INSTANCE *router, uint32_t file_offset, uint32_t event_size, REP_HEADER *hdr,
                         int type)
 {

server/modules/routing/binlogrouter/blr_master.c

@@ -127,7 +127,7 @@ static void blr_register_cache_response(ROUTER_INSTANCE *router,
                                         const char *save_tag,
                                         GWBUF *in_buf);
 static void blr_start_master_registration(ROUTER_INSTANCE *router, GWBUF *buf);
-static void blr_register_mariadb_gtid_domain(ROUTER_INSTANCE *router,
+static void blr_register_mariadb_gtid_request(ROUTER_INSTANCE *router,
                                              GWBUF *buf);
 static bool blr_handle_fake_rotate(ROUTER_INSTANCE *router,
                                    REP_HEADER *hdr,
@@ -135,7 +135,11 @@ static bool blr_handle_fake_rotate(ROUTER_INSTANCE *router,
 static void blr_handle_fake_gtid_list(ROUTER_INSTANCE *router,
                                       REP_HEADER *hdr,
                                       uint8_t *ptr);
-
+extern int blr_write_special_event(ROUTER_INSTANCE *router,
+                                   uint32_t file_offset,
+                                   uint32_t hole_size,
+                                   REP_HEADER *hdr,
+                                   int type);
 
 static void worker_cb_start_master(int worker_id, void* data);
 static void blr_start_master_in_main(void* data);
@@ -2792,8 +2796,29 @@ static void blr_start_master_registration(ROUTER_INSTANCE *router, GWBUF *buf)
         }
         break;
     case BLRM_MARIADB10_GTID_DOMAIN: // MariaDB10 Only
-        // Next state is BLRM_LATIN1
+        // Next state is BLRM_MARIADB10_REQUEST_GTID
-        blr_register_mariadb_gtid_domain(router, buf);
+        blr_register_mariadb_gtid_request(router, buf);
+        break;
+    case BLRM_MARIADB10_REQUEST_GTID: // MariaDB10 Only
+        // Don't save GTID request
+        gwbuf_free(buf);
+        blr_register_send_command(router,
+                                  "SET @slave_gtid_strict_mode=1",
+                                  BLRM_MARIADB10_GTID_STRICT);
+        break;
+    case BLRM_MARIADB10_GTID_STRICT: // MariaDB10 Only
+        // Don't save GTID strict
+        gwbuf_free(buf);
+        blr_register_send_command(router,
+                                  "SET @slave_gtid_ignore_duplicates=1",
+                                  BLRM_MARIADB10_GTID_NO_DUP);
+        break;
+    case BLRM_MARIADB10_GTID_NO_DUP: // MariaDB10 Only
+        // Don't save GTID ignore
+        gwbuf_free(buf);
+        blr_register_send_command(router,
+                                  "SET NAMES latin1",
+                                  BLRM_LATIN1);
         break;
     case BLRM_GTIDMODE: // MySQL 5.6/5.7 only
         blr_register_serveruuid(router, buf);
@@ -3009,13 +3034,15 @@ static void blr_start_master_registration(ROUTER_INSTANCE *router, GWBUF *buf)
  * Slave Protocol registration to Master (MariaDB 10 compatibility):
  *
  * Handles previous reply from MariaDB10 Master (GTID Domain ID) and
- * sets the state to BLRM_LATIN1
+ * sends the SET @slave_connect_state='x-y-z' GTID registration.
+ *
+ * The next state is set to BLRM_MARIADB10_REQUEST_GTID
  *
  * @param router    Current router instance
  * @param buf       GWBUF with server reply to previous
  *                  registration command
  */
-static void blr_register_mariadb_gtid_domain(ROUTER_INSTANCE *router,
+static void blr_register_mariadb_gtid_request(ROUTER_INSTANCE *router,
                                              GWBUF *buf)
 {
     // Extract GTID domain
@@ -3025,9 +3052,20 @@ static void blr_register_mariadb_gtid_domain(ROUTER_INSTANCE *router,
     MXS_FREE(val);
     // Don't save the server response
     gwbuf_free(buf);
+
+    // SET the requested GTID
+    char set_gtid[GTID_MAX_LEN + 33 + 1];
+    sprintf(set_gtid,
+            "SET @slave_connect_state='%s'",
+            router->last_mariadb_gtid);
+
+     MXS_INFO("%s: Requesting GTID (%s) from master server.",
+                   router->service->name,
+                   router->last_mariadb_gtid);
+    // Send the request
     blr_register_send_command(router,
-                              "SET NAMES latin1",
+                              set_gtid,
-                              BLRM_LATIN1);
+                              BLRM_MARIADB10_REQUEST_GTID);
 }
 
 /**
@@ -3066,6 +3104,10 @@ static bool blr_handle_fake_rotate(ROUTER_INSTANCE *router,
     pos <<= 32;
     pos |= extract_field(ptr + BINLOG_EVENT_HDR_LEN, 32);
 
+    MXS_INFO("Fake ROTATE_EVENT received: file %s, pos %lu. Next event at pos %lu\n",
+             file,
+             (unsigned long)pos,
+             (unsigned long)hdr->next_pos);
     /**
      * TODO: Detect any missing file in sequence.
      */
@@ -3075,11 +3117,17 @@ static bool blr_handle_fake_rotate(ROUTER_INSTANCE *router,
     /* Set writing pos to 4 if Master GTID */
     if (router->mariadb10_master_gtid && pos == 4)
     {
-        // Set pos = 4
+        /**
+         * If a MariadB 10 Slave is connecting and reading the
+         * events from this binlog file, the router->binlog_position check
+         * might fail in blr_slave.c:blr_slave_binlog_dump()
+         * and the slave connection will be closed.
+         *
+         * The slave will automatically try to re-connect.
+         */
         router->last_written = BINLOG_MAGIC_SIZE;
         router->current_pos = BINLOG_MAGIC_SIZE;
         router->binlog_position = BINLOG_MAGIC_SIZE;
-        router->current_safe_event = BINLOG_MAGIC_SIZE;
         router->last_event_pos = BINLOG_MAGIC_SIZE;
     }
 
@@ -3111,14 +3159,45 @@ static void blr_handle_fake_gtid_list(ROUTER_INSTANCE *router,
 
     if (router->mariadb10_master_gtid)
     {
+        uint64_t binlog_file_eof = lseek(router->binlog_fd, 0L, SEEK_END);
+
         MXS_INFO("Fake GTID_LIST received: file %s, pos %lu. Next event at pos %lu\n",
                  router->binlog_name,
                  (unsigned long)router->current_pos,
                  (unsigned long)hdr->next_pos);
 
-        /* We can write in any (after FDE and STE) binlog file position */
+        /**
-        /* TODO: fill any GAP with an ignorable event */
+         * We could write in any binlog file position:
+         * fill any GAP with an ignorable event
+         * if GTID_LIST next_pos is greter than current EOF
+         */
 
+        if (hdr->next_pos && (hdr->next_pos > binlog_file_eof))
+        {
+             uint64_t hole_size = hdr->next_pos - binlog_file_eof;
+
+             MXS_INFO("Detected hole while processing"
+                      " a Fake GTID_LIST Event: hole size will be %lu bytes",
+                      (unsigned long)hole_size);
+
+             /* Set the offet for the write routine */
+             spinlock_acquire(&router->binlog_lock);
+
+             router->last_written = binlog_file_eof;
+
+             spinlock_release(&router->binlog_lock);
+
+             // Write One Hole
+             // TODO: write small holes
+             blr_write_special_event(router,
+                                     binlog_file_eof,
+                                     hole_size,
+                                     hdr,
+                                     BLRM_IGNORABLE);
+        }
+        else
+        {
+            // Increment internal offsets
             spinlock_acquire(&router->binlog_lock);
 
             router->last_written = hdr->next_pos;
@@ -3127,4 +3206,5 @@ static void blr_handle_fake_gtid_list(ROUTER_INSTANCE *router,
 
             spinlock_release(&router->binlog_lock);
         }
+    }
 }

server/modules/routing/binlogrouter/blr_slave.c

@@ -323,7 +323,6 @@ blr_slave_request(ROUTER_INSTANCE *router, ROUTER_SLAVE *slave, GWBUF *queue)
     case COM_QUERY:
         slave->stats.n_queries++;
         return blr_slave_query(router, slave, queue);
-
     case COM_REGISTER_SLAVE:
         if (router->master_state == BLRM_UNCONFIGURED)
         {
@@ -343,13 +342,16 @@ blr_slave_request(ROUTER_INSTANCE *router, ROUTER_SLAVE *slave, GWBUF *queue)
          * If Master is MariaDB10 don't allow registration from
          * MariaDB/Mysql 5 Slaves
          */
-
         if (router->mariadb10_compat && !slave->mariadb10_compat)
         {
             slave->state = BLRS_ERRORED;
-            blr_send_custom_error(slave->dcb, 1, 0,
+            /* Send error that stops slave replication */
+            blr_send_custom_error(slave->dcb,
+                                  ++slave->seqno,
+                                  0,
                                   "MariaDB 10 Slave is required for Slave registration",
-                                  "42000", 1064);
+                                  "42000",
+                                  1064);
 
             MXS_ERROR("%s: Slave %s: a MariaDB 10 Slave is required for Slave registration",
                       router->service->name,
@@ -358,18 +360,36 @@ blr_slave_request(ROUTER_INSTANCE *router, ROUTER_SLAVE *slave, GWBUF *queue)
             dcb_close(slave->dcb);
             return 1;
         }
+        else if (router->mariadb10_master_gtid && !slave->mariadb_gtid)
+        {
+            slave->state = BLRS_ERRORED;
+            /* Send error that stops slave replication */
+            blr_send_custom_error(slave->dcb,
+                                  ++slave->seqno,
+                                  0,
+                                  "MariaDB 10 Slave GTID is required for Slave registration.",
+                                  "HY000",
+                                  //BINLOG_FATAL_ERROR_READING);
+                                  1597);
+            MXS_ERROR("%s: Slave %s: a MariaDB 10 Slave GTID request"
+                      " is needed for Slave registration."
+                      " Please use: CHANGE MASTER TO master_use_gtid=slave_pos.",
+                      router->service->name,
+                      slave->dcb->remote);
+
+            dcb_close(slave->dcb);
+            return 1;
+        }
         else
         {
             /* Master and Slave version OK: continue with slave registration */
             return blr_slave_register(router, slave, queue);
         }
-
     case COM_BINLOG_DUMP:
         {
             char task_name[BLRM_TASK_NAME_LEN + 1] = "";
-            int rc = 0;
 
-            rc = blr_slave_binlog_dump(router, slave, queue);
+            int rc = blr_slave_binlog_dump(router, slave, queue);
 
             if (router->send_slave_heartbeat && rc && slave->heartbeat > 0)
             {
@@ -384,13 +404,10 @@ blr_slave_request(ROUTER_INSTANCE *router, ROUTER_SLAVE *slave, GWBUF *queue)
 
             return rc;
         }
-
     case COM_STATISTICS:
         return blr_statistics(router, slave, queue);
-
     case COM_PING:
         return blr_ping(router, slave, queue);
-
     case COM_QUIT:
         MXS_DEBUG("COM_QUIT received from slave with server_id %d",
                   slave->serverid);
@@ -6908,6 +6925,62 @@ static bool blr_handle_set_stmt(ROUTER_INSTANCE *router,
         }
         return true;
     }
+    else if (strstr(word, "@@global.gtid_slave_pos") != NULL)
+    {
+        if (slave->serverid != 0)
+        {
+            MXS_ERROR("Master GTID registration can be sent only via administration connection");
+            blr_slave_send_error_packet(slave,
+                                        "Master GTID registration cannot be issued by a regitrating slave.",
+                                        (unsigned int)1198, NULL);
+            return false;
+        }
+        if (router->master_state != BLRM_SLAVE_STOPPED)
+        {
+            MXS_ERROR("Master GTID registration needs stopped slave: issue STOP SLAVE first.");
+            blr_slave_send_error_packet(slave,
+                                        "Cannot use Master GTID registration with a running slave; "
+                                        "run STOP SLAVE first",
+                                        (unsigned int)1198, NULL);
+            return true;
+        }
+        /* If not mariadb GTID an error message will be returned */
+        if (router->mariadb10_master_gtid)
+        {
+            if ((word = strtok_r(NULL, sep, &brkb)) != NULL)
+            {
+                char heading[GTID_MAX_LEN + 1];
+                MXS_INFO("Binlog server requests GTID '%s' to master",
+                         word);
+
+                // TODO: gtid_strip_chars routine for this
+                strcpy(heading, word + 1);
+                heading[strlen(heading) - 1] = '\0';
+                if (!heading[0])
+                {
+                    MXS_ERROR("Cannot request empty GTID righ now");
+                    blr_slave_send_error_packet(slave,
+                                                "Empty GTID not implemented righ now",
+                                                (unsigned int)1198, NULL);
+                    return false;
+                }
+                else
+                {
+                    strcpy(router->last_mariadb_gtid, heading);
+                    blr_slave_send_ok(router, slave);
+                    return true;
+                }
+            }
+        }
+        else
+        {
+            MXS_ERROR("Master GTID registration needs 'mariadb10_master_gtid' option to be set.");
+            blr_slave_send_error_packet(slave,
+                                        "Master GTID registration needs 'mariadb10_master_gtid' option to be set.",
+                                        (unsigned int)1198, NULL);
+            return true;
+        }
+    }
     else if (strstr(word, "@slave_connect_state") != NULL)
     {
         /* If not mariadb an error message will be returned */
@@ -6934,6 +7007,11 @@ static bool blr_handle_set_stmt(ROUTER_INSTANCE *router,
              blr_slave_send_ok(router, slave);
              return true;
         }
+        else
+        {
+            MXS_ERROR("GTID Master registration is not enabled");
+            return false;
+        }
     }
     else if (strcasecmp(word, "@slave_gtid_strict_mode") == 0)
     {