From 61ffd3e0f0bc269d8952d0369b9522cbc5a7725e Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Tue, 9 May 2017 08:25:44 +0200 Subject: [PATCH] MXS-1209: Master GTID Registration, part1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New option ‘mariadb10_master_gtid’ in use. Only MariaDB 10 Slaves with GTID request can register if the option is set. When receiving a Master reply to a GTID request and the GTID_LIST is seen, an IGNORABLE event could be written at the end of file if GTID_LIST next_pos is beyond that EOF --- server/modules/routing/binlogrouter/blr.c | 7 +- server/modules/routing/binlogrouter/blr.h | 7 ++ .../modules/routing/binlogrouter/blr_file.c | 65 +++++++--- .../modules/routing/binlogrouter/blr_master.c | 114 +++++++++++++++--- .../modules/routing/binlogrouter/blr_slave.c | 98 +++++++++++++-- 5 files changed, 248 insertions(+), 43 deletions(-) diff --git a/server/modules/routing/binlogrouter/blr.c b/server/modules/routing/binlogrouter/blr.c index 704cff599..5c7de2dcc 100644 --- a/server/modules/routing/binlogrouter/blr.c +++ b/server/modules/routing/binlogrouter/blr.c @@ -538,6 +538,10 @@ createInstance(SERVICE *service, char **options) { inst->mariadb10_gtid = config_truth_value(value); } + else if (strcmp(options[i], "mariadb10_master_gtid") == 0) + { + inst->mariadb10_master_gtid = config_truth_value(value); + } else if (strcmp(options[i], "encryption_algorithm") == 0) { int ret = blr_check_encryption_algorithm(value); @@ -2429,7 +2433,8 @@ blr_ping(ROUTER_INSTANCE *router, ROUTER_SLAVE *slave, GWBUF *queue) * */ int -blr_send_custom_error(DCB *dcb, int packet_number, +blr_send_custom_error(DCB *dcb, + int packet_number, int affected_rows, char *msg, char *statemsg, diff --git a/server/modules/routing/binlogrouter/blr.h b/server/modules/routing/binlogrouter/blr.h index 70355f486..adbeb6fde 100644 --- a/server/modules/routing/binlogrouter/blr.h +++ b/server/modules/routing/binlogrouter/blr.h @@ -189,6 +189,13 @@ enum blr_aes_mode */ #define BLR_REQUEST_ANNOTATE_ROWS_EVENT 2 +/** MaxScale generated events */ +typedef enum +{ + BLRM_IGNORABLE, /*< Ignorable event */ + BLRM_START_ENCRYPTION /*< Start Encryption event */ +} generated_event_t; + /** * How often to call the binlog status function (seconds) */ diff --git a/server/modules/routing/binlogrouter/blr_file.c b/server/modules/routing/binlogrouter/blr_file.c index caa4d4093..e7b326f9c 100644 --- a/server/modules/routing/binlogrouter/blr_file.c +++ b/server/modules/routing/binlogrouter/blr_file.c @@ -188,7 +188,7 @@ static uint8_t *blr_create_ignorable_event(uint32_t event_size, REP_HEADER *hdr, uint32_t event_pos, bool do_checksum); -static int blr_write_special_event(ROUTER_INSTANCE *router, +int blr_write_special_event(ROUTER_INSTANCE *router, uint32_t file_offset, uint32_t hole_size, REP_HEADER *hdr, @@ -221,13 +221,6 @@ static int blr_binlog_event_check(ROUTER_INSTANCE *router, static void blr_report_checksum(REP_HEADER hdr, const uint8_t *buffer, char *output); -/** MaxScale generated events */ -typedef enum -{ - BLRM_IGNORABLE, /*< Ignorable event */ - BLRM_START_ENCRYPTION /*< Start Encryption event */ -} generated_event_t; - /** * MariaDB 10.1.7 Start Encryption event content * @@ -2069,7 +2062,9 @@ blr_read_events_all_events(ROUTER_INSTANCE *router, else { char mariadb_gtid[GTID_MAX_LEN + 1]; - snprintf(mariadb_gtid, GTID_MAX_LEN, "%u-%u-%lu", + snprintf(mariadb_gtid, + GTID_MAX_LEN, + "%u-%u-%lu", domainid, hdr.serverid, n_sequence); @@ -2098,11 +2093,51 @@ blr_read_events_all_events(ROUTER_INSTANCE *router, } /** - * Check QUERY_EVENT - * - * Check for BEGIN ( ONLY for mysql 5.6, mariadb 5.5 ) - * Check for COMMIT (not transactional engines) - */ + * Check for GTID_LIST_EVENT + */ + if (router->mariadb10_compat) + { + if (hdr.event_type == MARIADB10_GTID_GTID_LIST_EVENT) + { + uint32_t n_gtids; /* The lower 28 bits are the number of GTIDs */ + uint32_t domainid; /* 4 bytes */ + uint32_t serverid; /* 4 bytes */ + uint64_t n_sequence;/* 8 bytes */ + uint8_t flags; /* 1 byte, 4 bits */ + char mariadb_gtid[GTID_MAX_LEN + 1]; + + n_gtids = extract_field(ptr, 32); + n_gtids &= 0x01111111; + + domainid = extract_field(ptr + 4, 32); + serverid = extract_field(ptr + 4 + 4, 32); + n_sequence = extract_field(ptr + 4 + 4 + 4, 64); + + snprintf(mariadb_gtid, + GTID_MAX_LEN, + "%u-%u-%lu", + domainid, + serverid, + n_sequence); + + MXS_DEBUG("GTID List has %lu GTIDs, first is %s", + (unsigned long)n_gtids, + mariadb_gtid); + + /* Set MariaDB GTID */ + if (router->mariadb10_gtid) + { + strcpy(router->last_mariadb_gtid, mariadb_gtid); + } + } + } + + /** + * Check QUERY_EVENT + * + * Check for BEGIN ( ONLY for mysql 5.6, mariadb 5.5 ) + * Check for COMMIT (not transactional engines) + */ if (hdr.event_type == QUERY_EVENT) { @@ -2666,7 +2701,7 @@ blr_create_ignorable_event(uint32_t event_size, * @param type Type of special event to create and write * @return 1 on success, 0 on error */ -static int +int blr_write_special_event(ROUTER_INSTANCE *router, uint32_t file_offset, uint32_t event_size, REP_HEADER *hdr, int type) { diff --git a/server/modules/routing/binlogrouter/blr_master.c b/server/modules/routing/binlogrouter/blr_master.c index 7707916c9..567076f1c 100644 --- a/server/modules/routing/binlogrouter/blr_master.c +++ b/server/modules/routing/binlogrouter/blr_master.c @@ -127,7 +127,7 @@ static void blr_register_cache_response(ROUTER_INSTANCE *router, const char *save_tag, GWBUF *in_buf); static void blr_start_master_registration(ROUTER_INSTANCE *router, GWBUF *buf); -static void blr_register_mariadb_gtid_domain(ROUTER_INSTANCE *router, +static void blr_register_mariadb_gtid_request(ROUTER_INSTANCE *router, GWBUF *buf); static bool blr_handle_fake_rotate(ROUTER_INSTANCE *router, REP_HEADER *hdr, @@ -135,7 +135,11 @@ static bool blr_handle_fake_rotate(ROUTER_INSTANCE *router, static void blr_handle_fake_gtid_list(ROUTER_INSTANCE *router, REP_HEADER *hdr, uint8_t *ptr); - +extern int blr_write_special_event(ROUTER_INSTANCE *router, + uint32_t file_offset, + uint32_t hole_size, + REP_HEADER *hdr, + int type); static void worker_cb_start_master(int worker_id, void* data); static void blr_start_master_in_main(void* data); @@ -2792,8 +2796,29 @@ static void blr_start_master_registration(ROUTER_INSTANCE *router, GWBUF *buf) } break; case BLRM_MARIADB10_GTID_DOMAIN: // MariaDB10 Only - // Next state is BLRM_LATIN1 - blr_register_mariadb_gtid_domain(router, buf); + // Next state is BLRM_MARIADB10_REQUEST_GTID + blr_register_mariadb_gtid_request(router, buf); + break; + case BLRM_MARIADB10_REQUEST_GTID: // MariaDB10 Only + // Don't save GTID request + gwbuf_free(buf); + blr_register_send_command(router, + "SET @slave_gtid_strict_mode=1", + BLRM_MARIADB10_GTID_STRICT); + break; + case BLRM_MARIADB10_GTID_STRICT: // MariaDB10 Only + // Don't save GTID strict + gwbuf_free(buf); + blr_register_send_command(router, + "SET @slave_gtid_ignore_duplicates=1", + BLRM_MARIADB10_GTID_NO_DUP); + break; + case BLRM_MARIADB10_GTID_NO_DUP: // MariaDB10 Only + // Don't save GTID ignore + gwbuf_free(buf); + blr_register_send_command(router, + "SET NAMES latin1", + BLRM_LATIN1); break; case BLRM_GTIDMODE: // MySQL 5.6/5.7 only blr_register_serveruuid(router, buf); @@ -3009,13 +3034,15 @@ static void blr_start_master_registration(ROUTER_INSTANCE *router, GWBUF *buf) * Slave Protocol registration to Master (MariaDB 10 compatibility): * * Handles previous reply from MariaDB10 Master (GTID Domain ID) and - * sets the state to BLRM_LATIN1 + * sends the SET @slave_connect_state='x-y-z' GTID registration. + * + * The next state is set to BLRM_MARIADB10_REQUEST_GTID * * @param router Current router instance * @param buf GWBUF with server reply to previous * registration command */ -static void blr_register_mariadb_gtid_domain(ROUTER_INSTANCE *router, +static void blr_register_mariadb_gtid_request(ROUTER_INSTANCE *router, GWBUF *buf) { // Extract GTID domain @@ -3025,9 +3052,20 @@ static void blr_register_mariadb_gtid_domain(ROUTER_INSTANCE *router, MXS_FREE(val); // Don't save the server response gwbuf_free(buf); + + // SET the requested GTID + char set_gtid[GTID_MAX_LEN + 33 + 1]; + sprintf(set_gtid, + "SET @slave_connect_state='%s'", + router->last_mariadb_gtid); + + MXS_INFO("%s: Requesting GTID (%s) from master server.", + router->service->name, + router->last_mariadb_gtid); + // Send the request blr_register_send_command(router, - "SET NAMES latin1", - BLRM_LATIN1); + set_gtid, + BLRM_MARIADB10_REQUEST_GTID); } /** @@ -3066,6 +3104,10 @@ static bool blr_handle_fake_rotate(ROUTER_INSTANCE *router, pos <<= 32; pos |= extract_field(ptr + BINLOG_EVENT_HDR_LEN, 32); + MXS_INFO("Fake ROTATE_EVENT received: file %s, pos %lu. Next event at pos %lu\n", + file, + (unsigned long)pos, + (unsigned long)hdr->next_pos); /** * TODO: Detect any missing file in sequence. */ @@ -3075,11 +3117,17 @@ static bool blr_handle_fake_rotate(ROUTER_INSTANCE *router, /* Set writing pos to 4 if Master GTID */ if (router->mariadb10_master_gtid && pos == 4) { - // Set pos = 4 + /** + * If a MariadB 10 Slave is connecting and reading the + * events from this binlog file, the router->binlog_position check + * might fail in blr_slave.c:blr_slave_binlog_dump() + * and the slave connection will be closed. + * + * The slave will automatically try to re-connect. + */ router->last_written = BINLOG_MAGIC_SIZE; router->current_pos = BINLOG_MAGIC_SIZE; router->binlog_position = BINLOG_MAGIC_SIZE; - router->current_safe_event = BINLOG_MAGIC_SIZE; router->last_event_pos = BINLOG_MAGIC_SIZE; } @@ -3111,20 +3159,52 @@ static void blr_handle_fake_gtid_list(ROUTER_INSTANCE *router, if (router->mariadb10_master_gtid) { + uint64_t binlog_file_eof = lseek(router->binlog_fd, 0L, SEEK_END); + MXS_INFO("Fake GTID_LIST received: file %s, pos %lu. Next event at pos %lu\n", router->binlog_name, (unsigned long)router->current_pos, (unsigned long)hdr->next_pos); - /* We can write in any (after FDE and STE) binlog file position */ - /* TODO: fill any GAP with an ignorable event */ + /** + * We could write in any binlog file position: + * fill any GAP with an ignorable event + * if GTID_LIST next_pos is greter than current EOF + */ - spinlock_acquire(&router->binlog_lock); + if (hdr->next_pos && (hdr->next_pos > binlog_file_eof)) + { + uint64_t hole_size = hdr->next_pos - binlog_file_eof; - router->last_written = hdr->next_pos; - router->last_event_pos = router->current_pos; - router->current_pos = hdr->next_pos; + MXS_INFO("Detected hole while processing" + " a Fake GTID_LIST Event: hole size will be %lu bytes", + (unsigned long)hole_size); - spinlock_release(&router->binlog_lock); + /* Set the offet for the write routine */ + spinlock_acquire(&router->binlog_lock); + + router->last_written = binlog_file_eof; + + spinlock_release(&router->binlog_lock); + + // Write One Hole + // TODO: write small holes + blr_write_special_event(router, + binlog_file_eof, + hole_size, + hdr, + BLRM_IGNORABLE); + } + else + { + // Increment internal offsets + spinlock_acquire(&router->binlog_lock); + + router->last_written = hdr->next_pos; + router->last_event_pos = router->current_pos; + router->current_pos = hdr->next_pos; + + spinlock_release(&router->binlog_lock); + } } } diff --git a/server/modules/routing/binlogrouter/blr_slave.c b/server/modules/routing/binlogrouter/blr_slave.c index faeda5a14..de6ed03a0 100644 --- a/server/modules/routing/binlogrouter/blr_slave.c +++ b/server/modules/routing/binlogrouter/blr_slave.c @@ -323,7 +323,6 @@ blr_slave_request(ROUTER_INSTANCE *router, ROUTER_SLAVE *slave, GWBUF *queue) case COM_QUERY: slave->stats.n_queries++; return blr_slave_query(router, slave, queue); - case COM_REGISTER_SLAVE: if (router->master_state == BLRM_UNCONFIGURED) { @@ -343,13 +342,16 @@ blr_slave_request(ROUTER_INSTANCE *router, ROUTER_SLAVE *slave, GWBUF *queue) * If Master is MariaDB10 don't allow registration from * MariaDB/Mysql 5 Slaves */ - if (router->mariadb10_compat && !slave->mariadb10_compat) { slave->state = BLRS_ERRORED; - blr_send_custom_error(slave->dcb, 1, 0, + /* Send error that stops slave replication */ + blr_send_custom_error(slave->dcb, + ++slave->seqno, + 0, "MariaDB 10 Slave is required for Slave registration", - "42000", 1064); + "42000", + 1064); MXS_ERROR("%s: Slave %s: a MariaDB 10 Slave is required for Slave registration", router->service->name, @@ -358,18 +360,36 @@ blr_slave_request(ROUTER_INSTANCE *router, ROUTER_SLAVE *slave, GWBUF *queue) dcb_close(slave->dcb); return 1; } + else if (router->mariadb10_master_gtid && !slave->mariadb_gtid) + { + slave->state = BLRS_ERRORED; + /* Send error that stops slave replication */ + blr_send_custom_error(slave->dcb, + ++slave->seqno, + 0, + "MariaDB 10 Slave GTID is required for Slave registration.", + "HY000", + //BINLOG_FATAL_ERROR_READING); + 1597); + MXS_ERROR("%s: Slave %s: a MariaDB 10 Slave GTID request" + " is needed for Slave registration." + " Please use: CHANGE MASTER TO master_use_gtid=slave_pos.", + router->service->name, + slave->dcb->remote); + + dcb_close(slave->dcb); + return 1; + } else { /* Master and Slave version OK: continue with slave registration */ return blr_slave_register(router, slave, queue); } - case COM_BINLOG_DUMP: { char task_name[BLRM_TASK_NAME_LEN + 1] = ""; - int rc = 0; - rc = blr_slave_binlog_dump(router, slave, queue); + int rc = blr_slave_binlog_dump(router, slave, queue); if (router->send_slave_heartbeat && rc && slave->heartbeat > 0) { @@ -384,13 +404,10 @@ blr_slave_request(ROUTER_INSTANCE *router, ROUTER_SLAVE *slave, GWBUF *queue) return rc; } - case COM_STATISTICS: return blr_statistics(router, slave, queue); - case COM_PING: return blr_ping(router, slave, queue); - case COM_QUIT: MXS_DEBUG("COM_QUIT received from slave with server_id %d", slave->serverid); @@ -6908,6 +6925,62 @@ static bool blr_handle_set_stmt(ROUTER_INSTANCE *router, } return true; } + else if (strstr(word, "@@global.gtid_slave_pos") != NULL) + { + if (slave->serverid != 0) + { + MXS_ERROR("Master GTID registration can be sent only via administration connection"); + blr_slave_send_error_packet(slave, + "Master GTID registration cannot be issued by a regitrating slave.", + (unsigned int)1198, NULL); + return false; + } + if (router->master_state != BLRM_SLAVE_STOPPED) + { + MXS_ERROR("Master GTID registration needs stopped slave: issue STOP SLAVE first."); + blr_slave_send_error_packet(slave, + "Cannot use Master GTID registration with a running slave; " + "run STOP SLAVE first", + (unsigned int)1198, NULL); + return true; + } + /* If not mariadb GTID an error message will be returned */ + if (router->mariadb10_master_gtid) + { + if ((word = strtok_r(NULL, sep, &brkb)) != NULL) + { + char heading[GTID_MAX_LEN + 1]; + MXS_INFO("Binlog server requests GTID '%s' to master", + word); + + // TODO: gtid_strip_chars routine for this + strcpy(heading, word + 1); + heading[strlen(heading) - 1] = '\0'; + if (!heading[0]) + { + MXS_ERROR("Cannot request empty GTID righ now"); + blr_slave_send_error_packet(slave, + "Empty GTID not implemented righ now", + (unsigned int)1198, NULL); + return false; + } + else + { + strcpy(router->last_mariadb_gtid, heading); + blr_slave_send_ok(router, slave); + return true; + } + } + } + else + { + MXS_ERROR("Master GTID registration needs 'mariadb10_master_gtid' option to be set."); + blr_slave_send_error_packet(slave, + "Master GTID registration needs 'mariadb10_master_gtid' option to be set.", + (unsigned int)1198, NULL); + return true; + } + } else if (strstr(word, "@slave_connect_state") != NULL) { /* If not mariadb an error message will be returned */ @@ -6934,6 +7007,11 @@ static bool blr_handle_set_stmt(ROUTER_INSTANCE *router, blr_slave_send_ok(router, slave); return true; } + else + { + MXS_ERROR("GTID Master registration is not enabled"); + return false; + } } else if (strcasecmp(word, "@slave_gtid_strict_mode") == 0) {