From d5de149e32ee4e9ff71462abff9526d940909b16 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Wed, 2 Aug 2017 15:32:14 +0200 Subject: [PATCH] Update GTID repository while receiving ROTATE events and check the current file exists. Update GTID repository while receiving ROTATE events and check the current file exists. --- .../modules/routing/binlogrouter/blr_file.c | 107 +++++++++++++++--- .../modules/routing/binlogrouter/blr_master.c | 27 ++++- .../modules/routing/binlogrouter/blr_slave.c | 15 ++- 3 files changed, 125 insertions(+), 24 deletions(-) diff --git a/server/modules/routing/binlogrouter/blr_file.c b/server/modules/routing/binlogrouter/blr_file.c index b1c6fc58f..2ab742a69 100644 --- a/server/modules/routing/binlogrouter/blr_file.c +++ b/server/modules/routing/binlogrouter/blr_file.c @@ -238,6 +238,8 @@ bool blr_compare_binlogs(ROUTER_INSTANCE *router, const char *r_file, const char *s_file); +void blr_file_update_gtid(ROUTER_INSTANCE *router); + /** * MariaDB 10.1.7 Start Encryption event content * @@ -573,21 +575,7 @@ blr_file_create(ROUTER_INSTANCE *router, char *orig_file) if (router->mariadb10_compat && router->mariadb10_gtid) { - MARIADB_GTID_ELEMS gtid_elms = {}; - // Add GTID domain - gtid_elms.domain_id = router->mariadb10_gtid_domain; - // router->orig_masterid keeps the original ID - gtid_elms.server_id = router->orig_masterid; - // Pos 4 for start/end_pos - router->pending_transaction.end_pos = 4; - router->pending_transaction.start_pos = 4; - // Update all the gtid_elms - memcpy(&router->pending_transaction.gtid_elms, - >id_elms, - sizeof(MARIADB_GTID_ELEMS)); - - /* Save GTID into repo */ - blr_save_mariadb_gtid(router); + blr_file_update_gtid(router); } } else @@ -4487,3 +4475,92 @@ bool blr_is_current_binlog(ROUTER_INSTANCE *router, router->binlog_name, slave->binlogfile); } + +/** + * Check whether the current binlog file exists. + * + * The file could have been manually removed by mistake. + * The check is done when ROTATE event (real or fake) is seen by + * blr_rotate_event(). + * + * @param router The router instance + * @return True if file exists, false otherwise. + * + */ +bool blr_file_exists(ROUTER_INSTANCE *router) +{ + bool ret = true; + char path[PATH_MAX + 1] = ""; + strcpy(path, router->binlogdir); + strcat(path, "/"); + + /** + * Create file prefix using domain and server_id prefix + */ + if (router->storage_type == BLR_BINLOG_STORAGE_TREE) + { + char prefix[BINLOG_FILE_EXTRA_INFO]; + // Add prefix + sprintf(prefix, + "%" PRIu32 "/%" PRIu32 "/", + router->mariadb10_gtid_domain, + router->orig_masterid); + strcat(path, prefix); + } + + // Set final file name full path + strcat(path, router->binlog_name); + + // Check file + if (access(path, F_OK) == -1 && errno == ENOENT) + { + // No file found + MXS_WARNING("%s: ROTATE_EVENT, missing binlog file %s ", + router->service->name, + path); + ret = false; + } + + return ret; +} + +/** + * Add/Update binlog file details into GTID mapd db: + * + * binlog file name + * pos = 4 + * server_id = router->orig_masterid + * sequence = 0 + * + * @param router The router instance + */ +void blr_file_update_gtid(ROUTER_INSTANCE *router) +{ + MARIADB_GTID_ELEMS gtid_elms; + // Add GTID domain + gtid_elms.domain_id = router->mariadb10_gtid_domain; + //router->orig_masterid keeps the original ID + gtid_elms.server_id = router->orig_masterid; + // Set GTID sequence to 0 + gtid_elms.seq_no = 0; + // Pos 4 for start/end_pos + router->pending_transaction.end_pos = 4; + router->pending_transaction.start_pos = 4; + // Update all the gtid_elms + memcpy(&router->pending_transaction.gtid_elms, + >id_elms, + sizeof(MARIADB_GTID_ELEMS)); + + /** + * Save GTID into repo + * + * If router->orig_masterid is not set yet + * don't update the db + * This happens when mariadb10_master_gtid id Off + * and a new file has been created by blr_file_init() + */ + if (gtid_elms.server_id > 0) + { + blr_save_mariadb_gtid(router); + } +} diff --git a/server/modules/routing/binlogrouter/blr_master.c b/server/modules/routing/binlogrouter/blr_master.c index 196934deb..6d895bb0a 100644 --- a/server/modules/routing/binlogrouter/blr_master.c +++ b/server/modules/routing/binlogrouter/blr_master.c @@ -147,6 +147,8 @@ static bool blr_handle_missing_files(ROUTER_INSTANCE *router, static void worker_cb_start_master(int worker_id, void* data); static void blr_start_master_in_main(void* data); +extern bool blr_file_exists(ROUTER_INSTANCE *router); +extern void blr_file_update_gtid(ROUTER_INSTANCE *router); static int keepalive = 1; @@ -1492,7 +1494,9 @@ blr_rotate_event(ROUTER_INSTANCE *router, uint8_t *ptr, REP_HEADER *hdr) int rotated = 1; int remove_encrytion_ctx = 0; - if (strncmp(router->binlog_name, file, slen) != 0) + /* Different file name in rotate event or missing binlog file */ + if ((strncmp(router->binlog_name, file, slen) != 0) || + !blr_file_exists(router)) { remove_encrytion_ctx = 1; router->stats.n_rotates++; @@ -1501,6 +1505,23 @@ blr_rotate_event(ROUTER_INSTANCE *router, uint8_t *ptr, REP_HEADER *hdr) rotated = 0; } } + else + { + /** + * ROTATE_EVENT reports a binlog file which is the same + * as router->binlog_name. + * + * If mariadb10_gtid is On, let's Add/Update into GTID repo: + * this allows SHOW BINARY LOGS to list all files + * including the ones without GTID events. + */ + + if (router->mariadb10_compat && + router->mariadb10_gtid) + { + blr_file_update_gtid(router); + } + } spinlock_acquire(&router->binlog_lock); router->rotating = 0; @@ -3145,8 +3166,8 @@ static void blr_register_mariadb_gtid_request(ROUTER_INSTANCE *router, * False otherwise. */ static bool blr_handle_fake_rotate(ROUTER_INSTANCE *router, - REP_HEADER *hdr, - uint8_t *ptr) + REP_HEADER *hdr, + uint8_t *ptr) { ss_dassert(hdr->event_type == ROTATE_EVENT); diff --git a/server/modules/routing/binlogrouter/blr_slave.c b/server/modules/routing/binlogrouter/blr_slave.c index 8d04e41b4..4c5de5083 100644 --- a/server/modules/routing/binlogrouter/blr_slave.c +++ b/server/modules/routing/binlogrouter/blr_slave.c @@ -342,6 +342,7 @@ extern bool blr_compare_binlogs(ROUTER_INSTANCE *router, MARIADB_GTID_INFO *slave, const char *r_file, const char *s_file); + /** * Process a request packet from the slave server. * @@ -679,7 +680,7 @@ blr_slave_query(ROUTER_INSTANCE *router, ROUTER_SLAVE *slave, GWBUF *queue) { if (brkb && strlen(brkb) && blr_handle_complex_select(router, - slave, + slave, word, brkb)) { @@ -688,8 +689,8 @@ blr_slave_query(ROUTER_INSTANCE *router, ROUTER_SLAVE *slave, GWBUF *queue) } if (blr_handle_simple_select_stmt(router, - slave, - word)) + slave, + word)) { MXS_FREE(query_text); return 1; @@ -8221,10 +8222,12 @@ blr_show_binary_logs(ROUTER_INSTANCE *router, seqno = result.seq_no; /** - * Check whether the last file is the current binlog file. - * If not then add the new row. + * Check whether the last file is the current binlog file, + * GTID repo might also contain no data at all. + * + * Add the new row if needed. */ - if (strcmp(current_file, result.last_file) != 0) + if (!result.last_file || strcmp(current_file, result.last_file) != 0) { char pos[40]; // Buffer for a 64-bit integer. GWBUF *pkt;