hcq/MaxScale
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MXS-1142: maxbinlogcheck to remove transaction from binlog

Browse Source 
maxbinlogcheck with new -T $pos option can find the BEGIN of
transaction where $pos belongs to and then replace all events in
between with IGNORABLE events
This commit is contained in:
MassimilianoPinto
2017-02-16 17:33:27 +01:00
parent 755a3601ec
commit 446f65a9cf
3 changed files with 120 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
server/modules/routing/binlogrouter/blr.h
View File

@ -76,17 +76,18 @@ MXS_BEGIN_DECLS
#define BINLOG_DATA_TRUNCATED 2032
/* Binlog Encryption */
#define BINLOG_ENC_ALGO_NAME_LEN 13
#define BINLOG_FLAG_ENCRYPT 1
#define BINLOG_FLAG_DECRYPT 0
#define BINLOG_AES_MAX_KEY_LEN 32
#define BINLOG_MAX_CRYPTO_SCHEME 2
#define BINLOG_SYSTEM_DATA_CRYPTO_SCHEME 1
#define BINLOG_MAX_KEYFILE_LINE_LEN 130
#define BINLOG_ENC_ALGO_NAME_LEN 13
#define BINLOG_FLAG_ENCRYPT 1
#define BINLOG_FLAG_DECRYPT 0
#define BINLOG_AES_MAX_KEY_LEN 32
#define BINLOG_MAX_CRYPTO_SCHEME 2
#define BINLOG_SYSTEM_DATA_CRYPTO_SCHEME 1
#define BINLOG_MAX_KEYFILE_LINE_LEN 130
/* Event detail routine */
#define BLR_REPORT_CHECKSUM_FORMAT "CRC32 0x"
#define BLR_REPORT_REP_HEADER 0x02
#define BLR_REPORT_REP_HEADER 0x02
#define BLR_CHECK_ONLY 0x04
/* MariaDB GTID string len */
#define GTID_MAX_LEN 42
@ -827,7 +828,7 @@ extern int blr_send_custom_error(DCB *, int, int, char *, char *, unsigned int);
extern int blr_file_next_exists(ROUTER_INSTANCE *, ROUTER_SLAVE *);
uint32_t extract_field(uint8_t *src, int bits);
void blr_cache_read_master_data(ROUTER_INSTANCE *router);
int blr_read_events_all_events(ROUTER_INSTANCE *, const BINLOG_FILE_FIX *, int);
int blr_read_events_all_events(ROUTER_INSTANCE *, BINLOG_FILE_FIX *, int);
int blr_save_dbusers(const ROUTER_INSTANCE *router);
char *blr_get_event_description(ROUTER_INSTANCE *router, uint8_t event);
void blr_file_append(ROUTER_INSTANCE *router, char *file);

98
server/modules/routing/binlogrouter/blr_file.c
View File

@ -1304,7 +1304,7 @@ blr_file_next_exists(ROUTER_INSTANCE *router, ROUTER_SLAVE *slave)
*/
int
blr_read_events_all_events(ROUTER_INSTANCE *router,
const BINLOG_FILE_FIX *action,
BINLOG_FILE_FIX *action,
int debug)
{
unsigned long filelen = 0;
@ -1340,6 +1340,7 @@ blr_read_events_all_events(ROUTER_INSTANCE *router,
int fde_seen = 0;
int start_encryption_seen = 0;
bool fix = action ? action->fix : false;
bool replace_trx_events = false;
memset(&first_event, '\0', sizeof(first_event));
memset(&last_event, '\0', sizeof(last_event));
@ -1370,9 +1371,13 @@ blr_read_events_all_events(ROUTER_INSTANCE *router,
switch (n)
{
case 0:
MXS_DEBUG("End of binlog file [%s] at %llu.",
router->binlog_name,
pos);
if (!(debug & BLR_CHECK_ONLY))
{
MXS_DEBUG("End of binlog file [%s] at %llu.",
router->binlog_name,
pos);
}
if (n_transactions)
{
average_events = (double)((double)total_events / (double)n_transactions) * (1.0);
@ -1383,7 +1388,7 @@ blr_read_events_all_events(ROUTER_INSTANCE *router,
}
/* Report Binlog First and Last event */
if (pos > 4)
if (pos > 4 && !(debug & BLR_CHECK_ONLY))
{
if (first_event.event_type == 0)
{
@ -1396,7 +1401,7 @@ blr_read_events_all_events(ROUTER_INSTANCE *router,
}
/* Report Transaction Summary */
if (n_transactions != 0)
if (!(debug & BLR_CHECK_ONLY) && n_transactions != 0)
{
char total_label[2] = "";
char average_label[2] = "";
@ -1796,7 +1801,7 @@ blr_read_events_all_events(ROUTER_INSTANCE *router,
buf_t[strlen(buf_t) - 1] = '\0';
}
if (debug)
if (!(debug & BLR_CHECK_ONLY))
{
MXS_DEBUG("- Format Description event FDE @ %llu, size %lu, time %lu (%s)",
pos, (unsigned long)hdr.event_size, fde_event.event_time, buf_t);
@ -1837,7 +1842,7 @@ blr_read_events_all_events(ROUTER_INSTANCE *router,
/* Now remove from the calculated number of events the extra 5 bytes */
n_events -= fde_extra_bytes;
if (debug)
if (!(debug & BLR_CHECK_ONLY))
{
MXS_DEBUG(" FDE ServerVersion [%50s]", ptr + 2);
@ -1851,7 +1856,7 @@ blr_read_events_all_events(ROUTER_INSTANCE *router,
checksum = ptr + hdr.event_size - event_header_length - fde_extra_bytes;
check_alg = checksum[0];
if (debug)
if (!(debug & BLR_CHECK_ONLY))
{
MXS_DEBUG(" FDE Checksum alg desc %i, alg type %s",
check_alg,
@ -1872,8 +1877,11 @@ blr_read_events_all_events(ROUTER_INSTANCE *router,
if ((debug & BLR_REPORT_REP_HEADER))
{
char *event_desc = blr_get_event_description(router, hdr.event_type);
MXS_DEBUG("%8s==== Event Header ====\n%39sEvent time %lu\n%39sEvent Type %u (%s)\n%39sServer Id %lu\n%39sNextPos %lu\n%39sFlags %u",
" ", " ", (unsigned long)hdr.timestamp, " ", hdr.event_type,
MXS_DEBUG("%8s==== Event Header ====\n%39sEvent Pos %lu\n%39sEvent time %lu\n%39s"
"Event size %lu\n%39sEvent Type %u (%s)\n%39s"
"Server Id %lu\n%39sNextPos %lu\n%39sFlags %u",
" ", " ", (unsigned long) pos, " ", (unsigned long)hdr.timestamp, " ",
(unsigned long)hdr.event_size, " ", hdr.event_type,
event_desc ? event_desc : "NULL", " ",
(unsigned long)hdr.serverid, " ", (unsigned long)hdr.next_pos, " ", hdr.flags);
if (found_chksum)
@ -1914,7 +1922,7 @@ blr_read_events_all_events(ROUTER_INSTANCE *router,
memcpy(&new_encryption_ctx->binlog_key_version,
&ste_event.binlog_key_version, BLRM_KEY_VERSION_LENGTH);
if (debug)
if (!(debug & BLR_CHECK_ONLY))
{
/* Hex representation of nonce */
gw_bin2hex(nonce_hex, ste_event.nonce, BLRM_NONCE_LENGTH);
@ -1969,32 +1977,62 @@ blr_read_events_all_events(ROUTER_INSTANCE *router,
memcpy(file, ptr + 8, slen);
file[slen] = 0;
if (debug)
if (!(debug & BLR_CHECK_ONLY))
{
MXS_DEBUG("- Rotate event @ %llu, next file is [%s] @ %lu",
pos, file, new_pos);
}
}
/**
* Replace event at pos with an IGNORABLE EVENT
*/
/* Find and report Transaction start for event replacing only */
if (action->pos > 4 &&
action->replace_trx &&
pos == action->pos &&
pending_transaction)
{
MXS_NOTICE(">>> Position %lu belongs to a transaction started at pos %lu.",
(unsigned long)pos, (unsigned long)last_known_commit);
MXS_NOTICE("This position will be used for replacing all related events.");
/* Set Transaction start as the stating pos for events replacing */
action->pos = last_known_commit;
/* Free resources */
gwbuf_free(result);
gwbuf_free(decrypted_event);
return 0;
}
/**
* Replace one event at pos or transaction events from pos:
* All events will be replaced by IGNORABLE events
*/
if (fix &&
action->pos > 4 &&
pos == action->pos)
(pos == action->pos || replace_trx_events))
{
char *event_desc = blr_get_event_description(router, hdr.event_type);
MXS_DEBUG("*** Filling event (%s) at pos %lu with an IGNORABLE EVENT\n",
event_desc ? event_desc : "unknown",
action->pos);
if (action->replace_trx && !replace_trx_events)
{
MXS_NOTICE("=== Replacing all events of Transaction at pos %lu"
" with IGNORABLE EVENT event type",
action->pos);
}
router->last_written = action->pos;
MXS_NOTICE("=== Replace event (%s) at pos %lu with an IGNORABLE EVENT\n",
event_desc ? event_desc : "unknown",
(unsigned long)pos);
router->last_written = pos;
router->master_chksum = found_chksum;
/* Create and write Ingonrable event into binlog file at action->pos */
blr_write_special_event(router, pos, hdr.event_size, &hdr, BLRM_IGNORABLE);
/* Set replace indicator: when COMMIT is seen later, it will be set to false */
replace_trx_events = action->replace_trx ? true : false;
}
/* If MariaDB 10 compatibility:
@ -2041,8 +2079,7 @@ blr_read_events_all_events(ROUTER_INSTANCE *router,
transaction_events = 0;
event_bytes = 0;
if (debug)
if (!(debug & BLR_CHECK_ONLY))
{
MXS_DEBUG("> MariaDB 10 Transaction (GTID %u-%u-%lu)"
" starts @ pos %llu",
@ -2098,8 +2135,7 @@ blr_read_events_all_events(ROUTER_INSTANCE *router,
transaction_events = 0;
event_bytes = 0;
if (debug)
if (!(debug & BLR_CHECK_ONLY))
{
MXS_DEBUG("> Transaction starts @ pos %llu", pos);
}
@ -2113,7 +2149,7 @@ blr_read_events_all_events(ROUTER_INSTANCE *router,
{
pending_transaction = 3;
if (debug)
if (!(debug & BLR_CHECK_ONLY))
{
MXS_DEBUG(" Transaction @ pos %llu, closing @ %llu",
last_known_commit, pos);
@ -2138,7 +2174,8 @@ blr_read_events_all_events(ROUTER_INSTANCE *router,
if (pending_transaction > 0)
{
pending_transaction = 2;
if (debug)
if (!(debug & BLR_CHECK_ONLY))
{
MXS_DEBUG(" Transaction XID @ pos %llu, closing @ %llu",
last_known_commit, pos);
@ -2148,7 +2185,7 @@ blr_read_events_all_events(ROUTER_INSTANCE *router,
if (pending_transaction > 1)
{
if (debug)
if (!(debug & BLR_CHECK_ONLY))
{
MXS_DEBUG("< Transaction @ pos %llu, is now closed @ %llu. %lu events seen",
last_known_commit, pos, transaction_events);
@ -2156,6 +2193,9 @@ blr_read_events_all_events(ROUTER_INSTANCE *router,
pending_transaction = 0;
last_known_commit = pos;
/* Reset the event replacing indicator */
replace_trx_events = false;
total_events += transaction_events;
if (transaction_events > max_events)
@ -3044,7 +3084,7 @@ const char *blr_get_encryption_algorithm(int algo)
/**
* Return the encryption algorithm value
*
* @param name The alogorithm string
* @param name The algorithm string
* @return The numeric value or -1 on error
*/
int blr_check_encryption_algorithm(char *name)

64
server/modules/routing/binlogrouter/maxbinlogcheck.c
View File

@ -58,19 +58,20 @@ static int set_encryption_options(ROUTER_INSTANCE *inst, char *key_file, char *a
static struct option long_options[] =
{
{"debug", no_argument, 0, 'd'},
{"version", no_argument, 0, 'V'},
{"fix", no_argument, 0, 'f'},
{"mariadb10", no_argument, 0, 'M'},
{"header", no_argument, 0, 'H'},
{"key_file", required_argument, 0, 'K'},
{"aes_algo", required_argument, 0, 'A'},
{"replace", required_argument, 0, 'R'},
{"help", no_argument, 0, '?'},
{"debug", no_argument, 0, 'd'},
{"version", no_argument, 0, 'V'},
{"fix", no_argument, 0, 'f'},
{"mariadb10", no_argument, 0, 'M'},
{"header", no_argument, 0, 'H'},
{"key_file", required_argument, 0, 'K'},
{"aes_algo", required_argument, 0, 'A'},
{"replace-event", required_argument, 0, 'R'},
{"remove-trx", required_argument, 0, 'T'},
{"help", no_argument, 0, '?'},
{0, 0, 0, 0}
};
char *binlog_check_version = "2.2.0";
char *binlog_check_version = "2.2.1";
int
maxscale_uptime()
@ -89,7 +90,7 @@ int main(int argc, char **argv)
char c;
BINLOG_FILE_FIX binlog_file = {0, false, false};
while ((c = getopt_long(argc, argv, "dVfMHK:A:R:?", long_options, &option_index)) >= 0)
while ((c = getopt_long(argc, argv, "dVfMHK:A:R:T:?", long_options, &option_index)) >= 0)
{
switch (c)
{
@ -116,7 +117,9 @@ int main(int argc, char **argv)
aes_algo = optarg;
break;
case 'R':
case 'T':
binlog_file.pos = atol(optarg);
binlog_file.replace_trx = (c == 'T') ? true : false;
break;
case '?':
printUsage(*argv);
@ -206,7 +209,21 @@ int main(int argc, char **argv)
MXS_NOTICE("Checking %s (%s), size %lu bytes", path, inst->binlog_name, filelen);
/* read binary log */
/* Look first for a transaction that has an event at pos binlog_file.pos */
if (binlog_file.fix && binlog_file.pos && binlog_file.replace_trx)
{
/* Don't modify anything */
binlog_file.fix = false;
/* The routine call overwrites binlog_file.pos with transaction BEGIN pos */
blr_read_events_all_events(inst, &binlog_file, BLR_CHECK_ONLY);
binlog_file.fix = true;
mxs_log_flush_sync();
}
/* Now read/check/fix the binary log */
int ret = blr_read_events_all_events(inst, &binlog_file, debug_out | report_header);
mxs_log_flush_sync();
@ -219,7 +236,7 @@ int main(int argc, char **argv)
mxs_log_flush_sync();
mxs_log_finish();
return 0;
return ret;
}
/**
@ -240,16 +257,17 @@ printUsage(const char *progname)
printVersion(progname);
printf("The MaxScale binlog check utility.\n\n");
printf("Usage: %s [-f] [-M] [-d] [-V] [-H] [-K file] [-A algo] [-R pos] [<binlog file>]\n\n", progname);
printf(" -f|--fix Fix binlog file, require write permissions (truncate)\n");
printf(" -d|--debug Print debug messages\n");
printf(" -M|--mariadb10 MariaDB 10 binlog compatibility\n");
printf(" -V|--version Print version information and exit\n");
printf(" -K|--key_file AES Key file for MariaDB 10.1 binlog file decryption\n");
printf(" -A|--aes_algo AES Algorithm for MariaDB 10.1 binlog file decryption (default=AES_CBC, AES_CTR)\n");
printf(" -H|--header Print content of binlog event header\n");
printf(" -R|--replace Replace the event at pos with an IGNORABLE EVENT\n");
printf(" -?|--help Print this help text\n");
printf("Usage: %s [-f] [-M] [-d] [-V] [-H] [-K file] [-A algo] [-R pos] [-T pos] [<binlog file>]\n\n", progname);
printf(" -f|--fix Fix binlog file, require write permissions (truncate)\n");
printf(" -d|--debug Print debug messages\n");
printf(" -M|--mariadb10 MariaDB 10 binlog compatibility\n");
printf(" -V|--version Print version information and exit\n");
printf(" -K|--key_file AES Key file for MariaDB 10.1 binlog file decryption\n");
printf(" -A|--aes_algo AES Algorithm for MariaDB 10.1 binlog file decryption (default=AES_CBC, AES_CTR)\n");
printf(" -H|--header Print content of binlog event header\n");
printf(" -R|--replace-event Replace the event at pos with an IGNORABLE event\n");
printf(" -T|--remove-trx Replace all events in the transaction the specified pos belongs to, with IGNORABLE events\n");
printf(" -?|--help Print this help text\n");
}
/**