From 0c552bab29bd968fdc44d800f7da7cf5522d2d70 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 27 Aug 2015 14:22:58 +0300 Subject: [PATCH 01/38] Fix to MXS-35: https://mariadb.atlassian.net/browse/MXS-35 The parent process now correctly returns the value from the child process. --- server/core/gateway.c | 57 +++++++++++++++++++++++++++++++++++++++++- server/core/gw_utils.c | 5 ++-- server/include/gw.h | 2 +- 3 files changed, 60 insertions(+), 4 deletions(-) diff --git a/server/core/gateway.c b/server/core/gateway.c index 2eceea4b2..a79fddfbf 100644 --- a/server/core/gateway.c +++ b/server/core/gateway.c @@ -1056,6 +1056,9 @@ int main(int argc, char **argv) int n_services; int eno = 0; /*< local variable for errno */ int opt; + int daemon_pipe[2]; + bool parent_process; + int child_status; void** threads = NULL; /*< thread list */ char mysql_home[PATH_MAX+1]; char datadir_arg[10+PATH_MAX+1]; /*< '--datadir=' + PATH_MAX */ @@ -1301,6 +1304,13 @@ int main(int argc, char **argv) } else { + if(pipe(daemon_pipe) == -1) + { + fprintf(stderr,"Error: Failed to create pipe for inter-process communication: %d %s",errno,strerror(errno)); + rc = MAXSCALE_INTERNALERROR; + goto return_main; + } + /*< * Maxscale must be daemonized before opening files, initializing * embedded MariaDB and in general, as early as possible. @@ -1433,7 +1443,37 @@ int main(int argc, char **argv) rc = MAXSCALE_INTERNALERROR; goto return_main; } - gw_daemonize(); + + /** Daemonize the process and wait for the child process to notify + * the parent process of its exit status. */ + parent_process = gw_daemonize(); + + if(parent_process) + { + int nread = read(daemon_pipe[0],(void*)&child_status,sizeof(int)); + close(daemon_pipe[1]); + close(daemon_pipe[0]); + + if(nread == -1) + { + char* logerr = "Failed to read data from child process pipe."; + print_log_n_stderr(true, true, logerr, logerr, errno); + exit(MAXSCALE_INTERNALERROR); + } + else if(nread == 0) + { + /** Child process has exited or closed write pipe */ + char* logerr = "No data read from child process pipe."; + print_log_n_stderr(true, true, logerr, logerr, 0); + exit(MAXSCALE_INTERNALERROR); + } + + exit(child_status); + } + + /** This is the child process and we can close the read end of + * the pipe. */ + close(daemon_pipe[0]); } /*< * Set signal handlers for SIGHUP, SIGTERM, SIGINT and critical signals like SIGSEGV. @@ -1907,6 +1947,13 @@ int main(int argc, char **argv) CRYPTO_set_id_callback(pthread_self); #endif + /** + * Successful start, notify the parent process that it can exit. + */ + ss_dassert(rc == MAXSCALE_SHUTDOWN); + write(daemon_pipe[1],&rc,sizeof(int)); + close(daemon_pipe[1]); + MaxScaleStarted = time(0); /*< * Serve clients. @@ -1945,6 +1992,14 @@ int main(int argc, char **argv) unlink_pidfile(); return_main: + + if(daemon_mode && rc != MAXSCALE_SHUTDOWN) + { + /** Notify the parent process that an error has occurred */ + write(daemon_pipe[1],&rc,sizeof(int)); + close(daemon_pipe[1]); + } + if (threads) free(threads); if (cnf_file_path) diff --git a/server/core/gw_utils.c b/server/core/gw_utils.c index 5d4fb5ed2..1a412370c 100644 --- a/server/core/gw_utils.c +++ b/server/core/gw_utils.c @@ -141,7 +141,7 @@ setipaddress(struct in_addr *a, char *p) { * Daemonize the process by forking and putting the process into the * background. */ -void gw_daemonize(void) { +bool gw_daemonize(void) { pid_t pid; pid = fork(); @@ -153,13 +153,14 @@ void gw_daemonize(void) { if (pid != 0) { /* exit from main */ - exit(0); + return true; } if (setsid() < 0) { fprintf(stderr, "setsid() error %s\n", strerror(errno)); exit(1); } + return false; } /** diff --git a/server/include/gw.h b/server/include/gw.h index b51bb0e2a..4f61d4512 100644 --- a/server/include/gw.h +++ b/server/include/gw.h @@ -74,7 +74,7 @@ #include "dcb.h" -void gw_daemonize(void); +bool gw_daemonize(void); int do_read_dcb(DCB *dcb); void MySQLListener(int epfd, char *config_bind); int MySQLAccept(DCB *listener); From 7d00a3670df9e6a9f1e18ccdf8c589d5e530a2c4 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Thu, 10 Sep 2015 14:37:49 +0200 Subject: [PATCH 02/38] Addition of maxbinlogcheck, a new utility in bin Addition of maxbinlogcheck, a new utility in bin --- Documentation/Reference/MaxBinlogCheck.md | 330 +++++++++ server/modules/include/blr.h | 8 +- server/modules/routing/binlog/CMakeLists.txt | 6 + server/modules/routing/binlog/blr_file.c | 695 ++++++++++++++++++ .../modules/routing/binlog/maxbinlogcheck.c | 249 +++++++ 5 files changed, 1287 insertions(+), 1 deletion(-) create mode 100644 Documentation/Reference/MaxBinlogCheck.md create mode 100644 server/modules/routing/binlog/maxbinlogcheck.c diff --git a/Documentation/Reference/MaxBinlogCheck.md b/Documentation/Reference/MaxBinlogCheck.md new file mode 100644 index 000000000..3f1fc4c57 --- /dev/null +++ b/Documentation/Reference/MaxBinlogCheck.md @@ -0,0 +1,330 @@ +# Maxbinlogcheck + +# The MySQL/MariaDB binlog check utility + +Massimiliano Pinto + +Last Updated: 08th September 2015 + +# Overview + +Maxbinlogcheck is a command line utility for checking binlogfiles downloaded by MaxScale binlog router or the MySQL/MariaDB binlog files stored in a database server acting as a master in a replication environment. +It checks the binlog file against any corruption and incomplete transaction stored and reports a transaction summary after reading all the events. +It may optionally truncate binlog file. + +Maxbinlogcheck supports + +* MariaDB 5.5 and MySQL 5.6 + +* MariaDB 10.0 with a command line option + +# Running maxbinlogcheck +``` +# /usr/local/bin/maxbinlogcheck /path_to_file/bin.000002 +``` + +# Command Line Switches + +The maxbinlogcheck command accepts a number of switches + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
SwitchLong OptionDescription
-f--fixIf the option is set the binlog file will be truncated at last safe transaction pos in case of any error
-M--mariadb10Check the current binlog against MariaDB 10.0.x events
-d--debugSet the debug mode. If set the FD Events, Rotate events and opening/closing transactions are displayed.
-?--helpPrint usage information regarding maxbinlogcheck
-V--versionPrint the maxbinlogcheck version information
+ +## Example without debug: + +1) No transactions + +``` +[root@maxscale-02 build]# /usr/local/bin/maxbinlogcheck /servers/binlogs/new-trx/mar-bin.000002 +2015-09-08 09:38:03 maxbinlogcheck 1.0.0 +2015-09-08 09:38:03 Checking /servers/binlogs/new-trx/mar-bin.000002 (mar-bin.000002), size 290 bytes +2015-09-08 09:38:03 Check retcode: 0, Binlog Pos = 290 +``` + +2) With complete transactions + +``` +[root@maxscale-02 build]# /usr/local/bin/maxbinlogcheck /servers/binlogs/new-trx/mar-bin.000004 +2015-09-08 09:38:35 maxbinlogcheck 1.0.0 +2015-09-08 09:38:35 Checking /servers/binlogs/new-trx/mar-bin.000004 (mar-bin.000004), size 38738629 bytes +2015-09-08 09:38:36 Transaction Summary: + Description Total Average Max + No. of Transactions 21713 + No. of Events 186599 8.6 5082 + No. of Bytes 36M 1k 5M +2015-09-08 09:38:36 Check retcode: 0, Binlog Pos = 38738629 +``` + +## Example with debug: + +1) one complete transaction + +``` +[root@maxscale-02 build]# /usr/local/bin/maxbinlogcheck /servers/binlogs/new-trx/mar-bin.000001 -d +2015-09-08 09:36:49 maxbinlogcheck 1.0.0 +2015-09-08 09:36:49 Checking /servers/binlogs/new-trx/mar-bin.000001 (mar-bin.000001), size 590760698 bytes +2015-09-08 09:36:49 - Format Description event FDE @ 4, size 241 +2015-09-08 09:36:49 FDE ServerVersion [ 5.5.35-MariaDB-log] +2015-09-08 09:36:49 FDE Header EventLength 19, N. of supported MySQL/MariaDB events 160 +2015-09-08 09:36:49 FDE Checksum alg desc 1, alg type BINLOG_CHECKSUM_ALG_CRC32 +2015-09-08 09:36:49 > Transaction starts @ pos 572 +2015-09-08 09:36:49 Transaction XID @ pos 572, closing @ 590760644 +2015-09-08 09:36:49 < Transaction @ pos 572, is now closed @ 590760644. 18001 events seen +2015-09-08 09:36:49 End of binlog file [mar-bin.000001] at 590760698. +2015-09-08 09:36:49 Transaction Summary: + Description Total Average Max + No. of Transactions 1 + No. of Events 18001 18001.0 18001 + No. of Bytes 563M 563M 563M +2015-09-08 09:36:49 Check retcode: 0, Binlog Pos = 590760698 +``` + +2) some transactions + +``` +[root@maxscale-02 build]# /usr/local/bin/maxbinlogcheck /servers/binlogs/new-trx/mar-bin.000004 -d +2015-09-08 10:19:35 maxbinlogcheck 1.0.0 +2015-09-08 10:19:35 Checking /servers/binlogs/new-trx/mar-bin.000004 (mar-bin.000004), size 38738629 bytes +2015-09-08 10:19:35 - Format Description event FDE @ 4, size 241 +2015-09-08 10:19:35 FDE ServerVersion [ 5.5.35-MariaDB-log] +2015-09-08 10:19:35 FDE Header EventLength 19, N. of supported MySQL/MariaDB events 160 +2015-09-08 10:19:35 FDE Checksum alg desc 1, alg type BINLOG_CHECKSUM_ALG_CRC32 +2015-09-08 10:19:35 > Transaction starts @ pos 971 +2015-09-08 10:19:35 Transaction XID @ pos 971, closing @ 1128 +2015-09-08 10:19:35 < Transaction @ pos 971, is now closed @ 1128. 3 events seen +... +2015-09-08 10:19:51 > Transaction starts @ pos 33440561 +2015-09-08 10:19:51 Transaction XID @ pos 33440561, closing @ 33440763 +2015-09-08 10:19:51 < Transaction @ pos 33440561, is now closed @ 33440763. 3 events seen +2015-09-08 10:19:51 > Transaction starts @ pos 33440794 +2015-09-08 10:19:51 Transaction XID @ pos 33440794, closing @ 38738553 +2015-09-08 10:19:51 < Transaction @ pos 33440794, is now closed @ 38738553. 5082 events seen +2015-09-08 10:19:51 - Rotate event @ 38738584, next file is [mar-bin.000005] @ 4 +2015-09-08 10:19:51 End of binlog file [mar-bin.000004] at 38738629. +2015-09-08 10:19:51 Transaction Summary: + Description Total Average Max + No. of Transactions 21713 + No. of Events 186599 8.6 5082 + No. of Bytes 36M 1k 5M +2015-09-08 10:19:51 Check retcode: 0, Binlog Pos = 38738629 +``` + +3) No transactions + +``` +2015-09-08 09:41:02 Check retcode: 0, Binlog Pos = 290 +[root@maxscale-02 build]# /usr/local/bin/maxbinlogcheck /servers/binlogs/new-trx/mar-bin.000002 -d +2015-09-08 09:41:08 maxbinlogcheck 1.0.0 +2015-09-08 09:41:08 Checking /servers/binlogs/new-trx/mar-bin.000002 (mar-bin.000002), size 290 bytes +2015-09-08 09:41:08 - Format Description event FDE @ 4, size 241 +2015-09-08 09:41:08 FDE ServerVersion [ 5.5.35-MariaDB-log] +2015-09-08 09:41:08 FDE Header EventLength 19, N. of supported MySQL/MariaDB events 160 +2015-09-08 09:41:08 FDE Checksum alg desc 1, alg type BINLOG_CHECKSUM_ALG_CRC32 +2015-09-08 09:41:08 - Rotate event @ 245, next file is [mar-bin.000003] @ 4 +2015-09-08 09:41:08 End of binlog file [mar-bin.000002] at 290. +2015-09-08 09:41:08 Check retcode: 0, Binlog Pos = 290 +``` + +## Fixing a corrupted binlog file + +This file is corrupted, as reported by the utility: + +``` +[root@maxscale-02 build]# /usr/local/bin/maxbinlogcheck /servers/binlogs/new-trx/bin.000002 +2015-09-08 10:03:16 maxbinlogcheck 1.0.0 +2015-09-08 10:03:16 Checking /servers/binlogs/new-trx/bin.000002 (bin.000002), size 109498 bytes +2015-09-08 10:03:16 Event size error: size 0 at 290. +2015-09-08 10:03:16 warning : an error has been found. Setting safe pos to 245, current pos 290 +2015-09-08 10:03:16 Check retcode: 1, Binlog Pos = 245 +``` + +The suggested safe pos is 245 + +Use -f option for fix with debug: + +``` +[root@maxscale-02 build]# /usr/local/bin/maxbinlogcheck /servers/binlogs/new-trx/bin.000002 -d -f +2015-09-08 09:56:52 maxbinlogcheck 1.0.0 +2015-09-08 09:56:52 Checking /servers/binlogs/new-trx/bin.000002 (bin.000002), size 109498 bytes +2015-09-08 09:56:52 - Format Description event FDE @ 4, size 241 +2015-09-08 09:56:52 FDE ServerVersion [ 5.5.35-MariaDB-log] +2015-09-08 09:56:52 FDE Header EventLength 19, N. of supported MySQL/MariaDB events 160 +2015-09-08 09:56:52 FDE Checksum alg desc 1, alg type BINLOG_CHECKSUM_ALG_CRC32 +2015-09-08 09:56:52 - Rotate event @ 245, next file is [mar-bin.000003] @ 4 +2015-09-08 09:56:52 Event size error: size 0 at 290. +2015-09-08 09:56:52 warning : an error has been found. Setting safe pos to 245, current pos 290 +2015-09-09 09:56:52 Binlog file bin.000002 has been truncated at 245 +2015-09-08 09:56:52 Check retcode: 1, Binlog Pos = 245 +``` + +Check it again, last pos will be 245 and no errors will be reported: + +``` +[root@maxscale-02 build]# /usr/local/bin/maxbinlogcheck /servers/binlogs/new-trx/bin.000002 -d +2015-09-08 09:56:56 maxbinlogcheck 1.0.0 +2015-09-08 09:56:56 Checking /servers/binlogs/new-trx/bin.000002 (bin.000002), size 245 bytes +2015-09-08 09:56:56 - Format Description event FDE @ 4, size 241 +2015-09-08 09:56:56 FDE ServerVersion [ 5.5.35-MariaDB-log] +2015-09-08 09:56:56 FDE Header EventLength 19, N. of supported MySQL/MariaDB events 160 +2015-09-08 09:56:56 FDE Checksum alg desc 1, alg type BINLOG_CHECKSUM_ALG_CRC32 +2015-09-08 09:56:56 End of binlog file [bin.000002] at 245. +2015-09-08 09:56:56 Check retcode: 0, Binlog Pos = 245 +``` + +## Detection of an incomplete big transaction + +``` +[root@maxscale-02 build]# /usr/local/bin/maxbinlogcheck /servers/binlogs/new-trx/mar-bin.000003 +2015-09-08 10:10:21 maxbinlogcheck 1.0.0 +2015-09-08 10:10:21 Checking /servers/binlogs/new-trx/mar-bin.000003 (mar-bin.000003), size 16476284 bytes +2015-09-08 10:10:21 Warning : pending transaction has been found. Setting safe pos to 572, current pos 16476284 +2015-09-08 10:10:21 Check retcode: 0, Binlog Pos = 572 +``` + +with debug option: + +``` +[root@maxscale-02 build]# /usr/local/bin/maxbinlogcheck /servers/binlogs/new-trx/mar-bin.000003 -d +2015-09-08 10:11:08 maxbinlogcheck 1.0.0 +2015-09-08 10:11:08 Checking /servers/binlogs/new-trx/mar-bin.000003 (mar-bin.000003), size 16476284 bytes +2015-09-08 10:11:08 - Format Description event FDE @ 4, size 241 +2015-09-08 10:11:08 FDE ServerVersion [ 5.5.35-MariaDB-log] +2015-09-08 10:11:08 FDE Header EventLength 19, N. of supported MySQL/MariaDB events 160 +2015-09-08 10:11:08 FDE Checksum alg desc 1, alg type BINLOG_CHECKSUM_ALG_CRC32 +2015-09-08 10:11:08 > Transaction starts @ pos 572 +2015-09-08 10:11:08 Warning : pending transaction has been found. Setting safe pos to 572, current pos 16476284 +2015-09-08 10:11:08 End of binlog file [mar-bin.000003] at 16476284. +2015-09-08 10:11:08 Check retcode: 0, Binlog Pos = 572 +``` + +Retcode is 0 as the transaction may proceed over time, example: + +Another check ... + +``` +[root@maxscale-02 build]# /usr/local/bin/maxbinlogcheck /servers/binlogs/new-trx/mar-bin.000003 -d +2015-09-08 10:17:13 maxbinlogcheck 1.0.0 +2015-09-08 10:17:13 Checking /servers/binlogs/new-trx/mar-bin.000003 (mar-bin.000003), size 569296364 bytes +2015-09-08 10:17:13 - Format Description event FDE @ 4, size 241 +2015-09-08 10:17:13 FDE ServerVersion [ 5.5.35-MariaDB-log] +2015-09-08 10:17:13 FDE Header EventLength 19, N. of supported MySQL/MariaDB events 160 +2015-09-08 10:17:13 FDE Checksum alg desc 1, alg type BINLOG_CHECKSUM_ALG_CRC32 +2015-09-08 10:17:13 > Transaction starts @ pos 572 +2015-09-08 10:17:14 End of binlog file [mar-bin.000003] at 577567062. +2015-09-08 10:17:14 Check retcode: 0, Binlog Pos = 572 +``` + +And finally big transaction is now done. + +``` +[root@maxscale-02 build]# /usr/local/bin/maxbinlogcheck /servers/binlogs/new-trx/mar-bin.000003 -d +2015-09-08 10:17:15 maxbinlogcheck 1.0.0 +2015-09-08 10:17:15 Checking /servers/binlogs/new-trx/mar-bin.000003 (mar-bin.000003), size 590760698 bytes +2015-09-08 10:17:15 - Format Description event FDE @ 4, size 241 +2015-09-08 10:17:15 FDE ServerVersion [ 5.5.35-MariaDB-log] +2015-09-08 10:17:15 FDE Header EventLength 19, N. of supported MySQL/MariaDB events 160 +2015-09-08 10:17:15 FDE Checksum alg desc 1, alg type BINLOG_CHECKSUM_ALG_CRC32 +2015-09-08 10:17:15 > Transaction starts @ pos 572 +2015-09-08 10:17:16 Transaction XID @ pos 572, closing @ 590760644 +2015-09-08 10:17:16 < Transaction @ pos 572, is now closed @ 590760644. 18001 events seen +2015-09-08 10:17:16 End of binlog file [mar-bin.000003] at 590760698. +2015-09-08 10:17:16 Transaction Summary: + Description Total Average Max + No. of Transactions 1 + No. of Events 18001 18001.0 18001 + No. of Bytes 563M 563M 563M +2015-09-08 10:17:16 Check retcode: 0, Binlog Pos = 590760698 +``` + +**Note** +with current maxbinlogcheck it's not possible to fix a binlog with incomplete transaction and no other errors + +If that is really desired it will be possible with UNIX command line: + +``` +# truncate /servers/binlogs/new-trx/mar-bin.000003 --size=572 +``` + +In case of an error and incomplete transaction, the fix will work + +``` +[root@maxscale-02 build]# /usr/local/bin/maxbinlogcheck /servers/binlogs/new-trx/mar-bin.000003 -d -f +2015-09-08 10:35:57 maxbinlogcheck 1.0.0 +2015-09-08 10:35:57 Checking /servers/binlogs/new-trx/mar-bin.000003 (mar-bin.000003), size 282580902 bytes +2015-09-08 10:35:57 - Format Description event FDE @ 4, size 241 +2015-09-08 10:35:57 FDE ServerVersion [ 5.5.35-MariaDB-log] +2015-09-08 10:35:57 FDE Header EventLength 19, N. of supported MySQL/MariaDB events 160 +2015-09-08 10:35:57 FDE Checksum alg desc 1, alg type BINLOG_CHECKSUM_ALG_CRC32 +2015-09-08 10:35:57 > Transaction starts @ pos 572 +2015-09-08 10:35:57 Short read when reading the event at 304898502 in mar-bin.000003. Expected 65563 bytes got 15911 bytes. +2015-09-08 10:35:57 warning : an error has been found. Setting safe pos to 572, current pos 304898502 +2015-09-09 10:35:57 Binlog file bin.000003 has been truncated at 572 +2015-09-08 10:35:57 Check retcode: 1, Binlog Pos = 572 +``` + +Check result: + +``` +[root@maxscale-02 build]# /usr/local/bin/maxbinlogcheck /servers/binlogs/new-trx/mar-bin.000003 -d +2015-09-08 10:54:17 maxbinlogcheck 1.0.0 +2015-09-08 10:54:17 Checking /servers/binlogs/new-trx/mar-bin.000003 (mar-bin.000003), size 572 bytes +2015-09-08 10:54:17 - Format Description event FDE @ 4, size 241 +2015-09-08 10:54:17 FDE ServerVersion [ 5.5.35-MariaDB-log] +2015-09-08 10:54:17 FDE Header EventLength 19, N. of supported MySQL/MariaDB events 160 +2015-09-08 10:54:17 FDE Checksum alg desc 1, alg type BINLOG_CHECKSUM_ALG_CRC32 +2015-09-08 10:54:17 End of binlog file [mar-bin.000003] at 572. +2015-09-08 10:54:17 Check retcode: 0, Binlog Pos = 572 +``` + +### MariaDB 10 binlog check + +``` +[root@maxscale-02 build]# /usr/local/bin/maxbinlogcheck -M -d /mariadb-10.0.11/data/mysql-bin.000008 +2015-09-08 12:49:18 maxbinlogcheck 1.0.0 +2015-09-08 12:49:18 Checking /mariadb-10.0.11/data/mysql-bin.000008 (mysql-bin.000008), size 1215327 bytes +2015-09-08 12:49:18 - Format Description event FDE @ 4, size 244 +2015-09-08 12:49:18 FDE ServerVersion [ 10.0.11-MariaDB-log] +2015-09-08 12:49:18 FDE Header EventLength 19, N. of supported MySQL/MariaDB events 163 +2015-09-08 12:49:18 FDE Checksum alg desc 0, alg type NONE or UNDEF +2015-09-08 12:49:18 > MariaDB 10 Transaction (GTID 0-29-60) starts @ pos 802 +2015-09-08 12:49:18 Transaction XID @ pos 802, closing @ 1214943 +2015-09-08 12:49:18 < Transaction @ pos 802, is now closed @ 1214943. 76 events seen +2015-09-08 12:49:18 End of binlog file [mysql-bin.000008] at 1215327. +2015-09-08 12:49:18 Transaction Summary: + Description Total Average Max + No. of Transactions 1 + No. of Events 76 76.0 76 + No. of Bytes 1.2M 1.2M 1.2M +2015-09-08 12:49:18 Check retcode: 0, Binlog Pos = 1215327 +``` + diff --git a/server/modules/include/blr.h b/server/modules/include/blr.h index 7df2c395f..03a87a698 100644 --- a/server/modules/include/blr.h +++ b/server/modules/include/blr.h @@ -15,7 +15,7 @@ * this program; if not, write to the Free Software Foundation, Inc., 51 * Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. * - * Copyright MariaDB Corporation Ab 2014 + * Copyright MariaDB Corporation Ab 2014-2015 */ /** @@ -142,6 +142,9 @@ #define BLR_MASTER_BACKOFF_TIME 10 #define BLR_MAX_BACKOFF 60 +/* string len for strerror_r message */ +#define BLRM_STRERROR_R_MSG_SIZE 128 + /** * Some useful macros for examining the MySQL Response packets */ @@ -340,9 +343,12 @@ typedef struct router_instance { MASTER_RESPONSES saved_master; /*< Saved master responses */ char *binlogdir; /*< The directory with the binlog files */ SPINLOCK binlog_lock; /*< Lock to control update of the binlog position */ + int pending_transaction; /*< Pending transaction */ char binlog_name[BINLOG_FNAMELEN+1]; /*< Name of the current binlog file */ uint64_t binlog_position; + /*< Current binlog position, safe pos */ + uint64_t current_pos; /*< Current binlog position */ int binlog_fd; /*< File descriptor of the binlog * file being written diff --git a/server/modules/routing/binlog/CMakeLists.txt b/server/modules/routing/binlog/CMakeLists.txt index 9a0c245de..1e65eb504 100644 --- a/server/modules/routing/binlog/CMakeLists.txt +++ b/server/modules/routing/binlog/CMakeLists.txt @@ -2,3 +2,9 @@ add_library(binlogrouter SHARED blr.c blr_master.c blr_cache.c blr_slave.c blr_f set_target_properties(binlogrouter PROPERTIES INSTALL_RPATH ${CMAKE_INSTALL_RPATH}:${MAXSCALE_LIBDIR}) target_link_libraries(binlogrouter ssl pthread log_manager) install(TARGETS binlogrouter DESTINATION ${MAXSCALE_LIBDIR}) + +add_executable(maxbinlogcheck maxbinlogcheck.c blr_file.c blr_cache.c blr_master.c blr_slave.c blr.c ${CMAKE_SOURCE_DIR}/server/core/service.c ${CMAKE_SOURCE_DIR}/server/core/spinlock.c ${CMAKE_SOURCE_DIR}/server/core/buffer.c ${CMAKE_SOURCE_DIR}/server/core/atomic.c ${CMAKE_SOURCE_DIR}/server/core/hint.c ${CMAKE_SOURCE_DIR}/server/core/gwdirs.c ${CMAKE_SOURCE_DIR}/server/core/server.c ${CMAKE_SOURCE_DIR}/server/core/dcb.c ${CMAKE_SOURCE_DIR}/server/core/users.c ${CMAKE_SOURCE_DIR}/server/core/dbusers.c ${CMAKE_SOURCE_DIR}/server/core/utils.c ${CMAKE_SOURCE_DIR}/server/core/hashtable.c ${CMAKE_SOURCE_DIR}/server/core/poll.c ${CMAKE_SOURCE_DIR}/server/core/gwbitmask.c ${CMAKE_SOURCE_DIR}/server/core/config.c ${CMAKE_SOURCE_DIR}/server/core/session.c ${CMAKE_SOURCE_DIR}/server/core/housekeeper.c ${CMAKE_SOURCE_DIR}/server/core/filter.c ${CMAKE_SOURCE_DIR}/server/core/resultset.c ${CMAKE_SOURCE_DIR}/server/core/load_utils.c ${CMAKE_SOURCE_DIR}/server/core/monitor.c ${CMAKE_SOURCE_DIR}/server/core/gw_utils.c ${CMAKE_SOURCE_DIR}/server/core/thread.c ${CMAKE_SOURCE_DIR}/server/core/secrets.c) + +target_link_libraries(maxbinlogcheck utils ssl pthread log_manager ${PCRE_LINK_FLAGS} aio rt crypt dl crypto inih z m stdc++ ${EMBEDDED_LIB} ${CURL_LIBRARIES}) + +install(TARGETS maxbinlogcheck DESTINATION bin) diff --git a/server/modules/routing/binlog/blr_file.c b/server/modules/routing/binlog/blr_file.c index a6aa50bdf..f6daf54d0 100644 --- a/server/modules/routing/binlog/blr_file.c +++ b/server/modules/routing/binlog/blr_file.c @@ -26,6 +26,8 @@ * Date Who Description * 14/04/2014 Mark Riddoch Initial implementation * 07/05/2015 Massimiliano Pinto Added MAX_EVENT_TYPE_MARIADB10 + * 10/09/2015 Massimiliano Pinto Added blr_read_events_all_events() + * It's called in maxbinlogcheck utility * * @endverbatim */ @@ -59,6 +61,7 @@ extern __thread log_info_t tls_log_info; static int blr_file_create(ROUTER_INSTANCE *router, char *file); static void blr_file_append(ROUTER_INSTANCE *router, char *file); static void blr_log_header(logfile_id_t file, char *msg, uint8_t *ptr); +static void blr_format_event_size(double *event_size, char *label); /** * Initialise the binlog file for this instance. MaxScale will look @@ -731,3 +734,695 @@ int filenum; return 0; return 1; } + +/** + * Read all replication events from a binlog file. + * + * Routine detects errors and pending transactions + * + * @param router The router instance + * @param fix Whether to fix or not errors + * @param debug Whether to enable or not the debug for events + * @return 0 on success, >0 on failure + */ +int +blr_read_events_all_events(ROUTER_INSTANCE *router, int fix, int debug) { +unsigned long filelen = 0; +struct stat statb; +uint8_t hdbuf[19]; +uint8_t *data; +GWBUF *result; +unsigned long long pos = 4; +unsigned long long last_known_commit = 4; + +REP_HEADER hdr; +int pending_transaction = 0; +int n; +int db_name_len; +char *statement_sql; +uint8_t *ptr; +int len; +int var_block_len; +int statement_len; +int checksum_len=0; +int found_chksum = 0; +int event_error = 0; +unsigned long transaction_events = 0; +unsigned long total_events = 0; +unsigned long total_bytes = 0; +unsigned long n_transactions = 0; +unsigned long max_events = 0; +unsigned long event_bytes = 0; +unsigned long max_bytes = 0; +double average_events = 0; +double average_bytes = 0; + + if (router->binlog_fd == -1) { + LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR, + "ERROR: Current binlog file %s is not open", + router->binlog_name))); + return 1; + } + + if (fstat(router->binlog_fd, &statb) == 0) + filelen = statb.st_size; + + router->current_pos = 4; + router->binlog_position = 4; + + while (1){ + + /* Read the header information from the file */ + if ((n = pread(router->binlog_fd, hdbuf, 19, pos)) != 19) { + switch (n) + { + case 0: + LOGIF(LD, (skygw_log_write_flush(LOGFILE_DEBUG, + "End of binlog file [%s] at %llu.", + router->binlog_name, + pos))); + if (n_transactions) + average_events = (double)((double)total_events / (double)n_transactions) * (1.0); + if (n_transactions) + average_bytes = (double)((double)total_bytes / (double)n_transactions) * (1.0); + + if (n_transactions != 0) { + char total_label[2]=""; + char average_label[2]=""; + char max_label[2]=""; + double format_total_bytes = total_bytes; + double format_max_bytes = max_bytes; + + blr_format_event_size(&format_total_bytes, total_label); + blr_format_event_size(&average_bytes, average_label); + blr_format_event_size(&format_max_bytes, max_label); + + LOGIF(LM, (skygw_log_write_flush(LOGFILE_MESSAGE, + "Transaction Summary for binlog '%s'\n" + "\t\t\tDescription %17s%17s%17s\n\t\t\t" + "No. of Transactions %16llu\n\t\t\t" + "No. of Events %16llu %16.1f %16llu\n\t\t\t" + "No. of Bytes %16.1f%s%16.1f%s%16.1f%s", router->binlog_name, + "Total", "Average", "Max", + n_transactions, total_events, + average_events, max_events, + format_total_bytes, total_label, average_bytes, average_label, format_max_bytes, max_label))); + } + + if (pending_transaction) { + LOGIF(LT, (skygw_log_write_flush(LOGFILE_TRACE, + "Warning : Binlog file %s contains a previous Opened Transaction" + " @ %llu. This pos is safe for slaves", + router->binlog_name, + last_known_commit))); + + } + + break; + case -1: + { + char err_msg[BLRM_STRERROR_R_MSG_SIZE+1] = ""; + strerror_r(errno, err_msg, BLRM_STRERROR_R_MSG_SIZE); + LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR, + "ERROR: Failed to read binlog file %s at position %llu" + " (%s).", router->binlog_name, + pos, err_msg))); + + if (errno == EBADF) + LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR, + "ERROR: Bad file descriptor in read binlog for file %s" + ", descriptor %d.", + router->binlog_name, router->binlog_fd))); + break; + } + default: + LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR, + "ERROR: Short read when reading the header. " + "Expected 19 bytes but got %d bytes. " + "Binlog file is %s, position %llu", + n, router->binlog_name, pos))); + break; + } + + /** + * Check for errors and force last_known_commit position + * and current pos + */ + + if (pending_transaction) { + router->binlog_position = last_known_commit; + router->current_pos = pos; + router->pending_transaction = 1; + pending_transaction = 0; + + LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR, + "Warning : pending transaction has been found. " + "Setting safe pos to %lu, current pos %lu", + router->binlog_position, router->current_pos))); + + return 0; + } else { + /* any error */ + if (n != 0) { + router->binlog_position = last_known_commit; + router->current_pos = pos; + + LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR, + "warning : an error has been found. " + "Setting safe pos to %lu, current pos %lu", + router->binlog_position, router->current_pos))); + if (fix) { + if (ftruncate(router->binlog_fd, router->binlog_position) == 0) { + LOGIF(LM, (skygw_log_write_flush(LOGFILE_MESSAGE, + "Binlog file %s has been truncated at %lu", + router->binlog_name, + router->binlog_position))); + fsync(router->binlog_fd); + } + } + + return 1; + } else { + router->binlog_position = pos; + router->current_pos = pos; + + return 0; + } + } + } + + /* fill replication header struct */ + hdr.timestamp = EXTRACT32(hdbuf); + hdr.event_type = hdbuf[4]; + hdr.serverid = EXTRACT32(&hdbuf[5]); + hdr.event_size = extract_field(&hdbuf[9], 32); + hdr.next_pos = EXTRACT32(&hdbuf[13]); + hdr.flags = EXTRACT16(&hdbuf[17]); + + /* Check event type against MAX_EVENT_TYPE */ + + if (router->mariadb10_compat) { + if (hdr.event_type > MAX_EVENT_TYPE_MARIADB10) { + LOGIF(LE, (skygw_log_write(LOGFILE_ERROR, + "Invalid MariaDB 10 event type 0x%x. " + "Binlog file is %s, position %d", + hdr.event_type, + router->binlog_name, pos))); + + event_error = 1; + } + } else { + if (hdr.event_type > MAX_EVENT_TYPE) { + LOGIF(LE, (skygw_log_write(LOGFILE_ERROR, + "Invalid event type 0x%x. " + "Binlog file is %s, position %d", + hdr.event_type, + router->binlog_name, pos))); + + event_error = 1; + } + } + + if (event_error) { + + router->binlog_position = last_known_commit; + router->current_pos = pos; + + LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR, + "warning : an error has been found in %s. " + "Setting safe pos to %lu, current pos %lu", + router->binlog_name, + router->binlog_position, + router->current_pos))); + + if (fix) { + if (ftruncate(router->binlog_fd, router->binlog_position) == 0) { + LOGIF(LM, (skygw_log_write_flush(LOGFILE_MESSAGE, + "Binlog file %s has been truncated at %lu", + router->binlog_name, + router->binlog_position))); + fsync(router->binlog_fd); + } + } + + return 1; + } + + if (hdr.event_size <= 0) + { + LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR, + "Event size error: " + "size %d at %llu.", + hdr.event_size, pos))); + + router->binlog_position = last_known_commit; + router->current_pos = pos; + + LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR, + "warning : an error has been found. " + "Setting safe pos to %lu, current pos %lu", + router->binlog_position, router->current_pos))); + if (fix) { + if (ftruncate(router->binlog_fd, router->binlog_position) == 0) { + LOGIF(LM, (skygw_log_write_flush(LOGFILE_MESSAGE, + "Binlog file %s has been truncated at %lu", + router->binlog_name, + router->binlog_position))); + fsync(router->binlog_fd); + } + } + + return 1; + } + + /* Allocate a GWBUF for the event */ + if ((result = gwbuf_alloc(hdr.event_size)) == NULL) + { + LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR, + "ERROR: Failed to allocate memory for binlog entry, " + "size %d at %llu.", + hdr.event_size, pos))); + + router->binlog_position = last_known_commit; + router->current_pos = pos; + + LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR, + "warning : an error has been found. " + "Setting safe pos to %lu, current pos %lu", + router->binlog_position, router->current_pos))); + + if (fix) { + if (ftruncate(router->binlog_fd, router->binlog_position) == 0) { + LOGIF(LM, (skygw_log_write_flush(LOGFILE_MESSAGE, + "Binlog file %s has been truncated at %lu", + router->binlog_name, + router->binlog_position))); + fsync(router->binlog_fd); + } + } + + return 1; + } + + /* Copy the header in the buffer */ + data = GWBUF_DATA(result); + memcpy(data, hdbuf, 19);// Copy the header in + + /* Read event data */ + if ((n = pread(router->binlog_fd, &data[19], hdr.event_size - 19, pos + 19)) != hdr.event_size - 19) + { + if (n == -1) + { + char err_msg[BLRM_STRERROR_R_MSG_SIZE+1] = ""; + strerror_r(errno, err_msg, BLRM_STRERROR_R_MSG_SIZE); + LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR, + "Error reading the event at %llu in %s. " + "%s, expected %d bytes.", + pos, router->binlog_name, + err_msg, hdr.event_size - 19))); + } + else + { + LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR, + "Short read when reading the event at %llu in %s. " + "Expected %d bytes got %d bytes.", + pos, router->binlog_name, hdr.event_size - 19, n))); + + if (filelen > 0 && filelen - pos < hdr.event_size) + { + LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR, + "Binlog event is close to the end of the binlog file %s, " + " size is %lu.", + router->binlog_name, filelen))); + } + } + + gwbuf_free(result); + + router->binlog_position = last_known_commit; + router->current_pos = pos; + + LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR, + "warning : an error has been found. " + "Setting safe pos to %lu, current pos %lu", + router->binlog_position, router->current_pos))); + if (fix) { + if (ftruncate(router->binlog_fd, router->binlog_position) == 0) { + LOGIF(LM, (skygw_log_write_flush(LOGFILE_MESSAGE, + "Binlog file %s has been truncated at %lu", + router->binlog_name, + router->binlog_position))); + fsync(router->binlog_fd); + } + } + + return 1; + } + + /* check for pending transaction */ + if (pending_transaction == 0) { + last_known_commit = pos; + } + + /* get event content */ + ptr = data+19; + + /* check for FORMAT DESCRIPTION EVENT */ + if(hdr.event_type == FORMAT_DESCRIPTION_EVENT) { + int event_header_length; + int event_header_ntypes; + int n_events; + int check_alg; + uint8_t *checksum; + + if(debug) + LOGIF(LD, (skygw_log_write_flush(LOGFILE_DEBUG, + "- Format Description event FDE @ %llu, size %lu", + pos, (unsigned long)hdr.event_size))); + + event_header_length = ptr[2 + 50 + 4]; + event_header_ntypes = hdr.event_size - event_header_length - (2 + 50 + 4 + 1); + + if (event_header_ntypes == 168) { + /* mariadb 10 LOG_EVENT_TYPES*/ + event_header_ntypes -= 163; + } else { + if (event_header_ntypes == 165) { + /* mariadb 5 LOG_EVENT_TYPES*/ + event_header_ntypes -= 160; + } else { + /* mysql 5.6 LOG_EVENT_TYPES = 35 */ + event_header_ntypes -= 35; + } + } + + n_events = hdr.event_size - event_header_length - (2 + 50 + 4 + 1); + + if(debug) { + LOGIF(LD, (skygw_log_write_flush(LOGFILE_DEBUG, + " FDE ServerVersion [%50s]", ptr + 2))); + + LOGIF(LD, (skygw_log_write_flush(LOGFILE_DEBUG, + " FDE Header EventLength %i" + ", N. of supported MySQL/MariaDB events %i", + event_header_length, + (n_events - event_header_ntypes)))); + } + + if (event_header_ntypes < n_events) { + checksum = ptr + hdr.event_size - event_header_length - event_header_ntypes; + check_alg = checksum[0]; + + if(debug) + LOGIF(LD, (skygw_log_write_flush(LOGFILE_DEBUG, + " FDE Checksum alg desc %i, alg type %s", + check_alg, + check_alg == 1 ? "BINLOG_CHECKSUM_ALG_CRC32" : "NONE or UNDEF"))); + if (check_alg == 1) { + checksum_len = 4; + found_chksum = 1; + } else { + found_chksum = 0; + } + } + } + /* Decode ROTATE EVENT */ + if(hdr.event_type == ROTATE_EVENT) { + int len, slen; + uint64_t new_pos; + char file[BINLOG_FNAMELEN+1]; + + len = hdr.event_size - 19; + new_pos = extract_field(ptr+4, 32); + new_pos <<= 32; + new_pos |= extract_field(ptr, 32); + slen = len - (8 + 4); // Allow for position and CRC + if (found_chksum == 0) + slen += 4; + if (slen > BINLOG_FNAMELEN) + slen = BINLOG_FNAMELEN; + memcpy(file, ptr + 8, slen); + file[slen] = 0; + + if(debug) + LOGIF(LD, (skygw_log_write_flush(LOGFILE_DEBUG, + "- Rotate event @ %llu, next file is [%s] @ %llu", + pos, file, new_pos))); + } + + /* If MariaDB 10 compatibility: + * check for MARIADB10_GTID_EVENT with flags = 0 + * This marks the transaction starts instead of + * QUERY_EVENT with "BEGIN" + */ + + if (router->mariadb10_compat) { + if (hdr.event_type == MARIADB10_GTID_EVENT) { + uint64_t n_sequence; /* 8 bytes */ + uint32_t domainid; /* 4 bytes */ + unsigned int flags; /* 1 byte */ + n_sequence = extract_field(ptr, 64); + domainid = extract_field(ptr + 8, 32); + flags = *(ptr + 8 + 4); + + if (flags == 0) { + if (pending_transaction > 0) { + LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR, + "ERROR: Transaction cannot be @ pos %llu: " + "Another MariaDB 10 transaction (GTID %lu-%lu-%llu)" + " was opened at %llu", + pos, domainid, hdr.serverid, n_sequence, last_known_commit))); + + gwbuf_free(result); + + break; + } else { + pending_transaction = 1; + + transaction_events = 0; + event_bytes = 0; + + if (debug) + LOGIF(LD, (skygw_log_write_flush(LOGFILE_DEBUG, + "> MariaDB 10 Transaction (GTID %lu-%lu-%llu)" + " starts @ pos %llu", + domainid, hdr.serverid, n_sequence, pos))); + } + } + } + } + + /** + * Check QUERY_EVENT + * + * Check for BEGIN ( ONLY for mysql 5.6, mariadb 5.5 ) + * Check for COMMIT (not transactional engines) + */ + + if(hdr.event_type == QUERY_EVENT) { + char *statement_sql; + db_name_len = ptr[4 + 4]; + var_block_len = ptr[4 + 4 + 1 + 2]; + + statement_len = hdr.event_size - 19 - (4+4+1+2+2+var_block_len+1+db_name_len); + //if (checksum_len) + // statement_len -= checksum_len; + + statement_sql = calloc(1, statement_len+1); + strncpy(statement_sql, (char *)ptr+4+4+1+2+2+var_block_len+1+db_name_len, statement_len); + + /* A transaction starts with this event */ + if (strncmp(statement_sql, "BEGIN", 5) == 0) { + if (pending_transaction > 0) { + LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR, + "ERROR: Transaction cannot be @ pos %llu: " + "Another transaction was opened at %llu", + pos, last_known_commit))); + + free(statement_sql); + gwbuf_free(result); + + break; + } else { + pending_transaction = 1; + + transaction_events = 0; + event_bytes = 0; + + if (debug) + LOGIF(LD, (skygw_log_write_flush(LOGFILE_DEBUG, + "> Transaction starts @ pos %llu", pos))); + } + } + + /* Commit received for non transactional tables, i.e. MyISAM */ + if (strncmp(statement_sql, "COMMIT", 6) == 0) { + if (pending_transaction > 0) { + pending_transaction = 3; + + if (debug) + LOGIF(LD, (skygw_log_write_flush(LOGFILE_DEBUG, + " Transaction @ pos %llu, closing @ %llu", last_known_commit, pos))); + } + } + free(statement_sql); + + } + + if(hdr.event_type == XID_EVENT) { + /* Commit received for a transactional tables, i.e. InnoDB */ + uint64_t xid; + xid = extract_field(ptr, 64); + + if (pending_transaction > 0) { + pending_transaction = 2; + if (debug) + LOGIF(LD, (skygw_log_write_flush(LOGFILE_DEBUG, + " Transaction XID @ pos %llu, closing @ %llu", last_known_commit, pos))); + } + } + + if (pending_transaction > 1) { + if (debug) + LOGIF(LD, (skygw_log_write_flush(LOGFILE_DEBUG, + "< Transaction @ pos %llu, is now closed @ %llu. %lu events seen", last_known_commit, pos, transaction_events))); + pending_transaction = 0; + last_known_commit = pos; + + total_events += transaction_events; + + if (transaction_events > max_events) + max_events = transaction_events; + + n_transactions++; + } + + gwbuf_free(result); + + /* pos and next_pos sanity checks */ + if (hdr.next_pos > 0 && hdr.next_pos < pos) { + LOGIF(LT, (skygw_log_write_flush(LOGFILE_TRACE, + "Binlog %s: next pos %llu < pos %llu, truncating to %llu", + router->binlog_name, + hdr.next_pos, + pos, + pos))); + + router->binlog_position = last_known_commit; + router->current_pos = pos; + + LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR, + "warning : an error has been found. " + "Setting safe pos to %lu, current pos %lu", + router->binlog_position, router->current_pos))); + if (fix) { + if (ftruncate(router->binlog_fd, router->binlog_position) == 0) { + LOGIF(LM, (skygw_log_write_flush(LOGFILE_MESSAGE, + "Binlog file %s has been truncated at %lu", + router->binlog_name, + router->binlog_position))); + fsync(router->binlog_fd); + } + } + + return 2; + } + + if (hdr.next_pos > 0 && hdr.next_pos != (pos + hdr.event_size)) { + LOGIF(LT, (skygw_log_write_flush(LOGFILE_TRACE, + "Binlog %s: next pos %llu != (pos %llu + event_size %llu), truncating to %llu", + router->binlog_name, + hdr.next_pos, + pos, + hdr.event_size, + pos))); + + router->binlog_position = last_known_commit; + router->current_pos = pos; + + LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR, + "warning : an error has been found. " + "Setting safe pos to %lu, current pos %lu", + router->binlog_position, router->current_pos))); + + if (fix) { + if (ftruncate(router->binlog_fd, router->binlog_position) == 0) { + LOGIF(LM, (skygw_log_write_flush(LOGFILE_MESSAGE, + "Binlog file %s has been truncated at %lu", + router->binlog_name, + router->binlog_position))); + fsync(router->binlog_fd); + } + } + + return 2; + } + + /* set pos to new value */ + if (hdr.next_pos > 0) { + + if (pending_transaction) { + total_bytes += hdr.event_size; + event_bytes += hdr.event_size; + + if (event_bytes > max_bytes) + max_bytes = event_bytes; + } + + pos = hdr.next_pos; + } else { + + LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR, + "Current event type %lu @ %llu has nex pos = %llu : exiting", hdr.event_type, pos, hdr.next_pos))); + break; + } + + transaction_events++; + } + + if (pending_transaction) { + LOGIF(LT, (skygw_log_write_flush(LOGFILE_TRACE, + "Binlog %s contains an Open Transaction, truncating to %llu", + router->binlog_name, + last_known_commit))); + + router->binlog_position = last_known_commit; + router->current_pos = pos; + router->pending_transaction = 1; + + LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR, + "warning : an error has been found. " + "Setting safe pos to %lu, current pos %lu", + router->binlog_position, router->current_pos))); + + return 0; + } else { + router->binlog_position = pos; + router->current_pos = pos; + + return 0; + } +} + +/** + * Format a number to G, M, k, or B size + * + * @param event_size The number to format + * @param label Label to use for display the formattted number + */ +static void +blr_format_event_size(double *event_size, char *label) +{ + if (*event_size > (1024 * 1024 * 1024)) { + *event_size = *event_size / (1024 * 1024 * 1024); + label[0] = 'G'; + } else if (*event_size > (1024 * 1024)) { + *event_size = *event_size / (1024 * 1024); + label[0] = 'M'; + } else if (*event_size > 1024) { + *event_size = *event_size / (1024); + label[0] = 'k'; + } else + label[0] = 'B'; +} + diff --git a/server/modules/routing/binlog/maxbinlogcheck.c b/server/modules/routing/binlog/maxbinlogcheck.c new file mode 100644 index 000000000..79cc30382 --- /dev/null +++ b/server/modules/routing/binlog/maxbinlogcheck.c @@ -0,0 +1,249 @@ +/* + * GNU General Public License as published by the Free Software Foundation, + * version 2. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 51 + * Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Copyright MariaDB Corporation Ab 2015 + */ + +/** + * @file maxbinlogcheck.c - The MaxScale binlog check utility + * + * This utility checks a MySQL 5.6 and MariaDB 10.0.X binlog file and reports + * any found error or an incomplete transaction. + * It suggests the pos the file should be trucatetd at. + * + * @verbatim + * Revision History + * + * Date Who Description + * 24/07/2015 Massimiliano Pinto Initial implementation + * 26/08/2015 Massimiliano Pinto Added mariadb10 option + * for MariaDB 10 binlog compatibility + * Currently MariadDB 10 starting transactions + * are detected checking GTID event + * with flags = 0 + * + * @endverbatim + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +extern int lm_enabled_logfiles_bitmask; +extern size_t log_ses_count[]; +extern __thread log_info_t tls_log_info; +extern int blr_read_events_all_events(ROUTER_INSTANCE *router, int fix, int debug); +extern uint32_t extract_field(uint8_t *src, int bits); +static void printVersion(const char *progname); +static void printUsage(const char *progname); + +static struct option long_options[] = { + {"debug", no_argument, 0, 'd'}, + {"version", no_argument, 0, 'V'}, + {"fix", no_argument, 0, 'f'}, + {"mariadb10", no_argument, 0, 'M'}, + {"help", no_argument, 0, '?'}, + {0, 0, 0, 0} +}; + +char *binlog_check_version = "1.0.0"; + +int main(int argc, char **argv) { + char** arg_vector; + int arg_count = 4; + ROUTER_INSTANCE *inst; + int fd; + int ret; + char *ptr; + char path[PATH_MAX+1] = ""; + unsigned long filelen = 0; + struct stat statb; + char c; + int option_index = 0; + int num_args = 0; + int debug_out = 0; + int fix_file = 0; + int mariadb10_compat = 0; + + while ((c = getopt_long(argc, argv, "dVfM?", long_options, &option_index)) >= 0) + { + switch (c) { + case 'd': + debug_out = 1; + break; + case 'V': + printVersion(*argv); + exit(EXIT_SUCCESS); + break; + case 'f': + fix_file = 1; + break; + case 'M': + mariadb10_compat = 1; + break; + case '?': + printUsage(*argv); + exit(optopt ? EXIT_FAILURE : EXIT_SUCCESS); + } + } + + num_args = optind; + + arg_vector = malloc(sizeof(char*)*(arg_count + 1)); + + if(arg_vector == NULL) + { + fprintf(stderr,"Error: Memory allocation failed for log manager arg_vector.\n"); + return 1; + } + + arg_vector[0] = "logmanager"; + arg_vector[1] = "-j"; + arg_vector[2] = "/tmp/maxbinlogcheck"; + arg_vector[3] = "-o"; + arg_vector[4] = NULL; + skygw_logmanager_init(arg_count,arg_vector); + + skygw_log_set_augmentation(0); + + free(arg_vector); + + if (!debug_out) + skygw_log_disable(LOGFILE_DEBUG); + else + skygw_log_enable(LOGFILE_DEBUG); + + if ((inst = calloc(1, sizeof(ROUTER_INSTANCE))) == NULL) { + LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR, + "Error: Memory allocation failed for ROUTER_INSTANCE"))); + + skygw_log_sync_all(); + skygw_logmanager_done(); + + return 1; + } + + if (argv[num_args] == NULL) { + printf("ERROR: No binlog file was specified\n"); + exit(EXIT_FAILURE); + } + + strncpy(path, argv[num_args], PATH_MAX); + + if (fix_file) + fd = open(path, O_RDWR, 0666); + else + fd = open(path, O_RDONLY, 0666); + + if (fd == -1) + { + LOGIF(LE, (skygw_log_write(LOGFILE_ERROR, + "Failed to open binlog file %s: %s", + path, strerror(errno)))); + + skygw_log_sync_all(); + skygw_logmanager_done(); + + free(inst); + + return 1; + } + + inst->binlog_fd = fd; + + if (mariadb10_compat == 1) + inst->mariadb10_compat = 1; + + ptr = strrchr(path, '/'); + if (ptr) + strncpy(inst->binlog_name, ptr+1, BINLOG_FNAMELEN); + else + strncpy(inst->binlog_name, path, BINLOG_FNAMELEN); + + LOGIF(LM, (skygw_log_write_flush(LOGFILE_MESSAGE, + "maxbinlogcheck %s", binlog_check_version))); + + if (fstat(inst->binlog_fd, &statb) == 0) + filelen = statb.st_size; + + LOGIF(LM, (skygw_log_write_flush(LOGFILE_MESSAGE, + "Checking %s (%s), size %lu bytes", path, inst->binlog_name, filelen))); + + /* read binary log */ + ret = blr_read_events_all_events(inst, fix_file, debug_out); + + close(inst->binlog_fd); + + skygw_log_sync_all(); + + LOGIF(LM, (skygw_log_write_flush(LOGFILE_MESSAGE, + "Check retcode: %i, Binlog Pos = %llu", ret, inst->binlog_position))); + + skygw_log_sync_all(); + skygw_logmanager_done(); + + free(inst); + + return 0; +} + +/** + * Print version information + */ +static void +printVersion(const char *progname) +{ + printf("%s Version %s\n", progname, binlog_check_version); +} + +/** + * Display the --help text. + */ +static void +printUsage(const char *progname) +{ + printVersion(progname); + + printf("The MaxScale binlog check utility.\n\n"); + printf("Usage: %s [-f] [-d] [-v] []\n\n", progname); + printf(" -f|--fix Fix binlog file, require write permissions (truncate)\n"); + printf(" -d|--debug Print debug messages\n"); + printf(" -M|--mariadb10 MariaDB 10 binlog compatibility\n"); + printf(" -V|--version print version information and exit\n"); + printf(" -?|--help Print this help text\n"); +} + From 96fdb5eb8d04502515100dab3b13930bd213208d Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 10 Sep 2015 18:36:59 +0300 Subject: [PATCH 03/38] Fixed SSL write failures not being handled properly. --- server/core/dcb.c | 41 +++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index 56ae46128..6d0617e05 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -108,7 +108,7 @@ static inline void dcb_write_fake_code(DCB *dcb); static inline void dcb_write_when_already_queued(DCB *dcb, GWBUF *queue); static void dcb_log_write_failure(DCB *dcb, GWBUF *queue, int eno); static inline void dcb_write_tidy_up(DCB *dcb, bool below_water); -static int dcb_write_SSL_error_report (DCB *dcb, int ret); +static void dcb_write_SSL_error_report (DCB *dcb, int ret, int ssl_errno); size_t dcb_get_session_id( DCB *dcb) @@ -1453,23 +1453,31 @@ dcb_write_SSL(DCB *dcb, GWBUF *queue) #if defined(FAKE_CODE) dcb_write_fake_code(dcb); #endif /* FAKE_CODE */ - do - { - w = gw_write_SSL(dcb->ssl, GWBUF_DATA(queue), GWBUF_LENGTH(queue)); - dcb->stats.n_writes++; + do + { + w = gw_write_SSL (dcb->ssl, GWBUF_DATA (queue), GWBUF_LENGTH (queue)); + dcb->stats.n_writes++; - if (w <= 0) - { - int ssl_errno = dcb_write_SSL_error_report (dcb, w); - if(ssl_errno != SSL_ERROR_WANT_WRITE) + if (w <= 0) + { + int ssl_errno = SSL_get_error (dcb->ssl, w); + dcb_write_SSL_error_report (dcb, w, ssl_errno); + if (ssl_errno != SSL_ERROR_WANT_WRITE) { - atomic_add(&dcb->writeqlen, gwbuf_length(queue)); + atomic_add (&dcb->writeqlen, gwbuf_length (queue)); dcb->stats.n_buffered++; - dcb_write_tidy_up(dcb, below_water); + dcb_write_tidy_up (dcb, below_water); return 1; } +#ifdef SS_DEBUG + else + { + skygw_log_write (LD, "SSL error: SSL_ERROR_WANT_WRITE, retrying SSL_write..."); + } +#endif } - } while(w <= 0); + } + while(w <= 0); /** Remove written bytes from the queue */ queue = gwbuf_consume(queue, w); @@ -1501,14 +1509,12 @@ dcb_write_SSL(DCB *dcb, GWBUF *queue) * * @param dcb The DCB of the client * @param ret The SSL operation return code - * @return The final SSL error number + * @param ssl_errno The SSL error code */ -static int -dcb_write_SSL_error_report (DCB *dcb, int ret) +static void +dcb_write_SSL_error_report (DCB *dcb, int ret, int ssl_errno) { - int ssl_errno; char errbuf[STRERROR_BUFLEN]; - ssl_errno = SSL_get_error(dcb->ssl,ret); if (LOG_IS_ENABLED(LOGFILE_DEBUG)) { @@ -1587,7 +1593,6 @@ dcb_write_SSL_error_report (DCB *dcb, int ret) } while((ssl_errno = ERR_get_error()) != 0); } } - return SSL_ERROR_NONE; } /** From 9fda8dcc2600ac4dc7f69f737f72d94e267c249e Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Fri, 11 Sep 2015 13:56:37 +0300 Subject: [PATCH 04/38] Changed link order. Did not link on Ubuntu. --- server/modules/routing/binlog/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/modules/routing/binlog/CMakeLists.txt b/server/modules/routing/binlog/CMakeLists.txt index 1e65eb504..bdf61aea1 100644 --- a/server/modules/routing/binlog/CMakeLists.txt +++ b/server/modules/routing/binlog/CMakeLists.txt @@ -5,6 +5,6 @@ install(TARGETS binlogrouter DESTINATION ${MAXSCALE_LIBDIR}) add_executable(maxbinlogcheck maxbinlogcheck.c blr_file.c blr_cache.c blr_master.c blr_slave.c blr.c ${CMAKE_SOURCE_DIR}/server/core/service.c ${CMAKE_SOURCE_DIR}/server/core/spinlock.c ${CMAKE_SOURCE_DIR}/server/core/buffer.c ${CMAKE_SOURCE_DIR}/server/core/atomic.c ${CMAKE_SOURCE_DIR}/server/core/hint.c ${CMAKE_SOURCE_DIR}/server/core/gwdirs.c ${CMAKE_SOURCE_DIR}/server/core/server.c ${CMAKE_SOURCE_DIR}/server/core/dcb.c ${CMAKE_SOURCE_DIR}/server/core/users.c ${CMAKE_SOURCE_DIR}/server/core/dbusers.c ${CMAKE_SOURCE_DIR}/server/core/utils.c ${CMAKE_SOURCE_DIR}/server/core/hashtable.c ${CMAKE_SOURCE_DIR}/server/core/poll.c ${CMAKE_SOURCE_DIR}/server/core/gwbitmask.c ${CMAKE_SOURCE_DIR}/server/core/config.c ${CMAKE_SOURCE_DIR}/server/core/session.c ${CMAKE_SOURCE_DIR}/server/core/housekeeper.c ${CMAKE_SOURCE_DIR}/server/core/filter.c ${CMAKE_SOURCE_DIR}/server/core/resultset.c ${CMAKE_SOURCE_DIR}/server/core/load_utils.c ${CMAKE_SOURCE_DIR}/server/core/monitor.c ${CMAKE_SOURCE_DIR}/server/core/gw_utils.c ${CMAKE_SOURCE_DIR}/server/core/thread.c ${CMAKE_SOURCE_DIR}/server/core/secrets.c) -target_link_libraries(maxbinlogcheck utils ssl pthread log_manager ${PCRE_LINK_FLAGS} aio rt crypt dl crypto inih z m stdc++ ${EMBEDDED_LIB} ${CURL_LIBRARIES}) +target_link_libraries(maxbinlogcheck utils ssl pthread log_manager ${EMBEDDED_LIB} ${PCRE_LINK_FLAGS} aio rt crypt dl crypto inih z m stdc++ ${CURL_LIBRARIES}) install(TARGETS maxbinlogcheck DESTINATION bin) From 535523f9d7cd69e406ff642e87f00f121b09a645 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 11 Sep 2015 14:27:46 +0300 Subject: [PATCH 05/38] Added an upper limit to trace log message length for queries. --- server/modules/routing/readwritesplit/readwritesplit.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index b8ad34719..42b64a34c 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -45,6 +45,8 @@ MODULE_INFO info = { # include #endif +#define RWSPLIT_TRACE_MSG_LEN 1000 + /** Defined in log_manager.cc */ extern int lm_enabled_logfiles_bitmask; extern size_t log_ses_count[]; @@ -2247,7 +2249,7 @@ static bool route_single_stmt( size_t len = MIN(GWBUF_LENGTH(querybuf), MYSQL_GET_PACKET_LEN((unsigned char *)querybuf->start)-1); char* data = (char*)&packet[5]; - char* contentstr = strndup(data, len); + char* contentstr = strndup(data, MIN(len, RWSPLIT_TRACE_MSG_LEN)); char* qtypestr = skygw_get_qtype_str(qtype); skygw_log_write( From 37f81485740f341333ce54a6eeb9949db5965e67 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Fri, 11 Sep 2015 15:21:31 +0300 Subject: [PATCH 06/38] MXS-362: Enable/disable log augmentation Log message augmentation (appending of function name) can now be enabled or disabled via the configuration file and command line. By default, the augmentation is disabled. --- .../Getting-Started/Configuration-Guide.md | 12 +++++++++ .../Tutorials/Administration-Tutorial.md | 1 + log_manager/log_manager.cc | 2 +- server/core/gateway.c | 26 ++++++++++++++++++- 4 files changed, 39 insertions(+), 2 deletions(-) diff --git a/Documentation/Getting-Started/Configuration-Guide.md b/Documentation/Getting-Started/Configuration-Guide.md index 01a8c1dba..0bb10617b 100644 --- a/Documentation/Getting-Started/Configuration-Guide.md +++ b/Documentation/Getting-Started/Configuration-Guide.md @@ -124,6 +124,18 @@ log_debug=1 To disable the log use the value 0 and to enable it use the value 1. +#### `log_augmentation` + +Enable or disable the augmentation of messages. If this is enabled, then each logged message is appended with the name of the function where the message was logged. This is primarily for development purposes and hence is disabled by default. + +``` +# Valid options are: +# log_augmentation=<0|1> +log_augmentation=1 +``` + +To disable the augmentation use the value 0 and to enable it use the value 1. + #### `logdir` Set the directory where the logfiles are stored. The folder needs to be both readable and writable by the user running MaxScale. diff --git a/Documentation/Tutorials/Administration-Tutorial.md b/Documentation/Tutorials/Administration-Tutorial.md index 364ca7e41..920c60f96 100644 --- a/Documentation/Tutorials/Administration-Tutorial.md +++ b/Documentation/Tutorials/Administration-Tutorial.md @@ -43,6 +43,7 @@ Switch|Long Option|Description `-U USER`|`--user=USER`|run MaxScale as another user. The user ID and group ID of this user are used to run MaxScale. `-s [yes no]`|`--syslog=[yes no]`|log messages to syslog (default:yes) `-S [yes no]`|`--maxscalelog=[yes no]`|log messages to MaxScale log (default: yes) +`-G [0 1]`|`--log_augmentation=[0 1]`|augment messages with the name of the function where the message was logged (default: 0). Primarily for development purposes. `-v`|`--version`|print version info and exit `-V`|`--version-full`|print version info and the commit ID the binary was built from `-?`|`--help`|show this help diff --git a/log_manager/log_manager.cc b/log_manager/log_manager.cc index c5c1f7bd6..4dc77f619 100644 --- a/log_manager/log_manager.cc +++ b/log_manager/log_manager.cc @@ -106,7 +106,7 @@ static bool flushall_done_flag; /** * Default augmentation. */ -static int default_log_augmentation = LOG_AUGMENT_WITH_FUNCTION; +static int default_log_augmentation = 0; static int log_augmentation = default_log_augmentation; /** Writer thread structure */ diff --git a/server/core/gateway.c b/server/core/gateway.c index 32c11584f..5a7b6f500 100644 --- a/server/core/gateway.c +++ b/server/core/gateway.c @@ -181,6 +181,7 @@ static struct option long_options[] = { {"version", no_argument, 0, 'v'}, {"help", no_argument, 0, '?'}, {"version-full", no_argument, 0, 'V'}, + {"log_augmentation", required_argument, 0, 'G'}, {0, 0, 0, 0} }; static int cnf_preparser(void* data, const char* section, const char* name, const char* value); @@ -197,6 +198,7 @@ static int ntfw_cb(const char*, const struct stat*, int, struct FTW*); static bool file_is_readable(char* absolute_pathname); static bool file_is_writable(char* absolute_pathname); bool handle_path_arg(char** dest, char* path, char* arg, bool rd, bool wr); +static void set_log_augmentation(const char* value); static void usage(void); static char* get_expanded_pathname( char** abs_path, @@ -1128,7 +1130,7 @@ int main(int argc, char **argv) } } - while ((opt = getopt_long(argc, argv, "dc:f:l:vVs:S:?L:D:C:B:U:A:P:", + while ((opt = getopt_long(argc, argv, "dc:f:l:vVs:S:?L:D:C:B:U:A:P:G:", long_options, &option_index)) != -1) { bool succp = true; @@ -1291,6 +1293,9 @@ int main(int argc, char **argv) succp = false; } break; + case 'G': + set_log_augmentation(optarg); + break; case '?': usage(); rc = EXIT_SUCCESS; @@ -2312,6 +2317,21 @@ bool handle_path_arg(char** dest, char* path, char* arg, bool rd, bool wr) return rval; } +void set_log_augmentation(const char* value) +{ + // Command line arguments are handled first, thus command line argument + // has priority. + + static bool augmentation_set = false; + + if (!augmentation_set) + { + skygw_log_set_augmentation(atoi(value)); + + augmentation_set = true; + } +} + /** * Pre-parse the MaxScale.cnf for config, log and module directories. * @param data Parameter passed by inih @@ -2423,6 +2443,10 @@ static int cnf_preparser(void* data, const char* section, const char* name, cons { cnf->maxlog = config_truth_value((char*)value); } + else if(strcmp(name, "log_augmentation") == 0) + { + set_log_augmentation(value); + } } return 1; From 743fb5774b812f01f4a11623cd0fc57d5dbb3084 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 11 Sep 2015 18:09:06 +0300 Subject: [PATCH 07/38] Improved logging. --- server/modules/protocol/mysql_backend.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/server/modules/protocol/mysql_backend.c b/server/modules/protocol/mysql_backend.c index 113e5dab0..391ec0cc7 100644 --- a/server/modules/protocol/mysql_backend.c +++ b/server/modules/protocol/mysql_backend.c @@ -713,6 +713,7 @@ gw_MySQLWrite_backend(DCB *dcb, GWBUF *queue) MySQLProtocol *backend_protocol = dcb->protocol; int rc = 0; + CHK_DCB(dcb); spinlock_acquire(&dcb->authlock); /** * Pick action according to state of protocol. @@ -722,10 +723,12 @@ gw_MySQLWrite_backend(DCB *dcb, GWBUF *queue) switch (backend_protocol->protocol_auth_state) { case MYSQL_HANDSHAKE_FAILED: case MYSQL_AUTH_FAILED: - LOGIF(LE, (skygw_log_write_flush( - LOGFILE_ERROR, - "Error : Unable to write to backend due to " - "authentication failure."))); + LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR, + "Error : Unable to write to backend '%s' due to " + "%s failure. Server in state %s.", + dcb->server->unique_name, + backend_protocol->protocol_auth_state == MYSQL_HANDSHAKE_FAILED ? "handshake" : "authentication", + STRSRVSTATUS(dcb->server)))); /** Consume query buffer */ while ((queue = gwbuf_consume( queue, From 3b0b740e4776108a1e983da9abc1705626498f07 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 11 Sep 2015 18:38:19 +0300 Subject: [PATCH 08/38] Fixed monitor scripts and added extra logging. --- server/core/externcmd.c | 1 + server/modules/monitor/monitor_common.c | 38 ++++++++++++++----------- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/server/core/externcmd.c b/server/core/externcmd.c index 1163fab2f..d58b06351 100644 --- a/server/core/externcmd.c +++ b/server/core/externcmd.c @@ -114,6 +114,7 @@ EXTERNCMD* externcmd_allocate(char* argstr) externcmd_free(cmd); return NULL; } + skygw_log_write(LT, "Executing script %s.", cmd->parameters[0]); } return cmd; } diff --git a/server/modules/monitor/monitor_common.c b/server/modules/monitor/monitor_common.c index cbf1def75..ae7c47e05 100644 --- a/server/modules/monitor/monitor_common.c +++ b/server/modules/monitor/monitor_common.c @@ -294,31 +294,37 @@ bool mon_print_fail_status( return succp; } -void monitor_launch_script(MONITOR* mon,MONITOR_SERVERS* ptr, char* script) +/** + * Launch a script + * @param mon Owning monitor + * @param ptr The server which has changed state + * @param script Script to execute + */ +void monitor_launch_script(MONITOR* mon, MONITOR_SERVERS* ptr, char* script) { char argstr[PATH_MAX + MON_ARG_MAX + 1]; EXTERNCMD* cmd; - snprintf(argstr,PATH_MAX + MON_ARG_MAX, - "%s --event=%s --initiator=%s:%d --nodelist=", - script, - mon_get_event_name(ptr), - ptr->server->name, - ptr->server->port); + snprintf(argstr, PATH_MAX + MON_ARG_MAX, + "%s --event=%s --initiator=%s:%d --nodelist=", + script, + mon_get_event_name(ptr), + ptr->server->name, + ptr->server->port); - mon_append_node_names(mon->databases,argstr,PATH_MAX + MON_ARG_MAX); - if((cmd = externcmd_allocate(argstr)) == NULL) + mon_append_node_names(mon->databases, argstr, PATH_MAX + MON_ARG_MAX); + if ((cmd = externcmd_allocate(argstr)) == NULL) { - skygw_log_write(LE,"Failed to initialize script: %s",script); - return; + skygw_log_write(LE, "Failed to initialize script: %s", script); + return; } - if(externcmd_execute(cmd)) + if (externcmd_execute(cmd)) { - skygw_log_write(LOGFILE_ERROR, - "Error: Failed to execute script " - "'%s' on server state change event %s.", - script,mon_get_event_type(ptr)); + skygw_log_write(LOGFILE_ERROR, + "Error: Failed to execute script " + "'%s' on server state change event %s.", + script, mon_get_event_name(ptr)); } externcmd_free(cmd); } From bb72b0c89298f225266a211672c93b25dcce2835 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Sat, 12 Sep 2015 04:46:47 +0300 Subject: [PATCH 09/38] Fixed compiler warnings. --- server/modules/routing/binlog/blr_file.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/server/modules/routing/binlog/blr_file.c b/server/modules/routing/binlog/blr_file.c index f6daf54d0..7d8ba20d1 100644 --- a/server/modules/routing/binlog/blr_file.c +++ b/server/modules/routing/binlog/blr_file.c @@ -764,7 +764,6 @@ uint8_t *ptr; int len; int var_block_len; int statement_len; -int checksum_len=0; int found_chksum = 0; int event_error = 0; unsigned long transaction_events = 0; @@ -1139,7 +1138,6 @@ double average_bytes = 0; check_alg, check_alg == 1 ? "BINLOG_CHECKSUM_ALG_CRC32" : "NONE or UNDEF"))); if (check_alg == 1) { - checksum_len = 4; found_chksum = 1; } else { found_chksum = 0; @@ -1225,8 +1223,6 @@ double average_bytes = 0; var_block_len = ptr[4 + 4 + 1 + 2]; statement_len = hdr.event_size - 19 - (4+4+1+2+2+var_block_len+1+db_name_len); - //if (checksum_len) - // statement_len -= checksum_len; statement_sql = calloc(1, statement_len+1); strncpy(statement_sql, (char *)ptr+4+4+1+2+2+var_block_len+1+db_name_len, statement_len); @@ -1271,8 +1267,8 @@ double average_bytes = 0; if(hdr.event_type == XID_EVENT) { /* Commit received for a transactional tables, i.e. InnoDB */ - uint64_t xid; - xid = extract_field(ptr, 64); + unsigned long long xid = (unsigned long long)extract_field(ptr, 64); + skygw_log_write(LD, "XID_EVENT: %llu", xid); if (pending_transaction > 0) { pending_transaction = 2; From c11c4733a43ee0f6353a81573b3689e7461287ff Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Sat, 12 Sep 2015 05:43:32 +0300 Subject: [PATCH 10/38] Fixed wrong error message in dbusers --- server/core/dbusers.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/core/dbusers.c b/server/core/dbusers.c index 701d59764..d55ca23b1 100644 --- a/server/core/dbusers.c +++ b/server/core/dbusers.c @@ -1238,7 +1238,7 @@ getUsers(SERVICE *service, USERS *users) LOGIF(LE, (skygw_log_write_flush( LOGFILE_ERROR, "Error : Unable to get user data from backend database " - "for service [%s]. Missing server information.", + "for service [%s]. Failed to connect to any of the backend databases.", service->name))); mysql_close(con); return -1; From 5d1895561ca299efa6c2ce4173463655f7523ddc Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 11 Sep 2015 14:28:39 +0300 Subject: [PATCH 11/38] Moved parts of dcb_read_SSL to separate functions. --- server/core/dcb.c | 379 +++++++++++++++++++++++----------------------- 1 file changed, 189 insertions(+), 190 deletions(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index 6d0617e05..398741638 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -109,6 +109,8 @@ static inline void dcb_write_when_already_queued(DCB *dcb, GWBUF *queue); static void dcb_log_write_failure(DCB *dcb, GWBUF *queue, int eno); static inline void dcb_write_tidy_up(DCB *dcb, bool below_water); static void dcb_write_SSL_error_report (DCB *dcb, int ret, int ssl_errno); +int dcb_bytes_readable_SSL (DCB *dcb, int nread); +void dcb_log_ssl_read_error(DCB *dcb, int ssl_errno, int rc); size_t dcb_get_session_id( DCB *dcb) @@ -909,208 +911,104 @@ int dcb_read( * @return -1 on error, otherwise the number of read bytes on the last * iteration of while loop. 0 is returned if no data available. */ -int dcb_read_SSL( - DCB *dcb, - GWBUF **head) +int dcb_read_SSL(DCB *dcb, + GWBUF **head) { - GWBUF *buffer = NULL; - int b,pending; - int rc; - int n; - int nread = 0; - int ssl_errno = 0; - CHK_DCB(dcb); + GWBUF *buffer = NULL; + int b, n, nread = 0; + CHK_DCB(dcb); - if (dcb->fd <= 0) - { - LOGIF(LE, (skygw_log_write_flush( - LOGFILE_ERROR, - "Error : Read failed, dcb is %s.", - dcb->fd == DCBFD_CLOSED ? "closed" : "cloned, not readable"))); - n = 0; - goto return_n; - } + if (dcb->fd <= 0) + { + LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR, + "Error : Read failed, dcb is %s.", + dcb->fd == DCBFD_CLOSED ? "closed" : "cloned, not readable"))); + return 0; + } - while (true) + while ((b = dcb_bytes_readable_SSL(dcb, nread)) > 0) + { + dcb->last_read = hkheartbeat; + int bufsize = MIN(b, MAX_BUFFER_SIZE); + + if ((buffer = gwbuf_alloc(bufsize)) == NULL) { - int bufsize; - ssl_errno = 0; - rc = ioctl(dcb->fd, FIONREAD, &b); - pending = SSL_pending(dcb->ssl); - if (rc == -1) - { - char errbuf[STRERROR_BUFLEN]; - LOGIF(LE, (skygw_log_write_flush( - LOGFILE_ERROR, - "Error : ioctl FIONREAD for dcb %p in " - "state %s fd %d failed due error %d, %s.", - dcb, - STRDCBSTATE(dcb->state), - dcb->fd, - errno, - strerror_r(errno, errbuf, sizeof(errbuf))))); - n = -1; - goto return_n; - } + /*< + * This is a fatal error which should cause shutdown. + * Todo shutdown if memory allocation fails. + */ + char errbuf[STRERROR_BUFLEN]; + LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR, + "Error : Failed to allocate read buffer " + "for dcb %p fd %d, due %d, %s.", + dcb, + dcb->fd, + errno, + strerror_r(errno, errbuf, sizeof (errbuf))))); - if (b == 0 && pending == 0 && nread == 0) - { - /** Handle closed client socket */ - if (dcb_isclient(dcb)) - { - char c = 0; - int r = -1; + return -1; + } - /* try to read 1 byte, without consuming the socket buffer */ - r = SSL_peek(dcb->ssl, &c, sizeof(char)); - if (r <= 0) - { - ssl_errno = SSL_get_error(dcb->ssl,r); - if(ssl_errno != SSL_ERROR_WANT_READ && - ssl_errno != SSL_ERROR_WANT_WRITE && - ssl_errno != SSL_ERROR_NONE) - n = -1; - else - n = 0; - goto return_n; - } - } - n = 0; - goto return_n; - } - else if (b == 0 && pending == 0) - { - n = 0; - goto return_n; - } + n = SSL_read(dcb->ssl, GWBUF_DATA(buffer), bufsize); + dcb->stats.n_reads++; + + if (n <= 0) + { + int ssl_errno = SSL_get_error(dcb->ssl, n); + dcb_log_ssl_read_error(dcb, ssl_errno, n); + + if (ssl_errno != SSL_ERROR_WANT_READ && + ssl_errno != SSL_ERROR_WANT_WRITE && + ssl_errno != SSL_ERROR_NONE) + { + nread = -1; + gwbuf_free(buffer); + } + return nread; + } + + buffer = gwbuf_rtrim(buffer, bufsize - n); + + if (buffer) + { #ifdef SS_DEBUG - else - { - skygw_log_write_flush(LD,"Total: %d Socket: %d Pending: %d", - nread,b,pending); - } + skygw_log_write(LD, "%lu SSL: Truncated buffer from %d to %d bytes. " + "Read %d bytes, %d bytes waiting.\n", pthread_self(), + bufsize, GWBUF_LENGTH(buffer), n, b); + + if (GWBUF_LENGTH(buffer) != n) + { + skygw_log_sync_all(); + } + + ss_info_dassert((buffer->start <= buffer->end), "Buffer start has passed end."); + ss_info_dassert(GWBUF_LENGTH(buffer) == n, "Buffer size not equal to read bytes."); #endif + nread += n; - dcb->last_read = hkheartbeat; + LOGIF(LD, (skygw_log_write(LOGFILE_DEBUG, + "%lu [dcb_read_SSL] Read %d bytes from dcb %p in state %s " + "fd %d.", + pthread_self(), + n, + dcb, + STRDCBSTATE(dcb->state), + dcb->fd))); - bufsize = MIN(b, MAX_BUFFER_SIZE); + /*< Append read data to the gwbuf */ + *head = gwbuf_append(*head, buffer); + } + } - if ((buffer = gwbuf_alloc(bufsize)) == NULL) - { - /*< - * This is a fatal error which should cause shutdown. - * Todo shutdown if memory allocation fails. - */ - char errbuf[STRERROR_BUFLEN]; - LOGIF(LE, (skygw_log_write_flush( - LOGFILE_ERROR, - "Error : Failed to allocate read buffer " - "for dcb %p fd %d, due %d, %s.", - dcb, - dcb->fd, - errno, - strerror_r(errno, errbuf, sizeof(errbuf))))); + ss_dassert(gwbuf_length(*head) == nread); + LOGIF(LD, skygw_log_write(LD, "%lu Read a total of %d bytes from dcb %p in state %s fd %d.", + pthread_self(), + nread, + dcb, + STRDCBSTATE(dcb->state), + dcb->fd)); - n = -1; - goto return_n; - } - - n = SSL_read(dcb->ssl, GWBUF_DATA(buffer), bufsize); - dcb->stats.n_reads++; - - if (n < 0) - { - char errbuf[200]; - ssl_errno = SSL_get_error(dcb->ssl,n); -#ifdef SS_DEBUG - if(ssl_errno == SSL_ERROR_SSL || - ssl_errno == SSL_ERROR_SYSCALL) - { - int eno; - while((eno = ERR_get_error()) != 0) - { - ERR_error_string_n(eno,errbuf,200); - skygw_log_write(LE, - "%s", - errbuf); - } - } -#endif - if(ssl_errno == SSL_ERROR_WANT_READ || - ssl_errno == SSL_ERROR_WANT_WRITE || - ssl_errno == SSL_ERROR_NONE) - { - n = 0; - } - else - { - char errbuf[STRERROR_BUFLEN]; - LOGIF(LE, (skygw_log_write_flush( - LOGFILE_ERROR, - "Error : Read failed, dcb %p in state " - "%s fd %d, SSL error %d: %s.", - dcb, - STRDCBSTATE(dcb->state), - dcb->fd, - ssl_errno, - strerror_r(errno, errbuf, sizeof(errbuf))))); - - if(ssl_errno == SSL_ERROR_SSL || - ssl_errno == SSL_ERROR_SYSCALL) - { - while((ssl_errno = ERR_get_error()) != 0) - { - ERR_error_string_n(ssl_errno,errbuf,200); - skygw_log_write(LE, - "%s", - errbuf); - } - } - } - n = -1; - gwbuf_free(buffer); - goto return_n; - } - else if(n == 0) - { - gwbuf_free(buffer); - goto return_n; - } - - buffer = gwbuf_rtrim(buffer,bufsize - n); - if(buffer == NULL) - { - goto return_n; - } -#ifdef SS_DEBUG - skygw_log_write(LD,"%lu SSL: Truncated buffer from %d to %d bytes. " - "Read %d bytes, %d bytes waiting.\n",pthread_self(), - bufsize,GWBUF_LENGTH(buffer),n,b); - - if(GWBUF_LENGTH(buffer) != n){ - skygw_log_sync_all(); - } - - ss_info_dassert((buffer->start <= buffer->end),"Buffer start has passed end."); - ss_info_dassert(GWBUF_LENGTH(buffer) == n,"Buffer size not equal to read bytes."); -#endif - nread += n; - - LOGIF(LD, (skygw_log_write( - LOGFILE_DEBUG, - "%lu [dcb_read_SSL] Read %d bytes from dcb %p in state %s " - "fd %d.", - pthread_self(), - n, - dcb, - STRDCBSTATE(dcb->state), - dcb->fd))); - - /*< Append read data to the gwbuf */ - *head = gwbuf_append(*head, buffer); - } /*< while (true) */ -return_n: - return nread; + return nread; } /** @@ -3143,3 +3041,104 @@ char *name = NULL; } return name; } + +/** + * Check how much data is readable from an SSL enabled DCB. + * @param dcb DCB to check + * @param nread Number of bytes we have already read + * @return Number of bytes readable or -1 on error + */ +int dcb_bytes_readable_SSL (DCB *dcb, int nread) +{ + int rval = 0; + int nbytes; + int rc = ioctl (dcb->fd, FIONREAD, &nbytes); + int pending = SSL_pending (dcb->ssl); + + if (rc == -1) + { + char errbuf[STRERROR_BUFLEN]; + LOGIF (LE, (skygw_log_write_flush (LOGFILE_ERROR, + "Error : ioctl FIONREAD for dcb %p in " + "state %s fd %d failed due error %d, %s.", + dcb, + STRDCBSTATE (dcb->state), + dcb->fd, + errno, + strerror_r (errno, errbuf, sizeof (errbuf))))); + rval = -1; + } + else + { + rval = nbytes + pending; + if (rval == 0 && nread == 0) + { + /** Handle closed client socket */ + if (dcb_isclient (dcb)) + { + char c = 0; + int r = -1; + + /* try to read 1 byte, without consuming the socket buffer */ + r = SSL_peek (dcb->ssl, &c, sizeof (char)); + if (r <= 0) + { + int ssl_errno = SSL_get_error (dcb->ssl, r); + if (ssl_errno != SSL_ERROR_WANT_READ && + ssl_errno != SSL_ERROR_WANT_WRITE && + ssl_errno != SSL_ERROR_NONE) + rval = -1; + } + } + } +#ifdef SS_DEBUG + else if (nbytes != 0 || pending != 0) + { + skygw_log_write_flush (LD, "Total: %d Socket: %d Pending: %d", + nread, nbytes, pending); + } + else + { + skygw_log_write(LD, "Tried to read from socket, no data left. %d bytes read in total.", nread); + } +#endif + } + return rval; +} + +/** + * Log SSL read error messages + * @param dcb DCB from which the SSL_read was attempted + * @param ssl_errno SSL error number + * @param rc Return value of SSL_read + */ +void dcb_log_ssl_read_error(DCB *dcb, int ssl_errno, int rc) +{ + if (ssl_errno != SSL_ERROR_WANT_READ && + ssl_errno != SSL_ERROR_WANT_WRITE && + ssl_errno != SSL_ERROR_NONE) + { + + char errbuf[STRERROR_BUFLEN]; + LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR, + "Error : Read failed, dcb %p in state " + "%s fd %d, SSL error %d: %s.", + dcb, + STRDCBSTATE(dcb->state), + dcb->fd, + ssl_errno, + strerror_r(errno, errbuf, sizeof (errbuf))))); + + if (ssl_errno == SSL_ERROR_SSL || + ssl_errno == SSL_ERROR_SYSCALL) + { + while ((ssl_errno = ERR_get_error()) != 0) + { + ERR_error_string_n(ssl_errno, errbuf, 200); + skygw_log_write(LE, + "%s", + errbuf); + } + } + } +} \ No newline at end of file From 955974d06b49fa9a504460a7c68866f17d6bf5cc Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Mon, 14 Sep 2015 09:43:13 +0200 Subject: [PATCH 12/38] Removed unused XID var Removed unused XID var --- server/modules/routing/binlog/blr_file.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/modules/routing/binlog/blr_file.c b/server/modules/routing/binlog/blr_file.c index 7d8ba20d1..36b0996ae 100644 --- a/server/modules/routing/binlog/blr_file.c +++ b/server/modules/routing/binlog/blr_file.c @@ -1267,8 +1267,6 @@ double average_bytes = 0; if(hdr.event_type == XID_EVENT) { /* Commit received for a transactional tables, i.e. InnoDB */ - unsigned long long xid = (unsigned long long)extract_field(ptr, 64); - skygw_log_write(LD, "XID_EVENT: %llu", xid); if (pending_transaction > 0) { pending_transaction = 2; From e443351341622fbc999cd9d690fe6a4b5c5619a9 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Mon, 14 Sep 2015 16:29:14 +0300 Subject: [PATCH 13/38] Fix to MXS-279: https://mariadb.atlassian.net/browse/MXS-279 Added NULL checks to block buffer initialization and checks for return values to functions that use it. --- log_manager/log_manager.cc | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/log_manager/log_manager.cc b/log_manager/log_manager.cc index 4dc77f619..fd2a74195 100644 --- a/log_manager/log_manager.cc +++ b/log_manager/log_manager.cc @@ -771,6 +771,8 @@ static int logmanager_write_log( wp = (char*)malloc(sizeof(char)*(timestamp_len-sizeof(char)+cmplen+str_len + 1)); } + if(wp == NULL) + return -1; #if defined (SS_LOG_DEBUG) { @@ -1042,7 +1044,9 @@ static char* blockbuf_get_writepos( /** * New node is created */ - bb = blockbuf_init(id); + if((bb = blockbuf_init(id)) == NULL) + return NULL; + CHK_BLOCKBUF(bb); /** @@ -1128,7 +1132,9 @@ static char* blockbuf_get_writepos( * Create the first block buffer to logfile's blockbuf list. */ - bb = blockbuf_init(id); + if((bb = blockbuf_init(id)) == NULL) + return NULL; + CHK_BLOCKBUF(bb); /** Lock buffer */ @@ -1210,9 +1216,10 @@ static void blockbuf_node_done( static blockbuf_t* blockbuf_init( logfile_id_t id) { - blockbuf_t* bb; + blockbuf_t* bb; - bb = (blockbuf_t *)calloc(1, sizeof(blockbuf_t)); + if ((bb = (blockbuf_t *) calloc(1, sizeof (blockbuf_t)))) + { bb->bb_fileid = id; #if defined(SS_DEBUG) bb->bb_chk_top = CHK_NUM_BLOCKBUF; @@ -1221,15 +1228,18 @@ static blockbuf_t* blockbuf_init( simple_mutex_init(&bb->bb_mutex, "Blockbuf mutex"); bb->bb_buf_left = MAX_LOGSTRLEN; bb->bb_buf_size = MAX_LOGSTRLEN; - #if defined(SS_LOG_DEBUG) - sprintf(bb->bb_buf,"[block:%d]",atomic_add(&block_start_index,1)); - bb->bb_buf_used += strlen(bb->bb_buf); - bb->bb_buf_left -= strlen(bb->bb_buf); + sprintf(bb->bb_buf, "[block:%d]", atomic_add(&block_start_index, 1)); + bb->bb_buf_used += strlen(bb->bb_buf); + bb->bb_buf_left -= strlen(bb->bb_buf); #endif - CHK_BLOCKBUF(bb); - return bb; + } + else + { + fprintf(stderr, "Error: Memory allocation failed when initializing log manager block buffers."); + } + return bb; } From 2829a8ce05d7b8428a21b0bd826e2687bcbd0afe Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Mon, 14 Sep 2015 21:20:22 +0300 Subject: [PATCH 14/38] Added missing check for empty result set in binlogrouter. --- server/modules/routing/binlog/blr_master.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/server/modules/routing/binlog/blr_master.c b/server/modules/routing/binlog/blr_master.c index 0a80a4d54..3001b2cef 100644 --- a/server/modules/routing/binlog/blr_master.c +++ b/server/modules/routing/binlog/blr_master.c @@ -1480,6 +1480,14 @@ char *rval; // Finally we have reached the row len = EXTRACT24(ptr); ptr += 4; + + /** The first EOF packet signals the start of the resultset rows and the second + EOF packet signals the end of the result set. If the resultset + contains a second EOF packet right after the first one, the result set is empty and + contains no rows. */ + if(len == 5 && *ptr == 0xfe) + return NULL; + while (--col > 0) { collen = *ptr++; From bbc0116cd24bef34b78e238cd11041755960e8c7 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 15 Sep 2015 10:52:37 +0300 Subject: [PATCH 15/38] Moved the writing to and closing of the parent-child pipe to a separate function. --- server/core/gateway.c | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/server/core/gateway.c b/server/core/gateway.c index a79fddfbf..7580c4a68 100644 --- a/server/core/gateway.c +++ b/server/core/gateway.c @@ -213,6 +213,7 @@ static bool resolve_maxscale_conf_fname( static char* check_dir_access(char* dirname,bool,bool); static int set_user(); bool pid_file_exists(); +void write_child_exit_code(int fd, int code); /** SSL multi-threading functions and structures */ static SPINLOCK* ssl_locks; @@ -1450,8 +1451,8 @@ int main(int argc, char **argv) if(parent_process) { - int nread = read(daemon_pipe[0],(void*)&child_status,sizeof(int)); close(daemon_pipe[1]); + int nread = read(daemon_pipe[0],(void*)&child_status,sizeof(int)); close(daemon_pipe[0]); if(nread == -1) @@ -1951,8 +1952,7 @@ int main(int argc, char **argv) * Successful start, notify the parent process that it can exit. */ ss_dassert(rc == MAXSCALE_SHUTDOWN); - write(daemon_pipe[1],&rc,sizeof(int)); - close(daemon_pipe[1]); + write_child_exit_code(daemon_pipe[1], rc); MaxScaleStarted = time(0); /*< @@ -1996,8 +1996,7 @@ return_main: if(daemon_mode && rc != MAXSCALE_SHUTDOWN) { /** Notify the parent process that an error has occurred */ - write(daemon_pipe[1],&rc,sizeof(int)); - close(daemon_pipe[1]); + write_child_exit_code(daemon_pipe[1], rc); } if (threads) @@ -2488,3 +2487,14 @@ static int set_user(char* user) return rval; } +/** + * Write the exit status of the child process to the parent process. + * @param fd File descriptor to write to + * @param code Exit status of the child process + */ +void write_child_exit_code(int fd, int code) +{ + /** Notify the parent process that an error has occurred */ + write(fd, &code, sizeof (int)); + close(fd); +} \ No newline at end of file From fe9babe9ecca312a97a00efab0fc189dcd265810 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Wed, 16 Sep 2015 10:22:29 +0300 Subject: [PATCH 16/38] Cleaned up code. --- server/core/dcb.c | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index 398741638..5740d4ffa 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -3048,42 +3048,42 @@ char *name = NULL; * @param nread Number of bytes we have already read * @return Number of bytes readable or -1 on error */ -int dcb_bytes_readable_SSL (DCB *dcb, int nread) +int dcb_bytes_readable_SSL(DCB *dcb, int nread) { int rval = 0; int nbytes; - int rc = ioctl (dcb->fd, FIONREAD, &nbytes); - int pending = SSL_pending (dcb->ssl); + int rc = ioctl(dcb->fd, FIONREAD, &nbytes); if (rc == -1) { char errbuf[STRERROR_BUFLEN]; - LOGIF (LE, (skygw_log_write_flush (LOGFILE_ERROR, - "Error : ioctl FIONREAD for dcb %p in " - "state %s fd %d failed due error %d, %s.", - dcb, - STRDCBSTATE (dcb->state), - dcb->fd, - errno, - strerror_r (errno, errbuf, sizeof (errbuf))))); + LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR, + "Error : ioctl FIONREAD for dcb %p in " + "state %s fd %d failed due error %d, %s.", + dcb, + STRDCBSTATE(dcb->state), + dcb->fd, + errno, + strerror_r(errno, errbuf, sizeof (errbuf))))); rval = -1; } else { + int pending = SSL_pending(dcb->ssl); rval = nbytes + pending; if (rval == 0 && nread == 0) { /** Handle closed client socket */ - if (dcb_isclient (dcb)) + if (dcb_isclient(dcb)) { char c = 0; int r = -1; /* try to read 1 byte, without consuming the socket buffer */ - r = SSL_peek (dcb->ssl, &c, sizeof (char)); + r = SSL_peek(dcb->ssl, &c, sizeof (char)); if (r <= 0) { - int ssl_errno = SSL_get_error (dcb->ssl, r); + int ssl_errno = SSL_get_error(dcb->ssl, r); if (ssl_errno != SSL_ERROR_WANT_READ && ssl_errno != SSL_ERROR_WANT_WRITE && ssl_errno != SSL_ERROR_NONE) @@ -3094,8 +3094,8 @@ int dcb_bytes_readable_SSL (DCB *dcb, int nread) #ifdef SS_DEBUG else if (nbytes != 0 || pending != 0) { - skygw_log_write_flush (LD, "Total: %d Socket: %d Pending: %d", - nread, nbytes, pending); + skygw_log_write_flush(LD, "Total: %d Socket: %d Pending: %d", + nread, nbytes, pending); } else { @@ -3134,7 +3134,7 @@ void dcb_log_ssl_read_error(DCB *dcb, int ssl_errno, int rc) { while ((ssl_errno = ERR_get_error()) != 0) { - ERR_error_string_n(ssl_errno, errbuf, 200); + ERR_error_string_n(ssl_errno, errbuf, STRERROR_BUFLEN); skygw_log_write(LE, "%s", errbuf); From 5aa882fc7e91ced1ec01ae19807f3e06043c65dc Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Wed, 16 Sep 2015 10:24:17 +0300 Subject: [PATCH 17/38] Made documentation cleaner. --- Documentation/Getting-Started/Configuration-Guide.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/Getting-Started/Configuration-Guide.md b/Documentation/Getting-Started/Configuration-Guide.md index 0bb10617b..cf2770a11 100644 --- a/Documentation/Getting-Started/Configuration-Guide.md +++ b/Documentation/Getting-Started/Configuration-Guide.md @@ -267,10 +267,10 @@ Query OK, 0 rows affected (0.00 sec) Additionally, `GRANT SELECT` on the `mysql.db` table and `SHOW DATABASES` privileges are required in order to load databases name and grants suitable for database name authorization. ``` -MariaDB [(none)]> GRANT SELECT ON mysql.db TO 'username'@'maxscalehost'; +MariaDB [(none)]> GRANT SELECT ON mysql.db TO 'maxscale'@'maxscalehost'; Query OK, 0 rows affected (0.00 sec) -MariaDB [(none)]> GRANT SHOW DATABASES ON *.* TO 'username'@'maxscalehost'; +MariaDB [(none)]> GRANT SHOW DATABASES ON *.* TO 'maxscale'@'maxscalehost'; Query OK, 0 rows affected (0.00 sec) ``` From fe400e7041519c6c1989ed723a47d98dae271d9a Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Wed, 16 Sep 2015 11:09:53 +0300 Subject: [PATCH 18/38] Fixed service permission checks failing without SELECT privilege on mysql.db for the servuce user. --- server/core/dbusers.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/core/dbusers.c b/server/core/dbusers.c index d55ca23b1..a30c82878 100644 --- a/server/core/dbusers.c +++ b/server/core/dbusers.c @@ -2428,9 +2428,8 @@ bool check_service_permissions(SERVICE* service) { if(mysql_errno(mysql) == ER_TABLEACCESS_DENIED_ERROR) { - skygw_log_write(LE,"%s: Error: User '%s' is missing SELECT privileges on mysql.db table. MySQL error message: %s", + skygw_log_write(LE,"%s: Warning: User '%s' is missing SELECT privileges on mysql.db table. Database name will be ignored in authentication. MySQL error message: %s", service->name,user,mysql_error(mysql)); - rval = false; } else { From ea2c45401087aa1b19895d10d0f60e4b908612e9 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Wed, 16 Sep 2015 11:10:47 +0300 Subject: [PATCH 19/38] Minor modification to documentation. --- Documentation/Monitors/MySQL-Monitor.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Documentation/Monitors/MySQL-Monitor.md b/Documentation/Monitors/MySQL-Monitor.md index 8e5bdeac5..d56082021 100644 --- a/Documentation/Monitors/MySQL-Monitor.md +++ b/Documentation/Monitors/MySQL-Monitor.md @@ -6,7 +6,7 @@ The MySQL Monitor is a monitoring module for MaxScale that monitors a Master-Sla ## Configuration -A minimal configuration for a monitor requires a set of servers for monitoring and a username and a password to connect to these servers. The user requires the REPLICATION CLIENT privilege to successfully monitor the state of the servers. +A minimal configuration for a monitor requires a set of servers for monitoring and a username and a password to connect to these servers. ``` [MySQL Monitor] @@ -18,6 +18,13 @@ passwd=mypwd ``` +The user requires the REPLICATION CLIENT privilege to successfully monitor the state of the servers. + +``` +MariaDB [(none)]> grant replication client on *.* to 'maxscale'@'maxscalehost'; +Query OK, 0 rows affected (0.00 sec) +``` + ## Optional parameters for all monitors Here are optional parameters that are common for all the monitors. From 6c75e09f292af7069133de51297712bab3c5fee7 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Wed, 16 Sep 2015 11:52:33 +0300 Subject: [PATCH 20/38] Fixed multi-source replication not working. --- server/modules/monitor/mysql_mon.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/modules/monitor/mysql_mon.c b/server/modules/monitor/mysql_mon.c index e31f2d57a..1dc8a7132 100644 --- a/server/modules/monitor/mysql_mon.c +++ b/server/modules/monitor/mysql_mon.c @@ -324,7 +324,7 @@ static inline bool connect_to_db(MONITOR* mon,MONITOR_SERVERS *database) static inline void monitor_mysql100_db(MONITOR_SERVERS* database) { - bool isslave = false; + int isslave = false; MYSQL_RES* result; MYSQL_ROW row; From 6ad89cb986b4e952e9dd901ef487283b02eca95d Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Wed, 16 Sep 2015 13:36:02 +0300 Subject: [PATCH 21/38] Removed boolean value assignment to an integer variable. --- server/modules/monitor/mysql_mon.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/modules/monitor/mysql_mon.c b/server/modules/monitor/mysql_mon.c index 1dc8a7132..4bb8a59b5 100644 --- a/server/modules/monitor/mysql_mon.c +++ b/server/modules/monitor/mysql_mon.c @@ -324,7 +324,7 @@ static inline bool connect_to_db(MONITOR* mon,MONITOR_SERVERS *database) static inline void monitor_mysql100_db(MONITOR_SERVERS* database) { - int isslave = false; + int isslave = 0; MYSQL_RES* result; MYSQL_ROW row; From d192cee0a1c72fa6693fa5c349e97921384d37f2 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Fri, 18 Sep 2015 09:49:22 +0200 Subject: [PATCH 22/38] Fixed Log message to 2015 Fixed Log message to 2015 --- server/core/gateway.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/core/gateway.c b/server/core/gateway.c index f7d639f17..ed1a3d0d4 100644 --- a/server/core/gateway.c +++ b/server/core/gateway.c @@ -1903,7 +1903,7 @@ int main(int argc, char **argv) } LOGIF(LM, (skygw_log_write( LOGFILE_MESSAGE, - "MariaDB Corporation MaxScale %s (C) MariaDB Corporation Ab 2013-2014", + "MariaDB Corporation MaxScale %s (C) MariaDB Corporation Ab 2013-2015", MAXSCALE_VERSION))); LOGIF(LM, (skygw_log_write( LOGFILE_MESSAGE, @@ -2569,4 +2569,5 @@ void write_child_exit_code(int fd, int code) /** Notify the parent process that an error has occurred */ write(fd, &code, sizeof (int)); close(fd); -} \ No newline at end of file +} + From 2a2582067a5a7cb670d578e90da8c55783d34cb4 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Fri, 18 Sep 2015 11:06:03 +0200 Subject: [PATCH 23/38] Added fix blr_slave_register reply Added fix blr_slave_register reply --- server/modules/routing/binlog/blr_slave.c | 52 ++++++++++------------- 1 file changed, 22 insertions(+), 30 deletions(-) diff --git a/server/modules/routing/binlog/blr_slave.c b/server/modules/routing/binlog/blr_slave.c index 6bd87d044..99d361afc 100644 --- a/server/modules/routing/binlog/blr_slave.c +++ b/server/modules/routing/binlog/blr_slave.c @@ -1158,20 +1158,12 @@ int slen; ptr += 2; slave->rank = extract_field(ptr, 32); - /* - * Now construct a response - */ - if ((resp = gwbuf_alloc(11)) == NULL) - return 0; - ptr = GWBUF_DATA(resp); - encode_value(ptr, 7, 24); // Payload length - ptr += 3; - *ptr++ = 1; // Sequence number - encode_value(ptr, 0, 24); - ptr += 3; - encode_value(ptr, slave->serverid, 32); slave->state = BLRS_REGISTERED; - return slave->dcb->func.write(slave->dcb, resp); + + /* + * Send OK response + */ + return blr_slave_send_ok(router, slave); } /** @@ -2094,31 +2086,31 @@ blr_slave_disconnect_all(ROUTER_INSTANCE *router, ROUTER_SLAVE *slave) return 1; } + /** - * Send a MySQL OK packet to the DCB + * Send a MySQL OK packet to the slave backend + * + * @param router The binlog router instance + * @param slave The slave server to which we are sending the response * - * @param dcb The DCB to send the OK packet to * @return result of a write call, non-zero if write was successful */ + static int blr_slave_send_ok(ROUTER_INSTANCE* router, ROUTER_SLAVE* slave) { GWBUF *pkt; uint8_t *ptr; +uint8_t ok_packet[] = {7, 0, 0, // Payload length + 1, // Seqno, + 0, // OK, + 0, 0, 2, 0, 0, 0}; - if ((pkt = gwbuf_alloc(11)) == NULL) - return 0; - ptr = GWBUF_DATA(pkt); - *ptr++ = 7; // Payload length - *ptr++ = 0; - *ptr++ = 0; - *ptr++ = 1; // Seqno - *ptr++ = 0; // ok - *ptr++ = 0; - *ptr++ = 0; - *ptr++ = 2; - *ptr++ = 0; - *ptr++ = 0; - *ptr++ = 0; - return slave->dcb->func.write(slave->dcb, pkt); + if ((pkt = gwbuf_alloc(sizeof(ok_packet))) == NULL) + return 0; + + memcpy(GWBUF_DATA(pkt), ok_packet, sizeof(ok_packet)); + + return slave->dcb->func.write(slave->dcb, pkt); } + From 67c9f306320f6ee0bcc6433b1e6a95265c78940b Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Fri, 18 Sep 2015 15:27:36 +0200 Subject: [PATCH 24/38] Added missing NULL byte for resource Added missing NULL byte for resource --- server/core/dbusers.c | 1 + 1 file changed, 1 insertion(+) diff --git a/server/core/dbusers.c b/server/core/dbusers.c index a30c82878..af663d1df 100644 --- a/server/core/dbusers.c +++ b/server/core/dbusers.c @@ -2174,6 +2174,7 @@ int tmp; free(dbkey); return NULL; } + dbkey->resource[tmp] = 0; // NULL Terminate } else // NULL is valid, so represent with a length of -1 { From 8d5ca1137498df8a69b1aeec2fea9aed5e7dd765 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Fri, 18 Sep 2015 16:05:22 +0200 Subject: [PATCH 25/38] Fixed: spinlock_release is before free Fixed: spinlock_release is before free --- server/modules/routing/binlog/blr_file.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/server/modules/routing/binlog/blr_file.c b/server/modules/routing/binlog/blr_file.c index 36b0996ae..907c9397e 100644 --- a/server/modules/routing/binlog/blr_file.c +++ b/server/modules/routing/binlog/blr_file.c @@ -596,9 +596,14 @@ blr_close_binlog(ROUTER_INSTANCE *router, BLFILE *file) close(file->fd); file->fd = -1; } - spinlock_release(&file->lock); - if (file->refcnt == 0) + + if (file->refcnt == 0) { + spinlock_release(&file->lock); + free(file); + } else { + spinlock_release(&file->lock); + } } /** From 601eac5a5dd2a73bfdd451326e730c1ca6dd4bb1 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 18 Sep 2015 17:20:50 +0300 Subject: [PATCH 26/38] Fixed parent-child code being called in non-daemon mode. --- server/core/gateway.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/core/gateway.c b/server/core/gateway.c index ed1a3d0d4..5c2355bee 100644 --- a/server/core/gateway.c +++ b/server/core/gateway.c @@ -2000,7 +2000,8 @@ int main(int argc, char **argv) * Successful start, notify the parent process that it can exit. */ ss_dassert(rc == MAXSCALE_SHUTDOWN); - write_child_exit_code(daemon_pipe[1], rc); + if(daemon_mode) + write_child_exit_code(daemon_pipe[1], rc); MaxScaleStarted = time(0); /*< From 2c12ba3d16a3d174500c1a32d8609bf7aec61b1e Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Mon, 21 Sep 2015 08:44:33 +0200 Subject: [PATCH 27/38] Various fixes Various fixes for initialisation and packet size allocation --- server/modules/routing/binlog/blr_file.c | 2 +- server/modules/routing/binlog/blr_slave.c | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/server/modules/routing/binlog/blr_file.c b/server/modules/routing/binlog/blr_file.c index 907c9397e..fa39aca02 100644 --- a/server/modules/routing/binlog/blr_file.c +++ b/server/modules/routing/binlog/blr_file.c @@ -334,7 +334,7 @@ BLFILE *file; return file; } - if ((file = (BLFILE *)malloc(sizeof(BLFILE))) == NULL) + if ((file = (BLFILE *)calloc(1, sizeof(BLFILE))) == NULL) { spinlock_release(&router->fileslock); return NULL; diff --git a/server/modules/routing/binlog/blr_slave.c b/server/modules/routing/binlog/blr_slave.c index 99d361afc..dd53eed7a 100644 --- a/server/modules/routing/binlog/blr_slave.c +++ b/server/modules/routing/binlog/blr_slave.c @@ -463,8 +463,9 @@ int query_len; "%s: Expected DISCONNECT SERVER $server_id", router->service->name))); } else { + int serverid = atoi(word); free(query_text); - return blr_slave_disconnect_server(router, slave, atoi(word)); + return blr_slave_disconnect_server(router, slave, serverid); } } } @@ -1075,7 +1076,7 @@ ROUTER_SLAVE *sptr; sprintf(port, "%d", sptr->port); sprintf(master_id, "%d", router->serverid); sprintf(slave_uuid, "%s", sptr->uuid ? sptr->uuid : ""); - len = 5 + strlen(server_id) + strlen(host) + strlen(port) + len = 4 + strlen(server_id) + strlen(host) + strlen(port) + strlen(master_id) + strlen(slave_uuid) + 5; if ((pkt = gwbuf_alloc(len)) == NULL) return 0; @@ -1907,7 +1908,7 @@ int len, id_len, seqno = 2; strcpy(state, "not found"); id_len = strlen(serverid); - len = 5 + id_len + strlen(state) + 1; + len = 4 + (1 + id_len) + (1 + strlen(state)); if ((pkt = gwbuf_alloc(len)) == NULL) return 0; @@ -1918,7 +1919,7 @@ int len, id_len, seqno = 2; blr_slave_send_eof(router, slave, seqno++); ptr = GWBUF_DATA(pkt); - encode_value(ptr, id_len + 2 + strlen(state), 24); // Add length of data packet + encode_value(ptr, len - 4), 24); // Add length of data packet ptr += 3; *ptr++ = seqno++; // Sequence number in response From de8e54da8b1fcd97a646efedd36eb5375c68028b Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Mon, 21 Sep 2015 08:52:39 +0200 Subject: [PATCH 28/38] Fix for compile error Fix for compile error --- server/modules/routing/binlog/blr_slave.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/modules/routing/binlog/blr_slave.c b/server/modules/routing/binlog/blr_slave.c index dd53eed7a..e92beb277 100644 --- a/server/modules/routing/binlog/blr_slave.c +++ b/server/modules/routing/binlog/blr_slave.c @@ -1919,7 +1919,7 @@ int len, id_len, seqno = 2; blr_slave_send_eof(router, slave, seqno++); ptr = GWBUF_DATA(pkt); - encode_value(ptr, len - 4), 24); // Add length of data packet + encode_value(ptr, len - 4, 24); // Add length of data packet ptr += 3; *ptr++ = seqno++; // Sequence number in response From 561891aac5f99688855ed8298259184328cfeef7 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Mon, 21 Sep 2015 13:41:22 +0300 Subject: [PATCH 29/38] Fix for double free, possible fix for bug601. In the end of execute_sescmd_in_backend the buffer was consumed in case the protocol function failed. Or actually if it returned something else but 1. In the case of mysql_backend, the buffer is always freed when authorizing and either consumed or placed on the dcb writequeue when the data is written. That is, it is never ok to consume the buffer in this function. The end-result is likely to be an abort. --- server/modules/routing/readwritesplit/readwritesplit.c | 1 - 1 file changed, 1 deletion(-) diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index 42b64a34c..9a322d1f3 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -4243,7 +4243,6 @@ static bool execute_sescmd_in_backend( } else { - while((buf = GWBUF_CONSUME_ALL(buf)) != NULL); succp = false; } return_succp: From 988a8d7008009a36a394df71ab84a11bbaa970f5 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 17 Sep 2015 07:13:24 +0300 Subject: [PATCH 30/38] Fixed error messages not being printed and cleaned up the function. --- server/modules/protocol/mysql_client.c | 300 +++++++++++++------------ 1 file changed, 161 insertions(+), 139 deletions(-) diff --git a/server/modules/protocol/mysql_client.c b/server/modules/protocol/mysql_client.c index 87b812c06..8219c7119 100644 --- a/server/modules/protocol/mysql_client.c +++ b/server/modules/protocol/mysql_client.c @@ -1331,164 +1331,186 @@ return_1: } /** - * set listener for mysql protocol, retur 1 on success and 0 in failure + * Bind the DCB to a network port or a UNIX Domain Socket. + * @param listen_dcb Listener DCB + * @param config_bind Bind address in either IP:PORT format for network sockets or PATH for UNIX Domain Sockets + * @return 1 on success, 0 on error */ -int gw_MySQLListener( - DCB *listen_dcb, - char *config_bind) +int gw_MySQLListener(DCB *listen_dcb, + char *config_bind) { - int l_so; - int syseno = 0; - struct sockaddr_in serv_addr; - struct sockaddr_un local_addr; - struct sockaddr *current_addr; - int one = 1; - int rc; - bool is_tcp = false; - memset(&serv_addr,0,sizeof(serv_addr)); - memset(&local_addr,0,sizeof(local_addr)); + int l_so; + struct sockaddr_in serv_addr; + struct sockaddr_un local_addr; + struct sockaddr *current_addr; + int one = 1; + int rc; + bool is_tcp = false; + memset(&serv_addr, 0, sizeof (serv_addr)); + memset(&local_addr, 0, sizeof (local_addr)); - if (strchr(config_bind, '/')) { - char *tmp = strrchr(config_bind, ':'); - if (tmp) - *tmp = '\0'; + if (strchr(config_bind, '/')) + { + char *tmp = strrchr(config_bind, ':'); + if (tmp) + *tmp = '\0'; - // UNIX socket create - if ((l_so = socket(AF_UNIX, SOCK_STREAM, 0)) < 0) { - char errbuf[STRERROR_BUFLEN]; - fprintf(stderr, - "\n* Error: can't create UNIX socket due " - "error %i, %s.\n\n\t", - errno, - strerror_r(errno, errbuf, sizeof(errbuf))); - return 0; - } - memset(&local_addr, 0, sizeof(local_addr)); - local_addr.sun_family = AF_UNIX; - strncpy(local_addr.sun_path, config_bind, sizeof(local_addr.sun_path) - 1); - - current_addr = (struct sockaddr *) &local_addr; - - } else { - /* MaxScale, as default, will bind on port 4406 */ - if (!parse_bindconfig(config_bind, 4406, &serv_addr)) { - fprintf(stderr, "Error in parse_bindconfig for [%s]\n", config_bind); - return 0; - } - // TCP socket create - if ((l_so = socket(AF_INET, SOCK_STREAM, 0)) < 0) { - char errbuf[STRERROR_BUFLEN]; - fprintf(stderr, - "\n* Error: can't create socket due " - "error %i, %s.\n\n\t", - errno, - strerror_r(errno, errbuf, sizeof(errbuf))); - return 0; - } - - current_addr = (struct sockaddr *) &serv_addr; - is_tcp = true; - } - - listen_dcb->fd = -1; - - // socket options - if((syseno = setsockopt(l_so, SOL_SOCKET, SO_REUSEADDR, (char *)&one, sizeof(one))) != 0){ - char errbuf[STRERROR_BUFLEN]; - LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR,"Error: Failed to set socket options. Error %d: %s", errno, strerror_r(errno, errbuf, sizeof(errbuf))))); - } - - if(is_tcp) - { + // UNIX socket create + if ((l_so = socket(AF_UNIX, SOCK_STREAM, 0)) < 0) + { char errbuf[STRERROR_BUFLEN]; - if((syseno = setsockopt(l_so, IPPROTO_TCP, TCP_NODELAY, (char *)&one, sizeof(one))) != 0){ - LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR,"Error: Failed to set socket options. Error %d: %s", errno, strerror_r(errno, errbuf, sizeof(errbuf))))); - } - } - // set NONBLOCKING mode - setnonblocking(l_so); + skygw_log_write(LE, + "Error: Can't create UNIX socket: %i, %s", + errno, + strerror_r(errno, errbuf, sizeof (errbuf))); + return 0; + } + memset(&local_addr, 0, sizeof (local_addr)); + local_addr.sun_family = AF_UNIX; + strncpy(local_addr.sun_path, config_bind, sizeof (local_addr.sun_path) - 1); - /* get the right socket family for bind */ - switch (current_addr->sa_family) { - case AF_UNIX: - rc = unlink(config_bind); - if ( (rc == -1) && (errno!=ENOENT) ) { - fprintf(stderr, "Error unlink Unix Socket %s\n", config_bind); - } + current_addr = (struct sockaddr *) &local_addr; - if (bind(l_so, (struct sockaddr *) &local_addr, sizeof(local_addr)) < 0) { - char errbuf[STRERROR_BUFLEN]; - fprintf(stderr, - "\n* Bind failed due error %i, %s.\n", - errno, - strerror_r(errno, errbuf, sizeof(errbuf))); - fprintf(stderr, "* Can't bind to %s\n\n", config_bind); - close(l_so); - return 0; - } + } + else + { + /* This is partially dead code, MaxScale will never start without explicit + * ports defined for all listeners. Thus the default port is never used. + */ + if (!parse_bindconfig(config_bind, 4406, &serv_addr)) + { + skygw_log_write(LE, "Error in parse_bindconfig for [%s]", config_bind); + return 0; + } - /* set permission for all users */ - if (chmod(config_bind, 0777) < 0) { - char errbuf[STRERROR_BUFLEN]; - fprintf(stderr, - "\n* chmod failed for %s due error %i, %s.\n\n", - config_bind, - errno, - strerror_r(errno, errbuf, sizeof(errbuf))); - } + /** Create the TCP socket */ + if ((l_so = socket(AF_INET, SOCK_STREAM, 0)) < 0) + { + char errbuf[STRERROR_BUFLEN]; + skygw_log_write(LE, + "Error: Can't create socket: %i, %s", + errno, + strerror_r(errno, errbuf, sizeof (errbuf))); + return 0; + } - break; + current_addr = (struct sockaddr *) &serv_addr; + is_tcp = true; + } - case AF_INET: - if (bind(l_so, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) { - char errbuf[STRERROR_BUFLEN]; - fprintf(stderr, - "\n* Bind failed due error %i, %s.\n", - errno, - strerror_r(errno, errbuf, sizeof(errbuf))); - fprintf(stderr, "* Can't bind to %s\n\n", config_bind); - close(l_so); - return 0; - } - break; + listen_dcb->fd = -1; - default: - fprintf(stderr, "* Socket Family %i not supported\n", current_addr->sa_family); - close(l_so); - return 0; - } + // socket options + if (setsockopt(l_so, SOL_SOCKET, SO_REUSEADDR, (char *) &one, sizeof (one)) != 0) + { + char errbuf[STRERROR_BUFLEN]; + LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR, "Error: Failed to set socket options. Error %d: %s", errno, strerror_r(errno, errbuf, sizeof (errbuf))))); + } - rc = listen(l_so, 10 * SOMAXCONN); + if (is_tcp) + { + char errbuf[STRERROR_BUFLEN]; + if (setsockopt(l_so, IPPROTO_TCP, TCP_NODELAY, (char *) &one, sizeof (one)) != 0) + { + LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR, "Error: Failed to set socket options. Error %d: %s", errno, strerror_r(errno, errbuf, sizeof (errbuf))))); + } + } + // set NONBLOCKING mode + if (setnonblocking(l_so) != 0) + { + skygw_log_write(LE, "Error: Failed to set socket to non-blocking mode."); + close(l_so); + return 0; + } - if (rc == 0) { - LOGIF(LM, (skygw_log_write_flush(LOGFILE_MESSAGE,"Listening MySQL connections at %s", config_bind))); - } else { - int eno = errno; - errno = 0; + /* get the right socket family for bind */ + switch (current_addr->sa_family) + { + case AF_UNIX: + rc = unlink(config_bind); + if ((rc == -1) && (errno != ENOENT)) + { char errbuf[STRERROR_BUFLEN]; - fprintf(stderr, - "\n* Failed to start listening MySQL due error %d, %s\n\n", - eno, - strerror_r(eno, errbuf, sizeof(errbuf))); - close(l_so); + skygw_log_write(LE, "Error: Failed to unlink Unix Socket %s: %d %s", + config_bind, errno, strerror_r(errno, errbuf, sizeof (errbuf))); + } + + if (bind(l_so, (struct sockaddr *) &local_addr, sizeof (local_addr)) < 0) + { + char errbuf[STRERROR_BUFLEN]; + skygw_log_write(LE, + "Error: Failed to bind to UNIX Domain socket '%s': %i, %s", + config_bind, + errno, + strerror_r(errno, errbuf, sizeof (errbuf))); + close(l_so); return 0; - } - // assign l_so to dcb - listen_dcb->fd = l_so; + } - // add listening socket to poll structure - if (poll_add_dcb(listen_dcb) == -1) { - fprintf(stderr, - "\n* MaxScale encountered system limit while " - "attempting to register on an epoll instance.\n\n"); - return 0; - } + /* set permission for all users */ + if (chmod(config_bind, 0777) < 0) + { + char errbuf[STRERROR_BUFLEN]; + skygw_log_write(LE, + "Error: Failed to change permissions on UNIX Domain socket '%s': %i, %s", + config_bind, + errno, + strerror_r(errno, errbuf, sizeof (errbuf))); + } + + break; + + case AF_INET: + if (bind(l_so, (struct sockaddr *) &serv_addr, sizeof (serv_addr)) < 0) + { + char errbuf[STRERROR_BUFLEN]; + skygw_log_write(LE, + "Error: Failed to bind on '%s': %i, %s", + config_bind, + errno, + strerror_r(errno, errbuf, sizeof (errbuf))); + close(l_so); + return 0; + } + break; + + default: + skygw_log_write(LE, "Error: Socket Family %i not supported\n", current_addr->sa_family); + close(l_so); + return 0; + } + + if (listen(l_so, 10 * SOMAXCONN) != 0) + { + char errbuf[STRERROR_BUFLEN]; + skygw_log_write(LE, + "Failed to start listening on '%s': %d, %s", + config_bind, + errno, + strerror_r(errno, errbuf, sizeof (errbuf))); + close(l_so); + return 0; + } + + LOGIF(LM, (skygw_log_write_flush(LOGFILE_MESSAGE, "Listening MySQL connections at %s", config_bind))); + + // assign l_so to dcb + listen_dcb->fd = l_so; + + // add listening socket to poll structure + if (poll_add_dcb(listen_dcb) != 0) + { + skygw_log_write(LE, + "MaxScale encountered system limit while " + "attempting to register on an epoll instance."); + return 0; + } #if defined(FAKE_CODE) - conn_open[l_so] = true; + conn_open[l_so] = true; #endif /* FAKE_CODE */ - listen_dcb->func.accept = gw_MySQLAccept; + listen_dcb->func.accept = gw_MySQLAccept; - return 1; + return 1; } From 11c8ef5b9251601537634be673019441253a3365 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Mon, 21 Sep 2015 13:56:56 +0300 Subject: [PATCH 31/38] Cleaned up code based on the review of the code. --- server/modules/protocol/mysql_client.c | 40 +++++++++++++++----------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/server/modules/protocol/mysql_client.c b/server/modules/protocol/mysql_client.c index 8219c7119..372a42f38 100644 --- a/server/modules/protocol/mysql_client.c +++ b/server/modules/protocol/mysql_client.c @@ -1346,8 +1346,8 @@ int gw_MySQLListener(DCB *listen_dcb, int one = 1; int rc; bool is_tcp = false; - memset(&serv_addr, 0, sizeof (serv_addr)); - memset(&local_addr, 0, sizeof (local_addr)); + memset(&serv_addr, 0, sizeof(serv_addr)); + memset(&local_addr, 0, sizeof(local_addr)); if (strchr(config_bind, '/')) { @@ -1362,12 +1362,12 @@ int gw_MySQLListener(DCB *listen_dcb, skygw_log_write(LE, "Error: Can't create UNIX socket: %i, %s", errno, - strerror_r(errno, errbuf, sizeof (errbuf))); + strerror_r(errno, errbuf, sizeof(errbuf))); return 0; } - memset(&local_addr, 0, sizeof (local_addr)); + memset(&local_addr, 0, sizeof(local_addr)); local_addr.sun_family = AF_UNIX; - strncpy(local_addr.sun_path, config_bind, sizeof (local_addr.sun_path) - 1); + strncpy(local_addr.sun_path, config_bind, sizeof(local_addr.sun_path) - 1); current_addr = (struct sockaddr *) &local_addr; @@ -1390,7 +1390,7 @@ int gw_MySQLListener(DCB *listen_dcb, skygw_log_write(LE, "Error: Can't create socket: %i, %s", errno, - strerror_r(errno, errbuf, sizeof (errbuf))); + strerror_r(errno, errbuf, sizeof(errbuf))); return 0; } @@ -1401,18 +1401,24 @@ int gw_MySQLListener(DCB *listen_dcb, listen_dcb->fd = -1; // socket options - if (setsockopt(l_so, SOL_SOCKET, SO_REUSEADDR, (char *) &one, sizeof (one)) != 0) + if (setsockopt(l_so, SOL_SOCKET, SO_REUSEADDR, (char *) &one, sizeof(one)) != 0) { char errbuf[STRERROR_BUFLEN]; - LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR, "Error: Failed to set socket options. Error %d: %s", errno, strerror_r(errno, errbuf, sizeof (errbuf))))); + LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR, + "Error: Failed to set socket options. Error %d: %s", + errno, + strerror_r(errno, errbuf, sizeof(errbuf))))); } if (is_tcp) { char errbuf[STRERROR_BUFLEN]; - if (setsockopt(l_so, IPPROTO_TCP, TCP_NODELAY, (char *) &one, sizeof (one)) != 0) + if (setsockopt(l_so, IPPROTO_TCP, TCP_NODELAY, (char *) &one, sizeof(one)) != 0) { - LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR, "Error: Failed to set socket options. Error %d: %s", errno, strerror_r(errno, errbuf, sizeof (errbuf))))); + LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR, + "Error: Failed to set socket options. Error %d: %s", + errno, + strerror_r(errno, errbuf, sizeof(errbuf))))); } } // set NONBLOCKING mode @@ -1432,17 +1438,17 @@ int gw_MySQLListener(DCB *listen_dcb, { char errbuf[STRERROR_BUFLEN]; skygw_log_write(LE, "Error: Failed to unlink Unix Socket %s: %d %s", - config_bind, errno, strerror_r(errno, errbuf, sizeof (errbuf))); + config_bind, errno, strerror_r(errno, errbuf, sizeof(errbuf))); } - if (bind(l_so, (struct sockaddr *) &local_addr, sizeof (local_addr)) < 0) + if (bind(l_so, (struct sockaddr *) &local_addr, sizeof(local_addr)) < 0) { char errbuf[STRERROR_BUFLEN]; skygw_log_write(LE, "Error: Failed to bind to UNIX Domain socket '%s': %i, %s", config_bind, errno, - strerror_r(errno, errbuf, sizeof (errbuf))); + strerror_r(errno, errbuf, sizeof(errbuf))); close(l_so); return 0; } @@ -1455,20 +1461,20 @@ int gw_MySQLListener(DCB *listen_dcb, "Error: Failed to change permissions on UNIX Domain socket '%s': %i, %s", config_bind, errno, - strerror_r(errno, errbuf, sizeof (errbuf))); + strerror_r(errno, errbuf, sizeof(errbuf))); } break; case AF_INET: - if (bind(l_so, (struct sockaddr *) &serv_addr, sizeof (serv_addr)) < 0) + if (bind(l_so, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) { char errbuf[STRERROR_BUFLEN]; skygw_log_write(LE, "Error: Failed to bind on '%s': %i, %s", config_bind, errno, - strerror_r(errno, errbuf, sizeof (errbuf))); + strerror_r(errno, errbuf, sizeof(errbuf))); close(l_so); return 0; } @@ -1487,7 +1493,7 @@ int gw_MySQLListener(DCB *listen_dcb, "Failed to start listening on '%s': %d, %s", config_bind, errno, - strerror_r(errno, errbuf, sizeof (errbuf))); + strerror_r(errno, errbuf, sizeof(errbuf))); close(l_so); return 0; } From 338b870cd1aa92c653b90fe9e1665d83101e4c86 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Mon, 21 Sep 2015 17:07:37 +0300 Subject: [PATCH 32/38] Fix to MXS-373: https://mariadb.atlassian.net/browse/MXS-373 The log manager is initialized only once and skygw_log_sync_all now checks if the log manager has been successfully started before interacting with the log manager --- log_manager/log_manager.cc | 23 +++++++++++++++++++---- server/core/gateway.c | 1 - 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/log_manager/log_manager.cc b/log_manager/log_manager.cc index fd2a74195..157c9ea04 100644 --- a/log_manager/log_manager.cc +++ b/log_manager/log_manager.cc @@ -109,6 +109,10 @@ static bool flushall_done_flag; static int default_log_augmentation = 0; static int log_augmentation = default_log_augmentation; +/** This is used to detect if the initialization of the log manager has failed + * and that it isn't initialized again after a failure has occurred. */ +static bool fatal_error = false; + /** Writer thread structure */ struct filewriter_st { #if defined(SS_DEBUG) @@ -1681,7 +1685,7 @@ static bool logmanager_register( * and its members which would probabaly lead to NULL pointer * reference. */ - if (!writep) { + if (!writep || fatal_error) { succp = false; goto return_succp; } @@ -1709,6 +1713,11 @@ static bool logmanager_register( } return_succp: + + if(!succp) + { + fatal_error = true; + } release_lock(&lmlock); return succp; } @@ -3232,9 +3241,15 @@ void flushall_logfiles(bool flush) void skygw_log_sync_all(void) { if(!use_stdout)skygw_log_write(LOGFILE_TRACE,"Starting log flushing to disk."); - flushall_logfiles(true); - skygw_message_send(lm->lm_logmes); - skygw_message_wait(lm->lm_clientmes); + + /** If initialization of the log manager has not been done, lm pointer can be + * NULL. */ + if(lm) + { + flushall_logfiles(true); + skygw_message_send(lm->lm_logmes); + skygw_message_wait(lm->lm_clientmes); + } } /** diff --git a/server/core/gateway.c b/server/core/gateway.c index 5c2355bee..0907d6365 100644 --- a/server/core/gateway.c +++ b/server/core/gateway.c @@ -1648,7 +1648,6 @@ int main(int argc, char **argv) if (!resolve_maxscale_conf_fname(&cnf_file_path, pathbuf, cnf_file_arg)) { - ss_dassert(cnf_file_path == NULL); rc = MAXSCALE_BADCONFIG; goto return_main; } From 065e2527679c6f8f896ac028f8effd07cdea4369 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 22 Sep 2015 07:36:26 +0300 Subject: [PATCH 33/38] Removed redundant text from documentation. --- .../Getting-Started/Configuration-Guide.md | 89 ++----------------- 1 file changed, 9 insertions(+), 80 deletions(-) diff --git a/Documentation/Getting-Started/Configuration-Guide.md b/Documentation/Getting-Started/Configuration-Guide.md index cf2770a11..c3a608b2d 100644 --- a/Documentation/Getting-Started/Configuration-Guide.md +++ b/Documentation/Getting-Started/Configuration-Guide.md @@ -736,6 +736,12 @@ Parsing within the router adds overhead to the cost of routing and makes this ty Currently a small number of query routers are available, these are in different stages of completion and offer different facilities. +* [ReadConnRoute](../Routers/ReadConnRoute.md) +* [ReadWriteSplit](../Routers/ReadWriteSplit.md) +* [SchemaRouter](../Routers/SchemaRouter.md) + +In addition to these routing modules, the binlogrouter module can act as a binary log proxy between a master and slave servers. + #### Readconnroute This is a connection based query router that was originally targeted at environments in which the clients already performed splitting of read and write queries into separate connections. @@ -926,86 +932,7 @@ The router stores all of the executed session commands so that in case of a slav Read/Write Split router-specific settings are specified in the configuration file of MaxScale in its specific section. The section can be freely named but the name is used later as a reference from listener section. -The configuration consists of mandatory and optional parameters. - -###### Mandatory parameters - -**`type`** specifies the type of service. For **readwritesplit** module the type is `router`: - - type=router - -**`service`** specifies the router module to be used. For **readwritesplit** the value is `readwritesplit`: - - service=readwritesplit - -**`servers`** provides a list of servers, which must include one master and available slaves: - - servers=server1,server2,server3 - -**NOTE: Each server on the list must have its own section in the configuration file where it is defined.** - -**`user`** is the username the router session uses for accessing backends in order to load the content of the `mysql.user` table (and `mysql.db` and database names as well) and optionally for creating, and using `maxscale_schema.replication_heartbeat` table. - -**`passwd`** specifies corresponding password for the user. Syntax for user and passwd is: - -``` -user= -passwd= -``` - -###### Optional parameters - -**`max_slave_connections`** sets the maximum number of slaves a router session uses at any moment. Default value is `1`. - - max_slave_connections= - -**`max_slave_replication_lag`** specifies how many seconds a slave is allowed to be behind the master. If the lag is bigger than configured value a slave can't be used for routing. - - max_slave_replication_lag= - -This applies to Master/Slave replication with MySQL monitor and `detect_replication_lag=1` options set. -Please note max_slave_replication_lag must be greater than monitor interval. - -**`router_options`** may include multiple **readwritesplit**-specific options. Values are either singular or parameter-value pairs. Currently available is a single option which specifies the criteria used in slave selection both in initialization of router session and per each query. Note that due to the current monitor implementation, the value specified here should be ** + 1. - - options=slave_selection_criteria= - -where ** is one of the following: - -* `LEAST_GLOBAL_CONNECTIONS`, the slave with least connections in total -* `LEAST_ROUTER_CONNECTIONS`, the slave with least connections from this router -* `LEAST_BEHIND_MASTER`, the slave with smallest replication lag -* `LEAST_CURRENT_OPERATIONS` (default), the slave with least active operations - -**`use_sql_variables_in`** specifies where should queries, which read session variable, be routed. The syntax for `use_sql_variable_in` is: - - use_sql_variables_in=[master|all] - -When value all is used, queries reading session variables can be routed to any available slave (depending on selection criteria). Note, that queries modifying session variables are routed to all backend servers by default, excluding write queries with embedded session variable modifications, such as: - - INSERT INTO test.t1 VALUES (@myid:=@myid+1) - -In above-mentioned case the user-defined variable would only be updated in the master where query would be routed due to `INSERT` statement. - -**`max_sescmd_history`** sets a limit on how many session commands each session can execute before the connection is closed. The default is an unlimited number of session commands. - - max_sescmd_history=1500 - -When a limitation is set, it effectively creates a cap on the session's memory consumption. This might be useful if connection pooling is used and the sessions use large amounts of session commands. - -**`disable_sescmd_history`** disables the session command history. This way nothing is stored and if a slave server fails and a new one is taken in its stead, the session on that server will be in an inconsistent state compared to the master server. Disabling session command history will allow connection pooling without causing a constant growth in the memory consumption. - -``` -# Disable the session command history -disable_sescmd_history=true -``` - -**`disable_slave_recovery`** disables the recovery and replacement of slave servers. If this option is enabled and a connection to a slave server in use is lost, no replacement slave will be taken. This allows the safe use of session state modifying statements when the session command history is disabled. This is mostly intended to be used with the `disable_sescmd_history` option enabled. - -``` -# Disable the session command history -disable_slave_recovery=true -``` +The configuration consists of mandatory and optional parameters. For a complete list of these, please read the [ReadWriteSplit](../Routers/ReadWriteSplit.md) documentation. An example of Read/Write Split router configuration : @@ -1024,6 +951,8 @@ filters=qla|fetch|from In addition to this, readwritesplit needs configuration for a listener, for all servers listed, and for each filter. Listener, server - and filter configurations are described in their own sections in this document. +An important parameter is the `max_slave_connections=50%` parameter. This sets the number of slaves each client connection will use. With the default values, client connections will only use a single slave for reads. For example, setting the parameter value to 100% will use all available slaves and read queries will be balanced evenly across all slaves. Changing the `max_slave_conections` parameter and `slave_selection_criteria` router option allows you to change the way MaxScale will balance reads. For more information about the `slave_selection_criteria` router option, please read the ReadWriteSplit documentation. + Below is a listener example for the "RWSplit Service" defined above: ``` From 618a641b965da1f5a4799e473d6a473b4948e58c Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Wed, 23 Sep 2015 08:00:58 +0300 Subject: [PATCH 34/38] Reformatted user authentication queries as multi-line strings instead of one long string. --- server/core/dbusers.c | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/server/core/dbusers.c b/server/core/dbusers.c index af663d1df..e95ebf70c 100644 --- a/server/core/dbusers.c +++ b/server/core/dbusers.c @@ -72,20 +72,39 @@ WHERE user.user IS NOT NULL AND user.user <> ''" #else -# define LOAD_MYSQL_USERS_QUERY "SELECT user, host, password, concat(user,host,password,Select_priv) AS userdata, Select_priv AS anydb FROM mysql.user WHERE user IS NOT NULL AND user <> ''" +# define LOAD_MYSQL_USERS_QUERY "SELECT \ + user, host, password, concat(user,host,password,Select_priv) AS userdata, \ + Select_priv AS anydb FROM mysql.user WHERE user IS NOT NULL AND user <> ''" #endif #define MYSQL_USERS_COUNT "SELECT COUNT(1) AS nusers FROM mysql.user" #define MYSQL_USERS_WITH_DB_ORDER " ORDER BY host DESC" -#define LOAD_MYSQL_USERS_WITH_DB_QUERY "SELECT user.user AS user,user.host AS host,user.password AS password,concat(user.user,user.host,user.password,user.Select_priv,IFNULL(db,'')) AS userdata, user.Select_priv AS anydb,db.db AS db FROM mysql.user LEFT JOIN mysql.db ON user.user=db.user AND user.host=db.host WHERE user.user IS NOT NULL" MYSQL_USERS_WITH_DB_ORDER +#define LOAD_MYSQL_USERS_WITH_DB_QUERY "SELECT \ + user.user AS user, \ + user.host AS host, \ + user.password AS password, \ + concat(user.user,user.host,user.password,user.Select_priv,IFNULL(db,'')) AS userdata, \ + user.Select_priv AS anydb, \ + db.db AS db \ + FROM mysql.user LEFT JOIN mysql.db \ + ON user.user=db.user AND user.host=db.host \ + WHERE user.user IS NOT NULL" MYSQL_USERS_WITH_DB_ORDER #define MYSQL_USERS_WITH_DB_COUNT "SELECT COUNT(1) AS nusers_db FROM (" LOAD_MYSQL_USERS_WITH_DB_QUERY ") AS tbl_count" -#define LOAD_MYSQL_USERS_WITH_DB_QUERY_NO_ROOT "SELECT * FROM (" LOAD_MYSQL_USERS_WITH_DB_QUERY ") AS t1 WHERE user NOT IN ('root')" MYSQL_USERS_WITH_DB_ORDER +#define LOAD_MYSQL_USERS_WITH_DB_QUERY_NO_ROOT "SELECT * \ + FROM (" LOAD_MYSQL_USERS_WITH_DB_QUERY ") AS t1 \ + WHERE user NOT IN ('root')" MYSQL_USERS_WITH_DB_ORDER -#define LOAD_MYSQL_DATABASE_NAMES "SELECT * FROM ( (SELECT COUNT(1) AS ndbs FROM INFORMATION_SCHEMA.SCHEMATA) AS tbl1, (SELECT GRANTEE,PRIVILEGE_TYPE from INFORMATION_SCHEMA.USER_PRIVILEGES WHERE privilege_type='SHOW DATABASES' AND REPLACE(GRANTEE, \'\\'\',\'\')=CURRENT_USER()) AS tbl2)" +#define LOAD_MYSQL_DATABASE_NAMES "SELECT * \ + FROM ( (SELECT COUNT(1) AS ndbs \ + FROM INFORMATION_SCHEMA.SCHEMATA) AS tbl1, \ + (SELECT GRANTEE,PRIVILEGE_TYPE from INFORMATION_SCHEMA.USER_PRIVILEGES \ + WHERE privilege_type='SHOW DATABASES' AND REPLACE(GRANTEE, \'\\'\',\'\')=CURRENT_USER()) AS tbl2)" -#define ERROR_NO_SHOW_DATABASES "%s: Unable to load database grant information, MaxScale authentication will proceed without including database permissions. To correct this GRANT SHOW DATABASES ON *.* privilege to the user %s." +#define ERROR_NO_SHOW_DATABASES "%s: Unable to load database grant information, \ + MaxScale authentication will proceed without including database permissions. \ + To correct this GRANT SHOW DATABASES ON *.* privilege to the user %s." /** Defined in log_manager.cc */ extern int lm_enabled_logfiles_bitmask; extern size_t log_ses_count[]; From f7946a05fa74044e2857ed28d906ec3d7257d23b Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Wed, 23 Sep 2015 14:05:02 +0200 Subject: [PATCH 35/38] Fix in extract_message Fix in extract_message --- server/modules/routing/binlog/blr.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/modules/routing/binlog/blr.c b/server/modules/routing/binlog/blr.c index 350b19988..e5bfbd23e 100644 --- a/server/modules/routing/binlog/blr.c +++ b/server/modules/routing/binlog/blr.c @@ -1057,7 +1057,8 @@ int len; return NULL; memcpy(rval, (char *)(errpkt->start) + 7, 6); rval[6] = ' '; - memcpy(&rval[7], (char *)(errpkt->start) + 13, len - 8); + /* message size is len - (1 byte field count + 2 bytes errno + 6 bytes status) */ + memcpy(&rval[7], (char *)(errpkt->start) + 13, len - 9); rval[len-2] = 0; return rval; } From 3b19acd1de8714bf6dfa2b7c95c60eee1a5be1aa Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Wed, 23 Sep 2015 15:05:16 +0300 Subject: [PATCH 36/38] Write queue emptied in dcb_final_free. NOTE: Invalid indentation for reduced conflicts with MXS-329. --- server/core/dcb.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index 5740d4ffa..6c836a6fd 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -374,6 +374,11 @@ dcb_final_free(DCB *dcb) GWBUF *queue = dcb->delayq; while ((queue = gwbuf_consume(queue, GWBUF_LENGTH(queue))) != NULL); } + if (dcb->writeq) { + GWBUF *queue = dcb->writeq; + while ((queue = gwbuf_consume(queue, GWBUF_LENGTH(queue))) != NULL); + dcb->writeq = NULL; + } if (dcb->dcb_readqueue) { GWBUF* queue = dcb->dcb_readqueue; @@ -3141,4 +3146,4 @@ void dcb_log_ssl_read_error(DCB *dcb, int ssl_errno, int rc) } } } -} \ No newline at end of file +} From 6a318836fb142340f82c04fd7fa8e873e05cb0f2 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Wed, 23 Sep 2015 15:11:50 +0300 Subject: [PATCH 37/38] Removed obsolete code. The read and write buffers are now emptied in dcb_final_free, so no need to do it explicitly here. --- server/modules/protocol/httpd.c | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/server/modules/protocol/httpd.c b/server/modules/protocol/httpd.c index db4467418..a9c9c3ded 100644 --- a/server/modules/protocol/httpd.c +++ b/server/modules/protocol/httpd.c @@ -383,16 +383,6 @@ int n_connect = 0; static int httpd_close(DCB *dcb) { - if(dcb->dcb_readqueue) - { - while((dcb->dcb_readqueue = GWBUF_CONSUME_ALL(dcb->dcb_readqueue))); - } - - if(dcb->writeq) - { - while((dcb->writeq = GWBUF_CONSUME_ALL(dcb->writeq))); - } - return 0; } From 2f0ef4ad31189dc1708aeadd94151c5cbb7f4aed Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 24 Sep 2015 14:35:48 +0300 Subject: [PATCH 38/38] Configuration files from older installations no longer overwrite existing files. --- etc/postrm.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/postrm.in b/etc/postrm.in index 7488ebe46..239715448 100755 --- a/etc/postrm.in +++ b/etc/postrm.in @@ -6,7 +6,7 @@ then rm -f /usr/lib/systemd/system/maxscale.service else # Copy and rename config from old location - if [ -f "/usr/local/mariadb-maxscale/etc/MaxScale.cnf" ] + if [ -f "/usr/local/mariadb-maxscale/etc/MaxScale.cnf" -a ! -f "/etc/maxscale.cnf" ] then cp "/usr/local/mariadb-maxscale/etc/MaxScale.cnf" "/etc/maxscale.cnf" fi