From 5c7b2a68e55d57f18bfd3c1fc824f636e7526f51 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Wed, 6 May 2015 12:19:18 +0200 Subject: [PATCH 001/100] mariadb10 compatibility test without GTID First implementation of mariadb10 compatibility test without GTID State machine to be modified for mysql5.6/mariadb10 compatibility router options for mariadb10 slave registration still missing --- server/modules/include/blr.h | 2 +- server/modules/routing/binlog/blr_master.c | 3 ++- server/modules/routing/binlog/blr_slave.c | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/server/modules/include/blr.h b/server/modules/include/blr.h index 62cda59a3..5c9c70181 100644 --- a/server/modules/include/blr.h +++ b/server/modules/include/blr.h @@ -395,7 +395,7 @@ static char *blrs_states[] = { "Created", "Unregistered", "Registered", #define ANONYMOUS_GTID_EVENT 0x22 #define PREVIOUS_GTIDS_EVENT 0x23 -#define MAX_EVENT_TYPE 0x23 +#define MAX_EVENT_TYPE 0xa3 /** * Binlog event flags diff --git a/server/modules/routing/binlog/blr_master.c b/server/modules/routing/binlog/blr_master.c index 2b164c349..70cd78686 100644 --- a/server/modules/routing/binlog/blr_master.c +++ b/server/modules/routing/binlog/blr_master.c @@ -448,7 +448,8 @@ char query[128]; GWBUF_CONSUME_ALL(router->saved_master.chksum2); router->saved_master.chksum2 = buf; blr_cache_response(router, "chksum2", buf); - buf = blr_make_query("SELECT @@GLOBAL.GTID_MODE"); + //buf = blr_make_query("SELECT @@GLOBAL.GTID_MODE"); + buf = blr_make_query("SET @mariadb_slave_capability=4); router->master_state = BLRM_GTIDMODE; router->master->func.write(router->master, buf); break; diff --git a/server/modules/routing/binlog/blr_slave.c b/server/modules/routing/binlog/blr_slave.c index 4de69b8d7..8a008e35f 100644 --- a/server/modules/routing/binlog/blr_slave.c +++ b/server/modules/routing/binlog/blr_slave.c @@ -369,7 +369,8 @@ int query_len; else if (strcasecmp(word, "@mariadb_slave_capability") == 0) { free(query_text); - return blr_slave_send_ok(router, slave); + return blr_slave_replay(router, slave, router->saved_master.gtid_mode); + //return blr_slave_send_ok(router, slave); } else if (strcasecmp(word, "@master_binlog_checksum") == 0) { From bc7cc2a4666a9ab8946b7134da072bf2703c002d Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 7 May 2015 12:56:58 +0300 Subject: [PATCH 002/100] Added variables for MariaDB 10 compatibility. --- server/modules/include/blr.h | 7 +++++-- server/modules/routing/binlog/blr.c | 5 +++++ server/modules/routing/binlog/blr_master.c | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/server/modules/include/blr.h b/server/modules/include/blr.h index 5c9c70181..48c48a194 100644 --- a/server/modules/include/blr.h +++ b/server/modules/include/blr.h @@ -233,6 +233,7 @@ typedef struct { GWBUF *selectvercom; /*< select @@version_comment */ GWBUF *selecthostname;/*< select @@hostname */ GWBUF *map; /*< select @@max_allowed_packet */ + GWBUF *mariadb10; /*< set @mariadb_slave_capability=4 */ uint8_t *fde_event; /*< Format Description Event */ int fde_len; /*< Length of fde_event */ } MASTER_RESPONSES; @@ -252,6 +253,7 @@ typedef struct router_instance { char *password; /*< Password to use with master */ char *fileroot; /*< Root of binlog filename */ bool master_chksum;/*< Does the master provide checksums */ + bool mariadb10_compat; /*< MariaDB 10.0 compatibility */ char *master_uuid; /*< UUID of the master */ DCB *master; /*< DCB for master connection */ DCB *client; /*< DCB for dummy client */ @@ -314,15 +316,16 @@ typedef struct router_instance { #define BLRM_MAP 0x0011 #define BLRM_REGISTER 0x0012 #define BLRM_BINLOGDUMP 0x0013 - +#define BLRM_MARIADB10 0x0014 #define BLRM_MAXSTATE 0x0013 +#define BLRM_MAXSTATE_MARIADB10 0x0014 static char *blrm_states[] = { "Unconnected", "Connecting", "Authenticated", "Timestamp retrieval", "Server ID retrieval", "HeartBeat Period setup", "binlog checksum config", "binlog checksum rerieval", "GTID Mode retrieval", "Master UUID retrieval", "Set Slave UUID", "Set Names latin1", "Set Names utf8", "select 1", "select version()", "select @@version_comment", "select @@hostname", - "select @@mx_allowed_packet", "Register slave", "Binlog Dump" }; + "select @@mx_allowed_packet", "Register slave", "Binlog Dump","Set MariaDB slave capability" }; #define BLRS_CREATED 0x0000 #define BLRS_UNREGISTERED 0x0001 diff --git a/server/modules/routing/binlog/blr.c b/server/modules/routing/binlog/blr.c index 2ba89689f..2f5a38bee 100644 --- a/server/modules/routing/binlog/blr.c +++ b/server/modules/routing/binlog/blr.c @@ -195,6 +195,7 @@ unsigned char *defuuid; inst->retry_backoff = 1; inst->binlogdir = NULL; inst->heartbeat = 300; // Default is every 5 minutes + inst->mariadb10_compat = false; inst->user = strdup(service->credentials.name); inst->password = strdup(service->credentials.authdata); @@ -282,6 +283,10 @@ unsigned char *defuuid; { inst->masterid = atoi(value); } + else if (strcmp(options[i], "mariadb10-compatibility") == 0) + { + inst->mariadb10_compat = config_truth_value(value); + } else if (strcmp(options[i], "filestem") == 0) { inst->fileroot = strdup(value); diff --git a/server/modules/routing/binlog/blr_master.c b/server/modules/routing/binlog/blr_master.c index 70cd78686..bc1e676d0 100644 --- a/server/modules/routing/binlog/blr_master.c +++ b/server/modules/routing/binlog/blr_master.c @@ -449,7 +449,7 @@ char query[128]; router->saved_master.chksum2 = buf; blr_cache_response(router, "chksum2", buf); //buf = blr_make_query("SELECT @@GLOBAL.GTID_MODE"); - buf = blr_make_query("SET @mariadb_slave_capability=4); + buf = blr_make_query("SET @mariadb_slave_capability=4"); router->master_state = BLRM_GTIDMODE; router->master->func.write(router->master, buf); break; From 8afa46b8b2436dd9bb3680b5642b1c0d3cec7f32 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 7 May 2015 13:00:34 +0300 Subject: [PATCH 003/100] Removed BLRM_MAXSTATE_MARIADB10 and set BLRM_MAXSTATE to 0x014 --- server/modules/include/blr.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/modules/include/blr.h b/server/modules/include/blr.h index 48c48a194..fa377c7e5 100644 --- a/server/modules/include/blr.h +++ b/server/modules/include/blr.h @@ -317,8 +317,7 @@ typedef struct router_instance { #define BLRM_REGISTER 0x0012 #define BLRM_BINLOGDUMP 0x0013 #define BLRM_MARIADB10 0x0014 -#define BLRM_MAXSTATE 0x0013 -#define BLRM_MAXSTATE_MARIADB10 0x0014 +#define BLRM_MAXSTATE 0x0014 static char *blrm_states[] = { "Unconnected", "Connecting", "Authenticated", "Timestamp retrieval", "Server ID retrieval", "HeartBeat Period setup", "binlog checksum config", From e9391ef48639c6607bfcd5472d8996cb74c41614 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Thu, 7 May 2015 15:16:37 +0200 Subject: [PATCH 004/100] MariaDB 10 optional compatibility MariaDB 10 optional compatibility with mariadb10-compatibility=1 --- server/modules/include/blr.h | 94 +++++++++++----------- server/modules/routing/binlog/blr_master.c | 21 ++++- server/modules/routing/binlog/blr_slave.c | 6 +- 3 files changed, 68 insertions(+), 53 deletions(-) diff --git a/server/modules/include/blr.h b/server/modules/include/blr.h index fa377c7e5..8aecd83f0 100644 --- a/server/modules/include/blr.h +++ b/server/modules/include/blr.h @@ -233,7 +233,7 @@ typedef struct { GWBUF *selectvercom; /*< select @@version_comment */ GWBUF *selecthostname;/*< select @@hostname */ GWBUF *map; /*< select @@max_allowed_packet */ - GWBUF *mariadb10; /*< set @mariadb_slave_capability=4 */ + GWBUF *mariadb10; /*< set @mariadb_slave_capability */ uint8_t *fde_event; /*< Format Description Event */ int fde_len; /*< Length of fde_event */ } MASTER_RESPONSES; @@ -242,55 +242,54 @@ typedef struct { * The per instance data for the router. */ typedef struct router_instance { - SERVICE *service; /*< Pointer to the service using this router */ - ROUTER_SLAVE *slaves; /*< Link list of all the slave connections */ - SPINLOCK lock; /*< Spinlock for the instance data */ - char *uuid; /*< UUID for the router to use w/master */ - int masterid; /*< Server ID of the master */ - int serverid; /*< Server ID to use with master */ - int initbinlog; /*< Initial binlog file number */ - char *user; /*< User name to use with master */ - char *password; /*< Password to use with master */ - char *fileroot; /*< Root of binlog filename */ - bool master_chksum;/*< Does the master provide checksums */ - bool mariadb10_compat; /*< MariaDB 10.0 compatibility */ - char *master_uuid; /*< UUID of the master */ - DCB *master; /*< DCB for master connection */ - DCB *client; /*< DCB for dummy client */ - SESSION *session; /*< Fake session for master connection */ - unsigned int master_state; /*< State of the master FSM */ - uint8_t lastEventReceived; - uint32_t lastEventTimestamp; /*< Timestamp from last event */ - GWBUF *residual; /*< Any residual binlog event */ - MASTER_RESPONSES saved_master; /*< Saved master responses */ - char *binlogdir; /*< The directory with the binlog files */ - SPINLOCK binlog_lock; /*< Lock to control update of the binlog position */ - char binlog_name[BINLOG_FNAMELEN+1]; + SERVICE *service; /*< Pointer to the service using this router */ + ROUTER_SLAVE *slaves; /*< Link list of all the slave connections */ + SPINLOCK lock; /*< Spinlock for the instance data */ + char *uuid; /*< UUID for the router to use w/master */ + int masterid; /*< Server ID of the master */ + int serverid; /*< Server ID to use with master */ + int initbinlog; /*< Initial binlog file number */ + char *user; /*< User name to use with master */ + char *password; /*< Password to use with master */ + char *fileroot; /*< Root of binlog filename */ + bool master_chksum; /*< Does the master provide checksums */ + bool mariadb10_compat; /*< MariaDB 10.0 compatibility */ + char *master_uuid; /*< UUID of the master */ + DCB *master; /*< DCB for master connection */ + DCB *client; /*< DCB for dummy client */ + SESSION *session; /*< Fake session for master connection */ + unsigned int master_state; /*< State of the master FSM */ + uint8_t lastEventReceived; + uint32_t lastEventTimestamp; /*< Timestamp from last event */ + GWBUF *residual; /*< Any residual binlog event */ + MASTER_RESPONSES saved_master; /*< Saved master responses */ + char *binlogdir; /*< The directory with the binlog files */ + SPINLOCK binlog_lock; /*< Lock to control update of the binlog position */ + char binlog_name[BINLOG_FNAMELEN+1]; /*< Name of the current binlog file */ - uint64_t binlog_position; + uint64_t binlog_position; /*< Current binlog position */ - int binlog_fd; /*< File descriptor of the binlog + int binlog_fd; /*< File descriptor of the binlog * file being written */ - uint64_t last_written; /*< Position of last event written */ - char prevbinlog[BINLOG_FNAMELEN+1]; - int rotating; /*< Rotation in progress flag */ - BLFILE *files; /*< Files used by the slaves */ - SPINLOCK fileslock; /*< Lock for the files queue above */ - unsigned int low_water; /*< Low water mark for client DCB */ - unsigned int high_water; /*< High water mark for client DCB */ - unsigned int short_burst; /*< Short burst for slave catchup */ - unsigned int long_burst; /*< Long burst for slave catchup */ - unsigned long burst_size; /*< Maximum size of burst to send */ - unsigned long heartbeat; /*< Configured heartbeat value */ - ROUTER_STATS stats; /*< Statistics for this router */ - int active_logs; - int reconnect_pending; - int retry_backoff; - time_t connect_time; - int handling_threads; - struct router_instance - *next; + uint64_t last_written; /*< Position of last event written */ + char prevbinlog[BINLOG_FNAMELEN+1]; + int rotating; /*< Rotation in progress flag */ + BLFILE *files; /*< Files used by the slaves */ + SPINLOCK fileslock; /*< Lock for the files queue above */ + unsigned int low_water; /*< Low water mark for client DCB */ + unsigned int high_water; /*< High water mark for client DCB */ + unsigned int short_burst; /*< Short burst for slave catchup */ + unsigned int long_burst; /*< Long burst for slave catchup */ + unsigned long burst_size; /*< Maximum size of burst to send */ + unsigned long heartbeat; /*< Configured heartbeat value */ + ROUTER_STATS stats; /*< Statistics for this router */ + int active_logs; + int reconnect_pending; + int retry_backoff; + time_t connect_time; + int handling_threads; + struct router_instance *next; } ROUTER_INSTANCE; /** @@ -317,6 +316,7 @@ typedef struct router_instance { #define BLRM_REGISTER 0x0012 #define BLRM_BINLOGDUMP 0x0013 #define BLRM_MARIADB10 0x0014 + #define BLRM_MAXSTATE 0x0014 static char *blrm_states[] = { "Unconnected", "Connecting", "Authenticated", "Timestamp retrieval", @@ -324,7 +324,7 @@ static char *blrm_states[] = { "Unconnected", "Connecting", "Authenticated", "Ti "binlog checksum rerieval", "GTID Mode retrieval", "Master UUID retrieval", "Set Slave UUID", "Set Names latin1", "Set Names utf8", "select 1", "select version()", "select @@version_comment", "select @@hostname", - "select @@mx_allowed_packet", "Register slave", "Binlog Dump","Set MariaDB slave capability" }; + "select @@mx_allowed_packet", "Register slave", "Binlog Dump", "Set MariaDB slave capability" }; #define BLRS_CREATED 0x0000 #define BLRS_UNREGISTERED 0x0001 diff --git a/server/modules/routing/binlog/blr_master.c b/server/modules/routing/binlog/blr_master.c index bc1e676d0..2efa417e4 100644 --- a/server/modules/routing/binlog/blr_master.c +++ b/server/modules/routing/binlog/blr_master.c @@ -448,12 +448,27 @@ char query[128]; GWBUF_CONSUME_ALL(router->saved_master.chksum2); router->saved_master.chksum2 = buf; blr_cache_response(router, "chksum2", buf); - //buf = blr_make_query("SELECT @@GLOBAL.GTID_MODE"); - buf = blr_make_query("SET @mariadb_slave_capability=4"); - router->master_state = BLRM_GTIDMODE; + + if (router->mariadb10_compat) { + buf = blr_make_query("SET @mariadb_slave_capability=4"); + router->master_state = BLRM_MARIADB10; + } else { + buf = blr_make_query("SELECT @@GLOBAL.GTID_MODE"); + router->master_state = BLRM_GTIDMODE; + } router->master->func.write(router->master, buf); break; } + case BLRM_MARIADB10: + // Response to the SET @mariadb_slave_capability=4, should be stored + if (router->saved_master.mariadb10) + GWBUF_CONSUME_ALL(router->saved_master.mariadb10); + router->saved_master.mariadb10 = buf; + blr_cache_response(router, "mariadb10", buf); + buf = blr_make_registration(router); + router->master_state = BLRM_REGISTER; + router->master->func.write(router->master, buf); + break; case BLRM_GTIDMODE: // Response to the GTID_MODE, should be stored if (router->saved_master.gtid_mode) diff --git a/server/modules/routing/binlog/blr_slave.c b/server/modules/routing/binlog/blr_slave.c index 8a008e35f..fe2bc2dff 100644 --- a/server/modules/routing/binlog/blr_slave.c +++ b/server/modules/routing/binlog/blr_slave.c @@ -366,11 +366,11 @@ int query_len; free(query_text); return blr_slave_replay(router, slave, router->saved_master.heartbeat); } - else if (strcasecmp(word, "@mariadb_slave_capability") == 0) + else if (strcasecmp(word, "@mariadb_slave_capability") == 0) { free(query_text); - return blr_slave_replay(router, slave, router->saved_master.gtid_mode); - //return blr_slave_send_ok(router, slave); + if (router->mariadb10_compat) + return blr_slave_replay(router, slave, router->saved_master.mariadb10); } else if (strcasecmp(word, "@master_binlog_checksum") == 0) { From 3f2876bde35d11464638c88cc3c0f52ff13316ee Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Thu, 7 May 2015 15:32:12 +0200 Subject: [PATCH 005/100] Fixed buffer free Fixed buffer free --- server/modules/routing/binlog/blr_slave.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/modules/routing/binlog/blr_slave.c b/server/modules/routing/binlog/blr_slave.c index fe2bc2dff..1928e5d9d 100644 --- a/server/modules/routing/binlog/blr_slave.c +++ b/server/modules/routing/binlog/blr_slave.c @@ -368,9 +368,10 @@ int query_len; } else if (strcasecmp(word, "@mariadb_slave_capability") == 0) { - free(query_text); - if (router->mariadb10_compat) + if (router->mariadb10_compat) { + free(query_text); return blr_slave_replay(router, slave, router->saved_master.mariadb10); + } } else if (strcasecmp(word, "@master_binlog_checksum") == 0) { From 2c2a03a6f696894cb015f32a0ae9d2ca50123806 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Thu, 7 May 2015 16:10:35 +0200 Subject: [PATCH 006/100] Always reply to SET @mariadb_slave_capability Always reply to SET @mariadb_slave_capability, with saved master reply for mariadb10 master or with OK otherwise --- server/modules/routing/binlog/blr_slave.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/modules/routing/binlog/blr_slave.c b/server/modules/routing/binlog/blr_slave.c index 1928e5d9d..c963d8ab4 100644 --- a/server/modules/routing/binlog/blr_slave.c +++ b/server/modules/routing/binlog/blr_slave.c @@ -368,9 +368,11 @@ int query_len; } else if (strcasecmp(word, "@mariadb_slave_capability") == 0) { + free(query_text); if (router->mariadb10_compat) { - free(query_text); return blr_slave_replay(router, slave, router->saved_master.mariadb10); + } else { + return blr_slave_send_ok(router, slave); } } else if (strcasecmp(word, "@master_binlog_checksum") == 0) From 7d48779913c7ebf62bd1eed7ce5a3f437bfee0ef Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Thu, 7 May 2015 17:02:33 +0200 Subject: [PATCH 007/100] Added MAX_EVENT_TYPE_MARIADB10 check Added MAX_EVENT_TYPE_MARIADB10 check for router->mariadb10_compat --- server/modules/include/blr.h | 3 ++- server/modules/routing/binlog/blr_file.c | 23 +++++++++++++++++------ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/server/modules/include/blr.h b/server/modules/include/blr.h index 8aecd83f0..7b57fd15f 100644 --- a/server/modules/include/blr.h +++ b/server/modules/include/blr.h @@ -397,7 +397,8 @@ static char *blrs_states[] = { "Created", "Unregistered", "Registered", #define ANONYMOUS_GTID_EVENT 0x22 #define PREVIOUS_GTIDS_EVENT 0x23 -#define MAX_EVENT_TYPE 0xa3 +#define MAX_EVENT_TYPE 0x23 +#define MAX_EVENT_TYPE_MARIADB10 0xa3 /** * Binlog event flags diff --git a/server/modules/routing/binlog/blr_file.c b/server/modules/routing/binlog/blr_file.c index 2473909ec..ec7864d5e 100644 --- a/server/modules/routing/binlog/blr_file.c +++ b/server/modules/routing/binlog/blr_file.c @@ -443,15 +443,26 @@ struct stat statb; hdr->next_pos = EXTRACT32(&hdbuf[13]); hdr->flags = EXTRACT16(&hdbuf[17]); - if (hdr->event_type > MAX_EVENT_TYPE) - { - LOGIF(LE, (skygw_log_write(LOGFILE_ERROR, - "Invalid event type 0x%x. " + if (router->mariadb10_compat) { + if (hdr->event_type > MAX_EVENT_TYPE_MARIADB10) { + LOGIF(LE, (skygw_log_write(LOGFILE_ERROR, + "Invalid MariaDB 10 event type 0x%x. " "Binlog file is %s, position %d", hdr->event_type, file->binlogname, pos))); - return NULL; - } + return NULL; + } + } else { + if (hdr->event_type > MAX_EVENT_TYPE) { + LOGIF(LE, (skygw_log_write(LOGFILE_ERROR, + "Invalid event type 0x%x. " + "Binlog file is %s, position %d", + hdr->event_type, + file->binlogname, pos))); + + return NULL; + } + } if (hdr->next_pos < pos && hdr->event_type != ROTATE_EVENT) { From 5d1e09ca4f22ba0a5402a089173cf4652ad71e08 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Thu, 7 May 2015 17:14:39 +0200 Subject: [PATCH 008/100] Added MariaDB 10 Compatibility without GTID Added MariaDB 10 Compatibility without GTID --- server/modules/routing/binlog/blr.c | 2 ++ server/modules/routing/binlog/blr_file.c | 3 ++- server/modules/routing/binlog/blr_master.c | 3 ++- server/modules/routing/binlog/blr_slave.c | 2 ++ 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/server/modules/routing/binlog/blr.c b/server/modules/routing/binlog/blr.c index 2f5a38bee..071563952 100644 --- a/server/modules/routing/binlog/blr.c +++ b/server/modules/routing/binlog/blr.c @@ -35,6 +35,8 @@ * 02/04/2014 Mark Riddoch Initial implementation * 17/02/2015 Massimiliano Pinto Addition of slave port and username in diagnostics * 18/02/2015 Massimiliano Pinto Addition of dcb_close in closeSession + * 07/05/2015 Massimiliano Pinto Addition of MariaDB 10 compatibility support + * * @endverbatim */ diff --git a/server/modules/routing/binlog/blr_file.c b/server/modules/routing/binlog/blr_file.c index ec7864d5e..e59a7be19 100644 --- a/server/modules/routing/binlog/blr_file.c +++ b/server/modules/routing/binlog/blr_file.c @@ -23,8 +23,9 @@ * @verbatim * Revision History * - * Date Who Description + * Date Who Description * 14/04/2014 Mark Riddoch Initial implementation + * 07/05/2015 Massimiliano Pinto Added MAX_EVENT_TYPE_MARIADB10 * * @endverbatim */ diff --git a/server/modules/routing/binlog/blr_master.c b/server/modules/routing/binlog/blr_master.c index 2efa417e4..f230442f1 100644 --- a/server/modules/routing/binlog/blr_master.c +++ b/server/modules/routing/binlog/blr_master.c @@ -31,8 +31,9 @@ * @verbatim * Revision History * - * Date Who Description + * Date Who Description * 02/04/2014 Mark Riddoch Initial implementation + * 07/05/2015 Massimiliano Pinto Added MariaDB 10 Compatibility * * @endverbatim */ diff --git a/server/modules/routing/binlog/blr_slave.c b/server/modules/routing/binlog/blr_slave.c index c963d8ab4..141ff3a03 100644 --- a/server/modules/routing/binlog/blr_slave.c +++ b/server/modules/routing/binlog/blr_slave.c @@ -36,9 +36,11 @@ * 18/02/2015 Massimiliano Pinto Addition of DISCONNECT ALL and DISCONNECT SERVER server_id * 18/03/2015 Markus Makela Better detection of CRC32 | NONE checksum * 19/03/2015 Massimiliano Pinto Addition of basic MariaDB 10 compatibility support + * 07/05/2015 Massimiliano Pinto Added MariaDB 10 Compatibility * * @endverbatim */ + #include #include #include From 230f88737cca9e6a5811f7261a2bc63eff0adf6a Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Thu, 7 May 2015 18:05:04 +0200 Subject: [PATCH 009/100] Added reading saved mariadb10 data Added reading saved mariadb10 data --- server/modules/routing/binlog/blr.c | 1 + 1 file changed, 1 insertion(+) diff --git a/server/modules/routing/binlog/blr.c b/server/modules/routing/binlog/blr.c index 071563952..becea18b0 100644 --- a/server/modules/routing/binlog/blr.c +++ b/server/modules/routing/binlog/blr.c @@ -395,6 +395,7 @@ unsigned char *defuuid; inst->saved_master.selectvercom = blr_cache_read_response(inst, "selectvercom"); inst->saved_master.selecthostname = blr_cache_read_response(inst, "selecthostname"); inst->saved_master.map = blr_cache_read_response(inst, "map"); + inst->saved_master.mariadb10 = blr_cache_read_response(inst, "mariadb10"); /* * Initialise the binlog file and position From f991e58b5714b82952b19d4283b0117f414e01d6 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Mon, 11 May 2015 11:43:21 +0200 Subject: [PATCH 010/100] MariaDB 10 master requires MariaDB 10 slaves Only MariaDB 10 slaves can register to binblog server with a MariaDB 10 Master --- server/modules/include/blr.h | 6 ++++-- server/modules/routing/binlog/blr.c | 1 + server/modules/routing/binlog/blr_slave.c | 26 ++++++++++++++++++++++- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/server/modules/include/blr.h b/server/modules/include/blr.h index 7b57fd15f..3dbc43853 100644 --- a/server/modules/include/blr.h +++ b/server/modules/include/blr.h @@ -24,8 +24,9 @@ * @verbatim * Revision History * - * Date Who Description - * 02/04/14 Mark Riddoch Initial implementation + * Date Who Description + * 02/04/14 Mark Riddoch Initial implementation + * 11/05/15 Massimilaino Pinto Added mariadb10_compat to master and slave structs * * @endverbatim */ @@ -175,6 +176,7 @@ typedef struct router_slave { uint32_t lastEventTimestamp;/*< Last event timestamp sent */ SPINLOCK catch_lock; /*< Event catchup lock */ unsigned int cstate; /*< Catch up state */ + bool mariadb10_compat;/*< MariaDB 10.0 compatibility */ SPINLOCK rses_lock; /*< Protects rses_deleted */ pthread_t pthread; struct router_instance diff --git a/server/modules/routing/binlog/blr.c b/server/modules/routing/binlog/blr.c index becea18b0..8dce5ea58 100644 --- a/server/modules/routing/binlog/blr.c +++ b/server/modules/routing/binlog/blr.c @@ -498,6 +498,7 @@ ROUTER_SLAVE *slave; strcpy(slave->binlogfile, "unassigned"); slave->connect_time = time(0); slave->lastEventTimestamp = 0; + slave->mariadb10_compat = false; /** * Add this session to the list of active sessions. diff --git a/server/modules/routing/binlog/blr_slave.c b/server/modules/routing/binlog/blr_slave.c index 141ff3a03..b932000d3 100644 --- a/server/modules/routing/binlog/blr_slave.c +++ b/server/modules/routing/binlog/blr_slave.c @@ -37,6 +37,7 @@ * 18/03/2015 Markus Makela Better detection of CRC32 | NONE checksum * 19/03/2015 Massimiliano Pinto Addition of basic MariaDB 10 compatibility support * 07/05/2015 Massimiliano Pinto Added MariaDB 10 Compatibility + * 11/05/2015 Massimiliano Pinto Only MariaDB 10 Slaves can register to binlog router with a MariaDB 10 Master * * @endverbatim */ @@ -125,7 +126,27 @@ blr_slave_request(ROUTER_INSTANCE *router, ROUTER_SLAVE *slave, GWBUF *queue) return blr_slave_query(router, slave, queue); break; case COM_REGISTER_SLAVE: - return blr_slave_register(router, slave, queue); + /* + * If Master is MariaDB10 don't allow registration from + * MariaDB/Mysql 5 Slaves + */ + + if (router->mariadb10_compat && !slave->mariadb10_compat) { + slave->state = BLRS_ERRORED; + blr_send_custom_error(slave->dcb, 1, 0, + "MariaDB 10 Slave is required for Slave registration"); + + LOGIF(LE, (skygw_log_write( + LOGFILE_ERROR, + "MariaDB 10 Slave is required for Slave registration", + MYSQL_COMMAND(queue)))); + + dcb_close(slave->dcb); + return 1; + } else { + /* Master and Slave version OK: continue with slave registration */ + return blr_slave_register(router, slave, queue); + } break; case COM_BINLOG_DUMP: return blr_slave_binlog_dump(router, slave, queue); @@ -370,6 +391,9 @@ int query_len; } else if (strcasecmp(word, "@mariadb_slave_capability") == 0) { + /* mariadb10 compatibility is set for the slave */ + slave->mariadb10_compat=true; + free(query_text); if (router->mariadb10_compat) { return blr_slave_replay(router, slave, router->saved_master.mariadb10); From a48e694dba044ae25debf846585c4079f17a679b Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Mon, 11 May 2015 12:42:14 +0200 Subject: [PATCH 011/100] Fix for log messages Fix for log messages about MariaDB 10 registration and unexpected query --- server/modules/routing/binlog/blr_slave.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/server/modules/routing/binlog/blr_slave.c b/server/modules/routing/binlog/blr_slave.c index b932000d3..87b795ea6 100644 --- a/server/modules/routing/binlog/blr_slave.c +++ b/server/modules/routing/binlog/blr_slave.c @@ -138,8 +138,9 @@ blr_slave_request(ROUTER_INSTANCE *router, ROUTER_SLAVE *slave, GWBUF *queue) LOGIF(LE, (skygw_log_write( LOGFILE_ERROR, - "MariaDB 10 Slave is required for Slave registration", - MYSQL_COMMAND(queue)))); + "%s: Slave %s: a MariaDB 10 Slave is required for Slave registration", + router->service->name, + slave->dcb->remote))); dcb_close(slave->dcb); return 1; @@ -472,7 +473,7 @@ int query_len; query_text = strndup(qtext, query_len); LOGIF(LE, (skygw_log_write( - LOGFILE_ERROR, "Unexpected query from slave server %s", query_text))); + LOGFILE_ERROR, "Unexpected query from slave %s: %s", slave->dcb->remote, query_text))); free(query_text); blr_slave_send_error(router, slave, "Unexpected SQL query received from slave."); return 1; From 16d6bd6d2c7f4d0b40eee154d6656902a238187d Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 28 May 2015 11:56:14 +0300 Subject: [PATCH 012/100] Added service SSL mode variables. --- server/core/config.c | 2 +- server/core/service.c | 13 +++++++++- server/include/service.h | 7 ++++++ .../include/mysql_client_server_protocol.h | 4 ++++ server/modules/protocol/mysql_client.c | 24 ++++++++++++++++--- 5 files changed, 45 insertions(+), 5 deletions(-) diff --git a/server/core/config.c b/server/core/config.c index 274522468..ccbeee0e0 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -420,7 +420,7 @@ hashtable_memory_fns(monitorhash,strdup,NULL,free,NULL); /** Add the 5.5.5- string to the start of the version string if * the version string starts with "10.". - * This mimics MariaDB 10.0 replication which adds 5.5.5- for backwards compatibility. */ + * This mimics MariaDB 10.0 behavior which adds 5.5.5- for backwards compatibility. */ if(strncmp(version_string,"10.",3) == 0) { ((SERVICE *)(obj->element))->version_string = malloc((strlen(version_string) + diff --git a/server/core/service.c b/server/core/service.c index 5ba5d539d..8297ea6fd 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -136,7 +136,8 @@ SERVICE *service; service->routerModule = strdup(router); service->users_from_all = false; service->resources = NULL; - + service->ssl_mode = SSL_REQUIRED; + if (service->name == NULL || service->routerModule == NULL) { if (service->name) @@ -855,6 +856,16 @@ serviceOptimizeWildcard(SERVICE *service, int action) return 1; } +/** Enable or disable the service SSL capability*/ +int +serviceSetSSL(SERVICE *service, int action) +{ + if(action) + service->ssl_mode = SSL_REQUIRED; + else + service->ssl_mode = SSL_DISABLED; +} + /** * Whether to strip escape characters from the name of the database the client * is connecting to. diff --git a/server/include/service.h b/server/include/service.h index f26c99806..a6fea6d56 100644 --- a/server/include/service.h +++ b/server/include/service.h @@ -105,6 +105,12 @@ typedef struct server_ref_t{ SERVER* server; }SERVER_REF; +typedef enum { + SSL_DISABLED, + SSL_ENABLED, + SSL_REQUIRED +} ssl_mode_t; + /** * Defines a service within the gateway. * @@ -149,6 +155,7 @@ typedef struct service { FILTER_DEF **filters; /**< Ordered list of filters */ int n_filters; /**< Number of filters */ int conn_timeout; /*< Session timeout in seconds */ + ssl_mode_t ssl_mode; /*< one of DISABLED, ENABLED or REQUIRED */ char *weightby; struct service *next; /**< The next service in the linked list */ } SERVICE; diff --git a/server/modules/include/mysql_client_server_protocol.h b/server/modules/include/mysql_client_server_protocol.h index 46bbe296c..de8118659 100644 --- a/server/modules/include/mysql_client_server_protocol.h +++ b/server/modules/include/mysql_client_server_protocol.h @@ -97,6 +97,10 @@ typedef enum { MYSQL_AUTH_RECV, MYSQL_AUTH_FAILED, MYSQL_HANDSHAKE_FAILED, + MYSQL_AUTH_SSL_REQ, /*< client requested SSL */ + MYSQL_AUTH_SSL_EXCHANGE_DONE, /*< SSL handshake done */ + MYSQL_AUTH_SSL_EXCHANGE_ERR, /*< SSL handshake failure */ + MYSQL_AUTH_SSL_RECV, /*< */ MYSQL_IDLE } mysql_auth_state_t; diff --git a/server/modules/protocol/mysql_client.c b/server/modules/protocol/mysql_client.c index abdb4422c..d1e188281 100644 --- a/server/modules/protocol/mysql_client.c +++ b/server/modules/protocol/mysql_client.c @@ -242,7 +242,7 @@ MySQLSendHandshake(DCB* dcb) char server_scramble[GW_MYSQL_SCRAMBLE_SIZE + 1]=""; char *version_string; int len_version_string=0; - + MySQLProtocol *protocol = DCB_PROTOCOL(dcb, MySQLProtocol); GWBUF *buf; @@ -319,7 +319,16 @@ MySQLSendHandshake(DCB* dcb) mysql_server_capabilities_one[0] &= ~GW_MYSQL_CAPABILITIES_COMPRESS; - mysql_server_capabilities_one[0] &= ~GW_MYSQL_CAPABILITIES_SSL; + + if(dcb->service->ssl_mode != SSL_DISABLED) + { + mysql_server_capabilities_one[1] |= GW_MYSQL_CAPABILITIES_SSL >> 8; + } + else + { + mysql_server_capabilities_one[0] &= ~GW_MYSQL_CAPABILITIES_SSL; + } + memcpy(mysql_handshake_payload, mysql_server_capabilities_one, sizeof(mysql_server_capabilities_one)); mysql_handshake_payload = mysql_handshake_payload + sizeof(mysql_server_capabilities_one); @@ -402,7 +411,7 @@ static int gw_mysql_do_authentication(DCB *dcb, GWBUF *queue) { uint8_t *stage1_hash = NULL; int auth_ret = -1; MYSQL_session *client_data = NULL; - + int ssl = 0; CHK_DCB(dcb); protocol = DCB_PROTOCOL(dcb, MySQLProtocol); @@ -451,6 +460,15 @@ static int gw_mysql_do_authentication(DCB *dcb, GWBUF *queue) { &protocol->client_capabilities); */ + + ssl = protocol->client_capabilities & GW_MYSQL_CAPABILITIES_SSL; + + /** Client didn't requested SSL when SSL mode was required*/ + if(!ssl && protocol->owner_dcb->service->ssl_mode == SSL_REQUIRED) + { + return 1; + } + username = get_username_from_auth(username, client_auth_packet); if (username == NULL) From 3d6259cb00d32818b9bc98f154a772543e524509 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 28 May 2015 16:33:51 +0300 Subject: [PATCH 013/100] Added configuration options for different SSL modes. --- server/core/config.c | 9 ++++++++- server/core/service.c | 20 ++++++++++++++++---- server/include/service.h | 1 + server/modules/protocol/mysql_client.c | 12 ++++++++++++ 4 files changed, 37 insertions(+), 5 deletions(-) diff --git a/server/core/config.c b/server/core/config.c index ccbeee0e0..f6721a28e 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -345,6 +345,7 @@ hashtable_memory_fns(monitorhash,strdup,NULL,free,NULL); char *weightby; char *version_string; char *subservices; + char* ssl; bool is_rwsplit = false; bool is_schemarouter = false; char *allow_localhost_match_wildcard_host; @@ -353,6 +354,8 @@ hashtable_memory_fns(monitorhash,strdup,NULL,free,NULL); user = config_get_value(obj->parameters, "user"); auth = config_get_value(obj->parameters, "passwd"); subservices = config_get_value(obj->parameters, "subservices"); + ssl = config_get_value(obj->parameters, "ssl"); + enable_root_user = config_get_value( obj->parameters, "enable_root_user"); @@ -443,7 +446,11 @@ hashtable_memory_fns(monitorhash,strdup,NULL,free,NULL); max_slave_rlag_str = config_get_value(obj->parameters, "max_slave_replication_lag"); - + + if(ssl) + if(serviceSetSSL(obj->element,ssl) != 0) + skygw_log_write(LE,"Error: Unknown parameter for service '%s': %s",obj->object,ssl); + if (enable_root_user) serviceEnableRootUser( obj->element, diff --git a/server/core/service.c b/server/core/service.c index 8297ea6fd..4ef9b3515 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -136,7 +136,7 @@ SERVICE *service; service->routerModule = strdup(router); service->users_from_all = false; service->resources = NULL; - service->ssl_mode = SSL_REQUIRED; + service->ssl_mode = SSL_DISABLED; if (service->name == NULL || service->routerModule == NULL) { @@ -858,12 +858,20 @@ serviceOptimizeWildcard(SERVICE *service, int action) /** Enable or disable the service SSL capability*/ int -serviceSetSSL(SERVICE *service, int action) +serviceSetSSL(SERVICE *service, char* action) { - if(action) + int rval = 0; + + if(strcasecmp(action,"required") == 0) service->ssl_mode = SSL_REQUIRED; - else + else if(strcasecmp(action,"enabled") == 0) + service->ssl_mode = SSL_ENABLED; + else if(strcasecmp(action,"disabled") == 0) service->ssl_mode = SSL_DISABLED; + else + rval = -1; + + return rval; } /** @@ -1029,6 +1037,8 @@ int i; printf("\tUsers data: %p\n", (void *)service->users); printf("\tTotal connections: %d\n", service->stats.n_sessions); printf("\tCurrently connected: %d\n", service->stats.n_current); + printf("\tSSL: %s\n", service->ssl_mode == SSL_DISABLED ? "Disabled": + (service->ssl_mode == SSL_ENABLED ? "Enabled":"Required")); } /** @@ -1138,6 +1148,8 @@ int i; service->stats.n_sessions); dcb_printf(dcb, "\tCurrently connected: %d\n", service->stats.n_current); + dcb_printf(dcb,"\tSSL: %s\n", service->ssl_mode == SSL_DISABLED ? "Disabled": + (service->ssl_mode == SSL_ENABLED ? "Enabled":"Required")); } /** diff --git a/server/include/service.h b/server/include/service.h index a6fea6d56..e0ae151cf 100644 --- a/server/include/service.h +++ b/server/include/service.h @@ -185,6 +185,7 @@ extern int serviceRestart(SERVICE *); extern int serviceSetUser(SERVICE *, char *, char *); extern int serviceGetUser(SERVICE *, char **, char **); extern void serviceSetFilters(SERVICE *, char *); +extern int serviceSetSSL(SERVICE *service, char* action); extern int serviceEnableRootUser(SERVICE *, int ); extern int serviceSetTimeout(SERVICE *, int ); extern void serviceWeightBy(SERVICE *, char *); diff --git a/server/modules/protocol/mysql_client.c b/server/modules/protocol/mysql_client.c index d1e188281..eaf061334 100644 --- a/server/modules/protocol/mysql_client.c +++ b/server/modules/protocol/mysql_client.c @@ -466,9 +466,21 @@ static int gw_mysql_do_authentication(DCB *dcb, GWBUF *queue) { /** Client didn't requested SSL when SSL mode was required*/ if(!ssl && protocol->owner_dcb->service->ssl_mode == SSL_REQUIRED) { + LOGIF(LT,(skygw_log_write(LT,"User %s@%s connected to service '%s' without SSL when SSL was required.", + protocol->owner_dcb->user, + protocol->owner_dcb->remote, + protocol->owner_dcb->service->name))); return 1; } + if(LOG_IS_ENABLED(LT)) + { + skygw_log_write(LT,"User %s@%s connected to service '%s' with SSL.", + protocol->owner_dcb->user, + protocol->owner_dcb->remote, + protocol->owner_dcb->service->name); + } + username = get_username_from_auth(username, client_auth_packet); if (username == NULL) From 449c186a668a6d48f9de484ada049af703fe43b0 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 28 May 2015 18:13:45 +0300 Subject: [PATCH 014/100] Added OpenSSL init function call. --- server/modules/protocol/mysql_client.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/server/modules/protocol/mysql_client.c b/server/modules/protocol/mysql_client.c index eaf061334..0d5c47b48 100644 --- a/server/modules/protocol/mysql_client.c +++ b/server/modules/protocol/mysql_client.c @@ -46,7 +46,9 @@ #include #include #include - +#include +#include +#include MODULE_INFO info = { MODULE_API_PROTOCOL, MODULE_GA, @@ -113,6 +115,7 @@ version() void ModuleInit() { + SSL_library_init(); } /** @@ -473,7 +476,7 @@ static int gw_mysql_do_authentication(DCB *dcb, GWBUF *queue) { return 1; } - if(LOG_IS_ENABLED(LT)) + if(LOG_IS_ENABLED(LT) && ssl) { skygw_log_write(LT,"User %s@%s connected to service '%s' with SSL.", protocol->owner_dcb->user, @@ -481,6 +484,11 @@ static int gw_mysql_do_authentication(DCB *dcb, GWBUF *queue) { protocol->owner_dcb->service->name); } + if(ssl && protocol->owner_dcb->service->ssl_mode != SSL_DISABLED) + { + + } + username = get_username_from_auth(username, client_auth_packet); if (username == NULL) From a572166ffd0a3152fdf7229e17fcf0242a633587 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 28 May 2015 22:19:50 +0300 Subject: [PATCH 015/100] Added ssl handshake to mysql_client --- server/core/config.c | 30 +++++++++-- server/core/service.c | 50 +++++++++++++++++++ server/include/service.h | 15 +++++- .../include/mysql_client_server_protocol.h | 5 +- server/modules/protocol/mysql_client.c | 44 ++++++++++++++-- 5 files changed, 134 insertions(+), 10 deletions(-) diff --git a/server/core/config.c b/server/core/config.c index f6721a28e..12d4c099e 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -345,7 +345,7 @@ hashtable_memory_fns(monitorhash,strdup,NULL,free,NULL); char *weightby; char *version_string; char *subservices; - char* ssl; + char *ssl,*ssl_cert,*ssl_key,*ssl_ca_cert; bool is_rwsplit = false; bool is_schemarouter = false; char *allow_localhost_match_wildcard_host; @@ -355,7 +355,9 @@ hashtable_memory_fns(monitorhash,strdup,NULL,free,NULL); auth = config_get_value(obj->parameters, "passwd"); subservices = config_get_value(obj->parameters, "subservices"); ssl = config_get_value(obj->parameters, "ssl"); - + ssl_cert = config_get_value(obj->parameters, "ssl_cert"); + ssl_key = config_get_value(obj->parameters, "ssl_key"); + ssl_ca_cert = config_get_value(obj->parameters, "ssl_ca_cert"); enable_root_user = config_get_value( obj->parameters, "enable_root_user"); @@ -448,8 +450,28 @@ hashtable_memory_fns(monitorhash,strdup,NULL,free,NULL); "max_slave_replication_lag"); if(ssl) - if(serviceSetSSL(obj->element,ssl) != 0) - skygw_log_write(LE,"Error: Unknown parameter for service '%s': %s",obj->object,ssl); + { + if(ssl_cert == NULL) + skygw_log_write(LE,"Error: Server certificate missing for service '%s'.",obj->object); + if(ssl_ca_cert == NULL) + skygw_log_write(LE,"Error: CA Certificate missing for service '%s'.",obj->object); + if(ssl_key == NULL) + skygw_log_write(LE,"Error: Server private key missing for service '%s'.",obj->object); + + if(ssl_ca_cert != NULL && ssl_cert != NULL && ssl_key != NULL) + { + + if(serviceSetSSL(obj->element,ssl) != 0) + { + skygw_log_write(LE,"Error: Unknown parameter for service '%s': %s",obj->object,ssl); + } + else + { + serviceSetCertificates(obj->element,ssl_cert,ssl_key,ssl_ca_cert); + } + } + + } if (enable_root_user) serviceEnableRootUser( diff --git a/server/core/service.c b/server/core/service.c index 4ef9b3515..ee6d1cf26 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -137,6 +137,10 @@ SERVICE *service; service->users_from_all = false; service->resources = NULL; service->ssl_mode = SSL_DISABLED; + service->ssl_init_done = false; + service->ssl_ca_cert = NULL; + service->ssl_cert = NULL; + service->ssl_key = NULL; if (service->name == NULL || service->routerModule == NULL) { @@ -856,6 +860,14 @@ serviceOptimizeWildcard(SERVICE *service, int action) return 1; } +void +serviceSetCertificates(SERVICE *service, char* cert,char* key, char* ca_cert) +{ + service->ssl_cert = strdup(cert); + service->ssl_key = strdup(key); + service->ssl_ca_cert = strdup(ca_cert); +} + /** Enable or disable the service SSL capability*/ int serviceSetSSL(SERVICE *service, char* action) @@ -1798,3 +1810,41 @@ int *data; return set; } + + +int serviceInitSSL(SERVICE* service) +{ + if(!service->ssl_init_done) + { + service->method = (SSL_METHOD*)SSLv23_server_method(); + service->ctx = SSL_CTX_new(service->method); + + if (SSL_CTX_use_certificate_file(service->ctx, service->ssl_cert, SSL_FILETYPE_PEM) <= 0) { + return -1; + } + + /* Load the private-key corresponding to the server certificate */ + if (SSL_CTX_use_PrivateKey_file(service->ctx, service->ssl_key, SSL_FILETYPE_PEM) <= 0) { + return -1; + } + + /* Check if the server certificate and private-key matches */ + if (!SSL_CTX_check_private_key(service->ctx)) { + return -1; + } + + + /* Load the RSA CA certificate into the SSL_CTX structure */ + if (!SSL_CTX_load_verify_locations(service->ctx, service->ssl_ca_cert, NULL)) { + return -1; + } + + /* Set to require peer (client) certificate verification */ + SSL_CTX_set_verify(service->ctx,SSL_VERIFY_PEER,NULL); + + /* Set the verification depth to 1 */ + SSL_CTX_set_verify_depth(service->ctx,10); + service->ssl_init_done = true; + } + return 0; +} diff --git a/server/include/service.h b/server/include/service.h index e0ae151cf..1c6614656 100644 --- a/server/include/service.h +++ b/server/include/service.h @@ -26,7 +26,9 @@ #include #include #include - +#include +#include +#include /** * @file service.h * @@ -158,6 +160,15 @@ typedef struct service { ssl_mode_t ssl_mode; /*< one of DISABLED, ENABLED or REQUIRED */ char *weightby; struct service *next; /**< The next service in the linked list */ + SSL_CTX *ctx; + SSL *ssl; + SSL_METHOD *method; /*< SSLv2/3 or TLSv1/2 methods + * see: https://www.openssl.org/docs/ssl/SSL_CTX_new.html */ + char* ssl_cert; + char* ssl_key; + char* ssl_ca_cert; + bool ssl_init_done; + } SERVICE; typedef enum count_spec_t {COUNT_NONE=0, COUNT_ATLEAST, COUNT_EXACT, COUNT_ATMOST} count_spec_t; @@ -186,6 +197,8 @@ extern int serviceSetUser(SERVICE *, char *, char *); extern int serviceGetUser(SERVICE *, char **, char **); extern void serviceSetFilters(SERVICE *, char *); extern int serviceSetSSL(SERVICE *service, char* action); +extern int serviceInitSSL(SERVICE* service); +extern void serviceSetCertificates(SERVICE *service, char* cert,char* key, char* ca_cert); extern int serviceEnableRootUser(SERVICE *, int ); extern int serviceSetTimeout(SERVICE *, int ); extern void serviceWeightBy(SERVICE *, char *); diff --git a/server/modules/include/mysql_client_server_protocol.h b/server/modules/include/mysql_client_server_protocol.h index de8118659..87dbc50ee 100644 --- a/server/modules/include/mysql_client_server_protocol.h +++ b/server/modules/include/mysql_client_server_protocol.h @@ -54,7 +54,9 @@ #include #include #include - +#include +#include +#include #include #include #include @@ -294,6 +296,7 @@ typedef struct { unsigned long tid; /*< MySQL Thread ID, in * handshake */ unsigned int charset; /*< MySQL character set at connect time */ + SSL* ssl; /*< SSL struct for client connection */ #if defined(SS_DEBUG) skygw_chk_t protocol_chk_tail; #endif diff --git a/server/modules/protocol/mysql_client.c b/server/modules/protocol/mysql_client.c index 0d5c47b48..5adb8bcd0 100644 --- a/server/modules/protocol/mysql_client.c +++ b/server/modules/protocol/mysql_client.c @@ -46,9 +46,7 @@ #include #include #include -#include -#include -#include + MODULE_INFO info = { MODULE_API_PROTOCOL, MODULE_GA, @@ -116,6 +114,7 @@ void ModuleInit() { SSL_library_init(); + SSL_load_error_strings(); } /** @@ -484,9 +483,47 @@ static int gw_mysql_do_authentication(DCB *dcb, GWBUF *queue) { protocol->owner_dcb->service->name); } + /** Do the SSL Handshake */ if(ssl && protocol->owner_dcb->service->ssl_mode != SSL_DISABLED) { + if(serviceInitSSL(protocol->owner_dcb->service) != 0) + { + skygw_log_write(LOGFILE_ERROR,"Error: SSL initialization for service '%s' failed.", + protocol->owner_dcb->service->name); + return 1; + } + protocol->ssl = SSL_new(protocol->owner_dcb->service->ctx); + SSL_set_fd(protocol->ssl,dcb->fd); + protocol->protocol_auth_state = MYSQL_AUTH_SSL_REQ; + printf("%s\n",SSL_get_version(protocol->ssl)); + int errnum,rval; + char errbuf[1024]; + switch((rval = SSL_accept(protocol->ssl))) + { + case 0: + errnum = SSL_get_error(protocol->ssl,rval); + ERR_error_string(errnum,errbuf); + skygw_log_write_flush(LOGFILE_ERROR,"SSL_accept: %s",errbuf); + ERR_print_errors_fp(stdout); + ERR_error_string(errnum,errbuf); + printf("%s\n",errbuf); + fflush(stdout); + break; + case 1: + protocol->protocol_auth_state = MYSQL_AUTH_SSL_EXCHANGE_DONE; + break; + default: + errnum = SSL_get_error(protocol->ssl,rval); + ERR_print_errors_fp(stdout); + ERR_error_string(errnum,errbuf); + printf("%s\n",errbuf); + fflush(stdout); + skygw_log_write_flush(LOGFILE_ERROR,"Error: Fatal error in SSL_accept: %s",errbuf); + protocol->protocol_auth_state = MYSQL_AUTH_SSL_EXCHANGE_ERR; + break; + + } } username = get_username_from_auth(username, client_auth_packet); @@ -1700,4 +1737,3 @@ return_str: return str; } #endif - From f946a44620b9f709d01dbcdb11c0769f1dab733d Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 28 May 2015 23:11:32 +0300 Subject: [PATCH 016/100] Added handling of partial SSL handshakes. --- server/modules/protocol/mysql_client.c | 91 +++++++++++++++++++++++--- 1 file changed, 83 insertions(+), 8 deletions(-) diff --git a/server/modules/protocol/mysql_client.c b/server/modules/protocol/mysql_client.c index 5adb8bcd0..387b59aa0 100644 --- a/server/modules/protocol/mysql_client.c +++ b/server/modules/protocol/mysql_client.c @@ -78,6 +78,8 @@ extern char* get_username_from_auth(char* ptr, uint8_t* data); extern int check_db_name_after_auth(DCB *, char *, int); extern char* create_auth_fail_str(char *username, char *hostaddr, char *sha1, char *db); +int do_ssl_accept(MySQLProtocol* protocol); + /* * The "module object" for the mysqld client protocol module. */ @@ -498,6 +500,7 @@ static int gw_mysql_do_authentication(DCB *dcb, GWBUF *queue) { printf("%s\n",SSL_get_version(protocol->ssl)); int errnum,rval; char errbuf[1024]; + switch((rval = SSL_accept(protocol->ssl))) { case 0: @@ -515,12 +518,22 @@ static int gw_mysql_do_authentication(DCB *dcb, GWBUF *queue) { default: errnum = SSL_get_error(protocol->ssl,rval); - ERR_print_errors_fp(stdout); - ERR_error_string(errnum,errbuf); - printf("%s\n",errbuf); - fflush(stdout); - skygw_log_write_flush(LOGFILE_ERROR,"Error: Fatal error in SSL_accept: %s",errbuf); - protocol->protocol_auth_state = MYSQL_AUTH_SSL_EXCHANGE_ERR; + if(errnum == SSL_ERROR_WANT_READ) + { + /** Not all of the data has been read. Go back to the poll + queue and wait for more.*/ + protocol->protocol_auth_state = MYSQL_AUTH_SSL_RECV; + return 0; + } + else + { + ERR_print_errors_fp(stdout); + ERR_error_string(errnum,errbuf); + printf("%s\n",errbuf); + fflush(stdout); + skygw_log_write_flush(LOGFILE_ERROR,"Error: Fatal error in SSL_accept: %s",errbuf); + protocol->protocol_auth_state = MYSQL_AUTH_SSL_EXCHANGE_ERR; + } break; } @@ -655,6 +668,10 @@ int gw_read_client_event( CHK_DCB(dcb); protocol = DCB_PROTOCOL(dcb, MySQLProtocol); CHK_PROTOCOL(protocol); + if(protocol->protocol_auth_state == MYSQL_AUTH_SSL_RECV) + { + goto do_auth; + } rc = dcb_read(dcb, &read_buffer); @@ -794,7 +811,7 @@ int gw_read_client_event( } } - + do_auth: /** * Now there should be at least one complete mysql packet in read_buffer. */ @@ -805,7 +822,10 @@ int gw_read_client_event( int auth_val; auth_val = gw_mysql_do_authentication(dcb, read_buffer); - + + if(protocol->protocol_auth_state == MYSQL_AUTH_SSL_RECV) + break; + if (auth_val == 0) { SESSION *session; @@ -899,6 +919,15 @@ int gw_read_client_event( } break; + case MYSQL_AUTH_SSL_RECV: + { + if(do_ssl_accept(protocol) == 1) + { + protocol->protocol_auth_state = MYSQL_AUTH_SSL_EXCHANGE_DONE; + } + } + break; + case MYSQL_IDLE: { uint8_t* payload = NULL; @@ -1737,3 +1766,49 @@ return_str: return str; } #endif + +int do_ssl_accept(MySQLProtocol* protocol) +{ + int rval,errnum; + char errbuf[2014]; + + switch((rval = SSL_accept(protocol->ssl))) + { + case 0: + errnum = SSL_get_error(protocol->ssl,rval); + ERR_error_string(errnum,errbuf); + skygw_log_write_flush(LOGFILE_ERROR,"SSL_accept: %s",errbuf); + ERR_print_errors_fp(stdout); + ERR_error_string(errnum,errbuf); + printf("%s\n",errbuf); + fflush(stdout); + break; + case 1: + protocol->protocol_auth_state = MYSQL_AUTH_SSL_EXCHANGE_DONE; + rval = 1; + break; + + default: + errnum = SSL_get_error(protocol->ssl,rval); + if(errnum == SSL_ERROR_WANT_READ) + { + /** Not all of the data has been read. Go back to the poll + queue and wait for more.*/ + rval = 0; + } + else + { + ERR_print_errors_fp(stdout); + ERR_error_string(errnum,errbuf); + printf("%s\n",errbuf); + fflush(stdout); + skygw_log_write_flush(LOGFILE_ERROR,"Error: Fatal error in SSL_accept: %s",errbuf); + protocol->protocol_auth_state = MYSQL_AUTH_SSL_EXCHANGE_ERR; + rval = -1; + } + break; + + } + + return rval; +} \ No newline at end of file From 0f814d3e73c8e57f1f78ef91a8999bf5672f099a Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 29 May 2015 13:00:37 +0300 Subject: [PATCH 017/100] Added SSL write and read functions. --- server/core/config.c | 19 +- server/core/dcb.c | 529 +++++++++++++++++- server/include/dcb.h | 8 +- .../include/mysql_client_server_protocol.h | 1 + server/modules/protocol/mysql_client.c | 245 +++++--- server/modules/protocol/mysql_common.c | 5 +- 6 files changed, 726 insertions(+), 81 deletions(-) diff --git a/server/core/config.c b/server/core/config.c index 12d4c099e..640437270 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -452,11 +452,17 @@ hashtable_memory_fns(monitorhash,strdup,NULL,free,NULL); if(ssl) { if(ssl_cert == NULL) - skygw_log_write(LE,"Error: Server certificate missing for service '%s'.",obj->object); + skygw_log_write(LE,"Error: Server certificate missing for service '%s'." + "Please provide the path to the server certificate by adding the ssl_cert= parameter", + obj->object); if(ssl_ca_cert == NULL) - skygw_log_write(LE,"Error: CA Certificate missing for service '%s'.",obj->object); + skygw_log_write(LE,"Error: CA Certificate missing for service '%s'." + "Please provide the path to the certificate authority certificate by adding the ssl_ca_cert= parameter", + obj->object); if(ssl_key == NULL) - skygw_log_write(LE,"Error: Server private key missing for service '%s'.",obj->object); + skygw_log_write(LE,"Error: Server private key missing for service '%s'. " + "Please provide the path to the server certificate key by adding the ssl_key= parameter" + ,obj->object); if(ssl_ca_cert != NULL && ssl_cert != NULL && ssl_key != NULL) { @@ -470,6 +476,13 @@ hashtable_memory_fns(monitorhash,strdup,NULL,free,NULL); serviceSetCertificates(obj->element,ssl_cert,ssl_key,ssl_ca_cert); } } + else + { + /** If SSL was configured wrong, the + * service needs to fail.*/ + skygw_log_write_flush(LE,"Error: Missing SSL certificate paths found in the configuration. " + "This service will not use SSL."); + } } diff --git a/server/core/dcb.c b/server/core/dcb.c index 6717aea41..ba2028de2 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -49,6 +49,7 @@ * backend * 07/05/2014 Mark Riddoch Addition of callback mechanism * 20/06/2014 Mark Riddoch Addition of dcb_clone + * 29/05/2015 Markus Makela Addition of dcb_write_SSL * * @endverbatim */ @@ -880,6 +881,152 @@ return_n: return n; } + +/** + * General purpose read routine to read data from a socket in the + * Descriptor Control Block and append it to a linked list of buffers. + * The list may be empty, in which case *head == NULL + * + * @param dcb The DCB to read from + * @param head Pointer to linked list to append data to + * @return -1 on error, otherwise the number of read bytes on the last + * iteration of while loop. 0 is returned if no data available. + */ +int dcb_read_SSL( + DCB *dcb, + SSL* ssl, + GWBUF **head) +{ + GWBUF *buffer = NULL; + int b; + int rc; + int n; + int nread = 0; + + CHK_DCB(dcb); + + if (dcb->fd <= 0) + { + LOGIF(LE, (skygw_log_write_flush( + LOGFILE_ERROR, + "Error : Read failed, dcb is %s.", + dcb->fd == DCBFD_CLOSED ? "closed" : "cloned, not readable"))); + n = 0; + goto return_n; + } + + while (true) + { + int bufsize; + + rc = ioctl(dcb->fd, FIONREAD, &b); + + if (rc == -1) + { + LOGIF(LE, (skygw_log_write_flush( + LOGFILE_ERROR, + "Error : ioctl FIONREAD for dcb %p in " + "state %s fd %d failed due error %d, %s.", + dcb, + STRDCBSTATE(dcb->state), + dcb->fd, + errno, + strerror(errno)))); + n = -1; + goto return_n; + } + + if (b == 0 && nread == 0) + { + /** Handle closed client socket */ + if (dcb_isclient(dcb)) + { + char c; + int l_errno = 0; + int r = -1; + + /* try to read 1 byte, without consuming the socket buffer */ + r = recv(dcb->fd, &c, sizeof(char), MSG_PEEK); + l_errno = errno; + + if (r <= 0 && + l_errno != EAGAIN && + l_errno != EWOULDBLOCK && + l_errno != 0) + { + n = -1; + goto return_n; + } + } + n = 0; + goto return_n; + } + else if (b == 0) + { + n = 0; + goto return_n; + } + + dcb->last_read = hkheartbeat; + + bufsize = MIN(b, MAX_BUFFER_SIZE); + + if ((buffer = gwbuf_alloc(bufsize)) == NULL) + { + /*< + * This is a fatal error which should cause shutdown. + * Todo shutdown if memory allocation fails. + */ + LOGIF(LE, (skygw_log_write_flush( + LOGFILE_ERROR, + "Error : Failed to allocate read buffer " + "for dcb %p fd %d, due %d, %s.", + dcb, + dcb->fd, + errno, + strerror(errno)))); + + n = -1; + goto return_n; + } + GW_NOINTR_CALL(n = SSL_read(ssl, GWBUF_DATA(buffer), bufsize); + dcb->stats.n_reads++); + + if (n <= 0) + { + int ssl_errno = ERR_get_error(); + if(ssl_errno != SSL_ERROR_WANT_READ) + { + LOGIF(LE, (skygw_log_write_flush( + LOGFILE_ERROR, + "Error : Read failed, dcb %p in state " + "%s fd %d: %s.", + dcb, + STRDCBSTATE(dcb->state), + dcb->fd, + ERR_error_string(ssl_errno,NULL)))); + + gwbuf_free(buffer); + goto return_n; + } + } + nread += n; + + LOGIF(LD, (skygw_log_write( + LOGFILE_DEBUG, + "%lu [dcb_read] Read %d bytes from dcb %p in state %s " + "fd %d.", + pthread_self(), + n, + dcb, + STRDCBSTATE(dcb->state), + dcb->fd))); + /*< Append read data to the gwbuf */ + *head = gwbuf_append(*head, buffer); + } /*< while (true) */ +return_n: + return n; +} /** * General purpose routine to write to a DCB * @@ -905,11 +1052,11 @@ int below_water; return 0; } /** - * SESSION_STATE_STOPPING means that one of the backends is closing - * the router session. Some backends may have not completed + * SESSION_STATE_STOPPING means that one of the backends is closing + * the router session. Some backends may have not completed * authentication yet and thus they have no information about router * being closed. Session state is changed to SESSION_STATE_STOPPING - * before router's closeSession is called and that tells that DCB may + * before router's closeSession is called and that tells that DCB may * still be writable. */ if (queue == NULL || @@ -932,9 +1079,9 @@ int below_water; //ss_dassert(false); return 0; } - + spinlock_acquire(&dcb->writeqlock); - + if (dcb->writeq != NULL) { /* @@ -949,7 +1096,7 @@ int below_water; if (queue) { int qlen; - + qlen = gwbuf_length(queue); atomic_add(&dcb->writeqlen, qlen); dcb->writeq = gwbuf_append(dcb->writeq, queue); @@ -998,7 +1145,7 @@ int below_water; w = gw_write(dcb, GWBUF_DATA(queue), qlen); dcb->stats.n_writes++; ); - + if (w < 0) { saved_errno = errno; @@ -1006,7 +1153,7 @@ int below_water; if (LOG_IS_ENABLED(LOGFILE_DEBUG)) { - if (saved_errno == EPIPE) + if (saved_errno == EPIPE) { LOGIF(LD, (skygw_log_write( LOGFILE_DEBUG, @@ -1019,9 +1166,9 @@ int below_water; dcb->fd, saved_errno, strerror(saved_errno)))); - } + } } - + if (LOG_IS_ENABLED(LOGFILE_ERROR)) { if (saved_errno != EPIPE && @@ -1066,7 +1213,7 @@ int below_water; if (queue) { int qlen; - + qlen = gwbuf_length(queue); atomic_add(&dcb->writeqlen, qlen); dcb->stats.n_buffered++; @@ -1086,7 +1233,7 @@ int below_water; if (GWBUF_IS_TYPE_MYSQL(queue)) { uint8_t* data = GWBUF_DATA(queue); - + if (data[4] == 0x01) { dolog = false; @@ -1116,6 +1263,262 @@ int below_water; return 1; } +/** + * General purpose routine to write to an SSL enabled DCB + * + * @param dcb The DCB of the client + * @param ssl The SSL structure for this DCB + * @param queue Queue of buffers to write + * @return 0 on failure, 1 on success + */ +int +dcb_write_SSL(DCB *dcb, SSL* ssl, GWBUF *queue) +{ + int w; + int saved_errno = 0; + int below_water; + + below_water = (dcb->high_water && dcb->writeqlen < dcb->high_water) ? 1 : 0; + ss_dassert(queue != NULL); + + if (dcb->fd <= 0) + { + LOGIF(LE, (skygw_log_write_flush( + LOGFILE_ERROR, + "Error : Write failed, dcb is %s.", + dcb->fd == DCBFD_CLOSED ? "closed" : "cloned, not writable"))); + return 0; + } + + /** + * SESSION_STATE_STOPPING means that one of the backends is closing + * the router session. Some backends may have not completed + * authentication yet and thus they have no information about router + * being closed. Session state is changed to SESSION_STATE_STOPPING + * before router's closeSession is called and that tells that DCB may + * still be writable. + */ + if (queue == NULL || + (dcb->state != DCB_STATE_ALLOC && + dcb->state != DCB_STATE_POLLING && + dcb->state != DCB_STATE_LISTENING && + dcb->state != DCB_STATE_NOPOLLING && + (dcb->session == NULL || + dcb->session->state != SESSION_STATE_STOPPING))) + { + LOGIF(LD, (skygw_log_write( + LOGFILE_DEBUG, + "%lu [dcb_write] Write aborted to dcb %p because " + "it is in state %s", + pthread_self(), + dcb->stats.n_buffered, + dcb, + STRDCBSTATE(dcb->state), + dcb->fd))); + //ss_dassert(false); + return 0; + } + + spinlock_acquire(&dcb->writeqlock); + + if (dcb->writeq != NULL) + { + /* + * We have some queued data, so add our data to + * the write queue and return. + * The assumption is that there will be an EPOLLOUT + * event to drain what is already queued. We are protected + * by the spinlock, which will also be acquired by the + * the routine that drains the queue data, so we should + * not have a race condition on the event. + */ + if (queue) + { + int qlen; + + qlen = gwbuf_length(queue); + atomic_add(&dcb->writeqlen, qlen); + dcb->writeq = gwbuf_append(dcb->writeq, queue); + dcb->stats.n_buffered++; + LOGIF(LD, (skygw_log_write( + LOGFILE_DEBUG, + "%lu [dcb_write] Append to writequeue. %d writes " + "buffered for dcb %p in state %s fd %d", + pthread_self(), + dcb->stats.n_buffered, + dcb, + STRDCBSTATE(dcb->state), + dcb->fd))); + } + } + else + { + /* + * Loop over the buffer chain that has been passed to us + * from the reading side. + * Send as much of the data in that chain as possible and + * add any balance to the write queue. + */ + while (queue != NULL) + { + int qlen; +#if defined(FAKE_CODE) + if (dcb->dcb_role == DCB_ROLE_REQUEST_HANDLER && + dcb->session != NULL) + { + if (dcb_isclient(dcb) && fail_next_client_fd) { + dcb_fake_write_errno[dcb->fd] = 32; + dcb_fake_write_ev[dcb->fd] = 29; + fail_next_client_fd = false; + } else if (!dcb_isclient(dcb) && + fail_next_backend_fd) + { + dcb_fake_write_errno[dcb->fd] = 32; + dcb_fake_write_ev[dcb->fd] = 29; + fail_next_backend_fd = false; + } + } +#endif /* FAKE_CODE */ + qlen = GWBUF_LENGTH(queue); + GW_NOINTR_CALL( + w = gw_write_SSL(ssl, GWBUF_DATA(queue), qlen); + dcb->stats.n_writes++; + ); + + if (w < 0) + { + int ssl_errno = ERR_get_error(); + + if (LOG_IS_ENABLED(LOGFILE_DEBUG)) + { + switch(ssl_errno) + { + case SSL_ERROR_WANT_READ: + LOGIF(LD, (skygw_log_write( + LOGFILE_DEBUG, + "%lu [dcb_write] Write to dcb " + "%p in state %s fd %d failed " + "due error SSL_ERROR_WANT_READ", + pthread_self(), + dcb, + STRDCBSTATE(dcb->state), + dcb->fd))); + break; + case SSL_ERROR_WANT_WRITE: + LOGIF(LD, (skygw_log_write( + LOGFILE_DEBUG, + "%lu [dcb_write] Write to dcb " + "%p in state %s fd %d failed " + "due error SSL_ERROR_WANT_WRITE", + pthread_self(), + dcb, + STRDCBSTATE(dcb->state), + dcb->fd))); + break; + default: + LOGIF(LD, (skygw_log_write( + LOGFILE_DEBUG, + "%lu [dcb_write] Write to dcb " + "%p in state %s fd %d failed " + "due error %d", + pthread_self(), + dcb, + STRDCBSTATE(dcb->state), + dcb->fd,ssl_errno))); + } + } + + if (LOG_IS_ENABLED(LOGFILE_ERROR)) + { + if (ssl_errno != 0) + { + LOGIF(LE, (skygw_log_write_flush( + LOGFILE_ERROR, + "Error : Write to dcb %p in " + "state %s fd %d failed due " + "SSL error %d", + dcb, + STRDCBSTATE(dcb->state), + dcb->fd, + ssl_errno))); + } + } + break; + } + /* + * Pull the number of bytes we have written from + * queue with have. + */ + queue = gwbuf_consume(queue, w); + LOGIF(LD, (skygw_log_write( + LOGFILE_DEBUG, + "%lu [dcb_write] Wrote %d Bytes to dcb %p in " + "state %s fd %d", + pthread_self(), + w, + dcb, + STRDCBSTATE(dcb->state), + dcb->fd))); + } /*< while (queue != NULL) */ + /*< + * What wasn't successfully written is stored to write queue + * for suspended write. + */ + dcb->writeq = queue; + + if (queue) + { + int qlen; + + qlen = gwbuf_length(queue); + atomic_add(&dcb->writeqlen, qlen); + dcb->stats.n_buffered++; + } + } /* if (dcb->writeq) */ + + if (saved_errno != 0 && + queue != NULL && + saved_errno != EAGAIN && + saved_errno != EWOULDBLOCK) + { + bool dolog = true; + + /** + * Do not log if writing COM_QUIT to backend failed. + */ + if (GWBUF_IS_TYPE_MYSQL(queue)) + { + uint8_t* data = GWBUF_DATA(queue); + + if (data[4] == 0x01) + { + dolog = false; + } + } + if (dolog) + { + LOGIF(LD, (skygw_log_write( + LOGFILE_DEBUG, + "%lu [dcb_write] Writing to %s socket failed due %d, %s.", + pthread_self(), + dcb_isclient(dcb) ? "client" : "backend server", + saved_errno, + strerror(saved_errno)))); + } + spinlock_release(&dcb->writeqlock); + return 0; + } + spinlock_release(&dcb->writeqlock); + + if (dcb->high_water && dcb->writeqlen > dcb->high_water && below_water) + { + atomic_add(&dcb->stats.n_high_water, 1); + dcb_call_callback(dcb, DCB_REASON_HIGH_WATER); + } + + return 1; +} + /** * Drain the write queue of a DCB. This is called as part of the EPOLLOUT handling * of a socket and will try to send any buffered data from the write queue @@ -1208,6 +1611,85 @@ int above_water; return n; } + +/** + * Drain the write queue of a DCB. This is called as part of the EPOLLOUT handling + * of a socket and will try to send any buffered data from the write queue + * up until the point the write would block. + * + * @param dcb DCB to drain the write queue of + * @return The number of bytes written + */ +int +dcb_drain_writeq_SSL(DCB *dcb, SSL* ssl) +{ + int n = 0; + int w; + int saved_errno = 0; + int above_water; + + above_water = (dcb->low_water && dcb->writeqlen > dcb->low_water) ? 1 : 0; + + spinlock_acquire(&dcb->writeqlock); + + if (dcb->writeq) + { + int len; + /* + * Loop over the buffer chain in the pending writeq + * Send as much of the data in that chain as possible and + * leave any balance on the write queue. + */ + while (dcb->writeq != NULL) + { + len = GWBUF_LENGTH(dcb->writeq); + GW_NOINTR_CALL(w = gw_write_SSL(ssl, GWBUF_DATA(dcb->writeq), len);); + + if (w < 0) + { + int ssl_errno = ERR_get_error(); + + if(ssl_errno == SSL_ERROR_WANT_WRITE || + ssl_errno == SSL_ERROR_WANT_ACCEPT || + ssl_errno == SSL_ERROR_WANT_READ) + { + break; + } + + skygw_log_write_flush( + LOGFILE_ERROR, + "Error : Write to dcb %p " + "in state %s fd %d failed: %s", + dcb, + STRDCBSTATE(dcb->state), + dcb->fd, + ERR_error_string(ssl_errno,NULL)); + break; + } + /* + * Pull the number of bytes we have written from + * queue with have. + */ + dcb->writeq = gwbuf_consume(dcb->writeq, w); + n += w; + } + } + spinlock_release(&dcb->writeqlock); + atomic_add(&dcb->writeqlen, -n); + + /* The write queue has drained, potentially need to call a callback function */ + if (dcb->writeq == NULL) + dcb_call_callback(dcb, DCB_REASON_DRAINED); + + if (above_water && dcb->writeqlen < dcb->low_water) + { + atomic_add(&dcb->stats.n_low_water, 1); + dcb_call_callback(dcb, DCB_REASON_LOW_WATER); + } + + return n; +} + /** * Removes dcb from poll set, and adds it to zombies list. As a consequense, * dcb first moves to DCB_STATE_NOPOLLING, and then to DCB_STATE_ZOMBIE state. @@ -1792,6 +2274,29 @@ static bool dcb_set_state_nomutex( return succp; } +/** + * Write data to a DCB + * + * @param ssl The SSL to write the buffer to + * @param buf Buffer to write + * @param nbytes Number of bytes to write + * @return Number of written bytes + */ +int +gw_write_SSL(SSL* ssl, const void *buf, size_t nbytes) +{ + int w = 0; + int fd = SSL_get_fd(ssl); + + if (fd > 0) + { + w = SSL_write(ssl, buf, nbytes); + } + return w; +} + + + /** * Write data to a DCB * diff --git a/server/include/dcb.h b/server/include/dcb.h index 7eedfbec3..38ebdc299 100644 --- a/server/include/dcb.h +++ b/server/include/dcb.h @@ -23,6 +23,9 @@ #include #include #include +#include +#include +#include #define ERRHANDLE @@ -337,7 +340,10 @@ bool dcb_set_state(DCB* dcb, dcb_state_t new_state, dcb_state_t* old_state); void dcb_call_foreach (struct server* server, DCB_REASON reason); size_t dcb_get_session_id(DCB* dcb); bool dcb_get_ses_log_info(DCB* dcb, size_t* sesid, int* enabled_logs); - +int gw_write_SSL(SSL* ssl, const void *buf, size_t nbytes); +int dcb_write_SSL(DCB *dcb, SSL* ssl, GWBUF *queue); +int dcb_read_SSL(DCB *dcb,SSL* ssl,GWBUF **head); +int dcb_drain_writeq_SSL(DCB *dcb, SSL* ssl); /** diff --git a/server/modules/include/mysql_client_server_protocol.h b/server/modules/include/mysql_client_server_protocol.h index 87dbc50ee..6c841d9c9 100644 --- a/server/modules/include/mysql_client_server_protocol.h +++ b/server/modules/include/mysql_client_server_protocol.h @@ -297,6 +297,7 @@ typedef struct { * handshake */ unsigned int charset; /*< MySQL character set at connect time */ SSL* ssl; /*< SSL struct for client connection */ + bool use_ssl; #if defined(SS_DEBUG) skygw_chk_t protocol_chk_tail; #endif diff --git a/server/modules/protocol/mysql_client.c b/server/modules/protocol/mysql_client.c index 387b59aa0..f1aa0aa94 100644 --- a/server/modules/protocol/mysql_client.c +++ b/server/modules/protocol/mysql_client.c @@ -37,7 +37,7 @@ * 09/09/2014 Massimiliano Pinto Added: 777 permission for socket path * 13/10/2014 Massimiliano Pinto Added: dbname authentication check * 10/11/2014 Massimiliano Pinto Added: client charset added to protocol struct - * + * 29/05/2015 Markus Makela Added SSL support */ #include #include @@ -69,7 +69,9 @@ static int gw_MySQLWrite_client(DCB *dcb, GWBUF *queue); static int gw_error_client_event(DCB *dcb); static int gw_client_close(DCB *dcb); static int gw_client_hangup_event(DCB *dcb); - +int gw_read_client_event_SSL(DCB* dcb); +int gw_MySQLWrite_client_SSL(DCB *dcb, GWBUF *queue); +int gw_write_client_event_SSL(DCB *dcb); int mysql_send_ok(DCB *dcb, int packet_number, int in_affected_rows, const char* mysql_message); int MySQLSendHandshake(DCB* dcb); static int gw_mysql_do_authentication(DCB *dcb, GWBUF *queue); @@ -464,6 +466,8 @@ static int gw_mysql_do_authentication(DCB *dcb, GWBUF *queue) { &protocol->client_capabilities); */ + if(protocol->protocol_auth_state == MYSQL_AUTH_SSL_EXCHANGE_DONE) + goto ssl_hs_done; ssl = protocol->client_capabilities & GW_MYSQL_CAPABILITIES_SSL; @@ -497,48 +501,19 @@ static int gw_mysql_do_authentication(DCB *dcb, GWBUF *queue) { protocol->ssl = SSL_new(protocol->owner_dcb->service->ctx); SSL_set_fd(protocol->ssl,dcb->fd); protocol->protocol_auth_state = MYSQL_AUTH_SSL_REQ; - printf("%s\n",SSL_get_version(protocol->ssl)); - int errnum,rval; - char errbuf[1024]; - switch((rval = SSL_accept(protocol->ssl))) + if(do_ssl_accept(protocol) < 0) { - case 0: - errnum = SSL_get_error(protocol->ssl,rval); - ERR_error_string(errnum,errbuf); - skygw_log_write_flush(LOGFILE_ERROR,"SSL_accept: %s",errbuf); - ERR_print_errors_fp(stdout); - ERR_error_string(errnum,errbuf); - printf("%s\n",errbuf); - fflush(stdout); - break; - case 1: - protocol->protocol_auth_state = MYSQL_AUTH_SSL_EXCHANGE_DONE; - break; - - default: - errnum = SSL_get_error(protocol->ssl,rval); - if(errnum == SSL_ERROR_WANT_READ) - { - /** Not all of the data has been read. Go back to the poll - queue and wait for more.*/ - protocol->protocol_auth_state = MYSQL_AUTH_SSL_RECV; - return 0; - } - else - { - ERR_print_errors_fp(stdout); - ERR_error_string(errnum,errbuf); - printf("%s\n",errbuf); - fflush(stdout); - skygw_log_write_flush(LOGFILE_ERROR,"Error: Fatal error in SSL_accept: %s",errbuf); - protocol->protocol_auth_state = MYSQL_AUTH_SSL_EXCHANGE_ERR; - } - break; - + return 1; + } + else + { + return 0; } } + ssl_hs_done: + username = get_username_from_auth(username, client_auth_packet); if (username == NULL) @@ -645,6 +620,65 @@ gw_MySQLWrite_client(DCB *dcb, GWBUF *queue) return dcb_write(dcb, queue); } + +/** + * Write function for client DCB: writes data from MaxScale to Client + * + * @param dcb The DCB of the client + * @param queue Queue of buffers to write + */ +int +gw_MySQLWrite_client_SSL(DCB *dcb, GWBUF *queue) +{ + MySQLProtocol *protocol = NULL; + CHK_DCB(dcb); + protocol = DCB_PROTOCOL(dcb, MySQLProtocol); + CHK_PROTOCOL(protocol); + return dcb_write_SSL(dcb, protocol->ssl, queue); +} + + +int gw_read_client_event_SSL( +DCB* dcb) +{ + SESSION *session = NULL; + ROUTER_OBJECT *router = NULL; + ROUTER *router_instance = NULL; + void *rsession = NULL; + MySQLProtocol *protocol = NULL; + GWBUF *read_buffer = NULL; + int rc = 0; + int nbytes_read = 0; + uint8_t cap = 0; + bool stmt_input = false; /*< router input type */ + + CHK_DCB(dcb); + protocol = DCB_PROTOCOL(dcb, MySQLProtocol); + CHK_PROTOCOL(protocol); + + + if(protocol->protocol_auth_state == MYSQL_AUTH_SSL_REQ) + { + if(do_ssl_accept(protocol) == 1) + { + spinlock_acquire(&protocol->protocol_lock); + protocol->protocol_auth_state = MYSQL_AUTH_SSL_EXCHANGE_DONE; + spinlock_release(&protocol->protocol_lock); + + spinlock_acquire(&dcb->authlock); + dcb->func.read = gw_read_client_event_SSL; + dcb->func.write = gw_MySQLWrite_client_SSL; + dcb->func.write_ready = gw_write_client_event; + spinlock_release(&dcb->authlock); + } + goto return_rc; + } + + return_rc: + + return rc; +} + /** * Client read event triggered by EPOLLIN * @@ -668,13 +702,36 @@ int gw_read_client_event( CHK_DCB(dcb); protocol = DCB_PROTOCOL(dcb, MySQLProtocol); CHK_PROTOCOL(protocol); - if(protocol->protocol_auth_state == MYSQL_AUTH_SSL_RECV) + + /** Let the OpenSSL API do the reading from the socket */ + if(protocol->protocol_auth_state == MYSQL_AUTH_SSL_REQ) { - goto do_auth; + if(do_ssl_accept(protocol) == 1) + { + spinlock_acquire(&protocol->protocol_lock); + protocol->protocol_auth_state = MYSQL_AUTH_SSL_EXCHANGE_DONE; + protocol->use_ssl = true; + spinlock_release(&protocol->protocol_lock); + + spinlock_acquire(&dcb->authlock); + //dcb->func.read = gw_read_client_event_SSL; + //dcb->func.write = gw_MySQLWrite_client_SSL; + //dcb->func.write_ready = gw_write_client_event_SSL; + spinlock_release(&dcb->authlock); + } + goto return_rc; } - rc = dcb_read(dcb, &read_buffer); - - + + + if(protocol->use_ssl) + { + rc = dcb_read_SSL(dcb,protocol->ssl, &read_buffer); + } + else + { + rc = dcb_read(dcb, &read_buffer); + } + if (rc < 0) { dcb_close(dcb); @@ -811,7 +868,7 @@ int gw_read_client_event( } } - do_auth: + /** * Now there should be at least one complete mysql packet in read_buffer. */ @@ -823,7 +880,7 @@ int gw_read_client_event( auth_val = gw_mysql_do_authentication(dcb, read_buffer); - if(protocol->protocol_auth_state == MYSQL_AUTH_SSL_RECV) + if(protocol->protocol_auth_state == MYSQL_AUTH_SSL_REQ) break; if (auth_val == 0) @@ -919,12 +976,9 @@ int gw_read_client_event( } break; - case MYSQL_AUTH_SSL_RECV: + case MYSQL_AUTH_SSL_EXCHANGE_DONE: { - if(do_ssl_accept(protocol) == 1) - { - protocol->protocol_auth_state = MYSQL_AUTH_SSL_EXCHANGE_DONE; - } + protocol->protocol_auth_state = MYSQL_AUTH_SSL_EXCHANGE_DONE; } break; @@ -1047,6 +1101,64 @@ return_rc: return rc; } + +/////////////////////////////////////////////// +// client write event to Client triggered by EPOLLOUT +////////////////////////////////////////////// +/** + * @node Client's fd became writable, and EPOLLOUT event + * arrived. As a consequence, client input buffer (writeq) is flushed. + * + * Parameters: + * @param dcb - in, use + * client dcb + * + * @return constantly 1 + * + * + * @details (write detailed description here) + * + */ +int gw_write_client_event(DCB *dcb) +{ + MySQLProtocol *protocol = NULL; + + CHK_DCB(dcb); + + ss_dassert(dcb->state != DCB_STATE_DISCONNECTED); + + if (dcb == NULL) { + goto return_1; + } + + if (dcb->state == DCB_STATE_DISCONNECTED) { + goto return_1; + } + + if (dcb->protocol == NULL) { + goto return_1; + } + protocol = (MySQLProtocol *)dcb->protocol; + CHK_PROTOCOL(protocol); + + if (protocol->protocol_auth_state == MYSQL_IDLE) + { + dcb_drain_writeq(dcb); + goto return_1; + } + +return_1: +#if defined(SS_DEBUG) + if (dcb->state == DCB_STATE_POLLING || + dcb->state == DCB_STATE_NOPOLLING || + dcb->state == DCB_STATE_ZOMBIE) + { + CHK_PROTOCOL(protocol); + } +#endif + return 1; +} + /////////////////////////////////////////////// // client write event to Client triggered by EPOLLOUT ////////////////////////////////////////////// @@ -1064,7 +1176,7 @@ return_rc: * @details (write detailed description here) * */ -int gw_write_client_event(DCB *dcb) +int gw_write_client_event_SSL(DCB *dcb) { MySQLProtocol *protocol = NULL; @@ -1088,7 +1200,7 @@ int gw_write_client_event(DCB *dcb) if (protocol->protocol_auth_state == MYSQL_IDLE) { - dcb_drain_writeq(dcb); + dcb_drain_writeq_SSL(dcb,protocol->ssl); goto return_1; } @@ -1776,16 +1888,17 @@ int do_ssl_accept(MySQLProtocol* protocol) { case 0: errnum = SSL_get_error(protocol->ssl,rval); - ERR_error_string(errnum,errbuf); - skygw_log_write_flush(LOGFILE_ERROR,"SSL_accept: %s",errbuf); - ERR_print_errors_fp(stdout); - ERR_error_string(errnum,errbuf); - printf("%s\n",errbuf); - fflush(stdout); + skygw_log_write_flush(LT,"SSL_accept ongoing for %s@%s", + protocol->owner_dcb->user, + protocol->owner_dcb->remote); break; case 1: protocol->protocol_auth_state = MYSQL_AUTH_SSL_EXCHANGE_DONE; rval = 1; + protocol->use_ssl = true; + skygw_log_write_flush(LT,"SSL_accept done for %s@%s", + protocol->owner_dcb->user, + protocol->owner_dcb->remote); break; default: @@ -1795,14 +1908,18 @@ int do_ssl_accept(MySQLProtocol* protocol) /** Not all of the data has been read. Go back to the poll queue and wait for more.*/ rval = 0; + protocol->protocol_auth_state = MYSQL_AUTH_SSL_REQ; + skygw_log_write_flush(LT,"SSL_accept partially done for %s@%s", + protocol->owner_dcb->user, + protocol->owner_dcb->remote); } else { - ERR_print_errors_fp(stdout); - ERR_error_string(errnum,errbuf); - printf("%s\n",errbuf); - fflush(stdout); - skygw_log_write_flush(LOGFILE_ERROR,"Error: Fatal error in SSL_accept: %s",errbuf); + skygw_log_write_flush(LE, + "Error: Fatal error in SSL_accept for %s@%s: %s", + protocol->owner_dcb->user, + protocol->owner_dcb->remote, + ERR_error_string(errnum,NULL)); protocol->protocol_auth_state = MYSQL_AUTH_SSL_EXCHANGE_ERR; rval = -1; } diff --git a/server/modules/protocol/mysql_common.c b/server/modules/protocol/mysql_common.c index ffd72c034..654d24692 100644 --- a/server/modules/protocol/mysql_common.c +++ b/server/modules/protocol/mysql_common.c @@ -137,7 +137,10 @@ void mysql_protocol_done ( goto retblock; } scmd = p->protocol_cmd_history; - + if(p->ssl) + { + SSL_free(p->ssl); + } while (scmd != NULL) { scmd2 = scmd->scom_next; From a2768955e75134914198bf956dc1a44cefd03b75 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Mon, 1 Jun 2015 13:50:22 +0300 Subject: [PATCH 018/100] SSL handshake now successfully completes when a client connects with SSL enabled. --- Documentation/Reference/MaxScale-and-SSL.md | 13 + server/core/dcb.c | 24 +- server/core/gateway.c | 37 +- server/core/service.c | 2 +- .../include/mysql_client_server_protocol.h | 9 +- server/modules/protocol/mysql_client.c | 347 ++++++++++-------- 6 files changed, 253 insertions(+), 179 deletions(-) create mode 100644 Documentation/Reference/MaxScale-and-SSL.md diff --git a/Documentation/Reference/MaxScale-and-SSL.md b/Documentation/Reference/MaxScale-and-SSL.md new file mode 100644 index 000000000..4c583793b --- /dev/null +++ b/Documentation/Reference/MaxScale-and-SSL.md @@ -0,0 +1,13 @@ +# MaxScale and SSL + +MaxScale supports client side SSL connections. Enabling is done on a per service basis and each service has its own set of certificates. + +## SSL Options + +Here are the options which relate to SSL and certificates. +Parameter|Values|Description +---------------------------- +ssl | disabled, enabled, required |`disable` disables SSL, `enabled` enables SSL for client connections but still allows non-SSL connections and `required` requires SSL from all client connections. With the `required` option, client connections that do not use SSL will be rejected. +ssl_cert | |Path to server certificate +ssl_key | |Path to server private key +ssl_ca_cert | |Path to Certificate Authority file diff --git a/server/core/dcb.c b/server/core/dcb.c index ba2028de2..562929774 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -941,18 +941,12 @@ int dcb_read_SSL( /** Handle closed client socket */ if (dcb_isclient(dcb)) { - char c; - int l_errno = 0; + char c = 0; int r = -1; /* try to read 1 byte, without consuming the socket buffer */ - r = recv(dcb->fd, &c, sizeof(char), MSG_PEEK); - l_errno = errno; - - if (r <= 0 && - l_errno != EAGAIN && - l_errno != EWOULDBLOCK && - l_errno != 0) + r = SSL_peek(ssl, &c, sizeof(char)); + if (r <= 0) { n = -1; goto return_n; @@ -989,13 +983,15 @@ int dcb_read_SSL( n = -1; goto return_n; } - GW_NOINTR_CALL(n = SSL_read(ssl, GWBUF_DATA(buffer), bufsize); - dcb->stats.n_reads++); + n = SSL_read(ssl, GWBUF_DATA(buffer), bufsize); + dcb->stats.n_reads++; + int ssl_errno = 0; if (n <= 0) { - int ssl_errno = ERR_get_error(); - if(ssl_errno != SSL_ERROR_WANT_READ) + ssl_errno = ERR_get_error(); + + if(ssl_errno != SSL_ERROR_WANT_READ && ssl_errno != SSL_ERROR_NONE) { LOGIF(LE, (skygw_log_write_flush( LOGFILE_ERROR, @@ -1023,6 +1019,8 @@ int dcb_read_SSL( dcb->fd))); /*< Append read data to the gwbuf */ *head = gwbuf_append(*head, buffer); + if(ssl_errno == SSL_ERROR_WANT_READ || ssl_errno == SSL_ERROR_NONE) + break; } /*< while (true) */ return_n: return n; diff --git a/server/core/gateway.c b/server/core/gateway.c index 9845713e1..0b07f507b 100644 --- a/server/core/gateway.c +++ b/server/core/gateway.c @@ -196,7 +196,9 @@ static bool resolve_maxscale_conf_fname( static char* check_dir_access(char* dirname,bool,bool); static int set_user(); - +static void maxscale_ssl_lock(int mode,int n,const char* file, int line); +static unsigned long maxscale_ssl_id(); +static SPINLOCK* ssl_locks; /** * Handler for SIGHUP signal. Reload the configuration for the * gateway. @@ -1370,7 +1372,23 @@ int main(int argc, char **argv) rc = MAXSCALE_INTERNALERROR; goto return_main; } + + /** OpenSSL initialization */ + SSL_library_init(); + SSL_load_error_strings(); + int n_locks = CRYPTO_num_locks(); + if((ssl_locks = malloc(n_locks*sizeof(SPINLOCK))) == NULL) + { + rc = MAXSCALE_INTERNALERROR; + goto return_main; + } + + for(i = 0;ictx,SSL_VERIFY_PEER,NULL); /* Set the verification depth to 1 */ - SSL_CTX_set_verify_depth(service->ctx,10); + SSL_CTX_set_verify_depth(service->ctx,1); service->ssl_init_done = true; } return 0; diff --git a/server/modules/include/mysql_client_server_protocol.h b/server/modules/include/mysql_client_server_protocol.h index 6c841d9c9..94db6e664 100644 --- a/server/modules/include/mysql_client_server_protocol.h +++ b/server/modules/include/mysql_client_server_protocol.h @@ -99,10 +99,11 @@ typedef enum { MYSQL_AUTH_RECV, MYSQL_AUTH_FAILED, MYSQL_HANDSHAKE_FAILED, - MYSQL_AUTH_SSL_REQ, /*< client requested SSL */ - MYSQL_AUTH_SSL_EXCHANGE_DONE, /*< SSL handshake done */ - MYSQL_AUTH_SSL_EXCHANGE_ERR, /*< SSL handshake failure */ - MYSQL_AUTH_SSL_RECV, /*< */ + MYSQL_AUTH_SSL_REQ, /*< client requested SSL but SSL_accept hasn't beed called */ + MYSQL_AUTH_SSL_HANDSHAKE_DONE, /*< SSL handshake has been fully completed */ + MYSQL_AUTH_SSL_HANDSHAKE_FAILED, /*< SSL handshake failed for any reason */ + MYSQL_AUTH_SSL_HANDSHAKE_ONGOING, /*< SSL_accept has been called but the + * SSL handshake hasn't been completed */ MYSQL_IDLE } mysql_auth_state_t; diff --git a/server/modules/protocol/mysql_client.c b/server/modules/protocol/mysql_client.c index f1aa0aa94..8ee4b5687 100644 --- a/server/modules/protocol/mysql_client.c +++ b/server/modules/protocol/mysql_client.c @@ -117,8 +117,6 @@ version() void ModuleInit() { - SSL_library_init(); - SSL_load_error_strings(); } /** @@ -466,7 +464,7 @@ static int gw_mysql_do_authentication(DCB *dcb, GWBUF *queue) { &protocol->client_capabilities); */ - if(protocol->protocol_auth_state == MYSQL_AUTH_SSL_EXCHANGE_DONE) + if(protocol->protocol_auth_state == MYSQL_AUTH_SSL_HANDSHAKE_DONE) goto ssl_hs_done; ssl = protocol->client_capabilities & GW_MYSQL_CAPABILITIES_SSL; @@ -637,48 +635,6 @@ gw_MySQLWrite_client_SSL(DCB *dcb, GWBUF *queue) return dcb_write_SSL(dcb, protocol->ssl, queue); } - -int gw_read_client_event_SSL( -DCB* dcb) -{ - SESSION *session = NULL; - ROUTER_OBJECT *router = NULL; - ROUTER *router_instance = NULL; - void *rsession = NULL; - MySQLProtocol *protocol = NULL; - GWBUF *read_buffer = NULL; - int rc = 0; - int nbytes_read = 0; - uint8_t cap = 0; - bool stmt_input = false; /*< router input type */ - - CHK_DCB(dcb); - protocol = DCB_PROTOCOL(dcb, MySQLProtocol); - CHK_PROTOCOL(protocol); - - - if(protocol->protocol_auth_state == MYSQL_AUTH_SSL_REQ) - { - if(do_ssl_accept(protocol) == 1) - { - spinlock_acquire(&protocol->protocol_lock); - protocol->protocol_auth_state = MYSQL_AUTH_SSL_EXCHANGE_DONE; - spinlock_release(&protocol->protocol_lock); - - spinlock_acquire(&dcb->authlock); - dcb->func.read = gw_read_client_event_SSL; - dcb->func.write = gw_MySQLWrite_client_SSL; - dcb->func.write_ready = gw_write_client_event; - spinlock_release(&dcb->authlock); - } - goto return_rc; - } - - return_rc: - - return rc; -} - /** * Client read event triggered by EPOLLIN * @@ -703,26 +659,26 @@ int gw_read_client_event( protocol = DCB_PROTOCOL(dcb, MySQLProtocol); CHK_PROTOCOL(protocol); - /** Let the OpenSSL API do the reading from the socket */ - if(protocol->protocol_auth_state == MYSQL_AUTH_SSL_REQ) + if(protocol->protocol_auth_state == MYSQL_AUTH_SSL_HANDSHAKE_ONGOING || + protocol->protocol_auth_state == MYSQL_AUTH_SSL_REQ) { - if(do_ssl_accept(protocol) == 1) + switch(do_ssl_accept(protocol)) { - spinlock_acquire(&protocol->protocol_lock); - protocol->protocol_auth_state = MYSQL_AUTH_SSL_EXCHANGE_DONE; - protocol->use_ssl = true; - spinlock_release(&protocol->protocol_lock); - - spinlock_acquire(&dcb->authlock); - //dcb->func.read = gw_read_client_event_SSL; - //dcb->func.write = gw_MySQLWrite_client_SSL; - //dcb->func.write_ready = gw_write_client_event_SSL; - spinlock_release(&dcb->authlock); + case 0: + return 0; + break; + case 1: + return 0; + break; + case -1: + return 1; + break; + default: + return 1; + break; } - goto return_rc; } - if(protocol->use_ssl) { rc = dcb_read_SSL(dcb,protocol->ssl, &read_buffer); @@ -880,8 +836,13 @@ int gw_read_client_event( auth_val = gw_mysql_do_authentication(dcb, read_buffer); - if(protocol->protocol_auth_state == MYSQL_AUTH_SSL_REQ) + if(protocol->protocol_auth_state == MYSQL_AUTH_SSL_REQ || + protocol->protocol_auth_state == MYSQL_AUTH_SSL_HANDSHAKE_ONGOING || + protocol->protocol_auth_state == MYSQL_AUTH_SSL_HANDSHAKE_DONE || + protocol->protocol_auth_state == MYSQL_AUTH_SSL_HANDSHAKE_FAILED) + { break; + } if (auth_val == 0) { @@ -976,9 +937,103 @@ int gw_read_client_event( } break; - case MYSQL_AUTH_SSL_EXCHANGE_DONE: + case MYSQL_AUTH_SSL_HANDSHAKE_DONE: { - protocol->protocol_auth_state = MYSQL_AUTH_SSL_EXCHANGE_DONE; + int auth_val; + + auth_val = gw_mysql_do_authentication(dcb, read_buffer); + + + if (auth_val == 0) + { + SESSION *session; + + protocol->protocol_auth_state = MYSQL_AUTH_RECV; + /** + * Create session, and a router session for it. + * If successful, there will be backend connection(s) + * after this point. + */ + session = session_alloc(dcb->service, dcb); + + if (session != NULL) + { + CHK_SESSION(session); + ss_dassert(session->state != SESSION_STATE_ALLOC); + + protocol->protocol_auth_state = MYSQL_IDLE; + /** + * Send an AUTH_OK packet to the client, + * packet sequence is # 2 + */ + mysql_send_ok(dcb, 2, 0, NULL); + } + else + { + protocol->protocol_auth_state = MYSQL_AUTH_FAILED; + LOGIF(LD, (skygw_log_write( + LOGFILE_DEBUG, + "%lu [gw_read_client_event] session " + "creation failed. fd %d, " + "state = MYSQL_AUTH_FAILED.", + pthread_self(), + protocol->owner_dcb->fd))); + + /** Send ERR 1045 to client */ + mysql_send_auth_error( + dcb, + 2, + 0, + "failed to create new session"); + + dcb_close(dcb); + } + } + else + { + char* fail_str = NULL; + + protocol->protocol_auth_state = MYSQL_AUTH_FAILED; + + if (auth_val == 2) { + /** Send error 1049 to client */ + int message_len = 25 + MYSQL_DATABASE_MAXLEN; + + fail_str = calloc(1, message_len+1); + snprintf(fail_str, message_len, "Unknown database '%s'", + (char*)((MYSQL_session *)dcb->data)->db); + + modutil_send_mysql_err_packet(dcb, 2, 0, 1049, "42000", fail_str); + } else { + /** Send error 1045 to client */ + fail_str = create_auth_fail_str((char *)((MYSQL_session *)dcb->data)->user, + dcb->remote, + (char*)((MYSQL_session *)dcb->data)->client_sha1, + (char*)((MYSQL_session *)dcb->data)->db); + modutil_send_mysql_err_packet(dcb, 2, 0, 1045, "28000", fail_str); + } + if (fail_str) + free(fail_str); + + LOGIF(LD, (skygw_log_write( + LOGFILE_DEBUG, + "%lu [gw_read_client_event] after " + "gw_mysql_do_authentication, fd %d, " + "state = MYSQL_AUTH_FAILED.", + protocol->owner_dcb->fd, + pthread_self()))); + /** + * Release MYSQL_session since it is not used anymore. + */ + if (!DCB_IS_CLONE(dcb)) + { + free(dcb->data); + } + dcb->data = NULL; + + dcb_close(dcb); + } + read_buffer = gwbuf_consume(read_buffer, nbytes_read); } break; @@ -1821,111 +1876,83 @@ return_rc: return rc; } - /** - * Create a character array including the query string. - * GWBUF given as input includes either one complete or partial query. - * Length of buffer is at most the query length+4 (length of packet header). + * Do the SSL authentication handshake. + * This functions + * @param protocol + * @return */ -#if defined(NOT_USED) -static char* gw_get_or_create_querystr ( - void* data, - bool* new_allocation) -{ - GWBUF* buf = (GWBUF *)data; - size_t buflen; /*< first gw buffer data length */ - size_t packetlen; /*< length of mysql packet */ - size_t querylen; /*< total buffer length- */ - size_t nbytes_copied; - char* startpos; /*< first byte of query in gw buffer */ - char* str; /*< resulting query string */ - - CHK_GWBUF(buf); - packetlen = MYSQL_GET_PACKET_LEN((uint8_t *)GWBUF_DATA(buf)); - str = (char *)malloc(packetlen); /*< leave space for terminating null */ - - if (str == NULL) - { - goto return_str; - } - *new_allocation = true; - /** - * First buffer includes 4 bytes header and a type indicator byte. - */ - buflen = GWBUF_LENGTH(buf); - querylen = packetlen-1; - ss_dassert(buflen<=querylen+5); /*< 5 == header+type indicator */ - startpos = (char *)GWBUF_DATA(buf)+5; - nbytes_copied = MIN(querylen, buflen-5); - memcpy(str, startpos, nbytes_copied); - memset(&str[querylen-1], 0, 1); - buf = gwbuf_consume(buf, querylen-1); - - /** - * In case of multi-packet statement whole buffer consists of query - * string. - */ - while (buf != NULL) - { - buflen = GWBUF_LENGTH(buf); - memcpy(str+nbytes_copied, GWBUF_DATA(buf), buflen); - nbytes_copied += buflen; - buf = gwbuf_consume(buf, buflen); - } - ss_dassert(str[querylen-1] == 0); - -return_str: - return str; -} -#endif - int do_ssl_accept(MySQLProtocol* protocol) { int rval,errnum; char errbuf[2014]; - - switch((rval = SSL_accept(protocol->ssl))) - { - case 0: - errnum = SSL_get_error(protocol->ssl,rval); - skygw_log_write_flush(LT,"SSL_accept ongoing for %s@%s", - protocol->owner_dcb->user, - protocol->owner_dcb->remote); - break; - case 1: - protocol->protocol_auth_state = MYSQL_AUTH_SSL_EXCHANGE_DONE; - rval = 1; - protocol->use_ssl = true; - skygw_log_write_flush(LT,"SSL_accept done for %s@%s", - protocol->owner_dcb->user, - protocol->owner_dcb->remote); - break; + DCB* dcb; - default: - errnum = SSL_get_error(protocol->ssl,rval); - if(errnum == SSL_ERROR_WANT_READ) - { - /** Not all of the data has been read. Go back to the poll - queue and wait for more.*/ - rval = 0; - protocol->protocol_auth_state = MYSQL_AUTH_SSL_REQ; - skygw_log_write_flush(LT,"SSL_accept partially done for %s@%s", - protocol->owner_dcb->user, - protocol->owner_dcb->remote); - } - else - { - skygw_log_write_flush(LE, - "Error: Fatal error in SSL_accept for %s@%s: %s", - protocol->owner_dcb->user, - protocol->owner_dcb->remote, - ERR_error_string(errnum,NULL)); - protocol->protocol_auth_state = MYSQL_AUTH_SSL_EXCHANGE_ERR; - rval = -1; - } - break; + rval = SSL_accept(protocol->ssl); - } + switch(rval) + { + case 0: + errnum = SSL_get_error(protocol->ssl,rval); + skygw_log_write_flush(LT,"SSL_accept shutdown for %s@%s", + protocol->owner_dcb->user, + protocol->owner_dcb->remote); + return -1; + break; + case 1: + spinlock_acquire(&protocol->protocol_lock); + dcb = protocol->owner_dcb; + protocol->protocol_auth_state = MYSQL_AUTH_SSL_HANDSHAKE_DONE; + protocol->use_ssl = true; + spinlock_release(&protocol->protocol_lock); + + spinlock_acquire(&dcb->authlock); + dcb->func.write = gw_MySQLWrite_client_SSL; + dcb->func.write_ready = gw_write_client_event_SSL; + spinlock_release(&dcb->authlock); + + rval = 1; + + skygw_log_write_flush(LT,"SSL_accept done for %s@%s", + protocol->owner_dcb->user, + protocol->owner_dcb->remote); + break; + + case -1: + errnum = SSL_get_error(protocol->ssl,rval); + + if(errnum == SSL_ERROR_WANT_READ || errnum == SSL_ERROR_WANT_WRITE || + errnum == SSL_ERROR_WANT_X509_LOOKUP) + { + /** Not all of the data has been read. Go back to the poll + queue and wait for more.*/ + + rval = 0; + skygw_log_write_flush(LT,"SSL_accept ongoing for %s@%s", + protocol->owner_dcb->user, + protocol->owner_dcb->remote); + } + else + { + spinlock_acquire(&protocol->protocol_lock); + protocol->protocol_auth_state = MYSQL_AUTH_SSL_HANDSHAKE_FAILED; + spinlock_release(&protocol->protocol_lock); + rval = -1; + + skygw_log_write_flush(LE, + "Error: Fatal error in SSL_accept for %s@%s: %s", + protocol->owner_dcb->user, + protocol->owner_dcb->remote, + ERR_error_string(errnum,NULL)); + } + break; + + default: + skygw_log_write_flush(LE, + "Error: Fatal error in SSL_accept, returned value was %d.", + rval); + break; + } return rval; } \ No newline at end of file From 4d5291c26329ebee8dd04a51bf0468a5f162727a Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Mon, 1 Jun 2015 13:58:26 +0300 Subject: [PATCH 019/100] Fixed wrong packet sequence number causing a disconnect from the client. --- server/modules/protocol/mysql_client.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/modules/protocol/mysql_client.c b/server/modules/protocol/mysql_client.c index 8ee4b5687..2d2633b90 100644 --- a/server/modules/protocol/mysql_client.c +++ b/server/modules/protocol/mysql_client.c @@ -966,7 +966,7 @@ int gw_read_client_event( * Send an AUTH_OK packet to the client, * packet sequence is # 2 */ - mysql_send_ok(dcb, 2, 0, NULL); + mysql_send_ok(dcb, 3, 0, NULL); } else { From d7232d8b6ecc2fab121b1326a7f489b84bb0722d Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Mon, 1 Jun 2015 20:51:26 +0300 Subject: [PATCH 020/100] Moved SSL structure to the DCB instead of the MySQL protocol. This allows for non-MySQL SSL connections. --- server/core/dcb.c | 139 ++++++++++++++++-- server/core/service.c | 6 +- server/include/dcb.h | 9 +- .../include/mysql_client_server_protocol.h | 1 - server/modules/protocol/mysql_client.c | 64 +++----- server/modules/protocol/mysql_common.c | 5 +- 6 files changed, 164 insertions(+), 60 deletions(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index 562929774..2948dae2e 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -73,6 +73,8 @@ #include #include +#include "mysql_client_server_protocol.h" + /** Defined in log_manager.cc */ extern int lm_enabled_logfiles_bitmask; extern size_t log_ses_count[]; @@ -433,7 +435,8 @@ DCB_CALLBACK *cb; free(cb); } spinlock_release(&dcb->cb_lock); - + if(dcb->ssl) + SSL_free(dcb->ssl); bitmask_free(&dcb->memdata.bitmask); free(dcb); } @@ -894,7 +897,6 @@ return_n: */ int dcb_read_SSL( DCB *dcb, - SSL* ssl, GWBUF **head) { GWBUF *buffer = NULL; @@ -945,7 +947,7 @@ int dcb_read_SSL( int r = -1; /* try to read 1 byte, without consuming the socket buffer */ - r = SSL_peek(ssl, &c, sizeof(char)); + r = SSL_peek(dcb->ssl, &c, sizeof(char)); if (r <= 0) { n = -1; @@ -983,11 +985,18 @@ int dcb_read_SSL( n = -1; goto return_n; } - n = SSL_read(ssl, GWBUF_DATA(buffer), bufsize); - dcb->stats.n_reads++; + + int npending; + n = 0; + do + { + n += SSL_read(dcb->ssl, GWBUF_DATA(buffer), bufsize); + dcb->stats.n_reads++; + }while((npending = SSL_pending(dcb->ssl)) > 0); int ssl_errno = 0; - if (n <= 0) + + if (n <= 0) { ssl_errno = ERR_get_error(); @@ -1006,6 +1015,15 @@ int dcb_read_SSL( goto return_n; } } + + if(n < b) + { + gwbuf_rtrim(buffer,b - n); + ss_dassert(gwbuf_length(buffer) == n); + LOGIF(LD,(skygw_log_write(LD,"[%lu] SSL: Truncated buffer to correct size from %d to %d bytes.\n", + b,gwbuf_length(buffer)))); + } + nread += n; LOGIF(LD, (skygw_log_write( @@ -1019,7 +1037,8 @@ int dcb_read_SSL( dcb->fd))); /*< Append read data to the gwbuf */ *head = gwbuf_append(*head, buffer); - if(ssl_errno == SSL_ERROR_WANT_READ || ssl_errno == SSL_ERROR_NONE) + if(ssl_errno == SSL_ERROR_WANT_READ || ssl_errno == SSL_ERROR_NONE || + ssl_errno == SSL_ERROR_WANT_X509_LOOKUP || SSL_ERROR_WANT_WRITE) break; } /*< while (true) */ return_n: @@ -1270,7 +1289,7 @@ int below_water; * @return 0 on failure, 1 on success */ int -dcb_write_SSL(DCB *dcb, SSL* ssl, GWBUF *queue) +dcb_write_SSL(DCB *dcb, GWBUF *queue) { int w; int saved_errno = 0; @@ -1379,7 +1398,7 @@ dcb_write_SSL(DCB *dcb, SSL* ssl, GWBUF *queue) #endif /* FAKE_CODE */ qlen = GWBUF_LENGTH(queue); GW_NOINTR_CALL( - w = gw_write_SSL(ssl, GWBUF_DATA(queue), qlen); + w = gw_write_SSL(dcb->ssl, GWBUF_DATA(queue), qlen); dcb->stats.n_writes++; ); @@ -1619,7 +1638,7 @@ int above_water; * @return The number of bytes written */ int -dcb_drain_writeq_SSL(DCB *dcb, SSL* ssl) +dcb_drain_writeq_SSL(DCB *dcb) { int n = 0; int w; @@ -1641,7 +1660,7 @@ dcb_drain_writeq_SSL(DCB *dcb, SSL* ssl) while (dcb->writeq != NULL) { len = GWBUF_LENGTH(dcb->writeq); - GW_NOINTR_CALL(w = gw_write_SSL(ssl, GWBUF_DATA(dcb->writeq), len);); + GW_NOINTR_CALL(w = gw_write_SSL(dcb->ssl, GWBUF_DATA(dcb->writeq), len);); if (w < 0) { @@ -2728,3 +2747,101 @@ DCB *ptr; spinlock_release(&dcbspin); return rval; } + +/** + * Create the SSL structure for this DCB. + * This function creates the SSL structure for the given SSL context. This context + * should be the service's context + * @param dcb + * @param context + * @return + */ +int dcb_create_SSL(DCB* dcb) +{ + + if(serviceInitSSL(dcb->service) != 0) + { + return -1; + } + + if((dcb->ssl = SSL_new(dcb->service->ctx)) == NULL) + { + skygw_log_write(LE,"Error: Failed to initialize SSL connection."); + return -1; + } + + if(SSL_set_fd(dcb->ssl,dcb->fd) == 0) + { + skygw_log_write(LE,"Error: Failed to set file descriptor for SSL connection."); + return -1; + } + + return 0; +} +/** + * Accept a SSL connection and do the SSL authentication handshake. + * This function accepts a client connection to a DCB. It assumes that the SSL + * structure has the underlying method of communication set and this method is ready + * for usage. It then proceeds with the SSL handshake and stops only if an error + * occurs or the client has not yet written enough data to complete the handshake. + * @param dcb DCB which should accept the SSL connection + * @return 1 if the handshake was successfully completed, 0 if the handshake is + * still ongoing and another call to dcb_SSL_accept should be made or -1 if an + * error occurred during the handshake and the connection should be terminated. + */ +int dcb_accept_SSL(DCB* dcb) +{ + int rval,errnum; + + rval = SSL_accept(dcb->ssl); + + switch(rval) + { + case 0: + errnum = SSL_get_error(dcb->ssl,rval); + LOGIF(LD,(skygw_log_write_flush(LD,"SSL_accept shutdown for %s@%s", + dcb->user, + dcb->remote))); + return -1; + break; + case 1: + rval = 1; + LOGIF(LD,(skygw_log_write_flush(LD,"SSL_accept done for %s@%s", + dcb->user, + dcb->remote))); + break; + + case -1: + errnum = SSL_get_error(dcb->ssl,rval); + + if(errnum == SSL_ERROR_WANT_READ || errnum == SSL_ERROR_WANT_WRITE || + errnum == SSL_ERROR_WANT_X509_LOOKUP) + { + /** Not all of the data has been read. Go back to the poll + queue and wait for more.*/ + + rval = 0; + LOGIF(LD,(skygw_log_write_flush(LD,"SSL_accept ongoing for %s@%s", + dcb->user, + dcb->remote))); + } + else + { + rval = -1; + skygw_log_write_flush(LE, + "Error: Fatal error in SSL_accept for %s@%s: %s", + dcb->user, + dcb->remote, + ERR_error_string(errnum,NULL)); + } + break; + + default: + skygw_log_write_flush(LE, + "Error: Fatal error in SSL_accept, returned value was %d.", + rval); + break; + } + + return rval; +} \ No newline at end of file diff --git a/server/core/service.c b/server/core/service.c index 77b581e81..f8b9d0e99 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -1818,24 +1818,28 @@ int serviceInitSSL(SERVICE* service) { service->method = (SSL_METHOD*)SSLv23_server_method(); service->ctx = SSL_CTX_new(service->method); - + SSL_CTX_set_read_ahead(service->ctx,1); if (SSL_CTX_use_certificate_file(service->ctx, service->ssl_cert, SSL_FILETYPE_PEM) <= 0) { + skygw_log_write(LE,"Error: Failed to set server SSL certificate."); return -1; } /* Load the private-key corresponding to the server certificate */ if (SSL_CTX_use_PrivateKey_file(service->ctx, service->ssl_key, SSL_FILETYPE_PEM) <= 0) { + skygw_log_write(LE,"Error: Failed to set server SSL key."); return -1; } /* Check if the server certificate and private-key matches */ if (!SSL_CTX_check_private_key(service->ctx)) { + skygw_log_write(LE,"Error: Server SSL certificate and key do not match."); return -1; } /* Load the RSA CA certificate into the SSL_CTX structure */ if (!SSL_CTX_load_verify_locations(service->ctx, service->ssl_ca_cert, NULL)) { + skygw_log_write(LE,"Error: Failed to set Certificate Authority file."); return -1; } diff --git a/server/include/dcb.h b/server/include/dcb.h index 38ebdc299..58a4eb532 100644 --- a/server/include/dcb.h +++ b/server/include/dcb.h @@ -268,6 +268,7 @@ typedef struct dcb { unsigned int high_water; /**< High water mark */ unsigned int low_water; /**< Low water mark */ struct server *server; /**< The associated backend server */ + SSL* ssl; /*< SSL struct for connection */ #if defined(SS_DEBUG) int dcb_port; /**< port of target server */ skygw_chk_t dcb_chk_tail; @@ -340,10 +341,12 @@ bool dcb_set_state(DCB* dcb, dcb_state_t new_state, dcb_state_t* old_state); void dcb_call_foreach (struct server* server, DCB_REASON reason); size_t dcb_get_session_id(DCB* dcb); bool dcb_get_ses_log_info(DCB* dcb, size_t* sesid, int* enabled_logs); +int dcb_create_SSL(DCB* dcb); +int dcb_accept_SSL(DCB* dcb); int gw_write_SSL(SSL* ssl, const void *buf, size_t nbytes); -int dcb_write_SSL(DCB *dcb, SSL* ssl, GWBUF *queue); -int dcb_read_SSL(DCB *dcb,SSL* ssl,GWBUF **head); -int dcb_drain_writeq_SSL(DCB *dcb, SSL* ssl); +int dcb_write_SSL(DCB *dcb,GWBUF *queue); +int dcb_read_SSL(DCB *dcb,GWBUF **head); +int dcb_drain_writeq_SSL(DCB *dcb); /** diff --git a/server/modules/include/mysql_client_server_protocol.h b/server/modules/include/mysql_client_server_protocol.h index 94db6e664..e5fd954a4 100644 --- a/server/modules/include/mysql_client_server_protocol.h +++ b/server/modules/include/mysql_client_server_protocol.h @@ -297,7 +297,6 @@ typedef struct { unsigned long tid; /*< MySQL Thread ID, in * handshake */ unsigned int charset; /*< MySQL character set at connect time */ - SSL* ssl; /*< SSL struct for client connection */ bool use_ssl; #if defined(SS_DEBUG) skygw_chk_t protocol_chk_tail; diff --git a/server/modules/protocol/mysql_client.c b/server/modules/protocol/mysql_client.c index 2d2633b90..b930d8c77 100644 --- a/server/modules/protocol/mysql_client.c +++ b/server/modules/protocol/mysql_client.c @@ -490,14 +490,7 @@ static int gw_mysql_do_authentication(DCB *dcb, GWBUF *queue) { /** Do the SSL Handshake */ if(ssl && protocol->owner_dcb->service->ssl_mode != SSL_DISABLED) { - if(serviceInitSSL(protocol->owner_dcb->service) != 0) - { - skygw_log_write(LOGFILE_ERROR,"Error: SSL initialization for service '%s' failed.", - protocol->owner_dcb->service->name); - return 1; - } - protocol->ssl = SSL_new(protocol->owner_dcb->service->ctx); - SSL_set_fd(protocol->ssl,dcb->fd); + protocol->protocol_auth_state = MYSQL_AUTH_SSL_REQ; if(do_ssl_accept(protocol) < 0) @@ -632,7 +625,7 @@ gw_MySQLWrite_client_SSL(DCB *dcb, GWBUF *queue) CHK_DCB(dcb); protocol = DCB_PROTOCOL(dcb, MySQLProtocol); CHK_PROTOCOL(protocol); - return dcb_write_SSL(dcb, protocol->ssl, queue); + return dcb_write_SSL(dcb, queue); } /** @@ -681,7 +674,7 @@ int gw_read_client_event( if(protocol->use_ssl) { - rc = dcb_read_SSL(dcb,protocol->ssl, &read_buffer); + rc = dcb_read_SSL(dcb, &read_buffer); } else { @@ -795,7 +788,7 @@ int gw_read_client_event( dcb->dcb_readqueue = gwbuf_append(dcb->dcb_readqueue, read_buffer); nbytes_read = gwbuf_length(dcb->dcb_readqueue); data = (uint8_t *)GWBUF_DATA(dcb->dcb_readqueue); - + int plen = MYSQL_GET_PACKET_LEN(data); if (nbytes_read < 3 || nbytes_read < MYSQL_GET_PACKET_LEN(data)) { rc = 0; @@ -1255,7 +1248,7 @@ int gw_write_client_event_SSL(DCB *dcb) if (protocol->protocol_auth_state == MYSQL_IDLE) { - dcb_drain_writeq_SSL(dcb,protocol->ssl); + dcb_drain_writeq_SSL(dcb); goto return_1; } @@ -1878,30 +1871,38 @@ return_rc: /** * Do the SSL authentication handshake. - * This functions - * @param protocol - * @return + * This creates the DCB SSL structure if one has not been created and starts the + * SSL handshake handling. + * @param protocol Protocol to connect with SSL + * @return 1 on success, 0 when the handshake is ongoing or -1 on error */ int do_ssl_accept(MySQLProtocol* protocol) { int rval,errnum; char errbuf[2014]; - DCB* dcb; - - rval = SSL_accept(protocol->ssl); + DCB* dcb = protocol->owner_dcb; + if(dcb->ssl == NULL) + { + if(dcb_create_SSL(dcb) != 0) + return -1; + } + rval = dcb_accept_SSL(dcb); + switch(rval) { case 0: - errnum = SSL_get_error(protocol->ssl,rval); - skygw_log_write_flush(LT,"SSL_accept shutdown for %s@%s", + /** Not all of the data has been read. Go back to the poll + queue and wait for more.*/ + + rval = 0; + skygw_log_write_flush(LT,"SSL_accept ongoing for %s@%s", protocol->owner_dcb->user, protocol->owner_dcb->remote); - return -1; + return 0; break; case 1: spinlock_acquire(&protocol->protocol_lock); - dcb = protocol->owner_dcb; protocol->protocol_auth_state = MYSQL_AUTH_SSL_HANDSHAKE_DONE; protocol->use_ssl = true; spinlock_release(&protocol->protocol_lock); @@ -1919,32 +1920,15 @@ int do_ssl_accept(MySQLProtocol* protocol) break; case -1: - errnum = SSL_get_error(protocol->ssl,rval); - if(errnum == SSL_ERROR_WANT_READ || errnum == SSL_ERROR_WANT_WRITE || - errnum == SSL_ERROR_WANT_X509_LOOKUP) - { - /** Not all of the data has been read. Go back to the poll - queue and wait for more.*/ - - rval = 0; - skygw_log_write_flush(LT,"SSL_accept ongoing for %s@%s", - protocol->owner_dcb->user, - protocol->owner_dcb->remote); - } - else - { spinlock_acquire(&protocol->protocol_lock); protocol->protocol_auth_state = MYSQL_AUTH_SSL_HANDSHAKE_FAILED; spinlock_release(&protocol->protocol_lock); rval = -1; - skygw_log_write_flush(LE, "Error: Fatal error in SSL_accept for %s@%s: %s", protocol->owner_dcb->user, - protocol->owner_dcb->remote, - ERR_error_string(errnum,NULL)); - } + protocol->owner_dcb->remote); break; default: diff --git a/server/modules/protocol/mysql_common.c b/server/modules/protocol/mysql_common.c index 654d24692..56988dc9a 100644 --- a/server/modules/protocol/mysql_common.c +++ b/server/modules/protocol/mysql_common.c @@ -137,10 +137,7 @@ void mysql_protocol_done ( goto retblock; } scmd = p->protocol_cmd_history; - if(p->ssl) - { - SSL_free(p->ssl); - } + while (scmd != NULL) { scmd2 = scmd->scom_next; From 76655e7136790bc189e678584681143108ada50b Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 2 Jun 2015 06:04:06 +0300 Subject: [PATCH 021/100] Added a call to a library function which adds all algorithms to OpenSSL to the SSL initialization code. --- server/core/gateway.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/core/gateway.c b/server/core/gateway.c index 0b07f507b..67d5ecf62 100644 --- a/server/core/gateway.c +++ b/server/core/gateway.c @@ -1377,6 +1377,8 @@ int main(int argc, char **argv) SSL_library_init(); SSL_load_error_strings(); + OPENSSL_add_all_algorithms_noconf(); + int n_locks = CRYPTO_num_locks(); if((ssl_locks = malloc(n_locks*sizeof(SPINLOCK))) == NULL) { From 6e01757455dd4ec94eeca482e85c616bf1572918 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 2 Jun 2015 06:39:51 +0300 Subject: [PATCH 022/100] Added error message to users when trying to connect to a MaxScale service that only allows SSL connections. --- .../include/mysql_client_server_protocol.h | 4 +++ server/modules/protocol/mysql_backend.c | 4 +-- server/modules/protocol/mysql_client.c | 27 ++++++++++++------- server/modules/protocol/mysql_common.c | 11 +++++++- 4 files changed, 33 insertions(+), 13 deletions(-) diff --git a/server/modules/include/mysql_client_server_protocol.h b/server/modules/include/mysql_client_server_protocol.h index e5fd954a4..f72416491 100644 --- a/server/modules/include/mysql_client_server_protocol.h +++ b/server/modules/include/mysql_client_server_protocol.h @@ -91,6 +91,10 @@ #define COM_QUIT_PACKET_SIZE (4+1) struct dcb; +#define MYSQL_FAILED_AUTH 1 +#define MYSQL_FAILED_AUTH_DB 2 +#define MYSQL_FAILED_AUTH_SSL 3 + typedef enum { MYSQL_ALLOC, MYSQL_PENDING_CONNECT, diff --git a/server/modules/protocol/mysql_backend.c b/server/modules/protocol/mysql_backend.c index ba5786851..d000474ab 100644 --- a/server/modules/protocol/mysql_backend.c +++ b/server/modules/protocol/mysql_backend.c @@ -72,7 +72,7 @@ static void backend_set_delayqueue(DCB *dcb, GWBUF *queue); static int gw_change_user(DCB *backend_dcb, SERVER *server, SESSION *in_session, GWBUF *queue); static GWBUF* process_response_data (DCB* dcb, GWBUF* readbuf, int nbytes_to_process); extern char* create_auth_failed_msg( GWBUF* readbuf, char* hostaddr, uint8_t* sha1); -extern char* create_auth_fail_str(char *username, char *hostaddr, char *sha1, char *db); +extern char* create_auth_fail_str(char *username, char *hostaddr, char *sha1, char *db,int); static bool sescmd_response_complete(DCB* dcb); @@ -1433,7 +1433,7 @@ static int gw_change_user( message = create_auth_fail_str(username, backend->session->client->remote, password_set, - ""); + "",auth_ret); if (message == NULL) { LOGIF(LE, (skygw_log_write_flush( diff --git a/server/modules/protocol/mysql_client.c b/server/modules/protocol/mysql_client.c index b930d8c77..6aeadd180 100644 --- a/server/modules/protocol/mysql_client.c +++ b/server/modules/protocol/mysql_client.c @@ -78,7 +78,7 @@ static int gw_mysql_do_authentication(DCB *dcb, GWBUF *queue); static int route_by_statement(SESSION *, GWBUF **); extern char* get_username_from_auth(char* ptr, uint8_t* data); extern int check_db_name_after_auth(DCB *, char *, int); -extern char* create_auth_fail_str(char *username, char *hostaddr, char *sha1, char *db); +extern char* create_auth_fail_str(char *username, char *hostaddr, char *sha1, char *db,int); int do_ssl_accept(MySQLProtocol* protocol); @@ -450,7 +450,7 @@ static int gw_mysql_do_authentication(DCB *dcb, GWBUF *queue) { /* Detect now if there are enough bytes to continue */ if (client_auth_packet_size < (4 + 4 + 4 + 1 + 23)) { - return 1; + return MYSQL_FAILED_AUTH; } memcpy(&protocol->client_capabilities, client_auth_packet + 4, 4); @@ -476,7 +476,7 @@ static int gw_mysql_do_authentication(DCB *dcb, GWBUF *queue) { protocol->owner_dcb->user, protocol->owner_dcb->remote, protocol->owner_dcb->service->name))); - return 1; + return MYSQL_FAILED_AUTH_SSL; } if(LOG_IS_ENABLED(LT) && ssl) @@ -495,7 +495,7 @@ static int gw_mysql_do_authentication(DCB *dcb, GWBUF *queue) { if(do_ssl_accept(protocol) < 0) { - return 1; + return MYSQL_FAILED_AUTH; } else { @@ -509,7 +509,7 @@ static int gw_mysql_do_authentication(DCB *dcb, GWBUF *queue) { if (username == NULL) { - return 1; + return MYSQL_FAILED_AUTH; } /* get charset */ @@ -902,7 +902,7 @@ int gw_read_client_event( fail_str = create_auth_fail_str((char *)((MYSQL_session *)dcb->data)->user, dcb->remote, (char*)((MYSQL_session *)dcb->data)->client_sha1, - (char*)((MYSQL_session *)dcb->data)->db); + (char*)((MYSQL_session *)dcb->data)->db,auth_val); modutil_send_mysql_err_packet(dcb, 2, 0, 1045, "28000", fail_str); } if (fail_str) @@ -996,14 +996,21 @@ int gw_read_client_event( snprintf(fail_str, message_len, "Unknown database '%s'", (char*)((MYSQL_session *)dcb->data)->db); - modutil_send_mysql_err_packet(dcb, 2, 0, 1049, "42000", fail_str); - } else { + modutil_send_mysql_err_packet(dcb, 3, 0, 1049, "42000", fail_str); + }else if(auth_val == 3){ /** Send error 1045 to client */ fail_str = create_auth_fail_str((char *)((MYSQL_session *)dcb->data)->user, dcb->remote, (char*)((MYSQL_session *)dcb->data)->client_sha1, - (char*)((MYSQL_session *)dcb->data)->db); - modutil_send_mysql_err_packet(dcb, 2, 0, 1045, "28000", fail_str); + (char*)((MYSQL_session *)dcb->data)->db,auth_val); + modutil_send_mysql_err_packet(dcb, 3, 0, 1045, "28000", fail_str); + }else { + /** Send error 1045 to client */ + fail_str = create_auth_fail_str((char *)((MYSQL_session *)dcb->data)->user, + dcb->remote, + (char*)((MYSQL_session *)dcb->data)->client_sha1, + (char*)((MYSQL_session *)dcb->data)->db,auth_val); + modutil_send_mysql_err_packet(dcb, 3, 0, 1045, "28000", fail_str); } if (fail_str) free(fail_str); diff --git a/server/modules/protocol/mysql_common.c b/server/modules/protocol/mysql_common.c index 56988dc9a..0a1d2195b 100644 --- a/server/modules/protocol/mysql_common.c +++ b/server/modules/protocol/mysql_common.c @@ -2199,7 +2199,8 @@ char *create_auth_fail_str( char *username, char *hostaddr, char *sha1, - char *db) + char *db, + int errcode) { char* errstr; const char* ferrstr; @@ -2214,6 +2215,10 @@ char *create_auth_fail_str( { ferrstr = "Access denied for user '%s'@'%s' (using password: %s) to database '%s'"; } + else if(errcode == MYSQL_FAILED_AUTH_SSL) + { + ferrstr = "Access without SSL denied"; + } else { ferrstr = "Access denied for user '%s'@'%s' (using password: %s)"; @@ -2233,6 +2238,10 @@ char *create_auth_fail_str( { sprintf(errstr, ferrstr, username, hostaddr, (*sha1 == '\0' ? "NO" : "YES"), db); } + else if(errcode == MYSQL_FAILED_AUTH_SSL) + { + sprintf(errstr, ferrstr); + } else { sprintf(errstr, ferrstr, username, hostaddr, (*sha1 == '\0' ? "NO" : "YES")); From 08e0a318268cd42fa0f70442c5642e9da682f1f2 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 2 Jun 2015 06:42:41 +0300 Subject: [PATCH 023/100] Fixed session creation failure messages using the wrong packet number when an SSL connection was made. --- server/modules/protocol/mysql_client.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/modules/protocol/mysql_client.c b/server/modules/protocol/mysql_client.c index 6aeadd180..45fc65ee0 100644 --- a/server/modules/protocol/mysql_client.c +++ b/server/modules/protocol/mysql_client.c @@ -975,7 +975,7 @@ int gw_read_client_event( /** Send ERR 1045 to client */ mysql_send_auth_error( dcb, - 2, + 3, 0, "failed to create new session"); From fc8918b1f2d77d32880ca0297dcb069fccc1e0e1 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 2 Jun 2015 09:15:08 +0300 Subject: [PATCH 024/100] Added a dcb_connect_SSL function which starts a client SSL connection. --- server/core/dcb.c | 66 ++++++++++++++++++++++++++++++++++++++++++++ server/include/dcb.h | 1 + 2 files changed, 67 insertions(+) diff --git a/server/core/dcb.c b/server/core/dcb.c index 2948dae2e..2dcbd9b4c 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -2843,5 +2843,71 @@ int dcb_accept_SSL(DCB* dcb) break; } + return rval; +} + +/** + * Initiate an SSL client connection to a server + * + * This functions starts an SSL client connection to a server which is expecting + * an SSL handshake. The DCB should already have a TCP connection to the server and + * this connection should be in a state that expects an SSL handshake. + * @param dcb DCB to connect + * @return 1 on success, -1 on error and 0 if the SSL handshake is still ongoing + */ +int dcb_connect_SSL(DCB* dcb) +{ + int rval,errnum; + + rval = SSL_connect(dcb->ssl); + + switch(rval) + { + case 0: + errnum = SSL_get_error(dcb->ssl,rval); + LOGIF(LD,(skygw_log_write_flush(LD,"SSL_connect shutdown for %s@%s", + dcb->user, + dcb->remote))); + return -1; + break; + case 1: + rval = 1; + LOGIF(LD,(skygw_log_write_flush(LD,"SSL_connect done for %s@%s", + dcb->user, + dcb->remote))); + break; + + case -1: + errnum = SSL_get_error(dcb->ssl,rval); + + if(errnum == SSL_ERROR_WANT_READ || errnum == SSL_ERROR_WANT_WRITE || + errnum == SSL_ERROR_WANT_X509_LOOKUP) + { + /** Not all of the data has been read. Go back to the poll + queue and wait for more.*/ + + rval = 0; + LOGIF(LD,(skygw_log_write_flush(LD,"SSL_connect ongoing for %s@%s", + dcb->user, + dcb->remote))); + } + else + { + rval = -1; + skygw_log_write_flush(LE, + "Error: Fatal error in SSL_connect for %s@%s: %s", + dcb->user, + dcb->remote, + ERR_error_string(errnum,NULL)); + } + break; + + default: + skygw_log_write_flush(LE, + "Error: Fatal error in SSL_connect, returned value was %d.", + rval); + break; + } + return rval; } \ No newline at end of file diff --git a/server/include/dcb.h b/server/include/dcb.h index 58a4eb532..cc96a2c0e 100644 --- a/server/include/dcb.h +++ b/server/include/dcb.h @@ -343,6 +343,7 @@ size_t dcb_get_session_id(DCB* dcb); bool dcb_get_ses_log_info(DCB* dcb, size_t* sesid, int* enabled_logs); int dcb_create_SSL(DCB* dcb); int dcb_accept_SSL(DCB* dcb); +int dcb_connect_SSL(DCB* dcb); int gw_write_SSL(SSL* ssl, const void *buf, size_t nbytes); int dcb_write_SSL(DCB *dcb,GWBUF *queue); int dcb_read_SSL(DCB *dcb,GWBUF **head); From 2f2c9c8cbc7d4f6fc343f99c8c5e9f1bd0f7b279 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Tue, 2 Jun 2015 09:45:26 +0200 Subject: [PATCH 025/100] Fix for MariaDB10 state machine Fix for MariaDB10 state machine --- server/modules/routing/binlog/blr_master.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/modules/routing/binlog/blr_master.c b/server/modules/routing/binlog/blr_master.c index f230442f1..3a0519108 100644 --- a/server/modules/routing/binlog/blr_master.c +++ b/server/modules/routing/binlog/blr_master.c @@ -466,8 +466,8 @@ char query[128]; GWBUF_CONSUME_ALL(router->saved_master.mariadb10); router->saved_master.mariadb10 = buf; blr_cache_response(router, "mariadb10", buf); - buf = blr_make_registration(router); - router->master_state = BLRM_REGISTER; + buf = blr_make_query("SHOW VARIABLES LIKE 'SERVER_UUID'"); + router->master_state = BLRM_MUUID; router->master->func.write(router->master, buf); break; case BLRM_GTIDMODE: From 94ba445d2fad80fd9dc7075db792ced8a3ea2001 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 28 May 2015 10:30:21 +0300 Subject: [PATCH 026/100] Added 5.5.5- string to the start of MariaDB 10.0 version strings. --- server/core/config.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/server/core/config.c b/server/core/config.c index d22bb777e..a2414ea7a 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -231,7 +231,7 @@ int rval; strcpy(version_string,tmp); } - ptr = strstr(tmp, "-embedded"); + ptr = strstr(version_string, "-embedded"); if (ptr) { *ptr = '\0'; } @@ -417,7 +417,21 @@ hashtable_memory_fns(monitorhash,strdup,NULL,free,NULL); } if (version_string) { + + /** Add the 5.5.5- string to the start of the version string if + * the version string starts with "10.". + * This mimics MariaDB 10.0 replication which adds 5.5.5- for backwards compatibility. */ + if(strncmp(version_string,"10.",3) == 0) + { + ((SERVICE *)(obj->element))->version_string = malloc((strlen(version_string) + + strlen("5.5.5-") + 1) * sizeof(char)); + strcpy(((SERVICE *)(obj->element))->version_string,"5.5.5-"); + strcat(((SERVICE *)(obj->element))->version_string,version_string); + } + else + { ((SERVICE *)(obj->element))->version_string = strdup(version_string); + } } else { if (gateway.version_string) ((SERVICE *)(obj->element))->version_string = strdup(gateway.version_string); From 57060cafecff9141e1883879c11cfe46d0b120d3 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 2 Jun 2015 17:00:39 +0300 Subject: [PATCH 027/100] Added SSL level configuration to services. --- .../Getting-Started/Configuration-Guide.md | 48 +++++++++++++++++ Documentation/Reference/MaxScale-and-SSL.md | 3 +- server/core/config.c | 16 ++++-- server/core/service.c | 51 ++++++++++++++++++- server/include/service.h | 12 +++++ 5 files changed, 124 insertions(+), 6 deletions(-) diff --git a/Documentation/Getting-Started/Configuration-Guide.md b/Documentation/Getting-Started/Configuration-Guide.md index 535eca2cc..e3cc4871b 100644 --- a/Documentation/Getting-Started/Configuration-Guide.md +++ b/Documentation/Getting-Started/Configuration-Guide.md @@ -326,6 +326,54 @@ Example: connection_timeout=300 ``` +### Service and SSL + +This section describes configuration parameters for services that control the SSL/TLS encrption method and the various certificate files involved in it. To enable SSL, you must configure the `ssl` parameter with either `enabled` or `required` and provide the three files for `ssl_cert`, `ssl_key` and `ssl_ca_cert`. After this, MySQL connections to this service can be encrypted with SSL. + +#### `ssl` + +This enables SSL connections to the service. If this parameter is set to either `required` or `enabled` and the three certificate files can be found (these are explained afterwards), then client connections will be encrypted with SSL. If the parameter is `enabled` then both SSL and non-SSL connections can connect to this service. If the parameter is set to `required` then only SSL connections can be used for this service and non-SSL connections will get an error when they try to connect to the service. + +#### `ssl_key` + +The SSL private key the service should use. This will be the private key that is used as the server side private key during a client-server SSL handshake. This is a required parameter for SSL enabled services. + +#### `ssl_cert` + +The SSL certificate the service should use. This will be the public certificate that is used as the server side certificate during a client-server SSL handshake. This is a required parameter for SSL enabled services. + +#### `ssl_ca_cert` + +This is the Certificate Authority file. It will be used to verify that both the client and the server certificates are valid. This is a required parameter for SSL enabled services. + +### `ssl_version` + +This parameter controls the level of encryption used. Accepted values are: + * SSLv2 + * SSLv3 + * TLSv10 + * TLSv11 + * TLSv12 + * MAX + +Example SSL enabled service configuration: + +``` +[ReadWriteSplitService] +type=service +router=readwritesplit +servers=server1,server2,server3 +user=myuser +passwd=mypasswd +ssl=required +ssl_cert=/home/markus/certs/server-cert.pem +ssl_key=/home/markus/certs/server-key.pem +ssl_ca_cert=/home/markus/certs/ca.pem +ssl_version=TLSv12 +``` + +This configuration requires all connections to be encryped with SSL. It also specifies that TLSv1.2 should be used as the encryption method. The paths to the server certificate files and the Certificate Authority file are also provided. + ### Server Server sections are used to define the backend database servers that can be formed into a service. A server may be a member of one or more services within MaxScale. Servers are identified by a server name which is the section name in the configuration file. Servers have a type parameter of server, plus address port and protocol parameters. diff --git a/Documentation/Reference/MaxScale-and-SSL.md b/Documentation/Reference/MaxScale-and-SSL.md index a4210b0de..ca61d52e2 100644 --- a/Documentation/Reference/MaxScale-and-SSL.md +++ b/Documentation/Reference/MaxScale-and-SSL.md @@ -5,9 +5,10 @@ MaxScale supports client side SSL connections. Enabling is done on a per service ## SSL Options Here are the options which relate to SSL and certificates. -Parameter|Values|Description +Parameter|Values |Description ---------|-----------|-------- ssl | disabled, enabled, required |`disable` disables SSL, `enabled` enables SSL for client connections but still allows non-SSL connections and `required` requires SSL from all client connections. With the `required` option, client connections that do not use SSL will be rejected. ssl_cert | |Path to server certificate ssl_key | |Path to server private key ssl_ca_cert | |Path to Certificate Authority file +ssl_version|SSLV2,SSLV3,TLSV10,TLSV11,TLSV12,MAX| The SSL method level, defaults to highest available encryption level which is TLSv1.2 diff --git a/server/core/config.c b/server/core/config.c index 640437270..5e03a3b19 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -345,7 +345,7 @@ hashtable_memory_fns(monitorhash,strdup,NULL,free,NULL); char *weightby; char *version_string; char *subservices; - char *ssl,*ssl_cert,*ssl_key,*ssl_ca_cert; + char *ssl,*ssl_cert,*ssl_key,*ssl_ca_cert,*ssl_version; bool is_rwsplit = false; bool is_schemarouter = false; char *allow_localhost_match_wildcard_host; @@ -358,6 +358,7 @@ hashtable_memory_fns(monitorhash,strdup,NULL,free,NULL); ssl_cert = config_get_value(obj->parameters, "ssl_cert"); ssl_key = config_get_value(obj->parameters, "ssl_key"); ssl_ca_cert = config_get_value(obj->parameters, "ssl_ca_cert"); + ssl_version = config_get_value(obj->parameters, "ssl_version"); enable_root_user = config_get_value( obj->parameters, "enable_root_user"); @@ -474,6 +475,10 @@ hashtable_memory_fns(monitorhash,strdup,NULL,free,NULL); else { serviceSetCertificates(obj->element,ssl_cert,ssl_key,ssl_ca_cert); + if(ssl_version) + { + serviceSetSSLVersion(obj->element,ssl_version); + } } } else @@ -1381,7 +1386,7 @@ int i; } else if (strcmp(name, "ms_timestamp") == 0) { - skygw_set_highp(config_truth_value(value)); + skygw_set_highp(config_truth_value((char*)value)); } else { @@ -1389,7 +1394,7 @@ int i; { if (strcasecmp(name, lognames[i].logname) == 0) { - if (config_truth_value(value)) + if (config_truth_value((char*)value)) skygw_log_enable(lognames[i].logfile); else skygw_log_disable(lognames[i].logfile); @@ -1967,6 +1972,11 @@ static char *service_params[] = "version_string", "filters", "weightby", + "ssl_cert", + "ssl_ca_cert", + "ssl", + "ssl_key", + "ssl_version", NULL }; diff --git a/server/core/service.c b/server/core/service.c index f8b9d0e99..a8df37f5b 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -141,7 +141,8 @@ SERVICE *service; service->ssl_ca_cert = NULL; service->ssl_cert = NULL; service->ssl_key = NULL; - + /** Use the highest possible SSL/TLS methods available */ + service->ssl_method_type = SERVICE_SSL_TLS_MAX; if (service->name == NULL || service->routerModule == NULL) { if (service->name) @@ -868,6 +869,22 @@ serviceSetCertificates(SERVICE *service, char* cert,char* key, char* ca_cert) service->ssl_ca_cert = strdup(ca_cert); } +void +serviceSetSSLVersion(SERVICE *service, char* version) +{ + if(strcasecmp(version,"SSLV2") == 0) + service->ssl_method_type = SERVICE_SSLV2; + else if(strcasecmp(version,"SSLV3") == 0) + service->ssl_method_type = SERVICE_SSLV3; + else if(strcasecmp(version,"TLSV10") == 0) + service->ssl_method_type = SERVICE_TLS10; + else if(strcasecmp(version,"TLSV11") == 0) + service->ssl_method_type = SERVICE_TLS11; + else if(strcasecmp(version,"TLSV12") == 0) + service->ssl_method_type = SERVICE_TLS12; + else if(strcasecmp(version,"MAX") == 0) + service->ssl_method_type = SERVICE_SSL_TLS_MAX; +} /** Enable or disable the service SSL capability*/ int serviceSetSSL(SERVICE *service, char* action) @@ -1816,7 +1833,37 @@ int serviceInitSSL(SERVICE* service) { if(!service->ssl_init_done) { - service->method = (SSL_METHOD*)SSLv23_server_method(); + switch(service->ssl_method_type) + { + case SERVICE_SSLV2: + service->method = (SSL_METHOD*)SSLv2_server_method(); + break; + case SERVICE_SSLV3: + service->method = (SSL_METHOD*)SSLv3_server_method(); + break; + case SERVICE_TLS10: + service->method = (SSL_METHOD*)TLSv1_server_method(); + break; + case SERVICE_TLS11: + service->method = (SSL_METHOD*)TLSv1_1_server_method(); + break; + case SERVICE_TLS12: + service->method = (SSL_METHOD*)TLSv1_2_server_method(); + break; + case SERVICE_SSL_MAX: + service->method = (SSL_METHOD*)SSLv23_server_method(); + break; + case SERVICE_TLS_MAX: + service->method = (SSL_METHOD*)SSLv23_server_method(); + break; + case SERVICE_SSL_TLS_MAX: + service->method = (SSL_METHOD*)SSLv23_server_method(); + break; + default: + service->method = (SSL_METHOD*)SSLv23_server_method(); + break; + } + service->ctx = SSL_CTX_new(service->method); SSL_CTX_set_read_ahead(service->ctx,1); if (SSL_CTX_use_certificate_file(service->ctx, service->ssl_cert, SSL_FILETYPE_PEM) <= 0) { diff --git a/server/include/service.h b/server/include/service.h index 1c6614656..7975518e7 100644 --- a/server/include/service.h +++ b/server/include/service.h @@ -113,6 +113,17 @@ typedef enum { SSL_REQUIRED } ssl_mode_t; +enum{ + SERVICE_SSLV2, + SERVICE_SSLV3, + SERVICE_TLS10, + SERVICE_TLS11, + SERVICE_TLS12, + SERVICE_SSL_MAX, + SERVICE_TLS_MAX, + SERVICE_SSL_TLS_MAX +}; + /** * Defines a service within the gateway. * @@ -164,6 +175,7 @@ typedef struct service { SSL *ssl; SSL_METHOD *method; /*< SSLv2/3 or TLSv1/2 methods * see: https://www.openssl.org/docs/ssl/SSL_CTX_new.html */ + int ssl_method_type; /*< Which of the SSLv2/3 or TLS1.0/1.1/1.2 methods to use */ char* ssl_cert; char* ssl_key; char* ssl_ca_cert; From 19ac70fc2fdb5f4276cd0ea4e62f85534c21b7e9 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Wed, 3 Jun 2015 13:15:45 +0300 Subject: [PATCH 028/100] Added unit tests for SSL. --- server/core/config.c | 46 ++++++++--- server/core/service.c | 12 ++- server/include/service.h | 1 + server/modules/protocol/CMakeLists.txt | 1 + server/modules/protocol/test/CMakeLists.txt | 10 +++ server/modules/protocol/test/bad_ca.cnf | 28 +++++++ server/modules/protocol/test/bad_cert.cnf | 28 +++++++ server/modules/protocol/test/bad_key.cnf | 28 +++++++ server/modules/protocol/test/bad_ssl.cnf | 28 +++++++ server/modules/protocol/test/no_ca.cnf | 28 +++++++ .../modules/protocol/test/no_server_cert.cnf | 28 +++++++ .../modules/protocol/test/no_server_key.cnf | 28 +++++++ server/modules/protocol/test/ok.cnf | 28 +++++++ server/modules/protocol/test/test_ssl.sh | 76 +++++++++++++++++++ 14 files changed, 360 insertions(+), 10 deletions(-) create mode 100644 server/modules/protocol/test/CMakeLists.txt create mode 100644 server/modules/protocol/test/bad_ca.cnf create mode 100644 server/modules/protocol/test/bad_cert.cnf create mode 100644 server/modules/protocol/test/bad_key.cnf create mode 100644 server/modules/protocol/test/bad_ssl.cnf create mode 100644 server/modules/protocol/test/no_ca.cnf create mode 100644 server/modules/protocol/test/no_server_cert.cnf create mode 100644 server/modules/protocol/test/no_server_key.cnf create mode 100644 server/modules/protocol/test/ok.cnf create mode 100755 server/modules/protocol/test/test_ssl.sh diff --git a/server/core/config.c b/server/core/config.c index 5e03a3b19..40be2c704 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -453,41 +453,69 @@ hashtable_memory_fns(monitorhash,strdup,NULL,free,NULL); if(ssl) { if(ssl_cert == NULL) + { + error_count++; skygw_log_write(LE,"Error: Server certificate missing for service '%s'." "Please provide the path to the server certificate by adding the ssl_cert= parameter", obj->object); + } if(ssl_ca_cert == NULL) + { + error_count++; skygw_log_write(LE,"Error: CA Certificate missing for service '%s'." "Please provide the path to the certificate authority certificate by adding the ssl_ca_cert= parameter", obj->object); + } if(ssl_key == NULL) + { + error_count++; skygw_log_write(LE,"Error: Server private key missing for service '%s'. " "Please provide the path to the server certificate key by adding the ssl_key= parameter" ,obj->object); + } - if(ssl_ca_cert != NULL && ssl_cert != NULL && ssl_key != NULL) + if(access(ssl_ca_cert,F_OK) != 0) { + skygw_log_write(LE,"Error: Certificate authority file for service '%s' not found: %s", + obj->object, + ssl_ca_cert); + error_count++; + } + if(access(ssl_cert,F_OK) != 0) + { + skygw_log_write(LE,"Error: Server certificate file for service '%s' not found: %s", + obj->object, + ssl_cert); + error_count++; + } + if(access(ssl_key,F_OK) != 0) + { + skygw_log_write(LE,"Error: Server private key file for service '%s' not found: %s", + obj->object, + ssl_key); + error_count++; + } + if(error_count == 0) + { if(serviceSetSSL(obj->element,ssl) != 0) { skygw_log_write(LE,"Error: Unknown parameter for service '%s': %s",obj->object,ssl); + error_count++; } else { serviceSetCertificates(obj->element,ssl_cert,ssl_key,ssl_ca_cert); if(ssl_version) { - serviceSetSSLVersion(obj->element,ssl_version); + if(serviceSetSSLVersion(obj->element,ssl_version) != 0) + { + skygw_log_write(LE,"Error: Unknown parameter value for 'ssl_version' for service '%s': %s",obj->object,ssl_version); + error_count++; + } } } } - else - { - /** If SSL was configured wrong, the - * service needs to fail.*/ - skygw_log_write_flush(LE,"Error: Missing SSL certificate paths found in the configuration. " - "This service will not use SSL."); - } } diff --git a/server/core/service.c b/server/core/service.c index a8df37f5b..7fc931297 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -864,12 +864,20 @@ serviceOptimizeWildcard(SERVICE *service, int action) void serviceSetCertificates(SERVICE *service, char* cert,char* key, char* ca_cert) { + if(service->ssl_cert) + free(service->ssl_cert); service->ssl_cert = strdup(cert); + + if(service->ssl_key) + free(service->ssl_key); service->ssl_key = strdup(key); + + if(service->ssl_ca_cert) + free(service->ssl_ca_cert); service->ssl_ca_cert = strdup(ca_cert); } -void +int serviceSetSSLVersion(SERVICE *service, char* version) { if(strcasecmp(version,"SSLV2") == 0) @@ -884,6 +892,8 @@ serviceSetSSLVersion(SERVICE *service, char* version) service->ssl_method_type = SERVICE_TLS12; else if(strcasecmp(version,"MAX") == 0) service->ssl_method_type = SERVICE_SSL_TLS_MAX; + else return -1; + return 0; } /** Enable or disable the service SSL capability*/ int diff --git a/server/include/service.h b/server/include/service.h index 7975518e7..af71bfe7d 100644 --- a/server/include/service.h +++ b/server/include/service.h @@ -210,6 +210,7 @@ extern int serviceGetUser(SERVICE *, char **, char **); extern void serviceSetFilters(SERVICE *, char *); extern int serviceSetSSL(SERVICE *service, char* action); extern int serviceInitSSL(SERVICE* service); +extern int serviceSetSSLVersion(SERVICE *service, char* version); extern void serviceSetCertificates(SERVICE *service, char* cert,char* key, char* ca_cert); extern int serviceEnableRootUser(SERVICE *, int ); extern int serviceSetTimeout(SERVICE *, int ); diff --git a/server/modules/protocol/CMakeLists.txt b/server/modules/protocol/CMakeLists.txt index 4ae3b8f2c..124071c44 100644 --- a/server/modules/protocol/CMakeLists.txt +++ b/server/modules/protocol/CMakeLists.txt @@ -17,6 +17,7 @@ install(TARGETS HTTPD DESTINATION ${MAXSCALE_LIBDIR}) if(BUILD_TESTS) add_library(testprotocol SHARED testprotocol.c) install(TARGETS testprotocol DESTINATION ${MAXSCALE_LIBDIR}) + add_subdirectory(test) endif() add_library(maxscaled SHARED maxscaled.c) diff --git a/server/modules/protocol/test/CMakeLists.txt b/server/modules/protocol/test/CMakeLists.txt new file mode 100644 index 000000000..f3eb3aa2b --- /dev/null +++ b/server/modules/protocol/test/CMakeLists.txt @@ -0,0 +1,10 @@ +configure_file(test_ssl.sh ${CMAKE_CURRENT_BINARY_DIR}/test_ssl.sh @ONLY) +configure_file(no_ca.cnf ${CMAKE_CURRENT_BINARY_DIR}/no_ca.cnf @ONLY) +configure_file(no_server_cert.cnf ${CMAKE_CURRENT_BINARY_DIR}/no_server_cert.cnf @ONLY) +configure_file(no_server_key.cnf ${CMAKE_CURRENT_BINARY_DIR}/no_server_key.cnf @ONLY) +configure_file(bad_ca.cnf ${CMAKE_CURRENT_BINARY_DIR}/bad_ca.cnf @ONLY) +configure_file(bad_cert.cnf ${CMAKE_CURRENT_BINARY_DIR}/bad_cert.cnf @ONLY) +configure_file(bad_key.cnf ${CMAKE_CURRENT_BINARY_DIR}/bad_key.cnf @ONLY) +configure_file(bad_ssl.cnf ${CMAKE_CURRENT_BINARY_DIR}/bad_ssl.cnf @ONLY) +configure_file(ok.cnf ${CMAKE_CURRENT_BINARY_DIR}/ok.cnf @ONLY) +add_test(NAME SSLTest COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test_ssl.sh) diff --git a/server/modules/protocol/test/bad_ca.cnf b/server/modules/protocol/test/bad_ca.cnf new file mode 100644 index 000000000..9206679a9 --- /dev/null +++ b/server/modules/protocol/test/bad_ca.cnf @@ -0,0 +1,28 @@ +[maxscale] +threads=1 +logdir=@CMAKE_CURRENT_BINARY_DIR@ +datadir=@CMAKE_CURRENT_BINARY_DIR@ +piddir=@CMAKE_CURRENT_BINARY_DIR@ +cachedir=@CMAKE_CURRENT_BINARY_DIR@ + +[Testservice] +type=service +router=readconnroute +servers=server1 +user=user +passwd=pwd +ssl=enabled +ssl_ca_cert=This is not a value +ssl_cert=@CMAKE_CURRENT_BINARY_DIR@/server-cert +ssl_key=@CMAKE_CURRENT_BINARY_DIR@/server-key + +[Testlistener] +type=listener +service=Testservice +protocol=MySQLBackend +port=12345 + +[server1] +type=server +address=127.0.0.1 +port=4321 diff --git a/server/modules/protocol/test/bad_cert.cnf b/server/modules/protocol/test/bad_cert.cnf new file mode 100644 index 000000000..1b4c776cc --- /dev/null +++ b/server/modules/protocol/test/bad_cert.cnf @@ -0,0 +1,28 @@ +[maxscale] +threads=1 +logdir=@CMAKE_CURRENT_BINARY_DIR@ +datadir=@CMAKE_CURRENT_BINARY_DIR@ +piddir=@CMAKE_CURRENT_BINARY_DIR@ +cachedir=@CMAKE_CURRENT_BINARY_DIR@ + +[Testservice] +type=service +router=readconnroute +servers=server1 +user=user +passwd=pwd +ssl=enabled +ssl_ca_cert=@CMAKE_CURRENT_BINARY_DIR@/ca +ssl_cert=This is not a value +ssl_key=@CMAKE_CURRENT_BINARY_DIR@/server-key + +[Testlistener] +type=listener +service=Testservice +protocol=MySQLBackend +port=12345 + +[server1] +type=server +address=127.0.0.1 +port=4321 diff --git a/server/modules/protocol/test/bad_key.cnf b/server/modules/protocol/test/bad_key.cnf new file mode 100644 index 000000000..4e0be5f05 --- /dev/null +++ b/server/modules/protocol/test/bad_key.cnf @@ -0,0 +1,28 @@ +[maxscale] +threads=1 +logdir=@CMAKE_CURRENT_BINARY_DIR@ +datadir=@CMAKE_CURRENT_BINARY_DIR@ +piddir=@CMAKE_CURRENT_BINARY_DIR@ +cachedir=@CMAKE_CURRENT_BINARY_DIR@ + +[Testservice] +type=service +router=readconnroute +servers=server1 +user=user +passwd=pwd +ssl=enabled +ssl_ca_cert=@CMAKE_CURRENT_BINARY_DIR@/ca +ssl_cert=@CMAKE_CURRENT_BINARY_DIR@/server-cert +ssl_key=This is not a value + +[Testlistener] +type=listener +service=Testservice +protocol=MySQLBackend +port=12345 + +[server1] +type=server +address=127.0.0.1 +port=4321 diff --git a/server/modules/protocol/test/bad_ssl.cnf b/server/modules/protocol/test/bad_ssl.cnf new file mode 100644 index 000000000..f6dcff1a1 --- /dev/null +++ b/server/modules/protocol/test/bad_ssl.cnf @@ -0,0 +1,28 @@ +[maxscale] +threads=1 +logdir=@CMAKE_CURRENT_BINARY_DIR@ +datadir=@CMAKE_CURRENT_BINARY_DIR@ +piddir=@CMAKE_CURRENT_BINARY_DIR@ +cachedir=@CMAKE_CURRENT_BINARY_DIR@ + +[Testservice] +type=service +router=readconnroute +servers=server1 +user=user +passwd=pwd +ssl=testing +ssl_ca_cert=@CMAKE_CURRENT_BINARY_DIR@/ca +ssl_cert=@CMAKE_CURRENT_BINARY_DIR@/server-cert +ssl_key=@CMAKE_CURRENT_BINARY_DIR@/server-key + +[Testlistener] +type=listener +service=Testservice +protocol=MySQLBackend +port=12345 + +[server1] +type=server +address=127.0.0.1 +port=4321 diff --git a/server/modules/protocol/test/no_ca.cnf b/server/modules/protocol/test/no_ca.cnf new file mode 100644 index 000000000..56f603f6a --- /dev/null +++ b/server/modules/protocol/test/no_ca.cnf @@ -0,0 +1,28 @@ +[maxscale] +threads=1 +logdir=@CMAKE_CURRENT_BINARY_DIR@ +datadir=@CMAKE_CURRENT_BINARY_DIR@ +piddir=@CMAKE_CURRENT_BINARY_DIR@ +cachedir=@CMAKE_CURRENT_BINARY_DIR@ + +[Testservice] +type=service +router=readconnroute +servers=server1 +user=user +passwd=pwd +ssl=enabled +#ssl_ca_cert=@CMAKE_CURRENT_BINARY_DIR@/ca +ssl_cert=@CMAKE_CURRENT_BINARY_DIR@/server-cert +ssl_key=@CMAKE_CURRENT_BINARY_DIR@/server-key + +[Testlistener] +type=listener +service=Testservice +protocol=MySQLBackend +port=12345 + +[server1] +type=server +address=127.0.0.1 +port=4321 diff --git a/server/modules/protocol/test/no_server_cert.cnf b/server/modules/protocol/test/no_server_cert.cnf new file mode 100644 index 000000000..f714a0b3f --- /dev/null +++ b/server/modules/protocol/test/no_server_cert.cnf @@ -0,0 +1,28 @@ +[maxscale] +threads=1 +logdir=@CMAKE_CURRENT_BINARY_DIR@ +datadir=@CMAKE_CURRENT_BINARY_DIR@ +piddir=@CMAKE_CURRENT_BINARY_DIR@ +cachedir=@CMAKE_CURRENT_BINARY_DIR@ + +[Testservice] +type=service +router=readconnroute +servers=server1 +user=user +passwd=pwd +ssl=enabled +ssl_ca_cert=@CMAKE_CURRENT_BINARY_DIR@/ca +#ssl_cert=@CMAKE_CURRENT_BINARY_DIR@/server-cert +ssl_key=@CMAKE_CURRENT_BINARY_DIR@/server-key + +[Testlistener] +type=listener +service=Testservice +protocol=MySQLBackend +port=12345 + +[server1] +type=server +address=127.0.0.1 +port=4321 diff --git a/server/modules/protocol/test/no_server_key.cnf b/server/modules/protocol/test/no_server_key.cnf new file mode 100644 index 000000000..a820ee414 --- /dev/null +++ b/server/modules/protocol/test/no_server_key.cnf @@ -0,0 +1,28 @@ +[maxscale] +threads=1 +logdir=@CMAKE_CURRENT_BINARY_DIR@ +datadir=@CMAKE_CURRENT_BINARY_DIR@ +piddir=@CMAKE_CURRENT_BINARY_DIR@ +cachedir=@CMAKE_CURRENT_BINARY_DIR@ + +[Testservice] +type=service +router=readconnroute +servers=server1 +user=user +passwd=pwd +ssl=enabled +ssl_ca_cert=@CMAKE_CURRENT_BINARY_DIR@/ca +ssl_cert=@CMAKE_CURRENT_BINARY_DIR@/server-cert +#ssl_key=@CMAKE_CURRENT_BINARY_DIR@/server-key + +[Testlistener] +type=listener +service=Testservice +protocol=MySQLBackend +port=12345 + +[server1] +type=server +address=127.0.0.1 +port=4321 diff --git a/server/modules/protocol/test/ok.cnf b/server/modules/protocol/test/ok.cnf new file mode 100644 index 000000000..089025c0d --- /dev/null +++ b/server/modules/protocol/test/ok.cnf @@ -0,0 +1,28 @@ +[maxscale] +threads=1 +logdir=@CMAKE_CURRENT_BINARY_DIR@ +datadir=@CMAKE_CURRENT_BINARY_DIR@ +piddir=@CMAKE_CURRENT_BINARY_DIR@ +cachedir=@CMAKE_CURRENT_BINARY_DIR@ + +[Testservice] +type=service +router=readconnroute +servers=server1 +user=user +passwd=pwd +ssl=enabled +ssl_ca_cert=@CMAKE_CURRENT_BINARY_DIR@/ca +ssl_cert=@CMAKE_CURRENT_BINARY_DIR@/server-cert +ssl_key=@CMAKE_CURRENT_BINARY_DIR@/server-key + +[Testlistener] +type=listener +service=Testservice +protocol=MySQLBackend +port=12345 + +[server1] +type=server +address=127.0.0.1 +port=4321 diff --git a/server/modules/protocol/test/test_ssl.sh b/server/modules/protocol/test/test_ssl.sh new file mode 100755 index 000000000..632da0f3a --- /dev/null +++ b/server/modules/protocol/test/test_ssl.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash + +function create_certs() +{ + echo "CA cert" > @CMAKE_CURRENT_BINARY_DIR@/ca.pem + echo "Server Certificate" > @CMAKE_CURRENT_BINARY_DIR@/server-cert.pem + echo "Server Key" > @CMAKE_CURRENT_BINARY_DIR@/server-key.pem +} + +function start_maxscale () +{ + local result=$(@CMAKE_INSTALL_PREFIX@/@MAXSCALE_BINDIR@/maxscale -d -f $1 &> $1.log;echo $?) + if [[ $result == "0" ]] + then + echo "Error: $1 exited with status $result!" + exit 1 + fi +} + +# No CA defined +printf "Testing No CA defined" +start_maxscale @CMAKE_CURRENT_BINARY_DIR@/no_ca.cnf +echo " OK" + +# No cert defined +printf "Testing No cert defined" +start_maxscale @CMAKE_CURRENT_BINARY_DIR@/no_cert.cnf +echo " OK" + +# No key defined +printf "Testing No key defined" +start_maxscale @CMAKE_CURRENT_BINARY_DIR@/no_key.cnf +echo " OK" + +# Bad SSL value +printf "Testing Bad SSL defined" +start_maxscale @CMAKE_CURRENT_BINARY_DIR@/bad_ssl.cnf +echo " OK" + +# Bad CA defined +printf "Testing Bad CA defined" +start_maxscale @CMAKE_CURRENT_BINARY_DIR@/bad_ca.cnf +echo " OK" + +# Bad cert defined +printf "Testing Bad cert defined" +start_maxscale @CMAKE_CURRENT_BINARY_DIR@/bad_cert.cnf +echo " OK" + +# Bad key defined +printf "Testing Bad key defined" +start_maxscale @CMAKE_CURRENT_BINARY_DIR@/bad_key.cnf +echo " OK" + +# No CA file +printf "Testing No CA file" +create_certs +rm @CMAKE_CURRENT_BINARY_DIR@/ca.pem +start_maxscale @CMAKE_CURRENT_BINARY_DIR@/ok.cnf +echo " OK" + +# No cert file +printf "Testing No cert file" +create_certs +rm @CMAKE_CURRENT_BINARY_DIR@/server-cert.pem +start_maxscale @CMAKE_CURRENT_BINARY_DIR@/ok.cnf +echo " OK" + +# No key file +printf "Testing No key file" +create_certs +rm @CMAKE_CURRENT_BINARY_DIR@/server-key.pem +start_maxscale @CMAKE_CURRENT_BINARY_DIR@/ok.cnf +echo " OK" + +exit 0 From 4d30cd5fd3b9bdb9a8bb03ec742b158dd558ae94 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Wed, 3 Jun 2015 13:28:35 +0300 Subject: [PATCH 029/100] Added unit test for SSL version. --- server/modules/protocol/test/CMakeLists.txt | 1 + .../modules/protocol/test/bad_ssl_version.cnf | 29 +++++++++++++++++++ server/modules/protocol/test/test_ssl.sh | 17 +++++++---- 3 files changed, 41 insertions(+), 6 deletions(-) create mode 100644 server/modules/protocol/test/bad_ssl_version.cnf diff --git a/server/modules/protocol/test/CMakeLists.txt b/server/modules/protocol/test/CMakeLists.txt index f3eb3aa2b..26653901c 100644 --- a/server/modules/protocol/test/CMakeLists.txt +++ b/server/modules/protocol/test/CMakeLists.txt @@ -6,5 +6,6 @@ configure_file(bad_ca.cnf ${CMAKE_CURRENT_BINARY_DIR}/bad_ca.cnf @ONLY) configure_file(bad_cert.cnf ${CMAKE_CURRENT_BINARY_DIR}/bad_cert.cnf @ONLY) configure_file(bad_key.cnf ${CMAKE_CURRENT_BINARY_DIR}/bad_key.cnf @ONLY) configure_file(bad_ssl.cnf ${CMAKE_CURRENT_BINARY_DIR}/bad_ssl.cnf @ONLY) +configure_file(bad_ssl_version.cnf ${CMAKE_CURRENT_BINARY_DIR}/bad_ssl_version.cnf @ONLY) configure_file(ok.cnf ${CMAKE_CURRENT_BINARY_DIR}/ok.cnf @ONLY) add_test(NAME SSLTest COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test_ssl.sh) diff --git a/server/modules/protocol/test/bad_ssl_version.cnf b/server/modules/protocol/test/bad_ssl_version.cnf new file mode 100644 index 000000000..6849b1904 --- /dev/null +++ b/server/modules/protocol/test/bad_ssl_version.cnf @@ -0,0 +1,29 @@ +[maxscale] +threads=1 +logdir=@CMAKE_CURRENT_BINARY_DIR@ +datadir=@CMAKE_CURRENT_BINARY_DIR@ +piddir=@CMAKE_CURRENT_BINARY_DIR@ +cachedir=@CMAKE_CURRENT_BINARY_DIR@ + +[Testservice] +type=service +router=readconnroute +servers=server1 +user=user +passwd=pwd +ssl=enabled +ssl_ca_cert=@CMAKE_CURRENT_BINARY_DIR@/ca +ssl_cert=@CMAKE_CURRENT_BINARY_DIR@/server-cert +ssl_key=@CMAKE_CURRENT_BINARY_DIR@/server-key +ssl_version=Don't use SSL, it's not needed! + +[Testlistener] +type=listener +service=Testservice +protocol=MySQLBackend +port=12345 + +[server1] +type=server +address=127.0.0.1 +port=4321 diff --git a/server/modules/protocol/test/test_ssl.sh b/server/modules/protocol/test/test_ssl.sh index 632da0f3a..b4b4c4d46 100755 --- a/server/modules/protocol/test/test_ssl.sh +++ b/server/modules/protocol/test/test_ssl.sh @@ -32,22 +32,27 @@ printf "Testing No key defined" start_maxscale @CMAKE_CURRENT_BINARY_DIR@/no_key.cnf echo " OK" -# Bad SSL value +# Bad SSL value defined printf "Testing Bad SSL defined" start_maxscale @CMAKE_CURRENT_BINARY_DIR@/bad_ssl.cnf echo " OK" -# Bad CA defined +# Bad SSL version defined +printf "Testing Bad SSL version defined" +start_maxscale @CMAKE_CURRENT_BINARY_DIR@/bad_ssl_version.cnf +echo " OK" + +# Bad CA value defined printf "Testing Bad CA defined" start_maxscale @CMAKE_CURRENT_BINARY_DIR@/bad_ca.cnf echo " OK" -# Bad cert defined +# Bad server certificate defined printf "Testing Bad cert defined" start_maxscale @CMAKE_CURRENT_BINARY_DIR@/bad_cert.cnf echo " OK" -# Bad key defined +# Bad server key defined printf "Testing Bad key defined" start_maxscale @CMAKE_CURRENT_BINARY_DIR@/bad_key.cnf echo " OK" @@ -59,14 +64,14 @@ rm @CMAKE_CURRENT_BINARY_DIR@/ca.pem start_maxscale @CMAKE_CURRENT_BINARY_DIR@/ok.cnf echo " OK" -# No cert file +# No server certificate file printf "Testing No cert file" create_certs rm @CMAKE_CURRENT_BINARY_DIR@/server-cert.pem start_maxscale @CMAKE_CURRENT_BINARY_DIR@/ok.cnf echo " OK" -# No key file +# No server key file printf "Testing No key file" create_certs rm @CMAKE_CURRENT_BINARY_DIR@/server-key.pem From a033cbf200ae059374f78fd09deef645306f4a86 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Wed, 3 Jun 2015 14:14:05 +0300 Subject: [PATCH 030/100] Added more informative error messages when SSL handshake fails. --- server/core/dcb.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index ade2affed..95130299f 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -2789,7 +2789,7 @@ int dcb_create_SSL(DCB* dcb) int dcb_accept_SSL(DCB* dcb) { int rval,errnum; - + char errbuf[140]; rval = SSL_accept(dcb->ssl); switch(rval) @@ -2819,23 +2819,25 @@ int dcb_accept_SSL(DCB* dcb) rval = 0; LOGIF(LD,(skygw_log_write_flush(LD,"SSL_accept ongoing for %s@%s", - dcb->user, + dcb->user?dcb->user:"a connection from ", dcb->remote))); } else { rval = -1; + ERR_error_string(errnum,errbuf); skygw_log_write_flush(LE, - "Error: Fatal error in SSL_accept for %s@%s: %s", + "Error: Fatal error in SSL_accept for %s@%s: (SSL error code: %d) %s", dcb->user, dcb->remote, - ERR_error_string(errnum,NULL)); + errnum, + errbuf); } break; default: skygw_log_write_flush(LE, - "Error: Fatal error in SSL_accept, returned value was %d.", + "Error: Fatal library error in SSL_accept, returned value was %d.", rval); break; } @@ -2855,7 +2857,7 @@ int dcb_accept_SSL(DCB* dcb) int dcb_connect_SSL(DCB* dcb) { int rval,errnum; - + char errbuf[140]; rval = SSL_connect(dcb->ssl); switch(rval) @@ -2891,11 +2893,13 @@ int dcb_connect_SSL(DCB* dcb) else { rval = -1; + ERR_error_string(errnum,errbuf); skygw_log_write_flush(LE, - "Error: Fatal error in SSL_connect for %s@%s: %s", + "Error: Fatal error in SSL_accept for %s@%s: (SSL error code: %d) %s", dcb->user, dcb->remote, - ERR_error_string(errnum,NULL)); + errnum, + errbuf); } break; From 98d98c589e5326306f6481c299f6ecb6ac2e0b35 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 4 Jun 2015 13:30:36 +0300 Subject: [PATCH 031/100] Added an Lsyncd configuration file. --- etc/lsyncd.conf | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 etc/lsyncd.conf diff --git a/etc/lsyncd.conf b/etc/lsyncd.conf new file mode 100644 index 000000000..40e860675 --- /dev/null +++ b/etc/lsyncd.conf @@ -0,0 +1,30 @@ + +-- Lsyncd will log to these two files. +settings{ + logfile = "/var/log/maxscale/maxscale-ha.log", + statusFile = "/var/log/maxscale/maxscale-ha-status.log" +} + +-- Copy and paste the sync section and change the host value to add new remote targets. +sync{ +default.rsyncssh, + +-- This is where the maxscale.cnf file is copied from. +source="/etc", + +-- This is the user and host where the maxscale.cnf is copied to. +host="user@127.0.0.1", + +-- This is where the maxscale.cnf is copied to on the remote host. +targetdir="/etc", + +-- This is an optional section which defines a custom SSH port. Uncomment to enable. +-- ssh={port=2222}, + +-- These are values passed to rsync. Only change these if you know what you are doing. +rsync={ +compress=true, +_extra = {[[--filter=+ *maxscale.cnf]], + [[--filter=- **]]} +} +} From a032c94d25c6557cfce3bd835d8c89d10bc22f26 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 4 Jun 2015 16:49:39 +0300 Subject: [PATCH 032/100] Added comments to SSL tests. --- server/modules/protocol/test/test_ssl.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/modules/protocol/test/test_ssl.sh b/server/modules/protocol/test/test_ssl.sh index b4b4c4d46..f14670689 100755 --- a/server/modules/protocol/test/test_ssl.sh +++ b/server/modules/protocol/test/test_ssl.sh @@ -17,6 +17,8 @@ function start_maxscale () fi } +# All test cases expect that MaxScale will not start with a bad configuration or missing certificates + # No CA defined printf "Testing No CA defined" start_maxscale @CMAKE_CURRENT_BINARY_DIR@/no_ca.cnf From dceccce2ef919d69b952fca810af621e6e68ebec Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 4 Jun 2015 16:52:43 +0300 Subject: [PATCH 033/100] Changed gwbuf_length function to GWBUF_LENGTH macro in dcb_read_SSL. --- server/core/dcb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index 95130299f..ae77707b9 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -1017,7 +1017,7 @@ int dcb_read_SSL( if(n < b) { gwbuf_rtrim(buffer,b - n); - ss_dassert(gwbuf_length(buffer) == n); + ss_dassert(GWBUF_LENGTH(buffer) == n); LOGIF(LD,(skygw_log_write(LD,"[%lu] SSL: Truncated buffer to correct size from %d to %d bytes.\n", b,gwbuf_length(buffer)))); } From 6f58f29494c2989330d02b4a36c7d2789dc53b56 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 4 Jun 2015 19:14:53 +0300 Subject: [PATCH 034/100] Added a guide on how to have a HA setup of MaxScale with lsyncd. --- CMakeLists.txt | 1 + Documentation/Documentation-Contents.md | 1 + .../Reference/MaxScale-HA-with-lsyncd.md | 117 ++++++++++++++++++ etc/{lsyncd.conf => lsyncd_example.conf} | 0 4 files changed, 119 insertions(+) create mode 100644 Documentation/Reference/MaxScale-HA-with-lsyncd.md rename etc/{lsyncd.conf => lsyncd_example.conf} (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 03cb541b6..d12de2908 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -159,6 +159,7 @@ install(FILES ${ERRMSG} DESTINATION ${MAXSCALE_VARDIR}/lib/maxscale install(FILES ${CMAKE_SOURCE_DIR}/COPYRIGHT DESTINATION ${MAXSCALE_SHAREDIR}) install(FILES ${CMAKE_SOURCE_DIR}/README DESTINATION ${MAXSCALE_SHAREDIR}) install(FILES ${CMAKE_SOURCE_DIR}/LICENSE DESTINATION ${MAXSCALE_SHAREDIR}) +install(FILES etc/lsyncd_example.conf DESTINATION ${MAXSCALE_SHAREDIR}) # Install startup scripts and ldconfig files if(WITH_SCRIPTS) diff --git a/Documentation/Documentation-Contents.md b/Documentation/Documentation-Contents.md index 1af3750c5..99f8ba38e 100644 --- a/Documentation/Documentation-Contents.md +++ b/Documentation/Documentation-Contents.md @@ -25,6 +25,7 @@ - [MaxAdmin](Reference/MaxAdmin.md) - [MaxScale HA with Corosync-Pacemaker](Reference/MaxScale-HA-with-Corosync-Pacemaker.md) + - [MaxScale HA with Lsyncd](Reference/MaxScale-HA-with-lsyncd.md) - [How Errors are Handled in MaxScale](Reference/How-errors-are-handled-in-MaxScale.md) - [Debug and Diagnostic Support](Reference/Debug-And-Diagnostic-Support.md) diff --git a/Documentation/Reference/MaxScale-HA-with-lsyncd.md b/Documentation/Reference/MaxScale-HA-with-lsyncd.md new file mode 100644 index 000000000..18ef233db --- /dev/null +++ b/Documentation/Reference/MaxScale-HA-with-lsyncd.md @@ -0,0 +1,117 @@ +# MaxScale HA with Lsyncd + +This document guides you in setting up multiple MaxScale instances and synchronizing the configuration files with Lsyncd. Lsyncd is a rsync wrapper which can synchronize files across the network. The lsyncd daemon uses a configuration file to control the files to synchronize and the remote targets where these files are synchronized to. + +Copying the configuration file and running the lsyncd daemon on all the hosts keeps all the configuration files in sync. Modifications in the configuration file on one of the hosts will be copied on the other hosts. This allows adinistrators to easily provide a highly available, disaster resistant MaxScale installation with up-to-date configuration files on all the hosts. + + +## Installing Lsyncd + +You can install lsyncd with either a package manager or by building from source code. This guide demonstrates installation using a package manager and those looking to build lsyncd from source should refer to its documentation: https://github.com/axkibe/lsyncd/wiki/Manual-to-Lsyncd-2.1.x + +Installing with Yum: + +``` +yum install lsyncd +``` + +Installing with Apt: + +``` +apt-get install lsyncd +``` + +Lsyncd needs no further configuration after installation. All necessary options are configured in the configuration file passed to the lsyncd daemon. + +## Creating the Lsyncd configuration file + +Lsyncd uses a configuration file to determine where to read files from and where to synchronize them if changes in them occur. Lsyncd is written in Lua and the configuration file is also valid Lua code. + +Here is an example configuration file with descriptions on the meaning of the values in it. + +``` +-- Lsyncd will log to these two files. +settings{ + logfile = "/var/log/maxscale/maxscale-ha.log", + statusFile = "/var/log/maxscale/maxscale-ha-status.log" +} + +-- Copy and paste the sync section and change the host value to add new remote targets. +sync{ +default.rsyncssh, + +-- This is where the maxscale.cnf file is copied from. +source="/etc", + +-- This is the user and host where the maxscale.cnf is copied to. +host="user@127.0.0.1", + +-- This is where the maxscale.cnf is copied to on the remote host. +targetdir="/etc", + +-- This is an optional section which defines a custom SSH port. Uncomment to enable. +-- ssh={port=2222}, + +-- These are values passed to rsync. Only change these if you know what you are doing. +rsync={ + compress=true, + _extra = {[[--filter=+ *maxscale.cnf]], + [[--filter=- **]] + } + } +} +``` + +The most important part is the `sync` section which defines a target for synchronization. The `default.rsyncssh` tells lsyncd to synchronize files using SSH. + +The `source` parameter tells lsyncd where to read the files from. This should be the location of the maxscale.cnf file. The `host` parameter defines the host where the files should be synchronized to and the user account lsyncd should use when synchronizing the files. The `targetdir` parameter defines the local directory on the remote target where the files should be synchronized to. This value should be the location on the remote host where the maxscale.cnf file is searched from. By default, this is the `/etc` folder. + +The optional `ssh` parameter and its sub-parameter `port`define a custom port for the SSH connection. Most users do not need this parameterer. The `rsycn` parameter contains an arra of options that are passed to the rsycn executable. These should not be changed unless you specifically know what you are doing. For more information on the options passed to rsync read the rsync(1) manpage. + +You can add multiple remote targets by defining multiple `sync` sections. Here is an example with two sync sections defining different hosts that have MaxScale installed and whose configuration files should be kep in sync. + +``` +settings{ + logfile = "/var/log/maxscale/maxscale-ha.log", + statusFile = "/var/log/maxscale/maxscale-ha-status.log" +} + +sync{ +default.rsyncssh, +source="/etc", +host="maxuser@192.168.0.50", +targetdir="/etc", +rsync={ + compress=true, + _extra = {[[--filter=+ *maxscale.cnf]], + [[--filter=- **]] + } + } +} + + +sync{ +default.rsyncssh, +source="/etc", +host="syncuser@192.168.122.105", +targetdir="/etc", +rsync={ + compress=true, + _extra = {[[--filter=+ *maxscale.cnf]], + [[--filter=- **]] + } + } +} +``` + +## Starting Lsyncd + +Starting lsyncd can be done from the command line or through a init script. To start syncd from the command like, execute the `lsyncd` command and pass the configuration file as the only parameter. + +Here is an example which start lsyncd and reads the configuration options from the `lsyncd.cnf` file. + +``` +lsyncd lsyncd.cnf +``` + +For more information on the lsyncd executable and its options, please see the --help output of lsyncd or the lsyncd(1) manpage. diff --git a/etc/lsyncd.conf b/etc/lsyncd_example.conf similarity index 100% rename from etc/lsyncd.conf rename to etc/lsyncd_example.conf From ce570685cda436d6a9b48e7ca39adde7c5b162c6 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 4 Jun 2015 19:31:58 +0300 Subject: [PATCH 035/100] Moved assertions around. --- server/core/dcb.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index ae77707b9..653799fa6 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -1014,12 +1014,14 @@ int dcb_read_SSL( } } - if(n < b) + if(n > 0 && b > 0 && n < b) { gwbuf_rtrim(buffer,b - n); - ss_dassert(GWBUF_LENGTH(buffer) == n); LOGIF(LD,(skygw_log_write(LD,"[%lu] SSL: Truncated buffer to correct size from %d to %d bytes.\n", b,gwbuf_length(buffer)))); + LOGIF(LD,(skygw_log_sync_all())); + ss_dassert(GWBUF_LENGTH(buffer) == n); + } nread += n; From 1f45eff135c4e657e6b0513790d25b3ff784561e Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 4 Jun 2015 19:38:33 +0300 Subject: [PATCH 036/100] Generated packages now have debug symbols. --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 03cb541b6..4f35031d8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -205,6 +205,7 @@ if(PACKAGE) else() # Generic CPack configuration variables + set(CPACK_STRIP_FILES FALSE) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "MaxScale") set(CPACK_PACKAGE_VERSION_MAJOR "${MAXSCALE_VERSION_MAJOR}") set(CPACK_PACKAGE_VERSION_MINOR "${MAXSCALE_VERSION_MINOR}") From cc1f720ea3e3fc6b992c75af28f200d35f2770de Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 4 Jun 2015 21:12:16 +0300 Subject: [PATCH 037/100] Removed log flushing on every dcb_read_SSL if debug log is enabled. --- server/core/dcb.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index 653799fa6..1a7727ff8 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -1019,7 +1019,11 @@ int dcb_read_SSL( gwbuf_rtrim(buffer,b - n); LOGIF(LD,(skygw_log_write(LD,"[%lu] SSL: Truncated buffer to correct size from %d to %d bytes.\n", b,gwbuf_length(buffer)))); - LOGIF(LD,(skygw_log_sync_all())); + LOGIF(LD, + if(GWBUF_LENGTH(buffer) != n){ + skygw_log_sync_all(); + } + ); ss_dassert(GWBUF_LENGTH(buffer) == n); } From e83799648a757d31c4b641365516de34b71ae321 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 5 Jun 2015 11:00:51 +0300 Subject: [PATCH 038/100] Fixed queries getting stuck when the SSL records were of the maximum allowed size. --- server/core/dcb.c | 85 ++++++++++++++------------ server/core/service.c | 1 - server/modules/protocol/mysql_client.c | 2 +- 3 files changed, 48 insertions(+), 40 deletions(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index 1a7727ff8..c1e77b58d 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -898,7 +898,7 @@ int dcb_read_SSL( GWBUF **head) { GWBUF *buffer = NULL; - int b; + int b,pending; int rc; int n; int nread = 0; @@ -918,9 +918,9 @@ int dcb_read_SSL( while (true) { int bufsize; - + int ssl_errno = 0; rc = ioctl(dcb->fd, FIONREAD, &b); - + pending = SSL_pending(dcb->ssl); if (rc == -1) { LOGIF(LE, (skygw_log_write_flush( @@ -936,7 +936,7 @@ int dcb_read_SSL( goto return_n; } - if (b == 0 && nread == 0) + if (b == 0 && pending == 0 && nread == 0) { /** Handle closed client socket */ if (dcb_isclient(dcb)) @@ -948,14 +948,20 @@ int dcb_read_SSL( r = SSL_peek(dcb->ssl, &c, sizeof(char)); if (r <= 0) { + ssl_errno = SSL_get_error(dcb->ssl,r); + if(ssl_errno != SSL_ERROR_WANT_READ && + ssl_errno != SSL_ERROR_WANT_WRITE && + ssl_errno != SSL_ERROR_NONE) + { n = -1; - goto return_n; + } + goto return_n; } } n = 0; goto return_n; } - else if (b == 0) + else if (b == 0 && pending == 0) { n = 0; goto return_n; @@ -984,39 +990,36 @@ int dcb_read_SSL( goto return_n; } - int npending; - n = 0; - do - { - n += SSL_read(dcb->ssl, GWBUF_DATA(buffer), bufsize); + n = SSL_read(dcb->ssl, GWBUF_DATA(buffer), bufsize); dcb->stats.n_reads++; - }while((npending = SSL_pending(dcb->ssl)) > 0); - int ssl_errno = 0; - - if (n <= 0) - { - ssl_errno = ERR_get_error(); - - if(ssl_errno != SSL_ERROR_WANT_READ && ssl_errno != SSL_ERROR_NONE) + if (n <= 0) { - LOGIF(LE, (skygw_log_write_flush( - LOGFILE_ERROR, - "Error : Read failed, dcb %p in state " - "%s fd %d: %s.", - dcb, - STRDCBSTATE(dcb->state), - dcb->fd, - ERR_error_string(ssl_errno,NULL)))); + ssl_errno = SSL_get_error(dcb->ssl,n); - gwbuf_free(buffer); - goto return_n; - } + if(ssl_errno != SSL_ERROR_WANT_READ && + ssl_errno != SSL_ERROR_WANT_WRITE && + ssl_errno != SSL_ERROR_NONE) + { + char errbuf[200]; + ERR_error_string(ssl_errno,errbuf); + LOGIF(LE, (skygw_log_write_flush( + LOGFILE_ERROR, + "Error : Read failed, dcb %p in state " + "%s fd %d, SSL error %d: %s.", + dcb, + STRDCBSTATE(dcb->state), + dcb->fd, + ssl_errno, + errbuf))); + + gwbuf_free(buffer); + goto return_n; + } } - if(n > 0 && b > 0 && n < b) - { - gwbuf_rtrim(buffer,b - n); + buffer->end = buffer->start + n; +#ifdef SS_DEBUG LOGIF(LD,(skygw_log_write(LD,"[%lu] SSL: Truncated buffer to correct size from %d to %d bytes.\n", b,gwbuf_length(buffer)))); LOGIF(LD, @@ -1025,9 +1028,7 @@ int dcb_read_SSL( } ); ss_dassert(GWBUF_LENGTH(buffer) == n); - - } - +#endif nread += n; LOGIF(LD, (skygw_log_write( @@ -1039,11 +1040,19 @@ int dcb_read_SSL( dcb, STRDCBSTATE(dcb->state), dcb->fd))); + /*< Append read data to the gwbuf */ *head = gwbuf_append(*head, buffer); - if(ssl_errno == SSL_ERROR_WANT_READ || ssl_errno == SSL_ERROR_NONE || - ssl_errno == SSL_ERROR_WANT_X509_LOOKUP || SSL_ERROR_WANT_WRITE) + rc = ioctl(dcb->fd, FIONREAD, &b); + pending = SSL_pending(dcb->ssl); + + if(ssl_errno == SSL_ERROR_WANT_READ || + ssl_errno == SSL_ERROR_WANT_WRITE || + (b == 0 && pending == 0)) + { break; + } + } /*< while (true) */ return_n: return n; diff --git a/server/core/service.c b/server/core/service.c index 7fc931297..f4880107d 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -1875,7 +1875,6 @@ int serviceInitSSL(SERVICE* service) } service->ctx = SSL_CTX_new(service->method); - SSL_CTX_set_read_ahead(service->ctx,1); if (SSL_CTX_use_certificate_file(service->ctx, service->ssl_cert, SSL_FILETYPE_PEM) <= 0) { skygw_log_write(LE,"Error: Failed to set server SSL certificate."); return -1; diff --git a/server/modules/protocol/mysql_client.c b/server/modules/protocol/mysql_client.c index 45fc65ee0..5a54ca4b3 100644 --- a/server/modules/protocol/mysql_client.c +++ b/server/modules/protocol/mysql_client.c @@ -789,7 +789,7 @@ int gw_read_client_event( nbytes_read = gwbuf_length(dcb->dcb_readqueue); data = (uint8_t *)GWBUF_DATA(dcb->dcb_readqueue); int plen = MYSQL_GET_PACKET_LEN(data); - if (nbytes_read < 3 || nbytes_read < MYSQL_GET_PACKET_LEN(data)) + if (nbytes_read < 3 || nbytes_read < MYSQL_GET_PACKET_LEN(data) + 4) { rc = 0; goto return_rc; From 518ef5050e5ef687c05eb83aa5cb068d78a7f31a Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 5 Jun 2015 12:15:19 +0300 Subject: [PATCH 039/100] Fixed debug asserts. --- server/core/dcb.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index c1e77b58d..c2f48b39d 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -1018,15 +1018,16 @@ int dcb_read_SSL( } } - buffer->end = buffer->start + n; + gwbuf_rtrim(buffer,bufsize - n); #ifdef SS_DEBUG LOGIF(LD,(skygw_log_write(LD,"[%lu] SSL: Truncated buffer to correct size from %d to %d bytes.\n", - b,gwbuf_length(buffer)))); + b,GWBUF_LENGTH(buffer)))); LOGIF(LD, if(GWBUF_LENGTH(buffer) != n){ skygw_log_sync_all(); } ); + ss_info_dassert((buffer->start <= buffer->end),"Buffer start has passed end."); ss_dassert(GWBUF_LENGTH(buffer) == n); #endif nread += n; From 61b1f3467160df1cb1f6e364ac4b860eaf81d0c0 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 5 Jun 2015 18:52:44 +0300 Subject: [PATCH 040/100] Added more descriptive debug output. --- server/core/dcb.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index c2f48b39d..728211abe 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -1020,13 +1020,14 @@ int dcb_read_SSL( gwbuf_rtrim(buffer,bufsize - n); #ifdef SS_DEBUG - LOGIF(LD,(skygw_log_write(LD,"[%lu] SSL: Truncated buffer to correct size from %d to %d bytes.\n", - b,GWBUF_LENGTH(buffer)))); - LOGIF(LD, - if(GWBUF_LENGTH(buffer) != n){ - skygw_log_sync_all(); - } - ); + skygw_log_write(LD,"[%lu] SSL: Truncated buffer from %d to %d bytes. " + "Read %d bytes, %d bytes waiting.\n", + bufsize,GWBUF_LENGTH(buffer),n,b); + + if(GWBUF_LENGTH(buffer) != n){ + skygw_log_sync_all(); + } + ss_info_dassert((buffer->start <= buffer->end),"Buffer start has passed end."); ss_dassert(GWBUF_LENGTH(buffer) == n); #endif From 1989a1482c3516cea06162b1aec640beadb6f1f3 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 5 Jun 2015 19:37:33 +0300 Subject: [PATCH 041/100] Fixed empty reads causing a debug assert with large packets. --- server/core/dcb.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index 728211abe..6beaec55d 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -966,6 +966,13 @@ int dcb_read_SSL( n = 0; goto return_n; } +#ifdef SS_DEBUG + else + { + skygw_log_write_flush(LD,"Total: %d Socket: %d Pending: %d", + nread,b,pending); + } +#endif dcb->last_read = hkheartbeat; @@ -993,7 +1000,7 @@ int dcb_read_SSL( n = SSL_read(dcb->ssl, GWBUF_DATA(buffer), bufsize); dcb->stats.n_reads++; - if (n <= 0) + if (n < 0) { ssl_errno = SSL_get_error(dcb->ssl,n); @@ -1016,12 +1023,17 @@ int dcb_read_SSL( gwbuf_free(buffer); goto return_n; } - } + } + else if(n == 0) + { + gwbuf_free(buffer); + goto return_n; + } gwbuf_rtrim(buffer,bufsize - n); #ifdef SS_DEBUG skygw_log_write(LD,"[%lu] SSL: Truncated buffer from %d to %d bytes. " - "Read %d bytes, %d bytes waiting.\n", + "Read %d bytes, %d bytes waiting.\n",pthread_self(), bufsize,GWBUF_LENGTH(buffer),n,b); if(GWBUF_LENGTH(buffer) != n){ @@ -1029,7 +1041,7 @@ int dcb_read_SSL( } ss_info_dassert((buffer->start <= buffer->end),"Buffer start has passed end."); - ss_dassert(GWBUF_LENGTH(buffer) == n); + ss_info_dassert(GWBUF_LENGTH(buffer) == n,"Buffer size not equal to read bytes."); #endif nread += n; From d19ccc6f846ec6d54f6ad2171acdee08a48aef63 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 5 Jun 2015 20:36:04 +0300 Subject: [PATCH 042/100] Fixed SSL_ERROR_WANT_READ, SSL_ERROR_WANT_WRITE and SSL_ERROR_NONE causing a debug assert. --- server/core/dcb.c | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index 6beaec55d..538b0531a 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -1002,13 +1002,21 @@ int dcb_read_SSL( if (n < 0) { + char errbuf[200]; ssl_errno = SSL_get_error(dcb->ssl,n); - - if(ssl_errno != SSL_ERROR_WANT_READ && - ssl_errno != SSL_ERROR_WANT_WRITE && - ssl_errno != SSL_ERROR_NONE) +#ifdef SS_DEBUG + ERR_error_string(ssl_errno,errbuf); + skygw_log_write_flush(LD,"[%lu]SSL error %d: %s", + pthread_self(),ssl_errno,errbuf); +#endif + if(ssl_errno == SSL_ERROR_WANT_READ || + ssl_errno == SSL_ERROR_WANT_WRITE || + ssl_errno == SSL_ERROR_NONE) + { + n = 0; + } + else { - char errbuf[200]; ERR_error_string(ssl_errno,errbuf); LOGIF(LE, (skygw_log_write_flush( LOGFILE_ERROR, @@ -1019,10 +1027,10 @@ int dcb_read_SSL( dcb->fd, ssl_errno, errbuf))); - - gwbuf_free(buffer); - goto return_n; } + + gwbuf_free(buffer); + goto return_n; } else if(n == 0) { From b8e55fe28d24327fd1b7d1641cdd2c736e8f26ad Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Sun, 7 Jun 2015 12:37:45 +0300 Subject: [PATCH 043/100] Fixed SSL_accept failing when more data was in the socket buffer than was used. --- server/core/dcb.c | 98 +++++++++++++++++++++++++---------------------- 1 file changed, 52 insertions(+), 46 deletions(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index 538b0531a..e7862dbd4 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -2825,60 +2825,66 @@ int dcb_create_SSL(DCB* dcb) */ int dcb_accept_SSL(DCB* dcb) { - int rval,errnum; + int rval = 0,ssl_rval,errnum,fd,b = 0; char errbuf[140]; - rval = SSL_accept(dcb->ssl); - - switch(rval) + fd = dcb->fd; + ioctl(fd,FIONREAD,&b); + while(b > 0 && rval != -1) { - case 0: - errnum = SSL_get_error(dcb->ssl,rval); - LOGIF(LD,(skygw_log_write_flush(LD,"SSL_accept shutdown for %s@%s", - dcb->user, - dcb->remote))); - return -1; - break; - case 1: - rval = 1; - LOGIF(LD,(skygw_log_write_flush(LD,"SSL_accept done for %s@%s", - dcb->user, - dcb->remote))); - break; + ssl_rval = SSL_accept(dcb->ssl); - case -1: - errnum = SSL_get_error(dcb->ssl,rval); - - if(errnum == SSL_ERROR_WANT_READ || errnum == SSL_ERROR_WANT_WRITE || - errnum == SSL_ERROR_WANT_X509_LOOKUP) + switch(ssl_rval) { - /** Not all of the data has been read. Go back to the poll - queue and wait for more.*/ - - rval = 0; - LOGIF(LD,(skygw_log_write_flush(LD,"SSL_accept ongoing for %s@%s", - dcb->user?dcb->user:"a connection from ", - dcb->remote))); - } - else - { - rval = -1; + case 0: + errnum = SSL_get_error(dcb->ssl,ssl_rval); ERR_error_string(errnum,errbuf); + LOGIF(LD,(skygw_log_write_flush(LD,"[%p] SSL_accept shutdown for %s:%s", + dcb, + dcb->remote, + errbuf))); + rval = -1; + break; + case 1: + rval = 1; + LOGIF(LD,(skygw_log_write_flush(LD,"[dcb_accept_SSL] SSL_accept done for %s", + dcb->remote))); + break; + + case -1: + errnum = SSL_get_error(dcb->ssl,ssl_rval); + + if(errnum == SSL_ERROR_WANT_READ || errnum == SSL_ERROR_WANT_WRITE) + { + /** Not all of the data has been read. Go back to the poll + queue and wait for more.*/ + rval = 0; + LOGIF(LD,(skygw_log_write_flush(LD,"[dcb_accept_SSL] SSL_accept ongoing for %s", + dcb->remote))); + return rval; + } + else + { + rval = -1; + ERR_error_string(errnum,errbuf); + skygw_log_write_flush(LE, + "Error: Fatal error in SSL_accept for %s: (SSL error code: %d) %s", + dcb->remote, + errnum, + errbuf); + } + break; + + default: skygw_log_write_flush(LE, - "Error: Fatal error in SSL_accept for %s@%s: (SSL error code: %d) %s", - dcb->user, - dcb->remote, - errnum, - errbuf); + "Error: Fatal library error in SSL_accept, returned value was %d.", + ssl_rval); + rval = -1; + break; } - break; - - default: - skygw_log_write_flush(LE, - "Error: Fatal library error in SSL_accept, returned value was %d.", - rval); - break; + ioctl(fd,FIONREAD,&b); + if(LOG_IS_ENABLED(LD) && b > 0) + skygw_log_write_flush(LD,"[dcb_accept_SSL] FD %d has %d bytes ",fd,b); } - return rval; } From 61ea0861ff8f403f14915218ceee3226e5691005 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Mon, 8 Jun 2015 14:35:31 +0300 Subject: [PATCH 044/100] Fixed some connections hanging with SSL. --- server/modules/protocol/mysql_client.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/modules/protocol/mysql_client.c b/server/modules/protocol/mysql_client.c index 5a54ca4b3..dc5c08d89 100644 --- a/server/modules/protocol/mysql_client.c +++ b/server/modules/protocol/mysql_client.c @@ -661,7 +661,6 @@ int gw_read_client_event( return 0; break; case 1: - return 0; break; case -1: return 1; @@ -1946,4 +1945,4 @@ int do_ssl_accept(MySQLProtocol* protocol) } return rval; -} \ No newline at end of file +} From 1c36cfb28592a549d526a97ea47a6064afba278b Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Mon, 8 Jun 2015 18:04:43 +0300 Subject: [PATCH 045/100] Added more debug output. --- server/core/dcb.c | 15 +++++++++++---- .../include/mysql_client_server_protocol.h | 4 ++-- server/modules/protocol/mysql_client.c | 11 +++++++++++ server/modules/protocol/mysql_common.c | 6 +++++- 4 files changed, 29 insertions(+), 7 deletions(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index e7862dbd4..cc8d77259 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -2825,14 +2825,19 @@ int dcb_create_SSL(DCB* dcb) */ int dcb_accept_SSL(DCB* dcb) { - int rval = 0,ssl_rval,errnum,fd,b = 0; + int rval = 0,ssl_rval,errnum = 0,fd,b = 0; char errbuf[140]; fd = dcb->fd; ioctl(fd,FIONREAD,&b); +#ifdef SS_DEBUG + skygw_log_write(LD,"[dcb_accept_SSL] fd %d bytes: %d",fd,b); +#endif while(b > 0 && rval != -1) { ssl_rval = SSL_accept(dcb->ssl); - +#ifdef SS_DEBUG + skygw_log_write(LD,"[dcb_accept_SSL] SSL_accept returned %d.",ssl_rval); +#endif switch(ssl_rval) { case 0: @@ -2882,8 +2887,10 @@ int dcb_accept_SSL(DCB* dcb) break; } ioctl(fd,FIONREAD,&b); - if(LOG_IS_ENABLED(LD) && b > 0) - skygw_log_write_flush(LD,"[dcb_accept_SSL] FD %d has %d bytes ",fd,b); +#ifdef SS_DEBUG + skygw_log_write_flush(LD,"[dcb_accept_SSL] fd %d: %d bytes",fd,b); + skygw_log_write_flush(LD,"[dcb_accept_SSL] SSL error: %d",errnum); +#endif } return rval; } diff --git a/server/modules/include/mysql_client_server_protocol.h b/server/modules/include/mysql_client_server_protocol.h index f72416491..905f57dd3 100644 --- a/server/modules/include/mysql_client_server_protocol.h +++ b/server/modules/include/mysql_client_server_protocol.h @@ -321,7 +321,7 @@ typedef struct { #define MYSQL_IS_CHANGE_USER(payload) (MYSQL_GET_COMMAND(payload)==0x11) #define MYSQL_GET_NATTR(payload) ((int)payload[4]) -#endif /** _MYSQL_PROTOCOL_H */ + MySQLProtocol* mysql_protocol_init(DCB* dcb, int fd); void mysql_protocol_done (DCB* dcb); @@ -417,4 +417,4 @@ void init_response_status ( int* npackets, ssize_t* nbytes); - +#endif /** _MYSQL_PROTOCOL_H */ \ No newline at end of file diff --git a/server/modules/protocol/mysql_client.c b/server/modules/protocol/mysql_client.c index dc5c08d89..fcbb1958e 100644 --- a/server/modules/protocol/mysql_client.c +++ b/server/modules/protocol/mysql_client.c @@ -652,9 +652,16 @@ int gw_read_client_event( protocol = DCB_PROTOCOL(dcb, MySQLProtocol); CHK_PROTOCOL(protocol); +#ifdef SS_DEBUG + skygw_log_write(LD,"[gw_read_client_event] Protocol state: %s", + gw_mysql_protocol_state2string(protocol->protocol_auth_state)); + +#endif + if(protocol->protocol_auth_state == MYSQL_AUTH_SSL_HANDSHAKE_ONGOING || protocol->protocol_auth_state == MYSQL_AUTH_SSL_REQ) { + switch(do_ssl_accept(protocol)) { case 0: @@ -1943,6 +1950,10 @@ int do_ssl_accept(MySQLProtocol* protocol) rval); break; } +#ifdef SS_DEBUG + skygw_log_write(LD,"[do_ssl_accept] Protocol state: %s", + gw_mysql_protocol_state2string(protocol->protocol_auth_state)); +#endif return rval; } diff --git a/server/modules/protocol/mysql_common.c b/server/modules/protocol/mysql_common.c index 0a1d2195b..d8f9a96ba 100644 --- a/server/modules/protocol/mysql_common.c +++ b/server/modules/protocol/mysql_common.c @@ -890,7 +890,11 @@ gw_mysql_protocol_state2string (int state) { case MYSQL_AUTH_FAILED: return "MySQL Authentication failed"; case MYSQL_IDLE: - return "MySQL authentication is succesfully done."; + return "MySQL authentication is succesfully done."; + case MYSQL_AUTH_SSL_REQ: return "MYSQL_AUTH_SSL_REQ"; + case MYSQL_AUTH_SSL_HANDSHAKE_DONE: return "MYSQL_AUTH_SSL_HANDSHAKE_DONE"; + case MYSQL_AUTH_SSL_HANDSHAKE_FAILED: return "MYSQL_AUTH_SSL_HANDSHAKE_FAILED"; + case MYSQL_AUTH_SSL_HANDSHAKE_ONGOING: return "MYSQL_AUTH_SSL_HANDSHAKE_ONGOING"; default: return "MySQL (unknown protocol state)"; } From 06c5da7b1728609f3d05610cfaf09a74c2dea4bc Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 9 Jun 2015 02:56:55 +0300 Subject: [PATCH 046/100] Minor fix to SSL authentication. --- server/core/dcb.c | 19 +++++++------------ server/modules/protocol/mysql_client.c | 13 +++++++++++++ 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index cc8d77259..cfee65371 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -1055,7 +1055,7 @@ int dcb_read_SSL( LOGIF(LD, (skygw_log_write( LOGFILE_DEBUG, - "%lu [dcb_read] Read %d bytes from dcb %p in state %s " + "%lu [dcb_read_SSL] Read %d bytes from dcb %p in state %s " "fd %d.", pthread_self(), n, @@ -2800,7 +2800,7 @@ int dcb_create_SSL(DCB* dcb) if((dcb->ssl = SSL_new(dcb->service->ctx)) == NULL) { - skygw_log_write(LE,"Error: Failed to initialize SSL connection."); + skygw_log_write(LE,"Error: Failed to initialize SSL for connection."); return -1; } @@ -2828,16 +2828,10 @@ int dcb_accept_SSL(DCB* dcb) int rval = 0,ssl_rval,errnum = 0,fd,b = 0; char errbuf[140]; fd = dcb->fd; - ioctl(fd,FIONREAD,&b); -#ifdef SS_DEBUG - skygw_log_write(LD,"[dcb_accept_SSL] fd %d bytes: %d",fd,b); -#endif - while(b > 0 && rval != -1) + + do { ssl_rval = SSL_accept(dcb->ssl); -#ifdef SS_DEBUG - skygw_log_write(LD,"[dcb_accept_SSL] SSL_accept returned %d.",ssl_rval); -#endif switch(ssl_rval) { case 0: @@ -2889,9 +2883,10 @@ int dcb_accept_SSL(DCB* dcb) ioctl(fd,FIONREAD,&b); #ifdef SS_DEBUG skygw_log_write_flush(LD,"[dcb_accept_SSL] fd %d: %d bytes",fd,b); - skygw_log_write_flush(LD,"[dcb_accept_SSL] SSL error: %d",errnum); + skygw_log_write(LD,"[dcb_accept_SSL] SSL_accept returned %d, SSL error: %d",ssl_rval,errnum); #endif - } + }while(b > 0 && rval != -1); + return rval; } diff --git a/server/modules/protocol/mysql_client.c b/server/modules/protocol/mysql_client.c index fcbb1958e..da5f04732 100644 --- a/server/modules/protocol/mysql_client.c +++ b/server/modules/protocol/mysql_client.c @@ -668,7 +668,18 @@ int gw_read_client_event( return 0; break; case 1: + { + int b = 0; + ioctl(dcb->fd,FIONREAD,&b); + if(b == 0) + { + skygw_log_write(LD, + "[gw_read_client_event] No data in socket after SSL auth"); + return 0; + } break; + } + case -1: return 1; break; @@ -1897,7 +1908,9 @@ int do_ssl_accept(MySQLProtocol* protocol) if(dcb->ssl == NULL) { if(dcb_create_SSL(dcb) != 0) + { return -1; + } } rval = dcb_accept_SSL(dcb); From ab120cb1de746b321f5ac662daf39aafddf40465 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 9 Jun 2015 17:04:51 +0300 Subject: [PATCH 047/100] Added Diffie-Hellman key exchange for MaxScale. --- server/core/service.c | 53 +++++++++++++++++++++++++++++++++++++++- server/include/service.h | 1 + 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/server/core/service.c b/server/core/service.c index f4880107d..826c79e66 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -91,7 +91,7 @@ static SPINLOCK service_spin = SPINLOCK_INIT; static SERVICE *allServices = NULL; static int find_type(typelib_t* tl, const char* needle, int maxlen); - +DH *ssl_get_dh2236(); static void service_add_qualified_param( SERVICE* svc, CONFIG_PARAMETER* param); @@ -1841,6 +1841,8 @@ int *data; int serviceInitSSL(SERVICE* service) { + DH* dh; + if(!service->ssl_init_done) { switch(service->ssl_method_type) @@ -1875,6 +1877,14 @@ int serviceInitSSL(SERVICE* service) } service->ctx = SSL_CTX_new(service->method); + + /** Enable the Diffie-Hellman algorithms */ + if((dh = ssl_get_dh2236()) != NULL) + { + SSL_CTX_set_tmp_dh(service->ctx,dh); + DH_free(dh); + } + if (SSL_CTX_use_certificate_file(service->ctx, service->ssl_cert, SSL_FILETYPE_PEM) <= 0) { skygw_log_write(LE,"Error: Failed to set server SSL certificate."); return -1; @@ -1908,3 +1918,44 @@ int serviceInitSSL(SERVICE* service) } return 0; } + +DH *ssl_get_dh2236() +{ + static unsigned char dh2236_p[]={ + 0x0B,0xC3,0xEC,0x3F,0xCB,0xD0,0x2E,0x43,0x7B,0x13,0xF9,0x0C, + 0x4D,0xE5,0xA3,0xA4,0xDB,0x68,0x13,0xBD,0xFC,0xD2,0x35,0x05, + 0xCB,0x62,0xA1,0x85,0x33,0x20,0xC4,0x88,0x3B,0x2B,0xD5,0x76, + 0x94,0xCD,0xEB,0x9C,0x5A,0xD1,0x16,0xDB,0x51,0x82,0x7A,0x1E, + 0xC6,0xC3,0xD9,0x52,0x8F,0x54,0x33,0xF4,0x50,0x96,0x01,0xF4, + 0x71,0xA1,0x8B,0x9B,0x43,0x85,0x9C,0x95,0xFF,0x53,0x1D,0x8D, + 0xDF,0xBC,0x60,0xEB,0x4D,0x96,0xD1,0x05,0x98,0x4A,0xEB,0xC9, + 0x33,0xF6,0xE9,0x74,0x73,0x29,0x27,0xCA,0x0D,0x6D,0xEA,0x36, + 0xB9,0x3B,0x54,0xF6,0x34,0x68,0x13,0xFA,0xAC,0x3B,0x57,0x55, + 0x76,0x41,0x67,0x48,0xEF,0x3C,0xE1,0xE1,0xAF,0x3C,0x68,0x05, + 0x9C,0x32,0xD9,0x14,0x8F,0xB2,0xEE,0xEE,0xBA,0x9F,0x0D,0x75, + 0xA7,0x33,0x1F,0x3A,0x0E,0xD1,0xA6,0x5A,0x29,0xC7,0x9B,0x5E, + 0x46,0xB1,0xA6,0xA5,0x1E,0x32,0xDB,0xAF,0x23,0x83,0x94,0x12, + 0x4F,0xE4,0xC2,0x8B,0x1B,0x2C,0x01,0x79,0x92,0x21,0xFF,0x01, + 0xED,0x46,0x27,0xF0,0x70,0x2A,0xA1,0xFD,0x5C,0x8F,0x8B,0x0C, + 0xC6,0x8F,0xFF,0x4C,0x99,0xAE,0x19,0xDB,0x58,0x4C,0xC0,0xE8, + 0x70,0xCC,0x7C,0x17,0xE8,0xBD,0x6B,0x19,0x93,0xB9,0x66,0xA9, + 0xD0,0x05,0x21,0x04,0x4C,0x7E,0x87,0x9F,0xF4,0xE9,0x23,0xE1, + 0x29,0x37,0xC5,0xE2,0x0A,0xC5,0xC1,0x92,0xC7,0x69,0xB4,0xFB, + 0x84,0x06,0xCE,0x0E,0xFC,0x65,0x70,0x2F,0xF6,0xB8,0x11,0xF9, + 0x0F,0x60,0x10,0xCA,0x94,0x29,0x44,0x5E,0x4A,0x05,0x46,0xE5, + 0xE6,0xA0,0xBD,0x14,0x45,0xA6,0xA7,0xCA,0x63,0x57,0xC6,0xB0, + 0x47,0xF9,0x71,0x24,0x19,0x75,0xD2,0x64,0x16,0xB1,0xBA,0x08, + 0xE9,0xE9,0xFB,0xF3, + }; + static unsigned char dh2236_g[]={ + 0x02, + }; + DH *dh; + + if ((dh=DH_new()) == NULL) return(NULL); + dh->p=BN_bin2bn(dh2236_p,sizeof(dh2236_p),NULL); + dh->g=BN_bin2bn(dh2236_g,sizeof(dh2236_g),NULL); + if ((dh->p == NULL) || (dh->g == NULL)) + { DH_free(dh); return(NULL); } + return(dh); +} \ No newline at end of file diff --git a/server/include/service.h b/server/include/service.h index af71bfe7d..936a73058 100644 --- a/server/include/service.h +++ b/server/include/service.h @@ -29,6 +29,7 @@ #include #include #include +#include /** * @file service.h * From 1ad1a31ed7a76ffe0345f7394202639df8094c1e Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 9 Jun 2015 17:18:25 +0300 Subject: [PATCH 048/100] Fixed the OpenSSL error stack being printed wrong. --- server/core/dcb.c | 35 +++++++++++++++++++++++++++++------ server/core/gateway.c | 2 +- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index cfee65371..a91645417 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -1026,7 +1026,19 @@ int dcb_read_SSL( STRDCBSTATE(dcb->state), dcb->fd, ssl_errno, - errbuf))); + strerror(errno)))); + + if(ssl_errno == SSL_ERROR_SSL || + ssl_errno == SSL_ERROR_SYSCALL) + { + while((ssl_errno = ERR_get_error()) != 0) + { + ERR_error_string(ssl_errno,errbuf); + skygw_log_write(LE, + "%s", + errbuf); + } + } } gwbuf_free(buffer); @@ -2850,6 +2862,7 @@ int dcb_accept_SSL(DCB* dcb) break; case -1: + errnum = SSL_get_error(dcb->ssl,ssl_rval); if(errnum == SSL_ERROR_WANT_READ || errnum == SSL_ERROR_WANT_WRITE) @@ -2864,12 +2877,22 @@ int dcb_accept_SSL(DCB* dcb) else { rval = -1; - ERR_error_string(errnum,errbuf); - skygw_log_write_flush(LE, - "Error: Fatal error in SSL_accept for %s: (SSL error code: %d) %s", - dcb->remote, - errnum, + skygw_log_write(LE, + "Error: Fatal error in SSL_accept for %s: (SSL error code: %d):%s", + dcb->remote, + errnum, + strerror(errno)); + if(errnum == SSL_ERROR_SSL || + errnum == SSL_ERROR_SYSCALL) + { + while((errnum = ERR_get_error()) != 0) + { + ERR_error_string(errnum,errbuf); + skygw_log_write(LE, + "%s", errbuf); + } + } } break; diff --git a/server/core/gateway.c b/server/core/gateway.c index 67d5ecf62..794ed5304 100644 --- a/server/core/gateway.c +++ b/server/core/gateway.c @@ -2038,4 +2038,4 @@ static void maxscale_ssl_lock(int mode,int n,const char* file, int line) static unsigned long maxscale_ssl_id() { return (unsigned long)pthread_self(); -} \ No newline at end of file +} From 1275a594acf71d276f6c3390cceb9ec58cd4f151 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 7 May 2015 05:56:28 +0300 Subject: [PATCH 049/100] Added missing utils library link from testmodutils. --- server/core/test/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/core/test/CMakeLists.txt b/server/core/test/CMakeLists.txt index 82e58919f..5c1f9a8ad 100644 --- a/server/core/test/CMakeLists.txt +++ b/server/core/test/CMakeLists.txt @@ -21,7 +21,7 @@ target_link_libraries(test_spinlock fullcore log_manager) target_link_libraries(test_filter fullcore) target_link_libraries(test_buffer fullcore log_manager) target_link_libraries(test_dcb fullcore) -target_link_libraries(test_modutil fullcore) +target_link_libraries(test_modutil fullcore utils log_manager) target_link_libraries(test_poll fullcore) target_link_libraries(test_service fullcore) target_link_libraries(test_server fullcore) From 196d41cb88c7858a31f83a530a8a4712447a340a Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 9 Jun 2015 20:02:45 +0300 Subject: [PATCH 050/100] More debug output. --- server/core/dcb.c | 53 ++++++++++++++++---------- server/modules/protocol/mysql_client.c | 3 +- 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index a91645417..0ce9ec053 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -1005,9 +1005,18 @@ int dcb_read_SSL( char errbuf[200]; ssl_errno = SSL_get_error(dcb->ssl,n); #ifdef SS_DEBUG - ERR_error_string(ssl_errno,errbuf); - skygw_log_write_flush(LD,"[%lu]SSL error %d: %s", - pthread_self(),ssl_errno,errbuf); + if(ssl_errno == SSL_ERROR_SSL || + ssl_errno == SSL_ERROR_SYSCALL) + { + int eno; + while((eno = ERR_get_error()) != 0) + { + ERR_error_string(eno,errbuf); + skygw_log_write(LE, + "%s", + errbuf); + } + } #endif if(ssl_errno == SSL_ERROR_WANT_READ || ssl_errno == SSL_ERROR_WANT_WRITE || @@ -1017,7 +1026,6 @@ int dcb_read_SSL( } else { - ERR_error_string(ssl_errno,errbuf); LOGIF(LE, (skygw_log_write_flush( LOGFILE_ERROR, "Error : Read failed, dcb %p in state " @@ -1052,7 +1060,7 @@ int dcb_read_SSL( gwbuf_rtrim(buffer,bufsize - n); #ifdef SS_DEBUG - skygw_log_write(LD,"[%lu] SSL: Truncated buffer from %d to %d bytes. " + skygw_log_write(LD,"%lu SSL: Truncated buffer from %d to %d bytes. " "Read %d bytes, %d bytes waiting.\n",pthread_self(), bufsize,GWBUF_LENGTH(buffer),n,b); @@ -1080,13 +1088,6 @@ int dcb_read_SSL( rc = ioctl(dcb->fd, FIONREAD, &b); pending = SSL_pending(dcb->ssl); - if(ssl_errno == SSL_ERROR_WANT_READ || - ssl_errno == SSL_ERROR_WANT_WRITE || - (b == 0 && pending == 0)) - { - break; - } - } /*< while (true) */ return_n: return n; @@ -2837,22 +2838,34 @@ int dcb_create_SSL(DCB* dcb) */ int dcb_accept_SSL(DCB* dcb) { - int rval = 0,ssl_rval,errnum = 0,fd,b = 0; + int rval = 0,ssl_rval,errnum = 0,fd,b = 0,pending; char errbuf[140]; fd = dcb->fd; do { ssl_rval = SSL_accept(dcb->ssl); + errnum = SSL_get_error(dcb->ssl,ssl_rval); + LOGIF(LD,(skygw_log_write_flush(LD,"[dcb_accept_SSL] SSL_accept %d, error %d", + ssl_rval,errnum))); switch(ssl_rval) { case 0: errnum = SSL_get_error(dcb->ssl,ssl_rval); - ERR_error_string(errnum,errbuf); - LOGIF(LD,(skygw_log_write_flush(LD,"[%p] SSL_accept shutdown for %s:%s", + skygw_log_write(LE,"Error: SSL authentication failed (SSL error %d):", dcb, dcb->remote, - errbuf))); + errnum); + + if(errnum == SSL_ERROR_SSL || + errnum == SSL_ERROR_SYSCALL) + { + while((errnum = ERR_get_error()) != 0) + { + ERR_error_string(errnum,errbuf); + skygw_log_write(LE,"%s",errbuf); + } + } rval = -1; break; case 1: @@ -2871,7 +2884,7 @@ int dcb_accept_SSL(DCB* dcb) queue and wait for more.*/ rval = 0; LOGIF(LD,(skygw_log_write_flush(LD,"[dcb_accept_SSL] SSL_accept ongoing for %s", - dcb->remote))); + dcb->remote))); return rval; } else @@ -2904,11 +2917,11 @@ int dcb_accept_SSL(DCB* dcb) break; } ioctl(fd,FIONREAD,&b); + pending = SSL_pending(dcb->ssl); #ifdef SS_DEBUG - skygw_log_write_flush(LD,"[dcb_accept_SSL] fd %d: %d bytes",fd,b); - skygw_log_write(LD,"[dcb_accept_SSL] SSL_accept returned %d, SSL error: %d",ssl_rval,errnum); + skygw_log_write_flush(LD,"[dcb_accept_SSL] fd %d: %d bytes, %d pending",fd,b,pending); #endif - }while(b > 0 && rval != -1); + }while((b > 0 || pending > 0) && rval != -1); return rval; } diff --git a/server/modules/protocol/mysql_client.c b/server/modules/protocol/mysql_client.c index da5f04732..187c33910 100644 --- a/server/modules/protocol/mysql_client.c +++ b/server/modules/protocol/mysql_client.c @@ -1952,8 +1952,7 @@ int do_ssl_accept(MySQLProtocol* protocol) spinlock_release(&protocol->protocol_lock); rval = -1; skygw_log_write_flush(LE, - "Error: Fatal error in SSL_accept for %s@%s: %s", - protocol->owner_dcb->user, + "Error: Fatal error in SSL_accept for %s", protocol->owner_dcb->remote); break; From de2910f75b2164ba6c5b606b036f5c7e46fd8a90 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 9 Jun 2015 22:27:15 +0300 Subject: [PATCH 051/100] Fixed SSL_accept failing if the GWBUF with the initial MySQL auth packet contains some of the SSL authentication data. --- server/core/dcb.c | 144 +++++++++++++++++++++++++ server/include/dcb.h | 1 + server/modules/protocol/mysql_client.c | 6 +- 3 files changed, 150 insertions(+), 1 deletion(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index 0ce9ec053..4c9dd0324 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -883,6 +883,150 @@ return_n: } +/** + * General purpose read routine to read data from a socket in the + * Descriptor Control Block and append it to a linked list of buffers. + * The list may be empty, in which case *head == NULL + * + * @param dcb The DCB to read from + * @param head Pointer to linked list to append data to + * @return -1 on error, otherwise the number of read bytes on the last + * iteration of while loop. 0 is returned if no data available. + */ +int dcb_read_n( + DCB *dcb, + GWBUF **head, + int nbytes) +{ + GWBUF *buffer = NULL; + int b; + int rc; + int n; + int nread = 0; + + CHK_DCB(dcb); + + if (dcb->fd <= 0) + { + LOGIF(LE, (skygw_log_write_flush( + LOGFILE_ERROR, + "Error : Read failed, dcb is %s.", + dcb->fd == DCBFD_CLOSED ? "closed" : "cloned, not readable"))); + n = 0; + goto return_n; + } + + int bufsize; + + rc = ioctl(dcb->fd, FIONREAD, &b); + + if (rc == -1) + { + LOGIF(LE, (skygw_log_write_flush( + LOGFILE_ERROR, + "Error : ioctl FIONREAD for dcb %p in " + "state %s fd %d failed due error %d, %s.", + dcb, + STRDCBSTATE(dcb->state), + dcb->fd, + errno, + strerror(errno)))); + n = -1; + goto return_n; + } + + if (b == 0 && nread == 0) + { + /** Handle closed client socket */ + if (dcb_isclient(dcb)) + { + char c; + int l_errno = 0; + int r = -1; + + /* try to read 1 byte, without consuming the socket buffer */ + r = recv(dcb->fd, &c, sizeof(char), MSG_PEEK); + l_errno = errno; + + if (r <= 0 && + l_errno != EAGAIN && + l_errno != EWOULDBLOCK && + l_errno != 0) + { + n = -1; + goto return_n; + } + } + n = 0; + goto return_n; + } + else if (b == 0) + { + n = 0; + goto return_n; + } + + dcb->last_read = hkheartbeat; + + bufsize = MIN(b, nbytes); + + if ((buffer = gwbuf_alloc(bufsize)) == NULL) + { + /*< + * This is a fatal error which should cause shutdown. + * Todo shutdown if memory allocation fails. + */ + LOGIF(LE, (skygw_log_write_flush( + LOGFILE_ERROR, + "Error : Failed to allocate read buffer " + "for dcb %p fd %d, due %d, %s.", + dcb, + dcb->fd, + errno, + strerror(errno)))); + + n = -1; + goto return_n; + } + GW_NOINTR_CALL(n = read(dcb->fd, GWBUF_DATA(buffer), bufsize); + dcb->stats.n_reads++); + + if (n <= 0) + { + if (errno != 0 && errno != EAGAIN && errno != EWOULDBLOCK) + { + LOGIF(LE, (skygw_log_write_flush( + LOGFILE_ERROR, + "Error : Read failed, dcb %p in state " + "%s fd %d, due %d, %s.", + dcb, + STRDCBSTATE(dcb->state), + dcb->fd, + errno, + strerror(errno)))); + } + gwbuf_free(buffer); + goto return_n; + } + nread += n; + + LOGIF(LD, (skygw_log_write( + LOGFILE_DEBUG, + "%lu [dcb_read] Read %d bytes from dcb %p in state %s " + "fd %d.", + pthread_self(), + n, + dcb, + STRDCBSTATE(dcb->state), + dcb->fd))); + /*< Append read data to the gwbuf */ + *head = gwbuf_append(*head, buffer); + +return_n: + return n; +} + + /** * General purpose read routine to read data from a socket in the * Descriptor Control Block and append it to a linked list of buffers. diff --git a/server/include/dcb.h b/server/include/dcb.h index cc96a2c0e..19f1e72ea 100644 --- a/server/include/dcb.h +++ b/server/include/dcb.h @@ -316,6 +316,7 @@ void dcb_free(DCB *); DCB *dcb_connect(struct server *, struct session *, const char *); DCB *dcb_clone(DCB *); int dcb_read(DCB *, GWBUF **); +int dcb_read_n(DCB*,GWBUF **,int); int dcb_drain_writeq(DCB *); void dcb_close(DCB *); DCB *dcb_process_zombies(int); /* Process Zombies except the one behind the pointer */ diff --git a/server/modules/protocol/mysql_client.c b/server/modules/protocol/mysql_client.c index 187c33910..15c572a0f 100644 --- a/server/modules/protocol/mysql_client.c +++ b/server/modules/protocol/mysql_client.c @@ -490,7 +490,6 @@ static int gw_mysql_do_authentication(DCB *dcb, GWBUF *queue) { /** Do the SSL Handshake */ if(ssl && protocol->owner_dcb->service->ssl_mode != SSL_DISABLED) { - protocol->protocol_auth_state = MYSQL_AUTH_SSL_REQ; if(do_ssl_accept(protocol) < 0) @@ -693,6 +692,11 @@ int gw_read_client_event( { rc = dcb_read_SSL(dcb, &read_buffer); } + else if(dcb->service->ssl_mode != SSL_DISABLED && + protocol->protocol_auth_state == MYSQL_AUTH_SENT) + { + rc = dcb_read_n(dcb, &read_buffer,(4 + 4 + 4 + 1 + 23)); + } else { rc = dcb_read(dcb, &read_buffer); From c15469013e89d31b9dd5878792d4013304716f90 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Wed, 10 Jun 2015 15:05:39 +0300 Subject: [PATCH 052/100] Fixed non-SSL connections to SSL enabled services failing. --- server/modules/protocol/mysql_client.c | 83 +++++++++++++++----------- 1 file changed, 48 insertions(+), 35 deletions(-) diff --git a/server/modules/protocol/mysql_client.c b/server/modules/protocol/mysql_client.c index 15c572a0f..dd6be0de2 100644 --- a/server/modules/protocol/mysql_client.c +++ b/server/modules/protocol/mysql_client.c @@ -74,7 +74,7 @@ int gw_MySQLWrite_client_SSL(DCB *dcb, GWBUF *queue); int gw_write_client_event_SSL(DCB *dcb); int mysql_send_ok(DCB *dcb, int packet_number, int in_affected_rows, const char* mysql_message); int MySQLSendHandshake(DCB* dcb); -static int gw_mysql_do_authentication(DCB *dcb, GWBUF *queue); +static int gw_mysql_do_authentication(DCB *dcb, GWBUF **queue); static int route_by_statement(SESSION *, GWBUF **); extern char* get_username_from_auth(char* ptr, uint8_t* data); extern int check_db_name_after_auth(DCB *, char *, int); @@ -402,7 +402,8 @@ MySQLSendHandshake(DCB* dcb) * @note in case of failure, dcb->data is freed before returning. If succeed, * dcb->data is freed in session.c:session_free. */ -static int gw_mysql_do_authentication(DCB *dcb, GWBUF *queue) { +static int gw_mysql_do_authentication(DCB *dcb, GWBUF **buf) { + GWBUF* queue = *buf; MySQLProtocol *protocol = NULL; /* int compress = -1; */ int connect_with_db = -1; @@ -464,46 +465,58 @@ static int gw_mysql_do_authentication(DCB *dcb, GWBUF *queue) { &protocol->client_capabilities); */ - if(protocol->protocol_auth_state == MYSQL_AUTH_SSL_HANDSHAKE_DONE) - goto ssl_hs_done; - - ssl = protocol->client_capabilities & GW_MYSQL_CAPABILITIES_SSL; - - /** Client didn't requested SSL when SSL mode was required*/ - if(!ssl && protocol->owner_dcb->service->ssl_mode == SSL_REQUIRED) + /** Skip this if the SSL handshake is already done. + * If not, start the SSL handshake. */ + if(protocol->protocol_auth_state != MYSQL_AUTH_SSL_HANDSHAKE_DONE) { - LOGIF(LT,(skygw_log_write(LT,"User %s@%s connected to service '%s' without SSL when SSL was required.", - protocol->owner_dcb->user, - protocol->owner_dcb->remote, - protocol->owner_dcb->service->name))); - return MYSQL_FAILED_AUTH_SSL; - } - if(LOG_IS_ENABLED(LT) && ssl) - { - skygw_log_write(LT,"User %s@%s connected to service '%s' with SSL.", - protocol->owner_dcb->user, - protocol->owner_dcb->remote, - protocol->owner_dcb->service->name); - } + ssl = protocol->client_capabilities & GW_MYSQL_CAPABILITIES_SSL; - /** Do the SSL Handshake */ - if(ssl && protocol->owner_dcb->service->ssl_mode != SSL_DISABLED) - { - protocol->protocol_auth_state = MYSQL_AUTH_SSL_REQ; - - if(do_ssl_accept(protocol) < 0) + /** Client didn't requested SSL when SSL mode was required*/ + if(!ssl && protocol->owner_dcb->service->ssl_mode == SSL_REQUIRED) { - return MYSQL_FAILED_AUTH; + LOGIF(LT,(skygw_log_write(LT,"User %s@%s connected to service '%s' without SSL when SSL was required.", + protocol->owner_dcb->user, + protocol->owner_dcb->remote, + protocol->owner_dcb->service->name))); + return MYSQL_FAILED_AUTH_SSL; } - else + + if(LOG_IS_ENABLED(LT) && ssl) { - return 0; + skygw_log_write(LT,"User %s@%s connected to service '%s' with SSL.", + protocol->owner_dcb->user, + protocol->owner_dcb->remote, + protocol->owner_dcb->service->name); + } + + /** Do the SSL Handshake */ + if(ssl && protocol->owner_dcb->service->ssl_mode != SSL_DISABLED) + { + protocol->protocol_auth_state = MYSQL_AUTH_SSL_REQ; + + if(do_ssl_accept(protocol) < 0) + { + return MYSQL_FAILED_AUTH; + } + else + { + return 0; + } + } + else if(dcb->service->ssl_mode == SSL_ENABLED) + { + /** This is a non-SSL connection to a SSL enabled service + * and we need to read the rest of the packet from the socket for the username */ + int bytes = dcb_read(dcb,&queue); + queue = gwbuf_make_contiguous(queue); + client_auth_packet = GWBUF_DATA(queue); + client_auth_packet_size = gwbuf_length(queue); + *buf = queue; + LOGIF(LD,(skygw_log_write(LD,"%lu Read %d bytes from fd %d",pthread_self(),bytes,dcb->fd))); } } - ssl_hs_done: - username = get_username_from_auth(username, client_auth_packet); if (username == NULL) @@ -848,7 +861,7 @@ int gw_read_client_event( { int auth_val; - auth_val = gw_mysql_do_authentication(dcb, read_buffer); + auth_val = gw_mysql_do_authentication(dcb, &read_buffer); if(protocol->protocol_auth_state == MYSQL_AUTH_SSL_REQ || protocol->protocol_auth_state == MYSQL_AUTH_SSL_HANDSHAKE_ONGOING || @@ -955,7 +968,7 @@ int gw_read_client_event( { int auth_val; - auth_val = gw_mysql_do_authentication(dcb, read_buffer); + auth_val = gw_mysql_do_authentication(dcb, &read_buffer); if (auth_val == 0) From bb427128a9f3a80f3545943e676d5422137770f6 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 11 Jun 2015 12:00:03 +0300 Subject: [PATCH 053/100] Fixed successful SSL_accept calls causing another call to SSL_accept. --- server/core/dcb.c | 34 ++++++++++------ server/core/gateway.c | 90 ++++++++++++++++++++++++++++--------------- 2 files changed, 81 insertions(+), 43 deletions(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index 4c9dd0324..e7c74b6bc 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -1046,7 +1046,7 @@ int dcb_read_SSL( int rc; int n; int nread = 0; - + int ssl_errno = 0; CHK_DCB(dcb); if (dcb->fd <= 0) @@ -1062,7 +1062,7 @@ int dcb_read_SSL( while (true) { int bufsize; - int ssl_errno = 0; + ssl_errno = 0; rc = ioctl(dcb->fd, FIONREAD, &b); pending = SSL_pending(dcb->ssl); if (rc == -1) @@ -1096,9 +1096,9 @@ int dcb_read_SSL( if(ssl_errno != SSL_ERROR_WANT_READ && ssl_errno != SSL_ERROR_WANT_WRITE && ssl_errno != SSL_ERROR_NONE) - { n = -1; - } + else + n = 0; goto return_n; } } @@ -1192,7 +1192,7 @@ int dcb_read_SSL( } } } - + n = -1; gwbuf_free(buffer); goto return_n; } @@ -1595,7 +1595,7 @@ dcb_write_SSL(DCB *dcb, GWBUF *queue) if (w < 0) { - int ssl_errno = ERR_get_error(); + int ssl_errno = SSL_get_error(dcb->ssl,w); if (LOG_IS_ENABLED(LOGFILE_DEBUG)) { @@ -1633,6 +1633,17 @@ dcb_write_SSL(DCB *dcb, GWBUF *queue) dcb, STRDCBSTATE(dcb->state), dcb->fd,ssl_errno))); + if(ssl_errno == SSL_ERROR_SSL || + ssl_errno == SSL_ERROR_SYSCALL) + { + while((ssl_errno = ERR_get_error()) != 0) + { + char errbuf[140]; + ERR_error_string(ssl_errno,errbuf); + skygw_log_write(LE,"%s",errbuf); + } + } + break; } } @@ -3016,7 +3027,7 @@ int dcb_accept_SSL(DCB* dcb) rval = 1; LOGIF(LD,(skygw_log_write_flush(LD,"[dcb_accept_SSL] SSL_accept done for %s", dcb->remote))); - break; + return rval; case -1: @@ -3035,10 +3046,10 @@ int dcb_accept_SSL(DCB* dcb) { rval = -1; skygw_log_write(LE, - "Error: Fatal error in SSL_accept for %s: (SSL error code: %d):%s", + "Error: Fatal error in SSL_accept for %s: (SSL version: %s SSL error code: %d)", dcb->remote, - errnum, - strerror(errno)); + SSL_get_version(dcb->ssl), + errnum); if(errnum == SSL_ERROR_SSL || errnum == SSL_ERROR_SYSCALL) { @@ -3104,8 +3115,7 @@ int dcb_connect_SSL(DCB* dcb) case -1: errnum = SSL_get_error(dcb->ssl,rval); - if(errnum == SSL_ERROR_WANT_READ || errnum == SSL_ERROR_WANT_WRITE || - errnum == SSL_ERROR_WANT_X509_LOOKUP) + if(errnum == SSL_ERROR_WANT_READ || errnum == SSL_ERROR_WANT_WRITE) { /** Not all of the data has been read. Go back to the poll queue and wait for more.*/ diff --git a/server/core/gateway.c b/server/core/gateway.c index 794ed5304..1805e483d 100644 --- a/server/core/gateway.c +++ b/server/core/gateway.c @@ -40,7 +40,16 @@ * @endverbatim */ #define _XOPEN_SOURCE 700 +#define OPENSSL_THREAD_DEFINES #include + + #include + #if defined(OPENSSL_THREADS) +#define HAVE_OPENSSL_THREADS 1 + #else +#define HAVE_OPENSSL_THREADS 0 + #endif + #include #include #include @@ -196,9 +205,46 @@ static bool resolve_maxscale_conf_fname( static char* check_dir_access(char* dirname,bool,bool); static int set_user(); -static void maxscale_ssl_lock(int mode,int n,const char* file, int line); -static unsigned long maxscale_ssl_id(); -static SPINLOCK* ssl_locks; + +/** SSL multi-threading functions and structures */ + +struct CRYPTO_dynlock_value +{ + SPINLOCK lock; +}; + +static struct CRYPTO_dynlock_value *ssl_create_dynlock(const char* file, int line) +{ + struct CRYPTO_dynlock_value* lock = malloc(sizeof(struct CRYPTO_dynlock_value)); + if(lock) + { + spinlock_init(&lock->lock); + } + return lock; +} + +static void ssl_lock_dynlock(int mode,struct CRYPTO_dynlock_value * n,const char* file, int line) +{ + if(mode & CRYPTO_LOCK) + { + spinlock_acquire(&n->lock); + } + else + { + spinlock_release(&n->lock); + } +} + +static void ssl_free_dynlock(struct CRYPTO_dynlock_value * n,const char* file, int line) +{ + free(n); +} + +static void maxscale_ssl_id(CRYPTO_THREADID* id) +{ + CRYPTO_THREADID_set_numeric(id,pthread_self()); +} + /** * Handler for SIGHUP signal. Reload the configuration for the * gateway. @@ -1374,23 +1420,21 @@ int main(int argc, char **argv) } /** OpenSSL initialization */ - - SSL_library_init(); - SSL_load_error_strings(); - OPENSSL_add_all_algorithms_noconf(); - - int n_locks = CRYPTO_num_locks(); - if((ssl_locks = malloc(n_locks*sizeof(SPINLOCK))) == NULL) + if(!HAVE_OPENSSL_THREADS) { + char* logerr = "OpenSSL library does not support multi-threading"; + print_log_n_stderr(true, true, logerr, logerr, eno); rc = MAXSCALE_INTERNALERROR; goto return_main; } + SSL_library_init(); + SSL_load_error_strings(); + OPENSSL_add_all_algorithms_noconf(); + CRYPTO_set_dynlock_create_callback(ssl_create_dynlock); + CRYPTO_set_dynlock_destroy_callback(ssl_free_dynlock); + CRYPTO_set_dynlock_lock_callback(ssl_lock_dynlock); + CRYPTO_THREADID_set_callback(maxscale_ssl_id); - for(i = 0;i Date: Thu, 11 Jun 2015 13:22:18 +0300 Subject: [PATCH 054/100] Added RSA key generator. --- server/core/service.c | 72 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 67 insertions(+), 5 deletions(-) diff --git a/server/core/service.c b/server/core/service.c index 826c79e66..fbffd27bd 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -69,6 +69,9 @@ extern int lm_enabled_logfiles_bitmask; extern size_t log_ses_count[]; extern __thread log_info_t tls_log_info; +static RSA *rsa_512 = NULL; +static RSA *rsa_1024 = NULL; + /** To be used with configuration type checks */ typedef struct typelib_st { int tl_nelems; @@ -418,6 +421,17 @@ serviceStart(SERVICE *service) SERV_PROTOCOL *port; int listeners = 0; +if(service->ssl_mode != SSL_DISABLED) +{ + if(serviceInitSSL(service) != 0) + { + LOGIF(LE, (skygw_log_write_flush(LOGFILE_ERROR, + "%s: SSL initialization failed. Service not started.", + service->name))); + service->state = SERVICE_STATE_FAILED; + return 0; + } +} if ((service->router_instance = service->router->createInstance(service, service->routerOptions)) == NULL) { @@ -1839,9 +1853,44 @@ int *data; } +/** + * + * @param s + * @param is_export + * @param keylength + * @return + */ + RSA *tmp_rsa_callback(SSL *s, int is_export, int keylength) + { + RSA *rsa_tmp=NULL; + + switch (keylength) { + case 512: + if (rsa_512) + rsa_tmp = rsa_512; + else { /* generate on the fly, should not happen in this example */ + rsa_tmp = RSA_generate_key(keylength,RSA_F4,NULL,NULL); + rsa_512 = rsa_tmp; /* Remember for later reuse */ + } + break; + case 1024: + if (rsa_1024) + rsa_tmp=rsa_1024; + break; + default: + /* Generating a key on the fly is very costly, so use what is there */ + if (rsa_1024) + rsa_tmp=rsa_1024; + else + rsa_tmp=rsa_512; /* Use at least a shorter key */ + } + return(rsa_tmp); + } + int serviceInitSSL(SERVICE* service) { DH* dh; + RSA* rsa; if(!service->ssl_init_done) { @@ -1878,12 +1927,21 @@ int serviceInitSSL(SERVICE* service) service->ctx = SSL_CTX_new(service->method); - /** Enable the Diffie-Hellman algorithms */ - if((dh = ssl_get_dh2236()) != NULL) + if(rsa_512 == NULL) { - SSL_CTX_set_tmp_dh(service->ctx,dh); - DH_free(dh); + rsa_512 = RSA_generate_key(512,RSA_F4,NULL,NULL); + if (rsa_512 == NULL) + skygw_log_write(LE,"Error: 512-bit RSA key generation failed."); } + if(rsa_1024 == NULL) + { + rsa_1024 = RSA_generate_key(1024,RSA_F4,NULL,NULL); + if (rsa_1024 == NULL) + skygw_log_write(LE,"Error: 1024-bit RSA key generation failed."); + } + + if(rsa_512 != NULL && rsa_1024 != NULL) + SSL_CTX_set_tmp_rsa_callback(service->ctx,tmp_rsa_callback); if (SSL_CTX_use_certificate_file(service->ctx, service->ssl_cert, SSL_FILETYPE_PEM) <= 0) { skygw_log_write(LE,"Error: Failed to set server SSL certificate."); @@ -1919,6 +1977,10 @@ int serviceInitSSL(SERVICE* service) return 0; } +/** + * Generated by OpenSSL. + * @return + */ DH *ssl_get_dh2236() { static unsigned char dh2236_p[]={ @@ -1958,4 +2020,4 @@ DH *ssl_get_dh2236() if ((dh->p == NULL) || (dh->g == NULL)) { DH_free(dh); return(NULL); } return(dh); -} \ No newline at end of file +} From 3f34d237cae6a367ac20c448d91e8d307e79e998 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 11 Jun 2015 15:26:05 +0300 Subject: [PATCH 055/100] enabled all bug fixes for OpenSSL. --- server/core/service.c | 1 + 1 file changed, 1 insertion(+) diff --git a/server/core/service.c b/server/core/service.c index fbffd27bd..bd09f06ee 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -1926,6 +1926,7 @@ int serviceInitSSL(SERVICE* service) } service->ctx = SSL_CTX_new(service->method); + SSL_CTX_set_options(service->ctx,SSL_OP_ALL); if(rsa_512 == NULL) { From 1c68a9a8729d625e495a6bf133f6c194d9b8dfaf Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 11 Jun 2015 15:54:09 +0300 Subject: [PATCH 056/100] Fixed dcb_connect_SSL calling SSL_connect again after a successful connection was already made. --- server/core/dcb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index e7c74b6bc..51dbb412f 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -3110,7 +3110,7 @@ int dcb_connect_SSL(DCB* dcb) LOGIF(LD,(skygw_log_write_flush(LD,"SSL_connect done for %s@%s", dcb->user, dcb->remote))); - break; + return rval; case -1: errnum = SSL_get_error(dcb->ssl,rval); From f602121459b021c4f597229add2dffbd2851aa34 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 12 Jun 2015 21:21:06 +0300 Subject: [PATCH 057/100] Added configurable SSL certificate verification depth and updated the documentation in the code. --- .../Getting-Started/Configuration-Guide.md | 9 ++ Documentation/Reference/MaxScale-and-SSL.md | 7 +- server/core/config.c | 11 ++ server/core/dcb.c | 13 +- server/core/gateway.c | 27 ++++ server/core/service.c | 120 ++++++++++-------- server/include/service.h | 13 +- server/modules/protocol/mysql_client.c | 53 ++++---- 8 files changed, 163 insertions(+), 90 deletions(-) diff --git a/Documentation/Getting-Started/Configuration-Guide.md b/Documentation/Getting-Started/Configuration-Guide.md index 23a9df70d..113e28c60 100644 --- a/Documentation/Getting-Started/Configuration-Guide.md +++ b/Documentation/Getting-Started/Configuration-Guide.md @@ -356,6 +356,15 @@ This parameter controls the level of encryption used. Accepted values are: * TLSv12 * MAX +### `ssl_cert_verification_depth` + +The maximum length of the certificate authority chain that will be accepted. Accepted values are positive integers. + +``` +# Example +ssl_cert_verification_depth=10 +``` + Example SSL enabled service configuration: ``` diff --git a/Documentation/Reference/MaxScale-and-SSL.md b/Documentation/Reference/MaxScale-and-SSL.md index ca61d52e2..d03a5af52 100644 --- a/Documentation/Reference/MaxScale-and-SSL.md +++ b/Documentation/Reference/MaxScale-and-SSL.md @@ -8,7 +8,8 @@ Here are the options which relate to SSL and certificates. Parameter|Values |Description ---------|-----------|-------- ssl | disabled, enabled, required |`disable` disables SSL, `enabled` enables SSL for client connections but still allows non-SSL connections and `required` requires SSL from all client connections. With the `required` option, client connections that do not use SSL will be rejected. -ssl_cert | |Path to server certificate -ssl_key | |Path to server private key -ssl_ca_cert | |Path to Certificate Authority file +ssl_cert | path to file |Path to server certificate +ssl_key | path to file |Path to server private key +ssl_ca_cert | path to file |Path to Certificate Authority file ssl_version|SSLV2,SSLV3,TLSV10,TLSV11,TLSV12,MAX| The SSL method level, defaults to highest available encryption level which is TLSv1.2 +ssl_cert_verify_depth|integer|Certificate authority certificate verification depth, default is 100. diff --git a/server/core/config.c b/server/core/config.c index 40be2c704..3e778d077 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -346,6 +346,7 @@ hashtable_memory_fns(monitorhash,strdup,NULL,free,NULL); char *version_string; char *subservices; char *ssl,*ssl_cert,*ssl_key,*ssl_ca_cert,*ssl_version; + char* ssl_cert_verify_depth; bool is_rwsplit = false; bool is_schemarouter = false; char *allow_localhost_match_wildcard_host; @@ -359,6 +360,7 @@ hashtable_memory_fns(monitorhash,strdup,NULL,free,NULL); ssl_key = config_get_value(obj->parameters, "ssl_key"); ssl_ca_cert = config_get_value(obj->parameters, "ssl_ca_cert"); ssl_version = config_get_value(obj->parameters, "ssl_version"); + ssl_cert_verify_depth = config_get_value(obj->parameters, "ssl_cert_verify_depth"); enable_root_user = config_get_value( obj->parameters, "enable_root_user"); @@ -514,6 +516,14 @@ hashtable_memory_fns(monitorhash,strdup,NULL,free,NULL); error_count++; } } + if(ssl_cert_verify_depth) + { + if(serviceSetSSLVerifyDepth(obj->element,atoi(ssl_cert_verify_depth)) != 0) + { + skygw_log_write(LE,"Error: Invalid parameter value for 'ssl_cert_verify_depth' for service '%s': %s",obj->object,ssl_cert_verify_depth); + error_count++; + } + } } } @@ -2005,6 +2015,7 @@ static char *service_params[] = "ssl", "ssl_key", "ssl_version", + "ssl_cert_verify_depth", NULL }; diff --git a/server/core/dcb.c b/server/core/dcb.c index 05114f48f..f600239ca 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -887,10 +887,13 @@ return_n: /** * General purpose read routine to read data from a socket in the * Descriptor Control Block and append it to a linked list of buffers. - * The list may be empty, in which case *head == NULL + * This function will read at most nbytes of data. + * + * The list may be empty, in which case *head == NULL. This * * @param dcb The DCB to read from * @param head Pointer to linked list to append data to + * @param nbytes Maximum number of bytes read * @return -1 on error, otherwise the number of read bytes on the last * iteration of while loop. 0 is returned if no data available. */ @@ -1835,7 +1838,8 @@ int above_water; /** * Drain the write queue of a DCB. This is called as part of the EPOLLOUT handling * of a socket and will try to send any buffered data from the write queue - * up until the point the write would block. + * up until the point the write would block. This function uses SSL encryption + * and the SSL handshake should have been completed prior to calling this function. * * @param dcb DCB to drain the write queue of * @return The number of bytes written @@ -2495,9 +2499,10 @@ static bool dcb_set_state_nomutex( } /** - * Write data to a DCB + * Write data to a socket through an SSL structure. The SSL structure is linked to a DCB's socket + * and all communication is encrypted and done via the SSL structure. * - * @param ssl The SSL to write the buffer to + * @param ssl The SSL structure to use for writing * @param buf Buffer to write * @param nbytes Number of bytes to write * @return Number of written bytes diff --git a/server/core/gateway.c b/server/core/gateway.c index 412714fbb..21c4472c0 100644 --- a/server/core/gateway.c +++ b/server/core/gateway.c @@ -208,11 +208,21 @@ static int set_user(); /** SSL multi-threading functions and structures */ +/** + * OpenSSL requires this struct to be defined in order to use dynamic locks + */ struct CRYPTO_dynlock_value { SPINLOCK lock; }; +/** + * Create a dynamic OpenSSL lock. The dynamic lock is just a wrapper structure + * around a SPINLOCK structure. + * @param file File name + * @param line Line number + * @return Pointer to new lock or NULL of an error occurred + */ static struct CRYPTO_dynlock_value *ssl_create_dynlock(const char* file, int line) { struct CRYPTO_dynlock_value* lock = malloc(sizeof(struct CRYPTO_dynlock_value)); @@ -223,6 +233,13 @@ static struct CRYPTO_dynlock_value *ssl_create_dynlock(const char* file, int lin return lock; } +/** + * Lock a dynamic lock for OpenSSL. + * @param mode + * @param n pointer to lock + * @param file File name + * @param line Line number + */ static void ssl_lock_dynlock(int mode,struct CRYPTO_dynlock_value * n,const char* file, int line) { if(mode & CRYPTO_LOCK) @@ -235,11 +252,21 @@ static void ssl_lock_dynlock(int mode,struct CRYPTO_dynlock_value * n,const char } } +/** + * Free a dynamic OpenSSL lock. + * @param n Lock to free + * @param file File name + * @param line Line number + */ static void ssl_free_dynlock(struct CRYPTO_dynlock_value * n,const char* file, int line) { free(n); } +/** + * The thread ID callback function for OpenSSL dynamic locks. + * @param id Id to modify + */ static void maxscale_ssl_id(CRYPTO_THREADID* id) { CRYPTO_THREADID_set_numeric(id,pthread_self()); diff --git a/server/core/service.c b/server/core/service.c index bd09f06ee..4215ce725 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -94,7 +94,7 @@ static SPINLOCK service_spin = SPINLOCK_INIT; static SERVICE *allServices = NULL; static int find_type(typelib_t* tl, const char* needle, int maxlen); -DH *ssl_get_dh2236(); + static void service_add_qualified_param( SERVICE* svc, CONFIG_PARAMETER* param); @@ -144,7 +144,8 @@ SERVICE *service; service->ssl_ca_cert = NULL; service->ssl_cert = NULL; service->ssl_key = NULL; - /** Use the highest possible SSL/TLS methods available */ + service->ssl_cert_verify_depth = DEFAULT_SSL_CERT_VERIFY_DEPTH; + /** Support the highest possible SSL/TLS methods available as the default */ service->ssl_method_type = SERVICE_SSL_TLS_MAX; if (service->name == NULL || service->routerModule == NULL) { @@ -875,6 +876,14 @@ serviceOptimizeWildcard(SERVICE *service, int action) return 1; } +/** + * Set the locations of the server's SSL certificate, server's private key and the CA + * certificate which both the client and the server should trust. + * @param service Service to configure + * @param cert SSL certificate + * @param key SSL private key + * @param ca_cert SSL CA certificate + */ void serviceSetCertificates(SERVICE *service, char* cert,char* key, char* ca_cert) { @@ -891,6 +900,12 @@ serviceSetCertificates(SERVICE *service, char* cert,char* key, char* ca_cert) service->ssl_ca_cert = strdup(ca_cert); } +/** + * Set the maximum SSL/TLS version the service will support + * @param service Service to configure + * @param version SSL/TLS version string + * @return 0 on success, -1 on invalid version string + */ int serviceSetSSLVersion(SERVICE *service, char* version) { @@ -909,7 +924,34 @@ serviceSetSSLVersion(SERVICE *service, char* version) else return -1; return 0; } -/** Enable or disable the service SSL capability*/ + +/** + * Set the service's SSL certificate verification depth. Depth of 0 means the peer + * certificate, 1 is the CA and 2 is a higher CA and so on. + * @param service Service to configure + * @param depth Certificate verification depth + * @return 0 on success, -1 on incorrect depth value + */ +int serviceSetSSLVerifyDepth(SERVICE* service, int depth) +{ + if(depth < 0) + return -1; + + service->ssl_cert_verify_depth = depth; + return 0; +} + +/** + * Enable or disable the service SSL capability of a service. + * The SSL mode string passed as a parameter should be one of required, enabled + * or disabled. Required requires all connections to use SSL encryption, enabled + * allows both SSL and non-SSL connections and disabled does not use SSL encryption. + * If the service SSL mode is set to enabled, then the client will decide whether + * SSL encryption is used. + * @param service Service to configure + * @param action Mode string. One of required, enabled or disabled. + * @return 0 on success, -1 on error + */ int serviceSetSSL(SERVICE *service, char* action) { @@ -1854,11 +1896,11 @@ int *data; /** - * - * @param s - * @param is_export - * @param keylength - * @return + * The RSA ket generation callback function for OpenSSL. + * @param s SSL structure + * @param is_export Not used + * @param keylength Length of the key + * @return Pointer to RSA structure */ RSA *tmp_rsa_callback(SSL *s, int is_export, int keylength) { @@ -1887,6 +1929,13 @@ int *data; return(rsa_tmp); } + /** + * Initialize the servce's SSL context. This sets up the generated RSA + * encryption keys, chooses the server encryption level and configures the server + * certificate, private key and certificate authority file. + * @param service + * @return + */ int serviceInitSSL(SERVICE* service) { DH* dh; @@ -1911,6 +1960,8 @@ int serviceInitSSL(SERVICE* service) case SERVICE_TLS12: service->method = (SSL_METHOD*)TLSv1_2_server_method(); break; + + /** Rest of these use the maximum available SSL/TLS methods */ case SERVICE_SSL_MAX: service->method = (SSL_METHOD*)SSLv23_server_method(); break; @@ -1926,8 +1977,11 @@ int serviceInitSSL(SERVICE* service) } service->ctx = SSL_CTX_new(service->method); + + /** Enable all OpenSSL bug fixes */ SSL_CTX_set_options(service->ctx,SSL_OP_ALL); + /** Generate the 512-bit and 1024-bit RSA keys */ if(rsa_512 == NULL) { rsa_512 = RSA_generate_key(512,RSA_F4,NULL,NULL); @@ -1944,6 +1998,7 @@ int serviceInitSSL(SERVICE* service) if(rsa_512 != NULL && rsa_1024 != NULL) SSL_CTX_set_tmp_rsa_callback(service->ctx,tmp_rsa_callback); + /** Load the server sertificate */ if (SSL_CTX_use_certificate_file(service->ctx, service->ssl_cert, SSL_FILETYPE_PEM) <= 0) { skygw_log_write(LE,"Error: Failed to set server SSL certificate."); return -1; @@ -1971,54 +2026,9 @@ int serviceInitSSL(SERVICE* service) /* Set to require peer (client) certificate verification */ SSL_CTX_set_verify(service->ctx,SSL_VERIFY_PEER,NULL); - /* Set the verification depth to 1 */ - SSL_CTX_set_verify_depth(service->ctx,1); + /* Set the verification depth */ + SSL_CTX_set_verify_depth(service->ctx,service->ssl_cert_verify_depth); service->ssl_init_done = true; } return 0; } - -/** - * Generated by OpenSSL. - * @return - */ -DH *ssl_get_dh2236() -{ - static unsigned char dh2236_p[]={ - 0x0B,0xC3,0xEC,0x3F,0xCB,0xD0,0x2E,0x43,0x7B,0x13,0xF9,0x0C, - 0x4D,0xE5,0xA3,0xA4,0xDB,0x68,0x13,0xBD,0xFC,0xD2,0x35,0x05, - 0xCB,0x62,0xA1,0x85,0x33,0x20,0xC4,0x88,0x3B,0x2B,0xD5,0x76, - 0x94,0xCD,0xEB,0x9C,0x5A,0xD1,0x16,0xDB,0x51,0x82,0x7A,0x1E, - 0xC6,0xC3,0xD9,0x52,0x8F,0x54,0x33,0xF4,0x50,0x96,0x01,0xF4, - 0x71,0xA1,0x8B,0x9B,0x43,0x85,0x9C,0x95,0xFF,0x53,0x1D,0x8D, - 0xDF,0xBC,0x60,0xEB,0x4D,0x96,0xD1,0x05,0x98,0x4A,0xEB,0xC9, - 0x33,0xF6,0xE9,0x74,0x73,0x29,0x27,0xCA,0x0D,0x6D,0xEA,0x36, - 0xB9,0x3B,0x54,0xF6,0x34,0x68,0x13,0xFA,0xAC,0x3B,0x57,0x55, - 0x76,0x41,0x67,0x48,0xEF,0x3C,0xE1,0xE1,0xAF,0x3C,0x68,0x05, - 0x9C,0x32,0xD9,0x14,0x8F,0xB2,0xEE,0xEE,0xBA,0x9F,0x0D,0x75, - 0xA7,0x33,0x1F,0x3A,0x0E,0xD1,0xA6,0x5A,0x29,0xC7,0x9B,0x5E, - 0x46,0xB1,0xA6,0xA5,0x1E,0x32,0xDB,0xAF,0x23,0x83,0x94,0x12, - 0x4F,0xE4,0xC2,0x8B,0x1B,0x2C,0x01,0x79,0x92,0x21,0xFF,0x01, - 0xED,0x46,0x27,0xF0,0x70,0x2A,0xA1,0xFD,0x5C,0x8F,0x8B,0x0C, - 0xC6,0x8F,0xFF,0x4C,0x99,0xAE,0x19,0xDB,0x58,0x4C,0xC0,0xE8, - 0x70,0xCC,0x7C,0x17,0xE8,0xBD,0x6B,0x19,0x93,0xB9,0x66,0xA9, - 0xD0,0x05,0x21,0x04,0x4C,0x7E,0x87,0x9F,0xF4,0xE9,0x23,0xE1, - 0x29,0x37,0xC5,0xE2,0x0A,0xC5,0xC1,0x92,0xC7,0x69,0xB4,0xFB, - 0x84,0x06,0xCE,0x0E,0xFC,0x65,0x70,0x2F,0xF6,0xB8,0x11,0xF9, - 0x0F,0x60,0x10,0xCA,0x94,0x29,0x44,0x5E,0x4A,0x05,0x46,0xE5, - 0xE6,0xA0,0xBD,0x14,0x45,0xA6,0xA7,0xCA,0x63,0x57,0xC6,0xB0, - 0x47,0xF9,0x71,0x24,0x19,0x75,0xD2,0x64,0x16,0xB1,0xBA,0x08, - 0xE9,0xE9,0xFB,0xF3, - }; - static unsigned char dh2236_g[]={ - 0x02, - }; - DH *dh; - - if ((dh=DH_new()) == NULL) return(NULL); - dh->p=BN_bin2bn(dh2236_p,sizeof(dh2236_p),NULL); - dh->g=BN_bin2bn(dh2236_g,sizeof(dh2236_g),NULL); - if ((dh->p == NULL) || (dh->g == NULL)) - { DH_free(dh); return(NULL); } - return(dh); -} diff --git a/server/include/service.h b/server/include/service.h index 936a73058..085c0c595 100644 --- a/server/include/service.h +++ b/server/include/service.h @@ -125,6 +125,8 @@ enum{ SERVICE_SSL_TLS_MAX }; +#define DEFAULT_SSL_CERT_VERIFY_DEPTH 100 /*< The default certificate verification depth */ + /** * Defines a service within the gateway. * @@ -173,14 +175,14 @@ typedef struct service { char *weightby; struct service *next; /**< The next service in the linked list */ SSL_CTX *ctx; - SSL *ssl; SSL_METHOD *method; /*< SSLv2/3 or TLSv1/2 methods * see: https://www.openssl.org/docs/ssl/SSL_CTX_new.html */ + int ssl_cert_verify_depth; /*< SSL certificate verification depth */ int ssl_method_type; /*< Which of the SSLv2/3 or TLS1.0/1.1/1.2 methods to use */ - char* ssl_cert; - char* ssl_key; - char* ssl_ca_cert; - bool ssl_init_done; + char* ssl_cert; /*< SSL certificate */ + char* ssl_key; /*< SSL private key */ + char* ssl_ca_cert; /*< SSL CA certificate */ + bool ssl_init_done; /*< If SSL has already been initialized for this service */ } SERVICE; @@ -212,6 +214,7 @@ extern void serviceSetFilters(SERVICE *, char *); extern int serviceSetSSL(SERVICE *service, char* action); extern int serviceInitSSL(SERVICE* service); extern int serviceSetSSLVersion(SERVICE *service, char* version); +extern int serviceSetSSLVerifyDepth(SERVICE* service, int depth); extern void serviceSetCertificates(SERVICE *service, char* cert,char* key, char* ca_cert); extern int serviceEnableRootUser(SERVICE *, int ); extern int serviceSetTimeout(SERVICE *, int ); diff --git a/server/modules/protocol/mysql_client.c b/server/modules/protocol/mysql_client.c index 3164f1642..c3e463139 100644 --- a/server/modules/protocol/mysql_client.c +++ b/server/modules/protocol/mysql_client.c @@ -389,15 +389,17 @@ MySQLSendHandshake(DCB* dcb) /** * gw_mysql_do_authentication * - * Performs the MySQL protocol 4.1 authentication, using data in GWBUF *queue + * Performs the MySQL protocol 4.1 authentication, using data in GWBUF **queue. * * (MYSQL_session*)client_data including: user, db, client_sha1 are copied into - * the dcb->data and later to dcb->session->data. - * - * client_capabilitiesa are copied into the dcb->protocol + * the dcb->data and later to dcb->session->data. client_capabilities are copied + * into the dcb->protocol. + * + * If SSL is enabled for the service, the SSL handshake will be done before the + * MySQL authentication. * * @param dcb Descriptor Control Block of the client - * @param queue The GWBUF with data from client + * @param queue Pointer to the location of the GWBUF with data from client * @return 0 If succeed, otherwise non-zero value * * @note in case of failure, dcb->data is freed before returning. If succeed, @@ -507,8 +509,11 @@ static int gw_mysql_do_authentication(DCB *dcb, GWBUF **buf) { } else if(dcb->service->ssl_mode == SSL_ENABLED) { - /** This is a non-SSL connection to a SSL enabled service - * and we need to read the rest of the packet from the socket for the username */ + /** This is a non-SSL connection to a SSL enabled service. + * We have only read enough of the packet to know that the client + * is not requesting SSL and the rest of the auth packet is still + * waiting in the socket. We need to read the data from the socket + * to find out the username of the connecting client. */ int bytes = dcb_read(dcb,&queue); queue = gwbuf_make_contiguous(queue); client_auth_packet = GWBUF_DATA(queue); @@ -626,7 +631,8 @@ gw_MySQLWrite_client(DCB *dcb, GWBUF *queue) /** - * Write function for client DCB: writes data from MaxScale to Client + * Write function for client DCB: writes data from MaxScale to Client using SSL + * encryption. The SSH handshake must have already been done. * * @param dcb The DCB of the client * @param queue Queue of buffers to write @@ -671,6 +677,8 @@ int gw_read_client_event( #endif + /** SSL authentication is still going on, we need to call do_ssl_accept + * until it return 1 for success or -1 for error */ if(protocol->protocol_auth_state == MYSQL_AUTH_SSL_HANDSHAKE_ONGOING || protocol->protocol_auth_state == MYSQL_AUTH_SSL_REQ) { @@ -704,15 +712,21 @@ int gw_read_client_event( if(protocol->use_ssl) { + /** SSL handshake is done, communication is now encrypted with SSL */ rc = dcb_read_SSL(dcb, &read_buffer); } else if(dcb->service->ssl_mode != SSL_DISABLED && protocol->protocol_auth_state == MYSQL_AUTH_SENT) { + /** The service allows both SSL and non-SSL connections. + * read only enough of the auth packet to know if the client is + * requesting SSL. If the client is not requesting SSL the rest of + the auth packet will be read later. */ rc = dcb_read_n(dcb, &read_buffer,(4 + 4 + 4 + 1 + 23)); } else { + /** Normal non-SSL connection */ rc = dcb_read(dcb, &read_buffer); } @@ -869,6 +883,9 @@ int gw_read_client_event( protocol->protocol_auth_state == MYSQL_AUTH_SSL_HANDSHAKE_DONE || protocol->protocol_auth_state == MYSQL_AUTH_SSL_HANDSHAKE_FAILED) { + /** SSL was requested and the handshake is either done or + * still ongoing. After the handshake is done, the client + * will send another auth packet. */ break; } @@ -1249,22 +1266,12 @@ return_1: return 1; } -/////////////////////////////////////////////// -// client write event to Client triggered by EPOLLOUT -////////////////////////////////////////////// -/** - * @node Client's fd became writable, and EPOLLOUT event - * arrived. As a consequence, client input buffer (writeq) is flushed. - * - * Parameters: - * @param dcb - in, use - * client dcb - * +/** + * EPOLLOUT event arrived and as a consequence, client input buffer (writeq) is + * flushed. The data is encrypted and SSL is used. The SSL handshake must have + * been successfully completed prior to this function being called. + * @param client dcb * @return constantly 1 - * - * - * @details (write detailed description here) - * */ int gw_write_client_event_SSL(DCB *dcb) { From 9b0a5f13285902a7ef7cdfd121f8d837ef53cb1f Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Mon, 15 Jun 2015 16:16:48 +0300 Subject: [PATCH 058/100] Added more comments. --- server/core/dcb.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index f600239ca..7f3651953 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -1032,9 +1032,10 @@ return_n: /** - * General purpose read routine to read data from a socket in the - * Descriptor Control Block and append it to a linked list of buffers. - * The list may be empty, in which case *head == NULL + * General purpose read routine to read data from a socket through the SSL + * structure lined with this DCB and append it to a linked list of buffers. + * The list may be empty, in which case *head == NULL. The SSL structure should + * be initialized and the SSL handshake should be done. * * @param dcb The DCB to read from * @param head Pointer to linked list to append data to From 2775145f377b9c3c66fb60b366c7561f7d502db6 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 16 Jun 2015 12:59:48 +0300 Subject: [PATCH 059/100] Fixed spelling errors. --- Documentation/Getting-Started/Configuration-Guide.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Documentation/Getting-Started/Configuration-Guide.md b/Documentation/Getting-Started/Configuration-Guide.md index 113e28c60..0ca3843c4 100644 --- a/Documentation/Getting-Started/Configuration-Guide.md +++ b/Documentation/Getting-Started/Configuration-Guide.md @@ -119,7 +119,7 @@ datadir=/home/user/maxscale_data/ #### `libdir` -Set the directory where MaxScale looks for modules. The library director is the only directory that MaxScale uses when it searches for modules. If you have custom modules for MaxScale, make sure you have them in this folder. +Set the directory where MaxScale looks for modules. The library directory is the only directory that MaxScale uses when it searches for modules. If you have custom modules for MaxScale, make sure you have them in this folder. ``` libdir=/home/user/lib64/ @@ -328,7 +328,7 @@ connection_timeout=300 ### Service and SSL -This section describes configuration parameters for services that control the SSL/TLS encrption method and the various certificate files involved in it. To enable SSL, you must configure the `ssl` parameter with either `enabled` or `required` and provide the three files for `ssl_cert`, `ssl_key` and `ssl_ca_cert`. After this, MySQL connections to this service can be encrypted with SSL. +This section describes configuration parameters for services that control the SSL/TLS encryption method and the various certificate files involved in it. To enable SSL, you must configure the `ssl` parameter with either `enabled` or `required` and provide the three files for `ssl_cert`, `ssl_key` and `ssl_ca_cert`. After this, MySQL connections to this service can be encrypted with SSL. #### `ssl` @@ -381,7 +381,7 @@ ssl_ca_cert=/home/markus/certs/ca.pem ssl_version=TLSv12 ``` -This configuration requires all connections to be encryped with SSL. It also specifies that TLSv1.2 should be used as the encryption method. The paths to the server certificate files and the Certificate Authority file are also provided. +This configuration requires all connections to be encrypted with SSL. It also specifies that TLSv1.2 should be used as the encryption method. The paths to the server certificate files and the Certificate Authority file are also provided. ### Server @@ -1129,7 +1129,7 @@ MariaDB [mysql]> grant REPLICATION CLIENT on *.* to 'maxscalemon'@'maxscalehost' Query OK, 0 rows affected (0.00 sec) ``` -MySQL monitor fetches the `@@server_id` variable and other informations from `SHOW SLAVE STATUS` in order to compute the replication topology tree that may include intermediate master servers, called relay servers. +MySQL monitor fetches the `@@server_id` variable and other information from `SHOW SLAVE STATUS` in order to compute the replication topology tree that may include intermediate master servers, called relay servers. The *Master* server used by router modules is the so called "root master": a server that has the `SERVER_MASTER` status bit set and it's at the lowest level of the replication depth. From 2ea7ff61a21880890e0f92dccc9ec31eaebc2c97 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 16 Jun 2015 14:07:34 +0300 Subject: [PATCH 060/100] Fixed init.d and systemd scripts. --- etc/maxscale.service.in | 1 + etc/ubuntu/init.d/maxscale.in | 14 +++++++------- server/include/gwdirs.h.in | 4 +++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/etc/maxscale.service.in b/etc/maxscale.service.in index 6717f6b25..1d0597c76 100644 --- a/etc/maxscale.service.in +++ b/etc/maxscale.service.in @@ -4,6 +4,7 @@ After=network.target [Service] Type=forking +Restart=on-failure PIDFile=@MAXSCALE_VARDIR@/run/maxscale/maxscale.pid ExecStart=@CMAKE_INSTALL_PREFIX@/@MAXSCALE_BINDIR@/maxscale --user=maxscale diff --git a/etc/ubuntu/init.d/maxscale.in b/etc/ubuntu/init.d/maxscale.in index f23ed144f..a60425890 100755 --- a/etc/ubuntu/init.d/maxscale.in +++ b/etc/ubuntu/init.d/maxscale.in @@ -44,8 +44,8 @@ _RETVAL_STATUS_NOT_RUNNING=3 # stop/start/status related vars ################################# NAME=maxscale -DAEMON=@CMAKE_INSTALL_PREFIX@/@MAXSCALE_BINDIR@/maxscale --user=maxscale - +DAEMON=@CMAKE_INSTALL_PREFIX@/@MAXSCALE_BINDIR@/maxscale +DAEMON_OPTS= --user=maxscale # Source function library. . /lib/lsb/init-functions @@ -57,11 +57,11 @@ RETVAL=0 start() { log_daemon_msg "Starting MaxScale" - start_daemon -p $MAXSCALE_PIDFILE $DAEMON 2> /dev/null > /dev/null + start_daemon -p "$MAXSCALE_PIDFILE" "$DAEMON" "$DAEMON_OPTS" 2> /dev/null > /dev/null sleep 2 - status_of_proc -p $MAXSCALE_PIDFILE $DAEMON $NAME + status_of_proc -p "$MAXSCALE_PIDFILE" "$DAEMON" $NAME log_end_msg $? } @@ -77,13 +77,13 @@ stop() { reload() { log_daemon_msg "Reloading MaxScale" - kill -HUP $(cat $MAXSCALE_PIDFILE) + kill -HUP $(cat "$MAXSCALE_PIDFILE") log_end_msg $? } maxscale_wait_stop() { - PIDTMP=$(pidofproc -p $MAXSCALE_PIDFILE @CMAKE_INSTALL_PREFIX@/@MAXSCALE_BINDIR@/maxscale) + PIDTMP=$(pidofproc -p "$MAXSCALE_PIDFILE" "$DAEMON") kill -TERM "${PIDTMP:-}" 2> /dev/null; if [ -n "${PIDTMP:-}" ] && kill -0 "${PIDTMP:-}" 2> /dev/null; then local i=0 @@ -115,7 +115,7 @@ case "$1" in # return 3 on any error log_daemon_msg "Checking MaxScale" - status_of_proc -p $MAXSCALE_PIDFILE $DAEMON $NAME + status_of_proc -p "$MAXSCALE_PIDFILE" "$DAEMON" $NAME RETVAL=$? if [ $RETVAL -ne 0 ]; then diff --git a/server/include/gwdirs.h.in b/server/include/gwdirs.h.in index fe911c71b..d9c17e152 100644 --- a/server/include/gwdirs.h.in +++ b/server/include/gwdirs.h.in @@ -26,7 +26,9 @@ /** Default file locations, configured by CMake */ static const char* default_cnf_fname = "maxscale.cnf"; static const char* default_configdir = "/etc/"; -static const char* default_piddir = "@MAXSCALE_VARDIR@/run/maxscale/"; +static const char* default_piddir = "@MAXSCALE_VARDIR@/run/maxscale/"; /*< This should be changed to just /run eventually, + * the /var/run folder is an old standard and the newe FSH 3.0 + * uses /run for PID files.*/ static const char* default_logdir = "@MAXSCALE_VARDIR@/log/maxscale/"; static const char* default_datadir = "@MAXSCALE_VARDIR@/lib/maxscale/"; static const char* default_libdir = "@CMAKE_INSTALL_PREFIX@/@MAXSCALE_LIBDIR@"; From 97a06e4006436cfd949ae54e3a6d9eac4b34fd7a Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 16 Jun 2015 14:24:38 +0300 Subject: [PATCH 061/100] Updated function documentation. --- server/core/service.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/server/core/service.c b/server/core/service.c index 4215ce725..4d2507cd6 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -1373,7 +1373,14 @@ void *router_obj; } } - +/** + * Refresh the database users for the service + * This function replaces the MySQL users used by the service with the latest + * version found on the backend servers. There is a limit on how often the users + * can be reloaded and if this limit is exceeded, the reload will fail. + * @param service Service to reload + * @return 0 on success and 1 on error + */ int service_refresh_users(SERVICE *service) { int ret = 1; /* check for another running getUsers request */ From 466ee46d8263de83c0d6f512df63a94f37095626 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 16 Jun 2015 17:10:00 +0300 Subject: [PATCH 062/100] Fix to MXS-125: https://mariadb.atlassian.net/browse/MXS-125 Unified the usages of maxkeys and maxpasswd. --- .../Getting-Started/Configuration-Guide.md | 14 ++++--- server/core/maxkeys.c | 38 ++++++++----------- server/core/maxpasswd.c | 8 ++-- server/core/secrets.c | 31 ++++++++------- server/include/secrets.h | 2 +- 5 files changed, 47 insertions(+), 46 deletions(-) diff --git a/Documentation/Getting-Started/Configuration-Guide.md b/Documentation/Getting-Started/Configuration-Guide.md index 0ca3843c4..afda397fe 100644 --- a/Documentation/Getting-Started/Configuration-Guide.md +++ b/Documentation/Getting-Started/Configuration-Guide.md @@ -1393,20 +1393,24 @@ In addition parameters may be added to define patterns to match against to eithe ## Encrypting Passwords -Passwords stored in the maxscale.cnf file may optionally be encrypted for added security. This is done by creation of an encryption key on installation of MaxScale. Encryption keys may be created manually by executing the maxkeys utility with the argument of the filename to store the key. The default location MaxScale stores the keys is `/var/cache/maxscale`. +Passwords stored in the maxscale.cnf file may optionally be encrypted for added security. This is done by creation of an encryption key on installation of MaxScale. Encryption keys may be created manually by executing the maxkeys utility with the argument of the filename to store the key. The default location MaxScale stores the keys is `/var/lib/maxscale`. ``` -maxkeys /var/cache/maxscale/.secrets + # Usage: maxkeys [PATH] +maxkeys /var/lib/maxscale/ ``` Changing the encryption key for MaxScale will invalidate any currently encrypted keys stored in the maxscale.cnf file. ### Creating Encrypted Passwords -Encrypted passwords are created by executing the maxpasswd command with the password you require to encrypt as an argument. +Encrypted passwords are created by executing the maxpasswd command with the location of the .secrets file and the password you require to encrypt as an argument. - maxpasswd MaxScalePw001 - 61DD955512C39A4A8BC4BB1E5F116705 +``` +# Usage: maxpasswd PATH PASSWORD +maxpasswd /var/lib/maxscale/ MaxScalePw001 +61DD955512C39A4A8BC4BB1E5F116705 +``` The output of the maxpasswd command is a hexadecimal string, this should be inserted into the maxscale.cnf file in place of the ordinary, plain text, password. MaxScale will determine this as an encrypted password and automatically decrypt it before sending it the database server. diff --git a/server/core/maxkeys.c b/server/core/maxkeys.c index 557920dbe..27b5c8382 100644 --- a/server/core/maxkeys.c +++ b/server/core/maxkeys.c @@ -32,19 +32,24 @@ #include #include #include + int main(int argc, char **argv) { - int arg_count = 6; + int arg_count = 4; char *home; + char *keyfile; char** arg_vector; int rval = 0; - if (argc != 2) + if (argc < 2) { - fprintf(stderr, "Usage: %s \n", argv[0]); - return 1; + keyfile = "/var/lib/maxscale/"; + fprintf(stderr, "Generating .secrets file in /var/lib/maxscale/ ...\n"); + } + else + { + keyfile = argv[1]; } - arg_vector = malloc(sizeof(char*)*(arg_count + 1)); if(arg_vector == NULL) @@ -53,27 +58,16 @@ int main(int argc, char **argv) return 1; } - if(access("/var/log/maxscale/maxkeys/",F_OK) != 0) - { - if(mkdir("/var/log/maxscale/maxkeys/",0777) == -1) - { - if(errno != EEXIST) - { - fprintf(stderr,"Error: %d - %s",errno,strerror(errno)); - return 1; - } - } - } - arg_vector[0] = strdup("logmanager"); - arg_vector[1] = strdup("-j"); - arg_vector[2] = strdup("/var/log/maxscale/maxkeys"); - arg_vector[3] = NULL; + arg_vector[0] = "logmanager"; + arg_vector[1] = "-j"; + arg_vector[2] = "/var/log/maxscale/maxkeys"; + arg_vector[3] = "-o"; + arg_vector[4] = NULL; skygw_logmanager_init(arg_count,arg_vector); - free(arg_vector[2]); free(arg_vector); - if (secrets_writeKeys(argv[1])) + if (secrets_writeKeys(keyfile)) { fprintf(stderr, "Failed to encode the password\n"); rval = 1; diff --git a/server/core/maxpasswd.c b/server/core/maxpasswd.c index 5c761f378..0a3d5ddea 100644 --- a/server/core/maxpasswd.c +++ b/server/core/maxpasswd.c @@ -46,9 +46,9 @@ main(int argc, char **argv) char** arg_vector; int rval = 0; - if (argc != 2) + if (argc != 3) { - fprintf(stderr, "Usage: %s \n", argv[0]); + fprintf(stderr, "Usage: %s \n", argv[0]); return 1; } @@ -79,9 +79,9 @@ main(int argc, char **argv) return 1; } - strncpy(pw,argv[1],80); + strncpy(pw,argv[2],80); - if ((enc = encryptPassword(pw)) != NULL){ + if ((enc = encryptPassword(argv[1],pw)) != NULL){ printf("%s\n", enc); }else{ fprintf(stderr, "Failed to encode the password\n"); diff --git a/server/core/secrets.c b/server/core/secrets.c index 2ec3e75a9..97cb49806 100644 --- a/server/core/secrets.c +++ b/server/core/secrets.c @@ -53,15 +53,14 @@ int i; } /** - * secrets_readKeys - * - * This routine reads data from a binary file and extracts the AES encryption key - * and the AES Init Vector - * + * This routine reads data from a binary file named ".secrets" and extracts the AES encryption key + * and the AES Init Vector. + * If the path parameter is not null the custom path is interpreted as a folder + * containing the .secrets file. Otherwise the default location is used. * @return The keys structure or NULL on error */ static MAXKEYS * -secrets_readKeys() +secrets_readKeys(char* path) { char secret_file[PATH_MAX+1]; char *home; @@ -70,9 +69,10 @@ struct stat secret_stats; int fd; int len; static int reported = 0; - - snprintf(secret_file, PATH_MAX, "%s/.secrets", get_datadir()); - + if(path != NULL) + snprintf(secret_file, PATH_MAX, "%s/.secrets", path); + else + snprintf(secret_file, PATH_MAX, "%s/.secrets", get_datadir()); /* Try to access secrets file */ if (access(secret_file, R_OK) == -1) { @@ -221,11 +221,14 @@ static int reported = 0; * @param secret_file The file with secret keys * @return 0 on success and 1 on failure */ -int secrets_writeKeys(char *secret_file) +int secrets_writeKeys(char *path) { int fd,randfd; unsigned int randval; MAXKEYS key; +char secret_file[PATH_MAX + 10]; + + sprintf(secret_file,"%s/.secrets",path); /* Open for writing | Create | Truncate the file for writing */ if ((fd = open(secret_file, O_CREAT | O_WRONLY | O_TRUNC, S_IRUSR)) < 0) @@ -328,7 +331,7 @@ char *ptr; unsigned char encrypted[80]; int enlen; - keys = secrets_readKeys(); + keys = secrets_readKeys(NULL); if (!keys) return strdup(crypt); /* @@ -365,12 +368,12 @@ int enlen; * Encrypt a password that can be stored in the MaxScale configuration file. * * Note the return is always a malloc'd string that the caller must free - * + * @param path Path the the .secrets file * @param password The password to encrypt * @return The encrypted password */ char * -encryptPassword(char *password) +encryptPassword(char* path, char *password) { MAXKEYS *keys; AES_KEY aeskey; @@ -379,7 +382,7 @@ char *hex_output; unsigned char padded_passwd[80]; unsigned char encrypted[80]; - if ((keys = secrets_readKeys()) == NULL) + if ((keys = secrets_readKeys(path)) == NULL) return NULL; memset(padded_passwd, 0, 80); diff --git a/server/include/secrets.h b/server/include/secrets.h index 505e79154..cb2912827 100644 --- a/server/include/secrets.h +++ b/server/include/secrets.h @@ -53,5 +53,5 @@ typedef struct maxkeys { extern int secrets_writeKeys(char *filename); extern char *decryptPassword(char *); -extern char *encryptPassword(char *); +extern char *encryptPassword(char*,char *); #endif From 2d63aa2842b0da54a8cb988963e7f461227314b3 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 16 Jun 2015 22:36:53 +0300 Subject: [PATCH 063/100] Fixed typo and added an example script. --- Documentation/monitors/Galera-Monitor.md | 2 +- Documentation/monitors/MM-Monitor.md | 2 +- Documentation/monitors/MySQL-Monitor.md | 30 ++++++++++++++++++- Documentation/monitors/NDB-Cluster-Monitor.md | 2 +- 4 files changed, 32 insertions(+), 4 deletions(-) diff --git a/Documentation/monitors/Galera-Monitor.md b/Documentation/monitors/Galera-Monitor.md index ffe54fb83..6a66fdcfc 100644 --- a/Documentation/monitors/Galera-Monitor.md +++ b/Documentation/monitors/Galera-Monitor.md @@ -108,7 +108,7 @@ master_down|A Master server has gone down master_up|A Master server has come up slave_down|A Slave server has gone down slave_up|A Slave server has come up -server_down|A server with no assigned role has done down +server_down|A server with no assigned role has gone down server_up|A server with no assigned role has come up synced_down|A synced Galera node has come up synced_up|A synced Galera node has gone down diff --git a/Documentation/monitors/MM-Monitor.md b/Documentation/monitors/MM-Monitor.md index 00375d4bf..ad9ee2c07 100644 --- a/Documentation/monitors/MM-Monitor.md +++ b/Documentation/monitors/MM-Monitor.md @@ -99,7 +99,7 @@ master_down|A Master server has gone down master_up|A Master server has come up slave_down|A Slave server has gone down slave_up|A Slave server has come up -server_down|A server with no assigned role has done down +server_down|A server with no assigned role has gone down server_up|A server with no assigned role has come up lost_master|A server lost Master status lost_slave|A server lost Slave status diff --git a/Documentation/monitors/MySQL-Monitor.md b/Documentation/monitors/MySQL-Monitor.md index ab643109e..1bfc06109 100644 --- a/Documentation/monitors/MySQL-Monitor.md +++ b/Documentation/monitors/MySQL-Monitor.md @@ -108,10 +108,38 @@ master_down|A Master server has gone down master_up|A Master server has come up slave_down|A Slave server has gone down slave_up|A Slave server has come up -server_down|A server with no assigned role has done down +server_down|A server with no assigned role has gone down server_up|A server with no assigned role has come up lost_master|A server lost Master status lost_slave|A server lost Slave status new_master|A new Master was detected new_slave|A new Slave was detected + +## Example 1 - Monitor script + +Here is an example shell script which sends an email to an admin when a server goes down. + +``` +#!/usr/bin/env bash + +#This script assumes that the local mail server is configured properly + +message="A server has gone down at `date`." +echo $message|mail -s "A server has gone down" admin@my.org + +``` + +Here is a monitor configuration that only triggers the script when a master or a slave server goes down. + +``` +[Database Monitor] +type=monitor +module=mysqlmon +servers=server1,server2 +script=mail_to_admin.sh +events=master_down,slave_down +``` + +When a master or a slave server goes down, the script is executed, a mail is sent and the administrator will be immediately notified of any possible problems. +This is just a simple example showing what you can do with MaxScale and monitor scripts. diff --git a/Documentation/monitors/NDB-Cluster-Monitor.md b/Documentation/monitors/NDB-Cluster-Monitor.md index a9f453678..9468ec323 100644 --- a/Documentation/monitors/NDB-Cluster-Monitor.md +++ b/Documentation/monitors/NDB-Cluster-Monitor.md @@ -89,7 +89,7 @@ master_down|A Master server has gone down master_up|A Master server has come up slave_down|A Slave server has gone down slave_up|A Slave server has come up -server_down|A server with no assigned role has done down +server_down|A server with no assigned role has gone down server_up|A server with no assigned role has come up ndb_down|A MySQL Cluster node has gone down ndb_up|A MySQL Cluster node has come up From f0aed1f66628c12fd3de1d1c6c759eac6ffb4a5d Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Wed, 17 Jun 2015 10:05:56 +0300 Subject: [PATCH 064/100] Fixed possible null pointer dereferences in mysql_backend. --- server/modules/protocol/mysql_backend.c | 46 +++++++++++++------------ 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/server/modules/protocol/mysql_backend.c b/server/modules/protocol/mysql_backend.c index d000474ab..477ca4cf0 100644 --- a/server/modules/protocol/mysql_backend.c +++ b/server/modules/protocol/mysql_backend.c @@ -1159,30 +1159,32 @@ gw_backend_close(DCB *dcb) * but client's close and adding client's DCB to zombies list is executed * only if client's DCB's state does _not_ change in parallel. */ - spinlock_acquire(&session->ses_lock); - /** - * If session->state is STOPPING, start closing client session. - * Otherwise only this backend connection is closed. - */ - if (session != NULL && - session->state == SESSION_STATE_STOPPING && - session->client != NULL) - { - if (session->client->state == DCB_STATE_POLLING) - { - spinlock_release(&session->ses_lock); - - /** Close client DCB */ - dcb_close(session->client); - } - else - { - spinlock_release(&session->ses_lock); - } - } - else + if(session != NULL) { + spinlock_acquire(&session->ses_lock); + /** + * If session->state is STOPPING, start closing client session. + * Otherwise only this backend connection is closed. + */ + if (session->state == SESSION_STATE_STOPPING && + session->client != NULL) + { + if (session->client->state == DCB_STATE_POLLING) + { + spinlock_release(&session->ses_lock); + + /** Close client DCB */ + dcb_close(session->client); + } + else + { + spinlock_release(&session->ses_lock); + } + } + else + { spinlock_release(&session->ses_lock); + } } return 1; } From 425dd8cb3b7c93b698a3395022638f299a6a4861 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Wed, 17 Jun 2015 10:20:00 +0300 Subject: [PATCH 065/100] Removed SSLv2 methods from serviceInitSSL because OpenSSL 1.1.0 does not support them. --- Documentation/Getting-Started/Configuration-Guide.md | 1 - Documentation/Reference/MaxScale-and-SSL.md | 2 +- server/core/service.c | 7 +------ server/include/service.h | 5 ++--- 4 files changed, 4 insertions(+), 11 deletions(-) diff --git a/Documentation/Getting-Started/Configuration-Guide.md b/Documentation/Getting-Started/Configuration-Guide.md index afda397fe..c2f815d09 100644 --- a/Documentation/Getting-Started/Configuration-Guide.md +++ b/Documentation/Getting-Started/Configuration-Guide.md @@ -349,7 +349,6 @@ This is the Certificate Authority file. It will be used to verify that both the ### `ssl_version` This parameter controls the level of encryption used. Accepted values are: - * SSLv2 * SSLv3 * TLSv10 * TLSv11 diff --git a/Documentation/Reference/MaxScale-and-SSL.md b/Documentation/Reference/MaxScale-and-SSL.md index d03a5af52..f293085c1 100644 --- a/Documentation/Reference/MaxScale-and-SSL.md +++ b/Documentation/Reference/MaxScale-and-SSL.md @@ -11,5 +11,5 @@ ssl | disabled, enabled, required |`disable` disables SSL, `enabled` ena ssl_cert | path to file |Path to server certificate ssl_key | path to file |Path to server private key ssl_ca_cert | path to file |Path to Certificate Authority file -ssl_version|SSLV2,SSLV3,TLSV10,TLSV11,TLSV12,MAX| The SSL method level, defaults to highest available encryption level which is TLSv1.2 +ssl_version|SSLV3,TLSV10,TLSV11,TLSV12,MAX| The SSL method level, defaults to highest available encryption level which is TLSv1.2 ssl_cert_verify_depth|integer|Certificate authority certificate verification depth, default is 100. diff --git a/server/core/service.c b/server/core/service.c index 4d2507cd6..b0959e6c3 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -909,9 +909,7 @@ serviceSetCertificates(SERVICE *service, char* cert,char* key, char* ca_cert) int serviceSetSSLVersion(SERVICE *service, char* version) { - if(strcasecmp(version,"SSLV2") == 0) - service->ssl_method_type = SERVICE_SSLV2; - else if(strcasecmp(version,"SSLV3") == 0) + if(strcasecmp(version,"SSLV3") == 0) service->ssl_method_type = SERVICE_SSLV3; else if(strcasecmp(version,"TLSV10") == 0) service->ssl_method_type = SERVICE_TLS10; @@ -1952,9 +1950,6 @@ int serviceInitSSL(SERVICE* service) { switch(service->ssl_method_type) { - case SERVICE_SSLV2: - service->method = (SSL_METHOD*)SSLv2_server_method(); - break; case SERVICE_SSLV3: service->method = (SSL_METHOD*)SSLv3_server_method(); break; diff --git a/server/include/service.h b/server/include/service.h index 085c0c595..3337ebfc0 100644 --- a/server/include/service.h +++ b/server/include/service.h @@ -115,7 +115,6 @@ typedef enum { } ssl_mode_t; enum{ - SERVICE_SSLV2, SERVICE_SSLV3, SERVICE_TLS10, SERVICE_TLS11, @@ -175,10 +174,10 @@ typedef struct service { char *weightby; struct service *next; /**< The next service in the linked list */ SSL_CTX *ctx; - SSL_METHOD *method; /*< SSLv2/3 or TLSv1/2 methods + SSL_METHOD *method; /*< SSLv3 or TLS1.0/1.1/1.2 methods * see: https://www.openssl.org/docs/ssl/SSL_CTX_new.html */ int ssl_cert_verify_depth; /*< SSL certificate verification depth */ - int ssl_method_type; /*< Which of the SSLv2/3 or TLS1.0/1.1/1.2 methods to use */ + int ssl_method_type; /*< Which of the SSLv3 or TLS1.0/1.1/1.2 methods to use */ char* ssl_cert; /*< SSL certificate */ char* ssl_key; /*< SSL private key */ char* ssl_ca_cert; /*< SSL CA certificate */ From acd649cdc4b7909cf8f144819f175215eb70d8e1 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Wed, 17 Jun 2015 18:18:40 +0300 Subject: [PATCH 066/100] Reverted back to older OpenSSL functions for compatibility support. --- server/core/gateway.c | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/server/core/gateway.c b/server/core/gateway.c index 21c4472c0..f206ca30e 100644 --- a/server/core/gateway.c +++ b/server/core/gateway.c @@ -263,15 +263,6 @@ static void ssl_free_dynlock(struct CRYPTO_dynlock_value * n,const char* file, i free(n); } -/** - * The thread ID callback function for OpenSSL dynamic locks. - * @param id Id to modify - */ -static void maxscale_ssl_id(CRYPTO_THREADID* id) -{ - CRYPTO_THREADID_set_numeric(id,pthread_self()); -} - /** * Handler for SIGHUP signal. Reload the configuration for the * gateway. @@ -1468,7 +1459,7 @@ int main(int argc, char **argv) CRYPTO_set_dynlock_create_callback(ssl_create_dynlock); CRYPTO_set_dynlock_destroy_callback(ssl_free_dynlock); CRYPTO_set_dynlock_lock_callback(ssl_lock_dynlock); - CRYPTO_THREADID_set_callback(maxscale_ssl_id); + CRYPTO_set_id_callback(pthread_self); /* register exit function for embedded MySQL library */ l = atexit(libmysqld_done); From fd42395d009208bc24ca51372a838c9e1174fbd1 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 18 Jun 2015 08:09:59 +0300 Subject: [PATCH 067/100] Removed mandb call from postinstall script and added cmake messages about systemd service file installation. --- CMakeLists.txt | 4 ++++ etc/postinst.in | 16 ++++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a21e9c745..f77fc79d7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -173,6 +173,7 @@ if(WITH_SCRIPTS) if(PACKAGE) message(STATUS "maxscale.conf will unpack to: /etc/ld.so.conf.d") message(STATUS "startup scripts will unpack to to: /etc/init.d") + message(STATUS "systemd service files will unpack to to: /usr/lib/systemd/system") install(FILES ${CMAKE_BINARY_DIR}/maxscale DESTINATION ${MAXSCALE_SHAREDIR} PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) install(FILES ${CMAKE_BINARY_DIR}/maxscale.conf DESTINATION ${MAXSCALE_SHAREDIR} @@ -184,8 +185,11 @@ if(WITH_SCRIPTS) PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) install(FILES ${CMAKE_BINARY_DIR}/maxscale.conf DESTINATION /etc/ld.so.conf.d PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) + install(FILES ${CMAKE_BINARY_DIR}/maxscale.service DESTINATION /usr/lib/systemd/system + PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) message(STATUS "Installing maxscale.conf to: /etc/ld.so.conf.d") message(STATUS "Installing startup scripts to: /etc/init.d") + message(STATUS "Installing systemd service files to: /usr/lib/systemd/system") endif() endif() diff --git a/etc/postinst.in b/etc/postinst.in index e1d3dc8ad..30e45f806 100755 --- a/etc/postinst.in +++ b/etc/postinst.in @@ -31,14 +31,22 @@ chmod 0755 @MAXSCALE_VARDIR@/cache/maxscale chmod 0755 @MAXSCALE_VARDIR@/run/maxscale # Copy init.d script and ldconfig file -cp @CMAKE_INSTALL_PREFIX@/@MAXSCALE_SHAREDIR@/maxscale /etc/init.d/ -cp @CMAKE_INSTALL_PREFIX@/@MAXSCALE_SHAREDIR@/maxscale.conf /etc/ld.so.conf.d/ -if [ -d "/usr/lib/systemd/system" ] +if [ -f "@CMAKE_INSTALL_PREFIX@/@MAXSCALE_SHAREDIR@/maxscale" ] +then + cp @CMAKE_INSTALL_PREFIX@/@MAXSCALE_SHAREDIR@/maxscale /etc/init.d/ +fi + +if [ -f "@CMAKE_INSTALL_PREFIX@/@MAXSCALE_SHAREDIR@/maxscale.conf" ] +then + cp @CMAKE_INSTALL_PREFIX@/@MAXSCALE_SHAREDIR@/maxscale.conf /etc/init.d/ +fi + +if [ -d "/usr/lib/systemd/system" -a -f @CMAKE_INSTALL_PREFIX@/@MAXSCALE_SHAREDIR@/maxscale.service ] then cp @CMAKE_INSTALL_PREFIX@/@MAXSCALE_SHAREDIR@/maxscale.service /usr/lib/systemd/system fi /sbin/ldconfig -mandb + cat <& 2 ********** Notice: MaxScale 1.2 Changes ************** From 09ffeff17e46cc4bd2e75e68bdf3cbb7448ab419 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 18 Jun 2015 13:25:30 +0300 Subject: [PATCH 068/100] Updated lsyncd document. --- .../Reference/MaxScale-HA-with-lsyncd.md | 57 ++++++++++++++++++- 1 file changed, 54 insertions(+), 3 deletions(-) diff --git a/Documentation/Reference/MaxScale-HA-with-lsyncd.md b/Documentation/Reference/MaxScale-HA-with-lsyncd.md index 18ef233db..d81d26790 100644 --- a/Documentation/Reference/MaxScale-HA-with-lsyncd.md +++ b/Documentation/Reference/MaxScale-HA-with-lsyncd.md @@ -1,9 +1,55 @@ # MaxScale HA with Lsyncd -This document guides you in setting up multiple MaxScale instances and synchronizing the configuration files with Lsyncd. Lsyncd is a rsync wrapper which can synchronize files across the network. The lsyncd daemon uses a configuration file to control the files to synchronize and the remote targets where these files are synchronized to. +This document guides you in setting up multiple MaxScale instances and synchronizing the configuration files with Lsyncd. Lsyncd is a rsync wrapper which can synchronize files across the network. The lsyncd daemon uses a configuration file to control the files to synchronize and the remote targets where these files are synchronized to. This guide was writted for lsyncd 2.1.5. Copying the configuration file and running the lsyncd daemon on all the hosts keeps all the configuration files in sync. Modifications in the configuration file on one of the hosts will be copied on the other hosts. This allows adinistrators to easily provide a highly available, disaster resistant MaxScale installation with up-to-date configuration files on all the hosts. +You will need to have SSH access to the remote servers for. + +## Creating SSH keys + +For lsyncd to work, we will need to either use an existing set of SSH keys or to create a new set of keys. If you already have a SSH key generated, you can skip this next step and go to the Copying Keys part. + +### Generating keys + +To generate a new set of SSH keys, we will use `ssh-keygen`. + +``` +[root@localhost ~]# ssh-keygen +Generating public/private rsa key pair. +Enter file in which to save the key (/root/.ssh/id_rsa): +Enter passphrase (empty for no passphrase): +Enter same passphrase again: +Your identification has been saved in /root/.ssh/id_rsa. +Your public key has been saved in /root/.ssh/id_rsa.pub. +The key fingerprint is: +f4:99:0a:cc:d4:ac:ea:ed:ff:0d:bb:e5:87:3e:38:df root@localhost.localdomain +The key's randomart image is: ++--[ RSA 2048]----+ +| | +| o | +| . + | +| + o . o | +| = S + | +| . . . | +| . . .... | +| . . o*o.. | +| ..o...+==oE | ++-----------------+ + +``` + +The keys will be generated in the .ssh folder and will automatically be used by ssh. + +### Copying keys + +To copy the SSH keys to the remote host we will use `ssh-copy-id`. + +Use the username and host of the remote server you wish to synchronize MaxScale's configuration files to. For example, if the server's address is 192.168.122.100 and the user we use for synchronization us `user` we can use the following command. + +``` +ssh-copy-id user@192.168.122.100 +``` ## Installing Lsyncd @@ -21,8 +67,6 @@ Installing with Apt: apt-get install lsyncd ``` -Lsyncd needs no further configuration after installation. All necessary options are configured in the configuration file passed to the lsyncd daemon. - ## Creating the Lsyncd configuration file Lsyncd uses a configuration file to determine where to read files from and where to synchronize them if changes in them occur. Lsyncd is written in Lua and the configuration file is also valid Lua code. @@ -44,6 +88,7 @@ default.rsyncssh, source="/etc", -- This is the user and host where the maxscale.cnf is copied to. +-- Change this to the user and destination host where you want maxscale.cnf to be synchronized to. host="user@127.0.0.1", -- This is where the maxscale.cnf is copied to on the remote host. @@ -108,6 +153,12 @@ rsync={ Starting lsyncd can be done from the command line or through a init script. To start syncd from the command like, execute the `lsyncd` command and pass the configuration file as the only parameter. +By default lsyncd will search for the configuration file in `/etc/lsyncd.conf`. By placing the configuration file we created in the `/etc` folder, we can start lsyncd with the following command. + +``` +service lsyncd start +``` + Here is an example which start lsyncd and reads the configuration options from the `lsyncd.cnf` file. ``` From e6b16121eb5f253fc93c163b4015713625e498d9 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 18 Jun 2015 13:49:12 +0300 Subject: [PATCH 069/100] Added instructions for SSH key generation. --- .../Reference/MaxScale-HA-with-lsyncd.md | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/Documentation/Reference/MaxScale-HA-with-lsyncd.md b/Documentation/Reference/MaxScale-HA-with-lsyncd.md index d81d26790..c9e6ac332 100644 --- a/Documentation/Reference/MaxScale-HA-with-lsyncd.md +++ b/Documentation/Reference/MaxScale-HA-with-lsyncd.md @@ -1,14 +1,26 @@ # MaxScale HA with Lsyncd -This document guides you in setting up multiple MaxScale instances and synchronizing the configuration files with Lsyncd. Lsyncd is a rsync wrapper which can synchronize files across the network. The lsyncd daemon uses a configuration file to control the files to synchronize and the remote targets where these files are synchronized to. This guide was writted for lsyncd 2.1.5. +***This guide was written for lsyncd 2.1.5.*** + +This document guides you in setting up multiple MaxScale instances and synchronizing the configuration files with lsyncd. Lsyncd is a rsync wrapper which can synchronize files across the network. The lsyncd daemon uses a configuration file to control the files to synchronize and the remote targets where these files are synchronized to. Copying the configuration file and running the lsyncd daemon on all the hosts keeps all the configuration files in sync. Modifications in the configuration file on one of the hosts will be copied on the other hosts. This allows adinistrators to easily provide a highly available, disaster resistant MaxScale installation with up-to-date configuration files on all the hosts. -You will need to have SSH access to the remote servers for. +### Requirements +You will need: + +* Access to the remote hosts. +* MaxScale installed on all systems +* Configured maxscale.cnf file in /etc +* SSH daemon and clients installed on all hosts + +The installation and configuration of MaxScale is covered in other documents. ## Creating SSH keys -For lsyncd to work, we will need to either use an existing set of SSH keys or to create a new set of keys. If you already have a SSH key generated, you can skip this next step and go to the Copying Keys part. +For lsyncd to work, we will need to either use an existing set of SSH keys or to create a new set of keys. The creation and copying of keys needs to be repeated on all of the hosts. + +If you already have a SSH key generated, you can skip this next step and go to the Copying Keys part. ### Generating keys @@ -51,7 +63,11 @@ Use the username and host of the remote server you wish to synchronize MaxScale' ssh-copy-id user@192.168.122.100 ``` -## Installing Lsyncd +Repeat the last command with the usernames and addresses of all the remote hosts you want to synchronize the configuration files to. + +## Installing lsyncd + +You will need to install lsyncd on all of the hosts for changes in the configuration file on one of the nodes to be synchronized to the other nodes. You can install lsyncd with either a package manager or by building from source code. This guide demonstrates installation using a package manager and those looking to build lsyncd from source should refer to its documentation: https://github.com/axkibe/lsyncd/wiki/Manual-to-Lsyncd-2.1.x @@ -89,7 +105,7 @@ source="/etc", -- This is the user and host where the maxscale.cnf is copied to. -- Change this to the user and destination host where you want maxscale.cnf to be synchronized to. -host="user@127.0.0.1", +host="user@192.168.122.100", -- This is where the maxscale.cnf is copied to on the remote host. targetdir="/etc", From f6369a1661a3ec785db2fd96c7b54881366deaad Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 18 Jun 2015 18:03:55 +0300 Subject: [PATCH 070/100] Changed from server unique name to ip:port in monitor script initiaion. --- server/modules/monitor/monitor_common.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/server/modules/monitor/monitor_common.c b/server/modules/monitor/monitor_common.c index f4566c35a..19980c5dd 100644 --- a/server/modules/monitor/monitor_common.c +++ b/server/modules/monitor/monitor_common.c @@ -230,7 +230,7 @@ void mon_append_node_names(MONITOR_SERVERS* start,char* str, int len) MONITOR_SERVERS* ptr = start; bool first = true; int slen = strlen(str); - + char arr[256]; while(ptr && slen < len) { if(!first) @@ -238,7 +238,8 @@ void mon_append_node_names(MONITOR_SERVERS* start,char* str, int len) strncat(str,",",len); } first = false; - strncat(str,ptr->server->unique_name,len); + sprintf(arr,"%s:%d",ptr->server->name,ptr->server->port); + strcat(str,arr); ptr = ptr->next; slen = strlen(str); } @@ -299,10 +300,11 @@ void monitor_launch_script(MONITOR* mon,MONITOR_SERVERS* ptr, char* script) EXTERNCMD* cmd; snprintf(argstr,PATH_MAX + MON_ARG_MAX, - "%s --event=%s --initiator=%s --nodelist=", + "%s --event=%s --initiator=%s:%d --nodelist=", script, mon_get_event_name(ptr), - ptr->server->unique_name); + ptr->server->name, + ptr->server->port); mon_append_node_names(mon->databases,argstr,PATH_MAX + MON_ARG_MAX + 1); if((cmd = externcmd_allocate(argstr)) == NULL) From 0cc2c13d7e124ee10deb837fe7d7e4a73c8c1928 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 18 Jun 2015 18:13:50 +0300 Subject: [PATCH 071/100] Updated documentation. --- Documentation/monitors/MySQL-Monitor.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Documentation/monitors/MySQL-Monitor.md b/Documentation/monitors/MySQL-Monitor.md index 1bfc06109..7228c1b10 100644 --- a/Documentation/monitors/MySQL-Monitor.md +++ b/Documentation/monitors/MySQL-Monitor.md @@ -124,9 +124,11 @@ Here is an example shell script which sends an email to an admin when a server g #!/usr/bin/env bash #This script assumes that the local mail server is configured properly - +#The second argument is the event type +event=${$2/.*=/} +server=${$3/.*=/} message="A server has gone down at `date`." -echo $message|mail -s "A server has gone down" admin@my.org +echo $message|mail -s "The event was $event for server $server." admin@my.org ``` From 79c20bfae82a2d0347b808c98ef42ec1f5d878d5 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 19 Jun 2015 04:45:02 +0300 Subject: [PATCH 072/100] Fixed trace and debug logs not being created in shared memory. --- log_manager/log_manager.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/log_manager/log_manager.cc b/log_manager/log_manager.cc index ba16db52f..b31ae6112 100644 --- a/log_manager/log_manager.cc +++ b/log_manager/log_manager.cc @@ -1761,7 +1761,6 @@ static bool fnames_conf_init( case 's': /** record list of log file ids for later use */ - if(do_syslog) shmem_id_str = optarg; break; case 'h': @@ -1793,12 +1792,14 @@ static bool fnames_conf_init( strdup(get_logpath_default()) : fn->fn_logpath; /** Set identity string for syslog if it is not set in config.*/ + if(do_syslog) + { syslog_ident_str = (syslog_ident_str == NULL ? (argv == NULL ? strdup(program_invocation_short_name) : strdup(*argv)) : syslog_ident_str); - + } /* ss_dfprintf(stderr, "\n\n\tCommand line : "); for (i=0; i Date: Sun, 21 Jun 2015 12:51:54 +0300 Subject: [PATCH 073/100] Fix to MXS-212: https://mariadb.atlassian.net/browse/MXS-212 The listener DCB is now properly closed instead of just being removed from the poll set. --- server/core/dcb.c | 8 +++++++- server/core/poll.c | 3 ++- server/core/service.c | 19 +++++++++---------- server/modules/protocol/mysql_client.c | 2 +- server/modules/protocol/mysql_common.c | 5 ++++- 5 files changed, 23 insertions(+), 14 deletions(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index 7f3651953..fc9329524 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -1949,13 +1949,15 @@ dcb_close(DCB *dcb) } ss_dassert(dcb->state == DCB_STATE_POLLING || + dcb->state == DCB_STATE_LISTENING || dcb->state == DCB_STATE_NOPOLLING || dcb->state == DCB_STATE_ZOMBIE); /*< * Stop dcb's listening and modify state accordingly. */ - if (dcb->state == DCB_STATE_POLLING) + if (dcb->state == DCB_STATE_POLLING || + dcb->state == DCB_STATE_LISTENING) { rc = poll_remove_dcb(dcb); @@ -2428,6 +2430,10 @@ static bool dcb_set_state_nomutex( case DCB_STATE_POLLING: /*< ok to try but state can't change */ succp = true; break; + case DCB_STATE_LISTENING: + dcb->state = new_state; + succp = true; + break; default: ss_dassert(old_state != NULL); break; diff --git a/server/core/poll.c b/server/core/poll.c index 377310cb0..9a1a5565d 100644 --- a/server/core/poll.c +++ b/server/core/poll.c @@ -330,7 +330,8 @@ poll_remove_dcb(DCB *dcb) CHK_DCB(dcb); /*< It is possible that dcb has already been removed from the set */ - if (dcb->state != DCB_STATE_POLLING) + if (dcb->state != DCB_STATE_POLLING && + dcb->state != DCB_STATE_LISTENING) { if (dcb->state == DCB_STATE_NOPOLLING || dcb->state == DCB_STATE_ZOMBIE) diff --git a/server/core/service.c b/server/core/service.c index b0959e6c3..c11c0bddc 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -534,11 +534,13 @@ int listeners = 0; port = service->ports; while (port) { - poll_remove_dcb(port->listener); - port->listener->session->state = SESSION_STATE_LISTENER_STOPPED; + if(port->listener) + { + dcb_close(port->listener); + port->listener = NULL; listeners++; - - port = port->next; + } + port = port->next; } service->state = SERVICE_STATE_STOPPED; @@ -562,13 +564,10 @@ int listeners = 0; port = service->ports; while (port) { - if (poll_add_dcb(port->listener) == 0) { - port->listener->session->state = SESSION_STATE_LISTENER; - listeners++; - } - port = port->next; + listeners += serviceStartPort(service,port); + port = port->next; } - + service->state = SERVICE_STATE_STARTED; return listeners; } diff --git a/server/modules/protocol/mysql_client.c b/server/modules/protocol/mysql_client.c index c3e463139..21a11dc42 100644 --- a/server/modules/protocol/mysql_client.c +++ b/server/modules/protocol/mysql_client.c @@ -1771,7 +1771,7 @@ gw_client_close(DCB *dcb) dcb->state == DCB_STATE_NOPOLLING || dcb->state == DCB_STATE_ZOMBIE) { - if (!DCB_IS_CLONE(dcb)) CHK_PROTOCOL(protocol); + if (!DCB_IS_CLONE(dcb) && protocol) CHK_PROTOCOL(protocol); } #endif LOGIF(LD, (skygw_log_write(LOGFILE_DEBUG, diff --git a/server/modules/protocol/mysql_common.c b/server/modules/protocol/mysql_common.c index 83e2912e5..eb044c1fb 100644 --- a/server/modules/protocol/mysql_common.c +++ b/server/modules/protocol/mysql_common.c @@ -128,7 +128,10 @@ void mysql_protocol_done ( MySQLProtocol* p; server_command_t* scmd; server_command_t* scmd2; - + + if(dcb->protocol == NULL) + return; + p = (MySQLProtocol *)dcb->protocol; spinlock_acquire(&p->protocol_lock); From c3aa5beeb45abc921bdc49a751e20269bea394ac Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Sun, 21 Jun 2015 19:32:19 +0300 Subject: [PATCH 074/100] Added missing initialization from MM monitor. --- server/modules/monitor/mmmon.c | 1 + 1 file changed, 1 insertion(+) diff --git a/server/modules/monitor/mmmon.c b/server/modules/monitor/mmmon.c index c0d54595d..545a77ca2 100644 --- a/server/modules/monitor/mmmon.c +++ b/server/modules/monitor/mmmon.c @@ -126,6 +126,7 @@ startMonitor(void *arg,void* opt) handle->shutdown = 0; handle->id = MONITOR_DEFAULT_ID; handle->master = NULL; + handle->script = NULL; memset(handle->events,false,sizeof(handle->events)); spinlock_init(&handle->lock); } From 8c900e73deab80691e004ee3319981adf173dd4e Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Mon, 22 Jun 2015 10:46:00 +0200 Subject: [PATCH 075/100] removed extra blr_file_add_magic removed extra blr_file_add_magic --- server/modules/include/blr.h | 1 + server/modules/routing/binlog/blr_file.c | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/server/modules/include/blr.h b/server/modules/include/blr.h index 62cda59a3..72171b253 100644 --- a/server/modules/include/blr.h +++ b/server/modules/include/blr.h @@ -34,6 +34,7 @@ #include #include +#include #define BINLOG_FNAMELEN 16 #define BLR_PROTOCOL "MySQLBackend" diff --git a/server/modules/routing/binlog/blr_file.c b/server/modules/routing/binlog/blr_file.c index ca92d6d06..68f47b1c3 100644 --- a/server/modules/routing/binlog/blr_file.c +++ b/server/modules/routing/binlog/blr_file.c @@ -210,9 +210,8 @@ int fd; close(router->binlog_fd); spinlock_acquire(&router->binlog_lock); strncpy(router->binlog_name, file,BINLOG_FNAMELEN); - blr_file_add_magic(router, fd); - spinlock_release(&router->binlog_lock); router->binlog_fd = fd; + spinlock_release(&router->binlog_lock); return 1; } @@ -254,12 +253,13 @@ int fd; LOGIF(LE, (skygw_log_write(LOGFILE_ERROR, "%s: binlog file %s has an invalid length %d.", router->service->name, path, router->binlog_position))); - close(fd); + close(fd); + spinlock_release(&router->binlog_lock); return; } } - spinlock_release(&router->binlog_lock); router->binlog_fd = fd; + spinlock_release(&router->binlog_lock); } /** From 15e6d6f9fcb53eb0d95bb3a4e6e082815e72f2ec Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Mon, 22 Jun 2015 18:24:39 +0200 Subject: [PATCH 076/100] fix for missing crc check in blr_slave_fake_rotate() fix for missing crc check in blr_slave_fake_rotate() --- server/modules/routing/binlog/blr_slave.c | 27 ++++++++++++++--------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/server/modules/routing/binlog/blr_slave.c b/server/modules/routing/binlog/blr_slave.c index 4de69b8d7..ee7d52099 100644 --- a/server/modules/routing/binlog/blr_slave.c +++ b/server/modules/routing/binlog/blr_slave.c @@ -1693,6 +1693,9 @@ uint32_t chksum; binlognamelen = strlen(slave->binlogfile); len = 19 + 8 + 4 + binlognamelen; + /* no slave crc, remove 4 bytes */ + if (slave->nocrc) + len -= 4; // Build a fake rotate event resp = gwbuf_alloc(len + 5); @@ -1711,17 +1714,19 @@ uint32_t chksum; memcpy(ptr, slave->binlogfile, binlognamelen); ptr += binlognamelen; - /* - * Now add the CRC to the fake binlog rotate event. - * - * The algorithm is first to compute the checksum of an empty buffer - * and then the checksum of the event portion of the message, ie we do not - * include the length, sequence number and ok byte that makes up the first - * 5 bytes of the message. We also do not include the 4 byte checksum itself. - */ - chksum = crc32(0L, NULL, 0); - chksum = crc32(chksum, GWBUF_DATA(resp) + 5, hdr.event_size - 4); - encode_value(ptr, chksum, 32); + if (!slave->nocrc) { + /* + * Now add the CRC to the fake binlog rotate event. + * + * The algorithm is first to compute the checksum of an empty buffer + * and then the checksum of the event portion of the message, ie we do not + * include the length, sequence number and ok byte that makes up the first + * 5 bytes of the message. We also do not include the 4 byte checksum itself. + */ + chksum = crc32(0L, NULL, 0); + chksum = crc32(chksum, GWBUF_DATA(resp) + 5, hdr.event_size - 4); + encode_value(ptr, chksum, 32); + } slave->dcb->func.write(slave->dcb, resp); return 1; From c22c6ea46aaaad1cfba993ccd19a41cbea4abe6e Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 23 Jun 2015 09:15:06 +0300 Subject: [PATCH 077/100] ServiceStop only removed DCBs from the polling system This removes the need to establish new DCBs for each of the listeners while still blocking new session creation for a service which is shut down. The client will not receive an error and the connection will be accepted when the service is restarted. --- server/core/service.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/server/core/service.c b/server/core/service.c index c11c0bddc..cfce442b8 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -534,12 +534,8 @@ int listeners = 0; port = service->ports; while (port) { - if(port->listener) - { - dcb_close(port->listener); - port->listener = NULL; + if(poll_remove_dcb(port->listener) == 0) listeners++; - } port = port->next; } service->state = SERVICE_STATE_STOPPED; @@ -564,7 +560,8 @@ int listeners = 0; port = service->ports; while (port) { - listeners += serviceStartPort(service,port); + if(poll_add_dcb(port->listener) == 0) + listeners++; port = port->next; } service->state = SERVICE_STATE_STARTED; From dc43a7d9da855015b1013980d1227c76c845e015 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 23 Jun 2015 11:25:59 +0300 Subject: [PATCH 078/100] Removed unnecessary code from dcb_close and dcb_set_state_nomutex. --- server/core/dcb.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index fc9329524..7f3651953 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -1949,15 +1949,13 @@ dcb_close(DCB *dcb) } ss_dassert(dcb->state == DCB_STATE_POLLING || - dcb->state == DCB_STATE_LISTENING || dcb->state == DCB_STATE_NOPOLLING || dcb->state == DCB_STATE_ZOMBIE); /*< * Stop dcb's listening and modify state accordingly. */ - if (dcb->state == DCB_STATE_POLLING || - dcb->state == DCB_STATE_LISTENING) + if (dcb->state == DCB_STATE_POLLING) { rc = poll_remove_dcb(dcb); @@ -2430,10 +2428,6 @@ static bool dcb_set_state_nomutex( case DCB_STATE_POLLING: /*< ok to try but state can't change */ succp = true; break; - case DCB_STATE_LISTENING: - dcb->state = new_state; - succp = true; - break; default: ss_dassert(old_state != NULL); break; From 3de7798fac7ffc74d36d50316e1449b3ab333bb2 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 23 Jun 2015 11:49:27 +0300 Subject: [PATCH 079/100] Added missing session state changes. --- server/core/service.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/core/service.c b/server/core/service.c index cfce442b8..bce7328cc 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -535,7 +535,10 @@ int listeners = 0; while (port) { if(poll_remove_dcb(port->listener) == 0) + { + port->listener->session->state = SESSION_STATE_LISTENER_STOPPED; listeners++; + } port = port->next; } service->state = SERVICE_STATE_STOPPED; @@ -561,7 +564,10 @@ int listeners = 0; while (port) { if(poll_add_dcb(port->listener) == 0) + { + port->listener->session->state = SESSION_STATE_LISTENER; listeners++; + } port = port->next; } service->state = SERVICE_STATE_STARTED; From 3d7b050f6ff3df7cc939903a8db202e938168be3 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 23 Jun 2015 11:53:29 +0300 Subject: [PATCH 080/100] Removed unnecessary NULL checks in mysql_client and mysql_common --- server/modules/protocol/mysql_client.c | 2 +- server/modules/protocol/mysql_common.c | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/server/modules/protocol/mysql_client.c b/server/modules/protocol/mysql_client.c index 21a11dc42..c3e463139 100644 --- a/server/modules/protocol/mysql_client.c +++ b/server/modules/protocol/mysql_client.c @@ -1771,7 +1771,7 @@ gw_client_close(DCB *dcb) dcb->state == DCB_STATE_NOPOLLING || dcb->state == DCB_STATE_ZOMBIE) { - if (!DCB_IS_CLONE(dcb) && protocol) CHK_PROTOCOL(protocol); + if (!DCB_IS_CLONE(dcb)) CHK_PROTOCOL(protocol); } #endif LOGIF(LD, (skygw_log_write(LOGFILE_DEBUG, diff --git a/server/modules/protocol/mysql_common.c b/server/modules/protocol/mysql_common.c index eb044c1fb..83e2912e5 100644 --- a/server/modules/protocol/mysql_common.c +++ b/server/modules/protocol/mysql_common.c @@ -128,10 +128,7 @@ void mysql_protocol_done ( MySQLProtocol* p; server_command_t* scmd; server_command_t* scmd2; - - if(dcb->protocol == NULL) - return; - + p = (MySQLProtocol *)dcb->protocol; spinlock_acquire(&p->protocol_lock); From abf39303d7590e908a54de7d3309183ab5efa5d7 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 23 Jun 2015 16:20:26 +0300 Subject: [PATCH 081/100] Fixed the wrong value being returned form dcb_read_SSL. --- server/core/dcb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index 7f3651953..9e94150a8 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -1239,7 +1239,7 @@ int dcb_read_SSL( } /*< while (true) */ return_n: - return n; + return nread; } /** * General purpose routine to write to a DCB From 039cbff1811315606bdec20dee421d9521830fe1 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 23 Jun 2015 16:45:10 +0300 Subject: [PATCH 082/100] Added missing null checks. --- server/core/service.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/core/service.c b/server/core/service.c index bce7328cc..7966d33ec 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -534,7 +534,7 @@ int listeners = 0; port = service->ports; while (port) { - if(poll_remove_dcb(port->listener) == 0) + if(port->listener && poll_remove_dcb(port->listener) == 0) { port->listener->session->state = SESSION_STATE_LISTENER_STOPPED; listeners++; @@ -563,7 +563,7 @@ int listeners = 0; port = service->ports; while (port) { - if(poll_add_dcb(port->listener) == 0) + if(port->listener && poll_add_dcb(port->listener) == 0) { port->listener->session->state = SESSION_STATE_LISTENER; listeners++; From c42d3d9f7ac086b5e012d9c2ee64f3b024c9ed17 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 23 Jun 2015 17:05:31 +0300 Subject: [PATCH 083/100] Added missing NULL checks. --- server/core/dcb.c | 3 +++ server/core/service.c | 16 ++++++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index 7f3651953..07bb1dc93 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -2423,6 +2423,9 @@ static bool dcb_set_state_nomutex( case DCB_STATE_NOPOLLING: switch (new_state) { + /** Stopped services which are restarting will go from + * DCB_STATE_NOPOLLING to DCB_STATE_LISTENING.*/ + case DCB_STATE_LISTENING: case DCB_STATE_ZOMBIE: /*< fall through */ dcb->state = new_state; case DCB_STATE_POLLING: /*< ok to try but state can't change */ diff --git a/server/core/service.c b/server/core/service.c index 7966d33ec..24d51542d 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -534,10 +534,14 @@ int listeners = 0; port = service->ports; while (port) { - if(port->listener && poll_remove_dcb(port->listener) == 0) + if(port->listener && + port->listener->session->state == SESSION_STATE_LISTENER) { - port->listener->session->state = SESSION_STATE_LISTENER_STOPPED; - listeners++; + if(poll_remove_dcb(port->listener) == 0) + { + port->listener->session->state = SESSION_STATE_LISTENER_STOPPED; + listeners++; + } } port = port->next; } @@ -563,11 +567,15 @@ int listeners = 0; port = service->ports; while (port) { - if(port->listener && poll_add_dcb(port->listener) == 0) + if(port->listener && + port->listener->session->state == SESSION_STATE_LISTENER_STOPPED) + { + if(poll_add_dcb(port->listener) == 0) { port->listener->session->state = SESSION_STATE_LISTENER; listeners++; } + } port = port->next; } service->state = SERVICE_STATE_STARTED; From 77c085fbd3be191f2b4679a628d89fb272814b02 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 23 Jun 2015 19:51:19 +0300 Subject: [PATCH 084/100] Improved routing hint documentation. --- Documentation/Reference/Hint-Syntax.md | 83 +++++++++++++++++++++----- 1 file changed, 69 insertions(+), 14 deletions(-) diff --git a/Documentation/Reference/Hint-Syntax.md b/Documentation/Reference/Hint-Syntax.md index 264d264f6..b84bb85fd 100644 --- a/Documentation/Reference/Hint-Syntax.md +++ b/Documentation/Reference/Hint-Syntax.md @@ -1,42 +1,97 @@ # Hint Syntax -Use either ’-- ’ (notice the whitespace) or ’#’ after the semicolon or ’/* .. */’ before -the semicolon. +## Enabling routing hints -The MySQL manual doesn’t specify if comment blocks, i.e. ’/* .. */’, should contain a w -hitespace character before or after the tags. +To enable routing hints for a service, the hintfilter module needs to be configured and the filter needs to be applied to the service. -All hints must start with the ’maxscale tag’: - -- maxscale - -The hints right now have two types, ones that route to a server and others that contain +Here is an example service which has the hint filter configured and applied. + +``` +[Read Service] +type=service +router=readconnroute +router_options=master +servers=server1 +user=maxuser +passwd=maxpwd +filter=Hint + +[Hint] +type=filter +module=hintfilter + +``` + +## Comments and comment types + +The client connection will need to have comments enabled. For example the `mysql` command line client has comments disabled by default. + +For comment types, use either `-- ` (notice the whitespace) or `#` after the semicolon or `/* .. */` before the semicolon. All comment types work with routing hints. + +The MySQL manual doesn`t specify if comment blocks, i.e. `/* .. */`, should contain a w +hitespace character before or after the tags, so adding whitespace at both the start and the end is advised. + +## Hint body + +All hints must start with the `maxscale` tag. + +``` +-- maxscale +``` + +The hints have two types, ones that route to a server and others that contain name-value pairs. -Routing queries to a server: +###Routing destination hints + +These hints will instruct the router to route a query to a certain type of a server. +``` -- maxscale route to [master | slave | server ] +``` -The name of the server is the same as in maxscale.cnf +A `master` value in a routing hint will route the query to a master server. This can be used to direct read queries to a master server for a up-to-date result with no replication lag. A `slave` value will route the query to a slave server. A `server` value will route the query to a named server. The value of needs to be the same as the server section name in maxscale.cnf. -Creating a name-value pair: +### Name-value hints + +These control the behavior and affect the routing decisions made by the router. + +``` -- maxscale = +``` -Currently the only accepted parameter is -’max_slave_replication_lag’ +Currently the only accepted parameter is `max_slave_replication_lag`. This will route the query to a server with lower replication lag then what is defined in the hint value. + +## Hint stack Hints can be either single-use hints, which makes them affect only one query, or named hints, which can be pushed on and off a stack of active hints. Defining named hints: + +``` -- maxscale prepare +``` Pushing a hint onto the stack: + +``` -- maxscale begin +``` Popping the topmost hint off the stack: + +``` -- maxscale end +``` You can define and activate a hint in a single command using the following: + +``` -- maxscale begin +``` You can also push anonymous hints onto the stack which are only used as long as they are on the stack: --- maxscale begin \ No newline at end of file + +``` +-- maxscale begin +``` From 29492426c3e48b30cb0273d0b20ea9315035751b Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 23 Jun 2015 20:06:02 +0300 Subject: [PATCH 085/100] Updated documentation contents. --- Documentation/Documentation-Contents.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Documentation/Documentation-Contents.md b/Documentation/Documentation-Contents.md index 99f8ba38e..fc51d83ec 100644 --- a/Documentation/Documentation-Contents.md +++ b/Documentation/Documentation-Contents.md @@ -5,7 +5,7 @@ ## About MaxScale - [About MaxScale](About/About-MaxScale.md) - - [Release Notes 1.1](Release-Notes/MaxScale-1.1-Release-Notes.md) + - [MaxScale 1.2.0 Release Notes](Release-Notes/MaxScale-1.2.0-Release-Notes.md) - [Changelog](Changelog.md) - [Limitations](About/Limitations.md) - [COPYRIGHT](About/COPYRIGHT.md) @@ -28,6 +28,7 @@ - [MaxScale HA with Lsyncd](Reference/MaxScale-HA-with-lsyncd.md) - [How Errors are Handled in MaxScale](Reference/How-errors-are-handled-in-MaxScale.md) - [Debug and Diagnostic Support](Reference/Debug-And-Diagnostic-Support.md) + - [Routing Hints](Reference/Hint-Syntax.md) ## Tutorials @@ -70,10 +71,6 @@ Here are detailed documents about the filters MaxScale offers. They contain conf - [RabbitMQ Consumer Client](filters/RabbitMQ-Consumer-Client.md) -## Routers - - - [Simple Schema Sharding Router](routers/schemarouter/SchemaRouter.md) - ## Design Documents - [Core Objects Design (in development)](http://mariadb-corporation.github.io/MaxScale/Design-Documents/core-objects-html-docs) @@ -89,4 +86,7 @@ Here are detailed documents about the filters MaxScale offers. They contain conf - [MaxScale 1.0 Release Notes](Release-Notes/MaxScale-1.0-Release-Notes.md) - [MaxScale 1.0.1 Release Notes](Release-Notes/MaxScale-1.0.1-Release-Notes.md) - [MaxScale 1.0.3 Release Notes](Release-Notes/MaxScale-1.0.3-Release-Notes.md) + - [MaxScale 1.1.0 Release Notes](Release-Notes/MaxScale-1.1-Release-Notes.md) + - [MaxScale 1.1.1 Release Notes](Release-Notes/MaxScale-1.1.1-Release-Notes.md) + - [MaxScale 1.2.0 Release Notes](Release-Notes/MaxScale-1.2.0-Release-Notes.md) From f44a3cf758ded11f85bb6fd2ab803e76061e2ac7 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 23 Jun 2015 20:10:43 +0300 Subject: [PATCH 086/100] Updated 1.2 release notes. --- Documentation/Release-Notes/MaxScale-1.2.0-Release-Notes.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Documentation/Release-Notes/MaxScale-1.2.0-Release-Notes.md b/Documentation/Release-Notes/MaxScale-1.2.0-Release-Notes.md index f85382b3c..a84f94c27 100644 --- a/Documentation/Release-Notes/MaxScale-1.2.0-Release-Notes.md +++ b/Documentation/Release-Notes/MaxScale-1.2.0-Release-Notes.md @@ -24,3 +24,8 @@ A quick list of changes in installation directories and file names: * Data directory is `/var/lib/maxscale`. This is the default location for MaxScale-specific data. * PID file can be found at `/var/run/maxscale` +### Client side SSL encryption +MaxScale now supports SSL/TLS encrypted connections to MaxScale. + +### Monitor scripts +State changes in backend servers can now trigger the execution of a custom script. With this you can easily customize MaxScale's behavior. From 13fb88ea076c80da79182ec2ee5d9123ae6cd075 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Wed, 24 Jun 2015 11:29:43 +0300 Subject: [PATCH 087/100] Added optional code for older OpenSSL library versions. --- CMakeLists.txt | 14 +++++++++++++- server/core/gateway.c | 14 +++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3b040f0df..c28b2c657 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,19 +31,31 @@ include(cmake/CheckPlatform.cmake) check_deps() check_dirs() +find_package(OpenSSL) find_package(Valgrind) find_package(MySQLClient) find_package(MySQL) find_package(Pandoc) find_package(TCMalloc) find_package(Jemalloc) +find_package(CURL) # You can find the variables set by this in the FindCURL.cmake file # which is a default module in CMake. -find_package(CURL) + if(NOT CURL_FOUND) message(FATAL_ERROR "Failed to locate dependency: libcurl") endif() +if(NOT OPENSSL_FOUND) + message(FATAL_ERROR "Failed to locate dependency: OpenSSL") +else() + if(OPENSSL_VERSION VERSION_LESS 1 AND NOT FORCE_OPENSSL100) + add_definitions("-DOPENSSL_0_9") + else() + add_definitions("-DOPENSSL_1_0") + endif() +endif() + set(CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_RPATH}:${CMAKE_INSTALL_PREFIX}/${MAXSCALE_LIBDIR}) # Make sure the release notes for this release are present if it is a stable one diff --git a/server/core/gateway.c b/server/core/gateway.c index f206ca30e..5997a9eeb 100644 --- a/server/core/gateway.c +++ b/server/core/gateway.c @@ -263,6 +263,15 @@ static void ssl_free_dynlock(struct CRYPTO_dynlock_value * n,const char* file, i free(n); } +/** + * The thread ID callback function for OpenSSL dynamic locks. + * @param id Id to modify + */ +static void maxscale_ssl_id(CRYPTO_THREADID* id) +{ + CRYPTO_THREADID_set_numeric(id,pthread_self()); +} + /** * Handler for SIGHUP signal. Reload the configuration for the * gateway. @@ -1459,8 +1468,11 @@ int main(int argc, char **argv) CRYPTO_set_dynlock_create_callback(ssl_create_dynlock); CRYPTO_set_dynlock_destroy_callback(ssl_free_dynlock); CRYPTO_set_dynlock_lock_callback(ssl_lock_dynlock); +#ifdef OPENSSL_1_0 + CRYPTO_THREADID_set_callback(maxscale_ssl_id); +#else CRYPTO_set_id_callback(pthread_self); - +#endif /* register exit function for embedded MySQL library */ l = atexit(libmysqld_done); From 0f199d924f937c81f057285b07ca2871d7df5aa9 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Wed, 24 Jun 2015 12:56:22 +0300 Subject: [PATCH 088/100] Removed unnecessary call to SSL_get_error from dcb_accept_SSL. --- server/core/dcb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index 4bd51d9a2..3597d0d3a 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -3010,7 +3010,7 @@ int dcb_accept_SSL(DCB* dcb) do { ssl_rval = SSL_accept(dcb->ssl); - errnum = SSL_get_error(dcb->ssl,ssl_rval); + LOGIF(LD,(skygw_log_write_flush(LD,"[dcb_accept_SSL] SSL_accept %d, error %d", ssl_rval,errnum))); switch(ssl_rval) From 067a62b24032e17dc1bae46bfba9eb0f95471496 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Wed, 24 Jun 2015 14:34:26 +0300 Subject: [PATCH 089/100] Added more error logging to dcb_write_SSL. --- server/core/dcb.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index 3597d0d3a..e63a73ffc 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -1645,7 +1645,7 @@ dcb_write_SSL(DCB *dcb, GWBUF *queue) { char errbuf[140]; ERR_error_string(ssl_errno,errbuf); - skygw_log_write(LE,"%s",errbuf); + skygw_log_write(LD,"%s",errbuf); } } break; @@ -1665,6 +1665,19 @@ dcb_write_SSL(DCB *dcb, GWBUF *queue) STRDCBSTATE(dcb->state), dcb->fd, ssl_errno))); + if(ssl_errno == SSL_ERROR_SSL) + { + while((ssl_errno = ERR_get_error()) != 0) + { + char errbuf[140]; + ERR_error_string(ssl_errno,errbuf); + skygw_log_write(LE,"%s",errbuf); + } + } + if(ssl_errno == SSL_ERROR_SYSCALL) + { + skygw_log_write(LE,"%d:%s",errno,strerror(errno)); + } } } break; From 484781a463913f242bba6d370f3b43d734132007 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Wed, 24 Jun 2015 14:46:46 +0300 Subject: [PATCH 090/100] More error logging for SSL connections. --- server/core/dcb.c | 47 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index e63a73ffc..4e2ee4800 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -1659,7 +1659,7 @@ dcb_write_SSL(DCB *dcb, GWBUF *queue) LOGIF(LE, (skygw_log_write_flush( LOGFILE_ERROR, "Error : Write to dcb %p in " - "state %s fd %d failed due " + "state %s fd %d failed due to " "SSL error %d", dcb, STRDCBSTATE(dcb->state), @@ -1881,28 +1881,47 @@ dcb_drain_writeq_SSL(DCB *dcb) while (dcb->writeq != NULL) { len = GWBUF_LENGTH(dcb->writeq); - w = gw_write_SSL(dcb->ssl, GWBUF_DATA(dcb->writeq), len); + w = gw_write_SSL(dcb->ssl, GWBUF_DATA(dcb->writeq), len); if (w < 0) { - int ssl_errno = ERR_get_error(); + int ssl_errno = SSL_get_error(dcb->ssl,w); - if(ssl_errno == SSL_ERROR_WANT_WRITE || - ssl_errno == SSL_ERROR_WANT_ACCEPT || - ssl_errno == SSL_ERROR_WANT_READ) + if(ssl_errno == SSL_ERROR_WANT_WRITE || ssl_errno == SSL_ERROR_WANT_READ) { break; } + skygw_log_write_flush(LOGFILE_ERROR, + "Error : Write to dcb failed due to " + "SSL error %d:", + dcb, + STRDCBSTATE(dcb->state), + dcb->fd, + ssl_errno); + switch(ssl_errno) + { + case SSL_ERROR_SSL: + case SSL_ERROR_SYSCALL: + while((ssl_errno = ERR_get_error()) != 0) + { + char errbuf[140]; + ERR_error_string(ssl_errno,errbuf); + skygw_log_write(LE,"%s",errbuf); + } + if(errno != 0) + skygw_log_write(LE,"%d:%s",errno,strerror(errno)); + break; + case SSL_ERROR_ZERO_RETURN: + skygw_log_write(LE,"Socket is closed."); + break; - skygw_log_write_flush( - LOGFILE_ERROR, - "Error : Write to dcb %p " - "in state %s fd %d failed: %s", - dcb, - STRDCBSTATE(dcb->state), - dcb->fd, - ERR_error_string(ssl_errno,NULL)); + default: + skygw_log_write(LE,"Unexpected error."); + break; + } break; + + } /* * Pull the number of bytes we have written from From 80d130ef0c6f9d65407519920d79dd6e5bbcaf14 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Wed, 24 Jun 2015 15:05:04 +0300 Subject: [PATCH 091/100] Fixed dcb_write_SSL being called multiple times on failure. --- server/core/dcb.c | 182 +++++++++++++++++++++++++--------------------- 1 file changed, 101 insertions(+), 81 deletions(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index 4e2ee4800..9142f7ee9 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -1594,108 +1594,128 @@ dcb_write_SSL(DCB *dcb, GWBUF *queue) } #endif /* FAKE_CODE */ qlen = GWBUF_LENGTH(queue); - - w = gw_write_SSL(dcb->ssl, GWBUF_DATA(queue), qlen); - dcb->stats.n_writes++; - - if (w < 0) + do { - int ssl_errno = SSL_get_error(dcb->ssl,w); + w = gw_write_SSL(dcb->ssl, GWBUF_DATA(queue), qlen); + dcb->stats.n_writes++; - if (LOG_IS_ENABLED(LOGFILE_DEBUG)) + if (w <= 0) { - switch(ssl_errno) + int ssl_errno = SSL_get_error(dcb->ssl,w); + + if (LOG_IS_ENABLED(LOGFILE_DEBUG)) { - case SSL_ERROR_WANT_READ: - LOGIF(LD, (skygw_log_write( - LOGFILE_DEBUG, - "%lu [dcb_write] Write to dcb " - "%p in state %s fd %d failed " - "due error SSL_ERROR_WANT_READ", - pthread_self(), - dcb, - STRDCBSTATE(dcb->state), - dcb->fd))); - break; - case SSL_ERROR_WANT_WRITE: - LOGIF(LD, (skygw_log_write( - LOGFILE_DEBUG, - "%lu [dcb_write] Write to dcb " - "%p in state %s fd %d failed " - "due error SSL_ERROR_WANT_WRITE", - pthread_self(), - dcb, - STRDCBSTATE(dcb->state), - dcb->fd))); - break; - default: - LOGIF(LD, (skygw_log_write( - LOGFILE_DEBUG, - "%lu [dcb_write] Write to dcb " - "%p in state %s fd %d failed " - "due error %d", - pthread_self(), - dcb, - STRDCBSTATE(dcb->state), - dcb->fd,ssl_errno))); - if(ssl_errno == SSL_ERROR_SSL || - ssl_errno == SSL_ERROR_SYSCALL) + switch(ssl_errno) { - while((ssl_errno = ERR_get_error()) != 0) + case SSL_ERROR_WANT_READ: + LOGIF(LD, (skygw_log_write( + LOGFILE_DEBUG, + "%lu [dcb_write] Write to dcb " + "%p in state %s fd %d failed " + "due error SSL_ERROR_WANT_READ", + pthread_self(), + dcb, + STRDCBSTATE(dcb->state), + dcb->fd))); + break; + case SSL_ERROR_WANT_WRITE: + LOGIF(LD, (skygw_log_write( + LOGFILE_DEBUG, + "%lu [dcb_write] Write to dcb " + "%p in state %s fd %d failed " + "due error SSL_ERROR_WANT_WRITE", + pthread_self(), + dcb, + STRDCBSTATE(dcb->state), + dcb->fd))); + break; + default: + LOGIF(LD, (skygw_log_write( + LOGFILE_DEBUG, + "%lu [dcb_write] Write to dcb " + "%p in state %s fd %d failed " + "due error %d", + pthread_self(), + dcb, + STRDCBSTATE(dcb->state), + dcb->fd,ssl_errno))); + if(ssl_errno == SSL_ERROR_SSL || + ssl_errno == SSL_ERROR_SYSCALL) { - char errbuf[140]; - ERR_error_string(ssl_errno,errbuf); - skygw_log_write(LD,"%s",errbuf); + while((ssl_errno = ERR_get_error()) != 0) + { + char errbuf[140]; + ERR_error_string(ssl_errno,errbuf); + skygw_log_write(LD,"%s",errbuf); + } } - } - break; - } - } - - if (LOG_IS_ENABLED(LOGFILE_ERROR)) - { - if (ssl_errno != 0) - { - LOGIF(LE, (skygw_log_write_flush( - LOGFILE_ERROR, - "Error : Write to dcb %p in " - "state %s fd %d failed due to " - "SSL error %d", - dcb, - STRDCBSTATE(dcb->state), - dcb->fd, - ssl_errno))); - if(ssl_errno == SSL_ERROR_SSL) - { - while((ssl_errno = ERR_get_error()) != 0) - { - char errbuf[140]; - ERR_error_string(ssl_errno,errbuf); - skygw_log_write(LE,"%s",errbuf); - } - } - if(ssl_errno == SSL_ERROR_SYSCALL) - { - skygw_log_write(LE,"%d:%s",errno,strerror(errno)); + break; } } + + if (LOG_IS_ENABLED(LOGFILE_ERROR) && ssl_errno != SSL_ERROR_WANT_WRITE) + { + if (ssl_errno == -1) + { + LOGIF(LE, (skygw_log_write_flush( + LOGFILE_ERROR, + "Error : Write to dcb %p in " + "state %s fd %d failed due to " + "SSL error %d", + dcb, + STRDCBSTATE(dcb->state), + dcb->fd, + ssl_errno))); + if(ssl_errno == SSL_ERROR_SSL) + { + do + { + char errbuf[140]; + ERR_error_string(ssl_errno,errbuf); + skygw_log_write(LE,"%s",errbuf); + }while((ssl_errno = ERR_get_error()) != 0); + } + if(ssl_errno == SSL_ERROR_SYSCALL) + { + skygw_log_write(LE,"%d:%s",errno,strerror(errno)); + } + } + else if(w == 0) + { + do + { + char errbuf[140]; + ERR_error_string(ssl_errno,errbuf); + skygw_log_write(LE,"%s",errbuf); + }while((ssl_errno = ERR_get_error()) != 0); + } + } + + if(ssl_errno != SSL_ERROR_WANT_WRITE) + break; } - break; - } + }while(w <= 0); /* * Pull the number of bytes we have written from * queue with have. */ - queue = gwbuf_consume(queue, w); - LOGIF(LD, (skygw_log_write( + if(w == -1) + { + while((queue = GWBUF_CONSUME_ALL(queue))); + } + else + { + queue = gwbuf_consume(queue, w); + LOGIF(LD, (skygw_log_write( LOGFILE_DEBUG, "%lu [dcb_write] Wrote %d Bytes to dcb %p in " - "state %s fd %d", + "state %s fd %d", pthread_self(), w, dcb, STRDCBSTATE(dcb->state), dcb->fd))); + } } /*< while (queue != NULL) */ /*< * What wasn't successfully written is stored to write queue From 047985fb91a21c94fc6446ba1e26de33ae8aa74c Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Wed, 24 Jun 2015 15:26:35 +0300 Subject: [PATCH 092/100] Fixed SSL thread locking functions not being used. --- server/core/gateway.c | 40 ++++++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/server/core/gateway.c b/server/core/gateway.c index 5997a9eeb..1b653d2f7 100644 --- a/server/core/gateway.c +++ b/server/core/gateway.c @@ -208,6 +208,15 @@ static int set_user(); /** SSL multi-threading functions and structures */ +static SPINLOCK* ssl_locks; + +static void ssl_locking_function(int mode,int n,const char* file, int line) +{ + if(mode & CRYPTO_LOCK) + spinlock_acquire(&ssl_locks[n]); + else + spinlock_release(&ssl_locks[n]); +} /** * OpenSSL requires this struct to be defined in order to use dynamic locks */ @@ -1465,14 +1474,19 @@ int main(int argc, char **argv) SSL_library_init(); SSL_load_error_strings(); OPENSSL_add_all_algorithms_noconf(); - CRYPTO_set_dynlock_create_callback(ssl_create_dynlock); - CRYPTO_set_dynlock_destroy_callback(ssl_free_dynlock); - CRYPTO_set_dynlock_lock_callback(ssl_lock_dynlock); -#ifdef OPENSSL_1_0 - CRYPTO_THREADID_set_callback(maxscale_ssl_id); -#else - CRYPTO_set_id_callback(pthread_self); -#endif + + int numlocks = CRYPTO_num_locks(); + if((ssl_locks = malloc(sizeof(SPINLOCK)*(numlocks + 1))) == NULL) + { + char* logerr = "Memory allocation failed"; + print_log_n_stderr(true, true, logerr, logerr, eno); + rc = MAXSCALE_INTERNALERROR; + goto return_main; + } + + for(i = 0;i Date: Wed, 24 Jun 2015 16:56:27 +0300 Subject: [PATCH 093/100] Moved SSL spinlock initialization to be done after thread initialization. --- server/core/gateway.c | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/server/core/gateway.c b/server/core/gateway.c index 1b653d2f7..6513bdb1f 100644 --- a/server/core/gateway.c +++ b/server/core/gateway.c @@ -1475,18 +1475,6 @@ int main(int argc, char **argv) SSL_load_error_strings(); OPENSSL_add_all_algorithms_noconf(); - int numlocks = CRYPTO_num_locks(); - if((ssl_locks = malloc(sizeof(SPINLOCK)*(numlocks + 1))) == NULL) - { - char* logerr = "Memory allocation failed"; - print_log_n_stderr(true, true, logerr, logerr, eno); - rc = MAXSCALE_INTERNALERROR; - goto return_main; - } - - for(i = 0;i Date: Wed, 24 Jun 2015 18:17:00 +0300 Subject: [PATCH 094/100] Updated readwritesplit documentation. --- Documentation/routers/ReadWriteSplit.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Documentation/routers/ReadWriteSplit.md b/Documentation/routers/ReadWriteSplit.md index 6058e0390..577bc2e36 100644 --- a/Documentation/routers/ReadWriteSplit.md +++ b/Documentation/routers/ReadWriteSplit.md @@ -4,7 +4,9 @@ This document provides a short overview of the **readwritesplit** router module ## Overview -The **readwritesplit** router is designed to be used with a Master-Slave replication cluster. It automatically detects changes in the master server and will use the current master server of the cluster. With a Galera cluster, one can achieve a resilient setup and easy master failover by using one of the Galera nodes as a Write-Master node, where are write queries are routed, and spreading the read load over all the nodes. +The **readwritesplit** router is designed to increase the read-only processing capability of a cluster while maintaining consistency. This is achieved by splitting the query load into read and write queries. Read queries, which do not modify data, are spread across multiple nodes while all write queries will be sent to a single node. + +The router is designed to be used with a traditional Master-Slave replication cluster. It automatically detects changes in the master server and will use the current master server of the cluster. With a Galera cluster, one can achieve a resilient setup and easy master failover by using one of the Galera nodes as a Write-Master node, where all write queries are routed, and spreading the read load over all the nodes. ## Configuration From e5d9abbdcbdb9a31bc18c1e5ea5fb64b3fe8c75f Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 25 Jun 2015 06:01:33 +0300 Subject: [PATCH 095/100] Fixes to Coverity defects. --- server/core/dcb.c | 82 +++++++----------------------- server/core/secrets.c | 6 +++ server/core/service.c | 2 +- server/modules/monitor/galeramon.c | 3 ++ utils/skygw_utils.cc | 2 + 5 files changed, 29 insertions(+), 66 deletions(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index 9142f7ee9..8f2f159d4 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -939,10 +939,10 @@ int dcb_read_n( goto return_n; } - if (b == 0 && nread == 0) + if (b == 0) { /** Handle closed client socket */ - if (dcb_isclient(dcb)) + if (nread == 0 && dcb_isclient(dcb)) { char c; int l_errno = 0; @@ -964,11 +964,6 @@ int dcb_read_n( n = 0; goto return_n; } - else if (b == 0) - { - n = 0; - goto return_n; - } dcb->last_read = hkheartbeat; @@ -1208,6 +1203,10 @@ int dcb_read_SSL( } gwbuf_rtrim(buffer,bufsize - n); + if(buffer == NULL) + { + goto return_n; + } #ifdef SS_DEBUG skygw_log_write(LD,"%lu SSL: Truncated buffer from %d to %d bytes. " "Read %d bytes, %d bytes waiting.\n",pthread_self(), @@ -1234,9 +1233,6 @@ int dcb_read_SSL( /*< Append read data to the gwbuf */ *head = gwbuf_append(*head, buffer); - rc = ioctl(dcb->fd, FIONREAD, &b); - pending = SSL_pending(dcb->ssl); - } /*< while (true) */ return_n: return nread; @@ -1639,16 +1635,6 @@ dcb_write_SSL(DCB *dcb, GWBUF *queue) dcb, STRDCBSTATE(dcb->state), dcb->fd,ssl_errno))); - if(ssl_errno == SSL_ERROR_SSL || - ssl_errno == SSL_ERROR_SYSCALL) - { - while((ssl_errno = ERR_get_error()) != 0) - { - char errbuf[140]; - ERR_error_string(ssl_errno,errbuf); - skygw_log_write(LD,"%s",errbuf); - } - } break; } } @@ -1666,19 +1652,19 @@ dcb_write_SSL(DCB *dcb, GWBUF *queue) STRDCBSTATE(dcb->state), dcb->fd, ssl_errno))); - if(ssl_errno == SSL_ERROR_SSL) + if(ssl_errno == SSL_ERROR_SSL || ssl_errno == SSL_ERROR_SYSCALL) { + if(ssl_errno == SSL_ERROR_SYSCALL) + { + skygw_log_write(LE,"%d:%s",errno,strerror(errno)); + } do { char errbuf[140]; ERR_error_string(ssl_errno,errbuf); - skygw_log_write(LE,"%s",errbuf); + skygw_log_write(LE,"%d:%s",ssl_errno,errbuf); }while((ssl_errno = ERR_get_error()) != 0); } - if(ssl_errno == SSL_ERROR_SYSCALL) - { - skygw_log_write(LE,"%d:%s",errno,strerror(errno)); - } } else if(w == 0) { @@ -1686,7 +1672,7 @@ dcb_write_SSL(DCB *dcb, GWBUF *queue) { char errbuf[140]; ERR_error_string(ssl_errno,errbuf); - skygw_log_write(LE,"%s",errbuf); + skygw_log_write(LE,"%d:%s",ssl_errno,errbuf); }while((ssl_errno = ERR_get_error()) != 0); } } @@ -1695,16 +1681,14 @@ dcb_write_SSL(DCB *dcb, GWBUF *queue) break; } }while(w <= 0); - /* - * Pull the number of bytes we have written from - * queue with have. - */ - if(w == -1) + + if(w <= 0) { - while((queue = GWBUF_CONSUME_ALL(queue))); + break; } else { + /** Remove written bytes from the queue */ queue = gwbuf_consume(queue, w); LOGIF(LD, (skygw_log_write( LOGFILE_DEBUG, @@ -1733,38 +1717,6 @@ dcb_write_SSL(DCB *dcb, GWBUF *queue) } } /* if (dcb->writeq) */ - if (saved_errno != 0 && - queue != NULL && - saved_errno != EAGAIN && - saved_errno != EWOULDBLOCK) - { - bool dolog = true; - - /** - * Do not log if writing COM_QUIT to backend failed. - */ - if (GWBUF_IS_TYPE_MYSQL(queue)) - { - uint8_t* data = GWBUF_DATA(queue); - - if (data[4] == 0x01) - { - dolog = false; - } - } - if (dolog) - { - LOGIF(LD, (skygw_log_write( - LOGFILE_DEBUG, - "%lu [dcb_write] Writing to %s socket failed due %d, %s.", - pthread_self(), - dcb_isclient(dcb) ? "client" : "backend server", - saved_errno, - strerror(saved_errno)))); - } - spinlock_release(&dcb->writeqlock); - return 0; - } spinlock_release(&dcb->writeqlock); if (dcb->high_water && dcb->writeqlen > dcb->high_water && below_water) diff --git a/server/core/secrets.c b/server/core/secrets.c index 97cb49806..b0e0c5542 100644 --- a/server/core/secrets.c +++ b/server/core/secrets.c @@ -228,6 +228,12 @@ unsigned int randval; MAXKEYS key; char secret_file[PATH_MAX + 10]; +if(strlen(path) > PATH_MAX) +{ + skygw_log_write(LOGFILE_ERROR,"Error: Pathname too long."); + return 1; +} + sprintf(secret_file,"%s/.secrets",path); /* Open for writing | Create | Truncate the file for writing */ diff --git a/server/core/service.c b/server/core/service.c index 24d51542d..be05932ae 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -268,7 +268,7 @@ GWPROTOCOL *funcs; /* Save authentication data to file cache */ char *ptr, path[PATH_MAX + 1]; int mkdir_rval = 0; - strcpy(path, get_cachedir()); + strncpy(path, get_cachedir(),PATH_MAX); strncat(path, "/", 4096); strncat(path, service->name, PATH_MAX); if (access(path, R_OK) == -1) diff --git a/server/modules/monitor/galeramon.c b/server/modules/monitor/galeramon.c index 07d9fd848..06a6e6de8 100644 --- a/server/modules/monitor/galeramon.c +++ b/server/modules/monitor/galeramon.c @@ -155,7 +155,10 @@ startMonitor(void *arg,void* opt) else if(!strcmp(params->name,"script")) { if(handle->script) + { free(handle->script); + handle->script = NULL; + } if(access(params->value,X_OK) == 0) { diff --git a/utils/skygw_utils.cc b/utils/skygw_utils.cc index 4e3f84dca..bce8c631a 100644 --- a/utils/skygw_utils.cc +++ b/utils/skygw_utils.cc @@ -396,6 +396,7 @@ mlist_cursor_t* mlist_cursor_init( if (c == NULL) { goto return_cursor; + simple_mutex_unlock(&list->mlist_mutex); } c->mlcursor_chk_top = CHK_NUM_MLIST_CURSOR; c->mlcursor_chk_tail = CHK_NUM_MLIST_CURSOR; @@ -581,6 +582,7 @@ bool mlist_cursor_move_to_first( simple_mutex_lock(&list->mlist_mutex, true); if (mc->mlcursor_list->mlist_deleted) { + simple_mutex_unlock(&list->mlist_mutex); return false; } /** Set position point to first node */ From 61bee570d17404ff84735068fff3ec664d59ca41 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 25 Jun 2015 11:56:27 +0300 Subject: [PATCH 096/100] Fixed build failures due to old OpenSSL libraries. --- server/core/gateway.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/core/gateway.c b/server/core/gateway.c index 6513bdb1f..c474083ee 100644 --- a/server/core/gateway.c +++ b/server/core/gateway.c @@ -272,6 +272,7 @@ static void ssl_free_dynlock(struct CRYPTO_dynlock_value * n,const char* file, i free(n); } +#ifdef OPENSSL_1_0 /** * The thread ID callback function for OpenSSL dynamic locks. * @param id Id to modify @@ -280,6 +281,7 @@ static void maxscale_ssl_id(CRYPTO_THREADID* id) { CRYPTO_THREADID_set_numeric(id,pthread_self()); } +#endif /** * Handler for SIGHUP signal. Reload the configuration for the From 80709ce039c3b07ec7cbc5461648d071be55f277 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 25 Jun 2015 16:43:50 +0300 Subject: [PATCH 097/100] Fixed compile errors for older SSL libraries. --- server/core/service.c | 3 ++- server/include/service.h | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/server/core/service.c b/server/core/service.c index be05932ae..9822ac881 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -1966,13 +1966,14 @@ int serviceInitSSL(SERVICE* service) case SERVICE_TLS10: service->method = (SSL_METHOD*)TLSv1_server_method(); break; +#ifdef OPENSSL_1_0 case SERVICE_TLS11: service->method = (SSL_METHOD*)TLSv1_1_server_method(); break; case SERVICE_TLS12: service->method = (SSL_METHOD*)TLSv1_2_server_method(); break; - +#endif /** Rest of these use the maximum available SSL/TLS methods */ case SERVICE_SSL_MAX: service->method = (SSL_METHOD*)SSLv23_server_method(); diff --git a/server/include/service.h b/server/include/service.h index 3337ebfc0..b64270b9e 100644 --- a/server/include/service.h +++ b/server/include/service.h @@ -117,8 +117,10 @@ typedef enum { enum{ SERVICE_SSLV3, SERVICE_TLS10, +#ifdef OPENSSL_1_0 SERVICE_TLS11, SERVICE_TLS12, +#endif SERVICE_SSL_MAX, SERVICE_TLS_MAX, SERVICE_SSL_TLS_MAX From cff01cad055a29a6116ea5d2de106a2aaa6eb38e Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 25 Jun 2015 17:13:10 +0300 Subject: [PATCH 098/100] Fixed compile errors with old SSL library. --- server/core/service.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/core/service.c b/server/core/service.c index 9822ac881..3dee01c91 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -923,10 +923,12 @@ serviceSetSSLVersion(SERVICE *service, char* version) service->ssl_method_type = SERVICE_SSLV3; else if(strcasecmp(version,"TLSV10") == 0) service->ssl_method_type = SERVICE_TLS10; +#ifdef OPENSSL_1_0 else if(strcasecmp(version,"TLSV11") == 0) service->ssl_method_type = SERVICE_TLS11; else if(strcasecmp(version,"TLSV12") == 0) service->ssl_method_type = SERVICE_TLS12; +#endif else if(strcasecmp(version,"MAX") == 0) service->ssl_method_type = SERVICE_SSL_TLS_MAX; else return -1; From 22053cfafbd3c933617b49505d3d5d5130768e09 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 26 Jun 2015 10:36:27 +0300 Subject: [PATCH 099/100] Updated MaxScale version to 1.2.0. --- cmake/macros.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/macros.cmake b/cmake/macros.cmake index c287343fb..b78502cf9 100644 --- a/cmake/macros.cmake +++ b/cmake/macros.cmake @@ -9,13 +9,13 @@ macro(set_maxscale_version) # MaxScale version number set(MAXSCALE_VERSION_MAJOR "1") - set(MAXSCALE_VERSION_MINOR "1") - set(MAXSCALE_VERSION_PATCH "1") + set(MAXSCALE_VERSION_MINOR "2") + set(MAXSCALE_VERSION_PATCH "0") set(MAXSCALE_VERSION_NUMERIC "${MAXSCALE_VERSION_MAJOR}.${MAXSCALE_VERSION_MINOR}.${MAXSCALE_VERSION_PATCH}") set(MAXSCALE_VERSION "${MAXSCALE_VERSION_MAJOR}.${MAXSCALE_VERSION_MINOR}.${MAXSCALE_VERSION_PATCH}") # This should be incremented each time a package is rebuilt - set(MAXSCALE_BUILD_NUMBER 2) + set(MAXSCALE_BUILD_NUMBER 1) endmacro() macro(set_variables) From fc1615c48931b7aab5fd255fcdae5f7e55f73eed Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 26 Jun 2015 13:35:14 +0300 Subject: [PATCH 100/100] Fixed the test header configuration failing. --- server/test/maxscale_test.h.in | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/test/maxscale_test.h.in b/server/test/maxscale_test.h.in index d41ff181f..b5448295c 100644 --- a/server/test/maxscale_test.h.in +++ b/server/test/maxscale_test.h.in @@ -1,9 +1,9 @@ #ifndef MAXSCALE_TEST_H #define MAXSCALE_TEST_H -#define TEST_DIR "${CMAKE_BINARY_DIR}" -#define TEST_LOG_DIR "${CMAKE_BINARY_DIR}/log" -#define TEST_BIN_DIR "${CMAKE_BINARY_DIR}/bin" -#define TEST_MOD_DIR "${CMAKE_BINARY_DIR}/modules" -#define TEST_LIB_DIR "${CMAKE_BINARY_DIR}/lib" -#define TEST_ETC_DIR "${CMAKE_BINARY_DIR}/etc" +#define TEST_DIR "@CMAKE_BINARY_DIR@" +#define TEST_LOG_DIR "@CMAKE_BINARY_DIR@/log" +#define TEST_BIN_DIR "@CMAKE_BINARY_DIR@/bin" +#define TEST_MOD_DIR "@CMAKE_BINARY_DIR@/modules" +#define TEST_LIB_DIR "@CMAKE_BINARY_DIR@/lib" +#define TEST_ETC_DIR "@CMAKE_BINARY_DIR@/etc" #endif