From 571919264fb74c6a4181c1d6f863d70335643abd Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 13 Oct 2016 23:10:59 +0300 Subject: [PATCH 1/6] MXS-862: Make packet sequence handling automatic Made the packet sequence number handling automatic so that it always uses the correct one. All functions now have documentation in them. Cleaned up code and added comments to GSSAPI code. --- include/maxscale/protocol/mysql.h | 17 ++- server/modules/authenticator/gssapi_auth.c | 16 ++- server/modules/authenticator/gssapi_auth.h | 7 +- .../authenticator/gssapi_auth_common.c | 1 + .../authenticator/gssapi_backend_auth.c | 115 ++++++++++++++---- .../protocol/MySQL/MySQLClient/mysql_client.c | 3 + 6 files changed, 122 insertions(+), 37 deletions(-) diff --git a/include/maxscale/protocol/mysql.h b/include/maxscale/protocol/mysql.h index 568e520c1..7a1c0af06 100644 --- a/include/maxscale/protocol/mysql.h +++ b/include/maxscale/protocol/mysql.h @@ -75,9 +75,15 @@ MXS_BEGIN_DECLS #define MYSQL_HEADER_LEN 4L #define MYSQL_CHECKSUM_LEN 4L -/** Offsets to various parts of the client packet */ -#define MYSQL_SEQ_OFFSET 3 -#define MYSQL_COM_OFFSET 4 +/** + * Offsets and sizes of various parts of the client packet. If the offset is + * defined but not the size, the size of the value is one byte. + */ +#define MYSQL_SEQ_OFFSET 3 +#define MYSQL_COM_OFFSET 4 +#define MYSQL_CHARSET_OFFSET 12 +#define MYSQL_CLIENT_CAP_OFFSET 4 +#define MYSQL_CLIENT_CAP_SIZE 4 #define GW_MYSQL_PROTOCOL_VERSION 10 // version is 10 #define GW_MYSQL_HANDSHAKE_FILLER 0x00 @@ -279,8 +285,9 @@ typedef struct } MySQLProtocol; /** Defines for response codes */ -#define MYSQL_REPLY_ERR 0xff -#define MYSQL_REPLY_OK 0x00 +#define MYSQL_REPLY_ERR 0xff +#define MYSQL_REPLY_OK 0x00 +#define MYSQL_REPLY_AUTHSWITCHREQUEST 0xfe /* * Let's try this with proper enums instead of numbers diff --git a/server/modules/authenticator/gssapi_auth.c b/server/modules/authenticator/gssapi_auth.c index 5f57d642a..e7264ff56 100644 --- a/server/modules/authenticator/gssapi_auth.c +++ b/server/modules/authenticator/gssapi_auth.c @@ -82,7 +82,7 @@ void* gssapi_auth_init(char **options) * @see https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchRequest * @see https://web.mit.edu/kerberos/krb5-1.5/krb5-1.5.4/doc/krb5-user/What-is-a-Kerberos-Principal_003f.html */ -static GWBUF* create_auth_change_packet(GSSAPI_INSTANCE *instance) +static GWBUF* create_auth_change_packet(GSSAPI_INSTANCE *instance, gssapi_auth_t *auth) { size_t principal_name_len = strlen(instance->principal_name); size_t plen = sizeof(auth_plugin_name) + 1 + principal_name_len; @@ -93,7 +93,7 @@ static GWBUF* create_auth_change_packet(GSSAPI_INSTANCE *instance) uint8_t *data = (uint8_t*)GWBUF_DATA(buffer); gw_mysql_set_byte3(data, plen); data += 3; - *data++ = 0x02; // Second packet + *data++ = ++auth->sequence; // Second packet *data++ = 0xfe; // AuthSwitchRequest command memcpy(data, auth_plugin_name, sizeof(auth_plugin_name)); // Plugin name data += sizeof(auth_plugin_name); @@ -143,10 +143,14 @@ static void copy_client_information(DCB *dcb, GWBUF *buffer) { size_t buflen = gwbuf_length(buffer); MySQLProtocol *protocol = (MySQLProtocol*)dcb->protocol; - /* Take data from fixed locations first */ - gwbuf_copy_data(buffer, 4, 4, (uint8_t*)&protocol->client_capabilities); + gssapi_auth_t *auth = (gssapi_auth_t*)dcb->authenticator_data; + + /* Store the connection characteristics and sequence number of the current packet */ protocol->charset = 0; - gwbuf_copy_data(buffer, 4 + 4 + 4, 1, (uint8_t*)&protocol->charset); + gwbuf_copy_data(buffer, MYSQL_CHARSET_OFFSET, 1, (uint8_t*)&protocol->charset); + gwbuf_copy_data(buffer, MYSQL_CLIENT_CAP_OFFSET, MYSQL_CLIENT_CAP_SIZE, + (uint8_t*)&protocol->client_capabilities); + gwbuf_copy_data(buffer, MYSQL_SEQ_OFFSET, 1, &auth->sequence); if (buflen > MYSQL_AUTH_PACKET_BASE_SIZE) { @@ -292,7 +296,7 @@ int gssapi_auth_authenticate(DCB *dcb) /** We need to send the authentication switch packet to change the * authentication to something other than the 'mysql_native_password' * method */ - GWBUF *buffer = create_auth_change_packet(instance); + GWBUF *buffer = create_auth_change_packet(instance, auth); if (buffer && dcb->func.write(dcb, buffer)) { diff --git a/server/modules/authenticator/gssapi_auth.h b/server/modules/authenticator/gssapi_auth.h index 317db08b7..be6bd7b0f 100644 --- a/server/modules/authenticator/gssapi_auth.h +++ b/server/modules/authenticator/gssapi_auth.h @@ -39,9 +39,10 @@ enum gssapi_auth_state /** Common state tracking structure */ typedef struct gssapi_auth { - enum gssapi_auth_state state; - uint8_t *principal_name; - size_t principal_name_len; + enum gssapi_auth_state state; /**< Authentication state*/ + uint8_t *principal_name; /**< Principal name */ + size_t principal_name_len; /**< Length of the principal name */ + uint8_t sequence; /**< The next packet seqence number */ } gssapi_auth_t; /** These functions can used for the `create` and `destroy` entry points */ diff --git a/server/modules/authenticator/gssapi_auth_common.c b/server/modules/authenticator/gssapi_auth_common.c index 8fa64627a..5ba0a35e5 100644 --- a/server/modules/authenticator/gssapi_auth_common.c +++ b/server/modules/authenticator/gssapi_auth_common.c @@ -24,6 +24,7 @@ void* gssapi_auth_alloc(void *instance) rval->state = GSSAPI_AUTH_INIT; rval->principal_name = NULL; rval->principal_name_len = 0; + rval->sequence = 0; } return rval; diff --git a/server/modules/authenticator/gssapi_backend_auth.c b/server/modules/authenticator/gssapi_backend_auth.c index d047b80f8..9b43a7f31 100644 --- a/server/modules/authenticator/gssapi_backend_auth.c +++ b/server/modules/authenticator/gssapi_backend_auth.c @@ -19,11 +19,14 @@ #include "gssapi_auth.h" /** - * @file gssapi_backend_auth.c GSSAPI backend authenticator + * @file gssapi_backend_auth.c - GSSAPI backend authenticator */ -/** TODO: Document functions and GSSAPI specific code */ - +/** + * @brief Create a new GSSAPI token + * @param dcb Backend DCB + * @return True on success, false on error + */ static bool send_new_auth_token(DCB *dcb) { bool rval = false; @@ -35,9 +38,11 @@ static bool send_new_auth_token(DCB *dcb) gss_name_t princ = GSS_C_NO_NAME; gssapi_auth_t *auth = (gssapi_auth_t*)dcb->authenticator_data; + /** The service principal name is sent by the backend server */ target.value = auth->principal_name; target.length = auth->principal_name_len + 1; + /** Convert the name into GSSAPI format */ major = gss_import_name(&minor, &target, GSS_C_NT_USER_NAME, &princ); if (GSS_ERROR(major)) @@ -45,6 +50,7 @@ static bool send_new_auth_token(DCB *dcb) report_error(major, minor); } + /** Request the token for the service */ major = gss_init_sec_context(&minor, GSS_C_NO_CREDENTIAL, &handle, princ, GSS_C_NO_OID, 0, 0, GSS_C_NO_CHANNEL_BINDINGS, &in, NULL, &out, 0, 0); @@ -54,6 +60,7 @@ static bool send_new_auth_token(DCB *dcb) } else { + /** We successfully requested the token, send it to the backend server */ GWBUF *buffer = gwbuf_alloc(MYSQL_HEADER_LEN + out.length); if (buffer) @@ -61,8 +68,9 @@ static bool send_new_auth_token(DCB *dcb) uint8_t *data = (uint8_t*)GWBUF_DATA(buffer); gw_mysql_set_byte3(data, out.length); data += 3; - *data++ = 0x03; + *data++ = ++auth->sequence; memcpy(data, out.value, out.length); + if (dcb_write(dcb, buffer)) { rval = true; @@ -77,6 +85,7 @@ static bool send_new_auth_token(DCB *dcb) } major = gss_release_name(&minor, &princ); + if (GSS_ERROR(major)) { report_error(major, minor); @@ -87,15 +96,48 @@ static bool send_new_auth_token(DCB *dcb) } +/** + * @brief Extract the principal name from the AuthSwitchRequest packet + * + * @param dcb Backend DCB + * @param buffer Buffer containing an AuthSwitchRequest packet + * @return True on success, false on error + */ bool extract_principal_name(DCB *dcb, GWBUF *buffer) { bool rval = false; size_t buflen = gwbuf_length(buffer) - MYSQL_HEADER_LEN; uint8_t databuf[buflen]; - gwbuf_copy_data(buffer, MYSQL_HEADER_LEN, buflen, databuf); uint8_t *data = databuf; + gssapi_auth_t *auth = (gssapi_auth_t*)dcb->authenticator_data; - while (*data) + /** Copy the payload and the current packet sequence number */ + gwbuf_copy_data(buffer, MYSQL_HEADER_LEN, buflen, databuf); + gwbuf_copy_data(buffer, MYSQL_SEQ_OFFSET, 1, &auth->sequence); + + if (databuf[0] != MYSQL_REPLY_AUTHSWITCHREQUEST) + { + /** Server responded with something we did not expect. If it's an OK packet, + * it's possible that the server authenticated us as the anonymous user. This + * means that the server is not secure. */ + MXS_ERROR("Server '%s' returned an unexpected authentication response.%s", + dcb->server->unique_name, databuf[0] == MYSQL_REPLY_OK ? + " Authentication was complete before it even started, " + "anonymous users might not be disabled." : ""); + return false; + } + + /** + * The AuthSwitchRequest packet + * + * 0xfe - Command byte + * string[NUL] - Auth plugin name + * string[EOF] - Auth plugin data + * + * Skip over the auth plugin name and copy the service principal name stored + * in the auth plugin data section. + */ + while (*data && data < databuf + buflen) { data++; } @@ -103,35 +145,51 @@ bool extract_principal_name(DCB *dcb, GWBUF *buffer) data++; buflen -= data - databuf; - uint8_t *principal = MXS_MALLOC(buflen); - - if (principal) + if (buflen > 0) { - memcpy(principal, data, buflen); - gssapi_auth_t *auth = (gssapi_auth_t*)dcb->authenticator_data; - auth->principal_name = principal; - auth->principal_name_len = buflen; - rval = true; + uint8_t *principal = MXS_MALLOC(buflen + 1); + + if (principal) + { + /** Store the principal name for later when we request the token + * from the GSSAPI server */ + memcpy(principal, data, buflen); + principal[buflen] = '\0'; + auth->principal_name = principal; + auth->principal_name_len = buflen; + rval = true; + } + } + else + { + MXS_ERROR("Backend server did not send any auth plugin data."); } return rval; } +/** + * @brief Extract data from a MySQL packet + * @param dcb Backend DCB + * @param buffer Buffer containing a complete packet + * @return MXS_AUTH_INCOMPLETE if authentication is ongoing, MXS_AUTH_SUCCEEDED + * if authentication is complete and MXS_AUTH_FAILED if authentication failed. + */ static int gssapi_backend_auth_extract(DCB *dcb, GWBUF *buffer) { int rval = MXS_AUTH_FAILED; - gssapi_auth_t *data = (gssapi_auth_t*)dcb->authenticator_data; + gssapi_auth_t *auth = (gssapi_auth_t*)dcb->authenticator_data; - if (data->state == GSSAPI_AUTH_INIT && extract_principal_name(dcb, buffer)) + if (auth->state == GSSAPI_AUTH_INIT && extract_principal_name(dcb, buffer)) { rval = MXS_AUTH_INCOMPLETE; } - else if (data->state == GSSAPI_AUTH_DATA_SENT) + else if (auth->state == GSSAPI_AUTH_DATA_SENT) { /** Read authentication response */ if (mxs_mysql_is_ok_packet(buffer)) { - data->state = GSSAPI_AUTH_OK; + auth->state = GSSAPI_AUTH_OK; rval = MXS_AUTH_SUCCEEDED; } } @@ -139,26 +197,37 @@ static int gssapi_backend_auth_extract(DCB *dcb, GWBUF *buffer) return rval; } +/** + * @brief Check whether the DCB supports SSL + * @param dcb Backend DCB + * @return True if DCB supports SSL + */ static bool gssapi_backend_auth_connectssl(DCB *dcb) { return dcb->server->server_ssl != NULL; } +/** + * @brief Authenticate the backend connection + * @param dcb Backend DCB + * @return MXS_AUTH_INCOMPLETE if authentication is ongoing, MXS_AUTH_SUCCEEDED + * if authentication is complete and MXS_AUTH_FAILED if authentication failed. + */ static int gssapi_backend_auth_authenticate(DCB *dcb) { int rval = MXS_AUTH_FAILED; - gssapi_auth_t *auth_data = (gssapi_auth_t*)dcb->authenticator_data; + gssapi_auth_t *auth = (gssapi_auth_t*)dcb->authenticator_data; - if (auth_data->state == GSSAPI_AUTH_INIT) + if (auth->state == GSSAPI_AUTH_INIT) { if (send_new_auth_token(dcb)) { rval = MXS_AUTH_INCOMPLETE; - auth_data->state = GSSAPI_AUTH_DATA_SENT; + auth->state = GSSAPI_AUTH_DATA_SENT; } } - else if (auth_data->state == GSSAPI_AUTH_OK) + else if (auth->state == GSSAPI_AUTH_OK) { rval = MXS_AUTH_SUCCEEDED; } @@ -171,7 +240,7 @@ static int gssapi_backend_auth_authenticate(DCB *dcb) */ static GWAUTHENTICATOR MyObject = { - NULL, /* TODO: implement initialize entry point */ + NULL, /* No initialize entry point */ gssapi_auth_alloc, /* Allocate authenticator data */ gssapi_backend_auth_extract, /* Extract data into structure */ gssapi_backend_auth_connectssl, /* Check if client supports SSL */ diff --git a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c index 53e240687..d57cbe677 100644 --- a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c +++ b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c @@ -479,6 +479,9 @@ int gw_read_client_event(DCB* dcb) static int gw_read_do_authentication(DCB *dcb, GWBUF *read_buffer, int nbytes_read) { + ss_debug(uint8_t hdr[MYSQL_HEADER_LEN]); + ss_dassert(gwbuf_copy_data(read_buffer, 0, MYSQL_HEADER_LEN, hdr) == MYSQL_HEADER_LEN && + MYSQL_GET_PACKET_LEN(hdr) + MYSQL_HEADER_LEN == gwbuf_length(read_buffer)); /** Allocate the shared session structure */ if (dcb->data == NULL && (dcb->data = mysql_session_alloc()) == NULL) { From 1333da071220b8eaf8c6920e107d780c4387b269 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Fri, 14 Oct 2016 19:41:50 +0300 Subject: [PATCH 2/6] Remove skygw_utils.h The general purpose stuff in skygw_utils.h was moved to utils.h and the corresponding implementation from skygw_utils.cc to utils.c. Includes updated accordingly. Skygw_utils.h is now only used by log_manager and by mlist, which is only used by log_manager. Consequently, skygw_utils.h was moved to server/maxscale. Utils.h needs a separate overhaul. --- include/maxscale/utils.h | 18 + query_classifier/qc_sqlite/qc_sqlite.c | 1 - .../test/canonical_tests/canonizer.c | 2 +- server/core/adminusers.c | 1 - server/core/buffer.c | 1 - server/core/config.c | 2 +- server/core/dbusers.c | 1 - server/core/dcb.c | 1 - server/core/filter.c | 1 - server/core/gateway.c | 2 +- server/core/gw_utils.c | 2 - server/core/load_utils.c | 1 - server/core/log_manager.cc | 1 - server/core/maxkeys.c | 1 - server/core/maxpasswd.c | 1 - server/core/maxscale/mlist.h | 2 +- .../core}/maxscale/skygw_utils.h | 16 - server/core/modutil.c | 1 - server/core/monitor.c | 1 - server/core/poll.c | 1 - server/core/query_classifier.c | 2 +- server/core/secrets.c | 1 - server/core/server.c | 1 - server/core/service.c | 1 - server/core/session.c | 1 - server/core/skygw_utils.cc | 485 +---------------- server/core/test/test_mysql_users.c | 1 - server/core/test/testadminusers.c | 2 +- server/core/test/testlog.c | 2 +- server/core/test/testlogorder.c | 2 +- server/core/utils.c | 494 +++++++++++++++++- server/modules/filter/ccrfilter/ccrfilter.c | 1 - server/modules/filter/hintfilter/hintparser.c | 1 - .../namedserverfilter/namedserverfilter.c | 1 - server/modules/filter/qlafilter/qlafilter.c | 2 +- .../modules/filter/regexfilter/regexfilter.c | 1 - server/modules/filter/tee/tee.c | 1 - server/modules/filter/topfilter/topfilter.c | 1 - server/modules/filter/tpmfilter/tpmfilter.c | 1 - server/modules/monitor/galeramon/galeramon.h | 1 - server/modules/monitor/mmmon/mmmon.h | 1 - server/modules/monitor/mysqlmon.h | 1 - .../monitor/ndbclustermon/ndbclustermon.h | 1 - .../MySQL/MySQLBackend/mysql_backend.c | 1 - .../protocol/MySQL/MySQLClient/mysql_client.c | 1 - server/modules/protocol/MySQL/mysql_common.c | 1 - server/modules/protocol/maxscaled/maxscaled.c | 1 - server/modules/protocol/telnetd/telnetd.c | 1 - server/modules/routing/avro/avro.c | 1 - server/modules/routing/avro/avro_client.c | 1 - server/modules/routing/avro/avro_file.c | 1 - server/modules/routing/avro/avro_schema.c | 1 - server/modules/routing/binlog/blr.c | 1 - server/modules/routing/binlog/blr_cache.c | 1 - server/modules/routing/binlog/blr_file.c | 1 - server/modules/routing/binlog/blr_master.c | 1 - server/modules/routing/binlog/blr_slave.c | 1 - .../modules/routing/binlog/test/testbinlog.c | 1 - server/modules/routing/cli/cli.c | 1 - server/modules/routing/debugcli/debugcli.c | 1 - server/modules/routing/debugcli/debugcmd.c | 1 - server/modules/routing/maxinfo/maxinfo.c | 1 - .../modules/routing/maxinfo/maxinfo_error.c | 1 - server/modules/routing/maxinfo/maxinfo_exec.c | 1 - .../modules/routing/maxinfo/maxinfo_parse.c | 1 - .../routing/readconnroute/readconnroute.c | 1 - .../routing/readwritesplit/readwritesplit.c | 1 - .../routing/readwritesplit/rwsplit_mysql.c | 1 - .../routing/schemarouter/schemarouter.c | 1 - 69 files changed, 521 insertions(+), 567 deletions(-) rename {include => server/core}/maxscale/skygw_utils.h (88%) diff --git a/include/maxscale/utils.h b/include/maxscale/utils.h index bca259700..1f608296d 100644 --- a/include/maxscale/utils.h +++ b/include/maxscale/utils.h @@ -37,6 +37,9 @@ MXS_BEGIN_DECLS #define MXS_ARRAY_NELEMS(array) ((size_t)(sizeof(array)/sizeof(array[0]))) +bool utils_init(); /*< Call this first before using any other function */ +void utils_end(); + int setnonblocking(int fd); char *gw_strend(register const char *s); static char gw_randomchar(); @@ -49,6 +52,21 @@ void gw_sha1_2_str(const uint8_t *in, int in_len, const uint8_t *in2, int in2_le int gw_getsockerrno(int fd); char *create_hex_sha1_sha1_passwd(char *passwd); +char* trim(char *str); +char* squeeze_whitespace(char* str); +bool strip_escape_chars(char*); + +bool is_valid_posix_path(char* path); + +char* remove_mysql_comments(const char** src, const size_t* srcsize, char** dest, + size_t* destsize); +char* replace_values(const char** src, const size_t* srcsize, char** dest, + size_t* destsize); +char* replace_literal(char* haystack, + const char* needle, + const char* replacement); +char* replace_quoted(const char** src, const size_t* srcsize, char** dest, size_t* destsize); + MXS_END_DECLS #endif diff --git a/query_classifier/qc_sqlite/qc_sqlite.c b/query_classifier/qc_sqlite/qc_sqlite.c index a5fbebd27..598bc3c2c 100644 --- a/query_classifier/qc_sqlite/qc_sqlite.c +++ b/query_classifier/qc_sqlite/qc_sqlite.c @@ -20,7 +20,6 @@ #include #include #include -#include #include #include #include "builtin_functions.h" diff --git a/query_classifier/test/canonical_tests/canonizer.c b/query_classifier/test/canonical_tests/canonizer.c index fc95105e2..56c59f1ea 100644 --- a/query_classifier/test/canonical_tests/canonizer.c +++ b/query_classifier/test/canonical_tests/canonizer.c @@ -18,7 +18,7 @@ #include #include #include -#include +#include int main(int argc, char** argv) { diff --git a/server/core/adminusers.c b/server/core/adminusers.c index 6b0f1693c..f0d1a1946 100644 --- a/server/core/adminusers.c +++ b/server/core/adminusers.c @@ -21,7 +21,6 @@ #include #include #include -#include #include #include #include diff --git a/server/core/buffer.c b/server/core/buffer.c index cf25bdb08..e91e79b58 100644 --- a/server/core/buffer.c +++ b/server/core/buffer.c @@ -43,7 +43,6 @@ #include #include #include -#include #include #include #include diff --git a/server/core/config.c b/server/core/config.c index 7cda0ed0a..4b9bbab2f 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -58,7 +58,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/server/core/dbusers.c b/server/core/dbusers.c index cd6491f77..4098474e2 100644 --- a/server/core/dbusers.c +++ b/server/core/dbusers.c @@ -43,7 +43,6 @@ #include #include #include -#include #include #include #include diff --git a/server/core/dcb.c b/server/core/dcb.c index acf811c7f..065213cec 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -81,7 +81,6 @@ #include #include #include -#include #include #include #include diff --git a/server/core/filter.c b/server/core/filter.c index 45fa3721a..c3f8812b1 100644 --- a/server/core/filter.c +++ b/server/core/filter.c @@ -30,7 +30,6 @@ #include #include #include -#include #include #include diff --git a/server/core/gateway.c b/server/core/gateway.c index 0320ebf10..6435e4b6d 100644 --- a/server/core/gateway.c +++ b/server/core/gateway.c @@ -74,7 +74,7 @@ #include #include -#include +#include #include #include diff --git a/server/core/gw_utils.c b/server/core/gw_utils.c index 78729e46a..2ef05638d 100644 --- a/server/core/gw_utils.c +++ b/server/core/gw_utils.c @@ -37,8 +37,6 @@ #include #include #include - -#include #include SPINLOCK tmplock = SPINLOCK_INIT; diff --git a/server/core/load_utils.c b/server/core/load_utils.c index ad83cfccf..c995ef085 100644 --- a/server/core/load_utils.c +++ b/server/core/load_utils.c @@ -36,7 +36,6 @@ #include #include #include -#include #include #include #include diff --git a/server/core/log_manager.cc b/server/core/log_manager.cc index 996e5e001..80263a5ef 100644 --- a/server/core/log_manager.cc +++ b/server/core/log_manager.cc @@ -28,7 +28,6 @@ #include #include #include -#include #include #include #include "maxscale/mlist.h" diff --git a/server/core/maxkeys.c b/server/core/maxkeys.c index 380338329..86c91d642 100644 --- a/server/core/maxkeys.c +++ b/server/core/maxkeys.c @@ -25,7 +25,6 @@ #include #include #include -#include #include #include diff --git a/server/core/maxpasswd.c b/server/core/maxpasswd.c index 4a7c77a2a..6fc98198d 100644 --- a/server/core/maxpasswd.c +++ b/server/core/maxpasswd.c @@ -26,7 +26,6 @@ #include #include #include -#include struct option options[] = { diff --git a/server/core/maxscale/mlist.h b/server/core/maxscale/mlist.h index c3ccc1695..be5a4fd55 100644 --- a/server/core/maxscale/mlist.h +++ b/server/core/maxscale/mlist.h @@ -15,7 +15,7 @@ */ #include -#include +#include "skygw_utils.h" MXS_BEGIN_DECLS diff --git a/include/maxscale/skygw_utils.h b/server/core/maxscale/skygw_utils.h similarity index 88% rename from include/maxscale/skygw_utils.h rename to server/core/maxscale/skygw_utils.h index e21a41afa..cbfad8a89 100644 --- a/include/maxscale/skygw_utils.h +++ b/server/core/maxscale/skygw_utils.h @@ -96,9 +96,6 @@ struct skygw_file_st skygw_chk_t sf_chk_tail; }; -bool utils_init(); /*< Call this first before using any other function */ -void utils_end(); - /** Skygw thread routines */ skygw_thread_t* skygw_thread_init(const char* name, void* (*sth_thrfun)(void* data), @@ -167,19 +164,6 @@ int skygw_rwlock_init(skygw_rwlock_t** rwlock); size_t get_decimal_len(size_t s); -char* remove_mysql_comments(const char** src, const size_t* srcsize, char** dest, - size_t* destsize); -char* replace_values(const char** src, const size_t* srcsize, char** dest, - size_t* destsize); -char* replace_literal(char* haystack, - const char* needle, - const char* replacement); -char* replace_quoted(const char** src, const size_t* srcsize, char** dest, size_t* destsize); -bool is_valid_posix_path(char* path); -bool strip_escape_chars(char*); -char* trim(char *str); -char* squeeze_whitespace(char* str); - MXS_END_DECLS #endif /* SKYGW_UTILS_H */ diff --git a/server/core/modutil.c b/server/core/modutil.c index 8f673d1a4..4e25999a6 100644 --- a/server/core/modutil.c +++ b/server/core/modutil.c @@ -30,7 +30,6 @@ #include #include #include -#include #include /** These are used when converting MySQL wildcards to regular expressions */ diff --git a/server/core/monitor.c b/server/core/monitor.c index 20f29323e..60bc4e238 100644 --- a/server/core/monitor.c +++ b/server/core/monitor.c @@ -33,7 +33,6 @@ #include #include #include -#include #include #include #include diff --git a/server/core/poll.c b/server/core/poll.c index 683b8cbf3..149b67fdb 100644 --- a/server/core/poll.c +++ b/server/core/poll.c @@ -24,7 +24,6 @@ #include #include #include -#include #include #include #include diff --git a/server/core/query_classifier.c b/server/core/query_classifier.c index 7a0ab82cb..071ddf0e9 100644 --- a/server/core/query_classifier.c +++ b/server/core/query_classifier.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include //#define QC_TRACE_ENABLED #undef QC_TRACE_ENABLED diff --git a/server/core/secrets.c b/server/core/secrets.c index 21e7318d1..30570044d 100644 --- a/server/core/secrets.c +++ b/server/core/secrets.c @@ -13,7 +13,6 @@ #include #include -#include #include #include #include diff --git a/server/core/server.c b/server/core/server.c index db3c49f94..101b5b5f0 100644 --- a/server/core/server.c +++ b/server/core/server.c @@ -40,7 +40,6 @@ #include #include #include -#include #include #include #include diff --git a/server/core/service.c b/server/core/service.c index cc908b90f..643011a31 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -55,7 +55,6 @@ #include #include #include -#include #include #include #include diff --git a/server/core/session.c b/server/core/session.c index f6ce149f6..a449c6998 100644 --- a/server/core/session.c +++ b/server/core/session.c @@ -40,7 +40,6 @@ #include #include #include -#include #include #include diff --git a/server/core/skygw_utils.cc b/server/core/skygw_utils.cc index 8b1142edd..fa0afb6fb 100644 --- a/server/core/skygw_utils.cc +++ b/server/core/skygw_utils.cc @@ -27,7 +27,7 @@ #include #include #include -#include +#include "maxscale/skygw_utils.h" #include #include #include @@ -40,8 +40,6 @@ # endif #endif -#define MAX_ERROR_MSG PATH_MAX - static void simple_mutex_free_memory(simple_mutex_t* sm); static void thread_free_memory(skygw_thread_t* th, char* name); /** End of static function declarations */ @@ -1006,298 +1004,6 @@ void skygw_file_close(skygw_file_t* file) } } -#define BUFFER_GROWTH_RATE 1.2 -static pcre2_code* remove_comments_re = NULL; -static const PCRE2_SPTR remove_comments_pattern = (PCRE2_SPTR) - "(?:`[^`]*`\\K)|(\\/[*](?!(M?!)).*?[*]\\/)|(?:#.*|--[[:space:]].*)"; - -/** - * Remove SQL comments from the end of a string - * - * The inline executable comments are not removed due to the fact that they can - * alter the behavior of the query. - * @param src Pointer to the string to modify. - * @param srcsize Pointer to a size_t variable which holds the length of the string to - * be modified. - * @param dest The address of the pointer where the result will be stored. If the - * value pointed by this parameter is NULL, new memory will be allocated as needed. - * @param Pointer to a size_t variable where the size of the result string is stored. - * @return Pointer to new modified string or NULL if memory allocation failed. - * If NULL is returned and the value pointed by @c dest was not NULL, no new - * memory will be allocated, the memory pointed by @dest will be freed and the - * contents of @c dest and @c destsize will be invalid. - */ -char* remove_mysql_comments(const char** src, const size_t* srcsize, char** dest, size_t* destsize) -{ - static const PCRE2_SPTR replace = (PCRE2_SPTR) ""; - pcre2_match_data* mdata; - char* output = *dest; - size_t orig_len = *srcsize; - size_t len = output ? *destsize : orig_len; - - if (orig_len > 0) - { - if ((output || (output = (char*) malloc(len * sizeof (char)))) && - (mdata = pcre2_match_data_create_from_pattern(remove_comments_re, NULL))) - { - while (pcre2_substitute(remove_comments_re, (PCRE2_SPTR) * src, orig_len, 0, - PCRE2_SUBSTITUTE_GLOBAL, mdata, NULL, - replace, PCRE2_ZERO_TERMINATED, - (PCRE2_UCHAR8*) output, &len) == PCRE2_ERROR_NOMEMORY) - { - char* tmp = (char*) realloc(output, (len = (size_t) (len * BUFFER_GROWTH_RATE + 1))); - if (tmp == NULL) - { - free(output); - output = NULL; - break; - } - output = tmp; - } - pcre2_match_data_free(mdata); - } - else - { - free(output); - output = NULL; - } - } - else if (output == NULL) - { - output = strdup(*src); - } - - if (output) - { - *destsize = strlen(output); - *dest = output; - } - - return output; -} - -static pcre2_code* replace_values_re = NULL; -static const PCRE2_SPTR replace_values_pattern = (PCRE2_SPTR) "(?i)([-=,+*/([:space:]]|\\b|[@])" - "(?:[0-9.-]+|(?<=[@])[a-z_0-9]+)([-=,+*/)[:space:];]|$)"; - -/** - * Replace literal numbers and user variables with a question mark. - * @param src Pointer to the string to modify. - * @param srcsize Pointer to a size_t variable which holds the length of the string to - * be modified. - * @param dest The address of the pointer where the result will be stored. If the - * value pointed by this parameter is NULL, new memory will be allocated as needed. - * @param Pointer to a size_t variable where the size of the result string is stored. - * @return Pointer to new modified string or NULL if memory allocation failed. - * If NULL is returned and the value pointed by @c dest was not NULL, no new - * memory will be allocated, the memory pointed by @dest will be freed and the - * contents of @c dest and @c destsize will be invalid. - */ -char* replace_values(const char** src, const size_t* srcsize, char** dest, size_t* destsize) -{ - static const PCRE2_SPTR replace = (PCRE2_SPTR) "$1?$2"; - pcre2_match_data* mdata; - char* output = *dest; - size_t orig_len = *srcsize; - size_t len = output ? *destsize : orig_len; - - if (orig_len > 0) - { - if ((output || (output = (char*) malloc(len * sizeof (char)))) && - (mdata = pcre2_match_data_create_from_pattern(replace_values_re, NULL))) - { - while (pcre2_substitute(replace_values_re, (PCRE2_SPTR) * src, orig_len, 0, - PCRE2_SUBSTITUTE_GLOBAL, mdata, NULL, - replace, PCRE2_ZERO_TERMINATED, - (PCRE2_UCHAR8*) output, &len) == PCRE2_ERROR_NOMEMORY) - { - char* tmp = (char*) realloc(output, (len = (size_t) (len * BUFFER_GROWTH_RATE + 1))); - if (tmp == NULL) - { - free(output); - output = NULL; - break; - } - output = tmp; - } - pcre2_match_data_free(mdata); - } - else - { - free(output); - output = NULL; - } - } - else if (output == NULL) - { - output = strdup(*src); - } - - if (output) - { - *destsize = strlen(output); - *dest = output; - } - - return output; -} - -/** - * Find the given needle - user-provided literal - and replace it with - * replacement string. Separate user-provided literals from matching table names - * etc. by searching only substrings preceded by non-letter and non-number. - * - * @param haystack Plain text query string, not to be freed - * @param needle Substring to be searched, not to be freed - * @param replacement Replacement text, not to be freed - * - * @return newly allocated string where needle is replaced - */ -char* replace_literal(char* haystack, const char* needle, const char* replacement) -{ - const char* prefix = "[ ='\",\\(]"; /*< ' ','=','(',''',''"',',' are allowed before needle */ - const char* suffix = "([^[:alnum:]]|$)"; /*< alpha-num chars aren't allowed after the needle */ - char* search_re; - char* newstr; - regex_t re; - regmatch_t match; - int rc; - size_t rlen = strlen(replacement); - size_t nlen = strlen(needle); - size_t hlen = strlen(haystack); - - search_re = (char *) malloc(strlen(prefix) + nlen + strlen(suffix) + 1); - - if (search_re == NULL) - { - char errbuf[MXS_STRERROR_BUFLEN]; - fprintf(stderr, "Regex memory allocation failed : %s\n", - strerror_r(errno, errbuf, sizeof (errbuf))); - newstr = haystack; - goto retblock; - } - - sprintf(search_re, "%s%s%s", prefix, needle, suffix); - /** Allocate memory for new string +1 for terminating byte */ - newstr = (char *) malloc(hlen - nlen + rlen + 1); - - if (newstr == NULL) - { - char errbuf[MXS_STRERROR_BUFLEN]; - fprintf(stderr, "Regex memory allocation failed : %s\n", - strerror_r(errno, errbuf, sizeof (errbuf))); - free(search_re); - free(newstr); - newstr = haystack; - goto retblock; - } - - rc = regcomp(&re, search_re, REG_EXTENDED | REG_ICASE); - ss_info_dassert(rc == 0, "Regex check"); - - if (rc != 0) - { - char error_message[MAX_ERROR_MSG]; - regerror(rc, &re, error_message, MAX_ERROR_MSG); - fprintf(stderr, "Regex error compiling '%s': %s\n", - search_re, error_message); - free(search_re); - free(newstr); - newstr = haystack; - goto retblock; - } - rc = regexec(&re, haystack, 1, &match, 0); - - if (rc != 0) - { - free(search_re); - free(newstr); - regfree(&re); - newstr = haystack; - goto retblock; - } - memcpy(newstr, haystack, match.rm_so + 1); - memcpy(newstr + match.rm_so + 1, replacement, rlen); - /** +1 is terminating byte */ - memcpy(newstr + match.rm_so + 1 + rlen, haystack + match.rm_so + 1 + nlen, hlen - (match.rm_so + 1) - nlen + 1); - - regfree(&re); - free(haystack); - free(search_re); -retblock: - return newstr; -} - -static pcre2_code* replace_quoted_re = NULL; -static const PCRE2_SPTR replace_quoted_pattern = (PCRE2_SPTR) - "(?>[^'\"]*)(?|(?:\"\\K(?:(?:(?<=\\\\)\")|[^\"])*(\"))|(?:'\\K(?:(?:(?<=\\\\)')|[^'])*(')))"; - -/** - * Replace contents of single or double quoted strings with question marks. - * @param src Pointer to the string to modify. - * @param srcsize Pointer to a size_t variable which holds the length of the string to - * be modified. - * @param dest The address of the pointer where the result will be stored. If the - * value pointed by this parameter is NULL, new memory will be allocated as needed. - * @param Pointer to a size_t variable where the size of the result string is stored. - * @return Pointer to new modified string or NULL if memory allocation failed. - * If NULL is returned and the value pointed by @c dest was not NULL, no new - * memory will be allocated, the memory pointed by @dest will be freed and the - * contents of @c dest and @c destsize will be invalid. - */ -char* replace_quoted(const char** src, const size_t* srcsize, char** dest, size_t* destsize) -{ - static const PCRE2_SPTR replace = (PCRE2_SPTR) "?$1"; - pcre2_match_data* mdata; - char* output = *dest; - size_t orig_len = *srcsize; - size_t len = output ? *destsize : orig_len; - - if (orig_len > 0) - { - if ((output || (output = (char*) malloc(len * sizeof (char)))) && - (mdata = pcre2_match_data_create_from_pattern(replace_quoted_re, NULL))) - { - while (pcre2_substitute(replace_quoted_re, (PCRE2_SPTR) * src, orig_len, 0, - PCRE2_SUBSTITUTE_GLOBAL, mdata, NULL, - replace, PCRE2_ZERO_TERMINATED, - (PCRE2_UCHAR8*) output, &len) == PCRE2_ERROR_NOMEMORY) - { - char* tmp = (char*) realloc(output, (len = (size_t) (len * BUFFER_GROWTH_RATE + 1))); - if (tmp == NULL) - { - free(output); - output = NULL; - break; - } - output = tmp; - } - pcre2_match_data_free(mdata); - } - else - { - free(output); - output = NULL; - } - } - else if (output == NULL) - { - output = strdup(*src); - } - - if (output) - { - *destsize = strlen(output); - *dest = output; - } - else - { - *dest = NULL; - } - - return output; -} - /** * Calculate the number of decimal numbers from a size_t value. * @@ -1312,192 +1018,3 @@ size_t get_decimal_len( { return value > 0 ? (size_t) log10((double) value) + 1 : 1; } - -/** - * Check if the provided pathname is POSIX-compliant. The valid characters - * are [a-z A-Z 0-9._-]. - * @param path A null-terminated string - * @return true if it is a POSIX-compliant pathname, otherwise false - */ -bool is_valid_posix_path(char* path) -{ - char* ptr = path; - while (*ptr != '\0') - { - if (isalnum(*ptr) || *ptr == '/' || *ptr == '.' || *ptr == '-' || *ptr == '_') - { - ptr++; - } - else - { - return false; - } - } - return true; -} - -/** - * Strip escape characters from a character string. - * @param String to parse. - * @return True if parsing was successful, false on errors. - */ -bool -strip_escape_chars(char* val) -{ - int cur, end; - - if (val == NULL) - { - return false; - } - - end = strlen(val) + 1; - cur = 0; - - while (cur < end) - { - if (val[cur] == '\\') - { - memmove(val + cur, val + cur + 1, end - cur - 1); - end--; - } - cur++; - } - return true; -} - -/** - * Trim leading and trailing whitespace from a string - * - * @param str String to trim - * @return Trimmed string - */ -char* trim(char *str) -{ - char* ptr = strchr(str, '\0') - 1; - - while (ptr > str && isspace(*ptr)) - { - ptr--; - } - - if (isspace(*(ptr + 1))) - { - *(ptr + 1) = '\0'; - } - - ptr = str; - - while (isspace(*ptr)) - { - ptr++; - } - - if (ptr != str) - { - memmove(str, ptr, strlen(ptr) + 1); - } - - return str; -} - -/** - * Replace all whitespace with spaces and squeeze repeating whitespace characters - * - * @param str String to squeeze - * @return Squeezed string - */ -char* squeeze_whitespace(char* str) -{ - char* store = str; - char* ptr = str; - - /** Remove leading whitespace */ - while (isspace(*ptr) && *ptr != '\0') - { - ptr++; - } - - /** Squeeze all repeating whitespace */ - while (*ptr != '\0') - { - while (isspace(*ptr) && isspace(*(ptr + 1))) - { - ptr++; - } - - if (isspace(*ptr)) - { - *store++ = ' '; - ptr++; - } - else - { - *store++ = *ptr++; - } - } - - *store = '\0'; - - /** Remove trailing whitespace */ - while (store > str && isspace(*(store - 1))) - { - store--; - *store = '\0'; - } - - return str; -} - -/** - * Initialize the utils library - * - * This function initializes structures used in various functions. - * @return true on success, false on error - */ -bool utils_init() -{ - bool rval = true; - - PCRE2_SIZE erroffset; - int errcode; - - ss_info_dassert(remove_comments_re == NULL, "utils_init called multiple times"); - remove_comments_re = pcre2_compile(remove_comments_pattern, PCRE2_ZERO_TERMINATED, 0, &errcode, - &erroffset, NULL); - if (remove_comments_re == NULL) - { - rval = false; - } - - ss_info_dassert(replace_quoted_re == NULL, "utils_init called multiple times"); - replace_quoted_re = pcre2_compile(replace_quoted_pattern, PCRE2_ZERO_TERMINATED, 0, &errcode, - &erroffset, NULL); - if (replace_quoted_re == NULL) - { - rval = false; - } - - ss_info_dassert(replace_values_re == NULL, "utils_init called multiple times"); - replace_values_re = pcre2_compile(replace_values_pattern, PCRE2_ZERO_TERMINATED, 0, &errcode, - &erroffset, NULL); - if (replace_values_re == NULL) - { - rval = false; - } - - return rval; -} - -/** - * Close the utils library. This should be the last call to this library. - */ -void utils_end() -{ - pcre2_code_free(remove_comments_re); - remove_comments_re = NULL; - pcre2_code_free(replace_quoted_re); - replace_quoted_re = NULL; - pcre2_code_free(replace_values_re); - replace_values_re = NULL; -} diff --git a/server/core/test/test_mysql_users.c b/server/core/test/test_mysql_users.c index c25b16205..49f980f53 100644 --- a/server/core/test/test_mysql_users.c +++ b/server/core/test/test_mysql_users.c @@ -31,7 +31,6 @@ #include #include #include -#include #include #include #include diff --git a/server/core/test/testadminusers.c b/server/core/test/testadminusers.c index 1164f6f0c..6fe85266e 100644 --- a/server/core/test/testadminusers.c +++ b/server/core/test/testadminusers.c @@ -37,7 +37,7 @@ #include #include #include -#include +#include /** * test1 default user diff --git a/server/core/test/testlog.c b/server/core/test/testlog.c index 2f48b71aa..cf409f0b4 100644 --- a/server/core/test/testlog.c +++ b/server/core/test/testlog.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include "../maxscale/skygw_utils.h" #include static void skygw_log_enable(int priority) diff --git a/server/core/test/testlogorder.c b/server/core/test/testlogorder.c index 42b9f78cb..c50e2ddbb 100644 --- a/server/core/test/testlogorder.c +++ b/server/core/test/testlogorder.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include static void skygw_log_enable(int priority) diff --git a/server/core/utils.c b/server/core/utils.c index e829b871a..ead727b8e 100644 --- a/server/core/utils.c +++ b/server/core/utils.c @@ -29,16 +29,27 @@ */ +#include #include #include #include #include #include #include -#include #include #include #include +#include + +#if !defined(PATH_MAX) +# if defined(__USE_POSIX) +# define PATH_MAX _POSIX_PATH_MAX +# else +# define PATH_MAX 256 +# endif +#endif + +#define MAX_ERROR_MSG PATH_MAX /* used in the hex2bin function */ #define char_val(X) (X >= '0' && X <= '9' ? X-'0' : \ @@ -50,6 +61,29 @@ char hex_upper[] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; char hex_lower[] = "0123456789abcdefghijklmnopqrstuvwxyz"; +/** + * Check if the provided pathname is POSIX-compliant. The valid characters + * are [a-z A-Z 0-9._-]. + * @param path A null-terminated string + * @return true if it is a POSIX-compliant pathname, otherwise false + */ +bool is_valid_posix_path(char* path) +{ + char* ptr = path; + while (*ptr != '\0') + { + if (isalnum(*ptr) || *ptr == '/' || *ptr == '.' || *ptr == '-' || *ptr == '_') + { + ptr++; + } + else + { + return false; + } + } + return true; +} + /***************************************** * backend read event triggered by EPOLLIN *****************************************/ @@ -388,3 +422,461 @@ bool mxs_mkdir_all(const char *path, int mask) return mkdir_all_internal(local_path, (mode_t)mask); } + +/** + * Trim leading and trailing whitespace from a string + * + * @param str String to trim + * @return Trimmed string + */ +char* trim(char *str) +{ + char* ptr = strchr(str, '\0') - 1; + + while (ptr > str && isspace(*ptr)) + { + ptr--; + } + + if (isspace(*(ptr + 1))) + { + *(ptr + 1) = '\0'; + } + + ptr = str; + + while (isspace(*ptr)) + { + ptr++; + } + + if (ptr != str) + { + memmove(str, ptr, strlen(ptr) + 1); + } + + return str; +} + +/** + * Replace all whitespace with spaces and squeeze repeating whitespace characters + * + * @param str String to squeeze + * @return Squeezed string + */ +char* squeeze_whitespace(char* str) +{ + char* store = str; + char* ptr = str; + + /** Remove leading whitespace */ + while (isspace(*ptr) && *ptr != '\0') + { + ptr++; + } + + /** Squeeze all repeating whitespace */ + while (*ptr != '\0') + { + while (isspace(*ptr) && isspace(*(ptr + 1))) + { + ptr++; + } + + if (isspace(*ptr)) + { + *store++ = ' '; + ptr++; + } + else + { + *store++ = *ptr++; + } + } + + *store = '\0'; + + /** Remove trailing whitespace */ + while (store > str && isspace(*(store - 1))) + { + store--; + *store = '\0'; + } + + return str; +} + +/** + * Strip escape characters from a character string. + * @param String to parse. + * @return True if parsing was successful, false on errors. + */ +bool +strip_escape_chars(char* val) +{ + int cur, end; + + if (val == NULL) + { + return false; + } + + end = strlen(val) + 1; + cur = 0; + + while (cur < end) + { + if (val[cur] == '\\') + { + memmove(val + cur, val + cur + 1, end - cur - 1); + end--; + } + cur++; + } + return true; +} + +#define BUFFER_GROWTH_RATE 1.2 +static pcre2_code* remove_comments_re = NULL; +static const PCRE2_SPTR remove_comments_pattern = (PCRE2_SPTR) + "(?:`[^`]*`\\K)|(\\/[*](?!(M?!)).*?[*]\\/)|(?:#.*|--[[:space:]].*)"; + +/** + * Remove SQL comments from the end of a string + * + * The inline executable comments are not removed due to the fact that they can + * alter the behavior of the query. + * @param src Pointer to the string to modify. + * @param srcsize Pointer to a size_t variable which holds the length of the string to + * be modified. + * @param dest The address of the pointer where the result will be stored. If the + * value pointed by this parameter is NULL, new memory will be allocated as needed. + * @param Pointer to a size_t variable where the size of the result string is stored. + * @return Pointer to new modified string or NULL if memory allocation failed. + * If NULL is returned and the value pointed by @c dest was not NULL, no new + * memory will be allocated, the memory pointed by @dest will be freed and the + * contents of @c dest and @c destsize will be invalid. + */ +char* remove_mysql_comments(const char** src, const size_t* srcsize, char** dest, size_t* destsize) +{ + static const PCRE2_SPTR replace = (PCRE2_SPTR) ""; + pcre2_match_data* mdata; + char* output = *dest; + size_t orig_len = *srcsize; + size_t len = output ? *destsize : orig_len; + + if (orig_len > 0) + { + if ((output || (output = (char*) malloc(len * sizeof (char)))) && + (mdata = pcre2_match_data_create_from_pattern(remove_comments_re, NULL))) + { + while (pcre2_substitute(remove_comments_re, (PCRE2_SPTR) * src, orig_len, 0, + PCRE2_SUBSTITUTE_GLOBAL, mdata, NULL, + replace, PCRE2_ZERO_TERMINATED, + (PCRE2_UCHAR8*) output, &len) == PCRE2_ERROR_NOMEMORY) + { + char* tmp = (char*) realloc(output, (len = (size_t) (len * BUFFER_GROWTH_RATE + 1))); + if (tmp == NULL) + { + free(output); + output = NULL; + break; + } + output = tmp; + } + pcre2_match_data_free(mdata); + } + else + { + free(output); + output = NULL; + } + } + else if (output == NULL) + { + output = strdup(*src); + } + + if (output) + { + *destsize = strlen(output); + *dest = output; + } + + return output; +} + +static pcre2_code* replace_values_re = NULL; +static const PCRE2_SPTR replace_values_pattern = (PCRE2_SPTR) "(?i)([-=,+*/([:space:]]|\\b|[@])" + "(?:[0-9.-]+|(?<=[@])[a-z_0-9]+)([-=,+*/)[:space:];]|$)"; + +/** + * Replace literal numbers and user variables with a question mark. + * @param src Pointer to the string to modify. + * @param srcsize Pointer to a size_t variable which holds the length of the string to + * be modified. + * @param dest The address of the pointer where the result will be stored. If the + * value pointed by this parameter is NULL, new memory will be allocated as needed. + * @param Pointer to a size_t variable where the size of the result string is stored. + * @return Pointer to new modified string or NULL if memory allocation failed. + * If NULL is returned and the value pointed by @c dest was not NULL, no new + * memory will be allocated, the memory pointed by @dest will be freed and the + * contents of @c dest and @c destsize will be invalid. + */ +char* replace_values(const char** src, const size_t* srcsize, char** dest, size_t* destsize) +{ + static const PCRE2_SPTR replace = (PCRE2_SPTR) "$1?$2"; + pcre2_match_data* mdata; + char* output = *dest; + size_t orig_len = *srcsize; + size_t len = output ? *destsize : orig_len; + + if (orig_len > 0) + { + if ((output || (output = (char*) malloc(len * sizeof (char)))) && + (mdata = pcre2_match_data_create_from_pattern(replace_values_re, NULL))) + { + while (pcre2_substitute(replace_values_re, (PCRE2_SPTR) * src, orig_len, 0, + PCRE2_SUBSTITUTE_GLOBAL, mdata, NULL, + replace, PCRE2_ZERO_TERMINATED, + (PCRE2_UCHAR8*) output, &len) == PCRE2_ERROR_NOMEMORY) + { + char* tmp = (char*) realloc(output, (len = (size_t) (len * BUFFER_GROWTH_RATE + 1))); + if (tmp == NULL) + { + free(output); + output = NULL; + break; + } + output = tmp; + } + pcre2_match_data_free(mdata); + } + else + { + free(output); + output = NULL; + } + } + else if (output == NULL) + { + output = strdup(*src); + } + + if (output) + { + *destsize = strlen(output); + *dest = output; + } + + return output; +} + +/** + * Find the given needle - user-provided literal - and replace it with + * replacement string. Separate user-provided literals from matching table names + * etc. by searching only substrings preceded by non-letter and non-number. + * + * @param haystack Plain text query string, not to be freed + * @param needle Substring to be searched, not to be freed + * @param replacement Replacement text, not to be freed + * + * @return newly allocated string where needle is replaced + */ +char* replace_literal(char* haystack, const char* needle, const char* replacement) +{ + const char* prefix = "[ ='\",\\(]"; /*< ' ','=','(',''',''"',',' are allowed before needle */ + const char* suffix = "([^[:alnum:]]|$)"; /*< alpha-num chars aren't allowed after the needle */ + char* search_re; + char* newstr; + regex_t re; + regmatch_t match; + int rc; + size_t rlen = strlen(replacement); + size_t nlen = strlen(needle); + size_t hlen = strlen(haystack); + + search_re = (char *) malloc(strlen(prefix) + nlen + strlen(suffix) + 1); + + if (search_re == NULL) + { + char errbuf[MXS_STRERROR_BUFLEN]; + fprintf(stderr, "Regex memory allocation failed : %s\n", + strerror_r(errno, errbuf, sizeof (errbuf))); + newstr = haystack; + goto retblock; + } + + sprintf(search_re, "%s%s%s", prefix, needle, suffix); + /** Allocate memory for new string +1 for terminating byte */ + newstr = (char *) malloc(hlen - nlen + rlen + 1); + + if (newstr == NULL) + { + char errbuf[MXS_STRERROR_BUFLEN]; + fprintf(stderr, "Regex memory allocation failed : %s\n", + strerror_r(errno, errbuf, sizeof (errbuf))); + free(search_re); + free(newstr); + newstr = haystack; + goto retblock; + } + + rc = regcomp(&re, search_re, REG_EXTENDED | REG_ICASE); + ss_info_dassert(rc == 0, "Regex check"); + + if (rc != 0) + { + char error_message[MAX_ERROR_MSG]; + regerror(rc, &re, error_message, MAX_ERROR_MSG); + fprintf(stderr, "Regex error compiling '%s': %s\n", + search_re, error_message); + free(search_re); + free(newstr); + newstr = haystack; + goto retblock; + } + rc = regexec(&re, haystack, 1, &match, 0); + + if (rc != 0) + { + free(search_re); + free(newstr); + regfree(&re); + newstr = haystack; + goto retblock; + } + memcpy(newstr, haystack, match.rm_so + 1); + memcpy(newstr + match.rm_so + 1, replacement, rlen); + /** +1 is terminating byte */ + memcpy(newstr + match.rm_so + 1 + rlen, haystack + match.rm_so + 1 + nlen, hlen - (match.rm_so + 1) - nlen + 1); + + regfree(&re); + free(haystack); + free(search_re); +retblock: + return newstr; +} + +static pcre2_code* replace_quoted_re = NULL; +static const PCRE2_SPTR replace_quoted_pattern = (PCRE2_SPTR) + "(?>[^'\"]*)(?|(?:\"\\K(?:(?:(?<=\\\\)\")|[^\"])*(\"))|(?:'\\K(?:(?:(?<=\\\\)')|[^'])*(')))"; + +/** + * Replace contents of single or double quoted strings with question marks. + * @param src Pointer to the string to modify. + * @param srcsize Pointer to a size_t variable which holds the length of the string to + * be modified. + * @param dest The address of the pointer where the result will be stored. If the + * value pointed by this parameter is NULL, new memory will be allocated as needed. + * @param Pointer to a size_t variable where the size of the result string is stored. + * @return Pointer to new modified string or NULL if memory allocation failed. + * If NULL is returned and the value pointed by @c dest was not NULL, no new + * memory will be allocated, the memory pointed by @dest will be freed and the + * contents of @c dest and @c destsize will be invalid. + */ +char* replace_quoted(const char** src, const size_t* srcsize, char** dest, size_t* destsize) +{ + static const PCRE2_SPTR replace = (PCRE2_SPTR) "?$1"; + pcre2_match_data* mdata; + char* output = *dest; + size_t orig_len = *srcsize; + size_t len = output ? *destsize : orig_len; + + if (orig_len > 0) + { + if ((output || (output = (char*) malloc(len * sizeof (char)))) && + (mdata = pcre2_match_data_create_from_pattern(replace_quoted_re, NULL))) + { + while (pcre2_substitute(replace_quoted_re, (PCRE2_SPTR) * src, orig_len, 0, + PCRE2_SUBSTITUTE_GLOBAL, mdata, NULL, + replace, PCRE2_ZERO_TERMINATED, + (PCRE2_UCHAR8*) output, &len) == PCRE2_ERROR_NOMEMORY) + { + char* tmp = (char*) realloc(output, (len = (size_t) (len * BUFFER_GROWTH_RATE + 1))); + if (tmp == NULL) + { + free(output); + output = NULL; + break; + } + output = tmp; + } + pcre2_match_data_free(mdata); + } + else + { + free(output); + output = NULL; + } + } + else if (output == NULL) + { + output = strdup(*src); + } + + if (output) + { + *destsize = strlen(output); + *dest = output; + } + else + { + *dest = NULL; + } + + return output; +} + +/** + * Initialize the utils library + * + * This function initializes structures used in various functions. + * @return true on success, false on error + */ +bool utils_init() +{ + bool rval = true; + + PCRE2_SIZE erroffset; + int errcode; + + ss_info_dassert(remove_comments_re == NULL, "utils_init called multiple times"); + remove_comments_re = pcre2_compile(remove_comments_pattern, PCRE2_ZERO_TERMINATED, 0, &errcode, + &erroffset, NULL); + if (remove_comments_re == NULL) + { + rval = false; + } + + ss_info_dassert(replace_quoted_re == NULL, "utils_init called multiple times"); + replace_quoted_re = pcre2_compile(replace_quoted_pattern, PCRE2_ZERO_TERMINATED, 0, &errcode, + &erroffset, NULL); + if (replace_quoted_re == NULL) + { + rval = false; + } + + ss_info_dassert(replace_values_re == NULL, "utils_init called multiple times"); + replace_values_re = pcre2_compile(replace_values_pattern, PCRE2_ZERO_TERMINATED, 0, &errcode, + &erroffset, NULL); + if (replace_values_re == NULL) + { + rval = false; + } + + return rval; +} + +/** + * Close the utils library. This should be the last call to this library. + */ +void utils_end() +{ + pcre2_code_free(remove_comments_re); + remove_comments_re = NULL; + pcre2_code_free(replace_quoted_re); + replace_quoted_re = NULL; + pcre2_code_free(replace_values_re); + replace_values_re = NULL; +} diff --git a/server/modules/filter/ccrfilter/ccrfilter.c b/server/modules/filter/ccrfilter/ccrfilter.c index e334e9f02..f9fe4f796 100644 --- a/server/modules/filter/ccrfilter/ccrfilter.c +++ b/server/modules/filter/ccrfilter/ccrfilter.c @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/filter/hintfilter/hintparser.c b/server/modules/filter/hintfilter/hintparser.c index bfc721a3a..1f3dbe79f 100644 --- a/server/modules/filter/hintfilter/hintparser.c +++ b/server/modules/filter/hintfilter/hintparser.c @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/filter/namedserverfilter/namedserverfilter.c b/server/modules/filter/namedserverfilter/namedserverfilter.c index 0dcbd662b..c30315a7f 100644 --- a/server/modules/filter/namedserverfilter/namedserverfilter.c +++ b/server/modules/filter/namedserverfilter/namedserverfilter.c @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/filter/qlafilter/qlafilter.c b/server/modules/filter/qlafilter/qlafilter.c index f1d7a9fc0..ec0690479 100644 --- a/server/modules/filter/qlafilter/qlafilter.c +++ b/server/modules/filter/qlafilter/qlafilter.c @@ -40,7 +40,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/server/modules/filter/regexfilter/regexfilter.c b/server/modules/filter/regexfilter/regexfilter.c index df84853e2..3266f129c 100644 --- a/server/modules/filter/regexfilter/regexfilter.c +++ b/server/modules/filter/regexfilter/regexfilter.c @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/filter/tee/tee.c b/server/modules/filter/tee/tee.c index 6ac656f49..304b8ae3d 100644 --- a/server/modules/filter/tee/tee.c +++ b/server/modules/filter/tee/tee.c @@ -46,7 +46,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/filter/topfilter/topfilter.c b/server/modules/filter/topfilter/topfilter.c index 4596f381c..5acbed3e8 100644 --- a/server/modules/filter/topfilter/topfilter.c +++ b/server/modules/filter/topfilter/topfilter.c @@ -37,7 +37,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/filter/tpmfilter/tpmfilter.c b/server/modules/filter/tpmfilter/tpmfilter.c index d082c89f6..97fddf10d 100644 --- a/server/modules/filter/tpmfilter/tpmfilter.c +++ b/server/modules/filter/tpmfilter/tpmfilter.c @@ -45,7 +45,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/monitor/galeramon/galeramon.h b/server/modules/monitor/galeramon/galeramon.h index 42a1c5198..967a2e52c 100644 --- a/server/modules/monitor/galeramon/galeramon.h +++ b/server/modules/monitor/galeramon/galeramon.h @@ -35,7 +35,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/monitor/mmmon/mmmon.h b/server/modules/monitor/mmmon/mmmon.h index f63aa221a..7c3a6fd9c 100644 --- a/server/modules/monitor/mmmon/mmmon.h +++ b/server/modules/monitor/mmmon/mmmon.h @@ -23,7 +23,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/monitor/mysqlmon.h b/server/modules/monitor/mysqlmon.h index 3d56ad4f6..92fc3584b 100644 --- a/server/modules/monitor/mysqlmon.h +++ b/server/modules/monitor/mysqlmon.h @@ -43,7 +43,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/monitor/ndbclustermon/ndbclustermon.h b/server/modules/monitor/ndbclustermon/ndbclustermon.h index 2b98613c4..efe557f4b 100644 --- a/server/modules/monitor/ndbclustermon/ndbclustermon.h +++ b/server/modules/monitor/ndbclustermon/ndbclustermon.h @@ -21,7 +21,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c b/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c index be8a20909..f99dfb605 100644 --- a/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c +++ b/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c @@ -12,7 +12,6 @@ */ #include -#include #include #include #include diff --git a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c index d57cbe677..519da671b 100644 --- a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c +++ b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c @@ -46,7 +46,6 @@ * 31/05/2016 Martin Brampton Implement connection throttling */ #include -#include #include #include #include diff --git a/server/modules/protocol/MySQL/mysql_common.c b/server/modules/protocol/MySQL/mysql_common.c index 16ea18d81..7d6958571 100644 --- a/server/modules/protocol/MySQL/mysql_common.c +++ b/server/modules/protocol/MySQL/mysql_common.c @@ -46,7 +46,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/protocol/maxscaled/maxscaled.c b/server/modules/protocol/maxscaled/maxscaled.c index df8ef451c..d6758604c 100644 --- a/server/modules/protocol/maxscaled/maxscaled.c +++ b/server/modules/protocol/maxscaled/maxscaled.c @@ -30,7 +30,6 @@ #include #include #include -#include #include #include #include "maxscaled.h" diff --git a/server/modules/protocol/telnetd/telnetd.c b/server/modules/protocol/telnetd/telnetd.c index f0f3a4ddd..95a3f96ea 100644 --- a/server/modules/protocol/telnetd/telnetd.c +++ b/server/modules/protocol/telnetd/telnetd.c @@ -31,7 +31,6 @@ #include #include #include -#include #include #include diff --git a/server/modules/routing/avro/avro.c b/server/modules/routing/avro/avro.c index 794cb3d26..122e3fa38 100644 --- a/server/modules/routing/avro/avro.c +++ b/server/modules/routing/avro/avro.c @@ -39,7 +39,6 @@ #include #include -#include #include #include diff --git a/server/modules/routing/avro/avro_client.c b/server/modules/routing/avro/avro_client.c index c72c34bce..cc0293b70 100644 --- a/server/modules/routing/avro/avro_client.c +++ b/server/modules/routing/avro/avro_client.c @@ -35,7 +35,6 @@ #include #include #include -#include #include #include #include "avrorouter.h" diff --git a/server/modules/routing/avro/avro_file.c b/server/modules/routing/avro/avro_file.c index fcaab652e..23ad1d904 100644 --- a/server/modules/routing/avro/avro_file.c +++ b/server/modules/routing/avro/avro_file.c @@ -40,7 +40,6 @@ #include #include #include -#include static const char *statefile_section = "avro-conversion"; static const char *ddl_list_name = "table-ddl.list"; diff --git a/server/modules/routing/avro/avro_schema.c b/server/modules/routing/avro/avro_schema.c index f4172d689..48739e4b0 100644 --- a/server/modules/routing/avro/avro_schema.c +++ b/server/modules/routing/avro/avro_schema.c @@ -27,7 +27,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/routing/binlog/blr.c b/server/modules/routing/binlog/blr.c index a781cb9bb..6d542e9a1 100644 --- a/server/modules/routing/binlog/blr.c +++ b/server/modules/routing/binlog/blr.c @@ -71,7 +71,6 @@ #include #include -#include #include #include diff --git a/server/modules/routing/binlog/blr_cache.c b/server/modules/routing/binlog/blr_cache.c index 9501f0f92..613bb35e2 100644 --- a/server/modules/routing/binlog/blr_cache.c +++ b/server/modules/routing/binlog/blr_cache.c @@ -43,7 +43,6 @@ #include #include -#include #include diff --git a/server/modules/routing/binlog/blr_file.c b/server/modules/routing/binlog/blr_file.c index 0db05df59..10a37989d 100644 --- a/server/modules/routing/binlog/blr_file.c +++ b/server/modules/routing/binlog/blr_file.c @@ -56,7 +56,6 @@ #include #include #include -#include #include #include diff --git a/server/modules/routing/binlog/blr_master.c b/server/modules/routing/binlog/blr_master.c index 7b02a66e3..c452dcefc 100644 --- a/server/modules/routing/binlog/blr_master.c +++ b/server/modules/routing/binlog/blr_master.c @@ -69,7 +69,6 @@ #include #include -#include #include #include diff --git a/server/modules/routing/binlog/blr_slave.c b/server/modules/routing/binlog/blr_slave.c index 6d4904b18..b41583c24 100644 --- a/server/modules/routing/binlog/blr_slave.c +++ b/server/modules/routing/binlog/blr_slave.c @@ -79,7 +79,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/routing/binlog/test/testbinlog.c b/server/modules/routing/binlog/test/testbinlog.c index f55d038c9..6f4b14d58 100644 --- a/server/modules/routing/binlog/test/testbinlog.c +++ b/server/modules/routing/binlog/test/testbinlog.c @@ -36,7 +36,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/routing/cli/cli.c b/server/modules/routing/cli/cli.c index 948cebc6a..c59f570a4 100644 --- a/server/modules/routing/cli/cli.c +++ b/server/modules/routing/cli/cli.c @@ -38,7 +38,6 @@ #include #include #include -#include #include diff --git a/server/modules/routing/debugcli/debugcli.c b/server/modules/routing/debugcli/debugcli.c index c360f2da4..eb1bd290f 100644 --- a/server/modules/routing/debugcli/debugcli.c +++ b/server/modules/routing/debugcli/debugcli.c @@ -37,7 +37,6 @@ #include #include #include -#include #include diff --git a/server/modules/routing/debugcli/debugcmd.c b/server/modules/routing/debugcli/debugcmd.c index b5d3d1d4c..14d25249d 100644 --- a/server/modules/routing/debugcli/debugcmd.c +++ b/server/modules/routing/debugcli/debugcmd.c @@ -72,7 +72,6 @@ #include #include -#include #include #include diff --git a/server/modules/routing/maxinfo/maxinfo.c b/server/modules/routing/maxinfo/maxinfo.c index ded66eff0..46114b2f7 100644 --- a/server/modules/routing/maxinfo/maxinfo.c +++ b/server/modules/routing/maxinfo/maxinfo.c @@ -44,7 +44,6 @@ #include #include #include "maxinfo.h" -#include #include #include #include diff --git a/server/modules/routing/maxinfo/maxinfo_error.c b/server/modules/routing/maxinfo/maxinfo_error.c index b16258a5b..aefd46f77 100644 --- a/server/modules/routing/maxinfo/maxinfo_error.c +++ b/server/modules/routing/maxinfo/maxinfo_error.c @@ -37,7 +37,6 @@ #include #include #include "maxinfo.h" -#include #include diff --git a/server/modules/routing/maxinfo/maxinfo_exec.c b/server/modules/routing/maxinfo/maxinfo_exec.c index d3c84ab0b..c5b031b94 100644 --- a/server/modules/routing/maxinfo/maxinfo_exec.c +++ b/server/modules/routing/maxinfo/maxinfo_exec.c @@ -41,7 +41,6 @@ #include #include #include "maxinfo.h" -#include #include #include #include diff --git a/server/modules/routing/maxinfo/maxinfo_parse.c b/server/modules/routing/maxinfo/maxinfo_parse.c index ccf3aea7a..e75d65e92 100644 --- a/server/modules/routing/maxinfo/maxinfo_parse.c +++ b/server/modules/routing/maxinfo/maxinfo_parse.c @@ -40,7 +40,6 @@ #include #include #include "maxinfo.h" -#include #include static MAXINFO_TREE *make_tree_node(MAXINFO_OPERATOR, char *, MAXINFO_TREE *, MAXINFO_TREE *); diff --git a/server/modules/routing/readconnroute/readconnroute.c b/server/modules/routing/readconnroute/readconnroute.c index bbb7d55b6..f95a24cf0 100644 --- a/server/modules/routing/readconnroute/readconnroute.c +++ b/server/modules/routing/readconnroute/readconnroute.c @@ -84,7 +84,6 @@ #include #include -#include #include #include diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index 7d9b8f3e4..8ad2c34d5 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -20,7 +20,6 @@ #include "readwritesplit.h" #include "rwsplit_internal.h" -#include #include #include #include diff --git a/server/modules/routing/readwritesplit/rwsplit_mysql.c b/server/modules/routing/readwritesplit/rwsplit_mysql.c index aa7050dcd..1025f44a9 100644 --- a/server/modules/routing/readwritesplit/rwsplit_mysql.c +++ b/server/modules/routing/readwritesplit/rwsplit_mysql.c @@ -22,7 +22,6 @@ #include "rwsplit_internal.h" #include -#include #include #include #include diff --git a/server/modules/routing/schemarouter/schemarouter.c b/server/modules/routing/schemarouter/schemarouter.c index 551841b5a..95e544a3c 100644 --- a/server/modules/routing/schemarouter/schemarouter.c +++ b/server/modules/routing/schemarouter/schemarouter.c @@ -21,7 +21,6 @@ #include "sharding_common.h" #include #include -#include #include #include #include From d5cf74bd24ed000df6aab90af44d9e432aee8e63 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Sat, 15 Oct 2016 11:14:11 +0300 Subject: [PATCH 3/6] Cleanup gw.h, part 1. Gw.h contained a fair amount of obsolete function declarations, duplicate declarations of functions declared in utils.h and declarations of functions that conceptually are similar to those in utils.h. The obsolete and duplicate ones were removed and all but one of the remaining moved to utils.h. Correspondingly the implementation was moved from gw_utils.c to utils.c. The one remaining function - gw_daemonize() - is not really worthy of a file of its own, so that is to be moved to gateway.c after which gw_utils.c can be removed. Gw.h still contains defines that are duplicated in maxscale/protocol/mysql.h. The ones in gw.h are to be removed. It appears that the entire gw.h will disappear. --- include/maxscale/cdefs.h | 1 + include/maxscale/gw.h | 16 ---- include/maxscale/utils.h | 11 +++ server/core/dcb.c | 1 + server/core/gw_utils.c | 167 ----------------------------------- server/core/gwdirs.c | 2 +- server/core/poll.c | 1 + server/core/utils.c | 185 +++++++++++++++++++++++++++++++++++++-- 8 files changed, 193 insertions(+), 191 deletions(-) diff --git a/include/maxscale/cdefs.h b/include/maxscale/cdefs.h index 62c28b328..a26342939 100644 --- a/include/maxscale/cdefs.h +++ b/include/maxscale/cdefs.h @@ -72,5 +72,6 @@ */ #include #include +#include #endif diff --git a/include/maxscale/gw.h b/include/maxscale/gw.h index 085c73e58..5e81ba2fe 100644 --- a/include/maxscale/gw.h +++ b/include/maxscale/gw.h @@ -69,23 +69,7 @@ MXS_BEGIN_DECLS #define MYSQL_CONN_DEBUG #undef MYSQL_CONN_DEBUG -#include "dcb.h" - bool gw_daemonize(void); -int do_read_dcb(DCB *dcb); -void MySQLListener(int epfd, char *config_bind); -int MySQLAccept(DCB *listener); -int do_read_dcb(DCB *dcb); -int do_read_10(DCB *dcb, uint8_t *buffer); -int MySQLWrite(DCB *dcb, GWBUF *queue); -int setnonblocking(int fd); -int gw_getsockerrno(int fd); -int parse_bindconfig(const char *, struct sockaddr_in *); -int setipaddress(struct in_addr *, char *); -char* get_libdir(); -long get_processor_count(); -void clean_up_pathname(char *path); -bool mxs_mkdir_all(const char *path, int mask); MXS_END_DECLS diff --git a/include/maxscale/utils.h b/include/maxscale/utils.h index 1f608296d..55ce9a34e 100644 --- a/include/maxscale/utils.h +++ b/include/maxscale/utils.h @@ -27,8 +27,10 @@ */ #include +#include #include #include +#include MXS_BEGIN_DECLS @@ -41,6 +43,9 @@ bool utils_init(); /*< Call this first before using any other function */ void utils_end(); int setnonblocking(int fd); +int parse_bindconfig(const char *, struct sockaddr_in *); +int setipaddress(struct in_addr *, char *); + char *gw_strend(register const char *s); static char gw_randomchar(); int gw_generate_random_str(char *output, int len); @@ -67,6 +72,12 @@ char* replace_literal(char* haystack, const char* replacement); char* replace_quoted(const char** src, const size_t* srcsize, char** dest, size_t* destsize); +void clean_up_pathname(char *path); + +bool mxs_mkdir_all(const char *path, int mask); + +long get_processor_count(); + MXS_END_DECLS #endif diff --git a/server/core/dcb.c b/server/core/dcb.c index 065213cec..675078798 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -90,6 +90,7 @@ #include #include #include +#include #if defined(FAKE_CODE) unsigned char dcb_fake_write_errno[10240]; diff --git a/server/core/gw_utils.c b/server/core/gw_utils.c index 2ef05638d..60cc9bed5 100644 --- a/server/core/gw_utils.c +++ b/server/core/gw_utils.c @@ -39,96 +39,6 @@ #include #include -SPINLOCK tmplock = SPINLOCK_INIT; - -/* - * Set IP address in socket structure in_addr - * - * @param a Pointer to a struct in_addr into which the address is written - * @param p The hostname to lookup - * @return 1 on success, 0 on failure - */ -int -setipaddress(struct in_addr *a, char *p) -{ -#ifdef __USE_POSIX - struct addrinfo *ai = NULL, hint; - int rc; - struct sockaddr_in *res_addr; - memset(&hint, 0, sizeof (hint)); - - hint.ai_socktype = SOCK_STREAM; - - /* - * This is for the listening socket, matching INADDR_ANY only for now. - * For future specific addresses bind, a dedicated routine woulbd be better - */ - - if (strcmp(p, "0.0.0.0") == 0) - { - hint.ai_flags = AI_PASSIVE; - hint.ai_family = AF_UNSPEC; - if ((rc = getaddrinfo(p, NULL, &hint, &ai)) != 0) - { - MXS_ERROR("Failed to obtain address for host %s, %s", - p, - gai_strerror(rc)); - - return 0; - } - } - else - { - hint.ai_flags = AI_CANONNAME; - hint.ai_family = AF_INET; - - if ((rc = getaddrinfo(p, NULL, &hint, &ai)) != 0) - { - MXS_ERROR("Failed to obtain address for host %s, %s", - p, - gai_strerror(rc)); - - return 0; - } - } - - /* take the first one */ - if (ai != NULL) - { - res_addr = (struct sockaddr_in *)(ai->ai_addr); - memcpy(a, &res_addr->sin_addr, sizeof(struct in_addr)); - - freeaddrinfo(ai); - - return 1; - } -#else - struct hostent *h; - - spinlock_acquire(&tmplock); - h = gethostbyname(p); - spinlock_release(&tmplock); - - if (h == NULL) - { - if ((a->s_addr = inet_addr(p)) == -1) - { - MXS_ERROR("gethostbyname failed for [%s]", p); - - return 0; - } - } - else - { - /* take the first one */ - memcpy(a, h->h_addr, h->h_length); - - return 1; - } -#endif - return 0; -} - /** * Daemonize the process by forking and putting the process into the * background. @@ -160,80 +70,3 @@ bool gw_daemonize(void) } return false; } - -/** - * Parse the bind config data. This is passed in a string as address:port. - * - * The address may be either a . separated IP address or a hostname to - * lookup. The address 0.0.0.0 is the wildcard address for SOCKADR_ANY. - * The ':' and port are required. - * - * @param config The bind address and port separated by a ':' - * @param addr The sockaddr_in in which the data is written - * @return 0 on failure - */ -int -parse_bindconfig(const char *config, struct sockaddr_in *addr) -{ - char buf[strlen(config) + 1]; - strcpy(buf, config); - - char *port = strrchr(buf, ':'); - short pnum; - if (port) - { - *port = 0; - port++; - pnum = atoi(port); - } - else - { - return 0; - } - - if (!strcmp(buf, "0.0.0.0")) - { - addr->sin_addr.s_addr = htonl(INADDR_ANY); - } - else - { - if (!inet_aton(buf, &addr->sin_addr)) - { - struct hostent *hp = gethostbyname(buf); - - if (hp) - { - bcopy(hp->h_addr, &(addr->sin_addr.s_addr), hp->h_length); - } - else - { - MXS_ERROR("Failed to lookup host '%s'.", buf); - return 0; - } - } - } - - addr->sin_family = AF_INET; - addr->sin_port = htons(pnum); - return 1; -} - -/** - * Return the number of processors available. - * @return Number of processors or 1 if the required definition of _SC_NPROCESSORS_CONF - * is not found - */ -long get_processor_count() -{ - long processors = 1; -#ifdef _SC_NPROCESSORS_ONLN - if ((processors = sysconf(_SC_NPROCESSORS_ONLN)) <= 0) - { - MXS_WARNING("Unable to establish the number of available cores. Defaulting to 1."); - processors = 1; - } -#else -#error _SC_NPROCESSORS_ONLN not available. -#endif - return processors; -} diff --git a/server/core/gwdirs.c b/server/core/gwdirs.c index d7c54be0f..eb8ca6394 100644 --- a/server/core/gwdirs.c +++ b/server/core/gwdirs.c @@ -13,7 +13,7 @@ #include #include -#include +#include /** * Set the configuration file directory diff --git a/server/core/poll.c b/server/core/poll.c index 149b67fdb..c32661f68 100644 --- a/server/core/poll.c +++ b/server/core/poll.c @@ -33,6 +33,7 @@ #include #include #include +#include #define PROFILE_POLL 0 diff --git a/server/core/utils.c b/server/core/utils.c index ead727b8e..0688da6c8 100644 --- a/server/core/utils.c +++ b/server/core/utils.c @@ -28,18 +28,21 @@ * @endverbatim */ - +#include +#include +#include #include -#include -#include -#include +#include +#include #include #include -#include +#include #include -#include -#include #include +#include +#include +#include +#include #if !defined(PATH_MAX) # if defined(__USE_POSIX) @@ -880,3 +883,171 @@ void utils_end() pcre2_code_free(replace_values_re); replace_values_re = NULL; } + +SPINLOCK tmplock = SPINLOCK_INIT; + +/* + * Set IP address in socket structure in_addr + * + * @param a Pointer to a struct in_addr into which the address is written + * @param p The hostname to lookup + * @return 1 on success, 0 on failure + */ +int +setipaddress(struct in_addr *a, char *p) +{ +#ifdef __USE_POSIX + struct addrinfo *ai = NULL, hint; + int rc; + struct sockaddr_in *res_addr; + memset(&hint, 0, sizeof (hint)); + + hint.ai_socktype = SOCK_STREAM; + + /* + * This is for the listening socket, matching INADDR_ANY only for now. + * For future specific addresses bind, a dedicated routine woulbd be better + */ + + if (strcmp(p, "0.0.0.0") == 0) + { + hint.ai_flags = AI_PASSIVE; + hint.ai_family = AF_UNSPEC; + if ((rc = getaddrinfo(p, NULL, &hint, &ai)) != 0) + { + MXS_ERROR("Failed to obtain address for host %s, %s", + p, + gai_strerror(rc)); + + return 0; + } + } + else + { + hint.ai_flags = AI_CANONNAME; + hint.ai_family = AF_INET; + + if ((rc = getaddrinfo(p, NULL, &hint, &ai)) != 0) + { + MXS_ERROR("Failed to obtain address for host %s, %s", + p, + gai_strerror(rc)); + + return 0; + } + } + + /* take the first one */ + if (ai != NULL) + { + res_addr = (struct sockaddr_in *)(ai->ai_addr); + memcpy(a, &res_addr->sin_addr, sizeof(struct in_addr)); + + freeaddrinfo(ai); + + return 1; + } +#else + struct hostent *h; + + spinlock_acquire(&tmplock); + h = gethostbyname(p); + spinlock_release(&tmplock); + + if (h == NULL) + { + if ((a->s_addr = inet_addr(p)) == -1) + { + MXS_ERROR("gethostbyname failed for [%s]", p); + + return 0; + } + } + else + { + /* take the first one */ + memcpy(a, h->h_addr, h->h_length); + + return 1; + } +#endif + return 0; +} + + +/** + * Parse the bind config data. This is passed in a string as address:port. + * + * The address may be either a . separated IP address or a hostname to + * lookup. The address 0.0.0.0 is the wildcard address for SOCKADR_ANY. + * The ':' and port are required. + * + * @param config The bind address and port separated by a ':' + * @param addr The sockaddr_in in which the data is written + * @return 0 on failure + */ +int +parse_bindconfig(const char *config, struct sockaddr_in *addr) +{ + char buf[strlen(config) + 1]; + strcpy(buf, config); + + char *port = strrchr(buf, ':'); + short pnum; + if (port) + { + *port = 0; + port++; + pnum = atoi(port); + } + else + { + return 0; + } + + if (!strcmp(buf, "0.0.0.0")) + { + addr->sin_addr.s_addr = htonl(INADDR_ANY); + } + else + { + if (!inet_aton(buf, &addr->sin_addr)) + { + struct hostent *hp = gethostbyname(buf); + + if (hp) + { + bcopy(hp->h_addr, &(addr->sin_addr.s_addr), hp->h_length); + } + else + { + MXS_ERROR("Failed to lookup host '%s'.", buf); + return 0; + } + } + } + + addr->sin_family = AF_INET; + addr->sin_port = htons(pnum); + return 1; +} + +/** + * Return the number of processors available. + * @return Number of processors or 1 if the required definition of _SC_NPROCESSORS_CONF + * is not found + */ +long get_processor_count() +{ + long processors = 1; +#ifdef _SC_NPROCESSORS_ONLN + if ((processors = sysconf(_SC_NPROCESSORS_ONLN)) <= 0) + { + MXS_WARNING("Unable to establish the number of available cores. Defaulting to 1."); + processors = 1; + } +#else +#error _SC_NPROCESSORS_ONLN not available. +#endif + return processors; +} From 4cb6d9227e1e95fa3991b11159a9b1ed0e0ac145 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Mon, 17 Oct 2016 07:20:35 +0300 Subject: [PATCH 4/6] MXS-778: Set read-only masters into Slave status When a master server is set into read-only mode, it can be treated as an always up-to-date slave. This gives us a somewhat graceful way to prevent writes to a master. Usually setting the master into read-only mode is done before a change in the replication topology to prevent stray writes arriving on the master. MaxScale should respect the read-only mode and not send any writes to a server that's in read-only mode. --- server/modules/monitor/mysqlmon/mysql_mon.c | 30 ++++++++++++++++----- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/server/modules/monitor/mysqlmon/mysql_mon.c b/server/modules/monitor/mysqlmon/mysql_mon.c index 901238f44..86967962d 100644 --- a/server/modules/monitor/mysqlmon/mysql_mon.c +++ b/server/modules/monitor/mysqlmon/mysql_mon.c @@ -49,6 +49,7 @@ #include #include #include +#include /** Column positions for SHOW SLAVE STATUS */ #define MYSQL55_STATUS_BINLOG_POS 5 @@ -1032,8 +1033,17 @@ void find_graph_cycles(MYSQL_MONITOR *handle, MONITOR_SERVERS *database, int nse * slave in this case can be either a normal slave or another * master. */ - monitor_set_pending_status(graph[i].db, SERVER_MASTER | SERVER_STALE_STATUS); - monitor_clear_pending_status(graph[i].db, SERVER_SLAVE); + if (graph[i].info->read_only) + { + /** The master is in read-only mode, set it into Slave state */ + monitor_set_pending_status(graph[i].db, SERVER_SLAVE); + monitor_clear_pending_status(graph[i].db, SERVER_MASTER | SERVER_STALE_STATUS); + } + else + { + monitor_set_pending_status(graph[i].db, SERVER_MASTER | SERVER_STALE_STATUS); + monitor_clear_pending_status(graph[i].db, SERVER_SLAVE); + } } } } @@ -1315,10 +1325,11 @@ monitorMain(void *arg) ptr = mon->databases; while (ptr) { - MYSQL_SERVER_INFO *serv_info = hashtable_fetch(handle->server_info, ptr->server->unique_name); - ss_dassert(serv_info); if (!SERVER_IN_MAINT(ptr->server)) { + MYSQL_SERVER_INFO *serv_info = hashtable_fetch(handle->server_info, ptr->server->unique_name); + ss_dassert(serv_info); + /** If "detect_stale_master" option is On, let's use the previous master. * * Multi-master mode detects the stale masters in find_graph_cycles(). @@ -1327,7 +1338,8 @@ monitorMain(void *arg) (strcmp(ptr->server->name, root_master->server->name) == 0 && ptr->server->port == root_master->server->port) && (ptr->server->status & SERVER_MASTER) && - !(ptr->pending_status & SERVER_MASTER)) + !(ptr->pending_status & SERVER_MASTER) && + !serv_info->read_only) { /** * In this case server->status will not be updated from pending_status @@ -1877,7 +1889,13 @@ static MONITOR_SERVERS *get_replication_tree(MONITOR *mon, int num_servers) monitor_clear_pending_status(handle->master, SERVER_MASTER); } - monitor_set_pending_status(master, SERVER_MASTER); + MYSQL_SERVER_INFO* info = hashtable_fetch(handle->server_info, + master->server->unique_name); + ss_dassert(info); + + /** Only set the Master status if read_only is disabled */ + monitor_set_pending_status(master, info->read_only ? SERVER_SLAVE : SERVER_MASTER); + handle->master = master; } else From 4faa7f8d21e25f8c2b7e38c547e501510e18c91f Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Mon, 17 Oct 2016 09:47:46 +0300 Subject: [PATCH 5/6] Obsolete defines removed --- include/maxscale/gw.h | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/include/maxscale/gw.h b/include/maxscale/gw.h index 5e81ba2fe..eb5ecb816 100644 --- a/include/maxscale/gw.h +++ b/include/maxscale/gw.h @@ -38,8 +38,6 @@ MXS_BEGIN_DECLS -#define EXIT_FAILURE 1 - // network buffer is 32K #define MAX_BUFFER_SIZE 32768 @@ -53,21 +51,6 @@ MXS_BEGIN_DECLS #define GW_CLIENT_SO_RCVBUF (128 * 1024) #define GW_NOINTR_CALL(A) do { errno = 0; A; } while (errno == EINTR) -#define GW_MYSQL_LOOP_TIMEOUT 300000000 -#define GW_MYSQL_READ 0 -#define GW_MYSQL_WRITE 1 - -#define GW_MYSQL_PROTOCOL_VERSION 10 // version is 10 -#define GW_MYSQL_HANDSHAKE_FILLER 0x00 -#define GW_MYSQL_SERVER_CAPABILITIES_BYTE1 0xff -#define GW_MYSQL_SERVER_CAPABILITIES_BYTE2 0xf7 -#define GW_MYSQL_SERVER_LANGUAGE 0x08 -#define GW_MYSQL_MAX_PACKET_LEN 0xffffffL; -#define GW_MYSQL_SCRAMBLE_SIZE 20 - -// debug for mysql_* functions -#define MYSQL_CONN_DEBUG -#undef MYSQL_CONN_DEBUG bool gw_daemonize(void); From 8aafa34d667be609619678d2c028aa0a35fcd341 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Mon, 17 Oct 2016 09:58:45 +0300 Subject: [PATCH 6/6] Hard limits moved to limits.h Limits used with SO_[RCV|SEND]BUF moved from gw.h to limits.h and renamed so that the prefix is MXS_. --- include/maxscale/gw.h | 9 ---- include/maxscale/limits.h | 47 +++++++++++++++++-- server/core/dcb.c | 4 +- .../MySQL/MySQLBackend/mysql_backend.c | 4 +- 4 files changed, 46 insertions(+), 18 deletions(-) diff --git a/include/maxscale/gw.h b/include/maxscale/gw.h index eb5ecb816..a560342fa 100644 --- a/include/maxscale/gw.h +++ b/include/maxscale/gw.h @@ -41,15 +41,6 @@ MXS_BEGIN_DECLS // network buffer is 32K #define MAX_BUFFER_SIZE 32768 -/** - * Configuration for send and receive socket buffer sizes for - * backend and cleint connections. - */ -#define GW_BACKEND_SO_SNDBUF (128 * 1024) -#define GW_BACKEND_SO_RCVBUF (128 * 1024) -#define GW_CLIENT_SO_SNDBUF (128 * 1024) -#define GW_CLIENT_SO_RCVBUF (128 * 1024) - #define GW_NOINTR_CALL(A) do { errno = 0; A; } while (errno == EINTR) bool gw_daemonize(void); diff --git a/include/maxscale/limits.h b/include/maxscale/limits.h index 6847683d9..ea3bc874c 100644 --- a/include/maxscale/limits.h +++ b/include/maxscale/limits.h @@ -18,12 +18,49 @@ MXS_BEGIN_DECLS -// This file defines hard limits of MaxScale. +/** + * @file lmits.h + * + * This file contains defines for hard limits of MaxScale. + */ -// Thread information is stored in a bitmask whose size must be a -// multiple of 8. The bitmask is indexed using the thread id that start -// from 1. Hence, the hard maximum number of threads must be a -// multiple of 8 minus 1. + +/** + * MXS_BACKEND_SO_RCVBUF + * + * The value used when setting SO_RCVBUF of backend sockets. + */ +#define MXS_BACKEND_SO_RCVBUF (128 * 1024) + +/** + * MSX_BACKEND_SO_SNDBUF + * + * The value used when setting SO_SNDBUF of backend sockets. + */ +#define MXS_BACKEND_SO_SNDBUF (128 * 1024) + +/** + * MXS_CLIENT_SO_RCVBUF + * + * The value used when setting SO_RCVBUF of client sockets. + */ +#define MXS_CLIENT_SO_RCVBUF (128 * 1024) + +/** + * MXS_CLIENT_SO_SNDBUF + * + * The value used when setting SO_SNDBUF of client sockets. + */ +#define MXS_CLIENT_SO_SNDBUF (128 * 1024) + +/** + * MXS_MAX_THREADS + * + * Thread information is stored in a bitmask whose size must be a + * multiple of 8. The bitmask is indexed using the thread id that start + * from 1. Hence, the hard maximum number of threads must be a + * multiple of 8 minus 1. + */ #define MXS_MAX_THREADS 255 MXS_END_DECLS diff --git a/server/core/dcb.c b/server/core/dcb.c index 675078798..9c9c74746 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -3101,7 +3101,7 @@ dcb_accept(DCB *listener, GWPROTOCOL *protocol_funcs) conn_open[c_sock] = true; #endif /* FAKE_CODE */ /* set nonblocking */ - sendbuf = GW_CLIENT_SO_SNDBUF; + sendbuf = MXS_CLIENT_SO_SNDBUF; if (setsockopt(c_sock, SOL_SOCKET, SO_SNDBUF, &sendbuf, optlen) != 0) { @@ -3109,7 +3109,7 @@ dcb_accept(DCB *listener, GWPROTOCOL *protocol_funcs) errno, strerror_r(errno, errbuf, sizeof(errbuf))); } - sendbuf = GW_CLIENT_SO_RCVBUF; + sendbuf = MXS_CLIENT_SO_RCVBUF; if (setsockopt(c_sock, SOL_SOCKET, SO_RCVBUF, &sendbuf, optlen) != 0) { diff --git a/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c b/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c index f99dfb605..c9ba4d89e 100644 --- a/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c +++ b/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c @@ -323,7 +323,7 @@ gw_do_connect_to_backend(char *host, int port, int *fd) /* prepare for connect */ setipaddress(&serv_addr.sin_addr, host); serv_addr.sin_port = htons(port); - bufsize = GW_BACKEND_SO_SNDBUF; + bufsize = MXS_BACKEND_SO_SNDBUF; if (setsockopt(so, SOL_SOCKET, SO_SNDBUF, &bufsize, sizeof(bufsize)) != 0) { @@ -340,7 +340,7 @@ gw_do_connect_to_backend(char *host, int port, int *fd) close_socket(so); goto return_rv; } - bufsize = GW_BACKEND_SO_RCVBUF; + bufsize = MXS_BACKEND_SO_RCVBUF; if (setsockopt(so, SOL_SOCKET, SO_RCVBUF, &bufsize, sizeof(bufsize)) != 0) {