From 53f01106aa0837eb7e9572a85dae588f860d5ab8 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 29 Sep 2016 23:05:55 +0300 Subject: [PATCH 001/215] Always try to build CDC modules The CDC protocol, authentication and router modules should be build if the required libraries are found. --- cmake/defaults.cmake | 2 +- server/modules/protocol/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/defaults.cmake b/cmake/defaults.cmake index 14ed66080..907bd099a 100644 --- a/cmake/defaults.cmake +++ b/cmake/defaults.cmake @@ -22,7 +22,7 @@ set(BUILD_RABBITMQ TRUE CACHE BOOL "Build RabbitMQ components") set(BUILD_BINLOG TRUE CACHE BOOL "Build binlog router") # Build the Avro router -set(BUILD_AVRO FALSE CACHE BOOL "Build Avro router") +set(BUILD_AVRO TRUE CACHE BOOL "Build Avro router") # Build the multimaster monitor set(BUILD_MMMON TRUE CACHE BOOL "Build multimaster monitor") diff --git a/server/modules/protocol/CMakeLists.txt b/server/modules/protocol/CMakeLists.txt index 423ab83df..a6f3e6a46 100644 --- a/server/modules/protocol/CMakeLists.txt +++ b/server/modules/protocol/CMakeLists.txt @@ -1,4 +1,4 @@ -if(BUILD_CDC) +if(BUILD_AVRO) add_subdirectory(CDC) add_subdirectory(examples) endif() From cd11971d5de3db70832f02f8bc121351928f87b1 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Sun, 9 Oct 2016 10:00:58 +0300 Subject: [PATCH 002/215] Log more precise DCB write error messages The DCB error messages now log the type of the DCB and the remote address in addition to the system error message. The file descriptor and memory address are no longer printed in the error message as they are not useful to the end user. The fd and address are now logged at debug level with a more verbose error message. --- server/core/dcb.c | 63 +++---------------- server/include/dcb.h | 5 ++ .../protocol/MySQLBackend/mysql_backend.c | 9 --- 3 files changed, 12 insertions(+), 65 deletions(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index e6ef6baf8..8db2db5e2 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -2412,55 +2412,6 @@ gw_write(DCB *dcb, GWBUF *writeq, bool *stop_writing) } #endif /* FAKE_CODE */ -#if defined(SS_DEBUG_MYSQL) - { - size_t len; - uint8_t* packet = (uint8_t *)buf; - char* str; - - /** Print only MySQL packets */ - if (written > 5) - { - str = (char *)&packet[5]; - len = packet[0]; - len += 256 * packet[1]; - len += 256 * 256 * packet[2]; - - if (strncmp(str, "insert", 6) == 0 || - strncmp(str, "create", 6) == 0 || - strncmp(str, "drop", 4) == 0) - { - ss_dassert((dcb->dcb_server_status & (SERVER_RUNNING | SERVER_MASTER | SERVER_SLAVE)) == - (SERVER_RUNNING | SERVER_MASTER)); - } - - if (strncmp(str, "set autocommit", 14) == 0 && nbytes > 17) - { - char* s = (char *)MXS_CALLOC(1, nbytes + 1); - MXS_ABORT_IF_NULL(s); - - if (nbytes - 5 > len) - { - size_t len2 = packet[4 + len]; - len2 += 256 * packet[4 + len + 1]; - len2 += 256 * 256 * packet[4 + len + 2]; - - char* str2 = (char *)&packet[4 + len + 5]; - snprintf(s, 5 + len + len2, "long %s %s", (char *)str, (char *)str2); - } - else - { - snprintf(s, len, "%s", (char *)str); - } - MXS_INFO("%lu [gw_write] Wrote %d bytes : %s ", - pthread_self(), - w, - s); - MXS_FREE(s); - } - } - } -#endif saved_errno = errno; errno = 0; @@ -2477,13 +2428,13 @@ gw_write(DCB *dcb, GWBUF *writeq, bool *stop_writing) #endif { char errbuf[STRERROR_BUFLEN]; - MXS_ERROR("Write to dcb %p " - "in state %s fd %d failed due errno %d, %s", - dcb, - STRDCBSTATE(dcb->state), - dcb->fd, - saved_errno, - strerror_r(saved_errno, errbuf, sizeof(errbuf))); + MXS_ERROR("Write to %s %s in state %s failed due errno %d, %s", + DCB_STRTYPE(dcb), dcb->remote, STRDCBSTATE(dcb->state), + saved_errno, strerror_r(saved_errno, errbuf, sizeof(errbuf))); + MXS_DEBUG("Write to %s %s in state %s failed due errno %d, %s (at %p, fd %d)", + DCB_STRTYPE(dcb), dcb->remote, STRDCBSTATE(dcb->state), + saved_errno, strerror_r(saved_errno, errbuf, sizeof(errbuf)), + dcb, dcb->fd); } } else diff --git a/server/include/dcb.h b/server/include/dcb.h index 2fbabd191..6da199e85 100644 --- a/server/include/dcb.h +++ b/server/include/dcb.h @@ -156,6 +156,11 @@ typedef enum DCB_ROLE_INTERNAL /*< Internal DCB not connected to the outside */ } dcb_role_t; +#define DCB_STRTYPE(dcb) (dcb->dcb_role == DCB_ROLE_CLIENT_HANDLER ? "Client DCB" : \ + dcb->dcb_role == DCB_ROLE_BACKEND_HANDLER ? "Backend DCB" : \ + dcb->dcb_role == DCB_ROLE_SERVICE_LISTENER ? "Listener DCB" : \ + dcb->dcb_role == DCB_ROLE_INTERNAL ? "Internal DCB" : "Unknown DCB") + /** * Callback reasons for the DCB callback mechanism. */ diff --git a/server/modules/protocol/MySQLBackend/mysql_backend.c b/server/modules/protocol/MySQLBackend/mysql_backend.c index 456d4b500..76d52ce2d 100644 --- a/server/modules/protocol/MySQLBackend/mysql_backend.c +++ b/server/modules/protocol/MySQLBackend/mysql_backend.c @@ -1549,12 +1549,6 @@ static int gw_backend_hangup(DCB *dcb) /* dcb_close(dcb); */ goto retblock; } -#if defined(SS_DEBUG) - if (ses_state != SESSION_STATE_STOPPING) - { - MXS_ERROR("Backend hangup error handling."); - } -#endif router->handleError(router_instance, rsession, @@ -1567,9 +1561,6 @@ static int gw_backend_hangup(DCB *dcb) /** There are no required backends available, close session. */ if (!succp) { -#if defined(SS_DEBUG) - MXS_ERROR("Backend hangup -> closing session."); -#endif spinlock_acquire(&session->ses_lock); session->state = SESSION_STATE_STOPPING; spinlock_release(&session->ses_lock); From 35d9b35609cda24cf79b784b44f045971d6290e8 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 22 Sep 2016 09:42:05 +0300 Subject: [PATCH 003/215] MXS-862: Refactor backend authentication handling The backend responses are now read in one place and the functions just read the data. The protocol level will now handle the packet gathering process and the authentication part just inspects the data. Backend connections now load authenticators when they are being connected. In the future, this enables the use of authentication modules for backend connection. --- server/core/config.c | 7 + server/core/dcb.c | 20 +- server/core/server.c | 1 + server/include/gw_authenticator.h | 23 + server/include/server.h | 1 + server/include/skygw_debug.h | 16 +- .../include/mysql_client_server_protocol.h | 32 +- .../protocol/MySQLBackend/mysql_backend.c | 1245 +++++++---------- .../protocol/MySQLClient/mysql_client.c | 18 +- server/modules/protocol/mysql_common.c | 79 +- 10 files changed, 599 insertions(+), 843 deletions(-) diff --git a/server/core/config.c b/server/core/config.c index df8fb44c7..82331c901 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -196,6 +196,7 @@ static char *server_params[] = "protocol", "port", "address", + "authenticator", "monitoruser", "monitorpw", "persistpoolmax", @@ -2504,6 +2505,7 @@ int create_new_server(CONFIG_CONTEXT *obj) char *protocol = config_get_value(obj->parameters, "protocol"); char *monuser = config_get_value(obj->parameters, "monitoruser"); char *monpw = config_get_value(obj->parameters, "monitorpw"); + char *auth = config_get_value(obj->parameters, "authenticator"); if (address && port && protocol) { @@ -2540,6 +2542,11 @@ int create_new_server(CONFIG_CONTEXT *obj) error_count++; } + if (auth && (server->authenticator = MXS_STRDUP(auth)) == NULL) + { + error_count++; + } + char *endptr; const char *poolmax = config_get_value_string(obj->parameters, "persistpoolmax"); if (poolmax) diff --git a/server/core/dcb.c b/server/core/dcb.c index 8db2db5e2..7e1384b33 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -793,6 +793,22 @@ dcb_connect(SERVER *server, SESSION *session, const char *protocol) memcpy(&(dcb->func), funcs, sizeof(GWPROTOCOL)); dcb->protoname = MXS_STRDUP_A(protocol); + const char *authenticator = server->authenticator ? + server->authenticator : dcb->func.auth_default ? + dcb->func.auth_default() : "NullAuthDeny"; + + GWAUTHENTICATOR *authfuncs = (GWAUTHENTICATOR*)load_module(authenticator, + MODULE_AUTHENTICATOR); + if (authfuncs == NULL) + { + + MXS_ERROR("Failed to load authenticator module '%s'.", authenticator); + dcb_close(dcb); + return NULL; + } + + memcpy(&dcb->authfunc, authfuncs, sizeof(GWAUTHENTICATOR)); + /** * Link dcb to session. Unlink is called in dcb_final_free */ @@ -3096,7 +3112,7 @@ dcb_accept(DCB *listener, GWPROTOCOL *protocol_funcs) } else { - const char *authenticator_name = "NullAuth"; + const char *authenticator_name = "NullAuthDeny"; GWAUTHENTICATOR *authfuncs; client_dcb->service = listener->session->service; @@ -3140,7 +3156,7 @@ dcb_accept(DCB *listener, GWPROTOCOL *protocol_funcs) if ((authfuncs = (GWAUTHENTICATOR *)load_module(authenticator_name, MODULE_AUTHENTICATOR)) == NULL) { - if ((authfuncs = (GWAUTHENTICATOR *)load_module("NullAuth", + if ((authfuncs = (GWAUTHENTICATOR *)load_module("NullAuthDeny", MODULE_AUTHENTICATOR)) == NULL) { MXS_ERROR("Failed to load authenticator module for %s, free dcb %p\n", diff --git a/server/core/server.c b/server/core/server.c index 93b15c875..9e0835e09 100644 --- a/server/core/server.c +++ b/server/core/server.c @@ -83,6 +83,7 @@ server_alloc(char *servname, char *protocol, unsigned short port) #endif server->name = servname; server->protocol = protocol; + server->authenticator = NULL; server->port = port; server->status = SERVER_RUNNING; server->node_id = -1; diff --git a/server/include/gw_authenticator.h b/server/include/gw_authenticator.h index a0cae4ce7..5f5ab04e4 100644 --- a/server/include/gw_authenticator.h +++ b/server/include/gw_authenticator.h @@ -77,6 +77,29 @@ typedef struct gw_authenticator #define MXS_AUTH_LOADUSERS_OK 0 /**< Users loaded successfully */ #define MXS_AUTH_LOADUSERS_ERROR 1 /**< Failed to load users */ +/** + * Authentication states + * + * The state usually goes from INIT to CONNECTED and alternates between + * MESSAGE_READ and RESPONSE_SENT until ending up in either FAILED or COMPLETE. + * + * If the server immediately rejects the connection, the state ends up in + * HANDSHAKE_FAILED. If the connection creation would block, instead of going to + * the CONNECTED state, the connection will be in PENDING_CONNECT state until + * the connection can be created. + */ +typedef enum +{ + MXS_AUTH_STATE_INIT, /**< Initial authentication state */ + MXS_AUTH_STATE_PENDING_CONNECT,/**< Connection creation is underway */ + MXS_AUTH_STATE_CONNECTED, /**< Network connection to server created */ + MXS_AUTH_STATE_MESSAGE_READ, /**< Read a authentication message from the server */ + MXS_AUTH_STATE_RESPONSE_SENT, /**< Responded to the read authentication message */ + MXS_AUTH_STATE_FAILED, /**< Authentication failed */ + MXS_AUTH_STATE_HANDSHAKE_FAILED, /**< Authentication failed immediately */ + MXS_AUTH_STATE_COMPLETE /**< Authentication is complete */ +} mxs_auth_state_t; + /** * The GWAUTHENTICATOR version data. The following should be updated whenever * the GWAUTHENTICATOR structure is changed. See the rules defined in modinfo.h diff --git a/server/include/server.h b/server/include/server.h index 1e634c5f1..f708b84ec 100644 --- a/server/include/server.h +++ b/server/include/server.h @@ -86,6 +86,7 @@ typedef struct server char *name; /**< Server name/IP address*/ unsigned short port; /**< Port to listen on */ char *protocol; /**< Protocol module to use */ + char *authenticator; /**< Authenticator module name */ SSL_LISTENER *server_ssl; /**< SSL data structure for server, if any */ unsigned int status; /**< Status flag bitmap for the server */ char *monuser; /**< User name to use to monitor the db */ diff --git a/server/include/skygw_debug.h b/server/include/skygw_debug.h index bff6685cd..d01bd5134 100644 --- a/server/include/skygw_debug.h +++ b/server/include/skygw_debug.h @@ -213,14 +213,14 @@ typedef enum skygw_chk_t (s) == SESSION_STATE_STOPPING ? "SESSION_STATE_STOPPING":\ "SESSION_STATE_UNKNOWN")))))) -#define STRPROTOCOLSTATE(s) ((s) == MYSQL_ALLOC ? "MYSQL_ALLOC" : \ - ((s) == MYSQL_PENDING_CONNECT ? "MYSQL_PENDING_CONNECT" : \ - ((s) == MYSQL_CONNECTED ? "MYSQL_CONNECTED" : \ - ((s) == MYSQL_AUTH_SENT ? "MYSQL_AUTH_SENT" : \ - ((s) == MYSQL_AUTH_RECV ? "MYSQL_AUTH_RECV" : \ - ((s) == MYSQL_AUTH_FAILED ? "MYSQL_AUTH_FAILED" : \ - ((s) == MYSQL_IDLE ? "MYSQL_IDLE" : \ - "UNKNOWN MYSQL STATE"))))))) +#define STRPROTOCOLSTATE(s) ((s) == MXS_AUTH_STATE_INIT ? "MXS_AUTH_STATE_INIT" : \ + ((s) == MXS_AUTH_STATE_PENDING_CONNECT ? "MXS_AUTH_STATE_PENDING_CONNECT" : \ + ((s) == MXS_AUTH_STATE_CONNECTED ? "MXS_AUTH_STATE_CONNECTED" : \ + ((s) == MXS_AUTH_STATE_MESSAGE_READ ? "MXS_AUTH_STATE_MESSAGE_READ" : \ + ((s) == MXS_AUTH_STATE_RESPONSE_SENT ? "MXS_AUTH_STATE_RESPONSE_SENT" : \ + ((s) == MXS_AUTH_STATE_FAILED ? "MXS_AUTH_STATE_FAILED" : \ + ((s) == MXS_AUTH_STATE_COMPLETE ? "MXS_AUTH_STATE_COMPLETE" : \ + "UNKNOWN AUTH STATE"))))))) #define STRITEMTYPE(t) ((t) == Item::FIELD_ITEM ? "FIELD_ITEM" : \ ((t) == Item::FUNC_ITEM ? "FUNC_ITEM" : \ diff --git a/server/modules/include/mysql_client_server_protocol.h b/server/modules/include/mysql_client_server_protocol.h index 715283ff3..857650c87 100644 --- a/server/modules/include/mysql_client_server_protocol.h +++ b/server/modules/include/mysql_client_server_protocol.h @@ -79,7 +79,7 @@ #define GW_MYSQL_SCRAMBLE_SIZE 20 #define GW_SCRAMBLE_LENGTH_323 8 -#define DEFAULT_AUTH_PLUGIN_NAME "mysql_native_password" +#define DEFAULT_MYSQL_AUTH_PLUGIN "mysql_native_password" /** Maximum length of a MySQL packet */ #define MYSQL_PACKET_LENGTH_MAX 0x00ffffff @@ -97,28 +97,6 @@ #define COM_QUIT_PACKET_SIZE (4+1) struct dcb; -typedef enum -{ - MYSQL_ALLOC, /* Initial state of protocol auth state */ - /* The following are used only for backend connections */ - MYSQL_PENDING_CONNECT, - MYSQL_CONNECTED, - /* The following can be used for either client or backend */ - /* The comments have only been checked for client use at present */ - MYSQL_AUTH_SENT, - MYSQL_AUTH_RECV, /* This is only ever a transient value */ - MYSQL_AUTH_FAILED, /* Once this is set, the connection */ - /* will be ended, so this is transient */ - /* The following is used only for backend connections */ - MYSQL_HANDSHAKE_FAILED, - /* The following are obsolete and will be removed */ - MYSQL_AUTH_SSL_REQ, /*< client requested SSL but SSL_accept hasn't beed called */ - MYSQL_AUTH_SSL_HANDSHAKE_DONE, /*< SSL handshake has been fully completed */ - MYSQL_AUTH_SSL_HANDSHAKE_FAILED, /*< SSL handshake failed for any reason */ - MYSQL_AUTH_SSL_HANDSHAKE_ONGOING, /*< SSL_accept has been called but the - * SSL handshake hasn't been completed */ - MYSQL_IDLE -} mysql_auth_state_t; typedef enum { @@ -272,7 +250,7 @@ typedef struct mysql_server_cmd_t current_command; /**< Current command being executed */ server_command_t protocol_command; /*< session command list */ server_command_t* protocol_cmd_history; /*< session command history */ - mysql_auth_state_t protocol_auth_state; /*< Authentication status */ + mxs_auth_state_t protocol_auth_state; /*< Authentication status */ mysql_protocol_state_t protocol_state; /*< Protocol struct status */ uint8_t scramble[MYSQL_SCRAMBLE_LEN]; /*< server scramble, * created or received */ @@ -288,6 +266,9 @@ typedef struct #endif } MySQLProtocol; +/** Defines for response codes */ +#define MYSQL_REPLY_ERR 0xff +#define MYSQL_REPLY_OK 0x00 /* * Let's try this with proper enums instead of numbers @@ -307,7 +288,7 @@ typedef struct #define MYSQL_GET_ERRCODE(payload) (gw_mysql_get_byte2(&payload[5])) #define MYSQL_GET_STMTOK_NPARAM(payload) (gw_mysql_get_byte2(&payload[9])) #define MYSQL_GET_STMTOK_NATTR(payload) (gw_mysql_get_byte2(&payload[11])) -#define MYSQL_IS_ERROR_PACKET(payload) ((int)MYSQL_GET_COMMAND(payload)==0xff) +#define MYSQL_IS_ERROR_PACKET(payload) ((int)MYSQL_GET_COMMAND(payload)==MYSQL_REPLY_ERR) #define MYSQL_IS_COM_QUIT(payload) (MYSQL_GET_COMMAND(payload)==MYSQL_COM_QUIT) #define MYSQL_IS_COM_INIT_DB(payload) (MYSQL_GET_COMMAND(payload)==MYSQL_COM_INIT_DB) #define MYSQL_IS_CHANGE_USER(payload) (MYSQL_GET_COMMAND(payload)==MYSQL_COM_CHANGE_USER) @@ -383,5 +364,6 @@ void init_response_status ( mysql_server_cmd_t cmd, int* npackets, ssize_t* nbytes); +bool read_complete_packet(DCB *dcb, GWBUF **readbuf); #endif /** _MYSQL_PROTOCOL_H */ diff --git a/server/modules/protocol/MySQLBackend/mysql_backend.c b/server/modules/protocol/MySQLBackend/mysql_backend.c index 76d52ce2d..e4616503b 100644 --- a/server/modules/protocol/MySQLBackend/mysql_backend.c +++ b/server/modules/protocol/MySQLBackend/mysql_backend.c @@ -19,9 +19,10 @@ #include #include #include +#include /* The following can be compared using memcmp to detect a null password */ -uint8_t null_client_sha1[MYSQL_SCRAMBLE_LEN]=""; +uint8_t null_client_sha1[MYSQL_SCRAMBLE_LEN] = ""; /* * MySQL Protocol module for handling the protocol between the gateway @@ -56,15 +57,16 @@ uint8_t null_client_sha1[MYSQL_SCRAMBLE_LEN]=""; #include #include - /* @see function load_module in load_utils.c for explanation of the following - * lint directives. - */ +/* @see function load_module in load_utils.c for explanation of the following + * lint directives. +*/ /*lint -e14 */ -MODULE_INFO info = { - MODULE_API_PROTOCOL, - MODULE_GA, - GWPROTOCOL_VERSION, - "The MySQL to backend server protocol" +MODULE_INFO info = +{ + MODULE_API_PROTOCOL, + MODULE_GA, + GWPROTOCOL_VERSION, + "The MySQL to backend server protocol" }; /*lint +e14 */ @@ -76,53 +78,51 @@ static int gw_MySQLWrite_backend(DCB *dcb, GWBUF *queue); static int gw_error_backend_event(DCB *dcb); static int gw_backend_close(DCB *dcb); static int gw_backend_hangup(DCB *dcb); -static int backend_write_delayqueue(DCB *dcb); +static int backend_write_delayqueue(DCB *dcb, GWBUF *buffer); static void backend_set_delayqueue(DCB *dcb, GWBUF *queue); static int gw_change_user(DCB *backend_dcb, SERVER *server, SESSION *in_session, GWBUF *queue); static char *gw_backend_default_auth(); static GWBUF* process_response_data(DCB* dcb, GWBUF* readbuf, int nbytes_to_process); extern char* create_auth_failed_msg(GWBUF* readbuf, char* hostaddr, uint8_t* sha1); static bool sescmd_response_complete(DCB* dcb); -static int gw_read_reply_or_error(DCB *dcb, MYSQL_session local_session); -static int gw_read_and_write(DCB *dcb, MYSQL_session local_session); -static int gw_read_backend_handshake(MySQLProtocol *conn); +static void gw_reply_on_error(DCB *dcb, mxs_auth_state_t state); +static int gw_read_and_write(DCB *dcb); +static mxs_auth_state_t gw_read_backend_handshake(MySQLProtocol *conn, GWBUF *buffer); static int gw_decode_mysql_server_handshake(MySQLProtocol *conn, uint8_t *payload); -static int gw_receive_backend_auth(MySQLProtocol *protocol); -static mysql_auth_state_t gw_send_authentication_to_backend(char *dbname, - char *user, - uint8_t *passwd, - MySQLProtocol *conn); +static mxs_auth_state_t gw_read_auth_response(DCB *dcb, GWBUF *buffer); +static mxs_auth_state_t gw_send_auth(MYSQL_session *ses, MySQLProtocol *conn); static uint32_t create_capabilities(MySQLProtocol *conn, bool db_specified, bool compress); static int response_length(MySQLProtocol *conn, char *user, uint8_t *passwd, char *dbname); static uint8_t *load_hashed_password(MySQLProtocol *conn, uint8_t *payload, uint8_t *passwd); static int gw_do_connect_to_backend(char *host, int port, int *fd); static void inline close_socket(int socket); static GWBUF *gw_create_change_user_packet(MYSQL_session* mses, - MySQLProtocol* protocol); + MySQLProtocol* protocol); static int gw_send_change_user_to_backend(char *dbname, - char *user, - uint8_t *passwd, - MySQLProtocol *conn); + char *user, + uint8_t *passwd, + MySQLProtocol *conn); #if defined(NOT_USED) static int gw_session(DCB *backend_dcb, void *data); #endif static bool gw_get_shared_session_auth_info(DCB* dcb, MYSQL_session* session); -static GWPROTOCOL MyObject = { - gw_read_backend_event, /* Read - EPOLLIN handler */ - gw_MySQLWrite_backend, /* Write - data from gateway */ - gw_write_backend_event, /* WriteReady - EPOLLOUT handler */ - gw_error_backend_event, /* Error - EPOLLERR handler */ - gw_backend_hangup, /* HangUp - EPOLLHUP handler */ - NULL, /* Accept */ - gw_create_backend_connection, /* Connect */ - gw_backend_close, /* Close */ - NULL, /* Listen */ - gw_change_user, /* Authentication */ - NULL, /* Session */ - gw_backend_default_auth, /* Default authenticator */ - NULL /**< Connection limit reached */ +static GWPROTOCOL MyObject = +{ + gw_read_backend_event, /* Read - EPOLLIN handler */ + gw_MySQLWrite_backend, /* Write - data from gateway */ + gw_write_backend_event, /* WriteReady - EPOLLOUT handler */ + gw_error_backend_event, /* Error - EPOLLERR handler */ + gw_backend_hangup, /* HangUp - EPOLLHUP handler */ + NULL, /* Accept */ + gw_create_backend_connection, /* Connect */ + gw_backend_close, /* Close */ + NULL, /* Listen */ + gw_change_user, /* Authentication */ + NULL, /* Session */ + gw_backend_default_auth, /* Default authenticator */ + NULL /* Connection limit reached */ }; /* @@ -169,7 +169,7 @@ GWPROTOCOL* GetModuleObject() */ static char *gw_backend_default_auth() { - return "NullBackendAuth"; + return "NullAuthAllow"; } /*lint +e14 */ @@ -248,7 +248,7 @@ static int gw_create_backend_connection(DCB *backend_dcb, case 0: ss_dassert(fd > 0); protocol->fd = fd; - protocol->protocol_auth_state = MYSQL_CONNECTED; + protocol->protocol_auth_state = MXS_AUTH_STATE_CONNECTED; MXS_DEBUG("%lu [gw_create_backend_connection] Established " "connection to %s:%i, protocol fd %d client " "fd %d.", @@ -264,7 +264,7 @@ static int gw_create_backend_connection(DCB *backend_dcb, /* as it means the calls have been successful but the connection */ /* has not yet completed and the calls are non-blocking. */ ss_dassert(fd > 0); - protocol->protocol_auth_state = MYSQL_PENDING_CONNECT; + protocol->protocol_auth_state = MXS_AUTH_STATE_PENDING_CONNECT; protocol->fd = fd; MXS_DEBUG("%lu [gw_create_backend_connection] Connection " "pending to %s:%i, protocol fd %d client fd %d.", @@ -278,7 +278,7 @@ static int gw_create_backend_connection(DCB *backend_dcb, default: /* Failure - the state reverts to its initial value */ ss_dassert(fd == -1); - ss_dassert(protocol->protocol_auth_state == MYSQL_ALLOC); + ss_dassert(protocol->protocol_auth_state == MXS_AUTH_STATE_INIT); MXS_DEBUG("%lu [gw_create_backend_connection] Connection " "failed to %s:%i, protocol fd %d client fd %d.", pthread_self(), @@ -424,6 +424,78 @@ return_rv: } +/** + * @brief Check if the response contain an error + * + * @param buffer Buffer with a complete response + * @return True if the reponse contains an MySQL error packet + */ +bool is_error_response(GWBUF *buffer) +{ + uint8_t cmd; + return gwbuf_copy_data(buffer, MYSQL_HEADER_LEN, 1, &cmd) && cmd == MYSQL_REPLY_ERR; +} + +/** + * @brief Log handshake failure + * + * @param dcb Backend DCB where authentication failed + * @param buffer Buffer containing the response from the backend + */ +void log_error_response(DCB *dcb, GWBUF *buffer) +{ + uint8_t *data = (uint8_t*)GWBUF_DATA(buffer); + size_t len = MYSQL_GET_PACKET_LEN(data); + uint16_t errcode = MYSQL_GET_ERRCODE(data); + char bufstr[len]; + memcpy(bufstr, data + 7, len - 3); + bufstr[len - 3] = '\0'; + + MXS_ERROR("Invalid authentication message from backend '%s'. Error code: %d, " + "Msg : %s", dcb->server->unique_name, errcode, bufstr); + + /** If the error is ER_HOST_IS_BLOCKED put the server into maintenace mode. + * This will prevent repeated authentication failures. */ + if (errcode == ER_HOST_IS_BLOCKED) + { + MXS_ERROR("Server %s has been put into maintenance mode due " + "to the server blocking connections from MaxScale. " + "Run 'mysqladmin -h %s -P %d flush-hosts' on this " + "server before taking this server out of maintenance " + "mode.", dcb->server->unique_name, + dcb->server->name, dcb->server->port); + + server_set_status(dcb->server, SERVER_MAINT); + } +} + +/** + * @brief Handle the server's handshake packet + * + * This function reads the server's handshake packet and does the first step of + * the authentication. + * + * @param dcb Backend DCB + * @param buffer Buffer containing the server's complete handshake + * @return MXS_AUTH_STATE_HANDSHAKE_FAILED on failure. + */ +mxs_auth_state_t handle_server_handshake(DCB *dcb, GWBUF *buffer) +{ + MySQLProtocol *proto = (MySQLProtocol *)dcb->protocol; + mxs_auth_state_t rval = gw_read_backend_handshake(proto, buffer); + + if (rval != MXS_AUTH_STATE_HANDSHAKE_FAILED) + { + MYSQL_session local_session; + gw_get_shared_session_auth_info(dcb, &local_session); + + /** First message read, decode password and send the auth credentials to backend */ + rval = gw_send_auth(&local_session, proto); + } + + return rval; +} + /******************************************************************************* ******************************************************************************* * @@ -454,266 +526,128 @@ return_rv: static int gw_read_backend_event(DCB *dcb) { - MySQLProtocol *backend_protocol; - MYSQL_session local_session; - CHK_DCB(dcb); if (dcb->persistentstart) { + /** If a DCB gets a read event when it's in the persistent pool, it is + * treated as if it were an error. */ dcb->dcb_errhandle_called = true; return 0; } - if (dcb->dcb_is_zombie || dcb->session == NULL) + if (dcb->dcb_is_zombie || dcb->session == NULL || + dcb->session->state == SESSION_STATE_DUMMY) { return 0; } CHK_SESSION(dcb->session); - /*< return only with complete session */ - if (!gw_get_shared_session_auth_info(dcb, &local_session)) + MySQLProtocol *proto = (MySQLProtocol *)dcb->protocol; + CHK_PROTOCOL(proto); + + MXS_DEBUG("%lu [gw_read_backend_event] Read dcb %p fd %d protocol state %d, %s.", + pthread_self(), dcb, dcb->fd, proto->protocol_auth_state, + STRPROTOCOLSTATE(proto->protocol_auth_state)); + + int rc = 0; + if (proto->protocol_auth_state == MXS_AUTH_STATE_COMPLETE) { - return 0; + rc = gw_read_and_write(dcb); } - - backend_protocol = (MySQLProtocol *) dcb->protocol; - CHK_PROTOCOL(backend_protocol); - - MXS_DEBUG("%lu [gw_read_backend_event] Read dcb %p fd %d protocol " - "state %d, %s.", - pthread_self(), - dcb, - dcb->fd, - backend_protocol->protocol_auth_state, - STRPROTOCOLSTATE(backend_protocol->protocol_auth_state)); - - /* backend is connected: - * - * 1. read server handshake - * 2. if (success) write auth request - * 3. and return - */ - - /*< - * If starting to auhenticate with backend server, lock dcb - * to prevent overlapping processing of auth messages. - */ - if (backend_protocol->protocol_auth_state == MYSQL_CONNECTED) + else { - spinlock_acquire(&dcb->authlock); - if (backend_protocol->protocol_auth_state == MYSQL_CONNECTED) - { - /** Read cached backend handshake */ - if (gw_read_backend_handshake(backend_protocol) != 0) - { - backend_protocol->protocol_auth_state = MYSQL_HANDSHAKE_FAILED; + GWBUF *readbuf = NULL; - MXS_DEBUG("%lu [gw_read_backend_event] after " - "gw_read_backend_handshake, fd %d, " - "state = MYSQL_HANDSHAKE_FAILED.", - pthread_self(), - backend_protocol->owner_dcb->fd); + if (!read_complete_packet(dcb, &readbuf)) + { + proto->protocol_auth_state = MXS_AUTH_STATE_FAILED; + gw_reply_on_error(dcb, proto->protocol_auth_state); + } + else if (readbuf) + { + /** We have a complete response from the server */ + /** TODO: add support for non-contiguous responses */ + readbuf = gwbuf_make_contiguous(readbuf); + MXS_ABORT_IF_NULL(readbuf); + + if (is_error_response(readbuf)) + { + /** The server responded with an error */ + proto->protocol_auth_state = MXS_AUTH_STATE_FAILED; + log_error_response(dcb, readbuf); + } + + if (proto->protocol_auth_state == MXS_AUTH_STATE_CONNECTED) + { + /** TODO: Convert this to a call to dcb->authfunc.authenticate */ + /** Read the first message from the server */ + proto->protocol_auth_state = handle_server_handshake(dcb, readbuf); + readbuf = NULL; } else { /** - * Decode password and send the auth credentials - * to backend. + * Authentication process is ongoing. The default authentication + * plugin, mysql_native_password, sends two messages: the initial + * handshake and then either an OK or an ERR packet which signals + * whether authentication was successful */ - backend_protocol->protocol_auth_state = - gw_send_authentication_to_backend( - local_session.db, - local_session.user, - local_session.client_sha1, - backend_protocol); - } - } - spinlock_release(&dcb->authlock); - } /*< backend_protocol->protocol_auth_state == MYSQL_CONNECTED */ - /* - * Now: - * -- check the authentication reply from backend - * OR - * -- handle a previous handshake error - */ - if (backend_protocol->protocol_auth_state != MYSQL_IDLE) - { - spinlock_acquire(&dcb->authlock); + if (proto->protocol_auth_state == MXS_AUTH_STATE_RESPONSE_SENT) + { + /** TODO: Convert this to a call to dcb->authfunc.authenticate */ + proto->protocol_auth_state = gw_read_auth_response(dcb, readbuf); + } - if (backend_protocol->protocol_auth_state != MYSQL_IDLE) - { - if (backend_protocol->protocol_auth_state == MYSQL_CONNECTED) - { - spinlock_release(&dcb->authlock); - return 0; - } - /* Function gw_read_reply_or_error will release dcb->authlock */ - int return_code = gw_read_reply_or_error(dcb, local_session); - /* Make decision whether to exit */ - if (return_code < 2) - { - return return_code; - } - } - else - { - spinlock_release(&dcb->authlock); - } - } /* MYSQL_AUTH_RECV || MYSQL_AUTH_FAILED || MYSQL_HANDSHAKE_FAILED */ + if (proto->protocol_auth_state == MXS_AUTH_STATE_COMPLETE) + { + /** Authentication completed successfully */ + spinlock_acquire(&dcb->authlock); + GWBUF *localq = dcb->delayq; + dcb->delayq = NULL; + spinlock_release(&dcb->authlock); - /* Reading MySQL command output from backend and writing to the client */ - return gw_read_and_write(dcb, local_session); + if (localq) + { + /** Send the queued commands to the backend */ + rc = backend_write_delayqueue(dcb, localq); + } + } + } + + if (proto->protocol_auth_state == MXS_AUTH_STATE_FAILED || + proto->protocol_auth_state == MXS_AUTH_STATE_HANDSHAKE_FAILED) + { + /** Authentication failed */ + gw_reply_on_error(dcb, proto->protocol_auth_state); + } + } + } + + return rc; } /** * Read the backend server MySQL handshake * + * TODO: Move this function inside a module + * * @param conn MySQL protocol structure * @return 0 on success, 1 on failure */ -static int -gw_read_backend_handshake(MySQLProtocol *conn) +static mxs_auth_state_t +gw_read_backend_handshake(MySQLProtocol *conn, GWBUF *buffer) { - GWBUF *head = NULL; - DCB *dcb = conn->owner_dcb; - uint8_t *payload = NULL; - int h_len = 0; - int success = 0; - int packet_len = 0; + mxs_auth_state_t rval = MXS_AUTH_STATE_HANDSHAKE_FAILED; + uint8_t *payload = GWBUF_DATA(buffer) + 4; - if (dcb_read(dcb, &head, 0) != -1) + if (gw_decode_mysql_server_handshake(conn, payload) >= 0) { - dcb->last_read = hkheartbeat; - - if (head) - { - payload = GWBUF_DATA(head); - h_len = gwbuf_length(head); - - /** - * The mysql packets content starts at byte fifth - * just return with less bytes - */ - - if (h_len <= 4) - { - /* log error this exit point */ - conn->protocol_auth_state = MYSQL_HANDSHAKE_FAILED; - MXS_DEBUG("%lu [gw_read_backend_handshake] after " - "dcb_read, fd %d, " - "state = MYSQL_HANDSHAKE_FAILED.", - pthread_self(), - dcb->fd); - - return 1; - } - - if (payload[4] == 0xff) - { - size_t len = MYSQL_GET_PACKET_LEN(payload); - uint16_t errcode = MYSQL_GET_ERRCODE(payload); - char* bufstr = strndup(&((char *)payload)[7], len - 3); - - conn->protocol_auth_state = MYSQL_HANDSHAKE_FAILED; - - MXS_DEBUG("%lu [gw_receive_backend_auth] Invalid " - "authentication message from backend dcb %p " - "fd %d, ptr[4] = %d, error code %d, msg %s.", - pthread_self(), - dcb, - dcb->fd, - payload[4], - errcode, - bufstr); - - MXS_ERROR("Invalid authentication message " - "from backend '%s'. Error code: %d, Msg : %s", - dcb->server->unique_name, - errcode, - bufstr); - - /** - * If ER_HOST_IS_BLOCKED is found - * the related server is put in maintenace mode - * This will avoid filling the error log. - */ - - if (errcode == 1129) - { - MXS_ERROR("Server %s has been put into maintenance mode due " - "to the server blocking connections from MaxScale. " - "Run 'mysqladmin -h %s -P %d flush-hosts' on this " - "server before taking this server out of maintenance " - "mode.", - dcb->server->unique_name, - dcb->server->name, - dcb->server->port); - - server_set_status(dcb->server, SERVER_MAINT); - } - - MXS_FREE(bufstr); - } - //get mysql packet size, 3 bytes - packet_len = gw_mysql_get_byte3(payload); - - if (h_len < (packet_len + 4)) - { - /* - * data in buffer less than expected in the - * packet. Log error this exit point - */ - - conn->protocol_auth_state = MYSQL_HANDSHAKE_FAILED; - - MXS_DEBUG("%lu [gw_read_backend_handshake] after " - "gw_mysql_get_byte3, fd %d, " - "state = MYSQL_HANDSHAKE_FAILED.", - pthread_self(), - dcb->fd); - - return 1; - } - - // skip the 4 bytes header - payload += 4; - - //Now decode mysql handshake - success = gw_decode_mysql_server_handshake(conn, payload); - - if (success < 0) - { - /* MySQL handshake has not been properly decoded - * we cannot continue - * log error this exit point - */ - conn->protocol_auth_state = MYSQL_HANDSHAKE_FAILED; - - MXS_DEBUG("%lu [gw_read_backend_handshake] after " - "gw_decode_mysql_server_handshake, fd %d, " - "state = MYSQL_HANDSHAKE_FAILED.", - pthread_self(), - conn->owner_dcb->fd); - gwbuf_free(head); - return 1; - } - - conn->protocol_auth_state = MYSQL_AUTH_SENT; - - // consume all the data here - gwbuf_free(head); - - return 0; - } - else if (SSL_ESTABLISHED == dcb->ssl_state) - { - return 0; - } + rval = MXS_AUTH_STATE_MESSAGE_READ; } - // Nothing done here, log error this - return 1; + gwbuf_free(buffer); + return rval; } /** @@ -725,33 +659,33 @@ gw_read_backend_handshake(MySQLProtocol *conn) * @param passwd The SHA1(real_password): Note real_password is unknown * @return MySQL authorisation state after operation */ -static mysql_auth_state_t -gw_send_authentication_to_backend(char *dbname, - char *user, - uint8_t *passwd, - MySQLProtocol *conn) +static mxs_auth_state_t +gw_send_auth(MYSQL_session *ses, MySQLProtocol *conn) { uint8_t *payload; long bytes; uint32_t capabilities; - uint8_t client_capabilities[4] = {0,0,0,0}; + uint8_t client_capabilities[4] = {0, 0, 0, 0}; GWBUF *buffer; - uint8_t *curr_passwd = memcmp(passwd, null_client_sha1, MYSQL_SCRAMBLE_LEN) ? passwd : NULL; + uint8_t *curr_passwd = memcmp(ses->client_sha1, null_client_sha1, MYSQL_SCRAMBLE_LEN) ? + ses->client_sha1 : NULL; /** - * If session is stopping return with error. + * If session is stopping or has failed return with error. */ if (conn->owner_dcb->session == NULL || (conn->owner_dcb->session->state != SESSION_STATE_READY && - conn->owner_dcb->session->state != SESSION_STATE_ROUTER_READY)) + conn->owner_dcb->session->state != SESSION_STATE_ROUTER_READY) || + (conn->owner_dcb->server->server_ssl && + conn->owner_dcb->ssl_state != SSL_HANDSHAKE_FAILED)) { - return MYSQL_AUTH_FAILED; + return MXS_AUTH_STATE_FAILED; } - capabilities = create_capabilities(conn, (dbname && strlen(dbname)), false); + capabilities = create_capabilities(conn, (ses->db && strlen(ses->db)), false); gw_mysql_set_byte4(client_capabilities, capabilities); - bytes = response_length(conn, user, passwd, dbname); + bytes = response_length(conn, ses->user, ses->client_sha1, ses->db); // allocating the GWBUF buffer = gwbuf_alloc(bytes); @@ -783,26 +717,19 @@ gw_send_authentication_to_backend(char *dbname, // 23 bytes of 0 payload += 23; - // 4 + 4 + 4 + 1 + 23 = 36, this includes the 4 bytes packet header if (conn->owner_dcb->server->server_ssl && conn->owner_dcb->ssl_state != SSL_ESTABLISHED) { - if (dcb_write(conn->owner_dcb, buffer)) + if (dcb_write(conn->owner_dcb, buffer) && dcb_connect_SSL(conn->owner_dcb) >= 0) { - switch (dcb_connect_SSL(conn->owner_dcb)) - { - case 1: - return MYSQL_CONNECTED; - case 0: - return MYSQL_CONNECTED; - default: - break; - } + return MXS_AUTH_STATE_CONNECTED; } - return MYSQL_AUTH_FAILED; + + return MXS_AUTH_STATE_FAILED; } - memcpy(payload, user, strlen(user)); - payload += strlen(user); + // 4 + 4 + 4 + 1 + 23 = 36, this includes the 4 bytes packet header + memcpy(payload, ses->user, strlen(ses->user)); + payload += strlen(ses->user); payload++; if (curr_passwd != NULL) @@ -815,20 +742,19 @@ gw_send_authentication_to_backend(char *dbname, } // if the db is not NULL append it - if (dbname && strlen(dbname)) + if (ses->db && strlen(ses->db)) { - memcpy(payload, dbname, strlen(dbname)); - payload += strlen(dbname); + memcpy(payload, ses->db, strlen(ses->db)); + payload += strlen(ses->db); payload++; } - memcpy(payload, - "mysql_native_password", - strlen("mysql_native_password")); - /* Following needed if payload is used again */ - /* payload += strlen("mysql_native_password"); */ + const char* auth_plugin_name = conn->owner_dcb->authfunc.plugin_name ? + conn->owner_dcb->authfunc.plugin_name : DEFAULT_MYSQL_AUTH_PLUGIN; - return dcb_write(conn->owner_dcb, buffer) ? MYSQL_AUTH_RECV : MYSQL_AUTH_FAILED; + memcpy(payload, auth_plugin_name, strlen(auth_plugin_name)); + + return dcb_write(conn->owner_dcb, buffer) ? MXS_AUTH_STATE_RESPONSE_SENT : MXS_AUTH_STATE_FAILED; } /** @@ -853,6 +779,7 @@ static bool gw_get_shared_session_auth_info(DCB* dcb, MYSQL_session* session) } else { + ss_dassert(false); MXS_ERROR("%lu [gw_get_shared_session_auth_info] Couldn't get " "session authentication info. Session in a wrong state %d.", pthread_self(), dcb->session->state); @@ -867,147 +794,48 @@ static bool gw_get_shared_session_auth_info(DCB* dcb, MYSQL_session* session) * * @param dcb Descriptor control block for backend server * @param local_session The current MySQL session data structure - * @return 0 = fail, 1 = success, 2 = success and data to be transferred + * @return */ -static int -gw_read_reply_or_error(DCB *dcb, MYSQL_session local_session) +static void +gw_reply_on_error(DCB *dcb, mxs_auth_state_t state) { - int return_code = 0; SESSION *session = dcb->session; - MySQLProtocol *backend_protocol = (MySQLProtocol *)dcb->protocol; - CHK_PROTOCOL(backend_protocol); + CHK_SESSION(session); - if (SESSION_STATE_DUMMY == session->state) - { - spinlock_release(&dcb->authlock); - return 0; - } - CHK_SESSION(session); + /* Only reload the users table if authentication failed and the + * client session is not stopping. It is possible that authentication + * fails because the client has closed the connection before all + * backends have done authentication. */ + if (state == MXS_AUTH_STATE_FAILED && session->state != SESSION_STATE_STOPPING) + { + service_refresh_users(session->service); + } - if (backend_protocol->protocol_auth_state == MYSQL_AUTH_RECV) - { - /** - * Read backend's reply to authentication message - */ - int receive_rc = gw_receive_backend_auth(backend_protocol); + GWBUF* errbuf = mysql_create_custom_error(1, 0, "Authentication with backend " + "failed. Session will be closed."); - switch (receive_rc) - { - case -1: - backend_protocol->protocol_auth_state = MYSQL_AUTH_FAILED; - MXS_ERROR("Backend server didn't " - "accept authentication for user " - "%s.", - local_session.user); - break; - case 1: - backend_protocol->protocol_auth_state = MYSQL_IDLE; - MXS_DEBUG("%lu [gw_read_backend_event] " - "gw_receive_backend_auth succeed. " - "dcb %p fd %d, user %s.", - pthread_self(), - dcb, - dcb->fd, - local_session.user); - break; - default: - ss_dassert(receive_rc == 0); - MXS_DEBUG("%lu [gw_read_backend_event] " - "gw_receive_backend_auth read " - "successfully " - "nothing. dcb %p fd %d, user %s.", - pthread_self(), - dcb, - dcb->fd, - local_session.user); - spinlock_release(&dcb->authlock); - return 0; - } /* switch */ - } + if (session->router_session) + { + bool succp = false; - if (backend_protocol->protocol_auth_state == MYSQL_AUTH_FAILED || - backend_protocol->protocol_auth_state == MYSQL_HANDSHAKE_FAILED) - { - GWBUF* errbuf; - bool succp; - /** - * protocol state won't change anymore, lock can be freed. - * First free delay queue - which is only ever processed while - * authlock is held. - */ - gwbuf_free(dcb->delayq); - dcb->delayq = NULL; - spinlock_release(&dcb->authlock); + session->service->router->handleError(session->service->router_instance, + session->router_session, + errbuf, dcb, ERRACT_REPLY_CLIENT, &succp); - /* Only reload the users table if authentication failed and the - * client session is not stopping. It is possible that authentication - * fails because the client has closed the connection before all - * backends have done authentication. */ - if (backend_protocol->protocol_auth_state == MYSQL_AUTH_FAILED && - dcb->session->state != SESSION_STATE_STOPPING) - { - service_refresh_users(dcb->session->service); - } -#if defined(SS_DEBUG) - MXS_DEBUG("%lu [gw_read_backend_event] " - "calling handleError. Backend " - "DCB %p, session %p", - pthread_self(), - dcb, - dcb->session); -#endif - errbuf = mysql_create_custom_error(1, - 0, - "Authentication with backend failed. " - "Session will be closed."); + spinlock_acquire(&session->ses_lock); + session->state = SESSION_STATE_STOPPING; + spinlock_release(&session->ses_lock); + ss_dassert(dcb->dcb_errhandle_called); + } + else + { + /** A NULL router_session can valid for a router if it declared the + * RCAP_TYPE_NO_RSESSION capability flag */ + dcb->dcb_errhandle_called = true; + } - if (session->router_session) - { - session->service->router->handleError( - session->service->router_instance, - session->router_session, - errbuf, - dcb, - ERRACT_REPLY_CLIENT, - &succp); - spinlock_acquire(&session->ses_lock); - session->state = SESSION_STATE_STOPPING; - spinlock_release(&session->ses_lock); - ss_dassert(dcb->dcb_errhandle_called); - } - else - { - dcb->dcb_errhandle_called = true; - /* - * I'm pretty certain this is best removed and - * causes trouble if present, but have left it - * here just for now as a comment. Martin - */ - /* dcb_close(dcb); */ - } - gwbuf_free(errbuf); - return 1; - } - else - { - MXS_DEBUG("%lu [gw_read_backend_event] " - "gw_receive_backend_auth succeed. Fd %d, " - "user %s.", - pthread_self(), - dcb->fd, - local_session.user); - - /* check the delay queue and flush the data */ - if (dcb->delayq) - { - return_code = backend_write_delayqueue(dcb); - spinlock_release(&dcb->authlock); - return return_code; - } - } - spinlock_release(&dcb->authlock); - return 2; -} /* MYSQL_AUTH_RECV || MYSQL_AUTH_FAILED */ + gwbuf_free(errbuf); +} /** * @brief With authentication completed, read new data and write to backend @@ -1017,153 +845,154 @@ gw_read_reply_or_error(DCB *dcb, MYSQL_session local_session) * @return 0 is fail, 1 is success */ static int -gw_read_and_write(DCB *dcb, MYSQL_session local_session) +gw_read_and_write(DCB *dcb) { - GWBUF *read_buffer = NULL; - SESSION *session = dcb->session; - int nbytes_read; - int return_code; + GWBUF *read_buffer = NULL; + SESSION *session = dcb->session; + int nbytes_read; + int return_code; - CHK_SESSION(session); + CHK_SESSION(session); - /* read available backend data */ - return_code = dcb_read(dcb, &read_buffer, 0); + /* read available backend data */ + return_code = dcb_read(dcb, &read_buffer, 0); - if (return_code < 0) - { - GWBUF* errbuf; - bool succp; + if (return_code < 0) + { + GWBUF* errbuf; + bool succp; #if defined(SS_DEBUG) - MXS_ERROR("Backend read error handling #2."); + MXS_ERROR("Backend read error handling #2."); #endif - errbuf = mysql_create_custom_error(1, - 0, - "Read from backend failed"); + errbuf = mysql_create_custom_error(1, + 0, + "Read from backend failed"); - session->service->router->handleError( - session->service->router_instance, - session->router_session, - errbuf, - dcb, - ERRACT_NEW_CONNECTION, - &succp); - gwbuf_free(errbuf); + session->service->router->handleError( + session->service->router_instance, + session->router_session, + errbuf, + dcb, + ERRACT_NEW_CONNECTION, + &succp); + gwbuf_free(errbuf); - if (!succp) - { - spinlock_acquire(&session->ses_lock); - session->state = SESSION_STATE_STOPPING; - spinlock_release(&session->ses_lock); - } - return_code = 0; - goto return_rc; - } - - nbytes_read = gwbuf_length(read_buffer); - if (nbytes_read == 0) + if (!succp) { - ss_dassert(read_buffer == NULL); + spinlock_acquire(&session->ses_lock); + session->state = SESSION_STATE_STOPPING; + spinlock_release(&session->ses_lock); + } + return_code = 0; + goto return_rc; + } + + nbytes_read = gwbuf_length(read_buffer); + if (nbytes_read == 0) + { + ss_dassert(read_buffer == NULL); + goto return_rc; + } + else + { + ss_dassert(read_buffer != NULL); + } + + if (nbytes_read < 3) + { + dcb->dcb_readqueue = read_buffer; + return_code = 0; + goto return_rc; + } + + { + GWBUF *tmp = modutil_get_complete_packets(&read_buffer); + /* Put any residue into the read queue */ + spinlock_acquire(&dcb->authlock); + dcb->dcb_readqueue = read_buffer; + spinlock_release(&dcb->authlock); + if (tmp == NULL) + { + /** No complete packets */ + return_code = 0; goto return_rc; } else { - ss_dassert(read_buffer != NULL); + read_buffer = tmp; } + } - if (nbytes_read < 3) + /** + * If protocol has session command set, concatenate whole + * response into one buffer. + */ + if (protocol_get_srv_command((MySQLProtocol *)dcb->protocol, false) != MYSQL_COM_UNDEFINED) + { + read_buffer = process_response_data(dcb, read_buffer, gwbuf_length(read_buffer)); + /** + * Received incomplete response to session command. + * Store it to readqueue and return. + */ + if (!sescmd_response_complete(dcb)) { - dcb->dcb_readqueue = read_buffer; return_code = 0; goto return_rc; } + if (!read_buffer) { - GWBUF *tmp = modutil_get_complete_packets(&read_buffer); - /* Put any residue into the read queue */ - spinlock_acquire(&dcb->authlock); - dcb->dcb_readqueue = read_buffer; - spinlock_release(&dcb->authlock); - if (tmp == NULL) - { - /** No complete packets */ - return_code = 0; - goto return_rc; - } - else - { - read_buffer = tmp; - } + MXS_NOTICE("%lu [gw_read_backend_event] " + "Read buffer unexpectedly null, even though response " + "not marked as complete. User: %s", + pthread_self(), dcb->session->client_dcb->user); + return_code = 0; + goto return_rc; } - - /** - * If protocol has session command set, concatenate whole - * response into one buffer. - */ - if (protocol_get_srv_command((MySQLProtocol *) dcb->protocol, false) != MYSQL_COM_UNDEFINED) + } + /** + * Check that session is operable, and that client DCB is + * still listening the socket for replies. + */ + if (dcb->session->state == SESSION_STATE_ROUTER_READY && + dcb->session->client_dcb != NULL && + dcb->session->client_dcb->state == DCB_STATE_POLLING && + (session->router_session || + session->service->router->getCapabilities() & (int)RCAP_TYPE_NO_RSESSION)) + { + MySQLProtocol *client_protocol = (MySQLProtocol *)dcb->session->client_dcb->protocol; + if (client_protocol != NULL) { - read_buffer = process_response_data(dcb, read_buffer, gwbuf_length(read_buffer)); - /** - * Received incomplete response to session command. - * Store it to readqueue and return. - */ - if (!sescmd_response_complete(dcb)) - { - return_code = 0; - goto return_rc; - } + CHK_PROTOCOL(client_protocol); - if (!read_buffer) - { - MXS_NOTICE("%lu [gw_read_backend_event] " - "Read buffer unexpectedly null, even though response " - "not marked as complete. User: %s", - pthread_self(), - local_session.user); - return_code = 0; - goto return_rc; - } - } - /** - * Check that session is operable, and that client DCB is - * still listening the socket for replies. - */ - if (dcb->session->state == SESSION_STATE_ROUTER_READY && - dcb->session->client_dcb != NULL && - dcb->session->client_dcb->state == DCB_STATE_POLLING && - (session->router_session || - session->service->router->getCapabilities() & (int)RCAP_TYPE_NO_RSESSION)) - { - MySQLProtocol *client_protocol = (MySQLProtocol *)dcb->session->client_dcb->protocol; - if (client_protocol != NULL) - { - CHK_PROTOCOL(client_protocol); - - if (client_protocol->protocol_auth_state == MYSQL_IDLE) - { - gwbuf_set_type(read_buffer, GWBUF_TYPE_MYSQL); - - session->service->router->clientReply( - session->service->router_instance, - session->router_session, - read_buffer, - dcb); - return_code = 1; - } - } - else if (dcb->session->client_dcb->dcb_role == DCB_ROLE_INTERNAL) + if (client_protocol->protocol_auth_state == MXS_AUTH_STATE_COMPLETE) { gwbuf_set_type(read_buffer, GWBUF_TYPE_MYSQL); + session->service->router->clientReply( - session->service->router_instance, - session->router_session, - read_buffer, dcb); + session->service->router_instance, + session->router_session, + read_buffer, + dcb); return_code = 1; } } - else /*< session is closing; replying to client isn't possible */ + else if (dcb->session->client_dcb->dcb_role == DCB_ROLE_INTERNAL) { - gwbuf_free(read_buffer); + gwbuf_set_type(read_buffer, GWBUF_TYPE_MYSQL); + + session->service->router->clientReply( + session->service->router_instance, + session->router_session, + read_buffer, + dcb); + return_code = 1; } + } + else /*< session is closing; replying to client isn't possible */ + { + gwbuf_free(read_buffer); + } return_rc: return return_code; @@ -1177,14 +1006,11 @@ return_rc: */ static int gw_write_backend_event(DCB *dcb) { - int rc = 0; - MySQLProtocol *backend_protocol = dcb->protocol; + int rc = 1; - /*< - * Don't write to backend if backend_dcb is not in poll set anymore. - */ if (dcb->state != DCB_STATE_POLLING) { + /** Don't write to backend if backend_dcb is not in poll set anymore */ uint8_t* data = NULL; bool com_quit = false; @@ -1193,49 +1019,46 @@ static int gw_write_backend_event(DCB *dcb) { data = (uint8_t *) GWBUF_DATA(dcb->writeq); com_quit = MYSQL_IS_COM_QUIT(data); - rc = 0; } spinlock_release(&dcb->writeqlock); - - if (data && !com_quit) + if (data) { - mysql_send_custom_error(dcb->session->client_dcb, 1, 0, - "Writing to backend failed due invalid Maxscale state."); - MXS_DEBUG("%lu [gw_write_backend_event] Write to backend " - "dcb %p fd %d failed due invalid state %s.", - pthread_self(), dcb, dcb->fd, STRDCBSTATE(dcb->state)); + rc = 0; - MXS_ERROR("Attempt to write buffered data to backend " - "failed due internal inconsistent state."); + if (!com_quit) + { + mysql_send_custom_error(dcb->session->client_dcb, 1, 0, + "Writing to backend failed due invalid Maxscale state."); + MXS_ERROR("Attempt to write buffered data to backend " + "failed due internal inconsistent state: %s", + STRDCBSTATE(dcb->state)); + } } else { MXS_DEBUG("%lu [gw_write_backend_event] Dcb %p in state %s " "but there's nothing to write either.", pthread_self(), dcb, STRDCBSTATE(dcb->state)); - rc = 1; + } + } + else + { + MySQLProtocol *backend_protocol = (MySQLProtocol*)dcb->protocol; + + if (backend_protocol->protocol_auth_state == MXS_AUTH_STATE_PENDING_CONNECT) + { + backend_protocol->protocol_auth_state = MXS_AUTH_STATE_CONNECTED; + } + else + { + dcb_drain_writeq(dcb); } - goto return_rc; + MXS_DEBUG("%lu [gw_write_backend_event] wrote to dcb %p fd %d, return %d", + pthread_self(), dcb, dcb->fd, rc); } - if (backend_protocol->protocol_auth_state == MYSQL_PENDING_CONNECT) - { - backend_protocol->protocol_auth_state = MYSQL_CONNECTED; - rc = 1; - goto return_rc; - } - dcb_drain_writeq(dcb); - rc = 1; -return_rc: - MXS_DEBUG("%lu [gw_write_backend_event] " - "wrote to dcb %p fd %d, return %d", - pthread_self(), - dcb, - dcb->fd, - rc); - return rc; } @@ -1260,29 +1083,24 @@ static int gw_MySQLWrite_backend(DCB *dcb, GWBUF *queue) */ switch (backend_protocol->protocol_auth_state) { - case MYSQL_HANDSHAKE_FAILED: - case MYSQL_AUTH_FAILED: + case MXS_AUTH_STATE_HANDSHAKE_FAILED: + case MXS_AUTH_STATE_FAILED: if (dcb->session->state != SESSION_STATE_STOPPING) { MXS_ERROR("Unable to write to backend '%s' due to " "%s failure. Server in state %s.", dcb->server->unique_name, - backend_protocol->protocol_auth_state == MYSQL_HANDSHAKE_FAILED ? + backend_protocol->protocol_auth_state == MXS_AUTH_STATE_HANDSHAKE_FAILED ? "handshake" : "authentication", STRSRVSTATUS(dcb->server)); } - /** Consume query buffer */ - while ((queue = gwbuf_consume( - queue, - GWBUF_LENGTH(queue))) != NULL) - { - ; - } + + gwbuf_free(queue); rc = 0; spinlock_release(&dcb->authlock); break; - case MYSQL_IDLE: + case MXS_AUTH_STATE_COMPLETE: { uint8_t* ptr = GWBUF_DATA(queue); mysql_server_cmd_t cmd = MYSQL_GET_COMMAND(ptr); @@ -1653,78 +1471,50 @@ static void backend_set_delayqueue(DCB *dcb, GWBUF *queue) * @param dcb The current backend DCB * @return The dcb_write status */ -static int backend_write_delayqueue(DCB *dcb) +static int backend_write_delayqueue(DCB *dcb, GWBUF *buffer) { - GWBUF *localq = NULL; - int rc; + ss_dassert(buffer); - if (dcb->delayq == NULL) + if (MYSQL_IS_CHANGE_USER(((uint8_t *)GWBUF_DATA(buffer)))) { - rc = 1; + /** Recreate the COM_CHANGE_USER packet with the scramble the backend sent to us */ + MYSQL_session mses; + gw_get_shared_session_auth_info(dcb, &mses); + gwbuf_free(buffer); + buffer = gw_create_change_user_packet(&mses, dcb->protocol); } - else - { - localq = dcb->delayq; - dcb->delayq = NULL; - if (MYSQL_IS_CHANGE_USER(((uint8_t *)GWBUF_DATA(localq)))) - { - MYSQL_session mses; - GWBUF* new_packet; - - gw_get_shared_session_auth_info(dcb, &mses); - new_packet = gw_create_change_user_packet(&mses, dcb->protocol); - /** - * Remove previous packet which lacks scramble - * and append the new. - */ - localq = gwbuf_consume(localq, GWBUF_LENGTH(localq)); - localq = gwbuf_append(localq, new_packet); - } - rc = dcb_write(dcb, localq); - } + int rc = dcb_write(dcb, buffer); if (rc == 0) { - GWBUF* errbuf; - bool succp; - ROUTER_OBJECT *router = NULL; - ROUTER *router_instance = NULL; - void *rsession = NULL; SESSION *session = dcb->session; - CHK_SESSION(session); + ROUTER_OBJECT *router = session->service->router; + ROUTER *router_instance = session->service->router_instance; + void *rsession = session->router_session; + bool succp = false; + GWBUF* errbuf = mysql_create_custom_error( + 1, 0, "Failed to write buffered data to back-end server. " + "Buffer was empty or back-end was disconnected during " + "operation. Attempting to find a new backend."); - if (session != NULL) + router->handleError(router_instance, + rsession, + errbuf, + dcb, + ERRACT_NEW_CONNECTION, + &succp); + gwbuf_free(errbuf); + + if (!succp) { - router = session->service->router; - router_instance = session->service->router_instance; - rsession = session->router_session; -#if defined(SS_DEBUG) - MXS_INFO("Backend write delayqueue error handling."); -#endif - errbuf = mysql_create_custom_error(1, - 0, - "Failed to write buffered data to back-end server. " - "Buffer was empty or back-end was disconnected during " - "operation. Attempting to find a new backend."); - - router->handleError(router_instance, - rsession, - errbuf, - dcb, - ERRACT_NEW_CONNECTION, - &succp); - gwbuf_free(errbuf); - - if (!succp) - { - spinlock_acquire(&session->ses_lock); - session->state = SESSION_STATE_STOPPING; - spinlock_release(&session->ses_lock); - } + spinlock_acquire(&session->ses_lock); + session->state = SESSION_STATE_STOPPING; + spinlock_release(&session->ses_lock); } } + return rc; } @@ -1842,11 +1632,11 @@ static int gw_change_user(DCB *backend, spinlock_acquire(&in_session->ses_lock); *current_session->db = 0; auth_ret = gw_check_mysql_scramble_data( - backend->session->client_dcb, - auth_token, auth_token_len, - client_protocol->scramble, - sizeof(client_protocol->scramble), - username, client_sha1); + backend->session->client_dcb, + auth_token, auth_token_len, + client_protocol->scramble, + sizeof(client_protocol->scramble), + username, client_sha1); strcpy(current_session->db, current_database); spinlock_release(&in_session->ses_lock); } @@ -2002,7 +1792,7 @@ static GWBUF* process_response_data(DCB* dcb, } nbytes_to_process = 0; } - /** Packet was read. All bytes belonged to the last packet. */ + /** Packet was read. All bytes belonged to the last packet. */ else if (nbytes_left == nbytes_to_process) { nbytes_left = 0; @@ -2048,7 +1838,7 @@ static GWBUF* process_response_data(DCB* dcb, /** Archive the command */ protocol_archive_srv_command(p); } - /** Read next packet */ + /** Read next packet */ else { uint8_t* data; @@ -2179,7 +1969,7 @@ gw_decode_mysql_server_handshake(MySQLProtocol *conn, uint8_t *payload) // get scramble len if (payload[0] > 0) { - scramble_len = payload[0] -1; + scramble_len = payload[0] - 1; ss_dassert(scramble_len > GW_SCRAMBLE_LENGTH_323); ss_dassert(scramble_len <= GW_MYSQL_SCRAMBLE_SIZE); @@ -2213,117 +2003,20 @@ gw_decode_mysql_server_handshake(MySQLProtocol *conn, uint8_t *payload) * Receive the MySQL authentication packet from backend, packet # is 2 * * @param protocol The MySQL protocol structure - * @return -1 in case of failure, 0 if there was nothing to read, 1 if read - * was successful. + * @return -1 in case of failure, 1 if authentication was successful. */ -static int -gw_receive_backend_auth(MySQLProtocol *protocol) +static mxs_auth_state_t +gw_read_auth_response(DCB *dcb, GWBUF *buffer) { - int n = -1; - GWBUF *head = NULL; - DCB *dcb = protocol->owner_dcb; - uint8_t *ptr = NULL; - int rc = 0; + mxs_auth_state_t rval = MXS_AUTH_STATE_FAILED; + uint8_t cmd; - n = dcb_read(dcb, &head, 0); - - dcb->last_read = hkheartbeat; - - /*< - * Read didn't fail and there is enough data for mysql packet. - */ - if (n != -1 && - head != NULL && - GWBUF_LENGTH(head) >= 5) + if (gwbuf_copy_data(buffer, MYSQL_HEADER_LEN, 1, &cmd) && cmd == MYSQL_REPLY_OK) { - ptr = GWBUF_DATA(head); - /*< - * 5th byte is 0x0 if successful. - */ - if (ptr[4] == 0x00) - { - rc = 1; - } - else if (ptr[4] == 0xff) - { - size_t len = MYSQL_GET_PACKET_LEN(ptr); - char* err = strndup(&((char *)ptr)[8], 5); - char* bufstr = strndup(&((char *)ptr)[13], len - 4 - 5); - - MXS_DEBUG("%lu [gw_receive_backend_auth] Invalid " - "authentication message from backend dcb %p " - "fd %d, ptr[4] = %d, error %s, msg %s.", - pthread_self(), - dcb, - dcb->fd, - ptr[4], - err, - bufstr); - - MXS_ERROR("Invalid authentication message " - "from backend. Error : %s, Msg : %s", - err, - bufstr); - - MXS_FREE(bufstr); - MXS_FREE(err); - rc = -1; - } - else - { - MXS_DEBUG("%lu [gw_receive_backend_auth] Invalid " - "authentication message from backend dcb %p " - "fd %d, ptr[4] = %d", - pthread_self(), - dcb, - dcb->fd, - ptr[4]); - - MXS_ERROR("Invalid authentication message " - "from backend. Packet type : %d", - ptr[4]); - } - /*< - * Remove data from buffer. - */ - while ((head = gwbuf_consume(head, GWBUF_LENGTH(head))) != NULL) - { - ; - } - } - else if (n == 0) - { - /*< - * This is considered as success because call didn't fail, - * although no bytes was read. - */ - rc = 0; - MXS_DEBUG("%lu [gw_receive_backend_auth] Read zero bytes from " - "backend dcb %p fd %d in state %s. n %d, head %p, len %ld", - pthread_self(), - dcb, - dcb->fd, - STRDCBSTATE(dcb->state), - n, - head, - (head == NULL) ? 0 : GWBUF_LENGTH(head)); - } - else - { - ss_dassert(n < 0 && head == NULL); - rc = -1; - MXS_DEBUG("%lu [gw_receive_backend_auth] Reading from backend dcb %p " - "fd %d in state %s failed. n %d, head %p, len %ld", - pthread_self(), - dcb, - dcb->fd, - STRDCBSTATE(dcb->state), - n, - head, - (head == NULL) ? 0 : GWBUF_LENGTH(head)); + rval = MXS_AUTH_STATE_COMPLETE; } - return rc; + return rval; } /** @@ -2447,9 +2140,9 @@ response_length(MySQLProtocol *conn, char *user, uint8_t *passwd, char *dbname) static uint8_t * load_hashed_password(MySQLProtocol *conn, uint8_t *payload, uint8_t *passwd) { - uint8_t hash1[GW_MYSQL_SCRAMBLE_SIZE]=""; - uint8_t hash2[GW_MYSQL_SCRAMBLE_SIZE]=""; - uint8_t new_sha[GW_MYSQL_SCRAMBLE_SIZE]=""; + uint8_t hash1[GW_MYSQL_SCRAMBLE_SIZE] = ""; + uint8_t hash2[GW_MYSQL_SCRAMBLE_SIZE] = ""; + uint8_t new_sha[GW_MYSQL_SCRAMBLE_SIZE] = ""; uint8_t client_scramble[GW_MYSQL_SCRAMBLE_SIZE]; // hash1 is the function input, SHA1(real_password) @@ -2483,9 +2176,9 @@ close_socket(int sock) { char errbuf[STRERROR_BUFLEN]; MXS_ERROR("Failed to close socket %d due %d, %s.", - sock, - errno, - strerror_r(errno, errbuf, sizeof(errbuf))); + sock, + errno, + strerror_r(errno, errbuf, sizeof(errbuf))); } } @@ -2502,7 +2195,7 @@ close_socket(int sock) */ static GWBUF * gw_create_change_user_packet(MYSQL_session* mses, - MySQLProtocol* protocol) + MySQLProtocol* protocol) { char* db; char* user; @@ -2512,7 +2205,7 @@ gw_create_change_user_packet(MYSQL_session* mses, uint8_t* payload = NULL; uint8_t* payload_start = NULL; long bytes; - char dbpass[MYSQL_USER_MAXLEN + 1]=""; + char dbpass[MYSQL_USER_MAXLEN + 1] = ""; char* curr_db = NULL; uint8_t* curr_passwd = NULL; unsigned int charset; @@ -2580,7 +2273,7 @@ gw_create_change_user_packet(MYSQL_session* mses, * Set correct type to GWBUF so that it will be handled like session * commands */ - buffer->gwbuf_type = GWBUF_TYPE_MYSQL|GWBUF_TYPE_SINGLE_STMT|GWBUF_TYPE_SESCMD; + buffer->gwbuf_type = GWBUF_TYPE_MYSQL | GWBUF_TYPE_SINGLE_STMT | GWBUF_TYPE_SESCMD; payload = GWBUF_DATA(buffer); memset(payload, '\0', bytes); payload_start = payload; @@ -2598,9 +2291,9 @@ gw_create_change_user_packet(MYSQL_session* mses, if (curr_passwd != NULL) { - uint8_t hash1[GW_MYSQL_SCRAMBLE_SIZE]=""; - uint8_t hash2[GW_MYSQL_SCRAMBLE_SIZE]=""; - uint8_t new_sha[GW_MYSQL_SCRAMBLE_SIZE]=""; + uint8_t hash1[GW_MYSQL_SCRAMBLE_SIZE] = ""; + uint8_t hash2[GW_MYSQL_SCRAMBLE_SIZE] = ""; + uint8_t new_sha[GW_MYSQL_SCRAMBLE_SIZE] = ""; uint8_t client_scramble[GW_MYSQL_SCRAMBLE_SIZE]; /** hash1 is the function input, SHA1(real_password) */ @@ -2658,7 +2351,7 @@ gw_create_change_user_packet(MYSQL_session* mses, /* Following needed if more to be added */ /* payload += strlen("mysql_native_password"); */ /** put here the paylod size: bytes to write - 4 bytes packet header */ - gw_mysql_set_byte3(payload_start, (bytes-4)); + gw_mysql_set_byte3(payload_start, (bytes - 4)); return buffer; } @@ -2674,9 +2367,9 @@ gw_create_change_user_packet(MYSQL_session* mses, */ static int gw_send_change_user_to_backend(char *dbname, - char *user, - uint8_t *passwd, - MySQLProtocol *conn) + char *user, + uint8_t *passwd, + MySQLProtocol *conn) { GWBUF *buffer; int rc; diff --git a/server/modules/protocol/MySQLClient/mysql_client.c b/server/modules/protocol/MySQLClient/mysql_client.c index 553a86a2f..ed4f3f6d3 100644 --- a/server/modules/protocol/MySQLClient/mysql_client.c +++ b/server/modules/protocol/MySQLClient/mysql_client.c @@ -312,7 +312,7 @@ int MySQLSendHandshake(DCB* dcb) memcpy(mysql_plugin_data, server_scramble + 8, 12); const char* plugin_name = dcb->authfunc.plugin_name ? - dcb->authfunc.plugin_name : DEFAULT_AUTH_PLUGIN_NAME; + dcb->authfunc.plugin_name : DEFAULT_MYSQL_AUTH_PLUGIN; int plugin_name_len = strlen(plugin_name); mysql_payload_size = @@ -512,7 +512,7 @@ int gw_read_client_event(DCB* dcb) * will be changed to MYSQL_IDLE (see below). * */ - case MYSQL_AUTH_SENT: + case MXS_AUTH_STATE_MESSAGE_READ: /* After this call read_buffer will point to freed data */ if (nbytes_read < 3 || (0 == max_bytes && nbytes_read < (MYSQL_GET_PACKET_LEN((uint8_t *) GWBUF_DATA(read_buffer)) + 4)) || @@ -533,12 +533,12 @@ int gw_read_client_event(DCB* dcb) * result in a call that comes to this section of code. * */ - case MYSQL_IDLE: + case MXS_AUTH_STATE_COMPLETE: /* After this call read_buffer will point to freed data */ return_code = gw_read_normal_data(dcb, read_buffer, nbytes_read); break; - case MYSQL_AUTH_FAILED: + case MXS_AUTH_STATE_FAILED: gwbuf_free(read_buffer); return_code = 1; break; @@ -601,7 +601,7 @@ gw_read_do_authentication(DCB *dcb, GWBUF *read_buffer, int nbytes_read) { SESSION *session; - protocol->protocol_auth_state = MYSQL_AUTH_RECV; + protocol->protocol_auth_state = MXS_AUTH_STATE_RESPONSE_SENT; /** * Create session, and a router session for it. * If successful, there will be backend connection(s) @@ -619,7 +619,7 @@ gw_read_do_authentication(DCB *dcb, GWBUF *read_buffer, int nbytes_read) ss_dassert(session->state != SESSION_STATE_ALLOC && session->state != SESSION_STATE_DUMMY); - protocol->protocol_auth_state = MYSQL_IDLE; + protocol->protocol_auth_state = MXS_AUTH_STATE_COMPLETE; /** * Send an AUTH_OK packet to the client, * packet sequence is # packet_number @@ -640,7 +640,7 @@ gw_read_do_authentication(DCB *dcb, GWBUF *read_buffer, int nbytes_read) MXS_AUTH_INCOMPLETE != auth_val && MXS_AUTH_SSL_INCOMPLETE != auth_val) { - protocol->protocol_auth_state = MYSQL_AUTH_FAILED; + protocol->protocol_auth_state = MXS_AUTH_STATE_FAILED; mysql_client_auth_error_handling(dcb, auth_val); /** * Close DCB and which will release MYSQL_session @@ -1106,7 +1106,7 @@ int gw_write_client_event(DCB *dcb) protocol = (MySQLProtocol *)dcb->protocol; CHK_PROTOCOL(protocol); - if (protocol->protocol_auth_state == MYSQL_IDLE) + if (protocol->protocol_auth_state == MXS_AUTH_STATE_COMPLETE) { dcb_drain_writeq(dcb); goto return_1; @@ -1207,7 +1207,7 @@ static void gw_process_one_new_client(DCB *client_dcb) MySQLSendHandshake(client_dcb); // client protocol state change - protocol->protocol_auth_state = MYSQL_AUTH_SENT; + protocol->protocol_auth_state = MXS_AUTH_STATE_MESSAGE_READ; /** * Set new descriptor to event set. At the same time, diff --git a/server/modules/protocol/mysql_common.c b/server/modules/protocol/mysql_common.c index 9da47aa58..5ddd6b665 100644 --- a/server/modules/protocol/mysql_common.c +++ b/server/modules/protocol/mysql_common.c @@ -50,6 +50,7 @@ #include #include #include +#include static server_command_t* server_command_init(server_command_t* srvcmd, mysql_server_cmd_t cmd); @@ -78,7 +79,7 @@ MySQLProtocol* mysql_protocol_init(DCB* dcb, int fd) goto return_p; } p->protocol_state = MYSQL_PROTOCOL_ALLOC; - p->protocol_auth_state = MYSQL_ALLOC; + p->protocol_auth_state = MXS_AUTH_STATE_INIT; p->current_command = MYSQL_COM_UNDEFINED; p->protocol_command.scom_cmd = MYSQL_COM_UNDEFINED; p->protocol_command.scom_nresponse_packets = 0; @@ -143,28 +144,20 @@ const char* gw_mysql_protocol_state2string (int state) { switch(state) { - case MYSQL_ALLOC: - return "MySQL Protocl struct allocated"; - case MYSQL_PENDING_CONNECT: - return "MySQL Backend socket PENDING connect"; - case MYSQL_CONNECTED: - return "MySQL Backend socket CONNECTED"; - case MYSQL_AUTH_SENT: - return "MySQL Authentication handshake has been sent"; - case MYSQL_AUTH_RECV: - return "MySQL Received user, password, db and capabilities"; - case MYSQL_AUTH_FAILED: - return "MySQL Authentication failed"; - case MYSQL_IDLE: - return "MySQL authentication is succesfully done."; - case MYSQL_AUTH_SSL_REQ: - return "MYSQL_AUTH_SSL_REQ"; - case MYSQL_AUTH_SSL_HANDSHAKE_DONE: - return "MYSQL_AUTH_SSL_HANDSHAKE_DONE"; - case MYSQL_AUTH_SSL_HANDSHAKE_FAILED: - return "MYSQL_AUTH_SSL_HANDSHAKE_FAILED"; - case MYSQL_AUTH_SSL_HANDSHAKE_ONGOING: - return "MYSQL_AUTH_SSL_HANDSHAKE_ONGOING"; + case MXS_AUTH_STATE_INIT: + return "Authentication initialized"; + case MXS_AUTH_STATE_PENDING_CONNECT: + return "Network connection pending"; + case MXS_AUTH_STATE_CONNECTED: + return "Network connection created"; + case MXS_AUTH_STATE_MESSAGE_READ: + return "Read server handshake"; + case MXS_AUTH_STATE_RESPONSE_SENT: + return "Response to handshake sent"; + case MXS_AUTH_STATE_FAILED: + return "Authentication failed"; + case MXS_AUTH_STATE_COMPLETE: + return "Authentication is complete."; default: return "MySQL (unknown protocol state)"; } @@ -992,3 +985,43 @@ char *create_auth_fail_str(char *username, retblock: return errstr; } + +/** + * @brief Read a complete packet from a DCB + * + * Read a complete packet from a connected DCB. If data was read, @c readbuf + * will point to the head of the read data. If no data was read, @c readbuf will + * be set to NULL. + * + * @param dcb DCB to read from + * @param readbuf Pointer to a buffer where the data is stored + * @return True on success, false if an error occurred while data was being read + */ +bool read_complete_packet(DCB *dcb, GWBUF **readbuf) +{ + bool rval = false; + GWBUF *localbuf = NULL; + + if (dcb_read(dcb, &localbuf, 0) >= 0) + { + rval = true; + dcb->last_read = hkheartbeat; + GWBUF *packets = modutil_get_complete_packets(&localbuf); + + if (packets) + { + /** A complete packet was read */ + *readbuf = packets; + } + + if (localbuf) + { + /** Store any extra data in the DCB's readqueue */ + spinlock_acquire(&dcb->authlock); + dcb->dcb_readqueue = gwbuf_append(dcb->dcb_readqueue, localbuf); + spinlock_release(&dcb->authlock); + } + } + + return rval; +} From 6d057f8152297465c4938c37790fe8e4cb9b028b Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Wed, 28 Sep 2016 10:20:39 +0300 Subject: [PATCH 004/215] MXS-862: Move backend authentication from MySQLBackend to MySQLBackendAuth The authentication for backend connections is now done in the MySQLBackendAuth module. This is also the default authentication module for backend connections created by MySQLBackend. --- server/core/dcb.c | 5 + server/include/dcb.h | 4 +- server/include/gw_authenticator.h | 4 +- server/modules/authenticator/CMakeLists.txt | 5 + .../authenticator/mysql_backend_auth.c | 275 +++++++ .../include/mysql_client_server_protocol.h | 9 + .../protocol/MySQLBackend/mysql_backend.c | 732 ++++++++---------- .../protocol/MySQLClient/mysql_client.c | 19 +- server/modules/protocol/mysql_common.c | 34 + 9 files changed, 671 insertions(+), 416 deletions(-) create mode 100644 server/modules/authenticator/mysql_backend_auth.c diff --git a/server/core/dcb.c b/server/core/dcb.c index 7e1384b33..c0f568f25 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -397,6 +397,11 @@ dcb_free_all_memory(DCB *dcb) dcb->authfunc.free(dcb); dcb->data = NULL; } + if (dcb->backend_data && dcb->authfunc.free && dcb->dcb_role == DCB_ROLE_BACKEND_HANDLER) + { + dcb->authfunc.free(dcb); + dcb->backend_data = NULL; + } if (dcb->protoname) { MXS_FREE(dcb->protoname); diff --git a/server/include/dcb.h b/server/include/dcb.h index 6da199e85..29c934829 100644 --- a/server/include/dcb.h +++ b/server/include/dcb.h @@ -253,6 +253,7 @@ typedef struct dcb time_t persistentstart; /**< Time when DCB placed in persistent pool */ struct service *service; /**< The related service */ void *data; /**< Specific client data */ + void *backend_data; /**< Specific backend data */ DCBMM memdata; /**< The data related to DCB memory management */ SPINLOCK cb_lock; /**< The lock for the callbacks linked list */ DCB_CALLBACK *callbacks; /**< The list of callbacks for the DCB */ @@ -282,7 +283,8 @@ typedef struct dcb .authlock = SPINLOCK_INIT, .stats = {0}, .memdata = DCBMM_INIT, \ .cb_lock = SPINLOCK_INIT, .pollinlock = SPINLOCK_INIT, \ .fd = DCBFD_CLOSED, .stats = DCBSTATS_INIT, .ssl_state = SSL_HANDSHAKE_UNKNOWN, \ - .state = DCB_STATE_ALLOC, .polloutlock = SPINLOCK_INIT, .dcb_chk_tail = CHK_NUM_DCB} + .state = DCB_STATE_ALLOC, .polloutlock = SPINLOCK_INIT, .dcb_chk_tail = CHK_NUM_DCB, \ + .backend_data = NULL} /** * The DCB usage filer used for returning DCB's in use for a certain reason diff --git a/server/include/gw_authenticator.h b/server/include/gw_authenticator.h index 5f5ab04e4..ee66c1764 100644 --- a/server/include/gw_authenticator.h +++ b/server/include/gw_authenticator.h @@ -67,8 +67,8 @@ typedef struct gw_authenticator /** Return values for extract and authenticate entry points */ #define MXS_AUTH_SUCCEEDED 0 /**< Authentication was successful */ #define MXS_AUTH_FAILED 1 /**< Authentication failed */ -#define MXS_AUTH_FAILED_DB 2 -#define MXS_AUTH_FAILED_SSL 3 +#define MXS_AUTH_FAILED_DB 2 /**< Authentication failed, database not found */ +#define MXS_AUTH_FAILED_SSL 3 /**< SSL authentication failed */ #define MXS_AUTH_INCOMPLETE 4 /**< Authentication is not yet complete */ #define MXS_AUTH_SSL_INCOMPLETE 5 /**< SSL connection is not yet complete */ #define MXS_AUTH_NO_SESSION 6 diff --git a/server/modules/authenticator/CMakeLists.txt b/server/modules/authenticator/CMakeLists.txt index abe32bec3..d15bd8310 100644 --- a/server/modules/authenticator/CMakeLists.txt +++ b/server/modules/authenticator/CMakeLists.txt @@ -3,6 +3,11 @@ target_link_libraries(MySQLAuth maxscale-common) set_target_properties(MySQLAuth PROPERTIES VERSION "1.0.0") install_module(MySQLAuth core) +add_library(MySQLBackendAuth SHARED mysql_backend_auth.c) +target_link_libraries(MySQLBackendAuth maxscale-common MySQLBackend) +set_target_properties(MySQLBackendAuth PROPERTIES VERSION "1.0.0") +install_module(MySQLBackendAuth core) + add_library(NullAuthAllow SHARED null_auth_allow.c) target_link_libraries(NullAuthAllow maxscale-common) set_target_properties(NullAuthAllow PROPERTIES VERSION "1.0.0") diff --git a/server/modules/authenticator/mysql_backend_auth.c b/server/modules/authenticator/mysql_backend_auth.c new file mode 100644 index 000000000..7b8b7111f --- /dev/null +++ b/server/modules/authenticator/mysql_backend_auth.c @@ -0,0 +1,275 @@ +/* + * Copyright (c) 2016 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl. + * + * Change Date: 2019-07-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ + +/** + * @file mysql_backend_auth.c - MySQL backend authenticator + * + * Backend authentication module for the MySQL protocol. Implements the + * client side of the 'mysql_native_password' authentication plugin. + * + * @verbatim + * Revision History + * Date Who Description + * 27/09/2016 Markus Makela Initial version + * + * @endverbatim + */ + +#include +#include +#include +#include + +/** Authentication states */ +enum mba_state +{ + MBA_NEED_HANDSHAKE, /**< Waiting for server's handshake packet */ + MBA_SEND_RESPONSE, /**< A response to the server's handshake has been sent */ + MBA_NEED_OK, /**< Waiting for server's OK packet */ + MBA_AUTH_OK, /**< Authentication completed successfully */ + MBA_AUTH_FAILED /**< Authentication failed */ +}; + +/** Structure representing the authentication state */ +typedef struct mysql_backend_auth +{ + enum mba_state state; /**< Authentication state */ +} mysql_backend_auth_t; + +/** + * @brief Allocate a new mysql_backend_auth object + * @return Allocated object or NULL if memory allocation failed + */ +mysql_backend_auth_t* mba_alloc() +{ + mysql_backend_auth_t* mba = MXS_MALLOC(sizeof(*mba)); + + if (mba) + { + mba->state = MBA_NEED_HANDSHAKE; + } + + return mba; +} + +/** + * Receive the MySQL authentication packet from backend, packet # is 2 + * + * @param protocol The MySQL protocol structure + * @return False in case of failure, true if authentication was successful. + */ +static bool gw_read_auth_response(DCB *dcb, GWBUF *buffer) +{ + bool rval = false; + uint8_t cmd; + + if (gwbuf_copy_data(buffer, MYSQL_HEADER_LEN, 1, &cmd) && cmd == MYSQL_REPLY_OK) + { + rval = true; + } + + return rval; +} + +/** + * @brief Extract backend response + * + * @param dcb Request handler DCB connected to the client + * @param buffer Buffer containing data from client + * @return Authentication status + * @see gw_quthenticator.h + * @see https://dev.mysql.com/doc/internals/en/client-server-protocol.html + */ +static int +auth_backend_extract(DCB *dcb, GWBUF *buf) +{ + int rval = MXS_AUTH_FAILED; + + if (dcb->backend_data || (dcb->backend_data = mba_alloc())) + { + mysql_backend_auth_t *mba = (mysql_backend_auth_t*)dcb->backend_data; + + switch (mba->state) + { + case MBA_NEED_HANDSHAKE: + if (gw_read_backend_handshake(dcb, buf)) + { + rval = MXS_AUTH_INCOMPLETE; + mba->state = MBA_SEND_RESPONSE; + } + else + { + mba->state = MBA_AUTH_FAILED; + } + break; + + case MBA_NEED_OK: + if (gw_read_auth_response(dcb, buf)) + { + rval = MXS_AUTH_SUCCEEDED; + mba->state = MBA_AUTH_OK; + } + else + { + mba->state = MBA_AUTH_FAILED; + } + break; + + default: + MXS_ERROR("Unexpected call to MySQLBackendAuth::extract"); + ss_dassert(false); + break; + } + } + + return rval; +} + +/** + * @brief Authenticates as a MySQL user + * + * @param dcb Backend DCB + * @return Authentication status + * @see gw_authenticator.h + */ +static int +auth_backend_authenticate(DCB *dcb) +{ + int rval = MXS_AUTH_FAILED; + mysql_backend_auth_t *mba = (mysql_backend_auth_t*)dcb->backend_data; + + if (mba->state == MBA_SEND_RESPONSE) + { + /** First message read, decode password and send the auth credentials to backend */ + switch (gw_send_backend_auth(dcb)) + { + case MXS_AUTH_STATE_CONNECTED: + rval = MXS_AUTH_SSL_INCOMPLETE; + break; + + case MXS_AUTH_STATE_RESPONSE_SENT: + mba->state = MBA_NEED_OK; + rval = MXS_AUTH_INCOMPLETE; + break; + + default: + /** Authentication failed */ + break; + } + } + else if (mba->state == MBA_AUTH_OK) + { + /** Authentication completed successfully */ + rval = MXS_AUTH_SUCCEEDED; + } + + return rval; +} + +/** + * @brief Determine whether the client is SSL capable + * + * The authentication request from the client will indicate whether the client + * is expecting to make an SSL connection. The information has been extracted + * in the previous functions. + * + * @param dcb Request handler DCB connected to the client + * @return Boolean indicating whether client is SSL capable + */ +static bool +auth_backend_ssl(DCB *dcb) +{ + return dcb->server->server_ssl != NULL; +} + +/** + * @brief Dummy function for the free entry point + */ +static void +auth_backend_free(DCB *dcb) +{ + MXS_FREE(dcb->backend_data); + dcb->backend_data = NULL; +} + +/** + * @brief Dummy function for the loadusers entry point + */ +static int auth_backend_load_users(SERV_LISTENER *port) +{ + return MXS_AUTH_LOADUSERS_OK; +} + +/* @see function load_module in load_utils.c for explanation of the following + * lint directives. +*/ +/*lint -e14 */ +MODULE_INFO info = +{ + MODULE_API_AUTHENTICATOR, + MODULE_GA, + GWAUTHENTICATOR_VERSION, + "The MySQL MaxScale to backend server authenticator" +}; +/*lint +e14 */ + +static char *version_str = "V1.0.0"; + +/* + * The "module object" for mysql client authenticator module. + */ +static GWAUTHENTICATOR MyObject = +{ + auth_backend_extract, /* Extract data into structure */ + auth_backend_ssl, /* Check if client supports SSL */ + auth_backend_authenticate, /* Authenticate user credentials */ + auth_backend_free, /* Free the client data held in DCB */ + auth_backend_load_users, /* Load users from backend databases */ + DEFAULT_MYSQL_AUTH_PLUGIN +}; + +/** + * Implementation of the mandatory version entry point + * + * @return version string of the module + * + * @see function load_module in load_utils.c for explanation of the following + * lint directives. + */ +/*lint -e14 */ +char* version() +{ + return version_str; +} + +/** + * The module initialisation routine, called when the module + * is first loaded. + */ +void ModuleInit() +{ +} + +/** + * The module entry point routine. It is this routine that + * must populate the structure that is referred to as the + * "module object", this is a structure with the set of + * external entry points for this module. + * + * @return The module object + */ +GWAUTHENTICATOR* GetModuleObject() +{ + return &MyObject; +} +/*lint +e14 */ diff --git a/server/modules/include/mysql_client_server_protocol.h b/server/modules/include/mysql_client_server_protocol.h index 857650c87..41c8ca05c 100644 --- a/server/modules/include/mysql_client_server_protocol.h +++ b/server/modules/include/mysql_client_server_protocol.h @@ -294,6 +294,8 @@ typedef struct #define MYSQL_IS_CHANGE_USER(payload) (MYSQL_GET_COMMAND(payload)==MYSQL_COM_CHANGE_USER) #define MYSQL_GET_NATTR(payload) ((int)payload[4]) +/* The following can be compared using memcmp to detect a null password */ +extern uint8_t null_client_sha1[MYSQL_SCRAMBLE_LEN]; MySQLProtocol* mysql_protocol_init(DCB* dcb, int fd); void mysql_protocol_done (DCB* dcb); @@ -365,5 +367,12 @@ void init_response_status ( int* npackets, ssize_t* nbytes); bool read_complete_packet(DCB *dcb, GWBUF **readbuf); +bool gw_get_shared_session_auth_info(DCB* dcb, MYSQL_session* session); + +/** Read the backend server's handshake */ +bool gw_read_backend_handshake(DCB *dcb, GWBUF *buffer); + +/** Send the server handshake response packet to the backend server */ +mxs_auth_state_t gw_send_backend_auth(DCB *dcb); #endif /** _MYSQL_PROTOCOL_H */ diff --git a/server/modules/protocol/MySQLBackend/mysql_backend.c b/server/modules/protocol/MySQLBackend/mysql_backend.c index e4616503b..08c7db9d5 100644 --- a/server/modules/protocol/MySQLBackend/mysql_backend.c +++ b/server/modules/protocol/MySQLBackend/mysql_backend.c @@ -21,9 +21,6 @@ #include #include -/* The following can be compared using memcmp to detect a null password */ -uint8_t null_client_sha1[MYSQL_SCRAMBLE_LEN] = ""; - /* * MySQL Protocol module for handling the protocol between the gateway * and the backend MySQL database. @@ -87,13 +84,7 @@ extern char* create_auth_failed_msg(GWBUF* readbuf, char* hostaddr, uint8_t* sha static bool sescmd_response_complete(DCB* dcb); static void gw_reply_on_error(DCB *dcb, mxs_auth_state_t state); static int gw_read_and_write(DCB *dcb); -static mxs_auth_state_t gw_read_backend_handshake(MySQLProtocol *conn, GWBUF *buffer); static int gw_decode_mysql_server_handshake(MySQLProtocol *conn, uint8_t *payload); -static mxs_auth_state_t gw_read_auth_response(DCB *dcb, GWBUF *buffer); -static mxs_auth_state_t gw_send_auth(MYSQL_session *ses, MySQLProtocol *conn); -static uint32_t create_capabilities(MySQLProtocol *conn, bool db_specified, bool compress); -static int response_length(MySQLProtocol *conn, char *user, uint8_t *passwd, char *dbname); -static uint8_t *load_hashed_password(MySQLProtocol *conn, uint8_t *payload, uint8_t *passwd); static int gw_do_connect_to_backend(char *host, int port, int *fd); static void inline close_socket(int socket); static GWBUF *gw_create_change_user_packet(MYSQL_session* mses, @@ -106,7 +97,6 @@ static int gw_send_change_user_to_backend(char *dbname, #if defined(NOT_USED) static int gw_session(DCB *backend_dcb, void *data); #endif -static bool gw_get_shared_session_auth_info(DCB* dcb, MYSQL_session* session); static GWPROTOCOL MyObject = { @@ -169,7 +159,7 @@ GWPROTOCOL* GetModuleObject() */ static char *gw_backend_default_auth() { - return "NullAuthAllow"; + return "MySQLBackendAuth"; } /*lint +e14 */ @@ -470,29 +460,41 @@ void log_error_response(DCB *dcb, GWBUF *buffer) } /** - * @brief Handle the server's handshake packet + * @brief Handle the server's response packet * - * This function reads the server's handshake packet and does the first step of + * This function reads the server's response packet and does the final step of * the authentication. * * @param dcb Backend DCB * @param buffer Buffer containing the server's complete handshake * @return MXS_AUTH_STATE_HANDSHAKE_FAILED on failure. */ -mxs_auth_state_t handle_server_handshake(DCB *dcb, GWBUF *buffer) +mxs_auth_state_t handle_server_response(DCB *dcb, GWBUF *buffer) { - MySQLProtocol *proto = (MySQLProtocol *)dcb->protocol; - mxs_auth_state_t rval = gw_read_backend_handshake(proto, buffer); + MySQLProtocol *proto = (MySQLProtocol*)dcb->protocol; + mxs_auth_state_t rval = proto->protocol_auth_state == MXS_AUTH_STATE_CONNECTED ? + MXS_AUTH_STATE_HANDSHAKE_FAILED : MXS_AUTH_STATE_FAILED; - if (rval != MXS_AUTH_STATE_HANDSHAKE_FAILED) + int rc = dcb->authfunc.extract(dcb, buffer); + + if (rc == MXS_AUTH_SUCCEEDED || rc == MXS_AUTH_INCOMPLETE) { - MYSQL_session local_session; - gw_get_shared_session_auth_info(dcb, &local_session); + switch (dcb->authfunc.authenticate(dcb)) + { + case MXS_AUTH_INCOMPLETE: + case MXS_AUTH_SSL_INCOMPLETE: + rval = MXS_AUTH_STATE_RESPONSE_SENT; + break; - /** First message read, decode password and send the auth credentials to backend */ - rval = gw_send_auth(&local_session, proto); + case MXS_AUTH_SUCCEEDED: + rval = MXS_AUTH_STATE_COMPLETE; + + default: + break; + } } + gwbuf_free(buffer); return rval; } @@ -578,45 +580,29 @@ gw_read_backend_event(DCB *dcb) log_error_response(dcb, readbuf); } - if (proto->protocol_auth_state == MXS_AUTH_STATE_CONNECTED) + if (proto->protocol_auth_state == MXS_AUTH_STATE_CONNECTED || + proto->protocol_auth_state == MXS_AUTH_STATE_RESPONSE_SENT) { - /** TODO: Convert this to a call to dcb->authfunc.authenticate */ /** Read the first message from the server */ - proto->protocol_auth_state = handle_server_handshake(dcb, readbuf); - readbuf = NULL; + proto->protocol_auth_state = handle_server_response(dcb, readbuf); } - else + + if (proto->protocol_auth_state == MXS_AUTH_STATE_COMPLETE) { - /** - * Authentication process is ongoing. The default authentication - * plugin, mysql_native_password, sends two messages: the initial - * handshake and then either an OK or an ERR packet which signals - * whether authentication was successful - */ - if (proto->protocol_auth_state == MXS_AUTH_STATE_RESPONSE_SENT) - { - /** TODO: Convert this to a call to dcb->authfunc.authenticate */ - proto->protocol_auth_state = gw_read_auth_response(dcb, readbuf); - } + /** Authentication completed successfully */ + spinlock_acquire(&dcb->authlock); + GWBUF *localq = dcb->delayq; + dcb->delayq = NULL; + spinlock_release(&dcb->authlock); - if (proto->protocol_auth_state == MXS_AUTH_STATE_COMPLETE) + if (localq) { - /** Authentication completed successfully */ - spinlock_acquire(&dcb->authlock); - GWBUF *localq = dcb->delayq; - dcb->delayq = NULL; - spinlock_release(&dcb->authlock); - - if (localq) - { - /** Send the queued commands to the backend */ - rc = backend_write_delayqueue(dcb, localq); - } + /** Send the queued commands to the backend */ + rc = backend_write_delayqueue(dcb, localq); } } - - if (proto->protocol_auth_state == MXS_AUTH_STATE_FAILED || - proto->protocol_auth_state == MXS_AUTH_STATE_HANDSHAKE_FAILED) + else if (proto->protocol_auth_state == MXS_AUTH_STATE_FAILED || + proto->protocol_auth_state == MXS_AUTH_STATE_HANDSHAKE_FAILED) { /** Authentication failed */ gw_reply_on_error(dcb, proto->protocol_auth_state); @@ -627,168 +613,6 @@ gw_read_backend_event(DCB *dcb) return rc; } -/** - * Read the backend server MySQL handshake - * - * TODO: Move this function inside a module - * - * @param conn MySQL protocol structure - * @return 0 on success, 1 on failure - */ -static mxs_auth_state_t -gw_read_backend_handshake(MySQLProtocol *conn, GWBUF *buffer) -{ - mxs_auth_state_t rval = MXS_AUTH_STATE_HANDSHAKE_FAILED; - uint8_t *payload = GWBUF_DATA(buffer) + 4; - - if (gw_decode_mysql_server_handshake(conn, payload) >= 0) - { - rval = MXS_AUTH_STATE_MESSAGE_READ; - } - - gwbuf_free(buffer); - return rval; -} - -/** - * Write MySQL authentication packet to backend server - * - * @param conn MySQL protocol structure - * @param dbname The selected database - * @param user The selected user - * @param passwd The SHA1(real_password): Note real_password is unknown - * @return MySQL authorisation state after operation - */ -static mxs_auth_state_t -gw_send_auth(MYSQL_session *ses, MySQLProtocol *conn) -{ - uint8_t *payload; - long bytes; - uint32_t capabilities; - uint8_t client_capabilities[4] = {0, 0, 0, 0}; - GWBUF *buffer; - uint8_t *curr_passwd = memcmp(ses->client_sha1, null_client_sha1, MYSQL_SCRAMBLE_LEN) ? - ses->client_sha1 : NULL; - - /** - * If session is stopping or has failed return with error. - */ - if (conn->owner_dcb->session == NULL || - (conn->owner_dcb->session->state != SESSION_STATE_READY && - conn->owner_dcb->session->state != SESSION_STATE_ROUTER_READY) || - (conn->owner_dcb->server->server_ssl && - conn->owner_dcb->ssl_state != SSL_HANDSHAKE_FAILED)) - { - return MXS_AUTH_STATE_FAILED; - } - - capabilities = create_capabilities(conn, (ses->db && strlen(ses->db)), false); - gw_mysql_set_byte4(client_capabilities, capabilities); - - bytes = response_length(conn, ses->user, ses->client_sha1, ses->db); - - // allocating the GWBUF - buffer = gwbuf_alloc(bytes); - payload = GWBUF_DATA(buffer); - - // clearing data - memset(payload, '\0', bytes); - - // put here the paylod size: bytes to write - 4 bytes packet header - gw_mysql_set_byte3(payload, (bytes - 4)); - - // set packet # = 1 - payload[3] = (SSL_ESTABLISHED == conn->owner_dcb->ssl_state) ? '\x02' : '\x01'; - payload += 4; - - // set client capabilities - memcpy(payload, client_capabilities, 4); - - // set now the max-packet size - payload += 4; - gw_mysql_set_byte4(payload, 16777216); - - // set the charset - payload += 4; - *payload = conn->charset; - - payload++; - - // 23 bytes of 0 - payload += 23; - - if (conn->owner_dcb->server->server_ssl && conn->owner_dcb->ssl_state != SSL_ESTABLISHED) - { - if (dcb_write(conn->owner_dcb, buffer) && dcb_connect_SSL(conn->owner_dcb) >= 0) - { - return MXS_AUTH_STATE_CONNECTED; - } - - return MXS_AUTH_STATE_FAILED; - } - - // 4 + 4 + 4 + 1 + 23 = 36, this includes the 4 bytes packet header - memcpy(payload, ses->user, strlen(ses->user)); - payload += strlen(ses->user); - payload++; - - if (curr_passwd != NULL) - { - payload = load_hashed_password(conn, payload, curr_passwd); - } - else - { - payload++; - } - - // if the db is not NULL append it - if (ses->db && strlen(ses->db)) - { - memcpy(payload, ses->db, strlen(ses->db)); - payload += strlen(ses->db); - payload++; - } - - const char* auth_plugin_name = conn->owner_dcb->authfunc.plugin_name ? - conn->owner_dcb->authfunc.plugin_name : DEFAULT_MYSQL_AUTH_PLUGIN; - - memcpy(payload, auth_plugin_name, strlen(auth_plugin_name)); - - return dcb_write(conn->owner_dcb, buffer) ? MXS_AUTH_STATE_RESPONSE_SENT : MXS_AUTH_STATE_FAILED; -} - -/** - * Copy shared session authentication info - * - * @param dcb A backend DCB - * @param session Destination where authentication data is copied - * @return bool true = success, false = fail - */ -static bool gw_get_shared_session_auth_info(DCB* dcb, MYSQL_session* session) -{ - bool rval = true; - CHK_DCB(dcb); - CHK_SESSION(dcb->session); - - spinlock_acquire(&dcb->session->ses_lock); - - if (dcb->session->state != SESSION_STATE_ALLOC && - dcb->session->state != SESSION_STATE_DUMMY) - { - memcpy(session, dcb->session->client_dcb->data, sizeof(MYSQL_session)); - } - else - { - ss_dassert(false); - MXS_ERROR("%lu [gw_get_shared_session_auth_info] Couldn't get " - "session authentication info. Session in a wrong state %d.", - pthread_self(), dcb->session->state); - rval = false; - } - spinlock_release(&dcb->session->ses_lock); - return rval; -} - /** * @brief Authentication of backend - read the reply, or handle an error * @@ -829,7 +653,7 @@ gw_reply_on_error(DCB *dcb, mxs_auth_state_t state) } else { - /** A NULL router_session can valid for a router if it declared the + /** A NULL router_session is valid if a router declares the * RCAP_TYPE_NO_RSESSION capability flag */ dcb->dcb_errhandle_called = true; } @@ -837,6 +661,286 @@ gw_reply_on_error(DCB *dcb, mxs_auth_state_t state) gwbuf_free(errbuf); } +/** + * @brief Computes the size of the response to the DB initial handshake + * + * When the connection is to be SSL, but an SSL connection has not yet been + * established, only a basic 36 byte response is sent, including the SSL + * capability flag. + * + * Otherwise, the packet size is computed, based on the minimum size and + * increased by the optional or variable elements. + * + * @param conn The MySQLProtocol structure for the connection + * @param user Name of the user seeking to connect + * @param passwd Password for the user seeking to connect + * @param dbname Name of the database to be made default, if any + * @return The length of the response packet + */ +static int +response_length(MySQLProtocol *conn, char *user, uint8_t *passwd, char *dbname, const char *auth_module) +{ + long bytes; + + if (conn->owner_dcb->server->server_ssl && conn->owner_dcb->ssl_state != SSL_ESTABLISHED) + { + return 36; + } + + // Protocol MySQL HandshakeResponse for CLIENT_PROTOCOL_41 + // 4 bytes capabilities + 4 bytes max packet size + 1 byte charset + 23 '\0' bytes + // 4 + 4 + 1 + 23 = 32 + bytes = 32; + + if (user) + { + bytes += strlen(user); + } + // the NULL + bytes++; + + // next will be + 1 (scramble_len) + 20 (fixed_scramble) + 1 (user NULL term) + 1 (db NULL term) + + if (passwd) + { + bytes += GW_MYSQL_SCRAMBLE_SIZE; + } + bytes++; + + if (dbname && strlen(dbname)) + { + bytes += strlen(dbname); + bytes++; + } + + bytes += strlen(auth_module); + bytes++; + + // the packet header + bytes += 4; + + return bytes; +} + +/** + * @brief Helper function to load hashed password + * @param conn DCB Protocol object + * @param payload Destination where hashed password is written + * @param passwd Client's double SHA1 password + * @return Address of the next byte after the end of the stored password + */ +static uint8_t * +load_hashed_password(uint8_t *scramble, uint8_t *payload, uint8_t *passwd) +{ + uint8_t hash1[GW_MYSQL_SCRAMBLE_SIZE] = ""; + uint8_t hash2[GW_MYSQL_SCRAMBLE_SIZE] = ""; + uint8_t new_sha[GW_MYSQL_SCRAMBLE_SIZE] = ""; + uint8_t client_scramble[GW_MYSQL_SCRAMBLE_SIZE]; + + // hash1 is the function input, SHA1(real_password) + memcpy(hash1, passwd, GW_MYSQL_SCRAMBLE_SIZE); + + // hash2 is the SHA1(input data), where input_data = SHA1(real_password) + gw_sha1_str(hash1, GW_MYSQL_SCRAMBLE_SIZE, hash2); + + // new_sha is the SHA1(CONCAT(scramble, hash2) + gw_sha1_2_str(scramble, GW_MYSQL_SCRAMBLE_SIZE, hash2, GW_MYSQL_SCRAMBLE_SIZE, new_sha); + + // compute the xor in client_scramble + gw_str_xor(client_scramble, new_sha, hash1, GW_MYSQL_SCRAMBLE_SIZE); + + // set the auth-length + *payload = GW_MYSQL_SCRAMBLE_SIZE; + payload++; + + //copy the 20 bytes scramble data after packet_buffer + 36 + user + NULL + 1 (byte of auth-length) + memcpy(payload, client_scramble, GW_MYSQL_SCRAMBLE_SIZE); + + payload += GW_MYSQL_SCRAMBLE_SIZE; + return payload; +} + +/** + * @brief Computes the capabilities bit mask for connecting to backend DB + * + * We start by taking the default bitmask and removing any bits not set in + * the bitmask contained in the connection structure. Then add SSL flag if + * the connection requires SSL (set from the MaxScale configuration). The + * compression flag may be set, although compression is NOT SUPPORTED. If a + * database name has been specified in the function call, the relevant flag + * is set. + * + * @param conn The MySQLProtocol structure for the connection + * @param db_specified Whether the connection request specified a database + * @param compress Whether compression is requested - NOT SUPPORTED + * @return Bit mask (32 bits) + * @note Capability bits are defined in mysql_client_server_protocol.h + */ +static uint32_t +create_capabilities(MySQLProtocol *conn, bool db_specified, bool compress) +{ + uint32_t final_capabilities; + + /** Copy client's flags to backend but with the known capabilities mask */ + final_capabilities = (conn->client_capabilities & (uint32_t)GW_MYSQL_CAPABILITIES_CLIENT); + + if (conn->owner_dcb->server->server_ssl) + { + final_capabilities |= (uint32_t)GW_MYSQL_CAPABILITIES_SSL; + /* Unclear whether we should include this */ + /* Maybe it should depend on whether CA certificate is provided */ + /* final_capabilities |= (uint32_t)GW_MYSQL_CAPABILITIES_SSL_VERIFY_SERVER_CERT; */ + } + + /* Compression is not currently supported */ + if (compress) + { + final_capabilities |= (uint32_t)GW_MYSQL_CAPABILITIES_COMPRESS; +#ifdef DEBUG_MYSQL_CONN + fprintf(stderr, ">>>> Backend Connection with compression\n"); +#endif + } + + if (db_specified) + { + /* With database specified */ + final_capabilities |= (int)GW_MYSQL_CAPABILITIES_CONNECT_WITH_DB; + } + else + { + /* Without database specified */ + final_capabilities &= ~(int)GW_MYSQL_CAPABILITIES_CONNECT_WITH_DB; + } + + final_capabilities |= (int)GW_MYSQL_CAPABILITIES_PLUGIN_AUTH; + + return final_capabilities; +} + +/** + * Write MySQL authentication packet to backend server + * + * @param dcb Backend DCB + * @return True on success, false on failure + */ +mxs_auth_state_t gw_send_backend_auth(DCB *dcb) +{ + MYSQL_session local_session; + gw_get_shared_session_auth_info(dcb, &local_session); + + uint8_t client_capabilities[4] = {0, 0, 0, 0}; + uint8_t *curr_passwd = memcmp(local_session.client_sha1, null_client_sha1, MYSQL_SCRAMBLE_LEN) ? + local_session.client_sha1 : NULL; + + /** + * If session is stopping or has failed return with error. + */ + if (dcb->session == NULL || + (dcb->session->state != SESSION_STATE_READY && + dcb->session->state != SESSION_STATE_ROUTER_READY) || + (dcb->server->server_ssl && + dcb->ssl_state != SSL_HANDSHAKE_FAILED)) + { + return MXS_AUTH_STATE_FAILED; + } + + MySQLProtocol *conn = (MySQLProtocol*)dcb->protocol; + uint32_t capabilities = create_capabilities(conn, (local_session.db && strlen(local_session.db)), false); + gw_mysql_set_byte4(client_capabilities, capabilities); + + const char* auth_plugin_name = dcb->authfunc.plugin_name ? + dcb->authfunc.plugin_name : DEFAULT_MYSQL_AUTH_PLUGIN; + + long bytes = response_length(conn, local_session.user, local_session.client_sha1, + local_session.db, auth_plugin_name); + + // allocating the GWBUF + GWBUF *buffer = gwbuf_alloc(bytes); + uint8_t *payload = GWBUF_DATA(buffer); + + // clearing data + memset(payload, '\0', bytes); + + // put here the paylod size: bytes to write - 4 bytes packet header + gw_mysql_set_byte3(payload, (bytes - 4)); + + // set packet # = 1 + payload[3] = (SSL_ESTABLISHED == dcb->ssl_state) ? '\x02' : '\x01'; + payload += 4; + + // set client capabilities + memcpy(payload, client_capabilities, 4); + + // set now the max-packet size + payload += 4; + gw_mysql_set_byte4(payload, 16777216); + + // set the charset + payload += 4; + *payload = conn->charset; + + payload++; + + // 23 bytes of 0 + payload += 23; + + if (dcb->server->server_ssl && dcb->ssl_state != SSL_ESTABLISHED) + { + if (dcb_write(dcb, buffer) && dcb_connect_SSL(dcb) >= 0) + { + return MXS_AUTH_STATE_CONNECTED; + } + + return MXS_AUTH_STATE_FAILED; + } + + // 4 + 4 + 4 + 1 + 23 = 36, this includes the 4 bytes packet header + memcpy(payload, local_session.user, strlen(local_session.user)); + payload += strlen(local_session.user); + payload++; + + if (curr_passwd != NULL) + { + payload = load_hashed_password(conn->scramble, payload, curr_passwd); + } + else + { + payload++; + } + + // if the db is not NULL append it + if (local_session.db && strlen(local_session.db)) + { + memcpy(payload, local_session.db, strlen(local_session.db)); + payload += strlen(local_session.db); + payload++; + } + + memcpy(payload, auth_plugin_name, strlen(auth_plugin_name)); + + return dcb_write(dcb, buffer) ? MXS_AUTH_STATE_RESPONSE_SENT : MXS_AUTH_STATE_FAILED; +} + +/** + * Read the backend server MySQL handshake + * + * @param dcb Backend DCB + * @return true on success, false on failure + */ +bool gw_read_backend_handshake(DCB *dcb, GWBUF *buffer) +{ + MySQLProtocol *proto = (MySQLProtocol *)dcb->protocol; + bool rval = false; + uint8_t *payload = GWBUF_DATA(buffer) + 4; + + if (gw_decode_mysql_server_handshake(proto, payload) >= 0) + { + rval = true; + } + + return rval; +} + /** * @brief With authentication completed, read new data and write to backend * @@ -869,12 +973,12 @@ gw_read_and_write(DCB *dcb) "Read from backend failed"); session->service->router->handleError( - session->service->router_instance, - session->router_session, - errbuf, - dcb, - ERRACT_NEW_CONNECTION, - &succp); + session->service->router_instance, + session->router_session, + errbuf, + dcb, + ERRACT_NEW_CONNECTION, + &succp); gwbuf_free(errbuf); if (!succp) @@ -970,22 +1074,21 @@ gw_read_and_write(DCB *dcb) gwbuf_set_type(read_buffer, GWBUF_TYPE_MYSQL); session->service->router->clientReply( - session->service->router_instance, - session->router_session, - read_buffer, - dcb); + session->service->router_instance, + session->router_session, + read_buffer, + dcb); return_code = 1; } } else if (dcb->session->client_dcb->dcb_role == DCB_ROLE_INTERNAL) { gwbuf_set_type(read_buffer, GWBUF_TYPE_MYSQL); - session->service->router->clientReply( - session->service->router_instance, - session->router_session, - read_buffer, - dcb); + session->service->router_instance, + session->router_session, + read_buffer, + dcb); return_code = 1; } } @@ -1999,175 +2102,6 @@ gw_decode_mysql_server_handshake(MySQLProtocol *conn, uint8_t *payload) return 0; } -/** - * Receive the MySQL authentication packet from backend, packet # is 2 - * - * @param protocol The MySQL protocol structure - * @return -1 in case of failure, 1 if authentication was successful. - */ -static mxs_auth_state_t -gw_read_auth_response(DCB *dcb, GWBUF *buffer) -{ - mxs_auth_state_t rval = MXS_AUTH_STATE_FAILED; - uint8_t cmd; - - if (gwbuf_copy_data(buffer, MYSQL_HEADER_LEN, 1, &cmd) && cmd == MYSQL_REPLY_OK) - { - rval = MXS_AUTH_STATE_COMPLETE; - } - - return rval; -} - -/** - * @brief Computes the capabilities bit mask for connecting to backend DB - * - * We start by taking the default bitmask and removing any bits not set in - * the bitmask contained in the connection structure. Then add SSL flag if - * the connection requires SSL (set from the MaxScale configuration). The - * compression flag may be set, although compression is NOT SUPPORTED. If a - * database name has been specified in the function call, the relevant flag - * is set. - * - * @param conn The MySQLProtocol structure for the connection - * @param db_specified Whether the connection request specified a database - * @param compress Whether compression is requested - NOT SUPPORTED - * @return Bit mask (32 bits) - * @note Capability bits are defined in mysql_client_server_protocol.h - */ -static uint32_t -create_capabilities(MySQLProtocol *conn, bool db_specified, bool compress) -{ - uint32_t final_capabilities; - - /** Copy client's flags to backend but with the known capabilities mask */ - final_capabilities = (conn->client_capabilities & (uint32_t)GW_MYSQL_CAPABILITIES_CLIENT); - - if (conn->owner_dcb->server->server_ssl) - { - final_capabilities |= (uint32_t)GW_MYSQL_CAPABILITIES_SSL; - /* Unclear whether we should include this */ - /* Maybe it should depend on whether CA certificate is provided */ - /* final_capabilities |= (uint32_t)GW_MYSQL_CAPABILITIES_SSL_VERIFY_SERVER_CERT; */ - } - - /* Compression is not currently supported */ - if (compress) - { - final_capabilities |= (uint32_t)GW_MYSQL_CAPABILITIES_COMPRESS; -#ifdef DEBUG_MYSQL_CONN - fprintf(stderr, ">>>> Backend Connection with compression\n"); -#endif - } - - if (db_specified) - { - /* With database specified */ - final_capabilities |= (int)GW_MYSQL_CAPABILITIES_CONNECT_WITH_DB; - } - else - { - /* Without database specified */ - final_capabilities &= ~(int)GW_MYSQL_CAPABILITIES_CONNECT_WITH_DB; - } - - final_capabilities |= (int)GW_MYSQL_CAPABILITIES_PLUGIN_AUTH; - - return final_capabilities; -} - -/** - * @brief Computes the size of the response to the DB initial handshake - * - * When the connection is to be SSL, but an SSL connection has not yet been - * established, only a basic 36 byte response is sent, including the SSL - * capability flag. - * - * Otherwise, the packet size is computed, based on the minimum size and - * increased by the optional or variable elements. - * - * @param conn The MySQLProtocol structure for the connection - * @param user Name of the user seeking to connect - * @param passwd Password for the user seeking to connect - * @param dbname Name of the database to be made default, if any - * @return The length of the response packet - */ -static int -response_length(MySQLProtocol *conn, char *user, uint8_t *passwd, char *dbname) -{ - long bytes; - - if (conn->owner_dcb->server->server_ssl && conn->owner_dcb->ssl_state != SSL_ESTABLISHED) - { - return 36; - } - - // Protocol MySQL HandshakeResponse for CLIENT_PROTOCOL_41 - // 4 bytes capabilities + 4 bytes max packet size + 1 byte charset + 23 '\0' bytes - // 4 + 4 + 1 + 23 = 32 - bytes = 32; - - if (user) - { - bytes += strlen(user); - } - // the NULL - bytes++; - - // next will be + 1 (scramble_len) + 20 (fixed_scramble) + 1 (user NULL term) + 1 (db NULL term) - - if (passwd) - { - bytes += GW_MYSQL_SCRAMBLE_SIZE; - } - bytes++; - - if (dbname && strlen(dbname)) - { - bytes += strlen(dbname); - bytes++; - } - - bytes += strlen("mysql_native_password"); - bytes++; - - // the packet header - bytes += 4; - - return bytes; -} - -static uint8_t * -load_hashed_password(MySQLProtocol *conn, uint8_t *payload, uint8_t *passwd) -{ - uint8_t hash1[GW_MYSQL_SCRAMBLE_SIZE] = ""; - uint8_t hash2[GW_MYSQL_SCRAMBLE_SIZE] = ""; - uint8_t new_sha[GW_MYSQL_SCRAMBLE_SIZE] = ""; - uint8_t client_scramble[GW_MYSQL_SCRAMBLE_SIZE]; - - // hash1 is the function input, SHA1(real_password) - memcpy(hash1, passwd, GW_MYSQL_SCRAMBLE_SIZE); - - // hash2 is the SHA1(input data), where input_data = SHA1(real_password) - gw_sha1_str(hash1, GW_MYSQL_SCRAMBLE_SIZE, hash2); - - // new_sha is the SHA1(CONCAT(scramble, hash2) - gw_sha1_2_str(conn->scramble, GW_MYSQL_SCRAMBLE_SIZE, hash2, GW_MYSQL_SCRAMBLE_SIZE, new_sha); - - // compute the xor in client_scramble - gw_str_xor(client_scramble, new_sha, hash1, GW_MYSQL_SCRAMBLE_SIZE); - - // set the auth-length - *payload = GW_MYSQL_SCRAMBLE_SIZE; - payload++; - - //copy the 20 bytes scramble data after packet_buffer + 36 + user + NULL + 1 (byte of auth-length) - memcpy(payload, client_scramble, GW_MYSQL_SCRAMBLE_SIZE); - - payload += GW_MYSQL_SCRAMBLE_SIZE; - return payload; -} - static void inline close_socket(int sock) { diff --git a/server/modules/protocol/MySQLClient/mysql_client.c b/server/modules/protocol/MySQLClient/mysql_client.c index ed4f3f6d3..492c84b1f 100644 --- a/server/modules/protocol/MySQLClient/mysql_client.c +++ b/server/modules/protocol/MySQLClient/mysql_client.c @@ -562,12 +562,6 @@ int gw_read_client_event(DCB* dcb) static int gw_read_do_authentication(DCB *dcb, GWBUF *read_buffer, int nbytes_read) { - MySQLProtocol *protocol; - int auth_val; - - protocol = (MySQLProtocol *)dcb->protocol; - /* int compress = -1; */ - /** * The first step in the authentication process is to extract the * relevant information from the buffer supplied and place it @@ -577,18 +571,15 @@ gw_read_do_authentication(DCB *dcb, GWBUF *read_buffer, int nbytes_read) * data extraction succeeds, then a call is made to the actual * authenticate function to carry out the user checks. */ - if (MXS_AUTH_SUCCEEDED == ( - auth_val = dcb->authfunc.extract(dcb, read_buffer))) + int auth_val = dcb->authfunc.extract(dcb, read_buffer); + + if (MXS_AUTH_SUCCEEDED == auth_val) { - /* - * Maybe this comment will be useful some day: - compress = - GW_MYSQL_CAPABILITIES_COMPRESS & gw_mysql_get_byte4( - &protocol->client_capabilities); - */ auth_val = dcb->authfunc.authenticate(dcb); } + MySQLProtocol *protocol = (MySQLProtocol *)dcb->protocol; + /** * At this point, if the auth_val return code indicates success * the user authentication has been successfully completed. diff --git a/server/modules/protocol/mysql_common.c b/server/modules/protocol/mysql_common.c index 5ddd6b665..1e6989af9 100644 --- a/server/modules/protocol/mysql_common.c +++ b/server/modules/protocol/mysql_common.c @@ -52,6 +52,8 @@ #include #include +uint8_t null_client_sha1[MYSQL_SCRAMBLE_LEN] = ""; + static server_command_t* server_command_init(server_command_t* srvcmd, mysql_server_cmd_t cmd); /** @@ -1025,3 +1027,35 @@ bool read_complete_packet(DCB *dcb, GWBUF **readbuf) return rval; } + +/** + * Copy shared session authentication info + * + * @param dcb A backend DCB + * @param session Destination where authentication data is copied + * @return bool true = success, false = fail + */ +bool gw_get_shared_session_auth_info(DCB* dcb, MYSQL_session* session) +{ + bool rval = true; + CHK_DCB(dcb); + CHK_SESSION(dcb->session); + + spinlock_acquire(&dcb->session->ses_lock); + + if (dcb->session->state != SESSION_STATE_ALLOC && + dcb->session->state != SESSION_STATE_DUMMY) + { + memcpy(session, dcb->session->client_dcb->data, sizeof(MYSQL_session)); + } + else + { + ss_dassert(false); + MXS_ERROR("%lu [gw_get_shared_session_auth_info] Couldn't get " + "session authentication info. Session in a wrong state %d.", + pthread_self(), dcb->session->state); + rval = false; + } + spinlock_release(&dcb->session->ses_lock); + return rval; +} From 829d5a7453100ff59912822b4a19a00a545cdde5 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 22 Sep 2016 16:08:31 +0300 Subject: [PATCH 005/215] Send COM_QUIT to backends if persistent connections are disabled The COM_QUIT packets should be sent to the backends if persistent connections aren't used. This allows for a controlled shutdown of the connections on both ends even if the client closes the connection before all backends have authenticated. --- .../protocol/MySQLBackend/mysql_backend.c | 27 +++- .../protocol/MySQLClient/mysql_client.c | 144 ++++++++---------- 2 files changed, 89 insertions(+), 82 deletions(-) diff --git a/server/modules/protocol/MySQLBackend/mysql_backend.c b/server/modules/protocol/MySQLBackend/mysql_backend.c index 08c7db9d5..a89158cf5 100644 --- a/server/modules/protocol/MySQLBackend/mysql_backend.c +++ b/server/modules/protocol/MySQLBackend/mysql_backend.c @@ -1229,8 +1229,18 @@ static int gw_MySQLWrite_backend(DCB *dcb, GWBUF *queue) /** Record the command to backend's protocol */ protocol_add_srv_command(backend_protocol, cmd); } - /** Write to backend */ - rc = dcb_write(dcb, queue); + + if (cmd == MYSQL_COM_QUIT && dcb->server->persistpoolmax) + { + /** We need to keep the pooled connections alive so we just ignore the COM_QUIT packet */ + gwbuf_free(queue); + rc = 1; + } + else + { + /** Write to backend */ + rc = dcb_write(dcb, queue); + } } break; @@ -1587,7 +1597,18 @@ static int backend_write_delayqueue(DCB *dcb, GWBUF *buffer) buffer = gw_create_change_user_packet(&mses, dcb->protocol); } - int rc = dcb_write(dcb, buffer); + int rc = 1; + + if (MYSQL_IS_COM_QUIT(((uint8_t*)GWBUF_DATA(buffer))) && dcb->server->persistpoolmax) + { + /** We need to keep the pooled connections alive so we just ignore the COM_QUIT packet */ + gwbuf_free(buffer); + rc = 1; + } + else + { + rc = dcb_write(dcb, buffer); + } if (rc == 0) { diff --git a/server/modules/protocol/MySQLClient/mysql_client.c b/server/modules/protocol/MySQLClient/mysql_client.c index 492c84b1f..4cfdb54cd 100644 --- a/server/modules/protocol/MySQLClient/mysql_client.c +++ b/server/modules/protocol/MySQLClient/mysql_client.c @@ -862,92 +862,78 @@ gw_read_finish_processing(DCB *dcb, GWBUF *read_buffer, uint8_t capabilities) SESSION *session = dcb->session; uint8_t *payload = GWBUF_DATA(read_buffer); MySQLProtocol *proto = (MySQLProtocol*)dcb->protocol; - CHK_PROTOCOL(proto) + CHK_PROTOCOL(proto); int return_code = 0; - /* Now, we are assuming in the first buffer there is - * the information for mysql command */ - /* Route COM_QUIT to backend */ + /** Reset error handler when routing of the new query begins */ + dcb->dcb_errhandle_called = false; + + if (capabilities & (int)RCAP_TYPE_STMT_INPUT) + { + /** + * Feed each statement completely and separately + * to router. The routing functions return 1 for + * success or 0 for failure. + */ + return_code = route_by_statement(session, &read_buffer) ? 0 : 1; + + if (read_buffer != NULL) + { + /* Must have been data left over */ + /* Add incomplete mysql packet to read queue */ + spinlock_acquire(&dcb->authlock); + dcb->dcb_readqueue = gwbuf_append(dcb->dcb_readqueue, read_buffer); + spinlock_release(&dcb->authlock); + } + } + else if (NULL != session->router_session || (capabilities & (int)RCAP_TYPE_NO_RSESSION)) + { + /** Feed whole packet to router, which will free it + * and return 1 for success, 0 for failure + */ + return_code = SESSION_ROUTE_QUERY(session, read_buffer) ? 0 : 1; + } + /* else return_code is still 0 from when it was originally set */ + /* Note that read_buffer has been freed or transferred by this point */ + + /** Routing failed */ + if (return_code != 0) + { + bool router_can_continue; + GWBUF* errbuf; + /** + * Create error to be sent to client if session + * can't be continued. + */ + errbuf = mysql_create_custom_error(1, 0, + "Routing failed. Session is closed."); + /** + * Ensure that there are enough backends + * available for router to continue operation. + */ + session->service->router->handleError(session->service->router_instance, + session->router_session, + errbuf, + dcb, + ERRACT_NEW_CONNECTION, + &router_can_continue); + gwbuf_free(errbuf); + /** + * If the router cannot continue, close session + */ + if (!router_can_continue) + { + MXS_ERROR("Routing the query failed. " + "Session will be closed."); + } + } + if (proto->current_command == MYSQL_COM_QUIT) { - /** - * Sends COM_QUIT packets since buffer is already - * created. A BREF_CLOSED flag is set so dcb_close won't - * send redundant COM_QUIT. - */ - /* Temporarily suppressed: SESSION_ROUTE_QUERY(session, read_buffer); */ - /* Replaced with freeing the read buffer. */ - gwbuf_free(read_buffer); - /** - * Close router session which causes closing of backends. - */ + /** Close router session which causes closing of backends */ dcb_close(dcb); } - else - { - /** Reset error handler when routing of the new query begins */ - dcb->dcb_errhandle_called = false; - if (capabilities & (int)RCAP_TYPE_STMT_INPUT) - { - /** - * Feed each statement completely and separately - * to router. The routing functions return 1 for - * success or 0 for failure. - */ - return_code = route_by_statement(session, &read_buffer) ? 0 : 1; - - if (read_buffer != NULL) - { - /* Must have been data left over */ - /* Add incomplete mysql packet to read queue */ - spinlock_acquire(&dcb->authlock); - dcb->dcb_readqueue = gwbuf_append(dcb->dcb_readqueue, read_buffer); - spinlock_release(&dcb->authlock); - } - } - else if (NULL != session->router_session || (capabilities & (int)RCAP_TYPE_NO_RSESSION)) - { - /** Feed whole packet to router, which will free it - * and return 1 for success, 0 for failure - */ - return_code = SESSION_ROUTE_QUERY(session, read_buffer) ? 0 : 1; - } - /* else return_code is still 0 from when it was originally set */ - /* Note that read_buffer has been freed or transferred by this point */ - - /** Routing failed */ - if (return_code != 0) - { - bool router_can_continue; - GWBUF* errbuf; - /** - * Create error to be sent to client if session - * can't be continued. - */ - errbuf = mysql_create_custom_error(1, 0, - "Routing failed. Session is closed."); - /** - * Ensure that there are enough backends - * available for router to continue operation. - */ - session->service->router->handleError(session->service->router_instance, - session->router_session, - errbuf, - dcb, - ERRACT_NEW_CONNECTION, - &router_can_continue); - gwbuf_free(errbuf); - /** - * If the router cannot continue, close session - */ - if (!router_can_continue) - { - MXS_ERROR("Routing the query failed. " - "Session will be closed."); - } - } - } return return_code; } From dfeb5c46c96964ebcc27baffcd0cfb12973b50a5 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 4 Oct 2016 12:06:52 +0300 Subject: [PATCH 006/215] MXS-862: Add create/destroy and remove plugin_name entry points The create and destroy entry points allow authenticators to store data in the DCB. This data is not shared by other DCBs related to the same session. The plugin_name entry point wasn't really useful as the plugins would still need to send a AuthSwitchRequest packet if they wanted to change the authentication mechanism. --- server/core/dcb.c | 19 ++++++-- server/core/service.c | 8 +++- server/include/dcb.h | 6 +-- server/include/gw_authenticator.h | 7 ++- server/modules/authenticator/cdc_plain_auth.c | 13 +++--- server/modules/authenticator/http_auth.c | 13 +++--- server/modules/authenticator/max_admin_auth.c | 13 +++--- server/modules/authenticator/mysql_auth.c | 33 +++++--------- .../authenticator/mysql_backend_auth.c | 43 ++++++++++--------- .../modules/authenticator/null_auth_allow.c | 13 +++--- server/modules/authenticator/null_auth_deny.c | 13 +++--- .../include/mysql_client_server_protocol.h | 5 +++ .../protocol/MySQLBackend/mysql_backend.c | 10 +++-- .../protocol/MySQLClient/mysql_client.c | 29 +++++++++++-- server/modules/protocol/mysql_common.c | 19 ++++++++ 15 files changed, 155 insertions(+), 89 deletions(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index c0f568f25..74f49b539 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -397,10 +397,10 @@ dcb_free_all_memory(DCB *dcb) dcb->authfunc.free(dcb); dcb->data = NULL; } - if (dcb->backend_data && dcb->authfunc.free && dcb->dcb_role == DCB_ROLE_BACKEND_HANDLER) + if (dcb->authfunc.destroy) { - dcb->authfunc.free(dcb); - dcb->backend_data = NULL; + dcb->authfunc.destroy(dcb->authenticator_data); + dcb->authenticator_data = NULL; } if (dcb->protoname) { @@ -814,6 +814,12 @@ dcb_connect(SERVER *server, SESSION *session, const char *protocol) memcpy(&dcb->authfunc, authfuncs, sizeof(GWAUTHENTICATOR)); + /** Allocate DCB specific authentication data */ + if (dcb->authfunc.create) + { + dcb->authenticator_data = dcb->authfunc.create(); + } + /** * Link dcb to session. Unlink is called in dcb_final_free */ @@ -3172,6 +3178,13 @@ dcb_accept(DCB *listener, GWPROTOCOL *protocol_funcs) } } memcpy(&(client_dcb->authfunc), authfuncs, sizeof(GWAUTHENTICATOR)); + + /** Allocate DCB specific authentication data */ + if (client_dcb->authfunc.create) + { + client_dcb->authenticator_data = client_dcb->authfunc.create(); + } + if (client_dcb->service->max_connections && client_dcb->service->client_count >= client_dcb->service->max_connections) { diff --git a/server/core/service.c b/server/core/service.c index 10535c157..b4304e238 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -288,6 +288,11 @@ serviceStartPort(SERVICE *service, SERV_LISTENER *port) memcpy(&port->listener->authfunc, authfuncs, sizeof(GWAUTHENTICATOR)); + /** + * Normally, we'd allocate the DCB specific authentication data. As the + * listeners aren't normal DCBs, we can skip that. + */ + if (port->address) { sprintf(config_bind, "%s:%d", port->address, port->port); @@ -1457,7 +1462,8 @@ int service_refresh_users(SERVICE *service) for (SERV_LISTENER *port = service->ports; port; port = port->next) { - if (port->listener->authfunc.loadusers(port) != MXS_AUTH_LOADUSERS_OK) + if (port->listener->authfunc.loadusers && + port->listener->authfunc.loadusers(port) != MXS_AUTH_LOADUSERS_OK) { MXS_ERROR("[%s] Failed to load users for listener '%s', authentication might not work.", service->name, port->name); diff --git a/server/include/dcb.h b/server/include/dcb.h index 29c934829..679ebb4cf 100644 --- a/server/include/dcb.h +++ b/server/include/dcb.h @@ -252,8 +252,8 @@ typedef struct dcb struct dcb *nextpersistent; /**< Next DCB in the persistent pool for SERVER */ time_t persistentstart; /**< Time when DCB placed in persistent pool */ struct service *service; /**< The related service */ - void *data; /**< Specific client data */ - void *backend_data; /**< Specific backend data */ + void *data; /**< Specific client data, shared between DCBs of this session */ + void *authenticator_data; /**< The authenticator data for this DCB */ DCBMM memdata; /**< The data related to DCB memory management */ SPINLOCK cb_lock; /**< The lock for the callbacks linked list */ DCB_CALLBACK *callbacks; /**< The list of callbacks for the DCB */ @@ -284,7 +284,7 @@ typedef struct dcb .cb_lock = SPINLOCK_INIT, .pollinlock = SPINLOCK_INIT, \ .fd = DCBFD_CLOSED, .stats = DCBSTATS_INIT, .ssl_state = SSL_HANDSHAKE_UNKNOWN, \ .state = DCB_STATE_ALLOC, .polloutlock = SPINLOCK_INIT, .dcb_chk_tail = CHK_NUM_DCB, \ - .backend_data = NULL} + .authenticator_data = NULL} /** * The DCB usage filer used for returning DCB's in use for a certain reason diff --git a/server/include/gw_authenticator.h b/server/include/gw_authenticator.h index ee66c1764..24c06b532 100644 --- a/server/include/gw_authenticator.h +++ b/server/include/gw_authenticator.h @@ -42,12 +42,14 @@ struct servlistener; * @verbatim * The operations that can be performed on the descriptor * + * create Create a data structure unique to this DCB, stored in dcb->authenticator_data * extract Extract the data from a buffer and place in a structure + * shared at the session level, stored in dcb->data * connectssl Determine whether the connection can support SSL * authenticate Carry out the authentication * free Free extracted data + * destroy Destroy the unique DCB data * loadusers Load or update authenticator user data - * plugin_name The protocol specific name of the authentication plugin. * @endverbatim * * This forms the "module object" for authenticator modules within the gateway. @@ -56,12 +58,13 @@ struct servlistener; */ typedef struct gw_authenticator { + void *(*create)(); int (*extract)(struct dcb *, GWBUF *); bool (*connectssl)(struct dcb *); int (*authenticate)(struct dcb *); void (*free)(struct dcb *); + void (*destroy)(void *); int (*loadusers)(struct servlistener *); - const char* plugin_name; } GWAUTHENTICATOR; /** Return values for extract and authenticate entry points */ diff --git a/server/modules/authenticator/cdc_plain_auth.c b/server/modules/authenticator/cdc_plain_auth.c index 3b6fc93ed..bdecec83f 100644 --- a/server/modules/authenticator/cdc_plain_auth.c +++ b/server/modules/authenticator/cdc_plain_auth.c @@ -67,12 +67,13 @@ extern char *decryptPassword(char *crypt); */ static GWAUTHENTICATOR MyObject = { - cdc_auth_set_protocol_data, /* Extract data into structure */ - cdc_auth_is_client_ssl_capable, /* Check if client supports SSL */ - cdc_auth_authenticate, /* Authenticate user credentials */ - cdc_auth_free_client_data, /* Free the client data held in DCB */ - cdc_replace_users, - NULL + NULL, /* No create entry point */ + cdc_auth_set_protocol_data, /* Extract data into structure */ + cdc_auth_is_client_ssl_capable, /* Check if client supports SSL */ + cdc_auth_authenticate, /* Authenticate user credentials */ + cdc_auth_free_client_data, /* Free the client data held in DCB */ + NULL, /* No destroy entry point */ + cdc_replace_users /* Load CDC users */ }; static int cdc_auth_check( diff --git a/server/modules/authenticator/http_auth.c b/server/modules/authenticator/http_auth.c index 9d902afbd..981f7148a 100644 --- a/server/modules/authenticator/http_auth.c +++ b/server/modules/authenticator/http_auth.c @@ -59,12 +59,13 @@ static void http_auth_free_client_data(DCB *dcb); */ static GWAUTHENTICATOR MyObject = { - http_auth_set_protocol_data, /* Extract data into structure */ - http_auth_is_client_ssl_capable, /* Check if client supports SSL */ - http_auth_authenticate, /* Authenticate user credentials */ - http_auth_free_client_data, /* Free the client data held in DCB */ - users_default_loadusers, - NULL + NULL, /* No create entry point */ + http_auth_set_protocol_data, /* Extract data into structure */ + http_auth_is_client_ssl_capable, /* Check if client supports SSL */ + http_auth_authenticate, /* Authenticate user credentials */ + http_auth_free_client_data, /* Free the client data held in DCB */ + NULL, /* No destroy entry point */ + users_default_loadusers /* Load generic users */ }; typedef struct http_auth diff --git a/server/modules/authenticator/max_admin_auth.c b/server/modules/authenticator/max_admin_auth.c index ce5b85e5d..42f3fef03 100644 --- a/server/modules/authenticator/max_admin_auth.c +++ b/server/modules/authenticator/max_admin_auth.c @@ -59,12 +59,13 @@ static void max_admin_auth_free_client_data(DCB *dcb); */ static GWAUTHENTICATOR MyObject = { - max_admin_auth_set_protocol_data, /* Extract data into structure */ - max_admin_auth_is_client_ssl_capable, /* Check if client supports SSL */ - max_admin_auth_authenticate, /* Authenticate user credentials */ - max_admin_auth_free_client_data, /* Free the client data held in DCB */ - users_default_loadusers, - NULL + NULL, /* No create entry point */ + max_admin_auth_set_protocol_data, /* Extract data into structure */ + max_admin_auth_is_client_ssl_capable, /* Check if client supports SSL */ + max_admin_auth_authenticate, /* Authenticate user credentials */ + max_admin_auth_free_client_data, /* Free the client data held in DCB */ + NULL, /* No destroy entry point */ + users_default_loadusers /* Load generic users */ }; /** diff --git a/server/modules/authenticator/mysql_auth.c b/server/modules/authenticator/mysql_auth.c index 253b337f3..874640f0d 100644 --- a/server/modules/authenticator/mysql_auth.c +++ b/server/modules/authenticator/mysql_auth.c @@ -62,12 +62,13 @@ static int mysql_auth_load_users(SERV_LISTENER *port); */ static GWAUTHENTICATOR MyObject = { - mysql_auth_set_protocol_data, /* Extract data into structure */ - mysql_auth_is_client_ssl_capable, /* Check if client supports SSL */ - mysql_auth_authenticate, /* Authenticate user credentials */ - mysql_auth_free_client_data, /* Free the client data held in DCB */ - mysql_auth_load_users, /* Load users from backend databases */ - "mysql_native_password" + NULL, /* No create entry point */ + mysql_auth_set_protocol_data, /* Extract data into structure */ + mysql_auth_is_client_ssl_capable, /* Check if client supports SSL */ + mysql_auth_authenticate, /* Authenticate user credentials */ + mysql_auth_free_client_data, /* Free the client data held in DCB */ + NULL, /* No destroy entry point */ + mysql_auth_load_users /* Load users from backend databases */ }; static int combined_auth_check( @@ -243,22 +244,8 @@ mysql_auth_set_protocol_data(DCB *dcb, GWBUF *buf) protocol = DCB_PROTOCOL(dcb, MySQLProtocol); CHK_PROTOCOL(protocol); - if (dcb->data == NULL) - { - if (NULL == (client_data = (MYSQL_session *)MXS_CALLOC(1, sizeof(MYSQL_session)))) - { - return MXS_AUTH_FAILED; - } -#if defined(SS_DEBUG) - client_data->myses_chk_top = CHK_NUM_MYSQLSES; - client_data->myses_chk_tail = CHK_NUM_MYSQLSES; -#endif - dcb->data = client_data; - } - else - { - client_data = (MYSQL_session *)dcb->data; - } + + client_data = (MYSQL_session *)dcb->data; client_auth_packet_size = gwbuf_length(buf); @@ -311,7 +298,7 @@ mysql_auth_set_client_data( gwbuf_copy_data(buffer, 0, client_auth_packet_size, client_auth_packet); /* The numbers are the fixed elements in the client handshake packet */ - int auth_packet_base_size = 4 + 4 + 4 + 1 + 23; + int auth_packet_base_size = MYSQL_AUTH_PACKET_BASE_SIZE; int packet_length_used = 0; /* Take data from fixed locations first */ diff --git a/server/modules/authenticator/mysql_backend_auth.c b/server/modules/authenticator/mysql_backend_auth.c index 7b8b7111f..dbabdd651 100644 --- a/server/modules/authenticator/mysql_backend_auth.c +++ b/server/modules/authenticator/mysql_backend_auth.c @@ -50,7 +50,7 @@ typedef struct mysql_backend_auth * @brief Allocate a new mysql_backend_auth object * @return Allocated object or NULL if memory allocation failed */ -mysql_backend_auth_t* mba_alloc() +void* auth_backend_create() { mysql_backend_auth_t* mba = MXS_MALLOC(sizeof(*mba)); @@ -62,6 +62,18 @@ mysql_backend_auth_t* mba_alloc() return mba; } +/** + * @brief Free allocated mysql_backend_auth object + * @param data Allocated mysql_backend_auth object + */ +void auth_backend_destroy(void *data) +{ + if (data) + { + MXS_FREE(data); + } +} + /** * Receive the MySQL authentication packet from backend, packet # is 2 * @@ -95,9 +107,9 @@ auth_backend_extract(DCB *dcb, GWBUF *buf) { int rval = MXS_AUTH_FAILED; - if (dcb->backend_data || (dcb->backend_data = mba_alloc())) + if (dcb->authenticator_data) { - mysql_backend_auth_t *mba = (mysql_backend_auth_t*)dcb->backend_data; + mysql_backend_auth_t *mba = (mysql_backend_auth_t*)dcb->authenticator_data; switch (mba->state) { @@ -146,7 +158,7 @@ static int auth_backend_authenticate(DCB *dcb) { int rval = MXS_AUTH_FAILED; - mysql_backend_auth_t *mba = (mysql_backend_auth_t*)dcb->backend_data; + mysql_backend_auth_t *mba = (mysql_backend_auth_t*)dcb->authenticator_data; if (mba->state == MBA_SEND_RESPONSE) { @@ -192,16 +204,6 @@ auth_backend_ssl(DCB *dcb) return dcb->server->server_ssl != NULL; } -/** - * @brief Dummy function for the free entry point - */ -static void -auth_backend_free(DCB *dcb) -{ - MXS_FREE(dcb->backend_data); - dcb->backend_data = NULL; -} - /** * @brief Dummy function for the loadusers entry point */ @@ -230,12 +232,13 @@ static char *version_str = "V1.0.0"; */ static GWAUTHENTICATOR MyObject = { - auth_backend_extract, /* Extract data into structure */ - auth_backend_ssl, /* Check if client supports SSL */ - auth_backend_authenticate, /* Authenticate user credentials */ - auth_backend_free, /* Free the client data held in DCB */ - auth_backend_load_users, /* Load users from backend databases */ - DEFAULT_MYSQL_AUTH_PLUGIN + auth_backend_create, /* Create authenticator */ + auth_backend_extract, /* Extract data into structure */ + auth_backend_ssl, /* Check if client supports SSL */ + auth_backend_authenticate, /* Authenticate user credentials */ + NULL, /* The shared data is freed by the client DCB */ + auth_backend_destroy, /* Destroy authenticator */ + NULL /* We don't need to load users */ }; /** diff --git a/server/modules/authenticator/null_auth_allow.c b/server/modules/authenticator/null_auth_allow.c index ff3b595b5..463500f74 100644 --- a/server/modules/authenticator/null_auth_allow.c +++ b/server/modules/authenticator/null_auth_allow.c @@ -58,12 +58,13 @@ static void null_auth_free_client_data(DCB *dcb); */ static GWAUTHENTICATOR MyObject = { - null_auth_set_protocol_data, /* Extract data into structure */ - null_auth_is_client_ssl_capable, /* Check if client supports SSL */ - null_auth_authenticate, /* Authenticate user credentials */ - null_auth_free_client_data, /* Free the client data held in DCB */ - users_default_loadusers, - NULL + NULL, /* No create entry point */ + null_auth_set_protocol_data, /* Extract data into structure */ + null_auth_is_client_ssl_capable, /* Check if client supports SSL */ + null_auth_authenticate, /* Authenticate user credentials */ + null_auth_free_client_data, /* Free the client data held in DCB */ + NULL, /* No destroy entry point */ + users_default_loadusers /* Load generic users */ }; /** diff --git a/server/modules/authenticator/null_auth_deny.c b/server/modules/authenticator/null_auth_deny.c index 59d11edcf..cdebefe49 100644 --- a/server/modules/authenticator/null_auth_deny.c +++ b/server/modules/authenticator/null_auth_deny.c @@ -58,12 +58,13 @@ static void null_auth_free_client_data(DCB *dcb); */ static GWAUTHENTICATOR MyObject = { - null_auth_set_protocol_data, /* Extract data into structure */ - null_auth_is_client_ssl_capable, /* Check if client supports SSL */ - null_auth_authenticate, /* Authenticate user credentials */ - null_auth_free_client_data, /* Free the client data held in DCB */ - users_default_loadusers, - NULL + NULL, /* No create entry point */ + null_auth_set_protocol_data, /* Extract data into structure */ + null_auth_is_client_ssl_capable, /* Check if client supports SSL */ + null_auth_authenticate, /* Authenticate user credentials */ + null_auth_free_client_data, /* Free the client data held in DCB */ + NULL, /* No destroy entry point */ + users_default_loadusers /* Load generic users */ }; /** diff --git a/server/modules/include/mysql_client_server_protocol.h b/server/modules/include/mysql_client_server_protocol.h index 41c8ca05c..c19292f9d 100644 --- a/server/modules/include/mysql_client_server_protocol.h +++ b/server/modules/include/mysql_client_server_protocol.h @@ -81,6 +81,9 @@ #define DEFAULT_MYSQL_AUTH_PLUGIN "mysql_native_password" +/** All authentication responses are at least this many bytes long */ +#define MYSQL_AUTH_PACKET_BASE_SIZE 36 + /** Maximum length of a MySQL packet */ #define MYSQL_PACKET_LENGTH_MAX 0x00ffffff @@ -297,6 +300,8 @@ typedef struct /* The following can be compared using memcmp to detect a null password */ extern uint8_t null_client_sha1[MYSQL_SCRAMBLE_LEN]; +MYSQL_session* mysql_session_alloc(); + MySQLProtocol* mysql_protocol_init(DCB* dcb, int fd); void mysql_protocol_done (DCB* dcb); const char *gw_mysql_protocol_state2string(int state); diff --git a/server/modules/protocol/MySQLBackend/mysql_backend.c b/server/modules/protocol/MySQLBackend/mysql_backend.c index a89158cf5..dfaec2b07 100644 --- a/server/modules/protocol/MySQLBackend/mysql_backend.c +++ b/server/modules/protocol/MySQLBackend/mysql_backend.c @@ -848,8 +848,12 @@ mxs_auth_state_t gw_send_backend_auth(DCB *dcb) uint32_t capabilities = create_capabilities(conn, (local_session.db && strlen(local_session.db)), false); gw_mysql_set_byte4(client_capabilities, capabilities); - const char* auth_plugin_name = dcb->authfunc.plugin_name ? - dcb->authfunc.plugin_name : DEFAULT_MYSQL_AUTH_PLUGIN; + /** + * Use the default authentication plugin name. If the server is using a + * different authentication mechanism, it will send an AuthSwitchRequest + * packet. + */ + const char* auth_plugin_name = DEFAULT_MYSQL_AUTH_PLUGIN; long bytes = response_length(conn, local_session.user, local_session.client_sha1, local_session.db, auth_plugin_name); @@ -909,7 +913,7 @@ mxs_auth_state_t gw_send_backend_auth(DCB *dcb) } // if the db is not NULL append it - if (local_session.db && strlen(local_session.db)) + if (local_session.db[0]) { memcpy(payload, local_session.db, strlen(local_session.db)); payload += strlen(local_session.db); diff --git a/server/modules/protocol/MySQLClient/mysql_client.c b/server/modules/protocol/MySQLClient/mysql_client.c index 4cfdb54cd..2c021a4f4 100644 --- a/server/modules/protocol/MySQLClient/mysql_client.c +++ b/server/modules/protocol/MySQLClient/mysql_client.c @@ -311,8 +311,12 @@ int MySQLSendHandshake(DCB* dcb) memcpy(mysql_plugin_data, server_scramble + 8, 12); - const char* plugin_name = dcb->authfunc.plugin_name ? - dcb->authfunc.plugin_name : DEFAULT_MYSQL_AUTH_PLUGIN; + /** + * Use the default authentication plugin name in the initial handshake. If the + * authenticator needs to change the authentication method, it should send + * an AuthSwitchRequest packet to the client. + */ + const char* plugin_name = DEFAULT_MYSQL_AUTH_PLUGIN; int plugin_name_len = strlen(plugin_name); mysql_payload_size = @@ -562,6 +566,13 @@ int gw_read_client_event(DCB* dcb) static int gw_read_do_authentication(DCB *dcb, GWBUF *read_buffer, int nbytes_read) { + /** Allocate the shared session structure */ + if (dcb->data == NULL && (dcb->data = mysql_session_alloc()) == NULL) + { + dcb_close(dcb); + return 1; + } + /** * The first step in the authentication process is to extract the * relevant information from the buffer supplied and place it @@ -590,7 +601,17 @@ gw_read_do_authentication(DCB *dcb, GWBUF *read_buffer, int nbytes_read) */ if (MXS_AUTH_SUCCEEDED == auth_val) { - SESSION *session; + if (dcb->user == NULL) + { + /** User authentication complete, copy the username to the DCB */ + MYSQL_session *ses = dcb->data; + if ((dcb->user = MXS_STRDUP(ses->user)) == NULL) + { + dcb_close(dcb); + gwbuf_free(read_buffer); + return 0; + } + } protocol->protocol_auth_state = MXS_AUTH_STATE_RESPONSE_SENT; /** @@ -600,7 +621,7 @@ gw_read_do_authentication(DCB *dcb, GWBUF *read_buffer, int nbytes_read) * is changed so that future data will go through the * normal data handling function instead of this one. */ - session = session_alloc(dcb->service, dcb); + SESSION *session = session_alloc(dcb->service, dcb); if (session != NULL) { diff --git a/server/modules/protocol/mysql_common.c b/server/modules/protocol/mysql_common.c index 1e6989af9..c0c416b67 100644 --- a/server/modules/protocol/mysql_common.c +++ b/server/modules/protocol/mysql_common.c @@ -56,6 +56,25 @@ uint8_t null_client_sha1[MYSQL_SCRAMBLE_LEN] = ""; static server_command_t* server_command_init(server_command_t* srvcmd, mysql_server_cmd_t cmd); +/** + * @brief Allocate a new MySQL_session + * @return New MySQL_session or NULL if memory allocation failed + */ +MYSQL_session* mysql_session_alloc() +{ + MYSQL_session *ses = MXS_CALLOC(1, sizeof(MYSQL_session)); + + if (ses) + { +#ifdef SS_DEBUG + ses->myses_chk_top = CHK_NUM_MYSQLSES; + ses->myses_chk_tail = CHK_NUM_MYSQLSES; +#endif + } + + return ses; +} + /** * Creates MySQL protocol structure * From d87f15b7f648e4d625088adb77d4d22ce564919f Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 4 Oct 2016 12:31:53 +0300 Subject: [PATCH 007/215] MXS-862: Add client side GSSAPI authenticator Initial implementation of the client side GSSAPI authenticator. The current version successfully authenticates clients using the same style as the MariaDB GSSAPI plugin does. Currently, it is not possible to acutally use the plugin as the backend server would require the matchig GSSAPI plugin. Also added skeleton code for the backend GSSAPI authenticator. It only implements the required entry points and sends the client auth packet to the backend. --- server/modules/authenticator/CMakeLists.txt | 11 + server/modules/authenticator/gssapi_auth.c | 382 ++++++++++++++++++ server/modules/authenticator/gssapi_auth.h | 43 ++ .../authenticator/gssapi_auth_common.c | 35 ++ .../authenticator/gssapi_backend_auth.c | 86 ++++ 5 files changed, 557 insertions(+) create mode 100644 server/modules/authenticator/gssapi_auth.c create mode 100644 server/modules/authenticator/gssapi_auth.h create mode 100644 server/modules/authenticator/gssapi_auth_common.c create mode 100644 server/modules/authenticator/gssapi_backend_auth.c diff --git a/server/modules/authenticator/CMakeLists.txt b/server/modules/authenticator/CMakeLists.txt index d15bd8310..bba4feb42 100644 --- a/server/modules/authenticator/CMakeLists.txt +++ b/server/modules/authenticator/CMakeLists.txt @@ -8,6 +8,17 @@ target_link_libraries(MySQLBackendAuth maxscale-common MySQLBackend) set_target_properties(MySQLBackendAuth PROPERTIES VERSION "1.0.0") install_module(MySQLBackendAuth core) +add_library(GSSAPIAuth SHARED gssapi_auth.c gssapi_auth_common.c) +target_link_libraries(GSSAPIAuth maxscale-common gssapi_krb5) +set_target_properties(GSSAPIAuth PROPERTIES VERSION "1.0.0") +install_module(GSSAPIAuth core) + +add_library(GSSAPIBackendAuth SHARED gssapi_backend_auth.c gssapi_auth_common.c) +target_link_libraries(GSSAPIBackendAuth maxscale-common gssapi_krb5 + MySQLBackend) # Needed for gw_send_backend_auth +set_target_properties(GSSAPIBackendAuth PROPERTIES VERSION "1.0.0") +install_module(GSSAPIBackendAuth core) + add_library(NullAuthAllow SHARED null_auth_allow.c) target_link_libraries(NullAuthAllow maxscale-common) set_target_properties(NullAuthAllow PROPERTIES VERSION "1.0.0") diff --git a/server/modules/authenticator/gssapi_auth.c b/server/modules/authenticator/gssapi_auth.c new file mode 100644 index 000000000..ac13f6b9c --- /dev/null +++ b/server/modules/authenticator/gssapi_auth.c @@ -0,0 +1,382 @@ +/* + * Copyright (c) 2016 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl. + * + * Change Date: 2019-07-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ + +#include +#include +#include +#include +#include +#include "gssapi_auth.h" + +/** + * @brief Create a AuthSwitchRequest packet + * + * This function also contains the first part of the GSSAPI authentication. + * The server (MaxScale) send the principal name that will be used to generate + * the token the client will send us. The principal name needs to exist in the + * GSSAPI server in order for the client to be able to request a token. + * + * @return Allocated packet or NULL if memory allocation failed + * @see https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchRequest + * @see https://web.mit.edu/kerberos/krb5-1.5/krb5-1.5.4/doc/krb5-user/What-is-a-Kerberos-Principal_003f.html + */ +static GWBUF* create_auth_change_packet() +{ + + size_t plen = sizeof(auth_plugin_name) + 1 + sizeof(default_princ_name) - 1; + GWBUF *buffer = gwbuf_alloc(plen + MYSQL_HEADER_LEN); + + if (buffer) + { + uint8_t *data = (uint8_t*)GWBUF_DATA(buffer); + gw_mysql_set_byte3(data, plen); + data += 3; + *data++ = 0x02; // Second packet + *data++ = 0xfe; // AuthSwitchRequest command + memcpy(data, auth_plugin_name, sizeof(auth_plugin_name)); // Plugin name + data += sizeof(auth_plugin_name); + memcpy(data, default_princ_name, sizeof(default_princ_name) - 1); // Plugin data + } + + return buffer; +} + +/** + * @brief Store the client's GSSAPI token + * + * This token will be shared with all the DCBs for this session when the backend + * GSSAPI authentication is done. + * + * @param dcb Client DCB + * @param buffer Buffer containing the key + * @return True on success, false if memory allocation failed + */ +bool store_client_token(DCB *dcb, GWBUF *buffer) +{ + bool rval = false; + uint8_t hdr[MYSQL_HEADER_LEN]; + + if (gwbuf_copy_data(buffer, 0, MYSQL_HEADER_LEN, hdr) == MYSQL_HEADER_LEN) + { + size_t plen = gw_mysql_get_byte3(hdr); + MYSQL_session *ses = (MYSQL_session*)dcb->data; + + if ((ses->auth_token = MXS_MALLOC(plen))) + { + gwbuf_copy_data(buffer, MYSQL_HEADER_LEN, plen, ses->auth_token); + ses->auth_token_len = plen; + rval = true; + } + } + + return rval; +} + +/** + * @brief Copy username to shared session data + * @param dcb Client DCB + * @param buffer Buffer containing the first authentication response + */ +static void copy_shared_username(DCB *dcb, GWBUF *buffer) +{ + size_t buflen = gwbuf_length(buffer); + + if (buflen > MYSQL_AUTH_PACKET_BASE_SIZE) + { + buflen -= MYSQL_AUTH_PACKET_BASE_SIZE; + + /** TODO: Implement something that can safely iterate bytes of a GWBUF + * so that we know where the terminating null character is. For the time + * being, we'll just copy everything. */ + uint8_t data[buflen]; + gwbuf_copy_data(buffer, MYSQL_AUTH_PACKET_BASE_SIZE, buflen, data); + + MYSQL_session *ses = (MYSQL_session*)dcb->data; + /** data is null-terminated so the strcpy is safe */ + strcpy(ses->user, (char*)data); + } +} + +/** + * @brief Extract data from client response + * + * @param dcb Client DCB + * @param read_buffer Buffer containing the client's response + * @return MXS_AUTH_SUCCEEDED if authentication can continue, MXS_AUTH_FAILED if + * authentication failed + */ +static int gssapi_auth_extract(DCB *dcb, GWBUF *read_buffer) +{ + int rval = MXS_AUTH_FAILED; + gssapi_auth_t *auth = (gssapi_auth_t*)dcb->authenticator_data; + + switch (auth->state) + { + case GSSAPI_AUTH_INIT: + copy_shared_username(dcb, read_buffer); + rval = MXS_AUTH_SUCCEEDED; + break; + + case GSSAPI_AUTH_DATA_SENT: + store_client_token(dcb, read_buffer); + rval = MXS_AUTH_SUCCEEDED; + break; + + default: + MXS_ERROR("Unexpected authentication state: %d", auth->state); + ss_dassert(false); + break; + } + + return rval; +} + +/** + * @brief Is the client SSL capable + * + * @param dcb Client DCB + * @return True if client supports SSL + */ +bool gssapi_auth_connectssl(DCB *dcb) +{ + MySQLProtocol *protocol = (MySQLProtocol*)dcb->protocol; + return protocol->client_capabilities & GW_MYSQL_CAPABILITIES_SSL; +} + +/** + * @brief Report GSSAPI errors + * + * @param major GSSAPI major error number + * @param minor GSSAPI minor error number + */ +static void report_error(OM_uint32 major, OM_uint32 minor) +{ + OM_uint32 status_maj = major; + OM_uint32 status_min = minor; + OM_uint32 res = 0; + gss_buffer_desc buf = {0, 0}; + + major = gss_display_status(&minor, status_maj, GSS_C_GSS_CODE, NULL, &res, &buf); + + { + char sbuf[buf.length + 1]; + memcpy(sbuf, buf.value, buf.length); + sbuf[buf.length] = '\0'; + MXS_ERROR("GSSAPI Major Error: %s", sbuf); + } + + major = gss_display_status(&minor, status_min, GSS_C_MECH_CODE, NULL, &res, &buf); + + { + char sbuf[buf.length + 1]; + memcpy(sbuf, buf.value, buf.length); + sbuf[buf.length] = '\0'; + MXS_ERROR("GSSAPI Minor Error: %s", sbuf); + } +} + +static gss_name_t server_name = GSS_C_NO_NAME; + +/** + * @brief Check if the client token is valid + * + * @param token Client token + * @param len Length of the token + * @return True if client token is valid + */ +static bool validate_gssapi_token(uint8_t* token, size_t len) +{ + OM_uint32 major = 0, minor = 0; + gss_buffer_desc server_buf = {0, 0}; + gss_cred_id_t credentials; + + /** TODO: Make this configurable */ + server_buf.value = (void*)default_princ_name; + server_buf.length = sizeof(default_princ_name); + + major = gss_import_name(&minor, &server_buf, GSS_C_NT_USER_NAME, &server_name); + + if (GSS_ERROR(major)) + { + report_error(major, minor); + return false; + } + + major = gss_acquire_cred(&minor, server_name, GSS_C_INDEFINITE, + GSS_C_NO_OID_SET, GSS_C_ACCEPT, + &credentials, NULL, NULL); + if (GSS_ERROR(major)) + { + report_error(major, minor); + return false; + } + + do + { + + gss_ctx_id_t handle = NULL; + gss_buffer_desc in = {0, 0}; + gss_buffer_desc out = {0, 0}; + gss_OID_desc *oid; + + + in.value = token; + in.length = len; + + major = gss_accept_sec_context(&minor, &handle, GSS_C_NO_CREDENTIAL, + &in, GSS_C_NO_CHANNEL_BINDINGS, + &server_name, &oid, &out, + 0, 0, NULL); + if (GSS_ERROR(major)) + { + return false; + report_error(major, minor); + } + } + while (major & GSS_S_CONTINUE_NEEDED); + + return true; +} + +/** + * @brief Authenticate the client + * + * @param dcb Client DCB + * @return MXS_AUTH_INCOMPLETE if authentication is not yet complete, MXS_AUTH_SUCCEEDED + * if authentication was successfully completed or MXS_AUTH_FAILED if authentication + * has failed. + */ +int gssapi_auth_authenticate(DCB *dcb) +{ + int rval = MXS_AUTH_FAILED; + gssapi_auth_t *auth = (gssapi_auth_t*)dcb->authenticator_data; + + if (auth->state == GSSAPI_AUTH_INIT) + { + /** We need to send the authentication switch packet to change the + * authentication to something other than the 'mysql_native_password' + * method */ + GWBUF *buffer = create_auth_change_packet(); + + if (buffer && dcb->func.write(dcb, buffer)) + { + auth->state = GSSAPI_AUTH_DATA_SENT; + rval = MXS_AUTH_INCOMPLETE; + } + } + else if (auth->state == GSSAPI_AUTH_DATA_SENT) + { + /** We sent the principal name and the client responded with the GSSAPI + * token that we must validate */ + + MYSQL_session *ses = (MYSQL_session*)dcb->data; + + if (validate_gssapi_token(ses->auth_token, ses->auth_token_len)) + { + /** Auth token is valid, send the OK packet + * @see https://dev.mysql.com/doc/internals/en/packet-OK_Packet.html */ + uint8_t ok_packet[] = + { + 0x07, 0x00, 0x00, 0x04, // Header + 0x00, // OK byte + 0x00, // Affected rows + 0x00, // Last insert id + 0x02, 0x00, // Status flags + 0x00, 0x00 // Warnings + }; + + GWBUF *buffer = gwbuf_alloc_and_load(sizeof(ok_packet), ok_packet); + + if (buffer && dcb->func.write(dcb, buffer)) + { + rval = MXS_AUTH_SUCCEEDED; + } + } + } + + return rval; +} + +/** + * @brief Free authenticator data from a DCB + * + * @param dcb DCB to free + */ +void gssapi_auth_free_data(DCB *dcb) +{ + if (dcb->data) + { + MYSQL_session *ses = dcb->data; + MXS_FREE(ses->auth_token); + MXS_FREE(ses); + dcb->data = NULL; + } +} + +/** + * @brief Dummy function for loadusers entry point + * + * @param listener Listener definition + * @return Always MXS_AUTH_LOADUSERS_OK + */ +int gssapi_auth_load_users(SERV_LISTENER *listener) +{ + return MXS_AUTH_LOADUSERS_OK; +} + +/** + * Implementation of the authenticator module interface + */ +static GWAUTHENTICATOR MyObject = +{ + gssapi_auth_alloc, /* Allocate authenticator data */ + gssapi_auth_extract, /* Extract data into structure */ + gssapi_auth_connectssl, /* Check if client supports SSL */ + gssapi_auth_authenticate, /* Authenticate user credentials */ + gssapi_auth_free_data, /* Free the client data held in DCB */ + gssapi_auth_free, /* Free authenticator data */ + gssapi_auth_load_users /* Dummy function */ +}; + +MODULE_INFO info = +{ + MODULE_API_AUTHENTICATOR, + MODULE_GA, + GWAUTHENTICATOR_VERSION, + "GSSAPI authenticator" +}; + +static char *version_str = "V1.0.0"; + +/** + * Version string entry point + */ +char* version() +{ + return version_str; +} + +/** + * Module initialization entry point + */ +void ModuleInit() +{ +} + +/** + * Module handle entry point + */ +GWAUTHENTICATOR* GetModuleObject() +{ + return &MyObject; +} diff --git a/server/modules/authenticator/gssapi_auth.h b/server/modules/authenticator/gssapi_auth.h new file mode 100644 index 000000000..cb2ca6084 --- /dev/null +++ b/server/modules/authenticator/gssapi_auth.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2016 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl. + * + * Change Date: 2019-07-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ + +#ifndef _GSSAPI_AUTH_H +#define _GSSAPI_AUTH_H + + +/** Client auth plugin name */ +static const char auth_plugin_name[] = "auth_gssapi_client"; + +/** This is mainly for testing purposes */ +static const char default_princ_name[] = "mariadb/localhost.localdomain"; + +/** GSSAPI authentication states */ +enum gssapi_auth_state +{ + GSSAPI_AUTH_INIT = 0, + GSSAPI_AUTH_DATA_SENT, + GSSAPI_AUTH_OK, + GSSAPI_AUTH_FAILED +}; + +/** Common state tracking structure */ +typedef struct gssapi_auth +{ + enum gssapi_auth_state state; +} gssapi_auth_t; + +/** These functions can used for the `create` and `destroy` entry points */ +void* gssapi_auth_alloc(); +void gssapi_auth_free(void *data); + +#endif diff --git a/server/modules/authenticator/gssapi_auth_common.c b/server/modules/authenticator/gssapi_auth_common.c new file mode 100644 index 000000000..a6bcef631 --- /dev/null +++ b/server/modules/authenticator/gssapi_auth_common.c @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2016 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl. + * + * Change Date: 2019-07-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ + +#include "gssapi_auth.h" +#include + +void* gssapi_auth_alloc() +{ + gssapi_auth_t* rval = MXS_MALLOC(sizeof(gssapi_auth_t)); + + if (rval) + { + rval->state = GSSAPI_AUTH_INIT; + } + + return rval; +} + +void gssapi_auth_free(void *data) +{ + if (data) + { + MXS_FREE(data); + } +} diff --git a/server/modules/authenticator/gssapi_backend_auth.c b/server/modules/authenticator/gssapi_backend_auth.c new file mode 100644 index 000000000..38b260aae --- /dev/null +++ b/server/modules/authenticator/gssapi_backend_auth.c @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2016 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl. + * + * Change Date: 2019-07-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ + +#include +#include +#include +#include +#include +#include "gssapi_auth.h" + +/** + * @file gssapi_backend_auth.c GSSAPI backend authenticator + */ + +static int gssapi_backend_auth_extract(DCB *dcb, GWBUF *buffer) +{ + gw_send_backend_auth(dcb); + return MXS_AUTH_SUCCEEDED; +} + +static bool gssapi_backend_auth_connectssl(DCB *dcb) +{ + return dcb->server->server_ssl != NULL; +} + +static int gssapi_backend_auth_authenticate(DCB *dcb) +{ + return MXS_AUTH_SUCCEEDED; +} + +/** + * Implementation of the authenticator module interface + */ +static GWAUTHENTICATOR MyObject = +{ + gssapi_auth_alloc, + gssapi_backend_auth_extract, /* Extract data into structure */ + gssapi_backend_auth_connectssl, /* Check if client supports SSL */ + gssapi_backend_auth_authenticate, /* Authenticate user credentials */ + NULL, + gssapi_auth_free, + NULL /* Load users from backend databases */ +}; + +MODULE_INFO info = +{ + MODULE_API_AUTHENTICATOR, + MODULE_GA, + GWAUTHENTICATOR_VERSION, + "GSSAPI backend authenticator" +}; + +static char *version_str = "V1.0.0"; + +/** + * Version string entry point + */ +char* version() +{ + return version_str; +} + +/** + * Module initialization entry point + */ +void ModuleInit() +{ +} + +/** + * Module handle entry point + */ +GWAUTHENTICATOR* GetModuleObject() +{ + return &MyObject; +} From cb7c112764bbe773d922714a811882beab9afa13 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Wed, 5 Oct 2016 18:02:15 +0300 Subject: [PATCH 008/215] MXS-862: Create common MySQL library The MySQLCommon library contains functions used by both the protocol and authenticator modules. The contents of the modutil.c file could also be moved to this file if the functions in that file are only used by modules and not the core. --- server/core/test/CMakeLists.txt | 2 +- server/modules/authenticator/CMakeLists.txt | 9 +- server/modules/authenticator/gssapi_auth.c | 22 +- server/modules/authenticator/mysql_auth.c | 2 + .../include/mysql_client_server_protocol.h | 3 + server/modules/protocol/CMakeLists.txt | 3 +- server/modules/protocol/MySQL/CMakeLists.txt | 7 + .../MySQL/MySQLBackend/CMakeLists.txt | 7 + .../{ => MySQL}/MySQLBackend/mysql_backend.c | 385 +-------------- .../protocol/MySQL/MySQLClient/CMakeLists.txt | 4 + .../{ => MySQL}/MySQLClient/mysql_client.c | 97 +--- .../protocol/{ => MySQL}/mysql_common.c | 462 ++++++++++++++++++ .../protocol/MySQLBackend/CMakeLists.txt | 4 - .../protocol/MySQLClient/CMakeLists.txt | 4 - 14 files changed, 497 insertions(+), 514 deletions(-) create mode 100644 server/modules/protocol/MySQL/CMakeLists.txt create mode 100644 server/modules/protocol/MySQL/MySQLBackend/CMakeLists.txt rename server/modules/protocol/{ => MySQL}/MySQLBackend/mysql_backend.c (84%) create mode 100644 server/modules/protocol/MySQL/MySQLClient/CMakeLists.txt rename server/modules/protocol/{ => MySQL}/MySQLClient/mysql_client.c (93%) rename server/modules/protocol/{ => MySQL}/mysql_common.c (68%) delete mode 100644 server/modules/protocol/MySQLBackend/CMakeLists.txt delete mode 100644 server/modules/protocol/MySQLClient/CMakeLists.txt diff --git a/server/core/test/CMakeLists.txt b/server/core/test/CMakeLists.txt index c35a887c7..b16fd2542 100644 --- a/server/core/test/CMakeLists.txt +++ b/server/core/test/CMakeLists.txt @@ -30,7 +30,7 @@ target_link_libraries(test_log maxscale-common) target_link_libraries(test_logorder maxscale-common) target_link_libraries(test_logthrottling maxscale-common) target_link_libraries(test_modutil maxscale-common) -target_link_libraries(test_mysql_users MySQLClient maxscale-common) +target_link_libraries(test_mysql_users MySQLAuth MySQLCommon maxscale-common) target_link_libraries(test_poll maxscale-common) target_link_libraries(test_queuemanager maxscale-common) target_link_libraries(test_server maxscale-common) diff --git a/server/modules/authenticator/CMakeLists.txt b/server/modules/authenticator/CMakeLists.txt index bba4feb42..c8928b726 100644 --- a/server/modules/authenticator/CMakeLists.txt +++ b/server/modules/authenticator/CMakeLists.txt @@ -1,21 +1,20 @@ add_library(MySQLAuth SHARED mysql_auth.c) -target_link_libraries(MySQLAuth maxscale-common) +target_link_libraries(MySQLAuth maxscale-common MySQLCommon) set_target_properties(MySQLAuth PROPERTIES VERSION "1.0.0") install_module(MySQLAuth core) add_library(MySQLBackendAuth SHARED mysql_backend_auth.c) -target_link_libraries(MySQLBackendAuth maxscale-common MySQLBackend) +target_link_libraries(MySQLBackendAuth maxscale-common MySQLCommon) set_target_properties(MySQLBackendAuth PROPERTIES VERSION "1.0.0") install_module(MySQLBackendAuth core) add_library(GSSAPIAuth SHARED gssapi_auth.c gssapi_auth_common.c) -target_link_libraries(GSSAPIAuth maxscale-common gssapi_krb5) +target_link_libraries(GSSAPIAuth maxscale-common gssapi_krb5 MySQLCommon) set_target_properties(GSSAPIAuth PROPERTIES VERSION "1.0.0") install_module(GSSAPIAuth core) add_library(GSSAPIBackendAuth SHARED gssapi_backend_auth.c gssapi_auth_common.c) -target_link_libraries(GSSAPIBackendAuth maxscale-common gssapi_krb5 - MySQLBackend) # Needed for gw_send_backend_auth +target_link_libraries(GSSAPIBackendAuth maxscale-common gssapi_krb5 MySQLCommon) set_target_properties(GSSAPIBackendAuth PROPERTIES VERSION "1.0.0") install_module(GSSAPIBackendAuth core) diff --git a/server/modules/authenticator/gssapi_auth.c b/server/modules/authenticator/gssapi_auth.c index ac13f6b9c..6c165f683 100644 --- a/server/modules/authenticator/gssapi_auth.c +++ b/server/modules/authenticator/gssapi_auth.c @@ -281,26 +281,10 @@ int gssapi_auth_authenticate(DCB *dcb) MYSQL_session *ses = (MYSQL_session*)dcb->data; - if (validate_gssapi_token(ses->auth_token, ses->auth_token_len)) + if (validate_gssapi_token(ses->auth_token, ses->auth_token_len) && + mxs_mysql_send_ok(dcb, 4, 0, NULL)) { - /** Auth token is valid, send the OK packet - * @see https://dev.mysql.com/doc/internals/en/packet-OK_Packet.html */ - uint8_t ok_packet[] = - { - 0x07, 0x00, 0x00, 0x04, // Header - 0x00, // OK byte - 0x00, // Affected rows - 0x00, // Last insert id - 0x02, 0x00, // Status flags - 0x00, 0x00 // Warnings - }; - - GWBUF *buffer = gwbuf_alloc_and_load(sizeof(ok_packet), ok_packet); - - if (buffer && dcb->func.write(dcb, buffer)) - { - rval = MXS_AUTH_SUCCEEDED; - } + rval = MXS_AUTH_SUCCEEDED; } } diff --git a/server/modules/authenticator/mysql_auth.c b/server/modules/authenticator/mysql_auth.c index 874640f0d..838e64346 100644 --- a/server/modules/authenticator/mysql_auth.c +++ b/server/modules/authenticator/mysql_auth.c @@ -192,6 +192,8 @@ mysql_auth_authenticate(DCB *dcb) if (MXS_AUTH_SUCCEEDED == auth_ret) { dcb->user = MXS_STRDUP_A(client_data->user); + /** Send an OK packet to the client */ + mxs_mysql_send_ok(dcb, ssl_required_by_dcb(dcb) ? 3 : 2, 0, NULL); } else if (dcb->service->log_auth_warnings) { diff --git a/server/modules/include/mysql_client_server_protocol.h b/server/modules/include/mysql_client_server_protocol.h index c19292f9d..08c9853ba 100644 --- a/server/modules/include/mysql_client_server_protocol.h +++ b/server/modules/include/mysql_client_server_protocol.h @@ -380,4 +380,7 @@ bool gw_read_backend_handshake(DCB *dcb, GWBUF *buffer); /** Send the server handshake response packet to the backend server */ mxs_auth_state_t gw_send_backend_auth(DCB *dcb); +/** Write an OK packet to a DCB */ +int mxs_mysql_send_ok(DCB *dcb, int sequence, int affected_rows, const char* message); + #endif /** _MYSQL_PROTOCOL_H */ diff --git a/server/modules/protocol/CMakeLists.txt b/server/modules/protocol/CMakeLists.txt index a6f3e6a46..3972c1509 100644 --- a/server/modules/protocol/CMakeLists.txt +++ b/server/modules/protocol/CMakeLists.txt @@ -5,8 +5,7 @@ endif() add_subdirectory(HTTPD) add_subdirectory(maxscaled) -add_subdirectory(MySQLBackend) -add_subdirectory(MySQLClient) +add_subdirectory(MySQL) add_subdirectory(telnetd) if(BUILD_TESTS) diff --git a/server/modules/protocol/MySQL/CMakeLists.txt b/server/modules/protocol/MySQL/CMakeLists.txt new file mode 100644 index 000000000..c8e967bf3 --- /dev/null +++ b/server/modules/protocol/MySQL/CMakeLists.txt @@ -0,0 +1,7 @@ +add_library(MySQLCommon SHARED mysql_common.c) +target_link_libraries(MySQLCommon maxscale-common) +set_target_properties(MySQLCommon PROPERTIES VERSION "2.0.0") +install_module(MySQLCommon core) + +add_subdirectory(MySQLBackend) +add_subdirectory(MySQLClient) diff --git a/server/modules/protocol/MySQL/MySQLBackend/CMakeLists.txt b/server/modules/protocol/MySQL/MySQLBackend/CMakeLists.txt new file mode 100644 index 000000000..3dc29a6e7 --- /dev/null +++ b/server/modules/protocol/MySQL/MySQLBackend/CMakeLists.txt @@ -0,0 +1,7 @@ +add_library(MySQLBackend SHARED mysql_backend.c) +# TODO: Refactor MySQLBackend so that COM_CHANGE_USER processing is +# transparent to the protocol module. After this change, we don't need to +# link against MySQLAuth. +target_link_libraries(MySQLBackend maxscale-common MySQLCommon MySQLAuth) +set_target_properties(MySQLBackend PROPERTIES VERSION "2.0.0") +install_module(MySQLBackend core) diff --git a/server/modules/protocol/MySQLBackend/mysql_backend.c b/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c similarity index 84% rename from server/modules/protocol/MySQLBackend/mysql_backend.c rename to server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c index dfaec2b07..4442b8add 100644 --- a/server/modules/protocol/MySQLBackend/mysql_backend.c +++ b/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c @@ -661,290 +661,6 @@ gw_reply_on_error(DCB *dcb, mxs_auth_state_t state) gwbuf_free(errbuf); } -/** - * @brief Computes the size of the response to the DB initial handshake - * - * When the connection is to be SSL, but an SSL connection has not yet been - * established, only a basic 36 byte response is sent, including the SSL - * capability flag. - * - * Otherwise, the packet size is computed, based on the minimum size and - * increased by the optional or variable elements. - * - * @param conn The MySQLProtocol structure for the connection - * @param user Name of the user seeking to connect - * @param passwd Password for the user seeking to connect - * @param dbname Name of the database to be made default, if any - * @return The length of the response packet - */ -static int -response_length(MySQLProtocol *conn, char *user, uint8_t *passwd, char *dbname, const char *auth_module) -{ - long bytes; - - if (conn->owner_dcb->server->server_ssl && conn->owner_dcb->ssl_state != SSL_ESTABLISHED) - { - return 36; - } - - // Protocol MySQL HandshakeResponse for CLIENT_PROTOCOL_41 - // 4 bytes capabilities + 4 bytes max packet size + 1 byte charset + 23 '\0' bytes - // 4 + 4 + 1 + 23 = 32 - bytes = 32; - - if (user) - { - bytes += strlen(user); - } - // the NULL - bytes++; - - // next will be + 1 (scramble_len) + 20 (fixed_scramble) + 1 (user NULL term) + 1 (db NULL term) - - if (passwd) - { - bytes += GW_MYSQL_SCRAMBLE_SIZE; - } - bytes++; - - if (dbname && strlen(dbname)) - { - bytes += strlen(dbname); - bytes++; - } - - bytes += strlen(auth_module); - bytes++; - - // the packet header - bytes += 4; - - return bytes; -} - -/** - * @brief Helper function to load hashed password - * @param conn DCB Protocol object - * @param payload Destination where hashed password is written - * @param passwd Client's double SHA1 password - * @return Address of the next byte after the end of the stored password - */ -static uint8_t * -load_hashed_password(uint8_t *scramble, uint8_t *payload, uint8_t *passwd) -{ - uint8_t hash1[GW_MYSQL_SCRAMBLE_SIZE] = ""; - uint8_t hash2[GW_MYSQL_SCRAMBLE_SIZE] = ""; - uint8_t new_sha[GW_MYSQL_SCRAMBLE_SIZE] = ""; - uint8_t client_scramble[GW_MYSQL_SCRAMBLE_SIZE]; - - // hash1 is the function input, SHA1(real_password) - memcpy(hash1, passwd, GW_MYSQL_SCRAMBLE_SIZE); - - // hash2 is the SHA1(input data), where input_data = SHA1(real_password) - gw_sha1_str(hash1, GW_MYSQL_SCRAMBLE_SIZE, hash2); - - // new_sha is the SHA1(CONCAT(scramble, hash2) - gw_sha1_2_str(scramble, GW_MYSQL_SCRAMBLE_SIZE, hash2, GW_MYSQL_SCRAMBLE_SIZE, new_sha); - - // compute the xor in client_scramble - gw_str_xor(client_scramble, new_sha, hash1, GW_MYSQL_SCRAMBLE_SIZE); - - // set the auth-length - *payload = GW_MYSQL_SCRAMBLE_SIZE; - payload++; - - //copy the 20 bytes scramble data after packet_buffer + 36 + user + NULL + 1 (byte of auth-length) - memcpy(payload, client_scramble, GW_MYSQL_SCRAMBLE_SIZE); - - payload += GW_MYSQL_SCRAMBLE_SIZE; - return payload; -} - -/** - * @brief Computes the capabilities bit mask for connecting to backend DB - * - * We start by taking the default bitmask and removing any bits not set in - * the bitmask contained in the connection structure. Then add SSL flag if - * the connection requires SSL (set from the MaxScale configuration). The - * compression flag may be set, although compression is NOT SUPPORTED. If a - * database name has been specified in the function call, the relevant flag - * is set. - * - * @param conn The MySQLProtocol structure for the connection - * @param db_specified Whether the connection request specified a database - * @param compress Whether compression is requested - NOT SUPPORTED - * @return Bit mask (32 bits) - * @note Capability bits are defined in mysql_client_server_protocol.h - */ -static uint32_t -create_capabilities(MySQLProtocol *conn, bool db_specified, bool compress) -{ - uint32_t final_capabilities; - - /** Copy client's flags to backend but with the known capabilities mask */ - final_capabilities = (conn->client_capabilities & (uint32_t)GW_MYSQL_CAPABILITIES_CLIENT); - - if (conn->owner_dcb->server->server_ssl) - { - final_capabilities |= (uint32_t)GW_MYSQL_CAPABILITIES_SSL; - /* Unclear whether we should include this */ - /* Maybe it should depend on whether CA certificate is provided */ - /* final_capabilities |= (uint32_t)GW_MYSQL_CAPABILITIES_SSL_VERIFY_SERVER_CERT; */ - } - - /* Compression is not currently supported */ - if (compress) - { - final_capabilities |= (uint32_t)GW_MYSQL_CAPABILITIES_COMPRESS; -#ifdef DEBUG_MYSQL_CONN - fprintf(stderr, ">>>> Backend Connection with compression\n"); -#endif - } - - if (db_specified) - { - /* With database specified */ - final_capabilities |= (int)GW_MYSQL_CAPABILITIES_CONNECT_WITH_DB; - } - else - { - /* Without database specified */ - final_capabilities &= ~(int)GW_MYSQL_CAPABILITIES_CONNECT_WITH_DB; - } - - final_capabilities |= (int)GW_MYSQL_CAPABILITIES_PLUGIN_AUTH; - - return final_capabilities; -} - -/** - * Write MySQL authentication packet to backend server - * - * @param dcb Backend DCB - * @return True on success, false on failure - */ -mxs_auth_state_t gw_send_backend_auth(DCB *dcb) -{ - MYSQL_session local_session; - gw_get_shared_session_auth_info(dcb, &local_session); - - uint8_t client_capabilities[4] = {0, 0, 0, 0}; - uint8_t *curr_passwd = memcmp(local_session.client_sha1, null_client_sha1, MYSQL_SCRAMBLE_LEN) ? - local_session.client_sha1 : NULL; - - /** - * If session is stopping or has failed return with error. - */ - if (dcb->session == NULL || - (dcb->session->state != SESSION_STATE_READY && - dcb->session->state != SESSION_STATE_ROUTER_READY) || - (dcb->server->server_ssl && - dcb->ssl_state != SSL_HANDSHAKE_FAILED)) - { - return MXS_AUTH_STATE_FAILED; - } - - MySQLProtocol *conn = (MySQLProtocol*)dcb->protocol; - uint32_t capabilities = create_capabilities(conn, (local_session.db && strlen(local_session.db)), false); - gw_mysql_set_byte4(client_capabilities, capabilities); - - /** - * Use the default authentication plugin name. If the server is using a - * different authentication mechanism, it will send an AuthSwitchRequest - * packet. - */ - const char* auth_plugin_name = DEFAULT_MYSQL_AUTH_PLUGIN; - - long bytes = response_length(conn, local_session.user, local_session.client_sha1, - local_session.db, auth_plugin_name); - - // allocating the GWBUF - GWBUF *buffer = gwbuf_alloc(bytes); - uint8_t *payload = GWBUF_DATA(buffer); - - // clearing data - memset(payload, '\0', bytes); - - // put here the paylod size: bytes to write - 4 bytes packet header - gw_mysql_set_byte3(payload, (bytes - 4)); - - // set packet # = 1 - payload[3] = (SSL_ESTABLISHED == dcb->ssl_state) ? '\x02' : '\x01'; - payload += 4; - - // set client capabilities - memcpy(payload, client_capabilities, 4); - - // set now the max-packet size - payload += 4; - gw_mysql_set_byte4(payload, 16777216); - - // set the charset - payload += 4; - *payload = conn->charset; - - payload++; - - // 23 bytes of 0 - payload += 23; - - if (dcb->server->server_ssl && dcb->ssl_state != SSL_ESTABLISHED) - { - if (dcb_write(dcb, buffer) && dcb_connect_SSL(dcb) >= 0) - { - return MXS_AUTH_STATE_CONNECTED; - } - - return MXS_AUTH_STATE_FAILED; - } - - // 4 + 4 + 4 + 1 + 23 = 36, this includes the 4 bytes packet header - memcpy(payload, local_session.user, strlen(local_session.user)); - payload += strlen(local_session.user); - payload++; - - if (curr_passwd != NULL) - { - payload = load_hashed_password(conn->scramble, payload, curr_passwd); - } - else - { - payload++; - } - - // if the db is not NULL append it - if (local_session.db[0]) - { - memcpy(payload, local_session.db, strlen(local_session.db)); - payload += strlen(local_session.db); - payload++; - } - - memcpy(payload, auth_plugin_name, strlen(auth_plugin_name)); - - return dcb_write(dcb, buffer) ? MXS_AUTH_STATE_RESPONSE_SENT : MXS_AUTH_STATE_FAILED; -} - -/** - * Read the backend server MySQL handshake - * - * @param dcb Backend DCB - * @return true on success, false on failure - */ -bool gw_read_backend_handshake(DCB *dcb, GWBUF *buffer) -{ - MySQLProtocol *proto = (MySQLProtocol *)dcb->protocol; - bool rval = false; - uint8_t *payload = GWBUF_DATA(buffer) + 4; - - if (gw_decode_mysql_server_handshake(proto, payload) >= 0) - { - rval = true; - } - - return rval; -} - /** * @brief With authentication completed, read new data and write to backend * @@ -1649,6 +1365,8 @@ static int backend_write_delayqueue(DCB *dcb, GWBUF *buffer) /** * This routine handles the COM_CHANGE_USER command * + * TODO: Move this into the authenticators + * * @param dcb The current backend DCB * @param server The backend server pointer * @param in_session The current session data (MYSQL_session) @@ -2028,105 +1746,6 @@ static bool sescmd_response_complete(DCB* dcb) return succp; } -/** - * gw_decode_mysql_server_handshake - * - * Decode mysql server handshake - * - * @param conn The MySQLProtocol structure - * @param payload The bytes just read from the net - * @return 0 on success, < 0 on failure - * - */ -static int -gw_decode_mysql_server_handshake(MySQLProtocol *conn, uint8_t *payload) -{ - uint8_t *server_version_end = NULL; - uint16_t mysql_server_capabilities_one = 0; - uint16_t mysql_server_capabilities_two = 0; - unsigned long tid = 0; - uint8_t scramble_data_1[GW_SCRAMBLE_LENGTH_323] = ""; - uint8_t scramble_data_2[GW_MYSQL_SCRAMBLE_SIZE - GW_SCRAMBLE_LENGTH_323] = ""; - uint8_t capab_ptr[4] = ""; - int scramble_len = 0; - uint8_t mxs_scramble[GW_MYSQL_SCRAMBLE_SIZE] = ""; - int protocol_version = 0; - - protocol_version = payload[0]; - - if (protocol_version != GW_MYSQL_PROTOCOL_VERSION) - { - return -1; - } - - payload++; - - // Get server version (string) - server_version_end = (uint8_t *) gw_strend((char*) payload); - - payload = server_version_end + 1; - - // get ThreadID: 4 bytes - tid = gw_mysql_get_byte4(payload); - memcpy(&conn->tid, &tid, 4); - - payload += 4; - - // scramble_part 1 - memcpy(scramble_data_1, payload, GW_SCRAMBLE_LENGTH_323); - payload += GW_SCRAMBLE_LENGTH_323; - - // 1 filler - payload++; - - mysql_server_capabilities_one = gw_mysql_get_byte2(payload); - - //Get capabilities_part 1 (2 bytes) + 1 language + 2 server_status - payload += 5; - - mysql_server_capabilities_two = gw_mysql_get_byte2(payload); - - memcpy(capab_ptr, &mysql_server_capabilities_one, 2); - - // get capabilities part 2 (2 bytes) - memcpy(&capab_ptr[2], &mysql_server_capabilities_two, 2); - - // 2 bytes shift - payload += 2; - - // get scramble len - if (payload[0] > 0) - { - scramble_len = payload[0] - 1; - ss_dassert(scramble_len > GW_SCRAMBLE_LENGTH_323); - ss_dassert(scramble_len <= GW_MYSQL_SCRAMBLE_SIZE); - - if ((scramble_len < GW_SCRAMBLE_LENGTH_323) || - scramble_len > GW_MYSQL_SCRAMBLE_SIZE) - { - /* log this */ - return -2; - } - } - else - { - scramble_len = GW_MYSQL_SCRAMBLE_SIZE; - } - // skip 10 zero bytes - payload += 11; - - // copy the second part of the scramble - memcpy(scramble_data_2, payload, scramble_len - GW_SCRAMBLE_LENGTH_323); - - memcpy(mxs_scramble, scramble_data_1, GW_SCRAMBLE_LENGTH_323); - memcpy(mxs_scramble + GW_SCRAMBLE_LENGTH_323, scramble_data_2, scramble_len - GW_SCRAMBLE_LENGTH_323); - - // full 20 bytes scramble is ready - memcpy(conn->scramble, mxs_scramble, GW_MYSQL_SCRAMBLE_SIZE); - - return 0; -} - static void inline close_socket(int sock) { diff --git a/server/modules/protocol/MySQL/MySQLClient/CMakeLists.txt b/server/modules/protocol/MySQL/MySQLClient/CMakeLists.txt new file mode 100644 index 000000000..31f7c937f --- /dev/null +++ b/server/modules/protocol/MySQL/MySQLClient/CMakeLists.txt @@ -0,0 +1,4 @@ +add_library(MySQLClient SHARED mysql_client.c) +target_link_libraries(MySQLClient maxscale-common MySQLCommon) +set_target_properties(MySQLClient PROPERTIES VERSION "1.0.0") +install_module(MySQLClient core) diff --git a/server/modules/protocol/MySQLClient/mysql_client.c b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c similarity index 93% rename from server/modules/protocol/MySQLClient/mysql_client.c rename to server/modules/protocol/MySQL/MySQLClient/mysql_client.c index 2c021a4f4..2ac029271 100644 --- a/server/modules/protocol/MySQLClient/mysql_client.c +++ b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c @@ -86,7 +86,6 @@ static int gw_client_close(DCB *dcb); static int gw_client_hangup_event(DCB *dcb); static char *gw_default_auth(); static int gw_connection_limit(DCB *dcb, int limit); -static int mysql_send_ok(DCB *dcb, int packet_number, int in_affected_rows, const char* mysql_message); static int MySQLSendHandshake(DCB* dcb); static int route_by_statement(SESSION *, GWBUF **); static void mysql_client_auth_error_handling(DCB *dcb, int auth_val); @@ -162,92 +161,6 @@ static char *gw_default_auth() { return "MySQLAuth"; } -/** - * mysql_send_ok - * - * Send a MySQL protocol OK message to the dcb (client) - * - * @param dcb Descriptor Control Block for the connection to which the OK is sent - * @param packet_number - * @param in_affected_rows - * @param mysql_message - * @return packet length - * - */ -int mysql_send_ok(DCB *dcb, int packet_number, int in_affected_rows, const char* mysql_message) -{ - uint8_t *outbuf = NULL; - uint32_t mysql_payload_size = 0; - uint8_t mysql_packet_header[4]; - uint8_t *mysql_payload = NULL; - uint8_t field_count = 0; - uint8_t affected_rows = 0; - uint8_t insert_id = 0; - uint8_t mysql_server_status[2]; - uint8_t mysql_warning_counter[2]; - GWBUF *buf; - - affected_rows = in_affected_rows; - - mysql_payload_size = - sizeof(field_count) + - sizeof(affected_rows) + - sizeof(insert_id) + - sizeof(mysql_server_status) + - sizeof(mysql_warning_counter); - - if (mysql_message != NULL) - { - mysql_payload_size += strlen(mysql_message); - } - - // allocate memory for packet header + payload - if ((buf = gwbuf_alloc(sizeof(mysql_packet_header) + mysql_payload_size)) == NULL) - { - return 0; - } - outbuf = GWBUF_DATA(buf); - - // write packet header with packet number - gw_mysql_set_byte3(mysql_packet_header, mysql_payload_size); - mysql_packet_header[3] = packet_number; - - // write header - memcpy(outbuf, mysql_packet_header, sizeof(mysql_packet_header)); - - mysql_payload = outbuf + sizeof(mysql_packet_header); - - mysql_server_status[0] = 2; - mysql_server_status[1] = 0; - mysql_warning_counter[0] = 0; - mysql_warning_counter[1] = 0; - - // write data - memcpy(mysql_payload, &field_count, sizeof(field_count)); - mysql_payload = mysql_payload + sizeof(field_count); - - memcpy(mysql_payload, &affected_rows, sizeof(affected_rows)); - mysql_payload = mysql_payload + sizeof(affected_rows); - - memcpy(mysql_payload, &insert_id, sizeof(insert_id)); - mysql_payload = mysql_payload + sizeof(insert_id); - - memcpy(mysql_payload, mysql_server_status, sizeof(mysql_server_status)); - mysql_payload = mysql_payload + sizeof(mysql_server_status); - - memcpy(mysql_payload, mysql_warning_counter, sizeof(mysql_warning_counter)); - mysql_payload = mysql_payload + sizeof(mysql_warning_counter); - - if (mysql_message != NULL) - { - memcpy(mysql_payload, mysql_message, strlen(mysql_message)); - } - - // writing data in the Client buffer queue - dcb->func.write(dcb, buf); - - return sizeof(mysql_packet_header) + mysql_payload_size; -} /** * MySQLSendHandshake @@ -625,18 +538,10 @@ gw_read_do_authentication(DCB *dcb, GWBUF *read_buffer, int nbytes_read) if (session != NULL) { - int packet_number = ssl_required_by_dcb(dcb) ? 3 : 2; - CHK_SESSION(session); ss_dassert(session->state != SESSION_STATE_ALLOC && - session->state != SESSION_STATE_DUMMY); - + session->state != SESSION_STATE_DUMMY); protocol->protocol_auth_state = MXS_AUTH_STATE_COMPLETE; - /** - * Send an AUTH_OK packet to the client, - * packet sequence is # packet_number - */ - mysql_send_ok(dcb, packet_number, 0, NULL); } else { diff --git a/server/modules/protocol/mysql_common.c b/server/modules/protocol/MySQL/mysql_common.c similarity index 68% rename from server/modules/protocol/mysql_common.c rename to server/modules/protocol/MySQL/mysql_common.c index c0c416b67..fbc6fc0b3 100644 --- a/server/modules/protocol/mysql_common.c +++ b/server/modules/protocol/MySQL/mysql_common.c @@ -1078,3 +1078,465 @@ bool gw_get_shared_session_auth_info(DCB* dcb, MYSQL_session* session) spinlock_release(&dcb->session->ses_lock); return rval; } + +/** + * @brief Send a MySQL protocol OK message to the dcb (client) + * + * @param dcb DCB where packet is written + * @param sequence Packet sequence number + * @param affected_rows Number of affected rows + * * @param message SQL message + * @return 1 on success, 0 on error + * + */ +int mxs_mysql_send_ok(DCB *dcb, int sequence, int affected_rows, const char* message) +{ + uint8_t *outbuf = NULL; + uint32_t mysql_payload_size = 0; + uint8_t mysql_packet_header[4]; + uint8_t *mysql_payload = NULL; + uint8_t field_count = 0; + uint8_t insert_id = 0; + uint8_t mysql_server_status[2]; + uint8_t mysql_warning_counter[2]; + GWBUF *buf; + + + mysql_payload_size = + sizeof(field_count) + + sizeof(affected_rows) + + sizeof(insert_id) + + sizeof(mysql_server_status) + + sizeof(mysql_warning_counter); + + if (message != NULL) + { + mysql_payload_size += strlen(message); + } + + // allocate memory for packet header + payload + if ((buf = gwbuf_alloc(sizeof(mysql_packet_header) + mysql_payload_size)) == NULL) + { + return 0; + } + outbuf = GWBUF_DATA(buf); + + // write packet header with packet number + gw_mysql_set_byte3(mysql_packet_header, mysql_payload_size); + mysql_packet_header[3] = sequence; + + // write header + memcpy(outbuf, mysql_packet_header, sizeof(mysql_packet_header)); + + mysql_payload = outbuf + sizeof(mysql_packet_header); + + mysql_server_status[0] = 2; + mysql_server_status[1] = 0; + mysql_warning_counter[0] = 0; + mysql_warning_counter[1] = 0; + + // write data + memcpy(mysql_payload, &field_count, sizeof(field_count)); + mysql_payload = mysql_payload + sizeof(field_count); + + memcpy(mysql_payload, &affected_rows, sizeof(affected_rows)); + mysql_payload = mysql_payload + sizeof(affected_rows); + + memcpy(mysql_payload, &insert_id, sizeof(insert_id)); + mysql_payload = mysql_payload + sizeof(insert_id); + + memcpy(mysql_payload, mysql_server_status, sizeof(mysql_server_status)); + mysql_payload = mysql_payload + sizeof(mysql_server_status); + + memcpy(mysql_payload, mysql_warning_counter, sizeof(mysql_warning_counter)); + mysql_payload = mysql_payload + sizeof(mysql_warning_counter); + + if (message != NULL) + { + memcpy(mysql_payload, message, strlen(message)); + } + + // writing data in the Client buffer queue + return dcb->func.write(dcb, buf); +} + +/** + * @brief Computes the size of the response to the DB initial handshake + * + * When the connection is to be SSL, but an SSL connection has not yet been + * established, only a basic 36 byte response is sent, including the SSL + * capability flag. + * + * Otherwise, the packet size is computed, based on the minimum size and + * increased by the optional or variable elements. + * + * @param conn The MySQLProtocol structure for the connection + * @param user Name of the user seeking to connect + * @param passwd Password for the user seeking to connect + * @param dbname Name of the database to be made default, if any + * @return The length of the response packet + */ +static int +response_length(MySQLProtocol *conn, char *user, uint8_t *passwd, char *dbname, const char *auth_module) +{ + long bytes; + + if (conn->owner_dcb->server->server_ssl && conn->owner_dcb->ssl_state != SSL_ESTABLISHED) + { + return MYSQL_AUTH_PACKET_BASE_SIZE; + } + + // Protocol MySQL HandshakeResponse for CLIENT_PROTOCOL_41 + // 4 bytes capabilities + 4 bytes max packet size + 1 byte charset + 23 '\0' bytes + // 4 + 4 + 1 + 23 = 32 + bytes = 32; + + if (user) + { + bytes += strlen(user); + } + // the NULL + bytes++; + + // next will be + 1 (scramble_len) + 20 (fixed_scramble) + 1 (user NULL term) + 1 (db NULL term) + + if (passwd) + { + bytes += GW_MYSQL_SCRAMBLE_SIZE; + } + bytes++; + + if (dbname && strlen(dbname)) + { + bytes += strlen(dbname); + bytes++; + } + + bytes += strlen(auth_module); + bytes++; + + // the packet header + bytes += 4; + + return bytes; +} + +/** + * @brief Helper function to load hashed password + * @param conn DCB Protocol object + * @param payload Destination where hashed password is written + * @param passwd Client's double SHA1 password + * @return Address of the next byte after the end of the stored password + */ +static uint8_t * +load_hashed_password(uint8_t *scramble, uint8_t *payload, uint8_t *passwd) +{ + uint8_t hash1[GW_MYSQL_SCRAMBLE_SIZE] = ""; + uint8_t hash2[GW_MYSQL_SCRAMBLE_SIZE] = ""; + uint8_t new_sha[GW_MYSQL_SCRAMBLE_SIZE] = ""; + uint8_t client_scramble[GW_MYSQL_SCRAMBLE_SIZE]; + + // hash1 is the function input, SHA1(real_password) + memcpy(hash1, passwd, GW_MYSQL_SCRAMBLE_SIZE); + + // hash2 is the SHA1(input data), where input_data = SHA1(real_password) + gw_sha1_str(hash1, GW_MYSQL_SCRAMBLE_SIZE, hash2); + + // new_sha is the SHA1(CONCAT(scramble, hash2) + gw_sha1_2_str(scramble, GW_MYSQL_SCRAMBLE_SIZE, hash2, GW_MYSQL_SCRAMBLE_SIZE, new_sha); + + // compute the xor in client_scramble + gw_str_xor(client_scramble, new_sha, hash1, GW_MYSQL_SCRAMBLE_SIZE); + + // set the auth-length + *payload = GW_MYSQL_SCRAMBLE_SIZE; + payload++; + + //copy the 20 bytes scramble data after packet_buffer + 36 + user + NULL + 1 (byte of auth-length) + memcpy(payload, client_scramble, GW_MYSQL_SCRAMBLE_SIZE); + + payload += GW_MYSQL_SCRAMBLE_SIZE; + return payload; +} + +/** + * @brief Computes the capabilities bit mask for connecting to backend DB + * + * We start by taking the default bitmask and removing any bits not set in + * the bitmask contained in the connection structure. Then add SSL flag if + * the connection requires SSL (set from the MaxScale configuration). The + * compression flag may be set, although compression is NOT SUPPORTED. If a + * database name has been specified in the function call, the relevant flag + * is set. + * + * @param conn The MySQLProtocol structure for the connection + * @param db_specified Whether the connection request specified a database + * @param compress Whether compression is requested - NOT SUPPORTED + * @return Bit mask (32 bits) + * @note Capability bits are defined in mysql_client_server_protocol.h + */ +static uint32_t +create_capabilities(MySQLProtocol *conn, bool db_specified, bool compress) +{ + uint32_t final_capabilities; + + /** Copy client's flags to backend but with the known capabilities mask */ + final_capabilities = (conn->client_capabilities & (uint32_t)GW_MYSQL_CAPABILITIES_CLIENT); + + if (conn->owner_dcb->server->server_ssl) + { + final_capabilities |= (uint32_t)GW_MYSQL_CAPABILITIES_SSL; + /* Unclear whether we should include this */ + /* Maybe it should depend on whether CA certificate is provided */ + /* final_capabilities |= (uint32_t)GW_MYSQL_CAPABILITIES_SSL_VERIFY_SERVER_CERT; */ + } + + /* Compression is not currently supported */ + if (compress) + { + final_capabilities |= (uint32_t)GW_MYSQL_CAPABILITIES_COMPRESS; +#ifdef DEBUG_MYSQL_CONN + fprintf(stderr, ">>>> Backend Connection with compression\n"); +#endif + } + + if (db_specified) + { + /* With database specified */ + final_capabilities |= (int)GW_MYSQL_CAPABILITIES_CONNECT_WITH_DB; + } + else + { + /* Without database specified */ + final_capabilities &= ~(int)GW_MYSQL_CAPABILITIES_CONNECT_WITH_DB; + } + + final_capabilities |= (int)GW_MYSQL_CAPABILITIES_PLUGIN_AUTH; + + return final_capabilities; +} + +/** + * Write MySQL authentication packet to backend server + * + * @param dcb Backend DCB + * @return True on success, false on failure + */ +mxs_auth_state_t gw_send_backend_auth(DCB *dcb) +{ + MYSQL_session local_session; + gw_get_shared_session_auth_info(dcb, &local_session); + + uint8_t client_capabilities[4] = {0, 0, 0, 0}; + uint8_t *curr_passwd = memcmp(local_session.client_sha1, null_client_sha1, MYSQL_SCRAMBLE_LEN) ? + local_session.client_sha1 : NULL; + + /** + * If session is stopping or has failed return with error. + */ + if (dcb->session == NULL || + (dcb->session->state != SESSION_STATE_READY && + dcb->session->state != SESSION_STATE_ROUTER_READY) || + (dcb->server->server_ssl && + dcb->ssl_state != SSL_HANDSHAKE_FAILED)) + { + return MXS_AUTH_STATE_FAILED; + } + + MySQLProtocol *conn = (MySQLProtocol*)dcb->protocol; + uint32_t capabilities = create_capabilities(conn, (local_session.db && strlen(local_session.db)), false); + gw_mysql_set_byte4(client_capabilities, capabilities); + + /** + * Use the default authentication plugin name. If the server is using a + * different authentication mechanism, it will send an AuthSwitchRequest + * packet. + */ + const char* auth_plugin_name = DEFAULT_MYSQL_AUTH_PLUGIN; + + long bytes = response_length(conn, local_session.user, local_session.client_sha1, + local_session.db, auth_plugin_name); + + // allocating the GWBUF + GWBUF *buffer = gwbuf_alloc(bytes); + uint8_t *payload = GWBUF_DATA(buffer); + + // clearing data + memset(payload, '\0', bytes); + + // put here the paylod size: bytes to write - 4 bytes packet header + gw_mysql_set_byte3(payload, (bytes - 4)); + + // set packet # = 1 + payload[3] = (SSL_ESTABLISHED == dcb->ssl_state) ? '\x02' : '\x01'; + payload += 4; + + // set client capabilities + memcpy(payload, client_capabilities, 4); + + // set now the max-packet size + payload += 4; + gw_mysql_set_byte4(payload, 16777216); + + // set the charset + payload += 4; + *payload = conn->charset; + + payload++; + + // 23 bytes of 0 + payload += 23; + + if (dcb->server->server_ssl && dcb->ssl_state != SSL_ESTABLISHED) + { + if (dcb_write(dcb, buffer) && dcb_connect_SSL(dcb) >= 0) + { + return MXS_AUTH_STATE_CONNECTED; + } + + return MXS_AUTH_STATE_FAILED; + } + + // 4 + 4 + 4 + 1 + 23 = 36, this includes the 4 bytes packet header + memcpy(payload, local_session.user, strlen(local_session.user)); + payload += strlen(local_session.user); + payload++; + + if (curr_passwd != NULL) + { + payload = load_hashed_password(conn->scramble, payload, curr_passwd); + } + else + { + payload++; + } + + // if the db is not NULL append it + if (local_session.db[0]) + { + memcpy(payload, local_session.db, strlen(local_session.db)); + payload += strlen(local_session.db); + payload++; + } + + memcpy(payload, auth_plugin_name, strlen(auth_plugin_name)); + + return dcb_write(dcb, buffer) ? MXS_AUTH_STATE_RESPONSE_SENT : MXS_AUTH_STATE_FAILED; +} + +/** + * Decode mysql server handshake + * + * @param conn The MySQLProtocol structure + * @param payload The bytes just read from the net + * @return 0 on success, < 0 on failure + * + */ +static int +gw_decode_mysql_server_handshake(MySQLProtocol *conn, uint8_t *payload) +{ + uint8_t *server_version_end = NULL; + uint16_t mysql_server_capabilities_one = 0; + uint16_t mysql_server_capabilities_two = 0; + unsigned long tid = 0; + uint8_t scramble_data_1[GW_SCRAMBLE_LENGTH_323] = ""; + uint8_t scramble_data_2[GW_MYSQL_SCRAMBLE_SIZE - GW_SCRAMBLE_LENGTH_323] = ""; + uint8_t capab_ptr[4] = ""; + int scramble_len = 0; + uint8_t mxs_scramble[GW_MYSQL_SCRAMBLE_SIZE] = ""; + int protocol_version = 0; + + protocol_version = payload[0]; + + if (protocol_version != GW_MYSQL_PROTOCOL_VERSION) + { + return -1; + } + + payload++; + + // Get server version (string) + server_version_end = (uint8_t *) gw_strend((char*) payload); + + payload = server_version_end + 1; + + // get ThreadID: 4 bytes + tid = gw_mysql_get_byte4(payload); + memcpy(&conn->tid, &tid, 4); + + payload += 4; + + // scramble_part 1 + memcpy(scramble_data_1, payload, GW_SCRAMBLE_LENGTH_323); + payload += GW_SCRAMBLE_LENGTH_323; + + // 1 filler + payload++; + + mysql_server_capabilities_one = gw_mysql_get_byte2(payload); + + //Get capabilities_part 1 (2 bytes) + 1 language + 2 server_status + payload += 5; + + mysql_server_capabilities_two = gw_mysql_get_byte2(payload); + + memcpy(capab_ptr, &mysql_server_capabilities_one, 2); + + // get capabilities part 2 (2 bytes) + memcpy(&capab_ptr[2], &mysql_server_capabilities_two, 2); + + // 2 bytes shift + payload += 2; + + // get scramble len + if (payload[0] > 0) + { + scramble_len = payload[0] - 1; + ss_dassert(scramble_len > GW_SCRAMBLE_LENGTH_323); + ss_dassert(scramble_len <= GW_MYSQL_SCRAMBLE_SIZE); + + if ((scramble_len < GW_SCRAMBLE_LENGTH_323) || + scramble_len > GW_MYSQL_SCRAMBLE_SIZE) + { + /* log this */ + return -2; + } + } + else + { + scramble_len = GW_MYSQL_SCRAMBLE_SIZE; + } + // skip 10 zero bytes + payload += 11; + + // copy the second part of the scramble + memcpy(scramble_data_2, payload, scramble_len - GW_SCRAMBLE_LENGTH_323); + + memcpy(mxs_scramble, scramble_data_1, GW_SCRAMBLE_LENGTH_323); + memcpy(mxs_scramble + GW_SCRAMBLE_LENGTH_323, scramble_data_2, scramble_len - GW_SCRAMBLE_LENGTH_323); + + // full 20 bytes scramble is ready + memcpy(conn->scramble, mxs_scramble, GW_MYSQL_SCRAMBLE_SIZE); + + return 0; +} + +/** + * Read the backend server MySQL handshake + * + * @param dcb Backend DCB + * @return true on success, false on failure + */ +bool gw_read_backend_handshake(DCB *dcb, GWBUF *buffer) +{ + MySQLProtocol *proto = (MySQLProtocol *)dcb->protocol; + bool rval = false; + uint8_t *payload = GWBUF_DATA(buffer) + 4; + + if (gw_decode_mysql_server_handshake(proto, payload) >= 0) + { + rval = true; + } + + return rval; +} diff --git a/server/modules/protocol/MySQLBackend/CMakeLists.txt b/server/modules/protocol/MySQLBackend/CMakeLists.txt deleted file mode 100644 index f23cb071b..000000000 --- a/server/modules/protocol/MySQLBackend/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -add_library(MySQLBackend SHARED mysql_backend.c ../mysql_common.c) -target_link_libraries(MySQLBackend maxscale-common MySQLAuth) -set_target_properties(MySQLBackend PROPERTIES VERSION "2.0.0") -install_module(MySQLBackend core) diff --git a/server/modules/protocol/MySQLClient/CMakeLists.txt b/server/modules/protocol/MySQLClient/CMakeLists.txt deleted file mode 100644 index 6a250cea0..000000000 --- a/server/modules/protocol/MySQLClient/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -add_library(MySQLClient SHARED mysql_client.c ../mysql_common.c) -target_link_libraries(MySQLClient maxscale-common MySQLAuth) -set_target_properties(MySQLClient PROPERTIES VERSION "1.0.0") -install_module(MySQLClient core) From 239b53e156da237e7bf57dc67c434ec18b17888d Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Wed, 5 Oct 2016 21:16:57 +0300 Subject: [PATCH 009/215] MXS-862: Do first part of authentication in MySQLBackend The first message exchange between the server and the client will almost always contain the same data. If the server is going to change authentication methods, it will send an AuthSwitchRequest packet instead of the OK/ERR packet that it would normally send. Only after this point the authenticator modules actually need to do something. In the case of the default 'mysql_native_password' plugin, the only thing that the plugin needs to do is to check whether the server responded with an OK packet. --- .../authenticator/mysql_backend_auth.c | 108 +++++------------- .../include/mysql_client_server_protocol.h | 3 + .../MySQL/MySQLBackend/mysql_backend.c | 21 +++- server/modules/protocol/MySQL/mysql_common.c | 19 +++ 4 files changed, 66 insertions(+), 85 deletions(-) diff --git a/server/modules/authenticator/mysql_backend_auth.c b/server/modules/authenticator/mysql_backend_auth.c index dbabdd651..0d0f20456 100644 --- a/server/modules/authenticator/mysql_backend_auth.c +++ b/server/modules/authenticator/mysql_backend_auth.c @@ -17,6 +17,10 @@ * Backend authentication module for the MySQL protocol. Implements the * client side of the 'mysql_native_password' authentication plugin. * + * The "heavy lifting" of the authentication is done by the protocol module so + * the only thing left for this module is to read the final OK packet from the + * server. + * * @verbatim * Revision History * Date Who Description @@ -33,8 +37,6 @@ /** Authentication states */ enum mba_state { - MBA_NEED_HANDSHAKE, /**< Waiting for server's handshake packet */ - MBA_SEND_RESPONSE, /**< A response to the server's handshake has been sent */ MBA_NEED_OK, /**< Waiting for server's OK packet */ MBA_AUTH_OK, /**< Authentication completed successfully */ MBA_AUTH_FAILED /**< Authentication failed */ @@ -56,7 +58,7 @@ void* auth_backend_create() if (mba) { - mba->state = MBA_NEED_HANDSHAKE; + mba->state = MBA_NEED_OK; } return mba; @@ -73,26 +75,6 @@ void auth_backend_destroy(void *data) MXS_FREE(data); } } - -/** - * Receive the MySQL authentication packet from backend, packet # is 2 - * - * @param protocol The MySQL protocol structure - * @return False in case of failure, true if authentication was successful. - */ -static bool gw_read_auth_response(DCB *dcb, GWBUF *buffer) -{ - bool rval = false; - uint8_t cmd; - - if (gwbuf_copy_data(buffer, MYSQL_HEADER_LEN, 1, &cmd) && cmd == MYSQL_REPLY_OK) - { - rval = true; - } - - return rval; -} - /** * @brief Extract backend response * @@ -102,46 +84,29 @@ static bool gw_read_auth_response(DCB *dcb, GWBUF *buffer) * @see gw_quthenticator.h * @see https://dev.mysql.com/doc/internals/en/client-server-protocol.html */ -static int -auth_backend_extract(DCB *dcb, GWBUF *buf) +static int auth_backend_extract(DCB *dcb, GWBUF *buf) { int rval = MXS_AUTH_FAILED; + mysql_backend_auth_t *mba = (mysql_backend_auth_t*)dcb->authenticator_data; - if (dcb->authenticator_data) + switch (mba->state) { - mysql_backend_auth_t *mba = (mysql_backend_auth_t*)dcb->authenticator_data; + case MBA_NEED_OK: + if (mxs_mysql_is_ok_packet(buf)) + { + rval = MXS_AUTH_SUCCEEDED; + mba->state = MBA_AUTH_OK; + } + else + { + mba->state = MBA_AUTH_FAILED; + } + break; - switch (mba->state) - { - case MBA_NEED_HANDSHAKE: - if (gw_read_backend_handshake(dcb, buf)) - { - rval = MXS_AUTH_INCOMPLETE; - mba->state = MBA_SEND_RESPONSE; - } - else - { - mba->state = MBA_AUTH_FAILED; - } - break; - - case MBA_NEED_OK: - if (gw_read_auth_response(dcb, buf)) - { - rval = MXS_AUTH_SUCCEEDED; - mba->state = MBA_AUTH_OK; - } - else - { - mba->state = MBA_AUTH_FAILED; - } - break; - - default: - MXS_ERROR("Unexpected call to MySQLBackendAuth::extract"); - ss_dassert(false); - break; - } + default: + MXS_ERROR("Unexpected call to MySQLBackendAuth::extract"); + ss_dassert(false); + break; } return rval; @@ -154,32 +119,12 @@ auth_backend_extract(DCB *dcb, GWBUF *buf) * @return Authentication status * @see gw_authenticator.h */ -static int -auth_backend_authenticate(DCB *dcb) +static int auth_backend_authenticate(DCB *dcb) { int rval = MXS_AUTH_FAILED; mysql_backend_auth_t *mba = (mysql_backend_auth_t*)dcb->authenticator_data; - if (mba->state == MBA_SEND_RESPONSE) - { - /** First message read, decode password and send the auth credentials to backend */ - switch (gw_send_backend_auth(dcb)) - { - case MXS_AUTH_STATE_CONNECTED: - rval = MXS_AUTH_SSL_INCOMPLETE; - break; - - case MXS_AUTH_STATE_RESPONSE_SENT: - mba->state = MBA_NEED_OK; - rval = MXS_AUTH_INCOMPLETE; - break; - - default: - /** Authentication failed */ - break; - } - } - else if (mba->state == MBA_AUTH_OK) + if (mba->state == MBA_AUTH_OK) { /** Authentication completed successfully */ rval = MXS_AUTH_SUCCEEDED; @@ -198,8 +143,7 @@ auth_backend_authenticate(DCB *dcb) * @param dcb Request handler DCB connected to the client * @return Boolean indicating whether client is SSL capable */ -static bool -auth_backend_ssl(DCB *dcb) +static bool auth_backend_ssl(DCB *dcb) { return dcb->server->server_ssl != NULL; } diff --git a/server/modules/include/mysql_client_server_protocol.h b/server/modules/include/mysql_client_server_protocol.h index 08c9853ba..b50a3fac4 100644 --- a/server/modules/include/mysql_client_server_protocol.h +++ b/server/modules/include/mysql_client_server_protocol.h @@ -383,4 +383,7 @@ mxs_auth_state_t gw_send_backend_auth(DCB *dcb); /** Write an OK packet to a DCB */ int mxs_mysql_send_ok(DCB *dcb, int sequence, int affected_rows, const char* message); +/** Check for OK packet */ +bool mxs_mysql_is_ok_packet(GWBUF *buffer); + #endif /** _MYSQL_PROTOCOL_H */ diff --git a/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c b/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c index 4442b8add..5f118af6a 100644 --- a/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c +++ b/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c @@ -580,10 +580,25 @@ gw_read_backend_event(DCB *dcb) log_error_response(dcb, readbuf); } - if (proto->protocol_auth_state == MXS_AUTH_STATE_CONNECTED || - proto->protocol_auth_state == MXS_AUTH_STATE_RESPONSE_SENT) + if (proto->protocol_auth_state == MXS_AUTH_STATE_CONNECTED) { - /** Read the first message from the server */ + mxs_auth_state_t state = MXS_AUTH_STATE_FAILED; + + /** Read the server handshake and send the standard response */ + if (gw_read_backend_handshake(dcb, readbuf)) + { + state = gw_send_backend_auth(dcb); + } + + proto->protocol_auth_state = state; + gwbuf_free(readbuf); + } + else if (proto->protocol_auth_state == MXS_AUTH_STATE_RESPONSE_SENT) + { + /** Read the message from the server. This will be the first + * packet that can contain authenticator specific data from the + * backend server. For 'mysql_native_password' it'll be an OK + * packet */ proto->protocol_auth_state = handle_server_response(dcb, readbuf); } diff --git a/server/modules/protocol/MySQL/mysql_common.c b/server/modules/protocol/MySQL/mysql_common.c index fbc6fc0b3..cfe6abe91 100644 --- a/server/modules/protocol/MySQL/mysql_common.c +++ b/server/modules/protocol/MySQL/mysql_common.c @@ -1540,3 +1540,22 @@ bool gw_read_backend_handshake(DCB *dcb, GWBUF *buffer) return rval; } + +/** + * @brief Check if the buffer contains an OK packet + * + * @param buffer Buffer containing a complete MySQL packet + * @return True if the buffer contains an OK packet + */ +bool mxs_mysql_is_ok_packet(GWBUF *buffer) +{ + bool rval = false; + uint8_t cmd; + + if (gwbuf_copy_data(buffer, MYSQL_HEADER_LEN, 1, &cmd) && cmd == MYSQL_REPLY_OK) + { + rval = true; + } + + return rval; +} From 5d96faedd88a12ea1ed8c7ae024aa95755940ff6 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 7 Oct 2016 07:40:11 +0300 Subject: [PATCH 010/215] MXS-862: Move sending of OK packet to mysql_client Moving the sending of the final OK packet of the authentication process to the client protocol plugin makes the authentication plugins simpler. By reading the client's sequence and incrementing that by one, the client protocol module will always send the correct sequence byte in the final OK packet. --- server/modules/authenticator/gssapi_auth.c | 3 +-- server/modules/authenticator/mysql_auth.c | 1 - server/modules/include/mysql_client_server_protocol.h | 4 ++++ server/modules/protocol/MySQL/MySQLClient/mysql_client.c | 6 ++++++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/server/modules/authenticator/gssapi_auth.c b/server/modules/authenticator/gssapi_auth.c index 6c165f683..27d574950 100644 --- a/server/modules/authenticator/gssapi_auth.c +++ b/server/modules/authenticator/gssapi_auth.c @@ -281,8 +281,7 @@ int gssapi_auth_authenticate(DCB *dcb) MYSQL_session *ses = (MYSQL_session*)dcb->data; - if (validate_gssapi_token(ses->auth_token, ses->auth_token_len) && - mxs_mysql_send_ok(dcb, 4, 0, NULL)) + if (validate_gssapi_token(ses->auth_token, ses->auth_token_len)) { rval = MXS_AUTH_SUCCEEDED; } diff --git a/server/modules/authenticator/mysql_auth.c b/server/modules/authenticator/mysql_auth.c index 838e64346..a84e20467 100644 --- a/server/modules/authenticator/mysql_auth.c +++ b/server/modules/authenticator/mysql_auth.c @@ -193,7 +193,6 @@ mysql_auth_authenticate(DCB *dcb) { dcb->user = MXS_STRDUP_A(client_data->user); /** Send an OK packet to the client */ - mxs_mysql_send_ok(dcb, ssl_required_by_dcb(dcb) ? 3 : 2, 0, NULL); } else if (dcb->service->log_auth_warnings) { diff --git a/server/modules/include/mysql_client_server_protocol.h b/server/modules/include/mysql_client_server_protocol.h index b50a3fac4..2ef9ad71d 100644 --- a/server/modules/include/mysql_client_server_protocol.h +++ b/server/modules/include/mysql_client_server_protocol.h @@ -70,6 +70,10 @@ #define MYSQL_HEADER_LEN 4L #define MYSQL_CHECKSUM_LEN 4L +/** Offsets to various parts of the client packet */ +#define MYSQL_SEQ_OFFSET 3 +#define MYSQL_COM_OFFSET 4 + #define GW_MYSQL_PROTOCOL_VERSION 10 // version is 10 #define GW_MYSQL_HANDSHAKE_FILLER 0x00 #define GW_MYSQL_SERVER_CAPABILITIES_BYTE1 0xff diff --git a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c index 2ac029271..d61c1c05d 100644 --- a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c +++ b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c @@ -486,6 +486,11 @@ gw_read_do_authentication(DCB *dcb, GWBUF *read_buffer, int nbytes_read) return 1; } + /** Read the client's packet sequence and increment that by one */ + uint8_t next_sequence; + gwbuf_copy_data(read_buffer, MYSQL_SEQ_OFFSET, 1, &next_sequence); + next_sequence++; + /** * The first step in the authentication process is to extract the * relevant information from the buffer supplied and place it @@ -542,6 +547,7 @@ gw_read_do_authentication(DCB *dcb, GWBUF *read_buffer, int nbytes_read) ss_dassert(session->state != SESSION_STATE_ALLOC && session->state != SESSION_STATE_DUMMY); protocol->protocol_auth_state = MXS_AUTH_STATE_COMPLETE; + mxs_mysql_send_ok(dcb, next_sequence, 0, NULL); } else { From a2a8562c393fb265715eed3f4dac6537c5b35887 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Wed, 5 Oct 2016 22:59:25 +0300 Subject: [PATCH 011/215] MXS-862: Implement GSSAPI backend authentication The GSSAPI backend authentication is based on tokens. The server first sends the service principal name which is used for token generation. The client then retrieves a token from the GSSAPI server which it sends to the backend server. If the server can verify the authenticity of the token, authentication is successful. This module can be used with both GSSAPIAuth and MySQLAuth modules. --- server/modules/authenticator/gssapi_auth.c | 42 +---- server/modules/authenticator/gssapi_auth.h | 8 + .../authenticator/gssapi_auth_common.c | 38 ++++- .../authenticator/gssapi_backend_auth.c | 149 ++++++++++++++++-- server/modules/protocol/MySQL/mysql_common.c | 2 +- 5 files changed, 191 insertions(+), 48 deletions(-) diff --git a/server/modules/authenticator/gssapi_auth.c b/server/modules/authenticator/gssapi_auth.c index 27d574950..d166c3ed8 100644 --- a/server/modules/authenticator/gssapi_auth.c +++ b/server/modules/authenticator/gssapi_auth.c @@ -15,7 +15,6 @@ #include #include #include -#include #include "gssapi_auth.h" /** @@ -87,9 +86,14 @@ bool store_client_token(DCB *dcb, GWBUF *buffer) * @param dcb Client DCB * @param buffer Buffer containing the first authentication response */ -static void copy_shared_username(DCB *dcb, GWBUF *buffer) +static void copy_client_information(DCB *dcb, GWBUF *buffer) { size_t buflen = gwbuf_length(buffer); + MySQLProtocol *protocol = (MySQLProtocol*)dcb->protocol; + /* Take data from fixed locations first */ + gwbuf_copy_data(buffer, 4, 4, (uint8_t*)&protocol->client_capabilities); + protocol->charset = 0; + gwbuf_copy_data(buffer, 4 + 4 + 4, 1, (uint8_t*)&protocol->charset); if (buflen > MYSQL_AUTH_PACKET_BASE_SIZE) { @@ -123,7 +127,7 @@ static int gssapi_auth_extract(DCB *dcb, GWBUF *read_buffer) switch (auth->state) { case GSSAPI_AUTH_INIT: - copy_shared_username(dcb, read_buffer); + copy_client_information(dcb, read_buffer); rval = MXS_AUTH_SUCCEEDED; break; @@ -153,38 +157,6 @@ bool gssapi_auth_connectssl(DCB *dcb) return protocol->client_capabilities & GW_MYSQL_CAPABILITIES_SSL; } -/** - * @brief Report GSSAPI errors - * - * @param major GSSAPI major error number - * @param minor GSSAPI minor error number - */ -static void report_error(OM_uint32 major, OM_uint32 minor) -{ - OM_uint32 status_maj = major; - OM_uint32 status_min = minor; - OM_uint32 res = 0; - gss_buffer_desc buf = {0, 0}; - - major = gss_display_status(&minor, status_maj, GSS_C_GSS_CODE, NULL, &res, &buf); - - { - char sbuf[buf.length + 1]; - memcpy(sbuf, buf.value, buf.length); - sbuf[buf.length] = '\0'; - MXS_ERROR("GSSAPI Major Error: %s", sbuf); - } - - major = gss_display_status(&minor, status_min, GSS_C_MECH_CODE, NULL, &res, &buf); - - { - char sbuf[buf.length + 1]; - memcpy(sbuf, buf.value, buf.length); - sbuf[buf.length] = '\0'; - MXS_ERROR("GSSAPI Minor Error: %s", sbuf); - } -} - static gss_name_t server_name = GSS_C_NO_NAME; /** diff --git a/server/modules/authenticator/gssapi_auth.h b/server/modules/authenticator/gssapi_auth.h index cb2ca6084..819768f1e 100644 --- a/server/modules/authenticator/gssapi_auth.h +++ b/server/modules/authenticator/gssapi_auth.h @@ -14,6 +14,9 @@ #ifndef _GSSAPI_AUTH_H #define _GSSAPI_AUTH_H +#include +#include +#include /** Client auth plugin name */ static const char auth_plugin_name[] = "auth_gssapi_client"; @@ -34,10 +37,15 @@ enum gssapi_auth_state typedef struct gssapi_auth { enum gssapi_auth_state state; + uint8_t *principal_name; + size_t principal_name_len; } gssapi_auth_t; /** These functions can used for the `create` and `destroy` entry points */ void* gssapi_auth_alloc(); void gssapi_auth_free(void *data); +/** Report GSSAPI errors */ +void report_error(OM_uint32 major, OM_uint32 minor); + #endif diff --git a/server/modules/authenticator/gssapi_auth_common.c b/server/modules/authenticator/gssapi_auth_common.c index a6bcef631..a147c7186 100644 --- a/server/modules/authenticator/gssapi_auth_common.c +++ b/server/modules/authenticator/gssapi_auth_common.c @@ -21,6 +21,8 @@ void* gssapi_auth_alloc() if (rval) { rval->state = GSSAPI_AUTH_INIT; + rval->principal_name = NULL; + rval->principal_name_len = 0; } return rval; @@ -30,6 +32,40 @@ void gssapi_auth_free(void *data) { if (data) { - MXS_FREE(data); + gssapi_auth_t *auth = (gssapi_auth_t*)data; + MXS_FREE(auth->principal_name); + MXS_FREE(auth); + } +} + +/** + * @brief Report GSSAPI errors + * + * @param major GSSAPI major error number + * @param minor GSSAPI minor error number + */ +void report_error(OM_uint32 major, OM_uint32 minor) +{ + OM_uint32 status_maj = major; + OM_uint32 status_min = minor; + OM_uint32 res = 0; + gss_buffer_desc buf = {0, 0}; + + major = gss_display_status(&minor, status_maj, GSS_C_GSS_CODE, NULL, &res, &buf); + + { + char sbuf[buf.length + 1]; + memcpy(sbuf, buf.value, buf.length); + sbuf[buf.length] = '\0'; + MXS_ERROR("GSSAPI Major Error: %s", sbuf); + } + + major = gss_display_status(&minor, status_min, GSS_C_MECH_CODE, NULL, &res, &buf); + + { + char sbuf[buf.length + 1]; + memcpy(sbuf, buf.value, buf.length); + sbuf[buf.length] = '\0'; + MXS_ERROR("GSSAPI Minor Error: %s", sbuf); } } diff --git a/server/modules/authenticator/gssapi_backend_auth.c b/server/modules/authenticator/gssapi_backend_auth.c index 38b260aae..a6c8e2c06 100644 --- a/server/modules/authenticator/gssapi_backend_auth.c +++ b/server/modules/authenticator/gssapi_backend_auth.c @@ -15,17 +15,127 @@ #include #include #include -#include #include "gssapi_auth.h" /** * @file gssapi_backend_auth.c GSSAPI backend authenticator */ +/** TODO: Document functions and GSSAPI specific code */ + +static bool send_new_auth_token(DCB *dcb) +{ + bool rval = false; + OM_uint32 major = 0, minor = 0; + gss_ctx_id_t handle = NULL; + gss_buffer_desc in = {0, 0}; + gss_buffer_desc out = {0, 0}; + gss_buffer_desc target = {0, 0}; + gss_name_t princ = GSS_C_NO_NAME; + gssapi_auth_t *auth = (gssapi_auth_t*)dcb->authenticator_data; + + target.value = auth->principal_name; + target.length = auth->principal_name_len + 1; + + major = gss_import_name(&minor, &target, GSS_C_NT_USER_NAME, &princ); + + if (GSS_ERROR(major)) + { + report_error(major, minor); + } + + major = gss_init_sec_context(&minor, GSS_C_NO_CREDENTIAL, + &handle, princ, GSS_C_NO_OID, 0, 0, + GSS_C_NO_CHANNEL_BINDINGS, &in, NULL, &out, 0, 0); + if (GSS_ERROR(major)) + { + report_error(major, minor); + } + else + { + GWBUF *buffer = gwbuf_alloc(MYSQL_HEADER_LEN + out.length); + + if (buffer) + { + uint8_t *data = (uint8_t*)GWBUF_DATA(buffer); + gw_mysql_set_byte3(data, out.length); + data += 3; + *data++ = 0x03; + memcpy(data, out.value, out.length); + if (dcb_write(dcb, buffer)) + { + rval = true; + } + } + + major = gss_delete_sec_context(&minor, &handle, &in); + + if (GSS_ERROR(major)) + { + report_error(major, minor); + } + + major = gss_release_name(&minor, &princ); + if (GSS_ERROR(major)) + { + report_error(major, minor); + } + } + + return rval; + +} + +bool extract_principal_name(DCB *dcb, GWBUF *buffer) +{ + bool rval = false; + size_t buflen = gwbuf_length(buffer) - MYSQL_HEADER_LEN; + uint8_t databuf[buflen]; + gwbuf_copy_data(buffer, MYSQL_HEADER_LEN, buflen, databuf); + uint8_t *data = databuf; + + while (*data) + { + data++; + } + + data++; + buflen -= data - databuf; + + uint8_t *principal = MXS_MALLOC(buflen); + + if (principal) + { + memcpy(principal, data, buflen); + gssapi_auth_t *auth = (gssapi_auth_t*)dcb->authenticator_data; + auth->principal_name = principal; + auth->principal_name_len = buflen; + rval = true; + } + + return rval; +} + static int gssapi_backend_auth_extract(DCB *dcb, GWBUF *buffer) { - gw_send_backend_auth(dcb); - return MXS_AUTH_SUCCEEDED; + int rval = MXS_AUTH_FAILED; + gssapi_auth_t *data = (gssapi_auth_t*)dcb->authenticator_data; + + if (data->state == GSSAPI_AUTH_INIT && extract_principal_name(dcb, buffer)) + { + rval = MXS_AUTH_INCOMPLETE; + } + else if (data->state == GSSAPI_AUTH_DATA_SENT) + { + /** Read authentication response */ + if (mxs_mysql_is_ok_packet(buffer)) + { + data->state = GSSAPI_AUTH_OK; + rval = MXS_AUTH_SUCCEEDED; + } + } + + return rval; } static bool gssapi_backend_auth_connectssl(DCB *dcb) @@ -35,7 +145,24 @@ static bool gssapi_backend_auth_connectssl(DCB *dcb) static int gssapi_backend_auth_authenticate(DCB *dcb) { - return MXS_AUTH_SUCCEEDED; + int rval = MXS_AUTH_FAILED; + gssapi_auth_t *auth_data = (gssapi_auth_t*)dcb->authenticator_data; + + if (auth_data->state == GSSAPI_AUTH_INIT) + { + if (send_new_auth_token(dcb)) + { + rval = MXS_AUTH_INCOMPLETE; + auth_data->state = GSSAPI_AUTH_DATA_SENT; + } + + } + else if (auth_data->state == GSSAPI_AUTH_OK) + { + rval = MXS_AUTH_SUCCEEDED; + } + + return rval; } /** @@ -43,13 +170,13 @@ static int gssapi_backend_auth_authenticate(DCB *dcb) */ static GWAUTHENTICATOR MyObject = { - gssapi_auth_alloc, - gssapi_backend_auth_extract, /* Extract data into structure */ - gssapi_backend_auth_connectssl, /* Check if client supports SSL */ - gssapi_backend_auth_authenticate, /* Authenticate user credentials */ - NULL, - gssapi_auth_free, - NULL /* Load users from backend databases */ + gssapi_auth_alloc, /* Allocate authenticator data */ + gssapi_backend_auth_extract, /* Extract data into structure */ + gssapi_backend_auth_connectssl, /* Check if client supports SSL */ + gssapi_backend_auth_authenticate, /* Authenticate user credentials */ + NULL, /* Client plugin will free shared data */ + gssapi_auth_free, /* Free authenticator data */ + NULL /* Load users from backend databases */ }; MODULE_INFO info = diff --git a/server/modules/protocol/MySQL/mysql_common.c b/server/modules/protocol/MySQL/mysql_common.c index cfe6abe91..834256fe4 100644 --- a/server/modules/protocol/MySQL/mysql_common.c +++ b/server/modules/protocol/MySQL/mysql_common.c @@ -1354,7 +1354,7 @@ mxs_auth_state_t gw_send_backend_auth(DCB *dcb) */ const char* auth_plugin_name = DEFAULT_MYSQL_AUTH_PLUGIN; - long bytes = response_length(conn, local_session.user, local_session.client_sha1, + long bytes = response_length(conn, local_session.user, curr_passwd, local_session.db, auth_plugin_name); // allocating the GWBUF From 9d8c5cd4107e747748d5f04033176520fd6c662f Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 11 Oct 2016 22:44:43 +0300 Subject: [PATCH 012/215] MXS-862: Add authenticator options and instances Authenticators now have a similar mechanism to the `router_options` parameter which enables configurable authentication. The authenticators also have a new initialize entry point which is similar to the createInstance entry point of the filters and routers. The value of `authenticator_options` is passed as a parameter to this function. The return vaulue of the `initialize` entry point is passed to the `create` entry point. --- server/core/CMakeLists.txt | 2 +- server/core/authenticator.c | 103 ++++++++++++++++++ server/core/config.c | 16 ++- server/core/dcb.c | 24 ++-- server/core/listener.c | 27 +++-- server/core/server.c | 30 ++++- server/core/service.c | 5 +- server/core/test/test_mysql_users.c | 2 +- server/core/test/testserver.c | 2 +- server/core/test/testservice.c | 2 +- server/include/gw_authenticator.h | 49 ++++++--- server/include/listener.h | 3 +- server/include/server.h | 3 +- server/include/service.h | 3 +- server/modules/authenticator/cdc_plain_auth.c | 1 + server/modules/authenticator/gssapi_auth.c | 1 + server/modules/authenticator/gssapi_auth.h | 2 +- .../authenticator/gssapi_auth_common.c | 2 +- .../authenticator/gssapi_backend_auth.c | 1 + server/modules/authenticator/http_auth.c | 1 + server/modules/authenticator/max_admin_auth.c | 1 + server/modules/authenticator/mysql_auth.c | 1 + .../authenticator/mysql_backend_auth.c | 11 +- .../modules/authenticator/null_auth_allow.c | 1 + server/modules/authenticator/null_auth_deny.c | 1 + server/modules/routing/binlog/blr.c | 2 +- .../modules/routing/binlog/test/testbinlog.c | 2 +- 27 files changed, 236 insertions(+), 62 deletions(-) create mode 100644 server/core/authenticator.c diff --git a/server/core/CMakeLists.txt b/server/core/CMakeLists.txt index 7a50e6e55..06302df63 100644 --- a/server/core/CMakeLists.txt +++ b/server/core/CMakeLists.txt @@ -1,4 +1,4 @@ -add_library(maxscale-common SHARED adminusers.c alloc.c atomic.c buffer.c config.c dbusers.c dcb.c filter.c externcmd.c gwbitmask.c gwdirs.c gw_utils.c hashtable.c hint.c housekeeper.c listmanager.c load_utils.c log_manager.cc maxscale_pcre2.c memlog.c misc.c mlist.c modutil.c monitor.c queuemanager.c query_classifier.c poll.c random_jkiss.c resultset.c secrets.c server.c service.c session.c slist.c spinlock.c thread.c users.c utils.c skygw_utils.cc statistics.c listener.c gw_ssl.c mysql_utils.c mysql_binlog.c) +add_library(maxscale-common SHARED adminusers.c alloc.c authenticator.c atomic.c buffer.c config.c dbusers.c dcb.c filter.c externcmd.c gwbitmask.c gwdirs.c gw_utils.c hashtable.c hint.c housekeeper.c listmanager.c load_utils.c log_manager.cc maxscale_pcre2.c memlog.c misc.c mlist.c modutil.c monitor.c queuemanager.c query_classifier.c poll.c random_jkiss.c resultset.c secrets.c server.c service.c session.c slist.c spinlock.c thread.c users.c utils.c skygw_utils.cc statistics.c listener.c gw_ssl.c mysql_utils.c mysql_binlog.c) target_link_libraries(maxscale-common ${MARIADB_CONNECTOR_LIBRARIES} ${LZMA_LINK_FLAGS} ${PCRE2_LIBRARIES} ${CURL_LIBRARIES} ssl pthread crypt dl crypto inih z rt m stdc++) diff --git a/server/core/authenticator.c b/server/core/authenticator.c new file mode 100644 index 000000000..d70bdb785 --- /dev/null +++ b/server/core/authenticator.c @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2016 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl. + * + * Change Date: 2019-07-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ + +#include +#include +#include +#include + +/** + * @file authenticator.c - Authenticator module functions + */ + +/** + * @brief Initialize an authenticator module + * + * Process the options into an array and pass them to the authenticator + * initialization function + * + * The authenticator must implement the @c initialize entry point if this + * function is called. If the authenticator does not implement this, behavior is + * undefined. + * + * @param func Authenticator entry point + * @param options Authenticator options + * @return Authenticator instance or NULL on error + */ +bool authenticator_init(void** dest, const char *authenticator, const char *options) +{ + bool rval = true; + void *instance = NULL; + GWAUTHENTICATOR *func = (GWAUTHENTICATOR*)load_module(authenticator, MODULE_AUTHENTICATOR); + + if (func == NULL) + { + rval = false; + } + else if (func->initialize) + { + char *optarray[AUTHENTICATOR_MAX_OPTIONS + 1]; + size_t optlen = options ? strlen(options) : 0; + char optcopy[optlen + 1]; + int optcount = 0; + + if (options) + { + strcpy(optcopy, options); + char *opt = optcopy; + + while (opt && optcount < AUTHENTICATOR_MAX_OPTIONS) + { + char *end = strnchr_esc(opt, ',', sizeof(optcopy) - (opt - optcopy)); + + if (end) + { + *end++ = '\0'; + } + + optarray[optcount++] = opt; + opt = end; + } + } + + optarray[optcount] = NULL; + + if ((instance = func->initialize(optarray)) == NULL) + { + rval = false; + } + } + + *dest = instance; + return rval; +} + +/** + * @brief Get the default authenticator for a protocol + * + * @param protocol Protocol to inspect + * @return The default authenticator for the protocol or NULL if the protocol + * does not provide one + */ +char* get_default_authenticator(const char *protocol) +{ + char *rval = NULL; + GWPROTOCOL *protofuncs = (GWPROTOCOL*)load_module(protocol, MODULE_PROTOCOL); + + if (protofuncs && protofuncs->auth_default) + { + rval = MXS_STRDUP(protofuncs->auth_default()); + } + + return rval; +} diff --git a/server/core/config.c b/server/core/config.c index 82331c901..e9848769f 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -147,6 +147,7 @@ static char *service_params[] = static char *listener_params[] = { + "authenticator_options", "type", "service", "protocol", @@ -2506,10 +2507,11 @@ int create_new_server(CONFIG_CONTEXT *obj) char *monuser = config_get_value(obj->parameters, "monitoruser"); char *monpw = config_get_value(obj->parameters, "monitorpw"); char *auth = config_get_value(obj->parameters, "authenticator"); + char *auth_opts = config_get_value(obj->parameters, "authenticator_options"); if (address && port && protocol) { - if ((obj->element = server_alloc(address, protocol, atoi(port)))) + if ((obj->element = server_alloc(address, protocol, atoi(port), auth, auth_opts))) { server_set_unique_name(obj->element, obj->object); } @@ -2542,11 +2544,6 @@ int create_new_server(CONFIG_CONTEXT *obj) error_count++; } - if (auth && (server->authenticator = MXS_STRDUP(auth)) == NULL) - { - error_count++; - } - char *endptr; const char *poolmax = config_get_value_string(obj->parameters, "persistpoolmax"); if (poolmax) @@ -2810,6 +2807,7 @@ int create_new_listener(CONFIG_CONTEXT *obj, bool startnow) char *protocol = config_get_value(obj->parameters, "protocol"); char *socket = config_get_value(obj->parameters, "socket"); char *authenticator = config_get_value(obj->parameters, "authenticator"); + char *authenticator_options = config_get_value(obj->parameters, "authenticator_options"); if (service_name && protocol && (socket || port)) { @@ -2828,7 +2826,7 @@ int create_new_listener(CONFIG_CONTEXT *obj, bool startnow) else { serviceAddProtocol(service, obj->object, protocol, socket, 0, - authenticator, ssl_info); + authenticator, authenticator_options, ssl_info); if (startnow) { serviceStartProtocol(service, protocol, 0); @@ -2848,8 +2846,8 @@ int create_new_listener(CONFIG_CONTEXT *obj, bool startnow) } else { - serviceAddProtocol(service, obj->object, protocol, address, - atoi(port), authenticator, ssl_info); + serviceAddProtocol(service, obj->object, protocol, address, atoi(port), + authenticator, authenticator_options, ssl_info); if (startnow) { serviceStartProtocol(service, protocol, atoi(port)); diff --git a/server/core/dcb.c b/server/core/dcb.c index 74f49b539..9f7e54f99 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -814,12 +814,6 @@ dcb_connect(SERVER *server, SESSION *session, const char *protocol) memcpy(&dcb->authfunc, authfuncs, sizeof(GWAUTHENTICATOR)); - /** Allocate DCB specific authentication data */ - if (dcb->authfunc.create) - { - dcb->authenticator_data = dcb->authfunc.create(); - } - /** * Link dcb to session. Unlink is called in dcb_final_free */ @@ -879,6 +873,16 @@ dcb_connect(SERVER *server, SESSION *session, const char *protocol) * is established. */ + /** Allocate DCB specific authentication data */ + if (dcb->authfunc.create && + (dcb->authenticator_data = dcb->authfunc.create(dcb->server->auth_instance)) == NULL) + { + MXS_ERROR("Failed to create authenticator for backend DCB."); + dcb->state = DCB_STATE_DISCONNECTED; + dcb_final_free(dcb); + return NULL; + } + /** * Add the dcb in the poll set */ @@ -3180,9 +3184,13 @@ dcb_accept(DCB *listener, GWPROTOCOL *protocol_funcs) memcpy(&(client_dcb->authfunc), authfuncs, sizeof(GWAUTHENTICATOR)); /** Allocate DCB specific authentication data */ - if (client_dcb->authfunc.create) + if (client_dcb->authfunc.create && + (client_dcb->authenticator_data = client_dcb->authfunc.create( + client_dcb->listener->auth_instance)) == NULL) { - client_dcb->authenticator_data = client_dcb->authfunc.create(); + MXS_ERROR("Failed to create authenticator for client DCB."); + dcb_close(client_dcb); + return NULL; } if (client_dcb->service->max_connections && diff --git a/server/core/listener.c b/server/core/listener.c index 71b7e35c7..249119d78 100644 --- a/server/core/listener.c +++ b/server/core/listener.c @@ -36,6 +36,7 @@ #include #include #include +#include static RSA *rsa_512 = NULL; static RSA *rsa_1024 = NULL; @@ -49,12 +50,13 @@ static RSA *tmp_rsa_callback(SSL *s, int is_export, int keylength); * @param address The address to listen with * @param port The port to listen on * @param authenticator Name of the authenticator to be used + * @param options Authenticator options * @param ssl SSL configuration * @return New listener object or NULL if unable to allocate */ SERV_LISTENER * listener_alloc(struct service* service, char* name, char *protocol, char *address, - unsigned short port, char *authenticator, SSL_LISTENER *ssl) + unsigned short port, char *authenticator, char* auth_options, SSL_LISTENER *ssl) { if (address) { @@ -68,20 +70,30 @@ listener_alloc(struct service* service, char* name, char *protocol, char *addres if (authenticator) { authenticator = MXS_STRDUP(authenticator); - if (!authenticator) - { - MXS_FREE(address); - return NULL; - } + } + else if ((authenticator = get_default_authenticator(protocol)) == NULL) + { + MXS_ERROR("No authenticator defined for listener '%s' and could not get " + "default authenticator for protocol '%s'.", name, protocol); + } + + void *auth_instance = NULL; + + if (!authenticator_init(&auth_instance, authenticator, auth_options)) + { + MXS_ERROR("Failed to initialize authenticator module '%s' for " + "listener '%s'.", authenticator, name); } protocol = MXS_STRDUP(protocol); name = MXS_STRDUP(name); SERV_LISTENER *proto = (SERV_LISTENER*)MXS_MALLOC(sizeof(SERV_LISTENER)); - if (!protocol || !proto || !name) + if (!protocol || !proto || !name || !authenticator) { + MXS_FREE(authenticator); MXS_FREE(protocol); + MXS_FREE(address); MXS_FREE(proto); MXS_FREE(name); return NULL; @@ -98,6 +110,7 @@ listener_alloc(struct service* service, char* name, char *protocol, char *addres proto->users = NULL; proto->resources = NULL; proto->next = NULL; + proto->auth_instance = auth_instance; spinlock_init(&proto->lock); return proto; diff --git a/server/core/server.c b/server/core/server.c index 9e0835e09..516ae7094 100644 --- a/server/core/server.c +++ b/server/core/server.c @@ -44,6 +44,7 @@ #include #include #include +#include static SPINLOCK server_spin = SPINLOCK_INIT; static SERVER *allServers = NULL; @@ -62,18 +63,40 @@ static void server_parameter_free(SERVER_PARAM *tofree); * @return The newly created server or NULL if an error occured */ SERVER * -server_alloc(char *servname, char *protocol, unsigned short port) +server_alloc(char *servname, char *protocol, unsigned short port, char *authenticator, + char *auth_options) { + if (authenticator) + { + authenticator = MXS_STRDUP(authenticator); + } + else if ((authenticator = get_default_authenticator(protocol)) == NULL) + { + MXS_ERROR("No authenticator defined for server at %s:%u and no default " + "authenticator for protocol '%s'.", servname, port, protocol); + } + + void *auth_instance = NULL; + + if (!authenticator_init(&auth_instance, authenticator, auth_options)) + { + MXS_ERROR("Failed to initialize authenticator module '%s' for server" + " at %s:%u.", authenticator, servname, port); + MXS_FREE(authenticator); + return NULL; + } + servname = MXS_STRNDUP(servname, MAX_SERVER_NAME_LEN); protocol = MXS_STRDUP(protocol); SERVER *server = (SERVER *)MXS_CALLOC(1, sizeof(SERVER)); - if (!servname || !protocol || !server) + if (!servname || !protocol || !server || !authenticator) { MXS_FREE(servname); MXS_FREE(protocol); MXS_FREE(server); + MXS_FREE(authenticator); return NULL; } @@ -83,7 +106,8 @@ server_alloc(char *servname, char *protocol, unsigned short port) #endif server->name = servname; server->protocol = protocol; - server->authenticator = NULL; + server->authenticator = authenticator; + server->auth_instance = auth_instance; server->port = port; server->status = SERVER_RUNNING; server->node_id = -1; diff --git a/server/core/service.c b/server/core/service.c index b4304e238..4452d6604 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -672,11 +672,12 @@ service_free(SERVICE *service) * @return TRUE if the protocol/port could be added */ int -serviceAddProtocol(SERVICE *service, char *name, char *protocol, char *address, unsigned short port, char *authenticator, +serviceAddProtocol(SERVICE *service, char *name, char *protocol, char *address, + unsigned short port, char *authenticator, char *options, SSL_LISTENER *ssl) { SERV_LISTENER *proto = listener_alloc(service, name, protocol, address, - port, authenticator, ssl); + port, authenticator, options, ssl); if (proto) { diff --git a/server/core/test/test_mysql_users.c b/server/core/test/test_mysql_users.c index 18df9eedd..8ab3439a4 100644 --- a/server/core/test/test_mysql_users.c +++ b/server/core/test/test_mysql_users.c @@ -213,7 +213,7 @@ int set_and_get_mysql_users_wildcards(char *username, char *hostname, char *pass return ret; } - SERV_LISTENER *port = listener_alloc(service, "testlistener", "MySQLClient", NULL, 4006, NULL, NULL); + SERV_LISTENER *port = listener_alloc(service, "testlistener", "MySQLClient", NULL, 4006, "MySQLAuth", NULL, NULL); dcb = dcb_alloc(DCB_ROLE_INTERNAL, port); diff --git a/server/core/test/testserver.c b/server/core/test/testserver.c index a959765f4..2424cedcb 100644 --- a/server/core/test/testserver.c +++ b/server/core/test/testserver.c @@ -50,7 +50,7 @@ test1() /* Server tests */ ss_dfprintf(stderr, "testserver : creating server called MyServer"); - server = server_alloc("MyServer", "HTTPD", 9876); + server = server_alloc("MyServer", "HTTPD", 9876, "NullAuthAccept", NULL); mxs_log_flush_sync(); //ss_info_dassert(NULL != service, "New server with valid protocol and port must not be null"); diff --git a/server/core/test/testservice.c b/server/core/test/testservice.c index e7dc7d765..cd896adc3 100644 --- a/server/core/test/testservice.c +++ b/server/core/test/testservice.c @@ -69,7 +69,7 @@ test1() ss_info_dassert(0 == strcmp("MyService", service_get_name(service)), "Service must have given name"); ss_dfprintf(stderr, "\t..done\nAdding protocol testprotocol."); ss_info_dassert(0 != serviceAddProtocol(service, "TestProtocol", "testprotocol", - "localhost", 9876, "MySQL", NULL), + "localhost", 9876, "MySQLClient", "MySQLAuth", NULL), "Add Protocol should succeed"); ss_info_dassert(0 != serviceHasProtocol(service, "testprotocol", "localhost", 9876), "Service should have new protocol as requested"); diff --git a/server/include/gw_authenticator.h b/server/include/gw_authenticator.h index 24c06b532..29a06ae40 100644 --- a/server/include/gw_authenticator.h +++ b/server/include/gw_authenticator.h @@ -33,6 +33,9 @@ #include #include +/** Maximum number of authenticator options */ +#define AUTHENTICATOR_MAX_OPTIONS 256 + struct dcb; struct server; struct session; @@ -42,14 +45,30 @@ struct servlistener; * @verbatim * The operations that can be performed on the descriptor * - * create Create a data structure unique to this DCB, stored in dcb->authenticator_data + * initialize Initialize the authenticator instance. The return value + * of this function will be given to the `create` entry point. + * If a module does not implement this entry point, the value + * given to the `create` is NULL. + * + * create Create a data structure unique to this DCB, stored in + * `dcb->authenticator_data`. If a module does not implement + * this entry point, `dcb->authenticator_data` will be set to NULL. + * * extract Extract the data from a buffer and place in a structure - * shared at the session level, stored in dcb->data - * connectssl Determine whether the connection can support SSL + * shared at the session level, stored in `dcb->data` + * + * connectSSL Determine whether the connection can support SSL + * * authenticate Carry out the authentication - * free Free extracted data - * destroy Destroy the unique DCB data - * loadusers Load or update authenticator user data + * + * free Free extracted data. This is only called for the client + * side authenticators so backend authenticators should not + * implement it. + * + * destroy Destroy the unique DCB data returned by the `create` + * entry point. + * + * loadUsers Load or update authenticator user data * @endverbatim * * This forms the "module object" for authenticator modules within the gateway. @@ -58,13 +77,14 @@ struct servlistener; */ typedef struct gw_authenticator { - void *(*create)(); - int (*extract)(struct dcb *, GWBUF *); - bool (*connectssl)(struct dcb *); - int (*authenticate)(struct dcb *); - void (*free)(struct dcb *); - void (*destroy)(void *); - int (*loadusers)(struct servlistener *); + void* (*initialize)(char **options); + void* (*create)(void* instance); + int (*extract)(struct dcb *, GWBUF *); + bool (*connectssl)(struct dcb *); + int (*authenticate)(struct dcb *); + void (*free)(struct dcb *); + void (*destroy)(void *); + int (*loadusers)(struct servlistener *); } GWAUTHENTICATOR; /** Return values for extract and authenticate entry points */ @@ -111,5 +131,8 @@ typedef enum #define GWAUTHENTICATOR_VERSION {1, 1, 0} +bool authenticator_init(void **instance, const char *authenticator, const char *options); +char* get_default_authenticator(const char *protocol); + #endif /* GW_AUTHENTICATOR_H */ diff --git a/server/include/listener.h b/server/include/listener.h index f243e534d..7e1c9c187 100644 --- a/server/include/listener.h +++ b/server/include/listener.h @@ -47,6 +47,7 @@ typedef struct servlistener unsigned short port; /**< Port to listen on */ char *address; /**< Address to listen with */ char *authenticator; /**< Name of authenticator */ + void *auth_instance; /**< Authenticator instance created in GWAUTHENTICATOR::initialize() */ SSL_LISTENER *ssl; /**< Structure of SSL data or NULL */ struct dcb *listener; /**< The DCB for the listener */ struct users *users; /**< The user data for this listener */ @@ -58,7 +59,7 @@ typedef struct servlistener SERV_LISTENER *listener_alloc(struct service* service, char *name, char *protocol, char *address, unsigned short port, char *authenticator, - SSL_LISTENER *ssl); + char* options, SSL_LISTENER *ssl); void listener_free(SERV_LISTENER* listener); int listener_set_ssl_version(SSL_LISTENER *ssl_listener, char* version); void listener_set_certificates(SSL_LISTENER *ssl_listener, char* cert, char* key, char* ca_cert); diff --git a/server/include/server.h b/server/include/server.h index f708b84ec..3b38abb70 100644 --- a/server/include/server.h +++ b/server/include/server.h @@ -87,6 +87,7 @@ typedef struct server unsigned short port; /**< Port to listen on */ char *protocol; /**< Protocol module to use */ char *authenticator; /**< Authenticator module name */ + void *auth_instance; /**< Authenticator instance */ SSL_LISTENER *server_ssl; /**< SSL data structure for server, if any */ unsigned int status; /**< Status flag bitmap for the server */ char *monuser; /**< User name to use to monitor the db */ @@ -193,7 +194,7 @@ typedef struct server (((server)->status & (SERVER_RUNNING|SERVER_MASTER|SERVER_SLAVE|SERVER_MAINT)) == \ (SERVER_RUNNING|SERVER_MASTER|SERVER_SLAVE)) -extern SERVER *server_alloc(char *, char *, unsigned short); +extern SERVER *server_alloc(char *, char *, unsigned short, char*, char*); extern int server_free(SERVER *); extern SERVER *server_find_by_unique_name(char *); extern SERVER *server_find(char *, unsigned short); diff --git a/server/include/service.h b/server/include/service.h index 8e0d9de89..385ac727f 100644 --- a/server/include/service.h +++ b/server/include/service.h @@ -173,7 +173,8 @@ extern SERVICE *service_find(char *); extern int service_isvalid(SERVICE *); extern int serviceAddProtocol(SERVICE *service, char *name, char *protocol, char *address, unsigned short port, - char *authenticator, SSL_LISTENER *ssl); + char *authenticator, char *options, + SSL_LISTENER *ssl); extern int serviceHasProtocol(SERVICE *service, const char *protocol, const char* address, unsigned short port); extern void serviceAddBackend(SERVICE *, SERVER *); diff --git a/server/modules/authenticator/cdc_plain_auth.c b/server/modules/authenticator/cdc_plain_auth.c index bdecec83f..909070c92 100644 --- a/server/modules/authenticator/cdc_plain_auth.c +++ b/server/modules/authenticator/cdc_plain_auth.c @@ -67,6 +67,7 @@ extern char *decryptPassword(char *crypt); */ static GWAUTHENTICATOR MyObject = { + NULL, /* No initialize entry point */ NULL, /* No create entry point */ cdc_auth_set_protocol_data, /* Extract data into structure */ cdc_auth_is_client_ssl_capable, /* Check if client supports SSL */ diff --git a/server/modules/authenticator/gssapi_auth.c b/server/modules/authenticator/gssapi_auth.c index d166c3ed8..fba19dda3 100644 --- a/server/modules/authenticator/gssapi_auth.c +++ b/server/modules/authenticator/gssapi_auth.c @@ -294,6 +294,7 @@ int gssapi_auth_load_users(SERV_LISTENER *listener) */ static GWAUTHENTICATOR MyObject = { + NULL, /* TODO: implement initialize entry point */ gssapi_auth_alloc, /* Allocate authenticator data */ gssapi_auth_extract, /* Extract data into structure */ gssapi_auth_connectssl, /* Check if client supports SSL */ diff --git a/server/modules/authenticator/gssapi_auth.h b/server/modules/authenticator/gssapi_auth.h index 819768f1e..4f2935c2b 100644 --- a/server/modules/authenticator/gssapi_auth.h +++ b/server/modules/authenticator/gssapi_auth.h @@ -42,7 +42,7 @@ typedef struct gssapi_auth } gssapi_auth_t; /** These functions can used for the `create` and `destroy` entry points */ -void* gssapi_auth_alloc(); +void* gssapi_auth_alloc(void *instance); void gssapi_auth_free(void *data); /** Report GSSAPI errors */ diff --git a/server/modules/authenticator/gssapi_auth_common.c b/server/modules/authenticator/gssapi_auth_common.c index a147c7186..60815288e 100644 --- a/server/modules/authenticator/gssapi_auth_common.c +++ b/server/modules/authenticator/gssapi_auth_common.c @@ -14,7 +14,7 @@ #include "gssapi_auth.h" #include -void* gssapi_auth_alloc() +void* gssapi_auth_alloc(void *instance) { gssapi_auth_t* rval = MXS_MALLOC(sizeof(gssapi_auth_t)); diff --git a/server/modules/authenticator/gssapi_backend_auth.c b/server/modules/authenticator/gssapi_backend_auth.c index a6c8e2c06..1f5d5883c 100644 --- a/server/modules/authenticator/gssapi_backend_auth.c +++ b/server/modules/authenticator/gssapi_backend_auth.c @@ -170,6 +170,7 @@ static int gssapi_backend_auth_authenticate(DCB *dcb) */ static GWAUTHENTICATOR MyObject = { + NULL, /* TODO: implement initialize entry point */ gssapi_auth_alloc, /* Allocate authenticator data */ gssapi_backend_auth_extract, /* Extract data into structure */ gssapi_backend_auth_connectssl, /* Check if client supports SSL */ diff --git a/server/modules/authenticator/http_auth.c b/server/modules/authenticator/http_auth.c index 981f7148a..c9e912a59 100644 --- a/server/modules/authenticator/http_auth.c +++ b/server/modules/authenticator/http_auth.c @@ -59,6 +59,7 @@ static void http_auth_free_client_data(DCB *dcb); */ static GWAUTHENTICATOR MyObject = { + NULL, /* No initialize entry point */ NULL, /* No create entry point */ http_auth_set_protocol_data, /* Extract data into structure */ http_auth_is_client_ssl_capable, /* Check if client supports SSL */ diff --git a/server/modules/authenticator/max_admin_auth.c b/server/modules/authenticator/max_admin_auth.c index 42f3fef03..3afc7ce97 100644 --- a/server/modules/authenticator/max_admin_auth.c +++ b/server/modules/authenticator/max_admin_auth.c @@ -59,6 +59,7 @@ static void max_admin_auth_free_client_data(DCB *dcb); */ static GWAUTHENTICATOR MyObject = { + NULL, /* No initialize entry point */ NULL, /* No create entry point */ max_admin_auth_set_protocol_data, /* Extract data into structure */ max_admin_auth_is_client_ssl_capable, /* Check if client supports SSL */ diff --git a/server/modules/authenticator/mysql_auth.c b/server/modules/authenticator/mysql_auth.c index a84e20467..aee9a65e3 100644 --- a/server/modules/authenticator/mysql_auth.c +++ b/server/modules/authenticator/mysql_auth.c @@ -62,6 +62,7 @@ static int mysql_auth_load_users(SERV_LISTENER *port); */ static GWAUTHENTICATOR MyObject = { + NULL, /* No initialize entry point */ NULL, /* No create entry point */ mysql_auth_set_protocol_data, /* Extract data into structure */ mysql_auth_is_client_ssl_capable, /* Check if client supports SSL */ diff --git a/server/modules/authenticator/mysql_backend_auth.c b/server/modules/authenticator/mysql_backend_auth.c index 0d0f20456..bde267850 100644 --- a/server/modules/authenticator/mysql_backend_auth.c +++ b/server/modules/authenticator/mysql_backend_auth.c @@ -52,7 +52,7 @@ typedef struct mysql_backend_auth * @brief Allocate a new mysql_backend_auth object * @return Allocated object or NULL if memory allocation failed */ -void* auth_backend_create() +void* auth_backend_create(void *instance) { mysql_backend_auth_t* mba = MXS_MALLOC(sizeof(*mba)); @@ -148,14 +148,6 @@ static bool auth_backend_ssl(DCB *dcb) return dcb->server->server_ssl != NULL; } -/** - * @brief Dummy function for the loadusers entry point - */ -static int auth_backend_load_users(SERV_LISTENER *port) -{ - return MXS_AUTH_LOADUSERS_OK; -} - /* @see function load_module in load_utils.c for explanation of the following * lint directives. */ @@ -176,6 +168,7 @@ static char *version_str = "V1.0.0"; */ static GWAUTHENTICATOR MyObject = { + NULL, /* No initialize entry point */ auth_backend_create, /* Create authenticator */ auth_backend_extract, /* Extract data into structure */ auth_backend_ssl, /* Check if client supports SSL */ diff --git a/server/modules/authenticator/null_auth_allow.c b/server/modules/authenticator/null_auth_allow.c index 463500f74..8d5329c18 100644 --- a/server/modules/authenticator/null_auth_allow.c +++ b/server/modules/authenticator/null_auth_allow.c @@ -58,6 +58,7 @@ static void null_auth_free_client_data(DCB *dcb); */ static GWAUTHENTICATOR MyObject = { + NULL, /* No initialize entry point */ NULL, /* No create entry point */ null_auth_set_protocol_data, /* Extract data into structure */ null_auth_is_client_ssl_capable, /* Check if client supports SSL */ diff --git a/server/modules/authenticator/null_auth_deny.c b/server/modules/authenticator/null_auth_deny.c index cdebefe49..424b625f8 100644 --- a/server/modules/authenticator/null_auth_deny.c +++ b/server/modules/authenticator/null_auth_deny.c @@ -58,6 +58,7 @@ static void null_auth_free_client_data(DCB *dcb); */ static GWAUTHENTICATOR MyObject = { + NULL, /* No initialize entry point */ NULL, /* No create entry point */ null_auth_set_protocol_data, /* Extract data into structure */ null_auth_is_client_ssl_capable, /* Check if client supports SSL */ diff --git a/server/modules/routing/binlog/blr.c b/server/modules/routing/binlog/blr.c index 273024aa1..d6dd22c2d 100644 --- a/server/modules/routing/binlog/blr.c +++ b/server/modules/routing/binlog/blr.c @@ -598,7 +598,7 @@ createInstance(SERVICE *service, char **options) { SERVER *server; SSL_LISTENER *ssl_cfg; - server = server_alloc("_none_", "MySQLBackend", (int)3306); + server = server_alloc("_none_", "MySQLBackend", 3306, "MySQLBackendAuth", NULL); if (server == NULL) { MXS_ERROR("%s: Error for server_alloc in createInstance", diff --git a/server/modules/routing/binlog/test/testbinlog.c b/server/modules/routing/binlog/test/testbinlog.c index cd4757d8e..a5bbe5897 100644 --- a/server/modules/routing/binlog/test/testbinlog.c +++ b/server/modules/routing/binlog/test/testbinlog.c @@ -105,7 +105,7 @@ int main(int argc, char **argv) { s = strtok_r(NULL, ",", &lasts); } - server = server_alloc("_none_", "MySQLBackend", (int)3306); + server = server_alloc("_none_", "MySQLBackend", 3306, "MySQLBackendAuth", NULL); if (server == NULL) { return 1; } From 4a4c65577cc0d6ab795d8a25e6a7e803c331f6c0 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 11 Oct 2016 22:46:01 +0300 Subject: [PATCH 013/215] MXS-862: Add initialize entry point to GSSAPIAuth The GSSAPIAuth module now creates an instance and stores the configured service principal name there. --- server/modules/authenticator/gssapi_auth.c | 65 ++++++++++++++++++++-- 1 file changed, 59 insertions(+), 6 deletions(-) diff --git a/server/modules/authenticator/gssapi_auth.c b/server/modules/authenticator/gssapi_auth.c index fba19dda3..81fedf769 100644 --- a/server/modules/authenticator/gssapi_auth.c +++ b/server/modules/authenticator/gssapi_auth.c @@ -17,6 +17,58 @@ #include #include "gssapi_auth.h" +typedef struct gssapi_instance +{ + char *principal_name; +}GSSAPI_INSTANCE; + +/** + * @brief Initialize the GSSAPI authenticator + * + * This function processes the service principal name that is given to the client. + * + * @param listener Listener port + * @param options Listener options + * @return Authenticator instance + */ +void* gssapi_auth_init(char **options) +{ + GSSAPI_INSTANCE *instance = MXS_MALLOC(sizeof(GSSAPI_INSTANCE)); + + if (instance) + { + instance->principal_name = NULL; + + for (int i = 0; options[i]; i++) + { + if (strstr(options[i], "principal_name")) + { + char *ptr = strchr(options[i], '='); + if (ptr) + { + ptr++; + instance->principal_name = MXS_STRDUP_A(ptr); + } + } + else + { + MXS_ERROR("Unknown option: %s", options[i]); + MXS_FREE(instance->principal_name); + MXS_FREE(instance); + return NULL; + } + } + + if (instance->principal_name == NULL) + { + instance->principal_name = MXS_STRDUP_A(default_princ_name); + MXS_NOTICE("Using default principal name: %s", instance->principal_name); + } + } + + return instance; +} + /** * @brief Create a AuthSwitchRequest packet * @@ -29,10 +81,10 @@ * @see https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchRequest * @see https://web.mit.edu/kerberos/krb5-1.5/krb5-1.5.4/doc/krb5-user/What-is-a-Kerberos-Principal_003f.html */ -static GWBUF* create_auth_change_packet() +static GWBUF* create_auth_change_packet(GSSAPI_INSTANCE *instance) { - - size_t plen = sizeof(auth_plugin_name) + 1 + sizeof(default_princ_name) - 1; + size_t principal_name_len = strlen(instance->principal_name); + size_t plen = sizeof(auth_plugin_name) + 1 + principal_name_len; GWBUF *buffer = gwbuf_alloc(plen + MYSQL_HEADER_LEN); if (buffer) @@ -44,7 +96,7 @@ static GWBUF* create_auth_change_packet() *data++ = 0xfe; // AuthSwitchRequest command memcpy(data, auth_plugin_name, sizeof(auth_plugin_name)); // Plugin name data += sizeof(auth_plugin_name); - memcpy(data, default_princ_name, sizeof(default_princ_name) - 1); // Plugin data + memcpy(data, instance->principal_name, principal_name_len); // Plugin data } return buffer; @@ -232,13 +284,14 @@ int gssapi_auth_authenticate(DCB *dcb) { int rval = MXS_AUTH_FAILED; gssapi_auth_t *auth = (gssapi_auth_t*)dcb->authenticator_data; + GSSAPI_INSTANCE *instance = (GSSAPI_INSTANCE*)dcb->listener->auth_instance; if (auth->state == GSSAPI_AUTH_INIT) { /** We need to send the authentication switch packet to change the * authentication to something other than the 'mysql_native_password' * method */ - GWBUF *buffer = create_auth_change_packet(); + GWBUF *buffer = create_auth_change_packet(instance); if (buffer && dcb->func.write(dcb, buffer)) { @@ -294,7 +347,7 @@ int gssapi_auth_load_users(SERV_LISTENER *listener) */ static GWAUTHENTICATOR MyObject = { - NULL, /* TODO: implement initialize entry point */ + gssapi_auth_init, /* Initialize authenticator */ gssapi_auth_alloc, /* Allocate authenticator data */ gssapi_auth_extract, /* Extract data into structure */ gssapi_auth_connectssl, /* Check if client supports SSL */ From db4b6793c4410714f05da60e909987201a53a85e Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Sun, 9 Oct 2016 11:48:59 +0300 Subject: [PATCH 014/215] MXS-862: Add authenticator documentation The authentication process in MaxScale is now described in Authentication-Modules.md. The GSSAPI authenticator also has its own document. --- Documentation/About/Limitations.md | 8 ++ .../Authenticators/Authentication-Modules.md | 106 ++++++++++++++++++ .../Authenticators/GSSAPI-Authenticator.md | 48 ++++++++ .../Getting-Started/Configuration-Guide.md | 18 +++ 4 files changed, 180 insertions(+) create mode 100644 Documentation/Authenticators/Authentication-Modules.md create mode 100644 Documentation/Authenticators/GSSAPI-Authenticator.md diff --git a/Documentation/About/Limitations.md b/Documentation/About/Limitations.md index 87f75101f..e7cf9af87 100644 --- a/Documentation/About/Limitations.md +++ b/Documentation/About/Limitations.md @@ -32,6 +32,14 @@ Issues [MXS-710](https://jira.mariadb.org/browse/MXS-710) and Compression is not included in MySQL server handshake +# Authenticator limitations + +## Limitations in the GSSAPI authentication + +Currently MaxScale only supports GSSAPI authentication when the backend +connections use GSSAPI authentication. Client side GSSAPI authentication with a +different backend authentication module is not supported. + # Monitor limitations A server can only be monitored by one monitor. If multiple monitors monitor the diff --git a/Documentation/Authenticators/Authentication-Modules.md b/Documentation/Authenticators/Authentication-Modules.md new file mode 100644 index 000000000..291e9030c --- /dev/null +++ b/Documentation/Authenticators/Authentication-Modules.md @@ -0,0 +1,106 @@ +# Authentication Modules in MaxScale + +This document describes the modular authentication in MaxScale. It contains +protocol specific information on authentication and how it is handled in +MaxScale. + +The constants described in this document are defined in the gw_authenticator.h +header unless otherwise mentioned. + +## Authenticator initialization + +When the authentication module is first loaded, the `initialize` entry point is +called. The return value of this function will be passed as the first argument +to the other entry points. + +The `loadUsers` entry point of the client side authenticator is called when a +service starts. The authenticator can load external user data when this entry +point is called. This entry point is also called when user authentication has +failed and the external user data needs to be refreshed. + +When a connection is created, the `create` entry point is called to create per +connection data. The return value of this function is stored in the +`dcb->authenticator_data` field of the DCB object. This data is freed in the +`destroy` entry point and the value returned by `create` will be given as the +first parameter. + +# MySQL Authentication Moduels + +The MySQL protocol authentication starts when the server sends the +[handshake packet](https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::Handshake) +to the client to which the client responds with a [handshake response packet](https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::HandshakeResponse). +If the server is using the default _mysql_native_password_ authentication plugin, the server responds with either an +[OK packet](https://dev.mysql.com/doc/internals/en/packet-OK_Packet.html) or an +[ERR packet](https://dev.mysql.com/doc/internals/en/packet-ERR_Packet.html) and +the authentication is complete. + +If a different authentication plugin is required to complete the authentication, instead of +sending an OK or ERR packet, the server responds with an +[AuthSwitchRequest packet](https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchRequest). +This is where the pluggable authentication in MaxScale starts. + +## Client authentication in MaxScale + +The first packet the client side authenticator plugins will receive is the +client's handshake response packet. + +The client protocol module will call the `extract` entry point of the +authenticator where the authenticator should extract client information. If the +`extract` entry point returns one of the following constants, the `authenticate` +entry point will be called. + +- MXS_AUTH_SUCCEEDED +- MXS_AUTH_INCOMPLETE +- MXS_AUTH_SSL_INCOMPLETE + +The `authenticate` entry point is where the authenticator plugin should +authenticate the client. If authentication is successful, the `authenticate` +entry point should return MXS_AUTH_SUCCEEDED. If authentication is not yet +complete or if the authentication module should be changed, the `authenticate` +entry point should return MXS_AUTH_INCOMPLETE. + +Authenticator plugins which do not use the default _mysql_native_password_ +authentication plugin should send an AuthSwitchRequest packet to the client and +return MXS_AUTH_INCOMPLETE. When more data is available, the `extract` and +`authenticate` entry points will be called again. + +If either of the aforementioned entry points returns one of the following +constants, the authentication is considered to have failed and the session will +be closed. + +- MXS_AUTH_FAILED +- MXS_AUTH_FAILED_DB +- MXS_AUTH_FAILED_SSL + +Read the individual authenticator module documentation for more details on the +authentication process of each authentication plugin. + +## Backend authentication in MaxScale + +The first packet the authentication plugins in MaxScale will receive is either +the AuthSwitchRequest packet or, in case of _mysql_native_password_, the OK +packet. At this point, the protocol plugin will call the `extract` entry point +of the backend authenticator. If the return value of the call is one of the +following constants, the protocol plugin will call the `authenticate` entry +point of the authenticator. + +- MXS_AUTH_SUCCEEDED +- MXS_AUTH_INCOMPLETE +- MXS_AUTH_SSL_INCOMPLETE + +If the `authenticate` entry point returns MXS_AUTH_SUCCEEDED, then +authentication is complete and any queued queries from the clients will be sent +to the backend server. If the return value is MXS_AUTH_INCOMPLETE or +MXS_AUTH_SSL_INCOMPLETE, the protocol module will continue the authentication by +calling the `extract` entry point once more data is available. + +If either of the aforementioned entry points returns one of the following +constants, the authentication is considered to have failed and the session will +be closed. + +- MXS_AUTH_FAILED +- MXS_AUTH_FAILED_DB +- MXS_AUTH_FAILED_SSL + +Read the individual authenticator module documentation for more details on the +authentication process of each authentication plugin. diff --git a/Documentation/Authenticators/GSSAPI-Authenticator.md b/Documentation/Authenticators/GSSAPI-Authenticator.md new file mode 100644 index 000000000..2d6e82f84 --- /dev/null +++ b/Documentation/Authenticators/GSSAPI-Authenticator.md @@ -0,0 +1,48 @@ +# GSSAPI Client Authenticator + +GSSAPI is an authentication protocol that is commonly implemented with +Kerberos on Unix or Active Directory on Windows. This document describes +the GSSAPI authentication in MaxScale. + +The _GSSAPIAuth_ module implements the client side authentication and the +_GSSAPIBackendAuth_ module implements the backend authentication. + +## Authenticator options + +The client side GSSAPIAuth authenticator supports one option, the service +principal name that MaxScale sends to the client. The backend authenticator +module has no options. + +### `principal_name` + +The service principal name to send to the client. This parameter is a +string parameter which is used by the client to request the token. + +The default value for this option is _mariadb/localhost.localdomain_. + +The parameter must be a valid GSSAPI principal name +e.g. `styx/pluto@EXAMPLE.COM`. The principal name can also be defined +without the realm part in which case the default realm will be used. + +## Implementation details + +Read the [Authentication Modules](Authentication-Modules.md) document for more +details on how authentication modules work in MaxScale. + +### GSSAPI authentication + +The GSSAPI plugin authentication starts when the database server sends the +service principal name in the AuthSwitchRequest packet. The principal name will +usually be in the form `service@REALM.COM`. + +The client will then request a token for this service from the GSSAPI server and +send the token to the database server. The database server will verify the +authenticity of the token by contacting the GSSAPI server and if the token is +authentic, the server sends the final OK packet. + +## Limitations + +Client side GSSAPI authentication is only supported when the backend +connections use GSSAPI authentication. + +See the [Limitations](../About/Limitations.md) document for more details. diff --git a/Documentation/Getting-Started/Configuration-Guide.md b/Documentation/Getting-Started/Configuration-Guide.md index cd8b50456..ea7fc03a4 100644 --- a/Documentation/Getting-Started/Configuration-Guide.md +++ b/Documentation/Getting-Started/Configuration-Guide.md @@ -435,6 +435,12 @@ router_options=master,slave A more complete description of router options and what is available for a given router is included with the documentation of the router itself. +#### `router_options` + +Option string given to the router module. The value of this parameter +should be a comma-separated list of key-value pairs. See router specific +documentation for more details. + #### `filters` The filters option allow a set of filters to be defined for a service; requests from the client are passed through these filters before being sent to the router for dispatch to the backend server. The filters parameter takes one or more filter names, as defined within the filter definition section of the configuration file. Multiple filters are separated using the | character. @@ -833,6 +839,18 @@ The `socket` option may be included in a listener definition, this configures th If a socket option and an address option is given then the listener will listen on both the specific IP address and the Unix socket. +#### `authenticator` + +The authenticator module to use. Each protocol module defines a default +authentication module which is used if no `authenticator` parameter is +found from the configuration. + +#### `authenticator_options` + +Option string given to the authenticator module. The value of this +parameter should be a comma-separated list of key-value pairs. See +authenticator specific documentation for more details. + #### Available Protocols The protocols supported by MariaDB MaxScale are implemented as external modules that are loaded dynamically into the MariaDB MaxScale core. They allow MariaDB MaxScale to communicate in various protocols both on the client side and the backend side. Each of the protocols can be either a client protocol or a backend protocol. Client protocols are used for client-MariaDB MaxScale communication and backend protocols are for MariaDB MaxScale-database communication. From ef5bbd935335ec9425758f3c929117c51fbd2d72 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Thu, 13 Oct 2016 13:21:36 +0300 Subject: [PATCH 015/215] Remove trailing NULLs from throttling message --- server/core/log_manager.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/core/log_manager.cc b/server/core/log_manager.cc index bd4eeb58c..ddcbcd1e5 100644 --- a/server/core/log_manager.cc +++ b/server/core/log_manager.cc @@ -2942,7 +2942,7 @@ int mxs_log_message(int priority, if (status == MESSAGE_SUPPRESSED) { suppression_len += sizeof(SUPPRESSION) - 1; // Remove trailing NULL - suppression_len += 3; // Remove the %lu + suppression_len -= 3; // Remove the %lu suppression_len += UINTLEN(suppress_ms); } From e41589be104dc33e56efbe9dc79633b2bc387039 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Thu, 13 Oct 2016 15:20:51 +0300 Subject: [PATCH 016/215] Move headers from server/include to include/maxscale - Headers now to be included as - First step, no cleanup of headers has been made. Only moving from one place to another + necessary modifications. --- CMakeLists.txt | 12 +++--- avro/maxavro.c | 2 +- avro/maxavro.h | 2 +- avro/maxavro_file.c | 2 +- avro/maxavro_record.c | 6 +-- avro/maxavro_schema.c | 4 +- avro/maxavro_write.c | 2 +- avro/maxavrocheck.c | 2 +- client/maxadmin.c | 2 +- .../maxscale}/adminusers.h.in | 2 +- {server/include => include}/maxscale/alloc.h | 2 +- {server/include => include/maxscale}/atomic.h | 0 {server/include => include/maxscale}/buffer.h | 6 +-- .../include => include/maxscale}/dbusers.h | 2 +- {server/include => include/maxscale}/dcb.h | 18 ++++----- .../maxscale}/def_monitor_event.h | 0 .../include => include/maxscale}/externcmd.h | 6 +-- {server/include => include/maxscale}/filter.h | 6 +-- {server/include => include/maxscale}/gw.h | 2 +- .../maxscale}/gw_authenticator.h | 2 +- .../maxscale}/gw_protocol.h | 2 +- {server/include => include/maxscale}/gw_ssl.h | 2 +- .../include => include/maxscale}/gwbitmask.h | 2 +- .../include => include/maxscale}/gwdirs.h.in | 2 +- .../include => include/maxscale}/hashtable.h | 4 +- {server/include => include/maxscale}/hint.h | 2 +- .../maxscale}/hk_heartbeat.h | 0 .../maxscale}/housekeeper.h | 4 +- {server/include => include}/maxscale/limits.h | 0 .../include => include/maxscale}/listener.h | 6 +-- .../maxscale}/listmanager.h | 4 +- .../maxscale}/log_manager.h | 0 .../include => include/maxscale}/maxconfig.h | 4 +- .../include => include/maxscale}/maxscale.h | 0 .../maxscale}/maxscale_pcre2.h | 0 {server/include => include/maxscale}/memlog.h | 2 +- {server/include => include/maxscale}/mlist.h | 2 +- .../include => include/maxscale}/modinfo.h | 0 .../include => include/maxscale}/modules.h | 8 ++-- .../include => include/maxscale}/modutil.h | 6 +-- .../include => include/maxscale}/monitor.h | 14 +++---- .../maxscale}/mysql_binlog.h | 0 .../maxscale}/mysql_utils.h | 2 +- .../maxscale}/notification.h | 0 .../include => include/maxscale}/platform.h | 0 {server/include => include}/maxscale/poll.h | 6 +-- .../maxscale}/query_classifier.h | 4 +- .../maxscale}/queuemanager.h | 2 +- .../maxscale}/random_jkiss.h | 0 {server/include => include/maxscale}/rdtsc.h | 0 .../include => include/maxscale}/resultset.h | 2 +- {server/include => include/maxscale}/router.h | 6 +-- .../include => include/maxscale}/secrets.h | 0 {server/include => include/maxscale}/server.h | 4 +- .../include => include/maxscale}/service.h | 20 +++++----- .../include => include/maxscale}/session.h | 14 +++---- .../maxscale}/skygw_debug.h | 2 +- .../maxscale}/skygw_types.h | 0 .../maxscale}/skygw_utils.h | 4 +- {server/include => include/maxscale}/slist.h | 2 +- .../include => include/maxscale}/spinlock.h | 2 +- .../include => include/maxscale}/statistics.h | 0 .../include => include/maxscale}/test_utils.h | 10 ++--- {server/include => include/maxscale}/thread.h | 0 {server/include => include/maxscale}/users.h | 6 +-- {server/include => include/maxscale}/utils.h | 0 .../include => include/maxscale}/version.h.in | 0 query_classifier/qc_dummy/qc_dummy.cc | 4 +- .../qc_mysqlembedded/qc_mysqlembedded.cc | 10 ++--- .../qc_sqlite/builtin_functions.c | 2 +- query_classifier/qc_sqlite/qc_sqlite.c | 12 +++--- .../test/canonical_tests/canonizer.c | 6 +-- query_classifier/test/classify.c | 8 ++-- query_classifier/test/compare.cc | 6 +-- server/core/adminusers.c | 10 ++--- server/core/alloc.c | 2 +- server/core/authenticator.c | 6 +-- server/core/buffer.c | 16 ++++---- server/core/config.c | 28 ++++++------- server/core/dbusers.c | 16 ++++---- server/core/dcb.c | 30 +++++++------- server/core/externcmd.c | 2 +- server/core/filter.c | 12 +++--- server/core/gateway.c | 36 ++++++++--------- server/core/gw_ssl.c | 6 +-- server/core/gw_utils.c | 10 ++--- server/core/gwbitmask.c | 2 +- server/core/gwdirs.c | 4 +- server/core/hashtable.c | 4 +- server/core/hint.c | 2 +- server/core/housekeeper.c | 8 ++-- server/core/listener.c | 12 +++--- server/core/listmanager.c | 8 ++-- server/core/load_utils.c | 16 ++++---- server/core/log_manager.cc | 16 ++++---- server/core/maxkeys.c | 8 ++-- server/core/maxpasswd.c | 8 ++-- server/core/maxscale_pcre2.c | 2 +- server/core/memlog.c | 4 +- server/core/misc.c | 2 +- server/core/mlist.c | 2 +- server/core/modutil.c | 4 +- server/core/monitor.c | 20 +++++----- server/core/mysql_binlog.c | 12 +++--- server/core/mysql_utils.c | 8 ++-- server/core/poll.c | 26 ++++++------ server/core/query_classifier.c | 8 ++-- server/core/queuemanager.c | 10 ++--- server/core/random_jkiss.c | 4 +- server/core/resultset.c | 6 +-- server/core/secrets.c | 12 +++--- server/core/server.c | 16 ++++---- server/core/service.c | 40 +++++++++---------- server/core/session.c | 20 +++++----- server/core/skygw_utils.cc | 10 ++--- server/core/slist.c | 4 +- server/core/spinlock.c | 6 +-- server/core/statistics.c | 6 +-- server/core/test/test_mysql_users.c | 16 ++++---- server/core/test/testadminusers.c | 4 +- server/core/test/testbuffer.c | 4 +- server/core/test/testdcb.c | 4 +- server/core/test/testfeedback.c | 10 ++--- server/core/test/testfilter.c | 2 +- server/core/test/testgwbitmask.c | 4 +- server/core/test/testhash.c | 4 +- server/core/test/testhint.c | 2 +- server/core/test/testlog.c | 4 +- server/core/test/testlogorder.c | 4 +- server/core/test/testlogthrottling.cc | 2 +- server/core/test/testmaxscalepcre2.c | 4 +- server/core/test/testmemlog.c | 2 +- server/core/test/testmodutil.c | 4 +- server/core/test/testpoll.c | 6 +-- server/core/test/testqueuemanager.c | 8 ++-- server/core/test/testserver.c | 4 +- server/core/test/testservice.c | 8 ++-- server/core/test/testspinlock.c | 4 +- server/core/test/testusers.c | 4 +- server/core/thread.c | 2 +- server/core/users.c | 6 +-- server/core/utils.c | 14 +++---- server/modules/authenticator/cdc_plain_auth.c | 6 +-- server/modules/authenticator/gssapi_auth.c | 4 +- .../authenticator/gssapi_backend_auth.c | 4 +- server/modules/authenticator/http_auth.c | 14 +++---- server/modules/authenticator/max_admin_auth.c | 12 +++--- server/modules/authenticator/mysql_auth.c | 12 +++--- .../authenticator/mysql_backend_auth.c | 4 +- .../modules/authenticator/null_auth_allow.c | 10 ++--- server/modules/authenticator/null_auth_deny.c | 10 ++--- server/modules/filter/cache/cache.c | 14 +++---- .../modules/filter/cache/cache_storage_api.h | 4 +- server/modules/filter/cache/rules.c | 6 +-- server/modules/filter/cache/rules.h | 6 +-- server/modules/filter/cache/storage.c | 4 +- .../storage/storage_rocksdb/rocksdbstorage.cc | 6 +-- .../storage/storage_rocksdb/storage_rocksdb.h | 2 +- server/modules/filter/cache/test/testrules.c | 4 +- server/modules/filter/ccrfilter/ccrfilter.c | 14 +++---- server/modules/filter/dbfwfilter/dbfwfilter.c | 18 ++++----- server/modules/filter/dbfwfilter/ruleparser.y | 4 +- server/modules/filter/gatekeeper/gatekeeper.c | 14 +++---- server/modules/filter/hintfilter/hintfilter.c | 6 +-- server/modules/filter/hintfilter/hintparser.c | 10 ++--- server/modules/filter/luafilter/luafilter.c | 16 ++++---- server/modules/filter/mqfilter/mqfilter.c | 18 ++++----- .../namedserverfilter/namedserverfilter.c | 12 +++--- server/modules/filter/qlafilter/qlafilter.c | 12 +++--- .../modules/filter/regexfilter/regexfilter.c | 14 +++---- server/modules/filter/tee/tee.c | 20 +++++----- server/modules/filter/testfilter/testfilter.c | 8 ++-- server/modules/filter/topfilter/topfilter.c | 12 +++--- server/modules/filter/tpmfilter/tpmfilter.c | 12 +++--- server/modules/include/avrorouter.h | 16 ++++---- server/modules/include/blr.h | 8 ++-- server/modules/include/cdc.h | 16 ++++---- server/modules/include/debugcli.h | 6 +-- server/modules/include/httpd.h | 14 +++---- server/modules/include/maxinfo.h | 6 +-- server/modules/include/maxscaled.h | 6 +-- server/modules/include/mysql_auth.h | 4 +- .../include/mysql_client_server_protocol.h | 14 +++---- server/modules/include/mysqlhint.h | 2 +- server/modules/include/readconnection.h | 2 +- server/modules/include/readwritesplit.h | 4 +- server/modules/include/rwsplit_internal.h | 2 +- server/modules/include/schemarouter.h | 4 +- server/modules/include/sharding_common.h | 10 ++--- server/modules/include/telnetd.h | 4 +- server/modules/monitor/auroramon/auroramon.c | 8 ++-- server/modules/monitor/galeramon/galeramon.c | 2 +- server/modules/monitor/galeramon/galeramon.h | 20 +++++----- server/modules/monitor/mmmon/mmmon.c | 2 +- server/modules/monitor/mmmon/mmmon.h | 20 +++++----- server/modules/monitor/mysqlmon.h | 22 +++++----- server/modules/monitor/mysqlmon/mysql_mon.c | 4 +- server/modules/protocol/CDC/cdc.c | 10 ++--- server/modules/protocol/HTTPD/httpd.c | 10 ++--- .../MySQL/MySQLBackend/mysql_backend.c | 18 ++++----- .../protocol/MySQL/MySQLClient/mysql_client.c | 16 ++++---- server/modules/protocol/MySQL/mysql_common.c | 12 +++--- server/modules/protocol/maxscaled/maxscaled.c | 24 +++++------ server/modules/protocol/telnetd/telnetd.c | 24 +++++------ .../protocol/testprotocol/testprotocol.c | 8 ++-- server/modules/routing/avro/avro.c | 24 +++++------ server/modules/routing/avro/avro_client.c | 24 +++++------ server/modules/routing/avro/avro_file.c | 4 +- server/modules/routing/avro/avro_index.c | 2 +- server/modules/routing/avro/avro_rbr.c | 2 +- server/modules/routing/avro/avro_schema.c | 8 ++-- server/modules/routing/binlog/binlog_common.c | 2 +- server/modules/routing/binlog/blr.c | 28 ++++++------- server/modules/routing/binlog/blr_cache.c | 20 +++++----- server/modules/routing/binlog/blr_file.c | 22 +++++----- server/modules/routing/binlog/blr_master.c | 28 ++++++------- server/modules/routing/binlog/blr_slave.c | 28 ++++++------- .../modules/routing/binlog/maxbinlogcheck.c | 2 +- .../modules/routing/binlog/test/testbinlog.c | 26 ++++++------ server/modules/routing/cli/cli.c | 20 +++++----- server/modules/routing/debugcli/debugcli.c | 20 +++++----- server/modules/routing/debugcli/debugcmd.c | 38 +++++++++--------- server/modules/routing/maxinfo/maxinfo.c | 40 +++++++++---------- .../modules/routing/maxinfo/maxinfo_error.c | 22 +++++----- server/modules/routing/maxinfo/maxinfo_exec.c | 32 +++++++-------- .../modules/routing/maxinfo/maxinfo_parse.c | 22 +++++----- .../routing/readconnroute/readconnroute.c | 24 +++++------ .../routing/readwritesplit/readwritesplit.c | 16 ++++---- .../routing/readwritesplit/rwsplit_mysql.c | 16 ++++---- .../readwritesplit/rwsplit_route_stmt.c | 2 +- .../readwritesplit/rwsplit_select_backends.c | 2 +- .../readwritesplit/rwsplit_session_cmd.c | 2 +- .../readwritesplit/rwsplit_tmp_table_multi.c | 4 +- .../routing/schemarouter/schemarouter.c | 18 ++++----- server/modules/routing/testroute/testroute.c | 4 +- 235 files changed, 1010 insertions(+), 1010 deletions(-) rename {server/include => include/maxscale}/adminusers.h.in (98%) rename {server/include => include}/maxscale/alloc.h (98%) rename {server/include => include/maxscale}/atomic.h (100%) rename {server/include => include/maxscale}/buffer.h (98%) rename {server/include => include/maxscale}/dbusers.h (98%) rename {server/include => include/maxscale}/dcb.h (98%) rename {server/include => include/maxscale}/def_monitor_event.h (100%) rename {server/include => include/maxscale}/externcmd.h (91%) rename {server/include => include/maxscale}/filter.h (98%) rename {server/include => include/maxscale}/gw.h (98%) rename {server/include => include/maxscale}/gw_authenticator.h (99%) rename {server/include => include/maxscale}/gw_protocol.h (98%) rename {server/include => include/maxscale}/gw_ssl.h (98%) rename {server/include => include/maxscale}/gwbitmask.h (98%) rename {server/include => include/maxscale}/gwdirs.h.in (98%) rename {server/include => include/maxscale}/hashtable.h (98%) rename {server/include => include/maxscale}/hint.h (98%) rename {server/include => include/maxscale}/hk_heartbeat.h (100%) rename {server/include => include/maxscale}/housekeeper.h (96%) rename {server/include => include}/maxscale/limits.h (100%) rename {server/include => include/maxscale}/listener.h (96%) rename {server/include => include/maxscale}/listmanager.h (98%) rename {server/include => include/maxscale}/log_manager.h (100%) rename {server/include => include/maxscale}/maxconfig.h (99%) rename {server/include => include/maxscale}/maxscale.h (100%) rename {server/include => include/maxscale}/maxscale_pcre2.h (100%) rename {server/include => include/maxscale}/memlog.h (97%) rename {server/include => include/maxscale}/mlist.h (98%) rename {server/include => include/maxscale}/modinfo.h (100%) rename {server/include => include/maxscale}/modules.h (95%) rename {server/include => include/maxscale}/modutil.h (97%) rename {server/include => include/maxscale}/monitor.h (97%) rename {server/include => include/maxscale}/mysql_binlog.h (100%) rename {server/include => include/maxscale}/mysql_utils.h (96%) rename {server/include => include/maxscale}/notification.h (100%) rename {server/include => include/maxscale}/platform.h (100%) rename {server/include => include}/maxscale/poll.h (95%) rename {server/include => include/maxscale}/query_classifier.h (98%) rename {server/include => include/maxscale}/queuemanager.h (98%) rename {server/include => include/maxscale}/random_jkiss.h (100%) rename {server/include => include/maxscale}/rdtsc.h (100%) rename {server/include => include/maxscale}/resultset.h (98%) rename {server/include => include/maxscale}/router.h (97%) rename {server/include => include/maxscale}/secrets.h (100%) rename {server/include => include/maxscale}/server.h (99%) rename {server/include => include/maxscale}/service.h (96%) rename {server/include => include/maxscale}/session.h (96%) rename {server/include => include/maxscale}/skygw_debug.h (99%) rename {server/include => include/maxscale}/skygw_types.h (100%) rename {server/include => include/maxscale}/skygw_utils.h (98%) rename {server/include => include/maxscale}/slist.h (98%) rename {server/include => include/maxscale}/spinlock.h (98%) rename {server/include => include/maxscale}/statistics.h (100%) rename {server/include => include/maxscale}/test_utils.h (80%) rename {server/include => include/maxscale}/thread.h (100%) rename {server/include => include/maxscale}/users.h (96%) rename {server/include => include/maxscale}/utils.h (100%) rename {server/include => include/maxscale}/version.h.in (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5ae651b70..745459863 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -113,10 +113,10 @@ if(${MAXSCALE_VERSION} MATCHES "-stable") endif() file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/server/include) -configure_file(${CMAKE_SOURCE_DIR}/server/include/version.h.in ${CMAKE_BINARY_DIR}/server/include/version.h @ONLY) -configure_file(${CMAKE_SOURCE_DIR}/server/include/gwdirs.h.in ${CMAKE_BINARY_DIR}/server/include/gwdirs.h @ONLY) -configure_file(${CMAKE_SOURCE_DIR}/server/include/adminusers.h.in ${CMAKE_BINARY_DIR}/server/include/adminusers.h @ONLY) -configure_file(${CMAKE_SOURCE_DIR}/server/test/maxscale_test.h.in ${CMAKE_BINARY_DIR}/server/include/maxscale_test.h @ONLY) +configure_file(${CMAKE_SOURCE_DIR}/include/maxscale/version.h.in ${CMAKE_BINARY_DIR}/include/maxscale/version.h @ONLY) +configure_file(${CMAKE_SOURCE_DIR}/include/maxscale/gwdirs.h.in ${CMAKE_BINARY_DIR}/include/maxscale/gwdirs.h @ONLY) +configure_file(${CMAKE_SOURCE_DIR}/include/maxscale/adminusers.h.in ${CMAKE_BINARY_DIR}/include/maxscale/adminusers.h @ONLY) +configure_file(${CMAKE_SOURCE_DIR}/server/test/maxscale_test.h.in ${CMAKE_BINARY_DIR}/include/maxscale/maxscale_test.h @ONLY) configure_file(${CMAKE_SOURCE_DIR}/etc/postinst.in ${CMAKE_BINARY_DIR}/postinst @ONLY) configure_file(${CMAKE_SOURCE_DIR}/etc/postrm.in ${CMAKE_BINARY_DIR}/postrm @ONLY) configure_file(${CMAKE_SOURCE_DIR}/server/test/maxscale_test.cnf ${CMAKE_BINARY_DIR}/maxscale.cnf @ONLY) @@ -177,10 +177,10 @@ set(CMAKE_CXX_FLAGS_RELEASE "") set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-ggdb") include_directories(avro) -include_directories(server/include) +include_directories(include) include_directories(server/inih) include_directories(server/modules/include) -include_directories(${CMAKE_BINARY_DIR}/server/include) +include_directories(${CMAKE_BINARY_DIR}/include) include_directories(${CURL_INCLUDE_DIRS}) if (BUILD_AVRO) diff --git a/avro/maxavro.c b/avro/maxavro.c index 251257fbb..5809d4c18 100644 --- a/avro/maxavro.c +++ b/avro/maxavro.c @@ -15,7 +15,7 @@ #include #include #include "maxavro.h" -#include +#include #include /** Maximum byte size of an integer value */ diff --git a/avro/maxavro.h b/avro/maxavro.h index 715eb1ee5..4d337835f 100644 --- a/avro/maxavro.h +++ b/avro/maxavro.h @@ -18,7 +18,7 @@ #include #include #include -#include +#include /** File magic and sync marker sizes block sizes */ #define AVRO_MAGIC_SIZE 4 diff --git a/avro/maxavro_file.c b/avro/maxavro_file.c index 457664333..efd91e2cf 100644 --- a/avro/maxavro_file.c +++ b/avro/maxavro_file.c @@ -14,7 +14,7 @@ #include "maxavro.h" #include #include -#include +#include static bool maxavro_read_sync(FILE *file, uint8_t* sync) diff --git a/avro/maxavro_record.c b/avro/maxavro_record.c index a5e0e06f9..961bc2add 100644 --- a/avro/maxavro_record.c +++ b/avro/maxavro_record.c @@ -12,10 +12,10 @@ */ #include "maxavro.h" -#include "skygw_utils.h" +#include #include -#include -#include +#include +#include #include bool maxavro_read_datablock_start(MAXAVRO_FILE *file); diff --git a/avro/maxavro_schema.c b/avro/maxavro_schema.c index 56c877b9b..e133b03c5 100644 --- a/avro/maxavro_schema.c +++ b/avro/maxavro_schema.c @@ -14,8 +14,8 @@ #include "maxavro.h" #include #include -#include -#include +#include +#include static const MAXAVRO_SCHEMA_FIELD types[MAXAVRO_TYPE_MAX] = { diff --git a/avro/maxavro_write.c b/avro/maxavro_write.c index 389fb06d7..cf4df1ce9 100644 --- a/avro/maxavro_write.c +++ b/avro/maxavro_write.c @@ -15,7 +15,7 @@ #include #include #include "maxavro.h" -#include +#include #include /** diff --git a/avro/maxavrocheck.c b/avro/maxavrocheck.c index b3c639720..23655be56 100644 --- a/avro/maxavrocheck.c +++ b/avro/maxavrocheck.c @@ -23,7 +23,7 @@ #include #include #include -#include +#include static int verbose = 0; static uint64_t seekto = 0; diff --git a/client/maxadmin.c b/client/maxadmin.c index 2079b62bf..32b1a6b3f 100644 --- a/client/maxadmin.c +++ b/client/maxadmin.c @@ -48,7 +48,7 @@ #include #include -#include +#include #ifdef HISTORY #include diff --git a/server/include/adminusers.h.in b/include/maxscale/adminusers.h.in similarity index 98% rename from server/include/adminusers.h.in rename to include/maxscale/adminusers.h.in index 6817b8e8a..25e6acbe2 100644 --- a/server/include/adminusers.h.in +++ b/include/maxscale/adminusers.h.in @@ -24,7 +24,7 @@ * * @endverbatim */ -#include +#include #define ADMIN_SALT "$1$MXS" diff --git a/server/include/maxscale/alloc.h b/include/maxscale/alloc.h similarity index 98% rename from server/include/maxscale/alloc.h rename to include/maxscale/alloc.h index 5a87268ef..e138d3af3 100644 --- a/server/include/maxscale/alloc.h +++ b/include/maxscale/alloc.h @@ -15,7 +15,7 @@ #include #include -#include +#include EXTERN_C_BLOCK_BEGIN diff --git a/server/include/atomic.h b/include/maxscale/atomic.h similarity index 100% rename from server/include/atomic.h rename to include/maxscale/atomic.h diff --git a/server/include/buffer.h b/include/maxscale/buffer.h similarity index 98% rename from server/include/buffer.h rename to include/maxscale/buffer.h index f3ae40134..935652671 100644 --- a/server/include/buffer.h +++ b/include/maxscale/buffer.h @@ -43,9 +43,9 @@ * @endverbatim */ #include -#include -#include -#include +#include +#include +#include #include EXTERN_C_BLOCK_BEGIN diff --git a/server/include/dbusers.h b/include/maxscale/dbusers.h similarity index 98% rename from server/include/dbusers.h rename to include/maxscale/dbusers.h index 7062c20af..ea669b4df 100644 --- a/server/include/dbusers.h +++ b/include/maxscale/dbusers.h @@ -13,7 +13,7 @@ * Public License. */ -#include +#include #include diff --git a/server/include/dcb.h b/include/maxscale/dcb.h similarity index 98% rename from server/include/dcb.h rename to include/maxscale/dcb.h index 679ebb4cf..3a6020003 100644 --- a/server/include/dcb.h +++ b/include/maxscale/dcb.h @@ -12,15 +12,15 @@ * of this software will be governed by version 2 or later of the General * Public License. */ -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #define ERRHANDLE diff --git a/server/include/def_monitor_event.h b/include/maxscale/def_monitor_event.h similarity index 100% rename from server/include/def_monitor_event.h rename to include/maxscale/def_monitor_event.h diff --git a/server/include/externcmd.h b/include/maxscale/externcmd.h similarity index 91% rename from server/include/externcmd.h rename to include/maxscale/externcmd.h index f6e2d1568..ab4ed2460 100644 --- a/server/include/externcmd.h +++ b/include/maxscale/externcmd.h @@ -16,9 +16,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include #define MAXSCALE_EXTCMD_ARG_MAX 256 diff --git a/server/include/filter.h b/include/maxscale/filter.h similarity index 98% rename from server/include/filter.h rename to include/maxscale/filter.h index fc6f7f257..3ccb8ac4c 100644 --- a/server/include/filter.h +++ b/include/maxscale/filter.h @@ -22,9 +22,9 @@ * 27/05/2014 Mark Riddoch Initial implementation * */ -#include -#include -#include +#include +#include +#include #include /** diff --git a/server/include/gw.h b/include/maxscale/gw.h similarity index 98% rename from server/include/gw.h rename to include/maxscale/gw.h index 616327563..17b011a28 100644 --- a/server/include/gw.h +++ b/include/maxscale/gw.h @@ -33,7 +33,7 @@ #include #include #include -#include +#include #define EXIT_FAILURE 1 diff --git a/server/include/gw_authenticator.h b/include/maxscale/gw_authenticator.h similarity index 99% rename from server/include/gw_authenticator.h rename to include/maxscale/gw_authenticator.h index 29a06ae40..5e999fb02 100644 --- a/server/include/gw_authenticator.h +++ b/include/maxscale/gw_authenticator.h @@ -27,7 +27,7 @@ * @endverbatim */ -#include +#include #include #include #include diff --git a/server/include/gw_protocol.h b/include/maxscale/gw_protocol.h similarity index 98% rename from server/include/gw_protocol.h rename to include/maxscale/gw_protocol.h index 7022d64a8..e2a3295da 100644 --- a/server/include/gw_protocol.h +++ b/include/maxscale/gw_protocol.h @@ -28,7 +28,7 @@ * @endverbatim */ -#include +#include struct dcb; struct server; diff --git a/server/include/gw_ssl.h b/include/maxscale/gw_ssl.h similarity index 98% rename from server/include/gw_ssl.h rename to include/maxscale/gw_ssl.h index 196f20a45..bfcc387cf 100644 --- a/server/include/gw_ssl.h +++ b/include/maxscale/gw_ssl.h @@ -27,7 +27,7 @@ * @endverbatim */ -#include +#include #include #include #include diff --git a/server/include/gwbitmask.h b/include/maxscale/gwbitmask.h similarity index 98% rename from server/include/gwbitmask.h rename to include/maxscale/gwbitmask.h index 782798907..31ee13655 100644 --- a/server/include/gwbitmask.h +++ b/include/maxscale/gwbitmask.h @@ -13,7 +13,7 @@ * Public License. */ -#include +#include #include /** diff --git a/server/include/gwdirs.h.in b/include/maxscale/gwdirs.h.in similarity index 98% rename from server/include/gwdirs.h.in rename to include/maxscale/gwdirs.h.in index 09824dac4..14592698c 100644 --- a/server/include/gwdirs.h.in +++ b/include/maxscale/gwdirs.h.in @@ -17,7 +17,7 @@ #endif #include #include -#include +#include EXTERN_C_BLOCK_BEGIN diff --git a/server/include/hashtable.h b/include/maxscale/hashtable.h similarity index 98% rename from server/include/hashtable.h rename to include/maxscale/hashtable.h index f6f2ab22e..0b271d731 100644 --- a/server/include/hashtable.h +++ b/include/maxscale/hashtable.h @@ -28,8 +28,8 @@ * * @endverbatim */ -#include -#include +#include +#include EXTERN_C_BLOCK_BEGIN diff --git a/server/include/hint.h b/include/maxscale/hint.h similarity index 98% rename from server/include/hint.h rename to include/maxscale/hint.h index 3f849d361..ea073292f 100644 --- a/server/include/hint.h +++ b/include/maxscale/hint.h @@ -25,7 +25,7 @@ * @endverbatim */ -#include +#include /** diff --git a/server/include/hk_heartbeat.h b/include/maxscale/hk_heartbeat.h similarity index 100% rename from server/include/hk_heartbeat.h rename to include/maxscale/hk_heartbeat.h diff --git a/server/include/housekeeper.h b/include/maxscale/housekeeper.h similarity index 96% rename from server/include/housekeeper.h rename to include/maxscale/housekeeper.h index 1cc6efd2e..6f0bea8db 100644 --- a/server/include/housekeeper.h +++ b/include/maxscale/housekeeper.h @@ -13,8 +13,8 @@ * Public License. */ #include -#include -#include +#include +#include /** * @file housekeeper.h A mechanism to have task run periodically * diff --git a/server/include/maxscale/limits.h b/include/maxscale/limits.h similarity index 100% rename from server/include/maxscale/limits.h rename to include/maxscale/limits.h diff --git a/server/include/listener.h b/include/maxscale/listener.h similarity index 96% rename from server/include/listener.h rename to include/maxscale/listener.h index 7e1c9c187..b2a2dbde5 100644 --- a/server/include/listener.h +++ b/include/maxscale/listener.h @@ -27,9 +27,9 @@ * @endverbatim */ -#include -#include -#include +#include +#include +#include struct dcb; struct service; diff --git a/server/include/listmanager.h b/include/maxscale/listmanager.h similarity index 98% rename from server/include/listmanager.h rename to include/maxscale/listmanager.h index a0ff8ff02..5afd28375 100644 --- a/server/include/listmanager.h +++ b/include/maxscale/listmanager.h @@ -26,8 +26,8 @@ * @endverbatim */ -#include -#include +#include +#include struct dcb; diff --git a/server/include/log_manager.h b/include/maxscale/log_manager.h similarity index 100% rename from server/include/log_manager.h rename to include/maxscale/log_manager.h diff --git a/server/include/maxconfig.h b/include/maxscale/maxconfig.h similarity index 99% rename from server/include/maxconfig.h rename to include/maxscale/maxconfig.h index 684af2376..2bf3c650b 100644 --- a/server/include/maxconfig.h +++ b/include/maxscale/maxconfig.h @@ -12,11 +12,11 @@ * of this software will be governed by version 2 or later of the General * Public License. */ -#include +#include #include #include #include -#include +#include /** * @file config.h The configuration handling elements * diff --git a/server/include/maxscale.h b/include/maxscale/maxscale.h similarity index 100% rename from server/include/maxscale.h rename to include/maxscale/maxscale.h diff --git a/server/include/maxscale_pcre2.h b/include/maxscale/maxscale_pcre2.h similarity index 100% rename from server/include/maxscale_pcre2.h rename to include/maxscale/maxscale_pcre2.h diff --git a/server/include/memlog.h b/include/maxscale/memlog.h similarity index 97% rename from server/include/memlog.h rename to include/maxscale/memlog.h index 49167eb14..ef832b673 100644 --- a/server/include/memlog.h +++ b/include/maxscale/memlog.h @@ -24,7 +24,7 @@ * * @endverbatim */ -#include +#include typedef enum { ML_INT, ML_LONG, ML_LONGLONG, ML_STRING } MEMLOGTYPE; diff --git a/server/include/mlist.h b/include/maxscale/mlist.h similarity index 98% rename from server/include/mlist.h rename to include/maxscale/mlist.h index 11af38864..36eec847e 100644 --- a/server/include/mlist.h +++ b/include/maxscale/mlist.h @@ -13,7 +13,7 @@ * Public License. */ -#include +#include EXTERN_C_BLOCK_BEGIN diff --git a/server/include/modinfo.h b/include/maxscale/modinfo.h similarity index 100% rename from server/include/modinfo.h rename to include/maxscale/modinfo.h diff --git a/server/include/modules.h b/include/maxscale/modules.h similarity index 95% rename from server/include/modules.h rename to include/maxscale/modules.h index c826e5a38..49abc562c 100644 --- a/server/include/modules.h +++ b/include/maxscale/modules.h @@ -12,10 +12,10 @@ * of this software will be governed by version 2 or later of the General * Public License. */ -#include -#include -#include -#include +#include +#include +#include +#include EXTERN_C_BLOCK_BEGIN diff --git a/server/include/modutil.h b/include/maxscale/modutil.h similarity index 97% rename from server/include/modutil.h rename to include/maxscale/modutil.h index 6d0c44e28..af48ca2ec 100644 --- a/server/include/modutil.h +++ b/include/maxscale/modutil.h @@ -26,10 +26,10 @@ * * @endverbatim */ -#include -#include +#include +#include #include -#include +#include #define PTR_IS_RESULTSET(b) (b[0] == 0x01 && b[1] == 0x0 && b[2] == 0x0 && b[3] == 0x01) #define PTR_IS_EOF(b) (b[0] == 0x05 && b[1] == 0x0 && b[2] == 0x0 && b[4] == 0xfe) diff --git a/server/include/monitor.h b/include/maxscale/monitor.h similarity index 97% rename from server/include/monitor.h rename to include/maxscale/monitor.h index 485d3baa9..7b2867391 100644 --- a/server/include/monitor.h +++ b/include/maxscale/monitor.h @@ -13,13 +13,13 @@ * Public License. */ #include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include /** * @file monitor.h The interface to the monitor module diff --git a/server/include/mysql_binlog.h b/include/maxscale/mysql_binlog.h similarity index 100% rename from server/include/mysql_binlog.h rename to include/maxscale/mysql_binlog.h diff --git a/server/include/mysql_utils.h b/include/maxscale/mysql_utils.h similarity index 96% rename from server/include/mysql_utils.h rename to include/maxscale/mysql_utils.h index 3cb26216f..329dfd6f8 100644 --- a/server/include/mysql_utils.h +++ b/include/maxscale/mysql_utils.h @@ -16,7 +16,7 @@ #include #include #include -#include +#include /** Length-encoded integers */ size_t leint_bytes(uint8_t* ptr); diff --git a/server/include/notification.h b/include/maxscale/notification.h similarity index 100% rename from server/include/notification.h rename to include/maxscale/notification.h diff --git a/server/include/platform.h b/include/maxscale/platform.h similarity index 100% rename from server/include/platform.h rename to include/maxscale/platform.h diff --git a/server/include/maxscale/poll.h b/include/maxscale/poll.h similarity index 95% rename from server/include/maxscale/poll.h rename to include/maxscale/poll.h index 296915022..2e9a1ec20 100644 --- a/server/include/maxscale/poll.h +++ b/include/maxscale/poll.h @@ -12,9 +12,9 @@ * of this software will be governed by version 2 or later of the General * Public License. */ -#include -#include -#include +#include +#include +#include #include /** diff --git a/server/include/query_classifier.h b/include/maxscale/query_classifier.h similarity index 98% rename from server/include/query_classifier.h rename to include/maxscale/query_classifier.h index 9d9ed0911..dbbc01fcb 100644 --- a/server/include/query_classifier.h +++ b/include/maxscale/query_classifier.h @@ -13,8 +13,8 @@ * Public License. */ -#include -#include +#include +#include EXTERN_C_BLOCK_BEGIN diff --git a/server/include/queuemanager.h b/include/maxscale/queuemanager.h similarity index 98% rename from server/include/queuemanager.h rename to include/maxscale/queuemanager.h index b91b2e6e8..462dd10e4 100644 --- a/server/include/queuemanager.h +++ b/include/maxscale/queuemanager.h @@ -27,7 +27,7 @@ */ #include -#include +#include #define CONNECTION_QUEUE_LIMIT 1000 diff --git a/server/include/random_jkiss.h b/include/maxscale/random_jkiss.h similarity index 100% rename from server/include/random_jkiss.h rename to include/maxscale/random_jkiss.h diff --git a/server/include/rdtsc.h b/include/maxscale/rdtsc.h similarity index 100% rename from server/include/rdtsc.h rename to include/maxscale/rdtsc.h diff --git a/server/include/resultset.h b/include/maxscale/resultset.h similarity index 98% rename from server/include/resultset.h rename to include/maxscale/resultset.h index 231d13227..58b4f7a35 100644 --- a/server/include/resultset.h +++ b/include/maxscale/resultset.h @@ -24,7 +24,7 @@ * * @endverbatim */ -#include +#include /** diff --git a/server/include/router.h b/include/maxscale/router.h similarity index 97% rename from server/include/router.h rename to include/maxscale/router.h index 64e90fe6a..9dafcc9aa 100644 --- a/server/include/router.h +++ b/include/maxscale/router.h @@ -27,9 +27,9 @@ * 27/10/2015 Martin Brampton Add RCAP_TYPE_NO_RSESSION * */ -#include -#include -#include +#include +#include +#include #include /** diff --git a/server/include/secrets.h b/include/maxscale/secrets.h similarity index 100% rename from server/include/secrets.h rename to include/maxscale/secrets.h diff --git a/server/include/server.h b/include/maxscale/server.h similarity index 99% rename from server/include/server.h rename to include/maxscale/server.h index 3b38abb70..3daaa10dd 100644 --- a/server/include/server.h +++ b/include/maxscale/server.h @@ -12,8 +12,8 @@ * of this software will be governed by version 2 or later of the General * Public License. */ -#include -#include +#include +#include /** * @file service.h diff --git a/server/include/service.h b/include/maxscale/service.h similarity index 96% rename from server/include/service.h rename to include/maxscale/service.h index 385ac727f..51b81ca6f 100644 --- a/server/include/service.h +++ b/include/maxscale/service.h @@ -14,16 +14,16 @@ */ #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include diff --git a/server/include/session.h b/include/maxscale/session.h similarity index 96% rename from server/include/session.h rename to include/maxscale/session.h index f808dfbf6..87497ad31 100644 --- a/server/include/session.h +++ b/include/maxscale/session.h @@ -34,13 +34,13 @@ * @endverbatim */ #include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include struct dcb; struct service; diff --git a/server/include/skygw_debug.h b/include/maxscale/skygw_debug.h similarity index 99% rename from server/include/skygw_debug.h rename to include/maxscale/skygw_debug.h index d01bd5134..da75fc057 100644 --- a/server/include/skygw_debug.h +++ b/include/maxscale/skygw_debug.h @@ -46,7 +46,7 @@ #endif /* SS_DEBUG || SS_PROF */ #if defined(SS_DEBUG) && defined(LOG_ASSERT) -#include +#include # define ss_dassert(exp) do { if(!(exp)){\ MXS_ERROR("debug assert %s:%d\n", (char*)__FILE__, __LINE__);\ mxs_log_flush_sync(); assert(exp);} } while (false) diff --git a/server/include/skygw_types.h b/include/maxscale/skygw_types.h similarity index 100% rename from server/include/skygw_types.h rename to include/maxscale/skygw_types.h diff --git a/server/include/skygw_utils.h b/include/maxscale/skygw_utils.h similarity index 98% rename from server/include/skygw_utils.h rename to include/maxscale/skygw_utils.h index 602f5fa03..a903466c3 100644 --- a/server/include/skygw_utils.h +++ b/include/maxscale/skygw_utils.h @@ -28,8 +28,8 @@ #endif #define FSYNCLIMIT 10 -#include "skygw_types.h" -#include "skygw_debug.h" +#include +#include #define DISKWRITE_LATENCY (5*MSEC_USEC) diff --git a/server/include/slist.h b/include/maxscale/slist.h similarity index 98% rename from server/include/slist.h rename to include/maxscale/slist.h index c7a179e95..725023b6d 100644 --- a/server/include/slist.h +++ b/include/maxscale/slist.h @@ -13,7 +13,7 @@ * Public License. */ -#include +#include EXTERN_C_BLOCK_BEGIN diff --git a/server/include/spinlock.h b/include/maxscale/spinlock.h similarity index 98% rename from server/include/spinlock.h rename to include/maxscale/spinlock.h index 34794165c..7df6064b7 100644 --- a/server/include/spinlock.h +++ b/include/maxscale/spinlock.h @@ -23,7 +23,7 @@ * for the lock to be released. However they are useful in that they do not involve * system calls and are light weight when the expected wait time for a lock is low. */ -#include +#include EXTERN_C_BLOCK_BEGIN diff --git a/server/include/statistics.h b/include/maxscale/statistics.h similarity index 100% rename from server/include/statistics.h rename to include/maxscale/statistics.h diff --git a/server/include/test_utils.h b/include/maxscale/test_utils.h similarity index 80% rename from server/include/test_utils.h rename to include/maxscale/test_utils.h index 7800ce1b3..d1ea666c8 100644 --- a/server/include/test_utils.h +++ b/include/maxscale/test_utils.h @@ -15,11 +15,11 @@ */ #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include void init_test_env(char *path) { diff --git a/server/include/thread.h b/include/maxscale/thread.h similarity index 100% rename from server/include/thread.h rename to include/maxscale/thread.h diff --git a/server/include/users.h b/include/maxscale/users.h similarity index 96% rename from server/include/users.h rename to include/maxscale/users.h index 19f802c26..78a4b3731 100644 --- a/server/include/users.h +++ b/include/maxscale/users.h @@ -12,9 +12,9 @@ * of this software will be governed by version 2 or later of the General * Public License. */ -#include -#include -#include +#include +#include +#include #include /** diff --git a/server/include/utils.h b/include/maxscale/utils.h similarity index 100% rename from server/include/utils.h rename to include/maxscale/utils.h diff --git a/server/include/version.h.in b/include/maxscale/version.h.in similarity index 100% rename from server/include/version.h.in rename to include/maxscale/version.h.in diff --git a/query_classifier/qc_dummy/qc_dummy.cc b/query_classifier/qc_dummy/qc_dummy.cc index 82f1381dd..2ee32e90c 100644 --- a/query_classifier/qc_dummy/qc_dummy.cc +++ b/query_classifier/qc_dummy/qc_dummy.cc @@ -11,8 +11,8 @@ * Public License. */ -#include -#include +#include +#include qc_parse_result_t qc_parse(GWBUF* querybuf) { diff --git a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc index b863b3f38..e9fd30a36 100644 --- a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc +++ b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc @@ -55,12 +55,12 @@ #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include -#include +#include #include #include diff --git a/query_classifier/qc_sqlite/builtin_functions.c b/query_classifier/qc_sqlite/builtin_functions.c index 2ed96ecfb..2cb2ed797 100644 --- a/query_classifier/qc_sqlite/builtin_functions.c +++ b/query_classifier/qc_sqlite/builtin_functions.c @@ -5,7 +5,7 @@ #include "builtin_functions.h" #include #include -#include +#include static struct { diff --git a/query_classifier/qc_sqlite/qc_sqlite.c b/query_classifier/qc_sqlite/qc_sqlite.c index eff117691..686b4821c 100644 --- a/query_classifier/qc_sqlite/qc_sqlite.c +++ b/query_classifier/qc_sqlite/qc_sqlite.c @@ -15,13 +15,13 @@ #include #include -#include -#include +#include +#include #include -#include -#include -#include -#include +#include +#include +#include +#include #include #include "builtin_functions.h" diff --git a/query_classifier/test/canonical_tests/canonizer.c b/query_classifier/test/canonical_tests/canonizer.c index 5609ecec7..9dc4a5c49 100644 --- a/query_classifier/test/canonical_tests/canonizer.c +++ b/query_classifier/test/canonical_tests/canonizer.c @@ -15,9 +15,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include int main(int argc, char** argv) { diff --git a/query_classifier/test/classify.c b/query_classifier/test/classify.c index 43449f969..9a51cc9fe 100644 --- a/query_classifier/test/classify.c +++ b/query_classifier/test/classify.c @@ -16,12 +16,12 @@ #include #include #include -#include -#include +#include +#include #include #include -#include -#include +#include +#include char* append(char* types, const char* type_name, size_t* lenp) { diff --git a/query_classifier/test/compare.cc b/query_classifier/test/compare.cc index b289881a6..b52d06a1f 100644 --- a/query_classifier/test/compare.cc +++ b/query_classifier/test/compare.cc @@ -20,10 +20,10 @@ #include #include #include -#include -#include +#include +#include #include -#include +#include using std::cerr; using std::cin; using std::cout; diff --git a/server/core/adminusers.c b/server/core/adminusers.c index ba7738964..6b0f1693c 100644 --- a/server/core/adminusers.c +++ b/server/core/adminusers.c @@ -19,11 +19,11 @@ #endif #include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include /** diff --git a/server/core/alloc.c b/server/core/alloc.c index 6ac095181..6ecaabcc3 100644 --- a/server/core/alloc.c +++ b/server/core/alloc.c @@ -13,7 +13,7 @@ #include #include -#include +#include /** * @brief Allocates memory; behaves exactly like `malloc`. diff --git a/server/core/authenticator.c b/server/core/authenticator.c index d70bdb785..30b766e29 100644 --- a/server/core/authenticator.c +++ b/server/core/authenticator.c @@ -11,9 +11,9 @@ * Public License. */ -#include -#include -#include +#include +#include +#include #include /** diff --git a/server/core/buffer.c b/server/core/buffer.c index 1b13b5fa5..9d55aaff9 100644 --- a/server/core/buffer.c +++ b/server/core/buffer.c @@ -37,19 +37,19 @@ * * @endverbatim */ -#include +#include #include #include #include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include #if defined(BUFFER_TRACE) -#include +#include #include static HASHTABLE *buffer_hashtable = NULL; diff --git a/server/core/config.c b/server/core/config.c index e9848769f..060386f97 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -50,30 +50,30 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include #include #include #include -#include -#include +#include +#include #include #include #include #include -#include -#include +#include +#include #include #include #define PCRE2_CODE_UNIT_WIDTH 8 diff --git a/server/core/dbusers.c b/server/core/dbusers.c index 7927e1a90..ab304107e 100644 --- a/server/core/dbusers.c +++ b/server/core/dbusers.c @@ -39,17 +39,17 @@ #include #include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include #include #include #include -#include +#include #include /** Don't include the root user */ diff --git a/server/core/dcb.c b/server/core/dcb.c index 9f7e54f99..27e324d37 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -69,23 +69,23 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include #include -#include +#include #include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include #include #include #include diff --git a/server/core/externcmd.c b/server/core/externcmd.c index c38041f21..9bcef24ae 100644 --- a/server/core/externcmd.c +++ b/server/core/externcmd.c @@ -11,7 +11,7 @@ * Public License. */ -#include +#include #include /** diff --git a/server/core/filter.c b/server/core/filter.c index 3571890c4..45fa3721a 100644 --- a/server/core/filter.c +++ b/server/core/filter.c @@ -26,12 +26,12 @@ #include #include #include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include #include static SPINLOCK filter_spin = SPINLOCK_INIT; /**< Protects the list of all filters */ diff --git a/server/core/gateway.c b/server/core/gateway.c index 657c37add..f8e567344 100644 --- a/server/core/gateway.c +++ b/server/core/gateway.c @@ -48,35 +48,35 @@ #include #include #include -#include +#include #include #include #include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include #include -#include -#include -#include -#include +#include +#include +#include +#include #include #include #include -#include -#include -#include +#include +#include +#include #include #include -#include -#include -#include +#include +#include +#include #include @@ -84,7 +84,7 @@ #include #include #include -#include +#include #include #define STRING_BUFFER_SIZE 1024 diff --git a/server/core/gw_ssl.c b/server/core/gw_ssl.c index 483b138dc..1951fcd7b 100644 --- a/server/core/gw_ssl.c +++ b/server/core/gw_ssl.c @@ -29,9 +29,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include #include /** diff --git a/server/core/gw_utils.c b/server/core/gw_utils.c index 29f8d96de..d7489d49a 100644 --- a/server/core/gw_utils.c +++ b/server/core/gw_utils.c @@ -34,12 +34,12 @@ *@endverbatim */ -#include -#include -#include +#include +#include +#include -#include -#include +#include +#include SPINLOCK tmplock = SPINLOCK_INIT; diff --git a/server/core/gwbitmask.c b/server/core/gwbitmask.c index ac163e395..bfa78d5eb 100644 --- a/server/core/gwbitmask.c +++ b/server/core/gwbitmask.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include /** diff --git a/server/core/gwdirs.c b/server/core/gwdirs.c index 2a9aef8c8..d7c54be0f 100644 --- a/server/core/gwdirs.c +++ b/server/core/gwdirs.c @@ -11,9 +11,9 @@ * Public License. */ -#include +#include #include -#include +#include /** * Set the configuration file directory diff --git a/server/core/hashtable.c b/server/core/hashtable.c index 650b2ee06..30975784b 100644 --- a/server/core/hashtable.c +++ b/server/core/hashtable.c @@ -18,8 +18,8 @@ #include #include #include -#include -#include +#include +#include /** * @file hashtable.c General purpose hashtable routines diff --git a/server/core/hint.c b/server/core/hint.c index 7cfb01f70..62e17a2f1 100644 --- a/server/core/hint.c +++ b/server/core/hint.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include /** diff --git a/server/core/housekeeper.c b/server/core/housekeeper.c index 19d96c7d3..48096b2f3 100644 --- a/server/core/housekeeper.c +++ b/server/core/housekeeper.c @@ -13,10 +13,10 @@ #include #include #include -#include -#include -#include -#include +#include +#include +#include +#include /** * @file housekeeper.c Provide a mechanism to run periodic tasks diff --git a/server/core/listener.c b/server/core/listener.c index 249119d78..a7de9403e 100644 --- a/server/core/listener.c +++ b/server/core/listener.c @@ -30,13 +30,13 @@ #include #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include -#include -#include +#include +#include static RSA *rsa_512 = NULL; static RSA *rsa_1024 = NULL; diff --git a/server/core/listmanager.c b/server/core/listmanager.c index 63c987300..373186fd7 100644 --- a/server/core/listmanager.c +++ b/server/core/listmanager.c @@ -31,10 +31,10 @@ #include #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include /** diff --git a/server/core/load_utils.c b/server/core/load_utils.c index 36ce7200c..ad83cfccf 100644 --- a/server/core/load_utils.c +++ b/server/core/load_utils.c @@ -34,17 +34,17 @@ #include #include #include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include #include #include #include -#include -#include +#include +#include #include static MODULES *registered = NULL; diff --git a/server/core/log_manager.cc b/server/core/log_manager.cc index ddcbcd1e5..75e855423 100644 --- a/server/core/log_manager.cc +++ b/server/core/log_manager.cc @@ -10,7 +10,7 @@ * of this software will be governed by version 2 or later of the General * Public License. */ -#include +#include #include #include #include @@ -23,14 +23,14 @@ #include #include #include -#include +#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include #include #define MAX_PREFIXLEN 250 diff --git a/server/core/maxkeys.c b/server/core/maxkeys.c index 5103f908c..380338329 100644 --- a/server/core/maxkeys.c +++ b/server/core/maxkeys.c @@ -24,10 +24,10 @@ */ #include #include -#include -#include -#include -#include +#include +#include +#include +#include struct option options[] = { diff --git a/server/core/maxpasswd.c b/server/core/maxpasswd.c index 0e1650b76..4a7c77a2a 100644 --- a/server/core/maxpasswd.c +++ b/server/core/maxpasswd.c @@ -23,10 +23,10 @@ * @endverbatim */ #include -#include -#include -#include -#include +#include +#include +#include +#include struct option options[] = { diff --git a/server/core/maxscale_pcre2.c b/server/core/maxscale_pcre2.c index c13eb323b..484767994 100644 --- a/server/core/maxscale_pcre2.c +++ b/server/core/maxscale_pcre2.c @@ -24,7 +24,7 @@ * @endverbatim */ -#include +#include #include /** diff --git a/server/core/memlog.c b/server/core/memlog.c index 20ed56ce5..a22cd36d5 100644 --- a/server/core/memlog.c +++ b/server/core/memlog.c @@ -22,13 +22,13 @@ * * @endverbatim */ -#include +#include #include #include #include #include #include -#include +#include static MEMLOG *memlogs = NULL; static SPINLOCK memlock = SPINLOCK_INIT; diff --git a/server/core/misc.c b/server/core/misc.c index b43b362e6..4f1e31774 100644 --- a/server/core/misc.c +++ b/server/core/misc.c @@ -11,7 +11,7 @@ * Public License. */ -#include +#include #include static time_t started; diff --git a/server/core/mlist.c b/server/core/mlist.c index 0242bcfff..c83df050b 100644 --- a/server/core/mlist.c +++ b/server/core/mlist.c @@ -11,7 +11,7 @@ * Public License. */ -#include +#include #include static void mlist_free_memory(mlist_t* ml, char* name); diff --git a/server/core/modutil.c b/server/core/modutil.c index 4026f3277..9a0daba9c 100644 --- a/server/core/modutil.c +++ b/server/core/modutil.c @@ -24,12 +24,12 @@ * * @endverbatim */ -#include +#include #include #include #include #include -#include +#include #include /** These are used when converting MySQL wildcards to regular expressions */ diff --git a/server/core/monitor.c b/server/core/monitor.c index e2143b319..8cdafe660 100644 --- a/server/core/monitor.c +++ b/server/core/monitor.c @@ -30,16 +30,16 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include #include -#include +#include #include /* @@ -52,7 +52,7 @@ #define ADDITEM( _event_type, _event_name ) { #_event_name } const monitor_def_t monitor_event_definitions[MAX_MONITOR_EVENT] = { -#include "def_monitor_event.h" +#include }; #undef ADDITEM diff --git a/server/core/mysql_binlog.c b/server/core/mysql_binlog.c index 342c7c34c..6e52cc47b 100644 --- a/server/core/mysql_binlog.c +++ b/server/core/mysql_binlog.c @@ -15,14 +15,14 @@ * @file mysql_binlog.c - Extracting information from binary logs */ -#include -#include +#include +#include #include -#include +#include #include -#include -#include -#include +#include +#include +#include #include /** diff --git a/server/core/mysql_utils.c b/server/core/mysql_utils.c index 7671bbf35..3c1b2159c 100644 --- a/server/core/mysql_utils.c +++ b/server/core/mysql_utils.c @@ -20,13 +20,13 @@ * row based replication. */ -#include +#include #include #include #include -#include -#include -#include +#include +#include +#include /** * @brief Calculate the length of a length-encoded integer in bytes diff --git a/server/core/poll.c b/server/core/poll.c index 1f79aa5fb..d68a44e26 100644 --- a/server/core/poll.c +++ b/server/core/poll.c @@ -21,20 +21,20 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include -#include -#include -#include -#include +#include +#include +#include +#include #define PROFILE_POLL 0 diff --git a/server/core/query_classifier.c b/server/core/query_classifier.c index 2e3de362d..0c984df43 100644 --- a/server/core/query_classifier.c +++ b/server/core/query_classifier.c @@ -11,10 +11,10 @@ * Public License. */ -#include -#include -#include -#include +#include +#include +#include +#include #include //#define QC_TRACE_ENABLED diff --git a/server/core/queuemanager.c b/server/core/queuemanager.c index ae615b1d7..9ddafb487 100644 --- a/server/core/queuemanager.c +++ b/server/core/queuemanager.c @@ -28,12 +28,12 @@ #include #include #include -#include +#include #include -#include -#include -#include -#include +#include +#include +#include +#include #if defined(SS_DEBUG) int debug_check_fail = 0; diff --git a/server/core/random_jkiss.c b/server/core/random_jkiss.c index 83b3d5e19..bc81b9fcc 100644 --- a/server/core/random_jkiss.c +++ b/server/core/random_jkiss.c @@ -31,8 +31,8 @@ #include #include #include -#include -#include +#include +#include /* Public domain code for JKISS RNG - Comment header added */ diff --git a/server/core/resultset.c b/server/core/resultset.c index 67e8a2da7..c8ccd8541 100644 --- a/server/core/resultset.c +++ b/server/core/resultset.c @@ -26,9 +26,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include static int mysql_send_fieldcount(DCB *, int); diff --git a/server/core/secrets.c b/server/core/secrets.c index 70014a2ee..f9716bc3a 100644 --- a/server/core/secrets.c +++ b/server/core/secrets.c @@ -11,17 +11,17 @@ * Public License. */ -#include +#include #include -#include -#include +#include +#include #include #include -#include -#include +#include +#include #include -#include "gw.h" +#include /** * Generate a random printable character diff --git a/server/core/server.c b/server/core/server.c index 516ae7094..db3c49f94 100644 --- a/server/core/server.c +++ b/server/core/server.c @@ -35,16 +35,16 @@ #include #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include -#include -#include -#include +#include +#include +#include #include -#include +#include static SPINLOCK server_spin = SPINLOCK_INIT; static SERVER *allServers = NULL; diff --git a/server/core/service.c b/server/core/service.c index 4452d6604..be80fc8c1 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -42,30 +42,30 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include -#include -#include +#include +#include #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include -#include -#include +#include +#include #include /** To be used with configuration type checks */ diff --git a/server/core/session.c b/server/core/session.c index 3aef74e1d..f6ce149f6 100644 --- a/server/core/session.c +++ b/server/core/session.c @@ -33,16 +33,16 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include /* This list of all sessions */ LIST_CONFIG SESSIONlist = diff --git a/server/core/skygw_utils.cc b/server/core/skygw_utils.cc index a8c13e9bd..c219d2521 100644 --- a/server/core/skygw_utils.cc +++ b/server/core/skygw_utils.cc @@ -23,12 +23,12 @@ #include #include #include -#include "skygw_debug.h" -#include +#include +#include #include -#include "skygw_utils.h" -#include -#include +#include +#include +#include #include static void simple_mutex_free_memory(simple_mutex_t* sm); diff --git a/server/core/slist.c b/server/core/slist.c index 7f43b6421..ab5cdd7ca 100644 --- a/server/core/slist.c +++ b/server/core/slist.c @@ -11,9 +11,9 @@ * Public License. */ -#include +#include #include -#include +#include static slist_cursor_t* slist_cursor_init(slist_t* list); static slist_t* slist_init_ex(bool create_cursors); diff --git a/server/core/spinlock.c b/server/core/spinlock.c index e2c59978a..9772eb65f 100644 --- a/server/core/spinlock.c +++ b/server/core/spinlock.c @@ -23,10 +23,10 @@ * @endverbatim */ -#include -#include +#include +#include #include -#include +#include /** * Initialise a spinlock. diff --git a/server/core/statistics.c b/server/core/statistics.c index 300f835d9..259608efa 100644 --- a/server/core/statistics.c +++ b/server/core/statistics.c @@ -23,11 +23,11 @@ * @endverbatim */ -#include +#include #include -#include +#include #include -#include +#include static int thread_count = 0; static bool stats_initialized = false; diff --git a/server/core/test/test_mysql_users.c b/server/core/test/test_mysql_users.c index 8ab3439a4..0defa3ff6 100644 --- a/server/core/test/test_mysql_users.c +++ b/server/core/test/test_mysql_users.c @@ -28,16 +28,16 @@ #include #include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include #include #include -#include +#include #include #include diff --git a/server/core/test/testadminusers.c b/server/core/test/testadminusers.c index 13f62804a..4ea8f5e88 100644 --- a/server/core/test/testadminusers.c +++ b/server/core/test/testadminusers.c @@ -34,8 +34,8 @@ #include #include #include -#include -#include +#include +#include #include diff --git a/server/core/test/testbuffer.c b/server/core/test/testbuffer.c index 6c6e4ce3d..119bd464d 100644 --- a/server/core/test/testbuffer.c +++ b/server/core/test/testbuffer.c @@ -34,8 +34,8 @@ #include #include -#include -#include +#include +#include /** * Generate predefined test data diff --git a/server/core/test/testdcb.c b/server/core/test/testdcb.c index c568380bc..756487297 100644 --- a/server/core/test/testdcb.c +++ b/server/core/test/testdcb.c @@ -32,8 +32,8 @@ #include #include #include -#include -#include +#include +#include /** * test1 Allocate a dcb and do lots of other things diff --git a/server/core/test/testfeedback.c b/server/core/test/testfeedback.c index 763332aa7..fe8736442 100644 --- a/server/core/test/testfeedback.c +++ b/server/core/test/testfeedback.c @@ -34,17 +34,17 @@ #include #include #include -#include +#include #include #include #include #include #include -#include -#include +#include +#include #include -#include -#include +#include +#include static char* server_options[] = { diff --git a/server/core/test/testfilter.c b/server/core/test/testfilter.c index bc96c21f2..377f5a7af 100644 --- a/server/core/test/testfilter.c +++ b/server/core/test/testfilter.c @@ -33,7 +33,7 @@ #include #include -#include +#include /** diff --git a/server/core/test/testgwbitmask.c b/server/core/test/testgwbitmask.c index 34edb5510..751920624 100644 --- a/server/core/test/testgwbitmask.c +++ b/server/core/test/testgwbitmask.c @@ -33,9 +33,9 @@ #include #include -#include +#include -#include +#include /** * test1 Create a bitmap and mess around with it diff --git a/server/core/test/testhash.c b/server/core/test/testhash.c index d7de067a5..902034216 100644 --- a/server/core/test/testhash.c +++ b/server/core/test/testhash.c @@ -36,8 +36,8 @@ #include #include -#include -#include +#include +#include static void read_lock(HASHTABLE *table) diff --git a/server/core/test/testhint.c b/server/core/test/testhint.c index 0174ddd92..afb9cc700 100644 --- a/server/core/test/testhint.c +++ b/server/core/test/testhint.c @@ -33,7 +33,7 @@ #include #include -#include +#include #include /** diff --git a/server/core/test/testlog.c b/server/core/test/testlog.c index c0549e8d4..2f48b71aa 100644 --- a/server/core/test/testlog.c +++ b/server/core/test/testlog.c @@ -15,8 +15,8 @@ #include #include #include -#include -#include +#include +#include static void skygw_log_enable(int priority) { diff --git a/server/core/test/testlogorder.c b/server/core/test/testlogorder.c index d8116a6ab..42b9f78cb 100644 --- a/server/core/test/testlogorder.c +++ b/server/core/test/testlogorder.c @@ -16,8 +16,8 @@ #include #include #include -#include -#include +#include +#include static void skygw_log_enable(int priority) { diff --git a/server/core/test/testlogthrottling.cc b/server/core/test/testlogthrottling.cc index 45c0982d4..cb0f227bb 100644 --- a/server/core/test/testlogthrottling.cc +++ b/server/core/test/testlogthrottling.cc @@ -19,7 +19,7 @@ #include #include #include -#include +#include using std::cerr; using std::cout; diff --git a/server/core/test/testmaxscalepcre2.c b/server/core/test/testmaxscalepcre2.c index 289de88d2..f75551f0c 100644 --- a/server/core/test/testmaxscalepcre2.c +++ b/server/core/test/testmaxscalepcre2.c @@ -34,8 +34,8 @@ #include #include #include -#include -#include +#include +#include #define test_assert(a, b) if(!(a)){fprintf(stderr, b);return 1;} diff --git a/server/core/test/testmemlog.c b/server/core/test/testmemlog.c index 21060661a..d37d0064d 100644 --- a/server/core/test/testmemlog.c +++ b/server/core/test/testmemlog.c @@ -33,7 +33,7 @@ #include #include #include -#include +#include /** * Count the number of lines in a file diff --git a/server/core/test/testmodutil.c b/server/core/test/testmodutil.c index 401b2147c..043629571 100644 --- a/server/core/test/testmodutil.c +++ b/server/core/test/testmodutil.c @@ -34,8 +34,8 @@ #include #include -#include -#include +#include +#include /** * test1 Allocate a service and do lots of other things diff --git a/server/core/test/testpoll.c b/server/core/test/testpoll.c index 0dce61a51..74c55b4f9 100644 --- a/server/core/test/testpoll.c +++ b/server/core/test/testpoll.c @@ -34,9 +34,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include /** * test1 Allocate a service and do lots of other things diff --git a/server/core/test/testqueuemanager.c b/server/core/test/testqueuemanager.c index cd250f816..e616d9c99 100644 --- a/server/core/test/testqueuemanager.c +++ b/server/core/test/testqueuemanager.c @@ -37,10 +37,10 @@ extern int debug_check_fail; #include #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include /** diff --git a/server/core/test/testserver.c b/server/core/test/testserver.c index 2424cedcb..d1e214de2 100644 --- a/server/core/test/testserver.c +++ b/server/core/test/testserver.c @@ -34,8 +34,8 @@ #include #include -#include -#include +#include +#include /** * test1 Allocate a server and do lots of other things * diff --git a/server/core/test/testservice.c b/server/core/test/testservice.c index cd896adc3..13ff6e9bc 100644 --- a/server/core/test/testservice.c +++ b/server/core/test/testservice.c @@ -32,10 +32,10 @@ #include #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include /** diff --git a/server/core/test/testspinlock.c b/server/core/test/testspinlock.c index 0992783c6..1d291a208 100644 --- a/server/core/test/testspinlock.c +++ b/server/core/test/testspinlock.c @@ -33,8 +33,8 @@ #include #include -#include -#include +#include +#include /** diff --git a/server/core/test/testusers.c b/server/core/test/testusers.c index a37ee2579..f6bea1ef6 100644 --- a/server/core/test/testusers.c +++ b/server/core/test/testusers.c @@ -33,9 +33,9 @@ #include #include -#include +#include -#include "log_manager.h" +#include /** * test1 Allocate table of users and mess around with it diff --git a/server/core/thread.c b/server/core/thread.c index 415faac3b..3cc32b09a 100644 --- a/server/core/thread.c +++ b/server/core/thread.c @@ -10,7 +10,7 @@ * of this software will be governed by version 2 or later of the General * Public License. */ -#include +#include /** * @file thread.c - Implementation of thread related operations diff --git a/server/core/users.c b/server/core/users.c index 66ebe3001..2c2a7ca56 100644 --- a/server/core/users.c +++ b/server/core/users.c @@ -13,10 +13,10 @@ #include #include #include -#include +#include #include -#include -#include +#include +#include /** * @file users.c User table maintenance routines diff --git a/server/core/utils.c b/server/core/utils.c index c01dcde89..b439f4fca 100644 --- a/server/core/utils.c +++ b/server/core/utils.c @@ -29,16 +29,16 @@ */ -#include -#include -#include +#include +#include +#include #include #include #include -#include -#include -#include -#include +#include +#include +#include +#include /* used in the hex2bin function */ #define char_val(X) (X >= '0' && X <= '9' ? X-'0' : \ diff --git a/server/modules/authenticator/cdc_plain_auth.c b/server/modules/authenticator/cdc_plain_auth.c index 909070c92..541d22441 100644 --- a/server/modules/authenticator/cdc_plain_auth.c +++ b/server/modules/authenticator/cdc_plain_auth.c @@ -25,10 +25,10 @@ * @endverbatim */ -#include +#include #include -#include -#include +#include +#include #include #include diff --git a/server/modules/authenticator/gssapi_auth.c b/server/modules/authenticator/gssapi_auth.c index 81fedf769..90abc0f33 100644 --- a/server/modules/authenticator/gssapi_auth.c +++ b/server/modules/authenticator/gssapi_auth.c @@ -11,9 +11,9 @@ * Public License. */ -#include +#include #include -#include +#include #include #include "gssapi_auth.h" diff --git a/server/modules/authenticator/gssapi_backend_auth.c b/server/modules/authenticator/gssapi_backend_auth.c index 1f5d5883c..d1b7903bc 100644 --- a/server/modules/authenticator/gssapi_backend_auth.c +++ b/server/modules/authenticator/gssapi_backend_auth.c @@ -11,9 +11,9 @@ * Public License. */ -#include +#include #include -#include +#include #include #include "gssapi_auth.h" diff --git a/server/modules/authenticator/http_auth.c b/server/modules/authenticator/http_auth.c index c9e912a59..a8e24b04d 100644 --- a/server/modules/authenticator/http_auth.c +++ b/server/modules/authenticator/http_auth.c @@ -24,15 +24,15 @@ * @endverbatim */ -#include +#include #include -#include -#include -#include +#include +#include +#include #include -#include -#include -#include +#include +#include +#include /* @see function load_module in load_utils.c for explanation of the following * lint directives. diff --git a/server/modules/authenticator/max_admin_auth.c b/server/modules/authenticator/max_admin_auth.c index 3afc7ce97..1ee62ce15 100644 --- a/server/modules/authenticator/max_admin_auth.c +++ b/server/modules/authenticator/max_admin_auth.c @@ -26,13 +26,13 @@ * @endverbatim */ -#include +#include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include /* @see function load_module in load_utils.c for explanation of the following * lint directives. diff --git a/server/modules/authenticator/mysql_auth.c b/server/modules/authenticator/mysql_auth.c index aee9a65e3..e92088f3c 100644 --- a/server/modules/authenticator/mysql_auth.c +++ b/server/modules/authenticator/mysql_auth.c @@ -27,14 +27,14 @@ #include #include -#include +#include #include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include /* @see function load_module in load_utils.c for explanation of the following * lint directives. diff --git a/server/modules/authenticator/mysql_backend_auth.c b/server/modules/authenticator/mysql_backend_auth.c index bde267850..ff7b37c69 100644 --- a/server/modules/authenticator/mysql_backend_auth.c +++ b/server/modules/authenticator/mysql_backend_auth.c @@ -29,10 +29,10 @@ * @endverbatim */ -#include +#include #include #include -#include +#include /** Authentication states */ enum mba_state diff --git a/server/modules/authenticator/null_auth_allow.c b/server/modules/authenticator/null_auth_allow.c index 8d5329c18..78d2c90ea 100644 --- a/server/modules/authenticator/null_auth_allow.c +++ b/server/modules/authenticator/null_auth_allow.c @@ -27,11 +27,11 @@ * @endverbatim */ -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include /* @see function load_module in load_utils.c for explanation of the following * lint directives. diff --git a/server/modules/authenticator/null_auth_deny.c b/server/modules/authenticator/null_auth_deny.c index 424b625f8..034b6c28e 100644 --- a/server/modules/authenticator/null_auth_deny.c +++ b/server/modules/authenticator/null_auth_deny.c @@ -27,11 +27,11 @@ * @endverbatim */ -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include /* @see function load_module in load_utils.c for explanation of the following * lint directives. diff --git a/server/modules/filter/cache/cache.c b/server/modules/filter/cache/cache.c index f500c5454..d0d76c0cd 100644 --- a/server/modules/filter/cache/cache.c +++ b/server/modules/filter/cache/cache.c @@ -13,13 +13,13 @@ #define MXS_MODULE_NAME "cache" #include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include #include "cache.h" #include "rules.h" #include "storage.h" diff --git a/server/modules/filter/cache/cache_storage_api.h b/server/modules/filter/cache/cache_storage_api.h index c6f889c3e..320e25f00 100644 --- a/server/modules/filter/cache/cache_storage_api.h +++ b/server/modules/filter/cache/cache_storage_api.h @@ -15,9 +15,9 @@ #include #include -#include +#include #include -#include +#include EXTERN_C_BLOCK_BEGIN diff --git a/server/modules/filter/cache/rules.c b/server/modules/filter/cache/rules.c index 27f769f54..e66ae84c1 100644 --- a/server/modules/filter/cache/rules.c +++ b/server/modules/filter/cache/rules.c @@ -16,10 +16,10 @@ #include #include #include -#include +#include #include -#include -#include +#include +#include #include "cache.h" static const char KEY_ATTRIBUTE[] = "attribute"; diff --git a/server/modules/filter/cache/rules.h b/server/modules/filter/cache/rules.h index d805b6451..58e04716c 100644 --- a/server/modules/filter/cache/rules.h +++ b/server/modules/filter/cache/rules.h @@ -15,9 +15,9 @@ #include #include -#include -#include -#include +#include +#include +#include typedef enum cache_rule_attribute diff --git a/server/modules/filter/cache/storage.c b/server/modules/filter/cache/storage.c index 7aa51b82e..165905dd1 100644 --- a/server/modules/filter/cache/storage.c +++ b/server/modules/filter/cache/storage.c @@ -15,8 +15,8 @@ #include #include #include -#include -#include +#include +#include CACHE_STORAGE_MODULE* cache_storage_open(const char *name) { diff --git a/server/modules/filter/cache/storage/storage_rocksdb/rocksdbstorage.cc b/server/modules/filter/cache/storage/storage_rocksdb/rocksdbstorage.cc index fdffe3b82..2fdbc5b1c 100644 --- a/server/modules/filter/cache/storage/storage_rocksdb/rocksdbstorage.cc +++ b/server/modules/filter/cache/storage/storage_rocksdb/rocksdbstorage.cc @@ -19,13 +19,13 @@ #include #include #include -#include +#include extern "C" { // TODO: Add extern "C" to modutil.h -#include +#include } -#include +#include #include "rocksdbinternals.h" using std::for_each; diff --git a/server/modules/filter/cache/storage/storage_rocksdb/storage_rocksdb.h b/server/modules/filter/cache/storage/storage_rocksdb/storage_rocksdb.h index 63b5d10bd..895421d4b 100644 --- a/server/modules/filter/cache/storage/storage_rocksdb/storage_rocksdb.h +++ b/server/modules/filter/cache/storage/storage_rocksdb/storage_rocksdb.h @@ -14,6 +14,6 @@ */ #define MXS_MODULE_NAME "storage_rocksdb" -#include +#include #endif diff --git a/server/modules/filter/cache/test/testrules.c b/server/modules/filter/cache/test/testrules.c index d8f075558..d717fd5a4 100644 --- a/server/modules/filter/cache/test/testrules.c +++ b/server/modules/filter/cache/test/testrules.c @@ -13,11 +13,11 @@ #include #include "rules.h" -#include +#include #if !defined(SS_DEBUG) #define SS_DEBUG #endif -#include +#include struct test_case { diff --git a/server/modules/filter/ccrfilter/ccrfilter.c b/server/modules/filter/ccrfilter/ccrfilter.c index ffcb9777e..e334e9f02 100644 --- a/server/modules/filter/ccrfilter/ccrfilter.c +++ b/server/modules/filter/ccrfilter/ccrfilter.c @@ -11,14 +11,14 @@ * Public License. */ #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include -#include -#include +#include +#include #include #include diff --git a/server/modules/filter/dbfwfilter/dbfwfilter.c b/server/modules/filter/dbfwfilter/dbfwfilter.c index c4baa3a58..1510af39c 100644 --- a/server/modules/filter/dbfwfilter/dbfwfilter.c +++ b/server/modules/filter/dbfwfilter/dbfwfilter.c @@ -62,20 +62,20 @@ #include #include -#include +#include #include -#include -#include -#include -#include +#include +#include +#include +#include #include -#include -#include +#include +#include #include #include #include -#include -#include +#include +#include "dbfwfilter.h" #include #include #include diff --git a/server/modules/filter/dbfwfilter/ruleparser.y b/server/modules/filter/dbfwfilter/ruleparser.y index 96dce952b..d90ab47b4 100644 --- a/server/modules/filter/dbfwfilter/ruleparser.y +++ b/server/modules/filter/dbfwfilter/ruleparser.y @@ -19,8 +19,8 @@ %{ #include -#include -#include +#include "dbfwfilter.h" +#include %} /** We need a reentrant scanner so no global variables are used */ diff --git a/server/modules/filter/gatekeeper/gatekeeper.c b/server/modules/filter/gatekeeper/gatekeeper.c index 35b9a67fa..f51ffa0e6 100644 --- a/server/modules/filter/gatekeeper/gatekeeper.c +++ b/server/modules/filter/gatekeeper/gatekeeper.c @@ -12,13 +12,13 @@ */ #include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include #include #define GK_DEFAULT_HASHTABLE_SIZE 1000 diff --git a/server/modules/filter/hintfilter/hintfilter.c b/server/modules/filter/hintfilter/hintfilter.c index 6669ab64f..87f74bbb3 100644 --- a/server/modules/filter/hintfilter/hintfilter.c +++ b/server/modules/filter/hintfilter/hintfilter.c @@ -11,10 +11,10 @@ * Public License. */ #include -#include +#include #include -#include -#include +#include +#include #include /** diff --git a/server/modules/filter/hintfilter/hintparser.c b/server/modules/filter/hintfilter/hintparser.c index bade7e684..ab72b8ccd 100644 --- a/server/modules/filter/hintfilter/hintparser.c +++ b/server/modules/filter/hintfilter/hintparser.c @@ -13,11 +13,11 @@ #include #include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include #include diff --git a/server/modules/filter/luafilter/luafilter.c b/server/modules/filter/luafilter/luafilter.c index 80f2de37e..941d2f87a 100644 --- a/server/modules/filter/luafilter/luafilter.c +++ b/server/modules/filter/luafilter/luafilter.c @@ -36,14 +36,14 @@ * or diagnostic being made for the session script. */ -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include #include "lua.h" #include "lualib.h" #include "lauxlib.h" diff --git a/server/modules/filter/mqfilter/mqfilter.c b/server/modules/filter/mqfilter/mqfilter.c index 734ab9ad4..71faca490 100644 --- a/server/modules/filter/mqfilter/mqfilter.c +++ b/server/modules/filter/mqfilter/mqfilter.c @@ -59,23 +59,23 @@ #include #include #include -#include -#include -#include +#include +#include +#include #include #include #include -#include +#include #include #include #include #include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include MODULE_INFO info = diff --git a/server/modules/filter/namedserverfilter/namedserverfilter.c b/server/modules/filter/namedserverfilter/namedserverfilter.c index 96b639237..0dcbd662b 100644 --- a/server/modules/filter/namedserverfilter/namedserverfilter.c +++ b/server/modules/filter/namedserverfilter/namedserverfilter.c @@ -13,14 +13,14 @@ #include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include #include -#include +#include #include /** diff --git a/server/modules/filter/qlafilter/qlafilter.c b/server/modules/filter/qlafilter/qlafilter.c index b24a32ed2..eece0cb0f 100644 --- a/server/modules/filter/qlafilter/qlafilter.c +++ b/server/modules/filter/qlafilter/qlafilter.c @@ -37,16 +37,16 @@ #include #include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include #include #include #include -#include +#include #include MODULE_INFO info = diff --git a/server/modules/filter/regexfilter/regexfilter.c b/server/modules/filter/regexfilter/regexfilter.c index de84270dd..c0e2a6ee6 100644 --- a/server/modules/filter/regexfilter/regexfilter.c +++ b/server/modules/filter/regexfilter/regexfilter.c @@ -13,16 +13,16 @@ #define PCRE2_CODE_UNIT_WIDTH 8 #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include #include -#include +#include #include -#include "maxconfig.h" +#include /** * @file regexfilter.c - a very simple regular expression rewrite filter. diff --git a/server/modules/filter/tee/tee.c b/server/modules/filter/tee/tee.c index 81eed1f60..481cd0559 100644 --- a/server/modules/filter/tee/tee.c +++ b/server/modules/filter/tee/tee.c @@ -43,23 +43,23 @@ #include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include #include #include -#include -#include -#include +#include +#include +#include #include #include #include -#include +#include #include -#include +#include #define MYSQL_COM_QUIT 0x01 #define MYSQL_COM_INITDB 0x02 diff --git a/server/modules/filter/testfilter/testfilter.c b/server/modules/filter/testfilter/testfilter.c index 00711b50a..8caa25b6e 100644 --- a/server/modules/filter/testfilter/testfilter.c +++ b/server/modules/filter/testfilter/testfilter.c @@ -11,11 +11,11 @@ * Public License. */ #include -#include +#include #include -#include -#include -#include +#include +#include +#include /** * @file testfilter.c - a very simple test filter. diff --git a/server/modules/filter/topfilter/topfilter.c b/server/modules/filter/topfilter/topfilter.c index d2e7d72c9..4596f381c 100644 --- a/server/modules/filter/topfilter/topfilter.c +++ b/server/modules/filter/topfilter/topfilter.c @@ -34,16 +34,16 @@ #include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include #include #include #include -#include +#include #include MODULE_INFO info = diff --git a/server/modules/filter/tpmfilter/tpmfilter.c b/server/modules/filter/tpmfilter/tpmfilter.c index d17823a0f..d803421a4 100644 --- a/server/modules/filter/tpmfilter/tpmfilter.c +++ b/server/modules/filter/tpmfilter/tpmfilter.c @@ -41,16 +41,16 @@ #include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include #include #include #include -#include +#include /** Defined in log_manager.cc */ extern int lm_enabled_logfiles_bitmask; diff --git a/server/modules/include/avrorouter.h b/server/modules/include/avrorouter.h index ab2f2b88e..83e018f7c 100644 --- a/server/modules/include/avrorouter.h +++ b/server/modules/include/avrorouter.h @@ -20,16 +20,16 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include #include #include -#include +#include #include #include #include diff --git a/server/modules/include/blr.h b/server/modules/include/blr.h index 5f0759a6a..7f423da0a 100644 --- a/server/modules/include/blr.h +++ b/server/modules/include/blr.h @@ -37,12 +37,12 @@ * * @endverbatim */ -#include -#include +#include +#include #include #include -#include -#include +#include +#include #include #include diff --git a/server/modules/include/cdc.h b/server/modules/include/cdc.h index 1702f8ac6..80f0d0c81 100644 --- a/server/modules/include/cdc.h +++ b/server/modules/include/cdc.h @@ -21,19 +21,19 @@ #include #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include #include #include #include #include -#include -#include -#include -#include +#include +#include +#include +#include #define CDC_SMALL_BUFFER 1024 #define CDC_METHOD_MAXLEN 128 diff --git a/server/modules/include/debugcli.h b/server/modules/include/debugcli.h index 1e956e9e1..f736038b3 100644 --- a/server/modules/include/debugcli.h +++ b/server/modules/include/debugcli.h @@ -13,9 +13,9 @@ * of this software will be governed by version 2 or later of the General * Public License. */ -#include -#include -#include +#include +#include +#include /** * @file debugcli.h The debug interface to the gateway diff --git a/server/modules/include/httpd.h b/server/modules/include/httpd.h index 3f8458db8..0bc8096af 100644 --- a/server/modules/include/httpd.h +++ b/server/modules/include/httpd.h @@ -21,19 +21,19 @@ #include #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include #include #include #include #include -#include +#include #include -#include -#include +#include +#include #define HTTPD_SMALL_BUFFER 1024 #define HTTPD_METHOD_MAXLEN 128 diff --git a/server/modules/include/maxinfo.h b/server/modules/include/maxinfo.h index 8c3751fbc..08fa63f24 100644 --- a/server/modules/include/maxinfo.h +++ b/server/modules/include/maxinfo.h @@ -12,9 +12,9 @@ * of this software will be governed by version 2 or later of the General * Public License. */ -#include -#include -#include +#include +#include +#include /** * @file maxinfo.h The MaxScale information schema provider diff --git a/server/modules/include/maxscaled.h b/server/modules/include/maxscaled.h index 660bd37db..8dff6aedb 100644 --- a/server/modules/include/maxscaled.h +++ b/server/modules/include/maxscaled.h @@ -24,9 +24,9 @@ * * @endverbatim */ -#include -#include -#include +#include +#include +#include /** * The maxscaled specific protocol structure to put in the DCB. */ diff --git a/server/modules/include/mysql_auth.h b/server/modules/include/mysql_auth.h index 7f0192c69..bd02dea59 100644 --- a/server/modules/include/mysql_auth.h +++ b/server/modules/include/mysql_auth.h @@ -23,8 +23,8 @@ * @endverbatim */ -#include -#include +#include +#include #include #include diff --git a/server/modules/include/mysql_client_server_protocol.h b/server/modules/include/mysql_client_server_protocol.h index 2ef9ad71d..45a18f959 100644 --- a/server/modules/include/mysql_client_server_protocol.h +++ b/server/modules/include/mysql_client_server_protocol.h @@ -54,13 +54,13 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include #include #define GW_MYSQL_VERSION "5.5.5-10.0.0 " MAXSCALE_VERSION "-maxscale" diff --git a/server/modules/include/mysqlhint.h b/server/modules/include/mysqlhint.h index b6dfcc538..85c9739fc 100644 --- a/server/modules/include/mysqlhint.h +++ b/server/modules/include/mysqlhint.h @@ -19,7 +19,7 @@ * Date Who Description * 17-07-2014 Mark Riddoch Initial implementation */ -#include +#include /* Parser tokens for the hint parser */ typedef enum diff --git a/server/modules/include/readconnection.h b/server/modules/include/readconnection.h index 7d7c56f4b..3fcbcb5d9 100644 --- a/server/modules/include/readconnection.h +++ b/server/modules/include/readconnection.h @@ -25,7 +25,7 @@ * * @endverbatim */ -#include +#include /** * Internal structure used to define the set of backend servers we are routing diff --git a/server/modules/include/readwritesplit.h b/server/modules/include/readwritesplit.h index d309f2f48..3787155c0 100644 --- a/server/modules/include/readwritesplit.h +++ b/server/modules/include/readwritesplit.h @@ -24,8 +24,8 @@ * @endverbatim */ -#include -#include +#include +#include #include #undef PREP_STMT_CACHING diff --git a/server/modules/include/rwsplit_internal.h b/server/modules/include/rwsplit_internal.h index 9fadc8982..c090914a9 100644 --- a/server/modules/include/rwsplit_internal.h +++ b/server/modules/include/rwsplit_internal.h @@ -25,7 +25,7 @@ extern "C" { #endif -#include +#include /* This needs to be removed along with dependency on it - see the * rwsplit_tmp_table_multi functions diff --git a/server/modules/include/schemarouter.h b/server/modules/include/schemarouter.h index 218499865..914d11960 100644 --- a/server/modules/include/schemarouter.h +++ b/server/modules/include/schemarouter.h @@ -27,8 +27,8 @@ #define PCRE2_CODE_UNIT_WIDTH 8 #endif -#include -#include +#include +#include #include #include /** diff --git a/server/modules/include/sharding_common.h b/server/modules/include/sharding_common.h index b676b31d6..0952f6a30 100644 --- a/server/modules/include/sharding_common.h +++ b/server/modules/include/sharding_common.h @@ -16,12 +16,12 @@ #include #include -#include -#include +#include +#include #include -#include -#include -#include +#include +#include +#include bool extract_database(GWBUF* buf, char* str); void create_error_reply(char* fail_str, DCB* dcb); diff --git a/server/modules/include/telnetd.h b/server/modules/include/telnetd.h index 4b6af5fda..eef8fc4bb 100644 --- a/server/modules/include/telnetd.h +++ b/server/modules/include/telnetd.h @@ -24,8 +24,8 @@ * * @endverbatim */ -#include -#include +#include +#include /** * The telnetd specific protocol structure to put in the DCB. */ diff --git a/server/modules/monitor/auroramon/auroramon.c b/server/modules/monitor/auroramon/auroramon.c index 9095d5d0e..784a7bd96 100644 --- a/server/modules/monitor/auroramon/auroramon.c +++ b/server/modules/monitor/auroramon/auroramon.c @@ -15,12 +15,12 @@ * @file auroramon.c - Amazon RDS Aurora monitor */ -#include -#include -#include +#include +#include +#include #include #include -#include +#include static char *version_str = (char*)"V1.0.0"; diff --git a/server/modules/monitor/galeramon/galeramon.c b/server/modules/monitor/galeramon/galeramon.c index cc1f44a98..781b7e891 100644 --- a/server/modules/monitor/galeramon/galeramon.c +++ b/server/modules/monitor/galeramon/galeramon.c @@ -35,7 +35,7 @@ #include "galeramon.h" -#include +#include #include static void monitorMain(void *); diff --git a/server/modules/monitor/galeramon/galeramon.h b/server/modules/monitor/galeramon/galeramon.h index 15ebe5a26..9fdcd8b27 100644 --- a/server/modules/monitor/galeramon/galeramon.h +++ b/server/modules/monitor/galeramon/galeramon.h @@ -16,18 +16,18 @@ #include #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include #include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include /** * @file galeramon.h - The Galera cluster monitor diff --git a/server/modules/monitor/mmmon/mmmon.c b/server/modules/monitor/mmmon/mmmon.c index a3809f42b..08197cc10 100644 --- a/server/modules/monitor/mmmon/mmmon.c +++ b/server/modules/monitor/mmmon/mmmon.c @@ -26,7 +26,7 @@ */ #include "mmmon.h" -#include +#include #include static void monitorMain(void *); diff --git a/server/modules/monitor/mmmon/mmmon.h b/server/modules/monitor/mmmon/mmmon.h index aaa6b2ca7..df3747c9a 100644 --- a/server/modules/monitor/mmmon/mmmon.h +++ b/server/modules/monitor/mmmon/mmmon.h @@ -16,18 +16,18 @@ #include #include #include -#include -#include -#include +#include +#include +#include #include #include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include /** * @file mmmon.h - The Multi-Master monitor diff --git a/server/modules/monitor/mysqlmon.h b/server/modules/monitor/mysqlmon.h index efbd140f3..b49cb8ba0 100644 --- a/server/modules/monitor/mysqlmon.h +++ b/server/modules/monitor/mysqlmon.h @@ -16,19 +16,19 @@ #include #include #include -#include -#include -#include +#include +#include +#include #include #include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include /** * @file mysqlmon.h - The MySQL monitor diff --git a/server/modules/monitor/mysqlmon/mysql_mon.c b/server/modules/monitor/mysqlmon/mysql_mon.c index f454ae84f..901238f44 100644 --- a/server/modules/monitor/mysqlmon/mysql_mon.c +++ b/server/modules/monitor/mysqlmon/mysql_mon.c @@ -46,8 +46,8 @@ */ #include "../mysqlmon.h" -#include -#include +#include +#include #include /** Column positions for SHOW SLAVE STATUS */ diff --git a/server/modules/protocol/CDC/cdc.c b/server/modules/protocol/CDC/cdc.c index c2d0b1897..43471caad 100644 --- a/server/modules/protocol/CDC/cdc.c +++ b/server/modules/protocol/CDC/cdc.c @@ -32,11 +32,11 @@ #include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include MODULE_INFO info = diff --git a/server/modules/protocol/HTTPD/httpd.c b/server/modules/protocol/HTTPD/httpd.c index 8430b90ae..c584fd27d 100644 --- a/server/modules/protocol/HTTPD/httpd.c +++ b/server/modules/protocol/HTTPD/httpd.c @@ -34,11 +34,11 @@ #include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include /* @see function load_module in load_utils.c for explanation of the following * lint directives. diff --git a/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c b/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c index 5f118af6a..598f71330 100644 --- a/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c +++ b/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c @@ -11,14 +11,14 @@ * Public License. */ -#include "mysql_client_server_protocol.h" -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include #include -#include +#include #include /* @@ -50,8 +50,8 @@ * */ #include -#include -#include +#include +#include #include /* @see function load_module in load_utils.c for explanation of the following diff --git a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c index d61c1c05d..10e76bf0d 100644 --- a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c +++ b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c @@ -45,21 +45,21 @@ * 07/02/2016 Martin Brampton Split off authentication and SSL. * 31/05/2016 Martin Brampton Implement connection throttling */ -#include -#include +#include +#include #include -#include +#include #include #include -#include +#include #include -#include -#include +#include +#include #include -#include +#include #include -#include "gw_authenticator.h" +#include /* @see function load_module in load_utils.c for explanation of the following * lint directives. diff --git a/server/modules/protocol/MySQL/mysql_common.c b/server/modules/protocol/MySQL/mysql_common.c index 834256fe4..0e16ef927 100644 --- a/server/modules/protocol/MySQL/mysql_common.c +++ b/server/modules/protocol/MySQL/mysql_common.c @@ -42,15 +42,15 @@ * */ -#include -#include +#include +#include #include #include -#include -#include -#include +#include +#include +#include #include -#include +#include uint8_t null_client_sha1[MYSQL_SCRAMBLE_LEN] = ""; diff --git a/server/modules/protocol/maxscaled/maxscaled.c b/server/modules/protocol/maxscaled/maxscaled.c index eb9d99169..75f2c660e 100644 --- a/server/modules/protocol/maxscaled/maxscaled.c +++ b/server/modules/protocol/maxscaled/maxscaled.c @@ -14,25 +14,25 @@ #include #include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include #include #include #include #include #include -#include +#include #include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include #include #include #include diff --git a/server/modules/protocol/telnetd/telnetd.c b/server/modules/protocol/telnetd/telnetd.c index 9d94335f2..f0f3a4ddd 100644 --- a/server/modules/protocol/telnetd/telnetd.c +++ b/server/modules/protocol/telnetd/telnetd.c @@ -15,25 +15,25 @@ #include #include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include #include #include #include #include -#include +#include #include -#include -#include +#include +#include #include -#include -#include -#include -#include +#include +#include +#include +#include /* @see function load_module in load_utils.c for explanation of the following * lint directives. diff --git a/server/modules/protocol/testprotocol/testprotocol.c b/server/modules/protocol/testprotocol/testprotocol.c index 78bf70c76..0ba45d825 100644 --- a/server/modules/protocol/testprotocol/testprotocol.c +++ b/server/modules/protocol/testprotocol/testprotocol.c @@ -25,10 +25,10 @@ * @endverbatim */ -#include -#include -#include -#include +#include +#include +#include +#include /* @see function load_module in load_utils.c for explanation of the following * lint directives. diff --git a/server/modules/routing/avro/avro.c b/server/modules/routing/avro/avro.c index b5944b640..c580c574e 100644 --- a/server/modules/routing/avro/avro.c +++ b/server/modules/routing/avro/avro.c @@ -29,26 +29,26 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include #include -#include -#include -#include +#include +#include +#include #include #include #include #include -#include +#include #include #include #include diff --git a/server/modules/routing/avro/avro_client.c b/server/modules/routing/avro/avro_client.c index ade20b539..505be1ec4 100644 --- a/server/modules/routing/avro/avro_client.c +++ b/server/modules/routing/avro/avro_client.c @@ -28,21 +28,21 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include -#include +#include extern char *blr_extract_column(GWBUF *buf, int col); extern uint32_t extract_field(uint8_t *src, int bits); diff --git a/server/modules/routing/avro/avro_file.c b/server/modules/routing/avro/avro_file.c index bdd4a5882..05f4b020b 100644 --- a/server/modules/routing/avro/avro_file.c +++ b/server/modules/routing/avro/avro_file.c @@ -34,8 +34,8 @@ #include #include #include -#include -#include +#include +#include #include #include #include diff --git a/server/modules/routing/avro/avro_index.c b/server/modules/routing/avro/avro_index.c index 9b88a3a3f..99da0cab2 100644 --- a/server/modules/routing/avro/avro_index.c +++ b/server/modules/routing/avro/avro_index.c @@ -33,7 +33,7 @@ */ #include -#include +#include #include void* safe_key_free(void *data); diff --git a/server/modules/routing/avro/avro_rbr.c b/server/modules/routing/avro/avro_rbr.c index da8f180da..27f68aa98 100644 --- a/server/modules/routing/avro/avro_rbr.c +++ b/server/modules/routing/avro/avro_rbr.c @@ -12,7 +12,7 @@ */ -#include +#include #include #include #include diff --git a/server/modules/routing/avro/avro_schema.c b/server/modules/routing/avro/avro_schema.c index a15c7f450..e7244fa4c 100644 --- a/server/modules/routing/avro/avro_schema.c +++ b/server/modules/routing/avro/avro_schema.c @@ -19,16 +19,16 @@ #endif #include -#include +#include #include #include #include #include -#include +#include #include #include -#include -#include +#include +#include #include #include #include diff --git a/server/modules/routing/binlog/binlog_common.c b/server/modules/routing/binlog/binlog_common.c index 11cf6a0c3..f17929fcf 100644 --- a/server/modules/routing/binlog/binlog_common.c +++ b/server/modules/routing/binlog/binlog_common.c @@ -18,7 +18,7 @@ #include #include #include -#include +#include /** * @file binlog_common.c - Common binary log code shared between multiple modules diff --git a/server/modules/routing/binlog/blr.c b/server/modules/routing/binlog/blr.c index d6dd22c2d..43f0facea 100644 --- a/server/modules/routing/binlog/blr.c +++ b/server/modules/routing/binlog/blr.c @@ -58,29 +58,29 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include #include -#include -#include -#include +#include +#include +#include #include -#include -#include -#include +#include +#include +#include #include #include #include #include #include -#include +#include static char *version_str = "V2.1.0"; diff --git a/server/modules/routing/binlog/blr_cache.c b/server/modules/routing/binlog/blr_cache.c index 08758ed72..2f8848ab6 100644 --- a/server/modules/routing/binlog/blr_cache.c +++ b/server/modules/routing/binlog/blr_cache.c @@ -34,18 +34,18 @@ #include #include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include -#include -#include +#include +#include -#include -#include -#include +#include +#include +#include /** diff --git a/server/modules/routing/binlog/blr_file.c b/server/modules/routing/binlog/blr_file.c index 7c353212c..14e259036 100644 --- a/server/modules/routing/binlog/blr_file.c +++ b/server/modules/routing/binlog/blr_file.c @@ -47,18 +47,18 @@ #include #include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include #include static int blr_file_create(ROUTER_INSTANCE *router, char *file); diff --git a/server/modules/routing/binlog/blr_master.c b/server/modules/routing/binlog/blr_master.c index b7edb5cb9..cc33bee65 100644 --- a/server/modules/routing/binlog/blr_master.c +++ b/server/modules/routing/binlog/blr_master.c @@ -55,26 +55,26 @@ #include #include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include -#include -#include -#include -#include +#include +#include +#include +#include #include #include -#include -#include -#include +#include +#include +#include -#include -#include +#include +#include /* Temporary requirement for auth data */ #include diff --git a/server/modules/routing/binlog/blr_slave.c b/server/modules/routing/binlog/blr_slave.c index 6a6c72798..d319d8514 100644 --- a/server/modules/routing/binlog/blr_slave.c +++ b/server/modules/routing/binlog/blr_slave.c @@ -68,24 +68,24 @@ #include #include #include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include #include -#include -#include -#include +#include +#include +#include #include -#include -#include -#include -#include +#include +#include +#include +#include #include #include -#include +#include extern int load_mysql_users(SERV_LISTENER *listener); extern void blr_master_close(ROUTER_INSTANCE* router); diff --git a/server/modules/routing/binlog/maxbinlogcheck.c b/server/modules/routing/binlog/maxbinlogcheck.c index 4b2007358..4c926ce75 100644 --- a/server/modules/routing/binlog/maxbinlogcheck.c +++ b/server/modules/routing/binlog/maxbinlogcheck.c @@ -38,7 +38,7 @@ #include #include -#include +#include #include diff --git a/server/modules/routing/binlog/test/testbinlog.c b/server/modules/routing/binlog/test/testbinlog.c index a5bbe5897..aaf060cd0 100644 --- a/server/modules/routing/binlog/test/testbinlog.c +++ b/server/modules/routing/binlog/test/testbinlog.c @@ -26,20 +26,20 @@ #include #include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include -#include -#include -#include +#include +#include +#include #include -#include -#include -#include -#include +#include +#include +#include +#include #include #include @@ -47,7 +47,7 @@ #include #include -#include +#include static void printVersion(const char *progname); static void printUsage(const char *progname); diff --git a/server/modules/routing/cli/cli.c b/server/modules/routing/cli/cli.c index 154ffff92..e5a88e759 100644 --- a/server/modules/routing/cli/cli.c +++ b/server/modules/routing/cli/cli.c @@ -27,19 +27,19 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include -#include -#include +#include +#include MODULE_INFO info = diff --git a/server/modules/routing/debugcli/debugcli.c b/server/modules/routing/debugcli/debugcli.c index 4e143a278..26208b328 100644 --- a/server/modules/routing/debugcli/debugcli.c +++ b/server/modules/routing/debugcli/debugcli.c @@ -26,19 +26,19 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include -#include -#include +#include +#include MODULE_INFO info = diff --git a/server/modules/routing/debugcli/debugcmd.c b/server/modules/routing/debugcli/debugcmd.c index 878047cd4..93bb67d8b 100644 --- a/server/modules/routing/debugcli/debugcmd.c +++ b/server/modules/routing/debugcli/debugcmd.c @@ -51,29 +51,29 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include -#include -#include -#include +#include +#include +#include #include -#include -#include +#include +#include #include -#include -#include +#include +#include -#include -#include +#include +#include #include #define MAXARGS 6 diff --git a/server/modules/routing/maxinfo/maxinfo.c b/server/modules/routing/maxinfo/maxinfo.c index 4be542584..21b1a391b 100644 --- a/server/modules/routing/maxinfo/maxinfo.c +++ b/server/modules/routing/maxinfo/maxinfo.c @@ -30,28 +30,28 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include MODULE_INFO info = diff --git a/server/modules/routing/maxinfo/maxinfo_error.c b/server/modules/routing/maxinfo/maxinfo_error.c index f8b813dd1..1dd4c3087 100644 --- a/server/modules/routing/maxinfo/maxinfo_error.c +++ b/server/modules/routing/maxinfo/maxinfo_error.c @@ -26,19 +26,19 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include -#include -#include +#include +#include /** diff --git a/server/modules/routing/maxinfo/maxinfo_exec.c b/server/modules/routing/maxinfo/maxinfo_exec.c index 6fe57476c..bdbfa3d47 100644 --- a/server/modules/routing/maxinfo/maxinfo_exec.c +++ b/server/modules/routing/maxinfo/maxinfo_exec.c @@ -27,24 +27,24 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include -#include -#include -#include -#include +#include +#include +#include +#include static void exec_show(DCB *dcb, MAXINFO_TREE *tree); static void exec_select(DCB *dcb, MAXINFO_TREE *tree); diff --git a/server/modules/routing/maxinfo/maxinfo_parse.c b/server/modules/routing/maxinfo/maxinfo_parse.c index 1a1f415bb..c80b2323c 100644 --- a/server/modules/routing/maxinfo/maxinfo_parse.c +++ b/server/modules/routing/maxinfo/maxinfo_parse.c @@ -27,19 +27,19 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include -#include -#include +#include +#include static MAXINFO_TREE *make_tree_node(MAXINFO_OPERATOR, char *, MAXINFO_TREE *, MAXINFO_TREE *); static void free_tree(MAXINFO_TREE *); diff --git a/server/modules/routing/readconnroute/readconnroute.c b/server/modules/routing/readconnroute/readconnroute.c index 37217b063..d15269635 100644 --- a/server/modules/routing/readconnroute/readconnroute.c +++ b/server/modules/routing/readconnroute/readconnroute.c @@ -74,23 +74,23 @@ #include #include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include -#include -#include -#include +#include +#include +#include -#include -#include -#include +#include +#include +#include #include -#include "modutil.h" +#include MODULE_INFO info = { diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index 83e8279b2..d9437339b 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -16,17 +16,17 @@ #include #include -#include +#include #include #include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include #include MODULE_INFO info = diff --git a/server/modules/routing/readwritesplit/rwsplit_mysql.c b/server/modules/routing/readwritesplit/rwsplit_mysql.c index 18c1f5a73..72228395a 100644 --- a/server/modules/routing/readwritesplit/rwsplit_mysql.c +++ b/server/modules/routing/readwritesplit/rwsplit_mysql.c @@ -17,18 +17,18 @@ #include #include -#include +#include #include #include #include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include #include #include #include diff --git a/server/modules/routing/readwritesplit/rwsplit_route_stmt.c b/server/modules/routing/readwritesplit/rwsplit_route_stmt.c index 2e2ca2ab1..bbf690b14 100644 --- a/server/modules/routing/readwritesplit/rwsplit_route_stmt.c +++ b/server/modules/routing/readwritesplit/rwsplit_route_stmt.c @@ -17,7 +17,7 @@ #include #include -#include +#include #include #include /** diff --git a/server/modules/routing/readwritesplit/rwsplit_select_backends.c b/server/modules/routing/readwritesplit/rwsplit_select_backends.c index dbf50da08..114e118fb 100644 --- a/server/modules/routing/readwritesplit/rwsplit_select_backends.c +++ b/server/modules/routing/readwritesplit/rwsplit_select_backends.c @@ -16,7 +16,7 @@ #include #include -#include +#include #include #include /** diff --git a/server/modules/routing/readwritesplit/rwsplit_session_cmd.c b/server/modules/routing/readwritesplit/rwsplit_session_cmd.c index d66e7f66f..ddfc0987d 100644 --- a/server/modules/routing/readwritesplit/rwsplit_session_cmd.c +++ b/server/modules/routing/readwritesplit/rwsplit_session_cmd.c @@ -16,7 +16,7 @@ #include #include -#include +#include #include #include diff --git a/server/modules/routing/readwritesplit/rwsplit_tmp_table_multi.c b/server/modules/routing/readwritesplit/rwsplit_tmp_table_multi.c index 04177f317..41d9e8526 100644 --- a/server/modules/routing/readwritesplit/rwsplit_tmp_table_multi.c +++ b/server/modules/routing/readwritesplit/rwsplit_tmp_table_multi.c @@ -18,9 +18,9 @@ #include /* Note that modutil contains much MySQL specific code */ -#include +#include -#include +#include #include #include /** diff --git a/server/modules/routing/schemarouter/schemarouter.c b/server/modules/routing/schemarouter/schemarouter.c index c7d7c8eb4..35882d9f9 100644 --- a/server/modules/routing/schemarouter/schemarouter.c +++ b/server/modules/routing/schemarouter/schemarouter.c @@ -16,18 +16,18 @@ #include #include #include -#include +#include #include #include -#include +#include #include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include #include #include #include diff --git a/server/modules/routing/testroute/testroute.c b/server/modules/routing/testroute/testroute.c index a553fe245..3fb823ec1 100644 --- a/server/modules/routing/testroute/testroute.c +++ b/server/modules/routing/testroute/testroute.c @@ -12,8 +12,8 @@ */ #include #include -#include -#include +#include +#include static char *version_str = "V1.0.0"; From 840575d1dc2f74f3d7c76833ba5cf3b7cedfca24 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 13 Oct 2016 16:28:57 +0300 Subject: [PATCH 017/215] MXS-862: Add FindGSSAPI.cmake and missing includes Added FindGSSAPI.cmake which allows the modules to be built only if the libraries are found. The log manager header was not included by the GSSAPI modules. --- CMakeLists.txt | 1 + cmake/FindGSSAPI.cmake | 98 +++++++++++++++++++ server/modules/authenticator/CMakeLists.txt | 20 ++-- server/modules/authenticator/gssapi_auth.c | 1 + .../authenticator/gssapi_auth_common.c | 1 + .../authenticator/gssapi_backend_auth.c | 1 + 6 files changed, 114 insertions(+), 8 deletions(-) create mode 100644 cmake/FindGSSAPI.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 745459863..c8f5a14a4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,6 +42,7 @@ find_package(CURL) find_package(RabbitMQ) find_package(LibUUID) find_package(Avro) +find_package(GSSAPI) # Find or build PCRE2 # Read BuildPCRE2 for details about how to add pcre2 as a dependency to a target diff --git a/cmake/FindGSSAPI.cmake b/cmake/FindGSSAPI.cmake new file mode 100644 index 000000000..d4323f839 --- /dev/null +++ b/cmake/FindGSSAPI.cmake @@ -0,0 +1,98 @@ +# - Try to detect the GSSAPI support +# Once done this will define +# +# GSSAPI_FOUND - system supports GSSAPI +# GSSAPI_INCS - the GSSAPI include directory +# GSSAPI_LIBS - the libraries needed to use GSSAPI +# GSSAPI_FLAVOR - the type of API - MIT or HEIMDAL + +# Copyright (c) 2006, Pino Toscano, +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. The name of the author may not be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +if(GSSAPI_LIBS AND GSSAPI_FLAVOR) + + # in cache already + set(GSSAPI_FOUND TRUE) + +else(GSSAPI_LIBS AND GSSAPI_FLAVOR) + + find_program(KRB5_CONFIG NAMES krb5-config heimdal-krb5-config PATHS + /opt/local/bin + ONLY_CMAKE_FIND_ROOT_PATH # this is required when cross compiling with cmake 2.6 and ignored with cmake 2.4, Alex + ) + mark_as_advanced(KRB5_CONFIG) + + #reset vars + set(GSSAPI_INCS) + set(GSSAPI_LIBS) + set(GSSAPI_FLAVOR) + + if(KRB5_CONFIG) + + set(HAVE_KRB5_GSSAPI TRUE) + exec_program(${KRB5_CONFIG} ARGS --libs gssapi RETURN_VALUE _return_VALUE OUTPUT_VARIABLE GSSAPI_LIBS) + if(_return_VALUE) + message(STATUS "GSSAPI configure check failed.") + set(HAVE_KRB5_GSSAPI FALSE) + endif(_return_VALUE) + + exec_program(${KRB5_CONFIG} ARGS --cflags gssapi RETURN_VALUE _return_VALUE OUTPUT_VARIABLE GSSAPI_INCS) + string(REGEX REPLACE "(\r?\n)+$" "" GSSAPI_INCS "${GSSAPI_INCS}") + string(REGEX REPLACE " *-I" ";" GSSAPI_INCS "${GSSAPI_INCS}") + + exec_program(${KRB5_CONFIG} ARGS --vendor RETURN_VALUE _return_VALUE OUTPUT_VARIABLE gssapi_flavor_tmp) + set(GSSAPI_FLAVOR_MIT) + if(gssapi_flavor_tmp MATCHES ".*Massachusetts.*") + set(GSSAPI_FLAVOR "MIT") + else(gssapi_flavor_tmp MATCHES ".*Massachusetts.*") + set(GSSAPI_FLAVOR "HEIMDAL") + endif(gssapi_flavor_tmp MATCHES ".*Massachusetts.*") + + if(NOT HAVE_KRB5_GSSAPI) + if (gssapi_flavor_tmp MATCHES "Sun Microsystems.*") + message(STATUS "Solaris Kerberos does not have GSSAPI; this is normal.") + set(GSSAPI_LIBS) + set(GSSAPI_INCS) + else(gssapi_flavor_tmp MATCHES "Sun Microsystems.*") + message(WARNING "${KRB5_CONFIG} failed unexpectedly.") + endif(gssapi_flavor_tmp MATCHES "Sun Microsystems.*") + endif(NOT HAVE_KRB5_GSSAPI) + + if(GSSAPI_LIBS) # GSSAPI_INCS can be also empty, so don't rely on that + set(GSSAPI_FOUND TRUE CACHE STRING "") + message(STATUS "Found GSSAPI: ${GSSAPI_LIBS}") + + set(GSSAPI_INCS ${GSSAPI_INCS} CACHE STRING "") + set(GSSAPI_LIBS ${GSSAPI_LIBS} CACHE STRING "") + set(GSSAPI_FLAVOR ${GSSAPI_FLAVOR} CACHE STRING "") + + mark_as_advanced(GSSAPI_INCS GSSAPI_LIBS GSSAPI_FLAVOR) + + endif(GSSAPI_LIBS) + + endif(KRB5_CONFIG) + +endif(GSSAPI_LIBS AND GSSAPI_FLAVOR) diff --git a/server/modules/authenticator/CMakeLists.txt b/server/modules/authenticator/CMakeLists.txt index c8928b726..a7f6da3e1 100644 --- a/server/modules/authenticator/CMakeLists.txt +++ b/server/modules/authenticator/CMakeLists.txt @@ -8,15 +8,19 @@ target_link_libraries(MySQLBackendAuth maxscale-common MySQLCommon) set_target_properties(MySQLBackendAuth PROPERTIES VERSION "1.0.0") install_module(MySQLBackendAuth core) -add_library(GSSAPIAuth SHARED gssapi_auth.c gssapi_auth_common.c) -target_link_libraries(GSSAPIAuth maxscale-common gssapi_krb5 MySQLCommon) -set_target_properties(GSSAPIAuth PROPERTIES VERSION "1.0.0") -install_module(GSSAPIAuth core) +if (GSSAPI_FOUND) + include_directories(${GSSAPI_INCS}) -add_library(GSSAPIBackendAuth SHARED gssapi_backend_auth.c gssapi_auth_common.c) -target_link_libraries(GSSAPIBackendAuth maxscale-common gssapi_krb5 MySQLCommon) -set_target_properties(GSSAPIBackendAuth PROPERTIES VERSION "1.0.0") -install_module(GSSAPIBackendAuth core) + add_library(GSSAPIAuth SHARED gssapi_auth.c gssapi_auth_common.c) + target_link_libraries(GSSAPIAuth maxscale-common ${GSSAPI_LIBS} MySQLCommon) + set_target_properties(GSSAPIAuth PROPERTIES VERSION "1.0.0") + install_module(GSSAPIAuth core) + + add_library(GSSAPIBackendAuth SHARED gssapi_backend_auth.c gssapi_auth_common.c) + target_link_libraries(GSSAPIBackendAuth maxscale-common ${GSSAPI_LIBS} MySQLCommon) + set_target_properties(GSSAPIBackendAuth PROPERTIES VERSION "1.0.0") + install_module(GSSAPIBackendAuth core) +endif() add_library(NullAuthAllow SHARED null_auth_allow.c) target_link_libraries(NullAuthAllow maxscale-common) diff --git a/server/modules/authenticator/gssapi_auth.c b/server/modules/authenticator/gssapi_auth.c index 90abc0f33..6b94c1b94 100644 --- a/server/modules/authenticator/gssapi_auth.c +++ b/server/modules/authenticator/gssapi_auth.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "gssapi_auth.h" diff --git a/server/modules/authenticator/gssapi_auth_common.c b/server/modules/authenticator/gssapi_auth_common.c index 60815288e..8fa64627a 100644 --- a/server/modules/authenticator/gssapi_auth_common.c +++ b/server/modules/authenticator/gssapi_auth_common.c @@ -13,6 +13,7 @@ #include "gssapi_auth.h" #include +#include void* gssapi_auth_alloc(void *instance) { diff --git a/server/modules/authenticator/gssapi_backend_auth.c b/server/modules/authenticator/gssapi_backend_auth.c index d1b7903bc..cd1a0a6e1 100644 --- a/server/modules/authenticator/gssapi_backend_auth.c +++ b/server/modules/authenticator/gssapi_backend_auth.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "gssapi_auth.h" From 1895c04fc627a31c452e5bb5f77d5c1538c9979e Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 13 Oct 2016 22:33:03 +0300 Subject: [PATCH 018/215] Fix luafilter build failure The luafilter included a non-existent header file when it needed . --- server/modules/filter/luafilter/luafilter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/modules/filter/luafilter/luafilter.c b/server/modules/filter/luafilter/luafilter.c index 941d2f87a..47f274326 100644 --- a/server/modules/filter/luafilter/luafilter.c +++ b/server/modules/filter/luafilter/luafilter.c @@ -40,7 +40,7 @@ #include #include #include -#include +#include #include #include #include From c433d11b39ba77e0409638cd3400fdce4ad23b97 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Thu, 13 Oct 2016 18:44:16 +0300 Subject: [PATCH 019/215] Module private headers moved to modules --- client/maxadmin.c | 2 +- .../include => include/maxscale}/maxadmin.h | 0 server/modules/filter/hintfilter/hintfilter.c | 2 +- server/modules/filter/hintfilter/hintparser.c | 2 +- .../hintfilter}/mysqlhint.h | 0 server/modules/include/shardrouter.h | 258 ------------------ server/modules/protocol/HTTPD/httpd.c | 2 +- .../{include => protocol/HTTPD}/httpd.h | 0 server/modules/protocol/maxscaled/maxscaled.c | 4 +- .../maxscaled}/maxscaled.h | 0 server/modules/routing/avro/avro.c | 2 +- server/modules/routing/avro/avro_client.c | 2 +- server/modules/routing/avro/avro_file.c | 2 +- server/modules/routing/avro/avro_index.c | 2 +- server/modules/routing/avro/avro_rbr.c | 2 +- server/modules/routing/avro/avro_schema.c | 2 +- .../{include => routing/avro}/avrorouter.h | 0 server/modules/routing/binlog/blr.c | 2 +- .../modules/{include => routing/binlog}/blr.h | 0 server/modules/routing/binlog/blr_cache.c | 2 +- server/modules/routing/binlog/blr_file.c | 2 +- server/modules/routing/binlog/blr_master.c | 2 +- server/modules/routing/binlog/blr_slave.c | 2 +- .../modules/routing/binlog/maxbinlogcheck.c | 2 +- .../modules/routing/binlog/test/testbinlog.c | 2 +- server/modules/routing/maxinfo/maxinfo.c | 2 +- .../{include => routing/maxinfo}/maxinfo.h | 0 .../modules/routing/maxinfo/maxinfo_error.c | 2 +- server/modules/routing/maxinfo/maxinfo_exec.c | 2 +- .../modules/routing/maxinfo/maxinfo_parse.c | 2 +- .../readconnroute}/readconnection.h | 0 .../routing/readconnroute/readconnroute.c | 2 +- .../routing/readwritesplit/readwritesplit.c | 6 +- .../readwritesplit}/readwritesplit.h | 0 .../readwritesplit}/rwsplit_internal.h | 0 .../routing/readwritesplit/rwsplit_mysql.c | 6 +- .../readwritesplit/rwsplit_route_stmt.c | 6 +- .../readwritesplit/rwsplit_select_backends.c | 6 +- .../readwritesplit/rwsplit_session_cmd.c | 6 +- .../readwritesplit/rwsplit_tmp_table_multi.c | 6 +- .../routing/schemarouter/schemarouter.c | 4 +- .../schemarouter}/schemarouter.h | 0 .../routing/schemarouter/sharding_common.c | 2 +- .../schemarouter}/sharding_common.h | 0 server/modules/routing/schemarouter/svcconn.c | 73 ----- 45 files changed, 45 insertions(+), 376 deletions(-) rename {server/modules/include => include/maxscale}/maxadmin.h (100%) rename server/modules/{include => filter/hintfilter}/mysqlhint.h (100%) delete mode 100644 server/modules/include/shardrouter.h rename server/modules/{include => protocol/HTTPD}/httpd.h (100%) rename server/modules/{include => protocol/maxscaled}/maxscaled.h (100%) rename server/modules/{include => routing/avro}/avrorouter.h (100%) rename server/modules/{include => routing/binlog}/blr.h (100%) rename server/modules/{include => routing/maxinfo}/maxinfo.h (100%) rename server/modules/{include => routing/readconnroute}/readconnection.h (100%) rename server/modules/{include => routing/readwritesplit}/readwritesplit.h (100%) rename server/modules/{include => routing/readwritesplit}/rwsplit_internal.h (100%) rename server/modules/{include => routing/schemarouter}/schemarouter.h (100%) rename server/modules/{include => routing/schemarouter}/sharding_common.h (100%) delete mode 100644 server/modules/routing/schemarouter/svcconn.c diff --git a/client/maxadmin.c b/client/maxadmin.c index 32b1a6b3f..13c92de7e 100644 --- a/client/maxadmin.c +++ b/client/maxadmin.c @@ -54,7 +54,7 @@ #include #endif -#include +#include /* * We need a common.h file that is included by every component. */ diff --git a/server/modules/include/maxadmin.h b/include/maxscale/maxadmin.h similarity index 100% rename from server/modules/include/maxadmin.h rename to include/maxscale/maxadmin.h diff --git a/server/modules/filter/hintfilter/hintfilter.c b/server/modules/filter/hintfilter/hintfilter.c index 87f74bbb3..c7efaabaa 100644 --- a/server/modules/filter/hintfilter/hintfilter.c +++ b/server/modules/filter/hintfilter/hintfilter.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include "mysqlhint.h" /** * hintfilter.c - a filter to parse the MaxScale hint syntax and attach those diff --git a/server/modules/filter/hintfilter/hintparser.c b/server/modules/filter/hintfilter/hintparser.c index ab72b8ccd..bfc721a3a 100644 --- a/server/modules/filter/hintfilter/hintparser.c +++ b/server/modules/filter/hintfilter/hintparser.c @@ -18,7 +18,7 @@ #include #include #include -#include +#include "mysqlhint.h" #include /** diff --git a/server/modules/include/mysqlhint.h b/server/modules/filter/hintfilter/mysqlhint.h similarity index 100% rename from server/modules/include/mysqlhint.h rename to server/modules/filter/hintfilter/mysqlhint.h diff --git a/server/modules/include/shardrouter.h b/server/modules/include/shardrouter.h deleted file mode 100644 index a57aa1603..000000000 --- a/server/modules/include/shardrouter.h +++ /dev/null @@ -1,258 +0,0 @@ -#ifndef _SHARDROUTER_H -#define _SHARDROUTER_H -/* - * - * Copyright (c) 2016 MariaDB Corporation Ab - * - * Use of this software is governed by the Business Source License included - * in the LICENSE.TXT file and at www.mariadb.com/bsl. - * - * Change Date: 2019-07-01 - * - * On the date above, in accordance with the Business Source License, use - * of this software will be governed by version 2 or later of the General - * Public License. - */ - -/** - * @file shardrouter.h - The sharding router module header file - * - * @verbatim - * Revision History - * - * See GitHub https://github.com/mariadb-corporation/MaxScale - * - * @endverbatim - */ - -#include -#include -#include - -struct router_instance; - -typedef enum -{ - TARGET_UNDEFINED = 0x00, - TARGET_MASTER = 0x01, - TARGET_SLAVE = 0x02, - TARGET_NAMED_SERVER = 0x04, - TARGET_ALL = 0x08, - TARGET_RLAG_MAX = 0x10, - TARGET_ANY = 0x20 -} route_target_t; - -#define TARGET_IS_UNDEFINED(t) (t == TARGET_UNDEFINED) -#define TARGET_IS_NAMED_SERVER(t) (t & TARGET_NAMED_SERVER) -#define TARGET_IS_ALL(t) (t & TARGET_ALL) -#define TARGET_IS_ANY(t) (t & TARGET_ANY) - -typedef struct rses_property_st rses_property_t; -typedef struct router_client_session ROUTER_CLIENT_SES; - -typedef enum rses_property_type_t -{ - RSES_PROP_TYPE_UNDEFINED = -1, - RSES_PROP_TYPE_SESCMD = 0, - RSES_PROP_TYPE_FIRST = RSES_PROP_TYPE_SESCMD, - RSES_PROP_TYPE_TMPTABLES, - RSES_PROP_TYPE_LAST = RSES_PROP_TYPE_TMPTABLES, - RSES_PROP_TYPE_COUNT = RSES_PROP_TYPE_LAST + 1 -} rses_property_type_t; - -/** default values for rwsplit configuration parameters */ -#define CONFIG_MAX_SLAVE_CONN 1 -#define CONFIG_MAX_SLAVE_RLAG -1 /*< not used */ -#define CONFIG_SQL_VARIABLES_IN TYPE_ALL - -#define GET_SELECT_CRITERIA(s) \ - (strncmp(s,"LEAST_GLOBAL_CONNECTIONS", strlen("LEAST_GLOBAL_CONNECTIONS")) == 0 ? \ - LEAST_GLOBAL_CONNECTIONS : ( \ - strncmp(s,"LEAST_BEHIND_MASTER", strlen("LEAST_BEHIND_MASTER")) == 0 ? \ - LEAST_BEHIND_MASTER : ( \ - strncmp(s,"LEAST_ROUTER_CONNECTIONS", strlen("LEAST_ROUTER_CONNECTIONS")) == 0 ? \ - LEAST_ROUTER_CONNECTIONS : ( \ - strncmp(s,"LEAST_CURRENT_OPERATIONS", strlen("LEAST_CURRENT_OPERATIONS")) == 0 ? \ - LEAST_CURRENT_OPERATIONS : UNDEFINED_CRITERIA)))) - - - -#define SUBSVC_IS_MAPPED(s) (s->state & SUBSVC_MAPPED) -#define SUBSVC_IS_CLOSED(s) (s->state & SUBSVC_CLOSED) -#define SUBSVC_IS_OK(s) (s->state & SUBSVC_OK) -#define SUBSVC_IS_WAITING(s) (s->state & SUBSVC_WAITING_RESULT) - -/** - * Session variable command - */ -typedef struct mysql_sescmd_st -{ -#if defined(SS_DEBUG) - skygw_chk_t my_sescmd_chk_top; -#endif - rses_property_t* my_sescmd_prop; /*< parent property */ - GWBUF* my_sescmd_buf; /*< query buffer */ - unsigned char my_sescmd_packet_type;/*< packet type */ - bool my_sescmd_is_replied; /*< is cmd replied to client */ -#if defined(SS_DEBUG) - skygw_chk_t my_sescmd_chk_tail; -#endif -} mysql_sescmd_t; - - -/** - * Property structure - */ -struct rses_property_st -{ -#if defined(SS_DEBUG) - skygw_chk_t rses_prop_chk_top; -#endif - ROUTER_CLIENT_SES* rses_prop_rsession; /*< parent router session */ - int rses_prop_refcount; - rses_property_type_t rses_prop_type; - union rses_prop_data - { - mysql_sescmd_t sescmd; - HASHTABLE* temp_tables; - } rses_prop_data; - rses_property_t* rses_prop_next; /*< next property of same type */ -#if defined(SS_DEBUG) - skygw_chk_t rses_prop_chk_tail; -#endif -}; - -typedef struct sescmd_cursor_st -{ -#if defined(SS_DEBUG) - skygw_chk_t scmd_cur_chk_top; -#endif - ROUTER_CLIENT_SES* scmd_cur_rses; /*< pointer to owning router session */ - rses_property_t** scmd_cur_ptr_property; /*< address of pointer to owner property */ - mysql_sescmd_t* scmd_cur_cmd; /*< pointer to current session command */ - bool scmd_cur_active; /*< true if command is being executed */ -#if defined(SS_DEBUG) - skygw_chk_t scmd_cur_chk_tail; -#endif -} sescmd_cursor_t; - -typedef struct shardrouter_config_st -{ - int rw_max_slave_conn_percent; - int rw_max_slave_conn_count; - target_t rw_use_sql_variables_in; -} shard_config_t; - -typedef enum -{ - SUBSVC_ALLOC = 0, - SUBSVC_OK = 1, - SUBSVC_CLOSED = (1 << 1), /* This is when the service was cleanly closed */ - SUBSVC_FAILED = (1 << 2), /* This is when something went wrong */ - SUBSVC_QUERY_ACTIVE = (1 << 3), - SUBSVC_WAITING_RESULT = (1 << 4), - SUBSVC_MAPPED = (1 << 5) -} subsvc_state_t; - -typedef struct subservice_t -{ - SERVICE* service; - SESSION* session; - DCB* dcb; - GWBUF* pending_cmd; - sescmd_cursor_t* scur; - int state; - int n_res_waiting; - bool mapped; -} SUBSERVICE; - -/** - * Bitmask values for the router session's initialization. These values are used - * to prevent responses from internal commands being forwarded to the client. - */ -typedef enum shard_init_mask -{ - INIT_READY = 0x0, - INIT_MAPPING = 0x1, - INIT_USE_DB = 0x02, - INIT_UNINT = 0x04 - -} shard_init_mask_t; - -/** - * The client session structure used within this router. - */ -struct router_client_session -{ -#if defined(SS_DEBUG) - skygw_chk_t rses_chk_top; -#endif - SPINLOCK rses_lock; /*< protects rses_deleted */ - int rses_versno; /*< even = no active update, else odd. not used 4/14 */ - bool rses_closed; /*< true when closeSession is called */ - DCB* rses_client_dcb; - DCB* replydcb; /* DCB used to send the client write messages from the router itself */ - DCB* routedcb; /* DCB used to send queued queries to the router */ - MYSQL_session* rses_mysql_session; - /** Properties listed by their type */ - rses_property_t* rses_properties[RSES_PROP_TYPE_COUNT]; - - shard_config_t rses_config; /*< copied config info from router instance */ - bool rses_autocommit_enabled; - bool rses_transaction_active; - struct router_instance *router; /*< The router instance */ - struct router_client_session* next; - HASHTABLE* dbhash; - SUBSERVICE * *subservice; - int n_subservice; - bool hash_init; - SESSION* session; - GWBUF* queue; - char connect_db[MYSQL_DATABASE_MAXLEN + 1]; /*< Database the user was trying to connect to */ - char current_db[MYSQL_DATABASE_MAXLEN + 1]; /*< Current active database */ - shard_init_mask_t init; /*< Initialization state bitmask */ -#if defined(SS_DEBUG) - skygw_chk_t rses_chk_tail; -#endif -}; - -/** - * The statistics for this router instance - */ -typedef struct -{ - int n_sessions; /*< Number sessions created */ - int n_queries; /*< Number of queries forwarded */ - int n_master; /*< Number of stmts sent to master */ - int n_slave; /*< Number of stmts sent to slave */ - int n_all; /*< Number of stmts sent to all */ -} ROUTER_STATS; - - -/** - * The per instance data for the router. - */ -typedef struct router_instance -{ - SERVICE* service; /*< Pointer to owning service */ - ROUTER_CLIENT_SES* connections; /*< List of client connections */ - SERVICE** services; /*< List of services to map for sharding */ - int n_services; - SUBSERVICE* all_subsvc; - SPINLOCK lock; /*< Lock for the instance data */ - shard_config_t shardrouter_config; /*< expanded config info from SERVICE */ - int shardrouter_version;/*< version number for router's config */ - unsigned int bitmask; /*< Bitmask to apply to server->status */ - unsigned int bitvalue; /*< Required value of server->status */ - ROUTER_STATS stats; /*< Statistics for this router */ - struct router_instance* next; /*< Next router on the list */ - bool available_slaves; /*< The router has some slaves available */ - DCB* dumy_backend; -} ROUTER_INSTANCE; - -#define BACKEND_TYPE(b) (SERVER_IS_MASTER((b)->backend_server) ? BE_MASTER : \ - (SERVER_IS_SLAVE((b)->backend_server) ? BE_SLAVE : BE_UNDEFINED)); - -bool subsvc_is_valid(SUBSERVICE*); - -#endif /*< _SHARDROUTER_H */ diff --git a/server/modules/protocol/HTTPD/httpd.c b/server/modules/protocol/HTTPD/httpd.c index c584fd27d..d3ae66ff9 100644 --- a/server/modules/protocol/HTTPD/httpd.c +++ b/server/modules/protocol/HTTPD/httpd.c @@ -32,7 +32,7 @@ * @endverbatim */ -#include +#include "httpd.h" #include #include #include diff --git a/server/modules/include/httpd.h b/server/modules/protocol/HTTPD/httpd.h similarity index 100% rename from server/modules/include/httpd.h rename to server/modules/protocol/HTTPD/httpd.h diff --git a/server/modules/protocol/maxscaled/maxscaled.c b/server/modules/protocol/maxscaled/maxscaled.c index 75f2c660e..9e8e6d4d2 100644 --- a/server/modules/protocol/maxscaled/maxscaled.c +++ b/server/modules/protocol/maxscaled/maxscaled.c @@ -33,8 +33,8 @@ #include #include #include -#include -#include +#include "maxscaled.h" +#include #include /* @see function load_module in load_utils.c for explanation of the following diff --git a/server/modules/include/maxscaled.h b/server/modules/protocol/maxscaled/maxscaled.h similarity index 100% rename from server/modules/include/maxscaled.h rename to server/modules/protocol/maxscaled/maxscaled.h diff --git a/server/modules/routing/avro/avro.c b/server/modules/routing/avro/avro.c index c580c574e..a4e5c5581 100644 --- a/server/modules/routing/avro/avro.c +++ b/server/modules/routing/avro/avro.c @@ -47,7 +47,7 @@ #include #include -#include +#include "avrorouter.h" #include #include #include diff --git a/server/modules/routing/avro/avro_client.c b/server/modules/routing/avro/avro_client.c index 505be1ec4..9a051728c 100644 --- a/server/modules/routing/avro/avro_client.c +++ b/server/modules/routing/avro/avro_client.c @@ -39,7 +39,7 @@ #include #include #include -#include +#include "avrorouter.h" #include #include #include diff --git a/server/modules/routing/avro/avro_file.c b/server/modules/routing/avro/avro_file.c index 05f4b020b..8fe34f3a7 100644 --- a/server/modules/routing/avro/avro_file.c +++ b/server/modules/routing/avro/avro_file.c @@ -33,7 +33,7 @@ #include #include #include -#include +#include "avrorouter.h" #include #include #include diff --git a/server/modules/routing/avro/avro_index.c b/server/modules/routing/avro/avro_index.c index 99da0cab2..70acf2353 100644 --- a/server/modules/routing/avro/avro_index.c +++ b/server/modules/routing/avro/avro_index.c @@ -32,7 +32,7 @@ * @endverbatim */ -#include +#include "avrorouter.h" #include #include diff --git a/server/modules/routing/avro/avro_rbr.c b/server/modules/routing/avro/avro_rbr.c index 27f68aa98..a64927290 100644 --- a/server/modules/routing/avro/avro_rbr.c +++ b/server/modules/routing/avro/avro_rbr.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include "avrorouter.h" #include #define WRITE_EVENT 0 diff --git a/server/modules/routing/avro/avro_schema.c b/server/modules/routing/avro/avro_schema.c index e7244fa4c..fdf48448a 100644 --- a/server/modules/routing/avro/avro_schema.c +++ b/server/modules/routing/avro/avro_schema.c @@ -18,7 +18,7 @@ #define _GNU_SOURCE #endif -#include +#include "avrorouter.h" #include #include #include diff --git a/server/modules/include/avrorouter.h b/server/modules/routing/avro/avrorouter.h similarity index 100% rename from server/modules/include/avrorouter.h rename to server/modules/routing/avro/avrorouter.h diff --git a/server/modules/routing/binlog/blr.c b/server/modules/routing/binlog/blr.c index 43f0facea..b8dc3d202 100644 --- a/server/modules/routing/binlog/blr.c +++ b/server/modules/routing/binlog/blr.c @@ -65,7 +65,7 @@ #include #include #include -#include +#include "blr.h" #include #include #include diff --git a/server/modules/include/blr.h b/server/modules/routing/binlog/blr.h similarity index 100% rename from server/modules/include/blr.h rename to server/modules/routing/binlog/blr.h diff --git a/server/modules/routing/binlog/blr_cache.c b/server/modules/routing/binlog/blr_cache.c index 2f8848ab6..3e6e5768d 100644 --- a/server/modules/routing/binlog/blr_cache.c +++ b/server/modules/routing/binlog/blr_cache.c @@ -39,7 +39,7 @@ #include #include #include -#include +#include "blr.h" #include #include diff --git a/server/modules/routing/binlog/blr_file.c b/server/modules/routing/binlog/blr_file.c index 14e259036..6edafcb3c 100644 --- a/server/modules/routing/binlog/blr_file.c +++ b/server/modules/routing/binlog/blr_file.c @@ -52,7 +52,7 @@ #include #include #include -#include +#include "blr.h" #include #include #include diff --git a/server/modules/routing/binlog/blr_master.c b/server/modules/routing/binlog/blr_master.c index cc33bee65..de9589879 100644 --- a/server/modules/routing/binlog/blr_master.c +++ b/server/modules/routing/binlog/blr_master.c @@ -60,7 +60,7 @@ #include #include #include -#include +#include "blr.h" #include #include #include diff --git a/server/modules/routing/binlog/blr_slave.c b/server/modules/routing/binlog/blr_slave.c index d319d8514..cbe4b3d07 100644 --- a/server/modules/routing/binlog/blr_slave.c +++ b/server/modules/routing/binlog/blr_slave.c @@ -74,7 +74,7 @@ #include #include #include -#include +#include "blr.h" #include #include #include diff --git a/server/modules/routing/binlog/maxbinlogcheck.c b/server/modules/routing/binlog/maxbinlogcheck.c index 4c926ce75..de8dbddb6 100644 --- a/server/modules/routing/binlog/maxbinlogcheck.c +++ b/server/modules/routing/binlog/maxbinlogcheck.c @@ -39,7 +39,7 @@ #include #include -#include +#include "blr.h" static void printVersion(const char *progname); diff --git a/server/modules/routing/binlog/test/testbinlog.c b/server/modules/routing/binlog/test/testbinlog.c index aaf060cd0..844dd2ba2 100644 --- a/server/modules/routing/binlog/test/testbinlog.c +++ b/server/modules/routing/binlog/test/testbinlog.c @@ -31,7 +31,7 @@ #include #include #include -#include +#include "../blr.h" #include #include #include diff --git a/server/modules/routing/maxinfo/maxinfo.c b/server/modules/routing/maxinfo/maxinfo.c index 21b1a391b..ded66eff0 100644 --- a/server/modules/routing/maxinfo/maxinfo.c +++ b/server/modules/routing/maxinfo/maxinfo.c @@ -43,7 +43,7 @@ #include #include #include -#include +#include "maxinfo.h" #include #include #include diff --git a/server/modules/include/maxinfo.h b/server/modules/routing/maxinfo/maxinfo.h similarity index 100% rename from server/modules/include/maxinfo.h rename to server/modules/routing/maxinfo/maxinfo.h diff --git a/server/modules/routing/maxinfo/maxinfo_error.c b/server/modules/routing/maxinfo/maxinfo_error.c index 1dd4c3087..b16258a5b 100644 --- a/server/modules/routing/maxinfo/maxinfo_error.c +++ b/server/modules/routing/maxinfo/maxinfo_error.c @@ -36,7 +36,7 @@ #include #include #include -#include +#include "maxinfo.h" #include #include diff --git a/server/modules/routing/maxinfo/maxinfo_exec.c b/server/modules/routing/maxinfo/maxinfo_exec.c index bdbfa3d47..950ae0183 100644 --- a/server/modules/routing/maxinfo/maxinfo_exec.c +++ b/server/modules/routing/maxinfo/maxinfo_exec.c @@ -40,7 +40,7 @@ #include #include #include -#include +#include "maxinfo.h" #include #include #include diff --git a/server/modules/routing/maxinfo/maxinfo_parse.c b/server/modules/routing/maxinfo/maxinfo_parse.c index c80b2323c..5fd8bfe05 100644 --- a/server/modules/routing/maxinfo/maxinfo_parse.c +++ b/server/modules/routing/maxinfo/maxinfo_parse.c @@ -37,7 +37,7 @@ #include #include #include -#include +#include "maxinfo.h" #include #include diff --git a/server/modules/include/readconnection.h b/server/modules/routing/readconnroute/readconnection.h similarity index 100% rename from server/modules/include/readconnection.h rename to server/modules/routing/readconnroute/readconnection.h diff --git a/server/modules/routing/readconnroute/readconnroute.c b/server/modules/routing/readconnroute/readconnroute.c index d15269635..04df306c4 100644 --- a/server/modules/routing/readconnroute/readconnroute.c +++ b/server/modules/routing/readconnroute/readconnroute.c @@ -79,7 +79,7 @@ #include #include #include -#include +#include "readconnection.h" #include #include #include diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index d9437339b..347ce8d15 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -4,7 +4,7 @@ * Use of this software is governed by the Business Source License included * in the LICENSE.TXT file and at www.mariadb.com/bsl. * - * Change Date: 2019-01-01 + * Change Date: 2019-07-01 * * On the date above, in accordance with the Business Source License, use * of this software will be governed by version 2 or later of the General @@ -17,8 +17,8 @@ #include #include -#include -#include +#include "readwritesplit.h" +#include "rwsplit_internal.h" #include #include diff --git a/server/modules/include/readwritesplit.h b/server/modules/routing/readwritesplit/readwritesplit.h similarity index 100% rename from server/modules/include/readwritesplit.h rename to server/modules/routing/readwritesplit/readwritesplit.h diff --git a/server/modules/include/rwsplit_internal.h b/server/modules/routing/readwritesplit/rwsplit_internal.h similarity index 100% rename from server/modules/include/rwsplit_internal.h rename to server/modules/routing/readwritesplit/rwsplit_internal.h diff --git a/server/modules/routing/readwritesplit/rwsplit_mysql.c b/server/modules/routing/readwritesplit/rwsplit_mysql.c index 72228395a..d7ff0ca95 100644 --- a/server/modules/routing/readwritesplit/rwsplit_mysql.c +++ b/server/modules/routing/readwritesplit/rwsplit_mysql.c @@ -4,7 +4,7 @@ * Use of this software is governed by the Business Source License included * in the LICENSE.TXT file and at www.mariadb.com/bsl. * - * Change Date: 2019-01-01 + * Change Date: 2019-07-01 * * On the date above, in accordance with the Business Source License, use * of this software will be governed by version 2 or later of the General @@ -18,8 +18,8 @@ #include #include -#include -#include +#include "readwritesplit.h" +#include "rwsplit_internal.h" #include #include diff --git a/server/modules/routing/readwritesplit/rwsplit_route_stmt.c b/server/modules/routing/readwritesplit/rwsplit_route_stmt.c index bbf690b14..73f0120f0 100644 --- a/server/modules/routing/readwritesplit/rwsplit_route_stmt.c +++ b/server/modules/routing/readwritesplit/rwsplit_route_stmt.c @@ -4,7 +4,7 @@ * Use of this software is governed by the Business Source License included * in the LICENSE.TXT file and at www.mariadb.com/bsl. * - * Change Date: 2019-01-01 + * Change Date: 2019-07-01 * * On the date above, in accordance with the Business Source License, use * of this software will be governed by version 2 or later of the General @@ -18,8 +18,8 @@ #include #include -#include -#include +#include "readwritesplit.h" +#include "rwsplit_internal.h" /** * @file rwsplit_route_stmt.c The functions that support the routing of * queries to back end servers. All the functions in this module are internal diff --git a/server/modules/routing/readwritesplit/rwsplit_select_backends.c b/server/modules/routing/readwritesplit/rwsplit_select_backends.c index 114e118fb..1198dec52 100644 --- a/server/modules/routing/readwritesplit/rwsplit_select_backends.c +++ b/server/modules/routing/readwritesplit/rwsplit_select_backends.c @@ -4,7 +4,7 @@ * Use of this software is governed by the Business Source License included * in the LICENSE.TXT file and at www.mariadb.com/bsl. * - * Change Date: 2019-01-01 + * Change Date: 2019-07-01 * * On the date above, in accordance with the Business Source License, use * of this software will be governed by version 2 or later of the General @@ -17,8 +17,8 @@ #include #include -#include -#include +#include "readwritesplit.h" +#include "rwsplit_internal.h" /** * @file rwsplit_select_backends.c The functions that implement back end * selection for the read write split router. All of these functions are diff --git a/server/modules/routing/readwritesplit/rwsplit_session_cmd.c b/server/modules/routing/readwritesplit/rwsplit_session_cmd.c index ddfc0987d..b37951543 100644 --- a/server/modules/routing/readwritesplit/rwsplit_session_cmd.c +++ b/server/modules/routing/readwritesplit/rwsplit_session_cmd.c @@ -4,7 +4,7 @@ * Use of this software is governed by the Business Source License included * in the LICENSE.TXT file and at www.mariadb.com/bsl. * - * Change Date: 2019-01-01 + * Change Date: 2019-07-01 * * On the date above, in accordance with the Business Source License, use * of this software will be governed by version 2 or later of the General @@ -17,8 +17,8 @@ #include #include -#include -#include +#include "readwritesplit.h" +#include "rwsplit_internal.h" /** * @file rwsplit_session_cmd.c The functions that provide session command diff --git a/server/modules/routing/readwritesplit/rwsplit_tmp_table_multi.c b/server/modules/routing/readwritesplit/rwsplit_tmp_table_multi.c index 41d9e8526..a82945982 100644 --- a/server/modules/routing/readwritesplit/rwsplit_tmp_table_multi.c +++ b/server/modules/routing/readwritesplit/rwsplit_tmp_table_multi.c @@ -4,7 +4,7 @@ * Use of this software is governed by the Business Source License included * in the LICENSE.TXT file and at www.mariadb.com/bsl. * - * Change Date: 2019-01-01 + * Change Date: 2019-07-01 * * On the date above, in accordance with the Business Source License, use * of this software will be governed by version 2 or later of the General @@ -21,8 +21,8 @@ #include #include -#include -#include +#include "readwritesplit.h" +#include "rwsplit_internal.h" /** * @file rwsplit_tmp_table.c The functions that carry out checks on * statements to see if they involve various operations involving temporary diff --git a/server/modules/routing/schemarouter/schemarouter.c b/server/modules/routing/schemarouter/schemarouter.c index 35882d9f9..895dd0388 100644 --- a/server/modules/routing/schemarouter/schemarouter.c +++ b/server/modules/routing/schemarouter/schemarouter.c @@ -17,8 +17,8 @@ #include #include #include -#include -#include +#include "schemarouter.h" +#include "sharding_common.h" #include #include #include diff --git a/server/modules/include/schemarouter.h b/server/modules/routing/schemarouter/schemarouter.h similarity index 100% rename from server/modules/include/schemarouter.h rename to server/modules/routing/schemarouter/schemarouter.h diff --git a/server/modules/routing/schemarouter/sharding_common.c b/server/modules/routing/schemarouter/sharding_common.c index 1c6fa75fc..43104a4b1 100644 --- a/server/modules/routing/schemarouter/sharding_common.c +++ b/server/modules/routing/schemarouter/sharding_common.c @@ -11,7 +11,7 @@ * Public License. */ -#include +#include "sharding_common.h" #include #include diff --git a/server/modules/include/sharding_common.h b/server/modules/routing/schemarouter/sharding_common.h similarity index 100% rename from server/modules/include/sharding_common.h rename to server/modules/routing/schemarouter/sharding_common.h diff --git a/server/modules/routing/schemarouter/svcconn.c b/server/modules/routing/schemarouter/svcconn.c deleted file mode 100644 index c88a29072..000000000 --- a/server/modules/routing/schemarouter/svcconn.c +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (c) 2016 MariaDB Corporation Ab - * - * Use of this software is governed by the Business Source License included - * in the LICENSE.TXT file and at www.mariadb.com/bsl. - * - * Change Date: 2019-07-01 - * - * On the date above, in accordance with the Business Source License, use - * of this software will be governed by version 2 or later of the General - * Public License. - */ - -#include - -void -subsvc_set_state(SUBSERVICE* svc, subsvc_state_t state) -{ - if (state & SUBSVC_WAITING_RESULT) - { - /** Increase waiter count */ - atomic_add(&svc->n_res_waiting, 1); - } - - svc->state |= state; -} - -void -subsvc_clear_state(SUBSERVICE* svc, subsvc_state_t state) -{ - if (state & SUBSVC_WAITING_RESULT) - { - /** Decrease waiter count */ - atomic_add(&svc->n_res_waiting, -1); - } - - svc->state &= ~state; -} - -bool -get_shard_subsvc(SUBSERVICE** subsvc, ROUTER_CLIENT_SES* session, char* target) -{ - int i; - - if (subsvc == NULL || session == NULL || target == NULL) - { - return false; - } - - for (i = 0; i < session->n_subservice; i++) - { - if (strcmp(session->subservice[i]->service->name, target) == 0) - { - - if (SUBSVC_IS_OK(session->subservice[i])) - { - if (subsvc_is_valid(session->subservice[i])) - { - *subsvc = session->subservice[i]; - return true; - } - - /** - * The service has failed - */ - - subsvc_set_state(session->subservice[i], SUBSVC_FAILED); - } - } - } - - return false; -} From 485675d065b6c6802a65b07ba9ee9f19e7110daf Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Thu, 13 Oct 2016 19:21:37 +0300 Subject: [PATCH 020/215] Duplicate declarations removed Duplicate declarations of functions declared in maxscale/users.h and maxscale/gw.h removed and corresponding includes added instead. Unimplemented declaration removed. Further cleanup will be needed to ensure that functions etc. are declared in the right place. --- .../include/mysql_client_server_protocol.h | 21 ++----------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/server/modules/include/mysql_client_server_protocol.h b/server/modules/include/mysql_client_server_protocol.h index 45a18f959..1dbc435b7 100644 --- a/server/modules/include/mysql_client_server_protocol.h +++ b/server/modules/include/mysql_client_server_protocol.h @@ -61,6 +61,8 @@ #include #include #include +#include +#include #include #define GW_MYSQL_VERSION "5.5.5-10.0.0 " MAXSCALE_VERSION "-maxscale" @@ -338,27 +340,8 @@ int mysql_send_auth_error ( int in_affected_rows, const char* mysql_message); -void gw_sha1_str(const uint8_t *in, int in_len, uint8_t *out); -void gw_sha1_2_str( - const uint8_t *in, - int in_len, - const uint8_t *in2, - int in2_len, - uint8_t *out); -void gw_str_xor( - uint8_t *output, - const uint8_t *input1, - const uint8_t *input2, - unsigned int len); - -char *gw_bin2hex(char *out, const uint8_t *in, unsigned int len); -int gw_hex2bin(uint8_t *out, const char *in, unsigned int len); -int gw_generate_random_str(char *output, int len); -int setnonblocking(int fd); -int setipaddress(struct in_addr *a, char *p); GWBUF* gw_MySQL_get_next_packet(GWBUF** p_readbuf); GWBUF* gw_MySQL_get_packets(GWBUF** p_readbuf, int* npackets); -GWBUF* gw_MySQL_discard_packets(GWBUF* buf, int npackets); void protocol_add_srv_command(MySQLProtocol* p, mysql_server_cmd_t cmd); void protocol_remove_srv_command(MySQLProtocol* p); bool protocol_waits_response(MySQLProtocol* p); From 3a18b64bddc882fec6c92f2e7daa4bb7d5001fdc Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Thu, 13 Oct 2016 19:40:00 +0300 Subject: [PATCH 021/215] mysql_client_server_protocol.h moved. Moved to include/maxscale/protocol/mysql.h --- .../maxscale/protocol/mysql.h | 4 ++-- .../qc_mysqlembedded/qc_mysqlembedded.cc | 2 +- query_classifier/qc_sqlite/qc_sqlite.c | 2 +- query_classifier/test/compare.cc | 2 +- server/core/dbusers.c | 2 +- server/core/modutil.c | 2 +- server/core/secrets.c | 2 +- server/core/test/test_mysql_users.c | 2 +- server/modules/authenticator/gssapi_auth.c | 2 +- server/modules/authenticator/gssapi_backend_auth.c | 2 +- server/modules/authenticator/mysql_auth.c | 14 +++++++------- server/modules/authenticator/mysql_backend_auth.c | 2 +- server/modules/filter/cache/cache_storage_api.h | 2 +- server/modules/filter/cache/rules.c | 2 +- server/modules/filter/dbfwfilter/dbfwfilter.c | 2 +- server/modules/filter/mqfilter/mqfilter.c | 2 +- server/modules/filter/tee/tee.c | 2 +- server/modules/include/mysql_auth.h | 2 +- .../protocol/MySQL/MySQLBackend/mysql_backend.c | 2 +- .../protocol/MySQL/MySQLClient/mysql_client.c | 4 ++-- server/modules/protocol/MySQL/mysql_common.c | 4 ++-- server/modules/routing/avro/avro.c | 2 +- server/modules/routing/avro/avrorouter.h | 2 +- server/modules/routing/binlog/blr.c | 2 +- server/modules/routing/binlog/blr.h | 2 +- server/modules/routing/binlog/blr_master.c | 2 +- server/modules/routing/binlog/test/testbinlog.c | 2 +- .../modules/routing/readconnroute/readconnroute.c | 4 ++-- .../routing/readwritesplit/rwsplit_internal.h | 2 +- .../modules/routing/readwritesplit/rwsplit_mysql.c | 4 ++-- server/modules/routing/schemarouter/schemarouter.c | 2 +- server/modules/routing/schemarouter/schemarouter.h | 2 +- .../modules/routing/schemarouter/sharding_common.h | 2 +- 33 files changed, 44 insertions(+), 44 deletions(-) rename server/modules/include/mysql_client_server_protocol.h => include/maxscale/protocol/mysql.h (99%) diff --git a/server/modules/include/mysql_client_server_protocol.h b/include/maxscale/protocol/mysql.h similarity index 99% rename from server/modules/include/mysql_client_server_protocol.h rename to include/maxscale/protocol/mysql.h index 1dbc435b7..f19230e89 100644 --- a/server/modules/include/mysql_client_server_protocol.h +++ b/include/maxscale/protocol/mysql.h @@ -1,5 +1,5 @@ -#ifndef _MYSQL_PROTOCOL_H -#define _MYSQL_PROTOCOL_H +#ifndef _MAXSCALE_PROTOCOL_MYSQL_H +#define _MAXSCALE_PROTOCOL_MYSQL_H /* * Copyright (c) 2016 MariaDB Corporation Ab * diff --git a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc index e9fd30a36..23dd508fc 100644 --- a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc +++ b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc @@ -59,7 +59,7 @@ #include #include #include -#include +#include #include #include diff --git a/query_classifier/qc_sqlite/qc_sqlite.c b/query_classifier/qc_sqlite/qc_sqlite.c index 686b4821c..a5fbebd27 100644 --- a/query_classifier/qc_sqlite/qc_sqlite.c +++ b/query_classifier/qc_sqlite/qc_sqlite.c @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/query_classifier/test/compare.cc b/query_classifier/test/compare.cc index b52d06a1f..964a41dd0 100644 --- a/query_classifier/test/compare.cc +++ b/query_classifier/test/compare.cc @@ -22,7 +22,7 @@ #include #include #include -#include +#include #include using std::cerr; using std::cin; diff --git a/server/core/dbusers.c b/server/core/dbusers.c index ab304107e..cd6491f77 100644 --- a/server/core/dbusers.c +++ b/server/core/dbusers.c @@ -46,7 +46,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/server/core/modutil.c b/server/core/modutil.c index 9a0daba9c..92d4d6bf6 100644 --- a/server/core/modutil.c +++ b/server/core/modutil.c @@ -26,7 +26,7 @@ */ #include #include -#include +#include #include #include #include diff --git a/server/core/secrets.c b/server/core/secrets.c index f9716bc3a..4ab38da52 100644 --- a/server/core/secrets.c +++ b/server/core/secrets.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/server/core/test/test_mysql_users.c b/server/core/test/test_mysql_users.c index 0defa3ff6..c25b16205 100644 --- a/server/core/test/test_mysql_users.c +++ b/server/core/test/test_mysql_users.c @@ -35,7 +35,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/server/modules/authenticator/gssapi_auth.c b/server/modules/authenticator/gssapi_auth.c index 6b94c1b94..5f57d642a 100644 --- a/server/modules/authenticator/gssapi_auth.c +++ b/server/modules/authenticator/gssapi_auth.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include "gssapi_auth.h" typedef struct gssapi_instance diff --git a/server/modules/authenticator/gssapi_backend_auth.c b/server/modules/authenticator/gssapi_backend_auth.c index cd1a0a6e1..d047b80f8 100644 --- a/server/modules/authenticator/gssapi_backend_auth.c +++ b/server/modules/authenticator/gssapi_backend_auth.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include "gssapi_auth.h" /** diff --git a/server/modules/authenticator/mysql_auth.c b/server/modules/authenticator/mysql_auth.c index e92088f3c..90ac2aa7b 100644 --- a/server/modules/authenticator/mysql_auth.c +++ b/server/modules/authenticator/mysql_auth.c @@ -26,7 +26,7 @@ */ #include -#include +#include #include #include #include @@ -133,7 +133,7 @@ GWAUTHENTICATOR* GetModuleObject() * * @param dcb Request handler DCB connected to the client * @return Authentication status - * @note Authentication status codes are defined in mysql_client_server_protocol.h + * @note Authentication status codes are defined in maxscale/protocol/mysql.h */ static int mysql_auth_authenticate(DCB *dcb) @@ -233,7 +233,7 @@ mysql_auth_authenticate(DCB *dcb) * @param dcb Request handler DCB connected to the client * @param buffer Pointer to pointer to buffer containing data from client * @return Authentication status - * @note Authentication status codes are defined in mysql_client_server_protocol.h + * @note Authentication status codes are defined in maxscale/protocol/mysql.h * @see https://dev.mysql.com/doc/internals/en/client-server-protocol.html */ static int @@ -286,7 +286,7 @@ mysql_auth_set_protocol_data(DCB *dcb, GWBUF *buf) * @param client_auth_packet The data from the buffer received from client * @param client_auth_packet size An integer giving the size of the data * @return Authentication status - * @note Authentication status codes are defined in mysql_client_server_protocol.h + * @note Authentication status codes are defined in maxscale/protocol/mysql.h * @see https://dev.mysql.com/doc/internals/en/client-server-protocol.html */ static int @@ -581,7 +581,7 @@ int gw_find_mysql_user_password_sha1(char *username, uint8_t *gateway_password, * @param username The current username in the authentication request * @param stage1_hash The SHA1(candidate_password) decoded by this routine * @return Authentication status - * @note Authentication status codes are defined in mysql_client_server_protocol.h + * @note Authentication status codes are defined in maxscale/protocol/mysql.h * */ int @@ -707,7 +707,7 @@ gw_check_mysql_scramble_data(DCB *dcb, * @param database A string containing the database name * @param auth_ret The authentication status prior to calling this function. * @return Authentication status - * @note Authentication status codes are defined in mysql_client_server_protocol.h + * @note Authentication status codes are defined in maxscale/protocol/mysql.h */ int check_db_name_after_auth(DCB *dcb, char *database, int auth_ret) @@ -764,7 +764,7 @@ check_db_name_after_auth(DCB *dcb, char *database, int auth_ret) * @param stage1_hash A password hash for authentication * @param database A string containing the name for the default database * @return Authentication status - * @note Authentication status codes are defined in mysql_client_server_protocol.h + * @note Authentication status codes are defined in maxscale/protocol/mysql.h */ static int combined_auth_check( DCB *dcb, diff --git a/server/modules/authenticator/mysql_backend_auth.c b/server/modules/authenticator/mysql_backend_auth.c index ff7b37c69..137692476 100644 --- a/server/modules/authenticator/mysql_backend_auth.c +++ b/server/modules/authenticator/mysql_backend_auth.c @@ -30,7 +30,7 @@ */ #include -#include +#include #include #include diff --git a/server/modules/filter/cache/cache_storage_api.h b/server/modules/filter/cache/cache_storage_api.h index 320e25f00..3755d58ec 100644 --- a/server/modules/filter/cache/cache_storage_api.h +++ b/server/modules/filter/cache/cache_storage_api.h @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include EXTERN_C_BLOCK_BEGIN diff --git a/server/modules/filter/cache/rules.c b/server/modules/filter/cache/rules.c index e66ae84c1..8e5c0179a 100644 --- a/server/modules/filter/cache/rules.c +++ b/server/modules/filter/cache/rules.c @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include #include #include "cache.h" diff --git a/server/modules/filter/dbfwfilter/dbfwfilter.c b/server/modules/filter/dbfwfilter/dbfwfilter.c index 1510af39c..72b30917f 100644 --- a/server/modules/filter/dbfwfilter/dbfwfilter.c +++ b/server/modules/filter/dbfwfilter/dbfwfilter.c @@ -68,7 +68,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/server/modules/filter/mqfilter/mqfilter.c b/server/modules/filter/mqfilter/mqfilter.c index 71faca490..3642eb3f3 100644 --- a/server/modules/filter/mqfilter/mqfilter.c +++ b/server/modules/filter/mqfilter/mqfilter.c @@ -70,7 +70,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/server/modules/filter/tee/tee.c b/server/modules/filter/tee/tee.c index 481cd0559..6ac656f49 100644 --- a/server/modules/filter/tee/tee.c +++ b/server/modules/filter/tee/tee.c @@ -56,7 +56,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/server/modules/include/mysql_auth.h b/server/modules/include/mysql_auth.h index bd02dea59..761fe530c 100644 --- a/server/modules/include/mysql_auth.h +++ b/server/modules/include/mysql_auth.h @@ -26,7 +26,7 @@ #include #include #include -#include +#include int gw_check_mysql_scramble_data(DCB *dcb, uint8_t *token, diff --git a/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c b/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c index 598f71330..30d763a12 100644 --- a/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c +++ b/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c @@ -11,7 +11,7 @@ * Public License. */ -#include +#include #include #include #include diff --git a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c index 10e76bf0d..53e240687 100644 --- a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c +++ b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c @@ -49,7 +49,7 @@ #include #include #include -#include +#include #include #include #include @@ -874,7 +874,7 @@ gw_read_finish_processing(DCB *dcb, GWBUF *read_buffer, uint8_t capabilities) * * @param dcb Request handler DCB connected to the client * @param auth_val The type of authentication failure - * @note Authentication status codes are defined in mysql_client_server_protocol.h + * @note Authentication status codes are defined in maxscale/protocol/mysql.h */ static void mysql_client_auth_error_handling(DCB *dcb, int auth_val) diff --git a/server/modules/protocol/MySQL/mysql_common.c b/server/modules/protocol/MySQL/mysql_common.c index 0e16ef927..08fb31b6e 100644 --- a/server/modules/protocol/MySQL/mysql_common.c +++ b/server/modules/protocol/MySQL/mysql_common.c @@ -44,7 +44,7 @@ #include #include -#include +#include #include #include #include @@ -1273,7 +1273,7 @@ load_hashed_password(uint8_t *scramble, uint8_t *payload, uint8_t *passwd) * @param db_specified Whether the connection request specified a database * @param compress Whether compression is requested - NOT SUPPORTED * @return Bit mask (32 bits) - * @note Capability bits are defined in mysql_client_server_protocol.h + * @note Capability bits are defined in maxscale/protocol/mysql.h */ static uint32_t create_capabilities(MySQLProtocol *conn, bool db_specified, bool compress) diff --git a/server/modules/routing/avro/avro.c b/server/modules/routing/avro/avro.c index a4e5c5581..8c6cb1807 100644 --- a/server/modules/routing/avro/avro.c +++ b/server/modules/routing/avro/avro.c @@ -43,7 +43,7 @@ #include #include -#include +#include #include #include diff --git a/server/modules/routing/avro/avrorouter.h b/server/modules/routing/avro/avrorouter.h index 83e018f7c..2180327cc 100644 --- a/server/modules/routing/avro/avrorouter.h +++ b/server/modules/routing/avro/avrorouter.h @@ -33,7 +33,7 @@ #include #include #include -#include +#include /** SQLite3 version 3.7.14 introduced the new v2 close interface */ #if SQLITE_VERSION_NUMBER < 3007014 diff --git a/server/modules/routing/binlog/blr.c b/server/modules/routing/binlog/blr.c index b8dc3d202..d4161b759 100644 --- a/server/modules/routing/binlog/blr.c +++ b/server/modules/routing/binlog/blr.c @@ -75,7 +75,7 @@ #include #include -#include +#include #include #include #include diff --git a/server/modules/routing/binlog/blr.h b/server/modules/routing/binlog/blr.h index 7f423da0a..5f95f957a 100644 --- a/server/modules/routing/binlog/blr.h +++ b/server/modules/routing/binlog/blr.h @@ -44,7 +44,7 @@ #include #include #include -#include +#include #define BINLOG_FNAMELEN 255 #define BLR_PROTOCOL "MySQLBackend" diff --git a/server/modules/routing/binlog/blr_master.c b/server/modules/routing/binlog/blr_master.c index de9589879..c96c30a70 100644 --- a/server/modules/routing/binlog/blr_master.c +++ b/server/modules/routing/binlog/blr_master.c @@ -77,7 +77,7 @@ #include /* Temporary requirement for auth data */ -#include +#include #include diff --git a/server/modules/routing/binlog/test/testbinlog.c b/server/modules/routing/binlog/test/testbinlog.c index 844dd2ba2..a9e3e2373 100644 --- a/server/modules/routing/binlog/test/testbinlog.c +++ b/server/modules/routing/binlog/test/testbinlog.c @@ -42,7 +42,7 @@ #include #include -#include +#include #include #include #include diff --git a/server/modules/routing/readconnroute/readconnroute.c b/server/modules/routing/readconnroute/readconnroute.c index 04df306c4..c55a82fe8 100644 --- a/server/modules/routing/readconnroute/readconnroute.c +++ b/server/modules/routing/readconnroute/readconnroute.c @@ -47,7 +47,7 @@ * and necessary headers. * 17/07/2013 Massimiliano Pinto Added clientReply routine: * called by backend server to send data to client - * Included mysql_client_server_protocol.h + * Included maxscale/protocol/mysql.h * with macros and MySQL commands with MYSQL_ prefix * avoiding any conflict with the standard ones * in mysql.h @@ -88,7 +88,7 @@ #include #include -#include +#include #include diff --git a/server/modules/routing/readwritesplit/rwsplit_internal.h b/server/modules/routing/readwritesplit/rwsplit_internal.h index c090914a9..b17048eae 100644 --- a/server/modules/routing/readwritesplit/rwsplit_internal.h +++ b/server/modules/routing/readwritesplit/rwsplit_internal.h @@ -30,7 +30,7 @@ extern "C" { /* This needs to be removed along with dependency on it - see the * rwsplit_tmp_table_multi functions */ -#include +#include /* * The following are implemented in rwsplit_mysql.c diff --git a/server/modules/routing/readwritesplit/rwsplit_mysql.c b/server/modules/routing/readwritesplit/rwsplit_mysql.c index d7ff0ca95..4397f9a11 100644 --- a/server/modules/routing/readwritesplit/rwsplit_mysql.c +++ b/server/modules/routing/readwritesplit/rwsplit_mysql.c @@ -29,12 +29,12 @@ #include #include #include -#include +#include #include #include #if defined(SS_DEBUG) -#include +#include #endif #define RWSPLIT_TRACE_MSG_LEN 1000 diff --git a/server/modules/routing/schemarouter/schemarouter.c b/server/modules/routing/schemarouter/schemarouter.c index 895dd0388..3ed3ba191 100644 --- a/server/modules/routing/schemarouter/schemarouter.c +++ b/server/modules/routing/schemarouter/schemarouter.c @@ -28,7 +28,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/server/modules/routing/schemarouter/schemarouter.h b/server/modules/routing/schemarouter/schemarouter.h index 914d11960..17a30729d 100644 --- a/server/modules/routing/schemarouter/schemarouter.h +++ b/server/modules/routing/schemarouter/schemarouter.h @@ -29,7 +29,7 @@ #include #include -#include +#include #include /** * Bitmask values for the router session's initialization. These values are used diff --git a/server/modules/routing/schemarouter/sharding_common.h b/server/modules/routing/schemarouter/sharding_common.h index 0952f6a30..f4fb06307 100644 --- a/server/modules/routing/schemarouter/sharding_common.h +++ b/server/modules/routing/schemarouter/sharding_common.h @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include #include #include From 76430e060f626f0589aef28a2d45c05a7434ff1f Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Thu, 13 Oct 2016 19:58:06 +0300 Subject: [PATCH 022/215] maxconfig.h renamed to config.h --- include/maxscale/{maxconfig.h => config.h} | 0 include/maxscale/monitor.h | 2 +- include/maxscale/service.h | 2 +- server/core/config.c | 2 +- server/core/gateway.c | 2 +- server/core/mysql_utils.c | 2 +- server/core/poll.c | 3 +-- server/core/statistics.c | 2 +- server/modules/filter/regexfilter/regexfilter.c | 2 +- server/modules/monitor/galeramon/galeramon.h | 2 +- server/modules/monitor/mmmon/mmmon.h | 2 +- server/modules/monitor/mysqlmon.h | 2 +- server/modules/monitor/ndbclustermon/ndbclustermon.h | 2 +- server/modules/routing/debugcli/debugcmd.c | 2 +- server/modules/routing/maxinfo/maxinfo_exec.c | 2 +- 15 files changed, 14 insertions(+), 15 deletions(-) rename include/maxscale/{maxconfig.h => config.h} (100%) diff --git a/include/maxscale/maxconfig.h b/include/maxscale/config.h similarity index 100% rename from include/maxscale/maxconfig.h rename to include/maxscale/config.h diff --git a/include/maxscale/monitor.h b/include/maxscale/monitor.h index 7b2867391..a5dac0abb 100644 --- a/include/maxscale/monitor.h +++ b/include/maxscale/monitor.h @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include #include diff --git a/include/maxscale/service.h b/include/maxscale/service.h index 51b81ca6f..eae46c1a7 100644 --- a/include/maxscale/service.h +++ b/include/maxscale/service.h @@ -22,7 +22,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/server/core/config.c b/server/core/config.c index 060386f97..e1cb5b747 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -50,7 +50,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/server/core/gateway.c b/server/core/gateway.c index f8e567344..c2919e03e 100644 --- a/server/core/gateway.c +++ b/server/core/gateway.c @@ -57,7 +57,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/server/core/mysql_utils.c b/server/core/mysql_utils.c index 3c1b2159c..7ae98160f 100644 --- a/server/core/mysql_utils.c +++ b/server/core/mysql_utils.c @@ -26,7 +26,7 @@ #include #include #include -#include +#include /** * @brief Calculate the length of a length-encoded integer in bytes diff --git a/server/core/poll.c b/server/core/poll.c index d68a44e26..e7a1fbb0f 100644 --- a/server/core/poll.c +++ b/server/core/poll.c @@ -27,9 +27,8 @@ #include #include #include -#include #include -#include +#include #include #include #include diff --git a/server/core/statistics.c b/server/core/statistics.c index 259608efa..c0e1dadfb 100644 --- a/server/core/statistics.c +++ b/server/core/statistics.c @@ -25,7 +25,7 @@ #include #include -#include +#include #include #include diff --git a/server/modules/filter/regexfilter/regexfilter.c b/server/modules/filter/regexfilter/regexfilter.c index c0e2a6ee6..df84853e2 100644 --- a/server/modules/filter/regexfilter/regexfilter.c +++ b/server/modules/filter/regexfilter/regexfilter.c @@ -22,7 +22,7 @@ #include #include #include -#include +#include /** * @file regexfilter.c - a very simple regular expression rewrite filter. diff --git a/server/modules/monitor/galeramon/galeramon.h b/server/modules/monitor/galeramon/galeramon.h index 9fdcd8b27..a9563e025 100644 --- a/server/modules/monitor/galeramon/galeramon.h +++ b/server/modules/monitor/galeramon/galeramon.h @@ -27,7 +27,7 @@ #include #include #include -#include +#include /** * @file galeramon.h - The Galera cluster monitor diff --git a/server/modules/monitor/mmmon/mmmon.h b/server/modules/monitor/mmmon/mmmon.h index df3747c9a..674672d89 100644 --- a/server/modules/monitor/mmmon/mmmon.h +++ b/server/modules/monitor/mmmon/mmmon.h @@ -26,7 +26,7 @@ #include #include #include -#include +#include #include /** diff --git a/server/modules/monitor/mysqlmon.h b/server/modules/monitor/mysqlmon.h index b49cb8ba0..357d1b235 100644 --- a/server/modules/monitor/mysqlmon.h +++ b/server/modules/monitor/mysqlmon.h @@ -26,7 +26,7 @@ #include #include #include -#include +#include #include #include diff --git a/server/modules/monitor/ndbclustermon/ndbclustermon.h b/server/modules/monitor/ndbclustermon/ndbclustermon.h index b491b13db..2b98613c4 100644 --- a/server/modules/monitor/ndbclustermon/ndbclustermon.h +++ b/server/modules/monitor/ndbclustermon/ndbclustermon.h @@ -26,7 +26,7 @@ #include #include #include -#include +#include #include /** diff --git a/server/modules/routing/debugcli/debugcmd.c b/server/modules/routing/debugcli/debugcmd.c index 93bb67d8b..247f69f87 100644 --- a/server/modules/routing/debugcli/debugcmd.c +++ b/server/modules/routing/debugcli/debugcmd.c @@ -64,7 +64,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/server/modules/routing/maxinfo/maxinfo_exec.c b/server/modules/routing/maxinfo/maxinfo_exec.c index 950ae0183..d3c84ab0b 100644 --- a/server/modules/routing/maxinfo/maxinfo_exec.c +++ b/server/modules/routing/maxinfo/maxinfo_exec.c @@ -44,7 +44,7 @@ #include #include #include -#include +#include static void exec_show(DCB *dcb, MAXINFO_TREE *tree); static void exec_select(DCB *dcb, MAXINFO_TREE *tree); From d04cb54b9fae4403b4dd05aada3cd38ec2ac175e Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Thu, 13 Oct 2016 21:11:35 +0300 Subject: [PATCH 023/215] maxscale/maxscale_pcre2.h renamed to maxscale/pcre2.h --- include/maxscale/externcmd.h | 2 +- include/maxscale/modutil.h | 2 +- include/maxscale/{maxscale_pcre2.h => pcre2.h} | 2 +- server/core/maxscale_pcre2.c | 2 +- server/core/modutil.c | 2 +- server/core/monitor.c | 2 +- server/core/test/testmaxscalepcre2.c | 2 +- server/modules/filter/cache/rules.h | 2 +- server/modules/filter/dbfwfilter/dbfwfilter.c | 2 +- server/modules/routing/avro/avro_file.c | 2 +- server/modules/routing/avro/avrorouter.h | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) rename include/maxscale/{maxscale_pcre2.h => pcre2.h} (93%) diff --git a/include/maxscale/externcmd.h b/include/maxscale/externcmd.h index ab4ed2460..eed094fdc 100644 --- a/include/maxscale/externcmd.h +++ b/include/maxscale/externcmd.h @@ -18,7 +18,7 @@ #include #include #include -#include +#include #define MAXSCALE_EXTCMD_ARG_MAX 256 diff --git a/include/maxscale/modutil.h b/include/maxscale/modutil.h index af48ca2ec..9ccc38345 100644 --- a/include/maxscale/modutil.h +++ b/include/maxscale/modutil.h @@ -29,7 +29,7 @@ #include #include #include -#include +#include #define PTR_IS_RESULTSET(b) (b[0] == 0x01 && b[1] == 0x0 && b[2] == 0x0 && b[3] == 0x01) #define PTR_IS_EOF(b) (b[0] == 0x05 && b[1] == 0x0 && b[2] == 0x0 && b[4] == 0xfe) diff --git a/include/maxscale/maxscale_pcre2.h b/include/maxscale/pcre2.h similarity index 93% rename from include/maxscale/maxscale_pcre2.h rename to include/maxscale/pcre2.h index 494b9e6a0..08c8e2b58 100644 --- a/include/maxscale/maxscale_pcre2.h +++ b/include/maxscale/pcre2.h @@ -21,7 +21,7 @@ #include /** - * @file maxscale_pcre2.h - Utility functions for regular expression matching + * @file pcre2.h - Utility functions for regular expression matching * with the bundled PCRE2 library. * * @verbatim diff --git a/server/core/maxscale_pcre2.c b/server/core/maxscale_pcre2.c index 484767994..04133f655 100644 --- a/server/core/maxscale_pcre2.c +++ b/server/core/maxscale_pcre2.c @@ -24,7 +24,7 @@ * @endverbatim */ -#include +#include #include /** diff --git a/server/core/modutil.c b/server/core/modutil.c index 92d4d6bf6..bb78c99e9 100644 --- a/server/core/modutil.c +++ b/server/core/modutil.c @@ -1099,7 +1099,7 @@ void prepare_pcre2_patterns() * @param string String to match * @return MXS_PCRE2_MATCH if the pattern matches, MXS_PCRE2_NOMATCH if it does * not match and MXS_PCRE2_ERROR if an error occurred - * @see maxscale_pcre2.h + * @see maxscale/pcre2.h */ mxs_pcre2_result_t modutil_mysql_wildcard_match(const char* pattern, const char* string) { diff --git a/server/core/monitor.c b/server/core/monitor.c index 8cdafe660..20f29323e 100644 --- a/server/core/monitor.c +++ b/server/core/monitor.c @@ -36,7 +36,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/server/core/test/testmaxscalepcre2.c b/server/core/test/testmaxscalepcre2.c index f75551f0c..b3c1b7395 100644 --- a/server/core/test/testmaxscalepcre2.c +++ b/server/core/test/testmaxscalepcre2.c @@ -34,7 +34,7 @@ #include #include #include -#include +#include #include #define test_assert(a, b) if(!(a)){fprintf(stderr, b);return 1;} diff --git a/server/modules/filter/cache/rules.h b/server/modules/filter/cache/rules.h index 58e04716c..c5590094f 100644 --- a/server/modules/filter/cache/rules.h +++ b/server/modules/filter/cache/rules.h @@ -17,7 +17,7 @@ #include #include #include -#include +#include typedef enum cache_rule_attribute diff --git a/server/modules/filter/dbfwfilter/dbfwfilter.c b/server/modules/filter/dbfwfilter/dbfwfilter.c index 72b30917f..01d13dfe1 100644 --- a/server/modules/filter/dbfwfilter/dbfwfilter.c +++ b/server/modules/filter/dbfwfilter/dbfwfilter.c @@ -74,7 +74,7 @@ #include #include #include -#include +#include #include "dbfwfilter.h" #include #include diff --git a/server/modules/routing/avro/avro_file.c b/server/modules/routing/avro/avro_file.c index 8fe34f3a7..bce83ffc8 100644 --- a/server/modules/routing/avro/avro_file.c +++ b/server/modules/routing/avro/avro_file.c @@ -35,7 +35,7 @@ #include #include "avrorouter.h" #include -#include +#include #include #include #include diff --git a/server/modules/routing/avro/avrorouter.h b/server/modules/routing/avro/avrorouter.h index 2180327cc..21812bf22 100644 --- a/server/modules/routing/avro/avrorouter.h +++ b/server/modules/routing/avro/avrorouter.h @@ -29,7 +29,7 @@ #include #include #include -#include +#include #include #include #include From 2fe58ebe5bf6f9d80dba0e930e7df47c851c8e62 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Thu, 13 Oct 2016 21:43:20 +0300 Subject: [PATCH 024/215] Move test_utils.h to server/core/test --- {include/maxscale => server/core/test}/test_utils.h | 0 server/core/test/testpoll.c | 2 +- server/core/test/testqueuemanager.c | 2 +- server/core/test/testservice.c | 2 +- server/modules/filter/dbfwfilter/dbfwfilter.c | 3 ++- 5 files changed, 5 insertions(+), 4 deletions(-) rename {include/maxscale => server/core/test}/test_utils.h (100%) diff --git a/include/maxscale/test_utils.h b/server/core/test/test_utils.h similarity index 100% rename from include/maxscale/test_utils.h rename to server/core/test/test_utils.h diff --git a/server/core/test/testpoll.c b/server/core/test/testpoll.c index 74c55b4f9..aea0cd1a5 100644 --- a/server/core/test/testpoll.c +++ b/server/core/test/testpoll.c @@ -35,8 +35,8 @@ #include #include #include -#include #include +#include "test_utils.h" /** * test1 Allocate a service and do lots of other things diff --git a/server/core/test/testqueuemanager.c b/server/core/test/testqueuemanager.c index e616d9c99..938010f54 100644 --- a/server/core/test/testqueuemanager.c +++ b/server/core/test/testqueuemanager.c @@ -39,9 +39,9 @@ extern int debug_check_fail; #include #include #include -#include #include #include +#include "test_utils.h" /** * test1 Allocate a queue and do lots of other things diff --git a/server/core/test/testservice.c b/server/core/test/testservice.c index 13ff6e9bc..b3101f1f7 100644 --- a/server/core/test/testservice.c +++ b/server/core/test/testservice.c @@ -33,10 +33,10 @@ #include #include #include -#include #include #include #include +#include "test_utils.h" /** * test1 Allocate a service and do lots of other things diff --git a/server/modules/filter/dbfwfilter/dbfwfilter.c b/server/modules/filter/dbfwfilter/dbfwfilter.c index 01d13dfe1..33e684bd1 100644 --- a/server/modules/filter/dbfwfilter/dbfwfilter.c +++ b/server/modules/filter/dbfwfilter/dbfwfilter.c @@ -2330,7 +2330,8 @@ diagnostic(FILTER *instance, void *fsession, DCB *dcb) } #ifdef BUILD_RULE_PARSER -#include +// TODO: Not ok to include file from other component's test directory. +#include "../../../core/test/test_utils.h" int main(int argc, char** argv) { From dc1f599b49bbb78b7586ea7c441a278ec655c957 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Fri, 14 Oct 2016 10:10:00 +0300 Subject: [PATCH 025/215] Add maxscale/cdefs.h The purpose of this file is to provide a header that *must* be included first (that is, also before any system headers) by all other headers. On the one hand this file provides a place where compilation environment dependent things can be defined, and, on the other hand, a place where things can be redefined globally, should that be necessary for whatever reason. It further provides a place where constants applicable across the line can be defined. --- include/maxscale/cdefs.h | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 include/maxscale/cdefs.h diff --git a/include/maxscale/cdefs.h b/include/maxscale/cdefs.h new file mode 100644 index 000000000..ed7de8ba7 --- /dev/null +++ b/include/maxscale/cdefs.h @@ -0,0 +1,38 @@ +#pragma once +#ifndef _MAXSCALE_CDEFS_H +#define _MAXSCALE_CDEFS_H +/* + * Copyright (c) 2016 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl. + * + * Change Date: 2019-07-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ + +/** + * @file cdefs.h + * + * This file has several purposes. + * + * - Its purpose is the same as that of x86_64-linux-gnu/sys/cdefs.h, that is, + * it defines things that are dependent upon the compilation environment. + * - Since this *must* be included as the very first header by all other MaxScale + * headers, it allows you to redfine things globally, should that be necessary, + * for instance, when debugging something. + * - Global constants applicable across the line can be defined here. + */ + +#ifdef __cplusplus +# define MXS_BEGIN_DECLS extern "C" { +# define MXS_END_DECLS } +#else +# define MXS_BEGIN_DECLS +# define MXS_END_DECLS +#endif + +#endif From 1a978be6b6577d74d3cc39d85ab4de624fff9a7f Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Fri, 14 Oct 2016 11:51:44 +0300 Subject: [PATCH 026/215] Cleanup header files - All now include maxscale/cdefs.h as the very first file. - MXS_[BEGIN|END]_DECLS added to all C-headers. Strictly speaking not necessary for private headers, but does not hurt either. - Include guards moved to the very top of the file. - #pragma once added. --- include/maxscale/adminusers.h.in | 11 ++++++-- include/maxscale/alloc.h | 7 +++--- include/maxscale/atomic.h | 19 ++++++++------ include/maxscale/buffer.h | 11 +++++--- include/maxscale/config.h | 7 ++++++ include/maxscale/dbusers.h | 9 +++++-- include/maxscale/dcb.h | 11 ++++++-- include/maxscale/externcmd.h | 10 ++++++-- include/maxscale/filter.h | 11 ++++++-- include/maxscale/gw.h | 12 ++++++--- include/maxscale/gw_authenticator.h | 10 ++++++-- include/maxscale/gw_protocol.h | 9 +++++-- include/maxscale/gw_ssl.h | 10 ++++++-- include/maxscale/gwbitmask.h | 10 ++++++-- include/maxscale/gwdirs.h.in | 11 +++++--- include/maxscale/hashtable.h | 10 +++++--- include/maxscale/hint.h | 10 ++++++-- include/maxscale/hk_heartbeat.h | 11 ++++++-- include/maxscale/housekeeper.h | 12 +++++++-- include/maxscale/limits.h | 7 ++++++ include/maxscale/listener.h | 10 ++++++-- include/maxscale/listmanager.h | 9 +++++-- include/maxscale/log_manager.h | 14 +++++------ include/maxscale/maxadmin.h | 11 ++++++-- include/maxscale/maxscale.h | 9 +++++-- include/maxscale/memlog.h | 11 ++++++-- include/maxscale/mlist.h | 10 +++++--- include/maxscale/modinfo.h | 12 +++++++-- include/maxscale/modules.h | 11 +++++--- include/maxscale/modutil.h | 11 ++++++-- include/maxscale/monitor.h | 11 ++++++-- include/maxscale/mysql_binlog.h | 10 ++++++-- include/maxscale/mysql_utils.h | 11 +++++--- include/maxscale/notification.h | 12 +++++++-- include/maxscale/pcre2.h | 7 ++++++ include/maxscale/platform.h | 11 ++++++-- include/maxscale/poll.h | 12 +++++++-- include/maxscale/query_classifier.h | 10 +++++--- include/maxscale/queuemanager.h | 10 ++++++-- include/maxscale/random_jkiss.h | 15 ++++++----- include/maxscale/rdtsc.h | 12 +++++++-- include/maxscale/resultset.h | 10 ++++++-- include/maxscale/router.h | 11 +++++--- include/maxscale/secrets.h | 11 +++++--- include/maxscale/server.h | 11 ++++++-- include/maxscale/service.h | 11 ++++++-- include/maxscale/session.h | 12 +++++++-- include/maxscale/skygw_debug.h | 20 ++++++--------- include/maxscale/skygw_types.h | 11 +++++--- include/maxscale/skygw_utils.h | 25 ++++++------------- include/maxscale/slist.h | 10 +++++--- include/maxscale/spinlock.h | 11 +++++--- include/maxscale/statistics.h | 10 ++++++-- include/maxscale/thread.h | 11 ++++++-- include/maxscale/users.h | 11 ++++++-- include/maxscale/utils.h | 11 ++++++-- server/core/test/test_utils.h | 3 ++- server/modules/authenticator/gssapi_auth.h | 11 +++++--- server/modules/filter/cache/cache.h | 9 +++++-- .../modules/filter/cache/cache_storage_api.h | 9 ++++--- server/modules/filter/cache/rules.h | 5 ++++ server/modules/filter/cache/storage.h | 6 +++++ .../storage_rocksdb/rocksdbinternals.h | 1 + .../storage/storage_rocksdb/rocksdbstorage.h | 2 ++ .../storage/storage_rocksdb/storage_rocksdb.h | 2 ++ server/modules/filter/dbfwfilter/dbfwfilter.h | 6 +++++ server/modules/filter/hintfilter/mysqlhint.h | 8 ++++-- server/modules/include/binlog_common.h | 11 ++++++-- server/modules/include/blr_constants.h | 14 ++++++++--- server/modules/include/cdc.h | 10 ++++++++ server/modules/include/debugcli.h | 8 ++++++ server/modules/include/mysql_auth.h | 6 +++++ server/modules/include/telnetd.h | 9 +++++++ server/modules/monitor/galeramon/galeramon.h | 6 +++++ server/modules/monitor/mmmon/mmmon.h | 6 +++++ server/modules/monitor/mysqlmon.h | 6 +++++ server/modules/protocol/HTTPD/httpd.h | 10 ++++++++ server/modules/protocol/maxscaled/maxscaled.h | 6 +++++ server/modules/routing/avro/avrorouter.h | 10 ++++++-- server/modules/routing/binlog/blr.h | 6 +++++ server/modules/routing/maxinfo/maxinfo.h | 7 ++++++ .../routing/readconnroute/readconnection.h | 8 ++++++ .../routing/readwritesplit/readwritesplit.h | 5 ++++ .../routing/readwritesplit/rwsplit_internal.h | 17 +++++++------ .../routing/schemarouter/schemarouter.h | 12 ++++++--- .../routing/schemarouter/sharding_common.h | 11 +++++--- 86 files changed, 647 insertions(+), 206 deletions(-) diff --git a/include/maxscale/adminusers.h.in b/include/maxscale/adminusers.h.in index 25e6acbe2..bc9302b13 100644 --- a/include/maxscale/adminusers.h.in +++ b/include/maxscale/adminusers.h.in @@ -1,5 +1,6 @@ -#ifndef _ADMINUSERS_H -#define _ADMINUSERS_H +#pragma once +#ifndef _MAXSCALE_ADMINUSERS_H +#define _MAXSCALE_ADMINUSERS_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -24,8 +25,12 @@ * * @endverbatim */ + +#include #include +MXS_BEGIN_DECLS + #define ADMIN_SALT "$1$MXS" /* Max length of fields in for admin users */ @@ -63,4 +68,6 @@ extern bool admin_verify_inet_user(const char *uname, const char *passwor extern void dcb_PrintAdminUsers(DCB *dcb); +MXS_END_DECLS + #endif diff --git a/include/maxscale/alloc.h b/include/maxscale/alloc.h index e138d3af3..49cd04a29 100644 --- a/include/maxscale/alloc.h +++ b/include/maxscale/alloc.h @@ -1,3 +1,4 @@ +#pragma once #ifndef _MAXSCALE_ALLOC_H #define _MAXSCALE_ALLOC_H /* @@ -13,11 +14,11 @@ * Public License. */ +#include #include #include -#include -EXTERN_C_BLOCK_BEGIN +MXS_BEGIN_DECLS /* * NOTE: Do not use these functions directly, use the macros below. @@ -77,6 +78,6 @@ char *mxs_strndup_a(const char *s, size_t n/*, const char *caller*/); */ #define MXS_ABORT_IF_FALSE(b) do { if (!b) { abort(); } } while (false) -EXTERN_C_BLOCK_END +MXS_END_DECLS #endif diff --git a/include/maxscale/atomic.h b/include/maxscale/atomic.h index bcd76f9a4..71127ee7c 100644 --- a/include/maxscale/atomic.h +++ b/include/maxscale/atomic.h @@ -1,5 +1,6 @@ -#ifndef _ATOMIC_H -#define _ATOMIC_H +#pragma once +#ifndef _MAXSCALE_ATOMIC_H +#define _MAXSCALE_ATOMIC_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -26,9 +27,13 @@ * @endverbatim */ -#ifdef __cplusplus -extern "C" int atomic_add(int *variable, int value); -#else -extern int atomic_add(int *variable, int value); -#endif +#include + +MXS_BEGIN_DECLS + +int atomic_add(int *variable, int value); +int atomic_add(int *variable, int value); + +MXS_END_DECLS + #endif diff --git a/include/maxscale/buffer.h b/include/maxscale/buffer.h index 935652671..3c71ea223 100644 --- a/include/maxscale/buffer.h +++ b/include/maxscale/buffer.h @@ -1,5 +1,6 @@ -#ifndef _BUFFER_H -#define _BUFFER_H +#pragma once +#ifndef _MAXSCALE_BUFFER_H +#define _MAXSCALE_BUFFER_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -42,13 +43,15 @@ * * @endverbatim */ + +#include #include #include #include #include #include -EXTERN_C_BLOCK_BEGIN +MXS_BEGIN_DECLS /** * Buffer properties - used to store properties related to the buffer @@ -214,7 +217,7 @@ void* gwbuf_get_buffer_object_data(GWBUF* buf, bufobj_id_t id) #if defined(BUFFER_TRACE) extern void dprintAllBuffers(void *pdcb); #endif -EXTERN_C_BLOCK_END +MXS_END_DECLS #endif diff --git a/include/maxscale/config.h b/include/maxscale/config.h index 2bf3c650b..caa9cdf84 100644 --- a/include/maxscale/config.h +++ b/include/maxscale/config.h @@ -1,3 +1,4 @@ +#pragma once #ifndef _MAXSCALE_CONFIG_H #define _MAXSCALE_CONFIG_H /* @@ -12,6 +13,8 @@ * of this software will be governed by version 2 or later of the General * Public License. */ + +#include #include #include #include @@ -33,6 +36,8 @@ * @endverbatim */ +MXS_BEGIN_DECLS + #define DEFAULT_NBPOLLS 3 /**< Default number of non block polls before we block */ #define DEFAULT_POLLSLEEP 1000 /**< Default poll wait time (milliseconds) */ #define _RELEASE_STR_LENGTH 256 /**< release len */ @@ -160,4 +165,6 @@ int config_truth_value(char *); void free_config_parameter(CONFIG_PARAMETER* p1); bool is_internal_service(const char *router); +MXS_END_DECLS + #endif diff --git a/include/maxscale/dbusers.h b/include/maxscale/dbusers.h index ea669b4df..7ef5a71a9 100644 --- a/include/maxscale/dbusers.h +++ b/include/maxscale/dbusers.h @@ -1,5 +1,6 @@ -#ifndef _DBUSERS_H -#define _DBUSERS_H +#pragma once +#ifndef _MAXSCALE_DBUSERS_H +#define _MAXSCALE_DBUSERS_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -13,9 +14,11 @@ * Public License. */ +#include #include #include +MXS_BEGIN_DECLS /** * @file dbusers.h Extarct user information form the backend database @@ -77,4 +80,6 @@ extern char *mysql_users_fetch(USERS *users, MYSQL_USER_HOST *key); extern int reload_mysql_users(SERV_LISTENER *listener); extern int replace_mysql_users(SERV_LISTENER *listener); +MXS_END_DECLS + #endif diff --git a/include/maxscale/dcb.h b/include/maxscale/dcb.h index 3a6020003..43aab8911 100644 --- a/include/maxscale/dcb.h +++ b/include/maxscale/dcb.h @@ -1,5 +1,6 @@ -#ifndef _DCB_H -#define _DCB_H +#pragma once +#ifndef _MAXSCALE_DCB_H +#define _MAXSCALE_DCB_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -12,6 +13,7 @@ * of this software will be governed by version 2 or later of the General * Public License. */ +#include #include #include #include @@ -23,6 +25,8 @@ #include #include +MXS_BEGIN_DECLS + #define ERRHANDLE struct session; @@ -369,4 +373,7 @@ void dcb_append_readqueue(DCB *dcb, GWBUF *buffer); #define DCB_IS_CLONE(d) ((d)->flags & DCBF_CLONE) #define DCB_REPLIED(d) ((d)->flags & DCBF_REPLIED) + +MXS_END_DECLS + #endif /* _DCB_H */ diff --git a/include/maxscale/externcmd.h b/include/maxscale/externcmd.h index eed094fdc..ed0a90f48 100644 --- a/include/maxscale/externcmd.h +++ b/include/maxscale/externcmd.h @@ -1,5 +1,6 @@ -#ifndef _EXTERN_CMD_HG -#define _EXTERN_CMD_HG +#pragma once +#ifndef _MAXSCALE_EXTERN_CMD_HG +#define _MAXSCALE_EXTERN_CMD_HG /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -13,6 +14,7 @@ * Public License. */ +#include #include #include #include @@ -20,6 +22,8 @@ #include #include +MXS_BEGIN_DECLS + #define MAXSCALE_EXTCMD_ARG_MAX 256 typedef struct extern_cmd_t @@ -38,4 +42,6 @@ bool externcmd_substitute_arg(EXTERNCMD* cmd, const char* re, const char* replac bool externcmd_can_execute(const char* argstr); bool externcmd_matches(const EXTERNCMD* cmd, const char* match); +MXS_END_DECLS + #endif diff --git a/include/maxscale/filter.h b/include/maxscale/filter.h index 3ccb8ac4c..780ed06f2 100644 --- a/include/maxscale/filter.h +++ b/include/maxscale/filter.h @@ -1,5 +1,6 @@ -#ifndef _FILTER_H -#define _FILTER_H +#pragma once +#ifndef _MAXSCALE_FILTER_H +#define _MAXSCALE_FILTER_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -22,11 +23,15 @@ * 27/05/2014 Mark Riddoch Initial implementation * */ + +#include #include #include #include #include +MXS_BEGIN_DECLS + /** * The FILTER handle points to module specific data, so the best we can do * is to make it a void * externally. @@ -116,4 +121,6 @@ void dprintAllFilters(DCB *); void dprintFilter(DCB *, FILTER_DEF *); void dListFilters(DCB *); +MXS_END_DECLS + #endif diff --git a/include/maxscale/gw.h b/include/maxscale/gw.h index 17b011a28..085c73e58 100644 --- a/include/maxscale/gw.h +++ b/include/maxscale/gw.h @@ -1,5 +1,6 @@ -#ifndef _GW_HG -#define _GW_HG +#pragma once +#ifndef _MAXSCALE_GW_HG +#define _MAXSCALE_GW_HG /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -13,7 +14,7 @@ * Public License. */ - +#include #include #include #include @@ -35,6 +36,8 @@ #include #include +MXS_BEGIN_DECLS + #define EXIT_FAILURE 1 // network buffer is 32K @@ -83,4 +86,7 @@ char* get_libdir(); long get_processor_count(); void clean_up_pathname(char *path); bool mxs_mkdir_all(const char *path, int mask); + +MXS_END_DECLS + #endif diff --git a/include/maxscale/gw_authenticator.h b/include/maxscale/gw_authenticator.h index 5e999fb02..79afbeebd 100644 --- a/include/maxscale/gw_authenticator.h +++ b/include/maxscale/gw_authenticator.h @@ -1,5 +1,6 @@ -#ifndef GW_AUTHENTICATOR_H -#define GW_AUTHENTICATOR_H +#pragma once +#ifndef _MAXSCALE_GW_AUTHENTICATOR_H +#define _MAXSCALE_GW_AUTHENTICATOR_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -27,12 +28,15 @@ * @endverbatim */ +#include #include #include #include #include #include +MXS_BEGIN_DECLS + /** Maximum number of authenticator options */ #define AUTHENTICATOR_MAX_OPTIONS 256 @@ -134,5 +138,7 @@ typedef enum bool authenticator_init(void **instance, const char *authenticator, const char *options); char* get_default_authenticator(const char *protocol); +MXS_END_DECLS + #endif /* GW_AUTHENTICATOR_H */ diff --git a/include/maxscale/gw_protocol.h b/include/maxscale/gw_protocol.h index e2a3295da..aa5ad2abf 100644 --- a/include/maxscale/gw_protocol.h +++ b/include/maxscale/gw_protocol.h @@ -1,5 +1,6 @@ -#ifndef GW_PROTOCOL_H -#define GW_PROTOCOL_H +#pragma once +#ifndef _MAXSCALE_GW_PROTOCOL_H +#define _MAXSCALE_GW_PROTOCOL_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -28,8 +29,11 @@ * @endverbatim */ +#include #include +MXS_BEGIN_DECLS + struct dcb; struct server; struct session; @@ -81,6 +85,7 @@ typedef struct gw_protocol */ #define GWPROTOCOL_VERSION {1, 1, 0} +MXS_END_DECLS #endif /* GW_PROTOCOL_H */ diff --git a/include/maxscale/gw_ssl.h b/include/maxscale/gw_ssl.h index bfcc387cf..65ac9ca82 100644 --- a/include/maxscale/gw_ssl.h +++ b/include/maxscale/gw_ssl.h @@ -1,5 +1,6 @@ -#ifndef _GW_SSL_H -#define _GW_SSL_H +#pragma once +#ifndef _MAXSCALE_GW_SSL_H +#define _MAXSCALE_GW_SSL_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -27,12 +28,15 @@ * @endverbatim */ +#include #include #include #include #include #include +MXS_BEGIN_DECLS + struct dcb; typedef enum ssl_method_type @@ -78,4 +82,6 @@ bool ssl_required_by_dcb(struct dcb *dcb); bool ssl_required_but_not_negotiated(struct dcb *dcb); const char* ssl_method_type_to_string(ssl_method_type_t method_type); +MXS_END_DECLS + #endif /* _GW_SSL_H */ diff --git a/include/maxscale/gwbitmask.h b/include/maxscale/gwbitmask.h index 31ee13655..1f9130f69 100644 --- a/include/maxscale/gwbitmask.h +++ b/include/maxscale/gwbitmask.h @@ -1,5 +1,6 @@ -#ifndef _GWBITMASK_H -#define _GWBITMASK_H +#pragma once +#ifndef _MAXSCALE_GWBITMASK_H +#define _MAXSCALE_GWBITMASK_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -13,9 +14,12 @@ * Public License. */ +#include #include #include +MXS_BEGIN_DECLS + /** * @file gwbitmask.h An implementation of an arbitrarily long bitmask * @@ -55,4 +59,6 @@ extern int bitmask_isallclear(GWBITMASK *); extern void bitmask_copy(GWBITMASK *, GWBITMASK *); extern char *bitmask_render_readable(GWBITMASK *); +MXS_END_DECLS + #endif diff --git a/include/maxscale/gwdirs.h.in b/include/maxscale/gwdirs.h.in index 14592698c..39e55b0b2 100644 --- a/include/maxscale/gwdirs.h.in +++ b/include/maxscale/gwdirs.h.in @@ -1,5 +1,6 @@ -#ifndef _GW_DIRS_HG -#define _GW_DIRS_HG +#pragma once +#ifndef _MAXSCALE_GW_DIRS_HG +#define _MAXSCALE_GW_DIRS_HG /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -15,11 +16,13 @@ #ifndef _GNU_SOURCE #define _GNU_SOURCE 1 #endif + +#include #include #include #include -EXTERN_C_BLOCK_BEGIN +MXS_BEGIN_DECLS /** * All of the following DEFAULT_* variables are defined in cmake/install_layout.cmake @@ -76,6 +79,6 @@ char* get_logdir(); char* get_langdir(); char* get_execdir(); -EXTERN_C_BLOCK_END +MXS_END_DECLS #endif diff --git a/include/maxscale/hashtable.h b/include/maxscale/hashtable.h index 0b271d731..0f176d8ec 100644 --- a/include/maxscale/hashtable.h +++ b/include/maxscale/hashtable.h @@ -1,5 +1,6 @@ -#ifndef _HASTABLE_H -#define _HASTABLE_H +#pragma once +#ifndef _MAXSCALE_HASTABLE_H +#define _MAXSCALE_HASTABLE_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -28,10 +29,11 @@ * * @endverbatim */ +#include #include #include -EXTERN_C_BLOCK_BEGIN +MXS_BEGIN_DECLS /** * The entries within a hashtable. @@ -150,6 +152,6 @@ extern int hashtable_item_strcmp(const void* str1, const void* str2); extern void* hashtable_item_strdup(const void *str); extern int hashtable_item_strhash(const void *str); -EXTERN_C_BLOCK_END +MXS_END_DECLS #endif diff --git a/include/maxscale/hint.h b/include/maxscale/hint.h index ea073292f..4cd2b8bbc 100644 --- a/include/maxscale/hint.h +++ b/include/maxscale/hint.h @@ -1,5 +1,6 @@ -#ifndef _HINT_H -#define _HINT_H +#pragma once +#ifndef _MAXSCALE_HINT_H +#define _MAXSCALE_HINT_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -25,8 +26,10 @@ * @endverbatim */ +#include #include +MXS_BEGIN_DECLS /** * The types of hint that are supported by the generic hinting mechanism. @@ -63,4 +66,7 @@ extern HINT *hint_create_route(HINT *, HINT_TYPE, char *); extern void hint_free(HINT *); extern HINT *hint_dup(HINT *); bool hint_exists(HINT **, HINT_TYPE); + +MXS_END_DECLS + #endif diff --git a/include/maxscale/hk_heartbeat.h b/include/maxscale/hk_heartbeat.h index ab9172074..bdc2567ef 100644 --- a/include/maxscale/hk_heartbeat.h +++ b/include/maxscale/hk_heartbeat.h @@ -1,5 +1,6 @@ -#ifndef _HK_HEARTBEAT_H -#define _HK_HEARTBEAT_H +#pragma once +#ifndef _MAXSCALE_HK_HEARTBEAT_H +#define _MAXSCALE_HK_HEARTBEAT_H /* * Copyright (c) 2016 MariaDB Corporation Ab @@ -14,6 +15,10 @@ * Public License. */ +#include + +MXS_BEGIN_DECLS + /** * The global housekeeper heartbeat value. This value is incremented * every 100 milliseconds and may be used for crude timing etc. @@ -21,4 +26,6 @@ extern long hkheartbeat; +MXS_END_DECLS + #endif diff --git a/include/maxscale/housekeeper.h b/include/maxscale/housekeeper.h index 6f0bea8db..131a0fbae 100644 --- a/include/maxscale/housekeeper.h +++ b/include/maxscale/housekeeper.h @@ -1,5 +1,6 @@ -#ifndef _HOUSEKEEPER_H -#define _HOUSEKEEPER_H +#pragma once +#ifndef _MAXSCALE_HOUSEKEEPER_H +#define _MAXSCALE_HOUSEKEEPER_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -12,9 +13,14 @@ * of this software will be governed by version 2 or later of the General * Public License. */ + +#include #include #include #include + +MXS_BEGIN_DECLS + /** * @file housekeeper.h A mechanism to have task run periodically * @@ -54,4 +60,6 @@ extern int hktask_remove(const char *name); extern void hkshutdown(); extern void hkshow_tasks(DCB *pdcb); +MXS_END_DECLS + #endif diff --git a/include/maxscale/limits.h b/include/maxscale/limits.h index 083a255dc..6847683d9 100644 --- a/include/maxscale/limits.h +++ b/include/maxscale/limits.h @@ -1,3 +1,4 @@ +#pragma once #ifndef _MAXSCALE_LIMITS_H #define _MAXSCALE_LIMITS_H /* @@ -13,6 +14,10 @@ * Public License. */ +#include + +MXS_BEGIN_DECLS + // This file defines hard limits of MaxScale. // Thread information is stored in a bitmask whose size must be a @@ -21,4 +26,6 @@ // multiple of 8 minus 1. #define MXS_MAX_THREADS 255 +MXS_END_DECLS + #endif diff --git a/include/maxscale/listener.h b/include/maxscale/listener.h index b2a2dbde5..0e8cb6097 100644 --- a/include/maxscale/listener.h +++ b/include/maxscale/listener.h @@ -1,5 +1,6 @@ -#ifndef _LISTENER_H -#define _LISTENER_H +#pragma once +#ifndef _MAXSCALE_LISTENER_H +#define _MAXSCALE_LISTENER_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -27,10 +28,13 @@ * @endverbatim */ +#include #include #include #include +MXS_BEGIN_DECLS + struct dcb; struct service; @@ -65,4 +69,6 @@ int listener_set_ssl_version(SSL_LISTENER *ssl_listener, char* version); void listener_set_certificates(SSL_LISTENER *ssl_listener, char* cert, char* key, char* ca_cert); int listener_init_SSL(SSL_LISTENER *ssl_listener); +MXS_END_DECLS + #endif diff --git a/include/maxscale/listmanager.h b/include/maxscale/listmanager.h index 5afd28375..10c3e6877 100644 --- a/include/maxscale/listmanager.h +++ b/include/maxscale/listmanager.h @@ -1,5 +1,6 @@ -#ifndef _LISTMANAGER_H -#define _LISTMANAGER_H +#pragma once +#ifndef _MAXSCALE_LISTMANAGER_H +#define _MAXSCALE_LISTMANAGER_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -26,9 +27,12 @@ * @endverbatim */ +#include #include #include +MXS_BEGIN_DECLS + struct dcb; /* @@ -115,5 +119,6 @@ void list_map(LIST_CONFIG *list_config, bool (*callback)(void *, ...)); list_entry_t *list_remove_first(LIST_CONFIG *list_config); list_entry_t *list_remove_last(LIST_CONFIG *list_config); +MXS_END_DECLS #endif /* LISTMANAGER_H */ diff --git a/include/maxscale/log_manager.h b/include/maxscale/log_manager.h index 127d3e985..357706c8f 100644 --- a/include/maxscale/log_manager.h +++ b/include/maxscale/log_manager.h @@ -1,3 +1,6 @@ +#pragma once +#ifndef _MAXSCALE_LOG_MANAGER_H +#define _MAXSCALE_LOG_MANAGER_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -10,16 +13,13 @@ * of this software will be governed by version 2 or later of the General * Public License. */ -#if !defined(LOG_MANAGER_H) -#define LOG_MANAGER_H +#include #include #include #include -#if defined(__cplusplus) -extern "C" { -#endif +MXS_BEGIN_DECLS /* * We need a common.h file that is included by every component. @@ -195,8 +195,6 @@ enum trailing NULL. If longer, it will be cut. */ }; -#if defined(__cplusplus) -} -#endif +MXS_END_DECLS #endif /** LOG_MANAGER_H */ diff --git a/include/maxscale/maxadmin.h b/include/maxscale/maxadmin.h index bef05a20d..8eb577b6d 100644 --- a/include/maxscale/maxadmin.h +++ b/include/maxscale/maxadmin.h @@ -1,5 +1,6 @@ -#ifndef _MAXADMIN_H -#define _MAXADMIN_H +#pragma once +#ifndef _MAXSCALE_MAXADMIN_H +#define _MAXSCALE_MAXADMIN_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -13,6 +14,10 @@ * Public License. */ +#include + +MXS_BEGIN_DECLS + #define MAXADMIN_DEFAULT_SOCKET "/tmp/maxadmin.sock" #define MAXADMIN_CONFIG_DEFAULT_SOCKET_TAG_LEN 7 @@ -28,4 +33,6 @@ #define MAXADMIN_AUTH_PASSWORD_PROMPT "PASSWORD" #define MAXADMIN_AUTH_PASSWORD_PROMPT_LEN 8 +MXS_END_DECLS + #endif diff --git a/include/maxscale/maxscale.h b/include/maxscale/maxscale.h index 020e858d0..5263409cf 100644 --- a/include/maxscale/maxscale.h +++ b/include/maxscale/maxscale.h @@ -1,5 +1,6 @@ -#ifndef _MAXSCALE_H -#define _MAXSCALE_H +#pragma once +#ifndef _MAXSCALE_MAXSCALE_H +#define _MAXSCALE_MAXSCALE_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -27,8 +28,10 @@ * @endverbatim */ +#include #include +MXS_BEGIN_DECLS /* Exit status for MaxScale */ #define MAXSCALE_SHUTDOWN 0 /* Good shutdown */ @@ -43,4 +46,6 @@ void maxscale_reset_starttime(void); time_t maxscale_started(void); int maxscale_uptime(void); +MXS_END_DECLS + #endif diff --git a/include/maxscale/memlog.h b/include/maxscale/memlog.h index ef832b673..5c6fdfa6c 100644 --- a/include/maxscale/memlog.h +++ b/include/maxscale/memlog.h @@ -1,5 +1,6 @@ -#ifndef _MEMLOG_H -#define _MEMLOG_H +#pragma once +#ifndef _MAXSCALE_MEMLOG_H +#define _MAXSCALE_MEMLOG_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -24,8 +25,12 @@ * * @endverbatim */ + +#include #include +MXS_BEGIN_DECLS + typedef enum { ML_INT, ML_LONG, ML_LONGLONG, ML_STRING } MEMLOGTYPE; typedef struct memlog @@ -59,4 +64,6 @@ extern void memlog_log(MEMLOG *, void *); extern void memlog_flush_all(); extern void memlog_flush(MEMLOG *); +MXS_END_DECLS + #endif diff --git a/include/maxscale/mlist.h b/include/maxscale/mlist.h index 36eec847e..c3ccc1695 100644 --- a/include/maxscale/mlist.h +++ b/include/maxscale/mlist.h @@ -1,5 +1,6 @@ -#ifndef _MLIST_H -#define _MLIST_H +#pragma once +#ifndef _MAXSCALE_MLIST_H +#define _MAXSCALE_MLIST_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -13,9 +14,10 @@ * Public License. */ +#include #include -EXTERN_C_BLOCK_BEGIN +MXS_BEGIN_DECLS typedef struct mlist_node_st mlist_node_t; @@ -74,6 +76,6 @@ mlist_cursor_t* mlist_cursor_init(mlist_t* ml); void* mlist_cursor_get_data_nomutex(mlist_cursor_t* c); bool mlist_cursor_move_to_first(mlist_cursor_t* c); -EXTERN_C_BLOCK_END +MXS_END_DECLS #endif diff --git a/include/maxscale/modinfo.h b/include/maxscale/modinfo.h index 686ac9d45..99d68a37f 100644 --- a/include/maxscale/modinfo.h +++ b/include/maxscale/modinfo.h @@ -1,5 +1,6 @@ -#ifndef _MODINFO_H -#define _MODINFO_H +#pragma once +#ifndef _MAXSCALE_MODINFO_H +#define _MAXSCALE_MODINFO_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -25,6 +26,10 @@ * @endverbatim */ +#include + +MXS_BEGIN_DECLS + /** * The status of the module. This gives some idea of the module * maturity. @@ -83,4 +88,7 @@ typedef struct MODULE_VERSION api_version; char *description; } MODULE_INFO; + +MXS_END_DECLS + #endif diff --git a/include/maxscale/modules.h b/include/maxscale/modules.h index 49abc562c..9e62922c1 100644 --- a/include/maxscale/modules.h +++ b/include/maxscale/modules.h @@ -1,5 +1,6 @@ -#ifndef _MODULES_H -#define _MODULES_H +#pragma once +#ifndef _MAXSCALE_MODULES_H +#define _MAXSCALE_MODULES_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -12,12 +13,14 @@ * of this software will be governed by version 2 or later of the General * Public License. */ + +#include #include #include #include #include -EXTERN_C_BLOCK_BEGIN +MXS_BEGIN_DECLS /** * @file modules.h Utilities for loading modules @@ -71,6 +74,6 @@ extern RESULTSET *moduleGetList(); extern void module_feedback_send(void*); extern void moduleShowFeedbackReport(DCB *dcb); -EXTERN_C_BLOCK_END +MXS_END_DECLS #endif diff --git a/include/maxscale/modutil.h b/include/maxscale/modutil.h index 9ccc38345..9f609f795 100644 --- a/include/maxscale/modutil.h +++ b/include/maxscale/modutil.h @@ -1,5 +1,6 @@ -#ifndef _MODUTIL_H -#define _MODUTIL_H +#pragma once +#ifndef _MAXSCALE_MODUTIL_H +#define _MAXSCALE_MODUTIL_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -26,11 +27,15 @@ * * @endverbatim */ + +#include #include #include #include #include +MXS_BEGIN_DECLS + #define PTR_IS_RESULTSET(b) (b[0] == 0x01 && b[1] == 0x0 && b[2] == 0x0 && b[3] == 0x01) #define PTR_IS_EOF(b) (b[0] == 0x05 && b[1] == 0x0 && b[2] == 0x0 && b[4] == 0xfe) #define PTR_IS_OK(b) (b[4] == 0x00) @@ -70,4 +75,6 @@ bool is_mysql_statement_end(const char* start, int len); bool is_mysql_sp_end(const char* start, int len); char* modutil_get_canonical(GWBUF* querybuf); +MXS_END_DECLS + #endif diff --git a/include/maxscale/monitor.h b/include/maxscale/monitor.h index a5dac0abb..18093e457 100644 --- a/include/maxscale/monitor.h +++ b/include/maxscale/monitor.h @@ -1,5 +1,6 @@ -#ifndef _MONITOR_H -#define _MONITOR_H +#pragma once +#ifndef _MAXSCALE_MONITOR_H +#define _MAXSCALE_MONITOR_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -12,6 +13,8 @@ * of this software will be governed by version 2 or later of the General * Public License. */ + +#include #include #include #include @@ -21,6 +24,8 @@ #include #include +MXS_BEGIN_DECLS + /** * @file monitor.h The interface to the monitor module * @@ -226,4 +231,6 @@ connect_result_t mon_connect_to_db(MONITOR* mon, MONITOR_SERVERS *database); void mon_log_connect_error(MONITOR_SERVERS* database, connect_result_t rval); void mon_log_state_change(MONITOR_SERVERS *ptr); +MXS_END_DECLS + #endif diff --git a/include/maxscale/mysql_binlog.h b/include/maxscale/mysql_binlog.h index 62b981d64..2e7f4c5de 100644 --- a/include/maxscale/mysql_binlog.h +++ b/include/maxscale/mysql_binlog.h @@ -1,5 +1,6 @@ -#ifndef MYSQL_BINLOG_H -#define MYSQL_BINLOG_H +#pragma once +#ifndef _MAXSCALE_MYSQL_BINLOG_H +#define _MAXSCALE_MYSQL_BINLOG_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -17,10 +18,13 @@ * @file mysql_binlog.h - Extracting information from binary logs */ +#include #include #include #include +MXS_BEGIN_DECLS + /** Maximum GTID string length */ #define GTID_MAX_LEN 64 @@ -91,4 +95,6 @@ uint64_t unpack_bit(uint8_t *ptr, uint8_t *null_mask, uint32_t col_count, void format_temporal_value(char *str, size_t size, uint8_t type, struct tm *tm); +MXS_END_DECLS + #endif /* MYSQL_BINLOG_H */ diff --git a/include/maxscale/mysql_utils.h b/include/maxscale/mysql_utils.h index 329dfd6f8..f5b4d79ce 100644 --- a/include/maxscale/mysql_utils.h +++ b/include/maxscale/mysql_utils.h @@ -1,5 +1,6 @@ -#ifndef _MYSQL_UTILS_H -#define _MYSQL_UTILS_H +#pragma once +#ifndef _MAXSCALE_MYSQL_UTILS_H +#define _MAXSCALE_MYSQL_UTILS_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -13,11 +14,14 @@ * Public License. */ +#include #include #include #include #include +MXS_BEGIN_DECLS + /** Length-encoded integers */ size_t leint_bytes(uint8_t* ptr); uint64_t leint_value(uint8_t* c); @@ -27,7 +31,8 @@ uint64_t leint_consume(uint8_t ** c); char* lestr_consume_dup(uint8_t** c); char* lestr_consume(uint8_t** c, size_t *size); - MYSQL *mxs_mysql_real_connect(MYSQL *mysql, SERVER *server, const char *user, const char *passwd); +MXS_END_DECLS + #endif diff --git a/include/maxscale/notification.h b/include/maxscale/notification.h index 1014f84a1..db410910f 100644 --- a/include/maxscale/notification.h +++ b/include/maxscale/notification.h @@ -1,5 +1,6 @@ -#ifndef _NOTIFICATION_SERVICE_H -#define _NOTIFICATION_SERVICE_H +#pragma once +#ifndef _MAXSCALE_NOTIFICATION_H +#define _MAXSCALE_NOTIFICATION_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -27,6 +28,10 @@ * @endverbatim */ +#include + +MXS_BEGIN_DECLS + #define _NOTIFICATION_CONNECT_TIMEOUT 30 #define _NOTIFICATION_OPERATION_TIMEOUT 30 #define _NOTIFICATION_SEND_PENDING 0 @@ -57,4 +62,7 @@ typedef struct extern char *gw_bin2hex(char *out, const uint8_t *in, unsigned int len); extern void gw_sha1_str(const uint8_t *in, int in_len, uint8_t *out); extern FEEDBACK_CONF * config_get_feedback_data(); + +MXS_END_DECLS + #endif diff --git a/include/maxscale/pcre2.h b/include/maxscale/pcre2.h index 08c8e2b58..342d79bde 100644 --- a/include/maxscale/pcre2.h +++ b/include/maxscale/pcre2.h @@ -1,3 +1,4 @@ +#pragma once #ifndef _MAXSCALE_PCRE2_H #define _MAXSCALE_PCRE2_H /* @@ -14,12 +15,16 @@ * */ +#include + #ifndef PCRE2_CODE_UNIT_WIDTH #define PCRE2_CODE_UNIT_WIDTH 8 #endif #include +MXS_BEGIN_DECLS + /** * @file pcre2.h - Utility functions for regular expression matching * with the bundled PCRE2 library. @@ -44,4 +49,6 @@ mxs_pcre2_result_t mxs_pcre2_substitute(pcre2_code *re, const char *subject, mxs_pcre2_result_t mxs_pcre2_simple_match(const char* pattern, const char* subject, int options, int* error); +MXS_END_DECLS + #endif diff --git a/include/maxscale/platform.h b/include/maxscale/platform.h index 69011f3d4..486898d0b 100644 --- a/include/maxscale/platform.h +++ b/include/maxscale/platform.h @@ -1,5 +1,6 @@ -#ifndef _PLATFORM_H -#define _PLATFORM_H +#pragma once +#ifndef _MAXSCALE_PLATFORM_H +#define _MAXSCALE_PLATFORM_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -13,6 +14,10 @@ * Public License. */ +#include + +MXS_BEGIN_DECLS + #if !defined(__cplusplus) #if __STDC_VERSION__ >= 201112 @@ -48,4 +53,6 @@ #endif // __cplusplus +MXS_END_DECLS + #endif // _PLATFORM_H diff --git a/include/maxscale/poll.h b/include/maxscale/poll.h index 2e9a1ec20..eb3b7fbd7 100644 --- a/include/maxscale/poll.h +++ b/include/maxscale/poll.h @@ -1,5 +1,6 @@ -#ifndef _POLL_H -#define _POLL_H +#pragma once +#ifndef _MAXSCALE_POLL_H +#define _MAXSCALE_POLL_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -12,11 +13,15 @@ * of this software will be governed by version 2 or later of the General * Public License. */ + +#include #include #include #include #include +MXS_BEGIN_DECLS + /** * @file poll.h The poll related functionality * @@ -67,4 +72,7 @@ extern void poll_fake_event(DCB *dcb, enum EPOLL_EVENTS ev); extern void poll_fake_hangup_event(DCB *dcb); extern void poll_fake_write_event(DCB *dcb); extern void poll_fake_read_event(DCB *dcb); + +MXS_END_DECLS + #endif diff --git a/include/maxscale/query_classifier.h b/include/maxscale/query_classifier.h index dbbc01fcb..4ea60e365 100644 --- a/include/maxscale/query_classifier.h +++ b/include/maxscale/query_classifier.h @@ -1,5 +1,6 @@ -#ifndef QUERY_CLASSIFIER_HG -#define QUERY_CLASSIFIER_HG +#pragma once +#ifndef _MAXSCALE_QUERY_CLASSIFIER_HG +#define _MAXSCALE_QUERY_CLASSIFIER_HG /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -13,10 +14,11 @@ * Public License. */ +#include #include #include -EXTERN_C_BLOCK_BEGIN +MXS_BEGIN_DECLS typedef enum { @@ -130,6 +132,6 @@ struct query_classifier #define QUERY_CLASSIFIER_VERSION {1, 0, 0} -EXTERN_C_BLOCK_END +MXS_END_DECLS #endif diff --git a/include/maxscale/queuemanager.h b/include/maxscale/queuemanager.h index 462dd10e4..15f9ac39f 100644 --- a/include/maxscale/queuemanager.h +++ b/include/maxscale/queuemanager.h @@ -1,5 +1,6 @@ -#ifndef _QUEUEMANAGER_H -#define _QUEUEMANAGER_H +#pragma once +#ifndef _MAXSCALE_QUEUEMANAGER_H +#define _MAXSCALE_QUEUEMANAGER_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -26,9 +27,12 @@ * @endverbatim */ +#include #include #include +MXS_BEGIN_DECLS + #define CONNECTION_QUEUE_LIMIT 1000 typedef struct queue_entry @@ -67,4 +71,6 @@ mxs_queue_count(QUEUE_CONFIG *queue_config) return count < 0 ? (count + queue_config->queue_limit + 1): count; } +MXS_END_DECLS + #endif /* QUEUEMANAGER_H */ diff --git a/include/maxscale/random_jkiss.h b/include/maxscale/random_jkiss.h index baf9be28a..c1041ea8f 100644 --- a/include/maxscale/random_jkiss.h +++ b/include/maxscale/random_jkiss.h @@ -1,5 +1,6 @@ -#ifndef RANDOM_JKISS_H -#define RANDOM_JKISS_H +#pragma once +#ifndef _MAXSCALE_RANDOM_JKISS_H +#define _MAXSCALE_RANDOM_JKISS_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -20,14 +21,12 @@ * Created on 26 August 2015, 15:34 */ -#ifdef __cplusplus -extern "C" { -#endif +#include + +MXS_BEGIN_DECLS extern unsigned int random_jkiss(void); -#ifdef __cplusplus -} -#endif +MXS_END_DECLS #endif /* RANDOM_H */ diff --git a/include/maxscale/rdtsc.h b/include/maxscale/rdtsc.h index 213d73b7f..f928fd85b 100644 --- a/include/maxscale/rdtsc.h +++ b/include/maxscale/rdtsc.h @@ -1,5 +1,6 @@ -#ifndef _RDTSC_H -#define _RDTSC_H +#pragma once +#ifndef _MAXSCALE_RDTSC_H +#define _MAXSCALE_RDTSC_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -31,6 +32,10 @@ * @endverbatim */ +#include + +MXS_BEGIN_DECLS + typedef unsigned long long CYCLES; /** @@ -53,4 +58,7 @@ static __inline__ CYCLES rdtsc(void) __asm__ volatile (".byte 0x0f, 0x31" : "=A" (x)); return x; } + +MXS_END_DECLS + #endif diff --git a/include/maxscale/resultset.h b/include/maxscale/resultset.h index 58b4f7a35..90e9c8042 100644 --- a/include/maxscale/resultset.h +++ b/include/maxscale/resultset.h @@ -1,5 +1,6 @@ -#ifndef _RESULTSET_H -#define _RESULTSET_H +#pragma once +#ifndef _MAXSCALE_RESULTSET_H +#define _MAXSCALE_RESULTSET_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -24,8 +25,11 @@ * * @endverbatim */ + +#include #include +MXS_BEGIN_DECLS /** * Column types @@ -85,4 +89,6 @@ extern int resultset_row_set(RESULT_ROW *, int, const char *); extern void resultset_stream_mysql(RESULTSET *, DCB *); extern void resultset_stream_json(RESULTSET *, DCB *); +MXS_END_DECLS + #endif diff --git a/include/maxscale/router.h b/include/maxscale/router.h index 9dafcc9aa..9172074b4 100644 --- a/include/maxscale/router.h +++ b/include/maxscale/router.h @@ -1,5 +1,6 @@ -#ifndef _ROUTER_H -#define _ROUTER_H +#pragma once +#ifndef _MAXSCALE_ROUTER_H +#define _MAXSCALE_ROUTER_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -27,11 +28,15 @@ * 27/10/2015 Martin Brampton Add RCAP_TYPE_NO_RSESSION * */ + +#include #include #include #include #include +MXS_BEGIN_DECLS + /** * The ROUTER handle points to module specific data, so the best we can do * is to make it a void * externally. @@ -100,6 +105,6 @@ typedef enum router_capability_t users when the service is started */ } router_capability_t; - +MXS_END_DECLS #endif diff --git a/include/maxscale/secrets.h b/include/maxscale/secrets.h index 09286c00b..257718c6a 100644 --- a/include/maxscale/secrets.h +++ b/include/maxscale/secrets.h @@ -1,5 +1,6 @@ -#ifndef _SECRETS_H -#define _SECRETS_H +#pragma once +#ifndef _MAXSCALE_SECRETS_H +#define _MAXSCALE_SECRETS_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -25,6 +26,7 @@ * @endverbatim */ +#include #include #include #include @@ -32,9 +34,10 @@ #include #include #include - #include +MXS_BEGIN_DECLS + #define MAXSCALE_KEYLEN 32 #define MAXSCALE_IV_LEN 16 @@ -56,4 +59,6 @@ extern int secrets_writeKeys(const char *directory); extern char *decryptPassword(const char *); extern char *encryptPassword(const char*, const char *); +MXS_END_DECLS + #endif diff --git a/include/maxscale/server.h b/include/maxscale/server.h index 3daaa10dd..724879cd9 100644 --- a/include/maxscale/server.h +++ b/include/maxscale/server.h @@ -1,5 +1,6 @@ -#ifndef _SERVER_H -#define _SERVER_H +#pragma once +#ifndef _MAXSCALE_SERVER_H +#define _MAXSCALE_SERVER_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -12,6 +13,8 @@ * of this software will be governed by version 2 or later of the General * Public License. */ + +#include #include #include @@ -44,6 +47,8 @@ * @endverbatim */ +MXS_BEGIN_DECLS + #define MAX_SERVER_NAME_LEN 1024 #define MAX_NUM_SLAVES 128 /**< Maximum number of slaves under a single server*/ @@ -222,4 +227,6 @@ extern RESULTSET *serverGetList(); extern unsigned int server_map_status(char *str); extern bool server_set_version_string(SERVER* server, const char* string); +MXS_END_DECLS + #endif diff --git a/include/maxscale/service.h b/include/maxscale/service.h index eae46c1a7..4c58ce6eb 100644 --- a/include/maxscale/service.h +++ b/include/maxscale/service.h @@ -1,5 +1,6 @@ -#ifndef _SERVICE_H -#define _SERVICE_H +#pragma once +#ifndef _MAXSCALE_SERVICE_H +#define _MAXSCALE_SERVICE_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -13,6 +14,7 @@ * Public License. */ +#include #include #include #include @@ -53,6 +55,9 @@ * * @endverbatim */ + +MXS_BEGIN_DECLS + struct server; struct router; struct router_object; @@ -222,4 +227,6 @@ extern RESULTSET *serviceGetList(); extern RESULTSET *serviceGetListenerList(); extern bool service_all_services_have_listeners(); +MXS_END_DECLS + #endif diff --git a/include/maxscale/session.h b/include/maxscale/session.h index 87497ad31..988fb9efd 100644 --- a/include/maxscale/session.h +++ b/include/maxscale/session.h @@ -1,5 +1,6 @@ -#ifndef _SESSION_H -#define _SESSION_H +#pragma once +#ifndef _MAXSCALE_SESSION_H +#define _MAXSCALE_SESSION_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -33,6 +34,8 @@ * * @endverbatim */ + +#include #include #include #include @@ -42,6 +45,8 @@ #include #include +MXS_BEGIN_DECLS + struct dcb; struct service; struct filter_def; @@ -201,4 +206,7 @@ void session_disable_log_priority(SESSION* ses, int priority); RESULTSET *sessionGetList(SESSIONLISTFILTER); void process_idle_sessions(); void enable_session_timeouts(); + +MXS_END_DECLS + #endif diff --git a/include/maxscale/skygw_debug.h b/include/maxscale/skygw_debug.h index da75fc057..cadc9f56b 100644 --- a/include/maxscale/skygw_debug.h +++ b/include/maxscale/skygw_debug.h @@ -1,3 +1,6 @@ +#pragma once +#ifndef _MAXSCALE_SKYGW_DEBUG_H +#define _MAXSCALE_SKYGW_DEBUG_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -11,6 +14,7 @@ * Public License. */ +#include #include #include #include @@ -21,19 +25,7 @@ #include #include -#if !defined(SKYGW_DEBUG_H) -#define SKYGW_DEBUG_H - - -#ifdef __cplusplus -#define EXTERN_C_BLOCK_BEGIN extern "C" { -#define EXTERN_C_BLOCK_END } -#define EXTERN_C_FUNC extern "C" -#else -#define EXTERN_C_BLOCK_BEGIN -#define EXTERN_C_BLOCK_END -#define EXTERN_C_FUNC -#endif +MXS_BEGIN_DECLS #if defined(SS_DEBUG) # define SS_PROF @@ -578,4 +570,6 @@ typedef enum skygw_chk_t static bool conn_open[10240]; #endif /* FAKE_CODE */ +MXS_END_DECLS + #endif /* SKYGW_DEBUG_H */ diff --git a/include/maxscale/skygw_types.h b/include/maxscale/skygw_types.h index 0c8247eeb..21ddd0027 100644 --- a/include/maxscale/skygw_types.h +++ b/include/maxscale/skygw_types.h @@ -1,3 +1,6 @@ +#pragma once +#ifndef _MAXSCALE_SKYGW_TYPES_H +#define _MAXSCALE_SKYGW_TYPES_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -11,13 +14,13 @@ * Public License. */ -#if !defined(SKYGW_TYPES_H) -#define SKYGW_TYPES_H - +#include #include #include #include +MXS_BEGIN_DECLS + #define SECOND_USEC (1024*1024L) #define MSEC_USEC (1024L) @@ -44,4 +47,6 @@ #define MAX_ERROR_MSG PATH_MAX #define array_nelems(a) ((uint)(sizeof(a)/sizeof(a[0]))) +MXS_END_DECLS + #endif /* SKYGW_TYPES_H */ diff --git a/include/maxscale/skygw_utils.h b/include/maxscale/skygw_utils.h index a903466c3..9e3e38c56 100644 --- a/include/maxscale/skygw_utils.h +++ b/include/maxscale/skygw_utils.h @@ -1,5 +1,6 @@ -#ifndef _SKYGW_UTILS_H -#define _SKYGW_UTILS_H +#pragma once +#ifndef _MAXSCALE_SKYGW_UTILS_H +#define _MAXSCALE_SKYGW_UTILS_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -13,6 +14,10 @@ * Public License. */ +#include + +MXS_BEGIN_DECLS + /* * We need a common.h file that is included by every component. */ @@ -105,13 +110,9 @@ struct skygw_file_st skygw_chk_t sf_chk_tail; }; -EXTERN_C_BLOCK_BEGIN - bool utils_init(); /*< Call this first before using any other function */ void utils_end(); -EXTERN_C_BLOCK_END - /** Skygw thread routines */ skygw_thread_t* skygw_thread_init(const char* name, void* (*sth_thrfun)(void* data), @@ -126,8 +127,6 @@ size_t get_timestamp_len_hp(void); size_t snprint_timestamp(char* p_ts, size_t tslen); size_t snprint_timestamp_hp(char* p_ts, size_t tslen); -EXTERN_C_BLOCK_BEGIN - void skygw_thread_set_state(skygw_thread_t* thr, skygw_thr_state_t state); void* skygw_thread_get_data(skygw_thread_t* thr); @@ -136,8 +135,6 @@ bool skygw_thread_set_exitflag(skygw_thread_t* thr, skygw_message_t* sendmes, skygw_message_t* recmes); -EXTERN_C_BLOCK_END - /** Skygw thread routines */ /** Skygw file routines */ @@ -159,8 +156,6 @@ int skygw_file_write(skygw_file_t* file, bool flush); /** Skygw file routines */ -EXTERN_C_BLOCK_BEGIN - void acquire_lock(int* l); void release_lock(int* l); @@ -179,15 +174,11 @@ void skygw_message_reset(skygw_message_t* mes); /** Skygw message routines */ -EXTERN_C_BLOCK_END - int skygw_rwlock_wrlock(skygw_rwlock_t* rwlock); int skygw_rwlock_rdlock(skygw_rwlock_t* rwlock); int skygw_rwlock_unlock(skygw_rwlock_t* rwlock); int skygw_rwlock_init(skygw_rwlock_t** rwlock); -EXTERN_C_BLOCK_BEGIN - size_t get_decimal_len(size_t s); char* remove_mysql_comments(const char** src, const size_t* srcsize, char** dest, @@ -203,6 +194,6 @@ bool strip_escape_chars(char*); char* trim(char *str); char* squeeze_whitespace(char* str); -EXTERN_C_BLOCK_END +MXS_END_DECLS #endif /* SKYGW_UTILS_H */ diff --git a/include/maxscale/slist.h b/include/maxscale/slist.h index 725023b6d..93e43afd7 100644 --- a/include/maxscale/slist.h +++ b/include/maxscale/slist.h @@ -1,5 +1,6 @@ -#ifndef _SLIST_H -#define _SLIST_H +#pragma once +#ifndef _MAXSCALE_SLIST_H +#define _MAXSCALE_SLIST_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -13,9 +14,10 @@ * Public License. */ +#include #include -EXTERN_C_BLOCK_BEGIN +MXS_BEGIN_DECLS typedef struct slist_node_st slist_node_t; typedef struct slist_st slist_t; @@ -61,6 +63,6 @@ void slcursor_remove_data(slist_cursor_t* c); bool slcursor_move_to_begin(slist_cursor_t* c); bool slcursor_step_ahead(slist_cursor_t* c); -EXTERN_C_BLOCK_END +MXS_END_DECLS #endif diff --git a/include/maxscale/spinlock.h b/include/maxscale/spinlock.h index 7df6064b7..8fcbc1918 100644 --- a/include/maxscale/spinlock.h +++ b/include/maxscale/spinlock.h @@ -1,5 +1,6 @@ -#ifndef _SPINLOCK_H -#define _SPINLOCK_H +#pragma once +#ifndef _MAXSCALE_SPINLOCK_H +#define _MAXSCALE_SPINLOCK_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -23,9 +24,11 @@ * for the lock to be released. However they are useful in that they do not involve * system calls and are light weight when the expected wait time for a lock is low. */ + +#include #include -EXTERN_C_BLOCK_BEGIN +MXS_BEGIN_DECLS #define SPINLOCK_PROFILE 0 @@ -74,6 +77,6 @@ extern int spinlock_acquire_nowait(const SPINLOCK *lock); extern void spinlock_release(const SPINLOCK *lock); extern void spinlock_stats(const SPINLOCK *lock, void (*reporter)(void *, char *, int), void *hdl); -EXTERN_C_BLOCK_END +MXS_END_DECLS #endif diff --git a/include/maxscale/statistics.h b/include/maxscale/statistics.h index 3ba4d2f30..ffb70e808 100644 --- a/include/maxscale/statistics.h +++ b/include/maxscale/statistics.h @@ -1,5 +1,6 @@ -#ifndef _STATISTICS_HG -#define _STATISTICS_HG +#pragma once +#ifndef _MAXSCALE_STATISTICS_HG +#define _MAXSCALE_STATISTICS_HG /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -25,8 +26,11 @@ * @endverbatim */ +#include #include +MXS_BEGIN_DECLS + typedef void* ts_stats_t; /** stats_init should be called only once */ @@ -68,4 +72,6 @@ ts_stats_set(ts_stats_t stats, int value, int thread_id) ((int64_t*)stats)[thread_id] = value; } +MXS_END_DECLS + #endif diff --git a/include/maxscale/thread.h b/include/maxscale/thread.h index 3defe9f90..d775cfec2 100644 --- a/include/maxscale/thread.h +++ b/include/maxscale/thread.h @@ -1,5 +1,6 @@ -#ifndef _THREAD_H -#define _THREAD_H +#pragma once +#ifndef _MAXSCALE_THREAD_H +#define _MAXSCALE_THREAD_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -22,6 +23,10 @@ * of changes. */ +#include + +MXS_BEGIN_DECLS + /** * Thread type and thread identifier function macros */ @@ -33,4 +38,6 @@ extern THREAD *thread_start(THREAD *thd, void (*entry)(void *), void *arg); extern void thread_wait(THREAD thd); extern void thread_millisleep(int ms); +MXS_END_DECLS + #endif diff --git a/include/maxscale/users.h b/include/maxscale/users.h index 78a4b3731..ba9f7440d 100644 --- a/include/maxscale/users.h +++ b/include/maxscale/users.h @@ -1,5 +1,6 @@ -#ifndef _USERS_H -#define _USERS_H +#pragma once +#ifndef _MAXSCALE_USERS_H +#define _MAXSCALE_USERS_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -12,11 +13,15 @@ * of this software will be governed by version 2 or later of the General * Public License. */ + +#include #include #include #include #include +MXS_BEGIN_DECLS + /** * @file users.h The functions to manipulate the table of users maintained * for each service @@ -70,4 +75,6 @@ extern int users_default_loadusers(SERV_LISTENER *port); /**< A generic implemen extern void usersPrint(USERS *); /**< Print data about the users loaded */ extern void dcb_usersPrint(DCB *, USERS *); /**< Print data about the users loaded */ +MXS_END_DECLS + #endif diff --git a/include/maxscale/utils.h b/include/maxscale/utils.h index 273f352ca..eb6d29995 100644 --- a/include/maxscale/utils.h +++ b/include/maxscale/utils.h @@ -1,5 +1,6 @@ -#ifndef _UTILS_H -#define _UTILS_H +#pragma once +#ifndef _MAXSCALE_UTILS_H +#define _MAXSCALE_UTILS_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -25,6 +26,10 @@ * @endverbatim */ +#include + +MXS_BEGIN_DECLS + int setnonblocking(int fd); char *gw_strend(register const char *s); static char gw_randomchar(); @@ -37,4 +42,6 @@ void gw_sha1_2_str(const uint8_t *in, int in_len, const uint8_t *in2, int in2_le int gw_getsockerrno(int fd); char *create_hex_sha1_sha1_passwd(char *passwd); +MXS_END_DECLS + #endif diff --git a/server/core/test/test_utils.h b/server/core/test/test_utils.h index d1ea666c8..150cc7ecd 100644 --- a/server/core/test/test_utils.h +++ b/server/core/test/test_utils.h @@ -1,6 +1,6 @@ +#pragma once #ifndef TEST_UTILS_H #define TEST_UTILS_H - /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -14,6 +14,7 @@ * Public License. */ +#include #include #include #include diff --git a/server/modules/authenticator/gssapi_auth.h b/server/modules/authenticator/gssapi_auth.h index 4f2935c2b..317db08b7 100644 --- a/server/modules/authenticator/gssapi_auth.h +++ b/server/modules/authenticator/gssapi_auth.h @@ -1,3 +1,6 @@ +#pragma once +#ifndef _GSSAPI_AUTH_H +#define _GSSAPI_AUTH_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -11,13 +14,13 @@ * Public License. */ -#ifndef _GSSAPI_AUTH_H -#define _GSSAPI_AUTH_H - +#include #include #include #include +MXS_BEGIN_DECLS + /** Client auth plugin name */ static const char auth_plugin_name[] = "auth_gssapi_client"; @@ -48,4 +51,6 @@ void gssapi_auth_free(void *data); /** Report GSSAPI errors */ void report_error(OM_uint32 major, OM_uint32 minor); +MXS_END_DECLS + #endif diff --git a/server/modules/filter/cache/cache.h b/server/modules/filter/cache/cache.h index c9deff52e..a0fc8bf2b 100644 --- a/server/modules/filter/cache/cache.h +++ b/server/modules/filter/cache/cache.h @@ -1,5 +1,6 @@ -#ifndef CACHE_H -#define CACHE_H +#pragma once +#ifndef _MAXSCALE_FILTER_CACHE_CACHE_H +#define _MAXSCALE_FILTER_CACHE_CACHE_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -15,6 +16,8 @@ #include +MXS_BEGIN_DECLS + #define CACHE_DEBUG_NONE 0 #define CACHE_DEBUG_MATCHING 1 #define CACHE_DEBUG_NON_MATCHING 2 @@ -35,4 +38,6 @@ // Integer value #define CACHE_DEFAULT_DEBUG 0 +MXS_END_DECLS + #endif diff --git a/server/modules/filter/cache/cache_storage_api.h b/server/modules/filter/cache/cache_storage_api.h index 3755d58ec..ffffa080f 100644 --- a/server/modules/filter/cache/cache_storage_api.h +++ b/server/modules/filter/cache/cache_storage_api.h @@ -1,5 +1,6 @@ -#ifndef _MAXSCALE_FILTER_CACHE_CACHE_H -#define _MAXSCALE_FILTER_CACHE_CACHE_H +#pragma once +#ifndef _MAXSCALE_FILTER_CACHE_CACHE_STORAGE_API_H +#define _MAXSCALE_FILTER_CACHE_CACHE_STORAGE_API_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -19,7 +20,7 @@ #include #include -EXTERN_C_BLOCK_BEGIN +MXS_BEGIN_DECLS typedef enum cache_result { @@ -116,6 +117,6 @@ typedef struct cache_storage_api #define CACHE_STORAGE_ENTRY_POINT "CacheGetStorageAPI" typedef CACHE_STORAGE_API* (*CacheGetStorageAPIFN)(); -EXTERN_C_BLOCK_END +MXS_END_DECLS #endif diff --git a/server/modules/filter/cache/rules.h b/server/modules/filter/cache/rules.h index c5590094f..667ad344b 100644 --- a/server/modules/filter/cache/rules.h +++ b/server/modules/filter/cache/rules.h @@ -1,3 +1,4 @@ +#pragma once #ifndef _MAXSCALE_FILTER_CACHE_RULES_H #define _MAXSCALE_FILTER_CACHE_RULES_H /* @@ -13,12 +14,14 @@ * Public License. */ +#include #include #include #include #include #include +MXS_BEGIN_DECLS typedef enum cache_rule_attribute { @@ -71,4 +74,6 @@ CACHE_RULES *cache_rules_parse(const char *json, uint32_t debug); bool cache_rules_should_store(CACHE_RULES *rules, const char *default_db, const GWBUF* query); bool cache_rules_should_use(CACHE_RULES *rules, const SESSION *session); +MXS_END_DECLS + #endif diff --git a/server/modules/filter/cache/storage.h b/server/modules/filter/cache/storage.h index 3132938f3..f0d2f8bef 100644 --- a/server/modules/filter/cache/storage.h +++ b/server/modules/filter/cache/storage.h @@ -1,3 +1,4 @@ +#pragma once #ifndef _MAXSCALE_FILTER_CACHE_STORAGE_H #define _MAXSCALE_FILTER_CACHE_STORAGE_H /* @@ -13,8 +14,11 @@ * Public License. */ +#include #include "cache_storage_api.h" +MXS_BEGIN_DECLS + typedef struct cache_storage_module_t { void* handle; @@ -24,4 +28,6 @@ typedef struct cache_storage_module_t CACHE_STORAGE_MODULE* cache_storage_open(const char *name); void cache_storage_close(CACHE_STORAGE_MODULE *module); +MXS_END_DECLS + #endif diff --git a/server/modules/filter/cache/storage/storage_rocksdb/rocksdbinternals.h b/server/modules/filter/cache/storage/storage_rocksdb/rocksdbinternals.h index 9435f30d0..f82920732 100644 --- a/server/modules/filter/cache/storage/storage_rocksdb/rocksdbinternals.h +++ b/server/modules/filter/cache/storage/storage_rocksdb/rocksdbinternals.h @@ -14,6 +14,7 @@ * Public License. */ +#include #include "storage_rocksdb.h" #include #include diff --git a/server/modules/filter/cache/storage/storage_rocksdb/rocksdbstorage.h b/server/modules/filter/cache/storage/storage_rocksdb/rocksdbstorage.h index 6c2eb1a72..0967fb86b 100644 --- a/server/modules/filter/cache/storage/storage_rocksdb/rocksdbstorage.h +++ b/server/modules/filter/cache/storage/storage_rocksdb/rocksdbstorage.h @@ -1,3 +1,4 @@ +#pragma once #ifndef _ROCKSDBSTORAGE_H #define _ROCKSDBSTORAGE_H /* @@ -13,6 +14,7 @@ * Public License. */ +#include #include "storage_rocksdb.h" #include #include diff --git a/server/modules/filter/cache/storage/storage_rocksdb/storage_rocksdb.h b/server/modules/filter/cache/storage/storage_rocksdb/storage_rocksdb.h index 895421d4b..7ab92c51b 100644 --- a/server/modules/filter/cache/storage/storage_rocksdb/storage_rocksdb.h +++ b/server/modules/filter/cache/storage/storage_rocksdb/storage_rocksdb.h @@ -1,3 +1,4 @@ +#pragma once #ifndef _STORAGE_ROCKSDB_H #define _STORAGE_ROCKSDB_H /* @@ -14,6 +15,7 @@ */ #define MXS_MODULE_NAME "storage_rocksdb" +#include #include #endif diff --git a/server/modules/filter/dbfwfilter/dbfwfilter.h b/server/modules/filter/dbfwfilter/dbfwfilter.h index 9e828e361..4171f2612 100644 --- a/server/modules/filter/dbfwfilter/dbfwfilter.h +++ b/server/modules/filter/dbfwfilter/dbfwfilter.h @@ -1,3 +1,4 @@ +#pragma once #ifndef _DBFWFILTER_H #define _DBFWFILTER_H /* @@ -20,8 +21,11 @@ * the generater parser which is created from the ruleparser.y and token.l files. */ +#include #include +MXS_BEGIN_DECLS + /** Matching type */ enum match_type { @@ -49,4 +53,6 @@ bool add_active_rule(void* scanner, const char* name); void set_matching_mode(void* scanner, enum match_type mode); bool create_user_templates(void* scanner); +MXS_END_DECLS + #endif diff --git a/server/modules/filter/hintfilter/mysqlhint.h b/server/modules/filter/hintfilter/mysqlhint.h index 85c9739fc..5766e1d38 100644 --- a/server/modules/filter/hintfilter/mysqlhint.h +++ b/server/modules/filter/hintfilter/mysqlhint.h @@ -1,3 +1,4 @@ +#pragma once #ifndef _MYSQLHINT_H #define _MYSQLHINT_H /* @@ -19,8 +20,12 @@ * Date Who Description * 17-07-2014 Mark Riddoch Initial implementation */ + +#include #include +MXS_BEGIN_DECLS + /* Parser tokens for the hint parser */ typedef enum { @@ -105,11 +110,10 @@ typedef struct #define HS_PVALUE 5 #define HS_PREPARE 6 - extern HINT *hint_parser(HINT_SESSION *session, GWBUF *request); NAMEDHINTS* free_named_hint(NAMEDHINTS* named_hint); HINTSTACK* free_hint_stack(HINTSTACK* hint_stack); - +MXS_END_DECLS #endif diff --git a/server/modules/include/binlog_common.h b/server/modules/include/binlog_common.h index 7db1f79ae..0b7b301d6 100644 --- a/server/modules/include/binlog_common.h +++ b/server/modules/include/binlog_common.h @@ -1,5 +1,6 @@ -#ifndef BINLOG_COMMON_H -#define BINLOG_COMMON_H +#pragma once +#ifndef _BINLOG_COMMON_H +#define _BINLOG_COMMON_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -13,9 +14,13 @@ * Public License. */ +#include #include #include #include + +MXS_BEGIN_DECLS + /** * Packet header for replication messages */ @@ -45,4 +50,6 @@ bool binlog_next_file_exists(const char* binlogdir, const char* binlog); uint32_t extract_field(uint8_t *src, int bits); const char* binlog_event_name(int type); +MXS_END_DECLS + #endif /* BINLOG_COMMON_H */ diff --git a/server/modules/include/blr_constants.h b/server/modules/include/blr_constants.h index ae2dd7287..1a91e86ba 100644 --- a/server/modules/include/blr_constants.h +++ b/server/modules/include/blr_constants.h @@ -1,3 +1,4 @@ +#pragma once #ifndef _BLR_DEFINES_H #define _BLR_DEFINES_H /* @@ -13,16 +14,19 @@ * Public License. */ -/* +/** + * @file blr_defines.h - Various definitions for binlogrouter + * @verbatim * Revision History * * 26/04/16 Massimiliano Pinto Added MariaDB 10.0 and 10.1 GTID event flags detection + * @endverbatim */ -/** - * @file blr_defines.h - Various definitions for binlogrouter - */ +#include + +MXS_BEGIN_DECLS #define BINLOG_FNAMELEN 255 #define BLR_PROTOCOL "MySQLBackend" @@ -201,4 +205,6 @@ #define EXTRACT32(x) extract_field((x), 32) #endif +MXS_END_DECLS + #endif diff --git a/server/modules/include/cdc.h b/server/modules/include/cdc.h index 80f0d0c81..0785ccbd0 100644 --- a/server/modules/include/cdc.h +++ b/server/modules/include/cdc.h @@ -1,3 +1,6 @@ +#pragma once +#ifndef _CDC_H +#define _CDC_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -18,6 +21,7 @@ * 11-01-2016 Massimiliano Pinto First Implementation */ +#include #include #include #include @@ -35,6 +39,8 @@ #include #include +MXS_BEGIN_DECLS + #define CDC_SMALL_BUFFER 1024 #define CDC_METHOD_MAXLEN 128 #define CDC_USER_MAXLEN 128 @@ -87,3 +93,7 @@ typedef struct cdc_protocol /* routines */ extern int gw_hex2bin(uint8_t *out, const char *in, unsigned int len); + +MXS_END_DECLS + +#endif diff --git a/server/modules/include/debugcli.h b/server/modules/include/debugcli.h index f736038b3..31faeb353 100644 --- a/server/modules/include/debugcli.h +++ b/server/modules/include/debugcli.h @@ -1,3 +1,4 @@ +#pragma once #ifndef _DEBUGCLI_H #define _DEBUGCLI_H /* @@ -13,10 +14,14 @@ * of this software will be governed by version 2 or later of the General * Public License. */ + +#include #include #include #include +MXS_BEGIN_DECLS + /** * @file debugcli.h The debug interface to the gateway * @@ -63,4 +68,7 @@ typedef struct cli_session /* Command line interface modes */ #define CLIM_USER 1 #define CLIM_DEVELOPER 2 + +MXS_END_DECLS + #endif diff --git a/server/modules/include/mysql_auth.h b/server/modules/include/mysql_auth.h index 761fe530c..a487d4aa7 100644 --- a/server/modules/include/mysql_auth.h +++ b/server/modules/include/mysql_auth.h @@ -1,3 +1,4 @@ +#pragma once #ifndef _MYSQL_AUTH_H #define _MYSQL_AUTH_H /* @@ -23,11 +24,14 @@ * @endverbatim */ +#include #include #include #include #include +MXS_BEGIN_DECLS + int gw_check_mysql_scramble_data(DCB *dcb, uint8_t *token, unsigned int token_len, @@ -41,4 +45,6 @@ int gw_find_mysql_user_password_sha1( uint8_t *gateway_password, DCB *dcb); +MXS_END_DECLS + #endif /** _MYSQL_AUTH_H */ diff --git a/server/modules/include/telnetd.h b/server/modules/include/telnetd.h index eef8fc4bb..a8689147c 100644 --- a/server/modules/include/telnetd.h +++ b/server/modules/include/telnetd.h @@ -1,3 +1,4 @@ +#pragma once #ifndef _TELNETD_H #define _TELNETD_H /* @@ -24,8 +25,13 @@ * * @endverbatim */ + +#include #include #include + +MXS_BEGIN_DECLS + /** * The telnetd specific protocol structure to put in the DCB. */ @@ -57,4 +63,7 @@ typedef struct telnetd #define TELNET_IAC 255 #define TELNET_ECHO 1 #define TELNET_SUPPRESS_GO_AHEAD 3 + +MXS_END_DECLS + #endif diff --git a/server/modules/monitor/galeramon/galeramon.h b/server/modules/monitor/galeramon/galeramon.h index a9563e025..f0afc8cf4 100644 --- a/server/modules/monitor/galeramon/galeramon.h +++ b/server/modules/monitor/galeramon/galeramon.h @@ -1,3 +1,4 @@ +#pragma once #ifndef _GALERAMON_H #define _GALERAMON_H /* @@ -13,6 +14,7 @@ * Public License. */ +#include #include #include #include @@ -29,6 +31,8 @@ #include #include +MXS_BEGIN_DECLS + /** * @file galeramon.h - The Galera cluster monitor * @@ -59,4 +63,6 @@ typedef struct bool events[MAX_MONITOR_EVENT]; /*< enabled events */ } GALERA_MONITOR; +MXS_END_DECLS + #endif diff --git a/server/modules/monitor/mmmon/mmmon.h b/server/modules/monitor/mmmon/mmmon.h index 674672d89..f63aa221a 100644 --- a/server/modules/monitor/mmmon/mmmon.h +++ b/server/modules/monitor/mmmon/mmmon.h @@ -1,3 +1,4 @@ +#pragma once #ifndef _MMMON_H #define _MMMON_H /* @@ -13,6 +14,7 @@ * Public License. */ +#include #include #include #include @@ -33,6 +35,8 @@ * @file mmmon.h - The Multi-Master monitor */ +MXS_BEGIN_DECLS + /** * The handle for an instance of a Multi-Master Monitor module */ @@ -49,4 +53,6 @@ typedef struct bool events[MAX_MONITOR_EVENT]; /*< enabled events */ } MM_MONITOR; +MXS_END_DECLS + #endif diff --git a/server/modules/monitor/mysqlmon.h b/server/modules/monitor/mysqlmon.h index 357d1b235..68b73f708 100644 --- a/server/modules/monitor/mysqlmon.h +++ b/server/modules/monitor/mysqlmon.h @@ -1,3 +1,4 @@ +#pragma once #ifndef _MYSQLMON_H #define _MYSQLMON_H /* @@ -13,6 +14,7 @@ * Public License. */ +#include #include #include #include @@ -30,6 +32,8 @@ #include #include +MXS_BEGIN_DECLS + /** * @file mysqlmon.h - The MySQL monitor * @@ -80,4 +84,6 @@ typedef struct bool warn_failover; /**< Log a warning when failover happens */ } MYSQL_MONITOR; +MXS_END_DECLS + #endif diff --git a/server/modules/protocol/HTTPD/httpd.h b/server/modules/protocol/HTTPD/httpd.h index 0bc8096af..4131f3ccf 100644 --- a/server/modules/protocol/HTTPD/httpd.h +++ b/server/modules/protocol/HTTPD/httpd.h @@ -1,3 +1,6 @@ +#pragma once +#ifndef _HTTPD_H +#define _HTTPD_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -18,6 +21,7 @@ * 08-07-2013 Massimiliano Pinto Added HTTPD protocol header file */ +#include #include #include #include @@ -35,6 +39,8 @@ #include #include +MXS_BEGIN_DECLS + #define HTTPD_SMALL_BUFFER 1024 #define HTTPD_METHOD_MAXLEN 128 #define HTTPD_USER_MAXLEN 128 @@ -59,3 +65,7 @@ typedef struct httpd_session char *query_string; /*< the Query string, starts with ?, after path_info and document name */ int headers_received; /*< All the headers has been received, if 1 */ } HTTPD_session; + +MXS_END_DECLS + +#endif diff --git a/server/modules/protocol/maxscaled/maxscaled.h b/server/modules/protocol/maxscaled/maxscaled.h index 8dff6aedb..44af2e2aa 100644 --- a/server/modules/protocol/maxscaled/maxscaled.h +++ b/server/modules/protocol/maxscaled/maxscaled.h @@ -1,3 +1,4 @@ +#pragma once #ifndef _MAXSCALED_H #define _MAXSCALED_H /* @@ -27,6 +28,9 @@ #include #include #include + +MXS_BEGIN_DECLS + /** * The maxscaled specific protocol structure to put in the DCB. */ @@ -41,4 +45,6 @@ typedef struct maxscaled #define MAXSCALED_STATE_PASSWD 2 /**< Waiting for password */ #define MAXSCALED_STATE_DATA 3 /**< User logged in */ +MXS_END_DECLS + #endif diff --git a/server/modules/routing/avro/avrorouter.h b/server/modules/routing/avro/avrorouter.h index 21812bf22..63147af7d 100644 --- a/server/modules/routing/avro/avrorouter.h +++ b/server/modules/routing/avro/avrorouter.h @@ -1,3 +1,6 @@ +#pragma once +#ifndef _MXS_AVRO_H +#define _MXS_AVRO_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -15,8 +18,7 @@ * MaxScale AVRO router * */ -#ifndef _MXS_AVRO_H -#define _MXS_AVRO_H +#include #include #include #include @@ -35,6 +37,8 @@ #include #include +MXS_BEGIN_DECLS + /** SQLite3 version 3.7.14 introduced the new v2 close interface */ #if SQLITE_VERSION_NUMBER < 3007014 #define sqlite3_close_v2 sqlite3_close @@ -312,4 +316,6 @@ extern void table_map_remap(uint8_t *ptr, uint8_t hdr_len, TABLE_MAP *map); #define AVRO_CS_BUSY 0x0001 #define AVRO_WAIT_DATA 0x0002 +MXS_END_DECLS + #endif diff --git a/server/modules/routing/binlog/blr.h b/server/modules/routing/binlog/blr.h index 5f95f957a..ea456e372 100644 --- a/server/modules/routing/binlog/blr.h +++ b/server/modules/routing/binlog/blr.h @@ -1,3 +1,4 @@ +#pragma once #ifndef _BLR_H #define _BLR_H /* @@ -37,6 +38,7 @@ * * @endverbatim */ +#include #include #include #include @@ -46,6 +48,8 @@ #include #include +MXS_BEGIN_DECLS + #define BINLOG_FNAMELEN 255 #define BLR_PROTOCOL "MySQLBackend" #define BINLOG_MAGIC { 0xfe, 0x62, 0x69, 0x6e } @@ -676,4 +680,6 @@ extern bool blr_send_event(blr_thread_role_t role, REP_HEADER *hdr, uint8_t *buf); +MXS_END_DECLS + #endif diff --git a/server/modules/routing/maxinfo/maxinfo.h b/server/modules/routing/maxinfo/maxinfo.h index 08fa63f24..7d5467960 100644 --- a/server/modules/routing/maxinfo/maxinfo.h +++ b/server/modules/routing/maxinfo/maxinfo.h @@ -1,3 +1,4 @@ +#pragma once #ifndef _MAXINFO_H #define _MAXINFO_H /* @@ -12,10 +13,13 @@ * of this software will be governed by version 2 or later of the General * Public License. */ +#include #include #include #include +MXS_BEGIN_DECLS + /** * @file maxinfo.h The MaxScale information schema provider * @@ -138,4 +142,7 @@ extern void maxinfo_send_parse_error(DCB *, char *, PARSE_ERROR); extern void maxinfo_send_error(DCB *, int, char *); extern RESULTSET *maxinfo_variables(); extern RESULTSET *maxinfo_status(); + +MXS_END_DECLS + #endif diff --git a/server/modules/routing/readconnroute/readconnection.h b/server/modules/routing/readconnroute/readconnection.h index 3fcbcb5d9..e832b9c2e 100644 --- a/server/modules/routing/readconnroute/readconnection.h +++ b/server/modules/routing/readconnroute/readconnection.h @@ -1,3 +1,4 @@ +#pragma once #ifndef _READCONNECTION_H #define _READCONNECTION_H /* @@ -25,8 +26,12 @@ * * @endverbatim */ + +#include #include +MXS_BEGIN_DECLS + /** * Internal structure used to define the set of backend servers we are routing * connections to. This provides the storage for routing module specific data @@ -84,4 +89,7 @@ typedef struct router_instance struct router_instance *next; } ROUTER_INSTANCE; + +MXS_END_DECLS + #endif diff --git a/server/modules/routing/readwritesplit/readwritesplit.h b/server/modules/routing/readwritesplit/readwritesplit.h index 3787155c0..98020b667 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.h +++ b/server/modules/routing/readwritesplit/readwritesplit.h @@ -1,3 +1,4 @@ +#pragma once #ifndef _RWSPLITROUTER_H #define _RWSPLITROUTER_H /* @@ -24,10 +25,13 @@ * @endverbatim */ +#include #include #include #include +MXS_BEGIN_DECLS + #undef PREP_STMT_CACHING #if defined(PREP_STMT_CACHING) @@ -356,5 +360,6 @@ typedef struct router_instance #define BACKEND_TYPE(b) (SERVER_IS_MASTER((b)->backend_server) ? BE_MASTER : \ (SERVER_IS_SLAVE((b)->backend_server) ? BE_SLAVE : BE_UNDEFINED)); +MXS_END_DECLS #endif /*< _RWSPLITROUTER_H */ diff --git a/server/modules/routing/readwritesplit/rwsplit_internal.h b/server/modules/routing/readwritesplit/rwsplit_internal.h index b17048eae..cbaaa979b 100644 --- a/server/modules/routing/readwritesplit/rwsplit_internal.h +++ b/server/modules/routing/readwritesplit/rwsplit_internal.h @@ -1,3 +1,6 @@ +#pragma once +#ifndef _RWSPLIT_INTERNAL_H +#define _RWSPLIT_INTERNAL_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -18,15 +21,11 @@ * Created on 08 August 2016, 11:54 */ -#ifndef RWSPLIT_INTERNAL_H -#define RWSPLIT_INTERNAL_H - -#ifdef __cplusplus -extern "C" { -#endif - +#include #include - + +MXS_BEGIN_DECLS + /* This needs to be removed along with dependency on it - see the * rwsplit_tmp_table_multi functions */ @@ -145,5 +144,7 @@ qc_query_type_t determine_query_type(GWBUF *querybuf, int packet_type, bool non_ } #endif +MXS_END_DECLS + #endif /* RWSPLIT_INTERNAL_H */ diff --git a/server/modules/routing/schemarouter/schemarouter.h b/server/modules/routing/schemarouter/schemarouter.h index 17a30729d..88dd01eda 100644 --- a/server/modules/routing/schemarouter/schemarouter.h +++ b/server/modules/routing/schemarouter/schemarouter.h @@ -1,3 +1,4 @@ +#pragma once #ifndef _SCHEMAROUTER_H #define _SCHEMAROUTER_H /* @@ -23,14 +24,15 @@ * * @endverbatim */ -#ifndef PCRE2_CODE_UNIT_WIDTH -#define PCRE2_CODE_UNIT_WIDTH 8 -#endif +#include #include #include #include -#include +#include + +MXS_BEGIN_DECLS + /** * Bitmask values for the router session's initialization. These values are used * to prevent responses from internal commands being forwarded to the client. @@ -377,4 +379,6 @@ typedef struct router_instance #define BACKEND_TYPE(b) (SERVER_IS_MASTER((b)->backend_server) ? BE_MASTER : \ (SERVER_IS_SLAVE((b)->backend_server) ? BE_SLAVE : BE_UNDEFINED)); +MXS_END_DECLS + #endif /*< _SCHEMAROUTER_H */ diff --git a/server/modules/routing/schemarouter/sharding_common.h b/server/modules/routing/schemarouter/sharding_common.h index f4fb06307..de215b7a5 100644 --- a/server/modules/routing/schemarouter/sharding_common.h +++ b/server/modules/routing/schemarouter/sharding_common.h @@ -1,3 +1,6 @@ +#pragma once +#ifndef _SHARDING_COMMON_HG +#define _SHARDING_COMMON_HG /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -11,9 +14,7 @@ * Public License. */ -#ifndef _SHARDING_COMMON_HG -#define _SHARDING_COMMON_HG - +#include #include #include #include @@ -23,8 +24,12 @@ #include #include +MXS_BEGIN_DECLS + bool extract_database(GWBUF* buf, char* str); void create_error_reply(char* fail_str, DCB* dcb); bool change_current_db(char* dest, HASHTABLE* dbhash, GWBUF* buf); +MXS_END_DECLS + #endif From c03b8079fd818e5ada2789cccc00f17796b8daaa Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Fri, 14 Oct 2016 13:20:52 +0300 Subject: [PATCH 027/215] Move @file comment Where it exists, the @file comment has now been moved to be consistently right after the license blurb. --- include/maxscale/config.h | 13 ++++--- include/maxscale/dbusers.h | 12 +++--- include/maxscale/dcb.h | 41 ++++++++++---------- include/maxscale/gw_authenticator.h | 2 +- include/maxscale/gwbitmask.h | 12 +++--- include/maxscale/modules.h | 16 ++++---- include/maxscale/monitor.h | 24 ++++++------ include/maxscale/pcre2.h | 20 +++++----- include/maxscale/poll.h | 17 ++++---- include/maxscale/protocol/mysql.h | 5 +++ include/maxscale/server.h | 8 ++-- include/maxscale/service.h | 33 ++++++++-------- include/maxscale/users.h | 16 ++++---- server/modules/include/debugcli.h | 15 +++---- server/modules/monitor/galeramon/galeramon.h | 22 +++++------ server/modules/monitor/mysqlmon.h | 40 +++++++++---------- server/modules/routing/binlog/blr.h | 1 + server/modules/routing/maxinfo/maxinfo.h | 14 ++++--- 18 files changed, 162 insertions(+), 149 deletions(-) diff --git a/include/maxscale/config.h b/include/maxscale/config.h index caa9cdf84..ba4c374a4 100644 --- a/include/maxscale/config.h +++ b/include/maxscale/config.h @@ -14,12 +14,6 @@ * Public License. */ -#include -#include -#include -#include -#include -#include /** * @file config.h The configuration handling elements * @@ -36,6 +30,13 @@ * @endverbatim */ +#include +#include +#include +#include +#include +#include + MXS_BEGIN_DECLS #define DEFAULT_NBPOLLS 3 /**< Default number of non block polls before we block */ diff --git a/include/maxscale/dbusers.h b/include/maxscale/dbusers.h index 7ef5a71a9..22b7bc2d0 100644 --- a/include/maxscale/dbusers.h +++ b/include/maxscale/dbusers.h @@ -14,12 +14,6 @@ * Public License. */ -#include -#include -#include - -MXS_BEGIN_DECLS - /** * @file dbusers.h Extarct user information form the backend database * @@ -36,6 +30,12 @@ MXS_BEGIN_DECLS * @endverbatim */ +#include +#include +#include + +MXS_BEGIN_DECLS + /* Refresh rate limits for load users from database */ #define USERS_REFRESH_TIME 30 /* Allowed time interval (in seconds) after last update*/ #define USERS_REFRESH_MAX_PER_TIME 4 /* Max number of load calls within the time interval */ diff --git a/include/maxscale/dcb.h b/include/maxscale/dcb.h index 43aab8911..8ef080523 100644 --- a/include/maxscale/dcb.h +++ b/include/maxscale/dcb.h @@ -13,26 +13,6 @@ * of this software will be governed by version 2 or later of the General * Public License. */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -MXS_BEGIN_DECLS - -#define ERRHANDLE - -struct session; -struct server; -struct service; -struct servlistener; /** * @file dcb.h The Descriptor Control Block @@ -67,6 +47,27 @@ struct servlistener; * @endverbatim */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +MXS_BEGIN_DECLS + +#define ERRHANDLE + +struct session; +struct server; +struct service; +struct servlistener; + struct dcb; /** diff --git a/include/maxscale/gw_authenticator.h b/include/maxscale/gw_authenticator.h index 79afbeebd..a78523ff3 100644 --- a/include/maxscale/gw_authenticator.h +++ b/include/maxscale/gw_authenticator.h @@ -15,7 +15,7 @@ */ /** - * @file protocol.h + * @file gw_authenticator.h * * The authenticator module interface definitions for MaxScale * diff --git a/include/maxscale/gwbitmask.h b/include/maxscale/gwbitmask.h index 1f9130f69..d5d041eea 100644 --- a/include/maxscale/gwbitmask.h +++ b/include/maxscale/gwbitmask.h @@ -14,12 +14,6 @@ * Public License. */ -#include -#include -#include - -MXS_BEGIN_DECLS - /** * @file gwbitmask.h An implementation of an arbitrarily long bitmask * @@ -33,6 +27,12 @@ MXS_BEGIN_DECLS * @endverbatim */ +#include +#include +#include + +MXS_BEGIN_DECLS + /* This number MUST an be exact multiple of 8 */ #define MXS_BITMASK_LENGTH (MXS_MAX_THREADS + 1) /**< Number of bits in the bitmask */ diff --git a/include/maxscale/modules.h b/include/maxscale/modules.h index 9e62922c1..07749f305 100644 --- a/include/maxscale/modules.h +++ b/include/maxscale/modules.h @@ -14,14 +14,6 @@ * Public License. */ -#include -#include -#include -#include -#include - -MXS_BEGIN_DECLS - /** * @file modules.h Utilities for loading modules * @@ -41,6 +33,14 @@ MXS_BEGIN_DECLS * @endverbatim */ +#include +#include +#include +#include +#include + +MXS_BEGIN_DECLS + typedef struct modules { char *module; /**< The name of the module */ diff --git a/include/maxscale/monitor.h b/include/maxscale/monitor.h index 18093e457..9d0b9269b 100644 --- a/include/maxscale/monitor.h +++ b/include/maxscale/monitor.h @@ -14,18 +14,6 @@ * Public License. */ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -MXS_BEGIN_DECLS - /** * @file monitor.h The interface to the monitor module * @@ -48,6 +36,18 @@ MXS_BEGIN_DECLS * @endverbatim */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +MXS_BEGIN_DECLS + /** * The "Module Object" for a monitor module. * diff --git a/include/maxscale/pcre2.h b/include/maxscale/pcre2.h index 342d79bde..e2373cca4 100644 --- a/include/maxscale/pcre2.h +++ b/include/maxscale/pcre2.h @@ -15,16 +15,6 @@ * */ -#include - -#ifndef PCRE2_CODE_UNIT_WIDTH -#define PCRE2_CODE_UNIT_WIDTH 8 -#endif - -#include - -MXS_BEGIN_DECLS - /** * @file pcre2.h - Utility functions for regular expression matching * with the bundled PCRE2 library. @@ -37,6 +27,16 @@ MXS_BEGIN_DECLS * @endverbatim */ +#include + +#ifndef PCRE2_CODE_UNIT_WIDTH +#define PCRE2_CODE_UNIT_WIDTH 8 +#endif + +#include + +MXS_BEGIN_DECLS + typedef enum { MXS_PCRE2_MATCH, diff --git a/include/maxscale/poll.h b/include/maxscale/poll.h index eb3b7fbd7..ab0e6e385 100644 --- a/include/maxscale/poll.h +++ b/include/maxscale/poll.h @@ -14,14 +14,6 @@ * Public License. */ -#include -#include -#include -#include -#include - -MXS_BEGIN_DECLS - /** * @file poll.h The poll related functionality * @@ -34,6 +26,15 @@ MXS_BEGIN_DECLS * * @endverbatim */ + +#include +#include +#include +#include +#include + +MXS_BEGIN_DECLS + #define MAX_EVENTS 1000 /** diff --git a/include/maxscale/protocol/mysql.h b/include/maxscale/protocol/mysql.h index f19230e89..568e520c1 100644 --- a/include/maxscale/protocol/mysql.h +++ b/include/maxscale/protocol/mysql.h @@ -36,6 +36,7 @@ * */ +#include #include #include #include @@ -65,6 +66,8 @@ #include #include +MXS_BEGIN_DECLS + #define GW_MYSQL_VERSION "5.5.5-10.0.0 " MAXSCALE_VERSION "-maxscale" #define GW_MYSQL_LOOP_TIMEOUT 300000000 #define GW_MYSQL_READ 0 @@ -373,4 +376,6 @@ int mxs_mysql_send_ok(DCB *dcb, int sequence, int affected_rows, const char* mes /** Check for OK packet */ bool mxs_mysql_is_ok_packet(GWBUF *buffer); +MXS_END_DECLS + #endif /** _MYSQL_PROTOCOL_H */ diff --git a/include/maxscale/server.h b/include/maxscale/server.h index 724879cd9..f06167568 100644 --- a/include/maxscale/server.h +++ b/include/maxscale/server.h @@ -14,10 +14,6 @@ * Public License. */ -#include -#include -#include - /** * @file service.h * @@ -47,6 +43,10 @@ * @endverbatim */ +#include +#include +#include + MXS_BEGIN_DECLS #define MAX_SERVER_NAME_LEN 1024 diff --git a/include/maxscale/service.h b/include/maxscale/service.h index 4c58ce6eb..fa401628d 100644 --- a/include/maxscale/service.h +++ b/include/maxscale/service.h @@ -14,22 +14,6 @@ * Public License. */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include /** * @file service.h * @@ -56,6 +40,23 @@ * @endverbatim */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + MXS_BEGIN_DECLS struct server; diff --git a/include/maxscale/users.h b/include/maxscale/users.h index ba9f7440d..1d95016cd 100644 --- a/include/maxscale/users.h +++ b/include/maxscale/users.h @@ -14,14 +14,6 @@ * Public License. */ -#include -#include -#include -#include -#include - -MXS_BEGIN_DECLS - /** * @file users.h The functions to manipulate the table of users maintained * for each service @@ -38,6 +30,14 @@ MXS_BEGIN_DECLS * @endverbatim */ +#include +#include +#include +#include +#include + +MXS_BEGIN_DECLS + #define USERS_HASHTABLE_DEFAULT_SIZE 52 /** diff --git a/server/modules/include/debugcli.h b/server/modules/include/debugcli.h index 31faeb353..a32f47374 100644 --- a/server/modules/include/debugcli.h +++ b/server/modules/include/debugcli.h @@ -15,13 +15,6 @@ * Public License. */ -#include -#include -#include -#include - -MXS_BEGIN_DECLS - /** * @file debugcli.h The debug interface to the gateway * @@ -33,6 +26,14 @@ MXS_BEGIN_DECLS * * @endverbatim */ + +#include +#include +#include +#include + +MXS_BEGIN_DECLS + struct cli_session; /** diff --git a/server/modules/monitor/galeramon/galeramon.h b/server/modules/monitor/galeramon/galeramon.h index f0afc8cf4..42a1c5198 100644 --- a/server/modules/monitor/galeramon/galeramon.h +++ b/server/modules/monitor/galeramon/galeramon.h @@ -14,6 +14,17 @@ * Public License. */ +/** + * @file galeramon.h - The Galera cluster monitor + * + * @verbatim + * Revision History + * + * Date Who Description + * 07/05/15 Markus Makela Initial Implementation of galeramon.h + * @endverbatim + */ + #include #include #include @@ -33,17 +44,6 @@ MXS_BEGIN_DECLS -/** - * @file galeramon.h - The Galera cluster monitor - * - * @verbatim - * Revision History - * - * Date Who Description - * 07/05/15 Markus Makela Initial Implementation of galeramon.h - * @endverbatim - */ - /** * The handle for an instance of a Galera Monitor module */ diff --git a/server/modules/monitor/mysqlmon.h b/server/modules/monitor/mysqlmon.h index 68b73f708..3d56ad4f6 100644 --- a/server/modules/monitor/mysqlmon.h +++ b/server/modules/monitor/mysqlmon.h @@ -14,6 +14,26 @@ * Public License. */ +/** + * @file mysqlmon.h - The MySQL monitor + * + * @verbatim + * Revision History + * + * Date Who Description + * 08/07/13 Mark Riddoch Initial implementation + * 26/05/14 Massimiliano Pinto Default values for MONITOR_INTERVAL + * 28/05/14 Massimiliano Pinto Addition of new fields in MYSQL_MONITOR struct + * 24/06/14 Massimiliano Pinto Addition of master field in MYSQL_MONITOR struct and MONITOR_MAX_NUM_SLAVES + * 28/08/14 Massimiliano Pinto Addition of detectStaleMaster + * 30/10/14 Massimiliano Pinto Addition of disableMasterFailback + * 07/11/14 Massimiliano Pinto Addition of NetworkTimeout: connect, read, write + * 20/04/15 Guillaume Lefranc Addition of availableWhenDonor + * 22/04/15 Martin Brampton Addition of disableMasterRoleSetting + * 07/05/15 Markus Makela Addition of command execution on Master server failure + * @endverbatim + */ + #include #include #include @@ -34,26 +54,6 @@ MXS_BEGIN_DECLS -/** - * @file mysqlmon.h - The MySQL monitor - * - * @verbatim - * Revision History - * - * Date Who Description - * 08/07/13 Mark Riddoch Initial implementation - * 26/05/14 Massimiliano Pinto Default values for MONITOR_INTERVAL - * 28/05/14 Massimiliano Pinto Addition of new fields in MYSQL_MONITOR struct - * 24/06/14 Massimiliano Pinto Addition of master field in MYSQL_MONITOR struct and MONITOR_MAX_NUM_SLAVES - * 28/08/14 Massimiliano Pinto Addition of detectStaleMaster - * 30/10/14 Massimiliano Pinto Addition of disableMasterFailback - * 07/11/14 Massimiliano Pinto Addition of NetworkTimeout: connect, read, write - * 20/04/15 Guillaume Lefranc Addition of availableWhenDonor - * 22/04/15 Martin Brampton Addition of disableMasterRoleSetting - * 07/05/15 Markus Makela Addition of command execution on Master server failure - * @endverbatim - */ - #define MYSQLMON_DEFAULT_FAILCOUNT 5 /** diff --git a/server/modules/routing/binlog/blr.h b/server/modules/routing/binlog/blr.h index ea456e372..c7b0cc66d 100644 --- a/server/modules/routing/binlog/blr.h +++ b/server/modules/routing/binlog/blr.h @@ -38,6 +38,7 @@ * * @endverbatim */ + #include #include #include diff --git a/server/modules/routing/maxinfo/maxinfo.h b/server/modules/routing/maxinfo/maxinfo.h index 7d5467960..4e42292ae 100644 --- a/server/modules/routing/maxinfo/maxinfo.h +++ b/server/modules/routing/maxinfo/maxinfo.h @@ -13,12 +13,6 @@ * of this software will be governed by version 2 or later of the General * Public License. */ -#include -#include -#include -#include - -MXS_BEGIN_DECLS /** * @file maxinfo.h The MaxScale information schema provider @@ -31,6 +25,14 @@ MXS_BEGIN_DECLS * * @endverbatim */ + +#include +#include +#include +#include + +MXS_BEGIN_DECLS + struct maxinfo_session; /** From a30782227b8e6f96375d5874531f2578ee77a3aa Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Fri, 14 Oct 2016 12:20:57 +0200 Subject: [PATCH 028/215] Hole detection while receiving events and maxbinlogcheck support for encrypted binlog files When a hole is detected while receiving events an ignorable event is written in maxscale binlog. Such event is not sent ti slave servers. Maxbinlogcheck can now check encrypted binlog files without decrypting them --- server/modules/routing/binlog/blr.c | 4 +- server/modules/routing/binlog/blr.h | 33 +- server/modules/routing/binlog/blr_file.c | 381 +++++++++++++++++++--- server/modules/routing/binlog/blr_slave.c | 11 + 4 files changed, 382 insertions(+), 47 deletions(-) diff --git a/server/modules/routing/binlog/blr.c b/server/modules/routing/binlog/blr.c index d4161b759..7cf9d9448 100644 --- a/server/modules/routing/binlog/blr.c +++ b/server/modules/routing/binlog/blr.c @@ -50,6 +50,7 @@ * for connection error and authentication failure. * 11/07/2016 Massimiliano Pinto Added SSL backend support * 22/07/2016 Massimiliano Pinto Added semi_sync replication support + * 16/08/2016 Massimiliano Pinto Addition of Start Encription Event description * * @endverbatim */ @@ -1116,7 +1117,8 @@ static char *event_names_mariadb10[] = /* New MariaDB 10.x event numbers */ "Binlog Checkpoint Event", "GTID Event", - "GTID List Event" + "GTID List Event", + "Start Encryption Event" }; /** diff --git a/server/modules/routing/binlog/blr.h b/server/modules/routing/binlog/blr.h index 5f95f957a..cc85628e1 100644 --- a/server/modules/routing/binlog/blr.h +++ b/server/modules/routing/binlog/blr.h @@ -34,6 +34,7 @@ * 11/07/16 Massimiliano Pinto Added SSL backend support * 22/07/16 Massimiliano Pinto Added Semi-Sync replication support * 01/08/2016 Massimiliano Pinto Added support for ANNOTATE_ROWS_EVENT in COM_BINLOG_DUMP + * 16/09/2016 Massimiliano Pinto Addition of MARIADB10_START_ENCRYPTION_EVENT 0xa4 * * @endverbatim */ @@ -45,6 +46,7 @@ #include #include #include +#include #define BINLOG_FNAMELEN 255 #define BLR_PROTOCOL "MySQLBackend" @@ -53,7 +55,13 @@ #define BINLOG_NAMEFMT "%s.%06d" #define BINLOG_NAME_ROOT "mysql-bin" -#define BINLOG_EVENT_HDR_LEN 19 +#define BINLOG_EVENT_HDR_LEN 19 +#define BINLOG_EVENT_CRC_ALGO_TYPE 1 +#define BINLOG_EVENT_CRC_SIZE 4 +/* BINLOG_EVENT_LEN_OFFSET points to event_size in event_header */ +#define BINLOG_EVENT_LEN_OFFSET 9 +#define BINLOG_ENCRYPTION_ALGORYTHM_NAME_LEN 13 +#define BINLOG_FATAL_ERROR_READING 1236 /** * Binlog event types @@ -103,8 +111,9 @@ #define MARIADB10_BINLOG_CHECKPOINT_EVENT 0xa1 #define MARIADB10_GTID_EVENT 0xa2 #define MARIADB10_GTID_GTID_LIST_EVENT 0xa3 +#define MARIADB10_START_ENCRYPTION_EVENT 0xa4 -#define MAX_EVENT_TYPE_MARIADB10 0xa3 +#define MAX_EVENT_TYPE_MARIADB10 0xa4 /* Maximum event type so far */ #define MAX_EVENT_TYPE_END MAX_EVENT_TYPE_MARIADB10 @@ -537,6 +546,26 @@ typedef struct router_instance struct router_instance *next; } ROUTER_INSTANCE; +/** + * Defines and offsets for binlog encryption + * + * BLRM_FDE_EVENT_TYPES_OFFSET is the offset in FDE event content that points to + * the number of events the master server supports. + */ +#define BLR_FDE_EVENT_BINLOG_VERSION 2 +#define BLR_FDE_EVENT_SERVER_VERSION 50 +#define BLR_FDE_EVENT_BINLOG_TIME 4 +#define BLR_FDE_EVENT_BINLOG_EVENT_HDR_LEN 1 +#define BLRM_FDE_EVENT_TYPES_OFFSET (BLR_FDE_EVENT_BINLOG_VERSION + \ + BLR_FDE_EVENT_SERVER_VERSION + \ + BLR_FDE_EVENT_BINLOG_TIME + \ + BLR_FDE_EVENT_BINLOG_EVENT_HDR_LEN) +#define BLRM_CRYPTO_SCHEME_LENGTH 1 +#define BLRM_KEY_VERSION_LENGTH 4 +#define BLRM_IV_LENGTH AES_BLOCK_SIZE +#define BLRM_IV_OFFS_LENGTH 4 +#define BLRM_NONCE_LENGTH (BLRM_IV_LENGTH - BLRM_IV_OFFS_LENGTH) + /** * State machine for the master to MaxScale replication */ diff --git a/server/modules/routing/binlog/blr_file.c b/server/modules/routing/binlog/blr_file.c index 6edafcb3c..94fffabc5 100644 --- a/server/modules/routing/binlog/blr_file.c +++ b/server/modules/routing/binlog/blr_file.c @@ -35,6 +35,9 @@ * 23/10/2015 Markus Makela Added current_safe_event * 26/04/2016 Massimiliano Pinto Added MariaDB 10.0 and 10.1 GTID event flags detection * 11/07/2016 Massimiliano Pinto Added SSL backend support + * 16/09/2016 Massimiliano Pinto Addition of IGNORABLE_EVENT in case of a missing event + * detected from master binlog stream + * 19/09/2016 Massimiliano Pinto START_ENCRYPTION_EVENT is detected by maxbinlocheck. * * @endverbatim */ @@ -60,6 +63,7 @@ #include #include #include +#include static int blr_file_create(ROUTER_INSTANCE *router, char *file); static void blr_log_header(int priority, char *msg, uint8_t *ptr); @@ -70,10 +74,11 @@ int blr_file_write_master_config(ROUTER_INSTANCE *router, char *error); extern uint32_t extract_field(uint8_t *src, int bits); static void blr_format_event_size(double *event_size, char *label); extern int MaxScaleUptime(); +extern void encode_value(unsigned char *data, unsigned int value, int len); typedef struct binlog_event_desc { - unsigned long long event_pos; + uint64_t event_pos; uint8_t event_type; time_t event_time; } BINLOG_EVENT_DESC; @@ -82,6 +87,44 @@ static void blr_print_binlog_details(ROUTER_INSTANCE *router, BINLOG_EVENT_DESC first_event_time, BINLOG_EVENT_DESC last_event_time); +static uint8_t *blr_create_ignorable_event(uint32_t event_size, + REP_HEADER *hdr, + uint32_t event_pos, + bool do_checksum); +static int blr_write_special_event(ROUTER_INSTANCE *router, + uint32_t file_offset, + uint32_t hole_size, + REP_HEADER *hdr, + int type); + +/** MaxScale generated events */ +typedef enum +{ + BLRM_IGNORABLE, /*< Ignorable event */ + BLRM_START_ENCRYPTION /*< Start Encryption event */ +} generated_event_t; + +/** + * MariaDB 10.1.7 Start Encryption event content + * + * Event header: 19 bytes + * Content size: 17 bytes + * crypto scheme 1 byte + * key_version 4 bytes + * nonce random 12 bytes + * + * Event size is 19 + 17 = 36 bytes + */ +typedef struct start_encryption_event +{ + uint8_t header[BINLOG_EVENT_HDR_LEN]; /**< Replication event header */ + uint8_t binlog_crypto_scheme; /**< Encryption scheme */ + uint32_t binlog_key_version; /**< Encryption key version */ + uint8_t nonce[BLRM_NONCE_LENGTH]; /**< nonce (random bytes) of current binlog. + * These bytes + the binlog event current pos + * form the encrryption IV for the event */ +} START_ENCRYPTION_EVENT; + /** * Initialise the binlog file for this instance. MaxScale will look * for all the binlogs that it has on local disk, determine the next @@ -364,6 +407,24 @@ blr_write_binlog_record(ROUTER_INSTANCE *router, REP_HEADER *hdr, uint32_t size, { int n; + uint64_t file_offset = router->current_pos; + + /** + * Check for possible hole looking at current pos and next pos + * Fill the gap with a self generated ignorable event + * Binlog file position is incremented by blr_write_special_event() + */ + + if (hdr->next_pos && (hdr->next_pos > (file_offset + size))) + { + uint64_t hole_size = hdr->next_pos - file_offset - size; + if (!blr_write_special_event(router, file_offset, hole_size, hdr, BLRM_IGNORABLE)) + { + return 0; + } + } + + /* Write current received event form master */ if ((n = pwrite(router->binlog_fd, buf, size, router->last_written)) != size) { @@ -383,6 +444,8 @@ blr_write_binlog_record(ROUTER_INSTANCE *router, REP_HEADER *hdr, uint32_t size, } return 0; } + + /* Increment offsets */ spinlock_acquire(&router->binlog_lock); router->current_pos = hdr->next_pos; router->last_written += size; @@ -1032,6 +1095,7 @@ blr_read_events_all_events(ROUTER_INSTANCE *router, int fix, int debug) BINLOG_EVENT_DESC last_event; BINLOG_EVENT_DESC fde_event; int fde_seen = 0; + int start_encryption_seen = 0; memset(&first_event, '\0', sizeof(first_event)); memset(&last_event, '\0', sizeof(last_event)); @@ -1200,6 +1264,35 @@ blr_read_events_all_events(ROUTER_INSTANCE *router, int fix, int debug) } } + /* If binlog is encrypted just set pos = pos + event_size and continue */ + if (start_encryption_seen) + { + uint32_t event_size = EXTRACT32(hdbuf + BINLOG_EVENT_LEN_OFFSET); + + /** + * Events are encrypted. + * + * The routine doesn't decrypt them but follows + * next event based on the event_size (4 bytes) that is af offset + * of BINLOG_EVENT_LEN_OFFSET (9) and it's in clear. + * + */ + MXS_DEBUG("** Encrypted Event @ %lu: size is %lu, next event at %lu\n", + (unsigned long)pos, + (unsigned long)event_size, + (unsigned long)(pos + event_size)); + + /* Next event pos is ps + event size */ + pos = pos + event_size; + + /* Update other offsets as well */ + router->binlog_position = pos; + router->current_safe_event = pos; + router->current_pos = pos; + + continue; + } + /* fill replication header struct */ hdr.timestamp = EXTRACT32(hdbuf); hdr.event_type = hdbuf[4]; @@ -1395,8 +1488,6 @@ blr_read_events_all_events(ROUTER_INSTANCE *router, int fix, int debug) if (hdr.event_type == FORMAT_DESCRIPTION_EVENT) { int event_header_length; - int event_header_ntypes; - int n_events; int check_alg; uint8_t *checksum; char buf_t[40]; @@ -1421,29 +1512,40 @@ blr_read_events_all_events(ROUTER_INSTANCE *router, int fix, int debug) pos, (unsigned long)hdr.event_size, fde_event.event_time, buf_t); } - event_header_length = ptr[2 + 50 + 4]; - event_header_ntypes = hdr.event_size - event_header_length - (2 + 50 + 4 + 1); + /* FDE is: + * + * 2 bytes binlog-version + * string[50] mysql-server version + * 4 bytes create timestamp + * 1 event header length, 19 is the current length + * string[p] event type header lengths: + * an array indexed by [Binlog Event Type - 1] + */ - if (event_header_ntypes == 168) - { - /* mariadb 10 LOG_EVENT_TYPES*/ - event_header_ntypes -= 163; - } - else - { - if (event_header_ntypes == 165) - { - /* mariadb 5 LOG_EVENT_TYPES*/ - event_header_ntypes -= 160; - } - else - { - /* mysql 5.6 LOG_EVENT_TYPES = 35 */ - event_header_ntypes -= 35; - } - } + /* ptr now points to event_header_length byte. + * This offset is just 1 byte before the number of supported events offset + */ + event_header_length = ptr[BLRM_FDE_EVENT_TYPES_OFFSET - 1]; - n_events = hdr.event_size - event_header_length - (2 + 50 + 4 + 1); + /* The number of supported events formula: + * number_of_events = event_size - (event_header_len + BLRM_FDE_EVENT_TYPES_OFFSET) + */ + int n_events = hdr.event_size - event_header_length - BLRM_FDE_EVENT_TYPES_OFFSET; + + /** + * The FDE event also carries 5 additional bytes: + * + * 1 byte is the checksum_alg_type and 4 bytes are the computed crc32 + * + * These 5 bytes are always present even if alg_type is NONE/UNDEF: + * then the 4 crc32 bytes must not be checked, whatever the value is. + * + * In case of CRC32 algo_type the 4 bytes contain the event crc32. + */ + int fde_extra_bytes = BINLOG_EVENT_CRC_ALGO_TYPE + BINLOG_EVENT_CRC_SIZE; + + /* Now remove from the calculated number of events the extra 5 bytes */ + n_events -= fde_extra_bytes; if (debug) { @@ -1452,30 +1554,75 @@ blr_read_events_all_events(ROUTER_INSTANCE *router, int fix, int debug) MXS_DEBUG(" FDE Header EventLength %i" ", N. of supported MySQL/MariaDB events %i", event_header_length, - (n_events - event_header_ntypes)); + n_events); } - if (event_header_ntypes < n_events) + /* Check whether Master is sending events with CRC32 checksum */ + checksum = ptr + hdr.event_size - event_header_length - fde_extra_bytes; + check_alg = checksum[0]; + + if (debug) { - checksum = ptr + hdr.event_size - event_header_length - event_header_ntypes; - check_alg = checksum[0]; + MXS_DEBUG(" FDE Checksum alg desc %i, alg type %s", + check_alg, + check_alg == 1 ? + "BINLOG_CHECKSUM_ALG_CRC32" : "NONE or UNDEF"); + } + if (check_alg == 1) + { + found_chksum = 1; + } + else + { + found_chksum = 0; + } + } + + /* Detect possible Start Encryption Event */ + if (hdr.event_type == MARIADB10_START_ENCRYPTION_EVENT) + { + START_ENCRYPTION_EVENT ste_event; + memset(&ste_event, '\0', sizeof(START_ENCRYPTION_EVENT)); + char nonce_hex[BLRM_NONCE_LENGTH * 2 + 1] = ""; + /* The start encryption event data is 17 bytes long: + * Scheme = 1 + * Key Version: 4 + * nonce = 12 + */ + + /* Fill the event content, after the event header */ + ste_event.binlog_crypto_scheme = ptr[0]; + ste_event.binlog_key_version = extract_field(ptr + 1, 32); + memcpy(ste_event.nonce, ptr + 1 + BLRM_KEY_VERSION_LENGTH, BLRM_NONCE_LENGTH); + + gw_bin2hex(nonce_hex, ste_event.nonce, BLRM_NONCE_LENGTH); if (debug) { - MXS_DEBUG(" FDE Checksum alg desc %i, alg type %s", - check_alg, - check_alg == 1 ? - "BINLOG_CHECKSUM_ALG_CRC32" : "NONE or UNDEF"); + char *cksum_format = ", crc32 0x"; + char hex_checksum[BINLOG_EVENT_CRC_SIZE * 2 + strlen(cksum_format) + 1]; + uint8_t cksum_data[BINLOG_EVENT_CRC_SIZE]; + cksum_data[3] = *(ptr + hdr.event_size - 4 - BINLOG_EVENT_HDR_LEN); + cksum_data[2] = *(ptr + hdr.event_size - 3 - BINLOG_EVENT_HDR_LEN); + cksum_data[1] = *(ptr + hdr.event_size - 2 - BINLOG_EVENT_HDR_LEN); + cksum_data[0] = *(ptr + hdr.event_size - 1 - BINLOG_EVENT_HDR_LEN); + + if (found_chksum) + { + strcpy(hex_checksum, cksum_format); + gw_bin2hex(hex_checksum + strlen(cksum_format) , cksum_data, BINLOG_EVENT_CRC_SIZE); + } + + MXS_DEBUG("- START_ENCRYPTION event @ %llu, size %lu, next pos is @ %lu, flags %u%s", + pos, (unsigned long)hdr.event_size, (unsigned long)hdr.next_pos, hdr.flags, + hex_checksum); + + MXS_DEBUG(" Encryption scheme: %u, key_version: %u," + " nonce: %s\n", ste_event.binlog_crypto_scheme, + ste_event.binlog_key_version, nonce_hex); } - if (check_alg == 1) - { - found_chksum = 1; - } - else - { - found_chksum = 0; - } - } + + start_encryption_seen = 1; } /* set last event time, pos and type */ @@ -1994,7 +2141,7 @@ blr_file_write_master_config(ROUTER_INSTANCE *router, char *error) /** Print Binlog Details * - * @param router The router instance + * @param router The router instance * @param first_event First Event details * @param last_event First Event details */ @@ -2019,7 +2166,7 @@ blr_print_binlog_details(ROUTER_INSTANCE *router, event_desc = blr_get_event_description(router, first_event.event_type); - MXS_NOTICE("%lu @ %llu, %s, (%s), First EventTime", + MXS_NOTICE("%lu @ %" PRIu64 ", %s, (%s), First EventTime", first_event.event_time, first_event.event_pos, event_desc != NULL ? event_desc : "unknown", buf_t); @@ -2034,8 +2181,154 @@ blr_print_binlog_details(ROUTER_INSTANCE *router, event_desc = blr_get_event_description(router, last_event.event_type); - MXS_NOTICE("%lu @ %llu, %s, (%s), Last EventTime", + MXS_NOTICE("%lu @ %" PRIu64 ", %s, (%s), Last EventTime", last_event.event_time, last_event.event_pos, event_desc != NULL ? event_desc : "unknown", buf_t); } +/** Create an ignorable event + * + * @param event_size The size of the new event being created (crc32 4 bytes could be included) + * @param hdr Current replication event header, received from master + * @param event_pos The position in binlog file of the new event + * @param do_checksum Whether checksum must be calculated and stored + * @return Returns the pointer of new event + */ +static uint8_t * +blr_create_ignorable_event(uint32_t event_size, + REP_HEADER *hdr, + uint32_t event_pos, + bool do_checksum) +{ + uint8_t *new_event; + + if (event_size < BINLOG_EVENT_HDR_LEN) + { + MXS_ERROR("blr_create_ignorable_event an event of %lu bytes" + " is not valid in blr_file.c", (unsigned long)event_size); + return NULL; + } + + // Allocate space for event: size might contain the 4 crc32 + new_event = MXS_CALLOC(1, event_size); + if (new_event == NULL) + { + return NULL; + } + + // Populate Event header 19 bytes for Ignorable Event + encode_value(&new_event[0], hdr->timestamp, 32); // same timestamp as in current received event + new_event[4] = IGNORABLE_EVENT; // type is IGNORABLE_EVENT + encode_value(&new_event[5], hdr->serverid, 32); // same serverid as in current received event + encode_value(&new_event[9], event_size, 32); // event size + encode_value(&new_event[13], event_pos + event_size, 32); // next_pos + encode_value(&new_event[17], LOG_EVENT_IGNORABLE_F, 16); // flag is LOG_EVENT_IGNORABLE_F + + /* if checksum is required calculate the crc32 and add it in the last 4 bytes*/ + if (do_checksum) + { + /* + * Now add the CRC to the Ignorable binlog event. + * + * The algorithm is first to compute the checksum of an empty buffer + * and then the checksum of the real event: 4 byte less than event_size + */ + uint32_t chksum; + chksum = crc32(0L, NULL, 0); + chksum = crc32(chksum, new_event, event_size - 4); + + // checksum is stored after current event data using 4 bytes + encode_value(new_event + event_size - 4, chksum, 32); + } + return new_event; +} + +/** + * Create and write a special event (not received from master) into binlog file + * + * @param router The current router instance + * @param file_offset Position where event will be written + * @param event_size The size of new event (it might hold the 4 bytes crc32) + * @param hdr Replication header of the current reived event (from Master) + * @param type Type of special event to create and write + * @return 1 on success, 0 on error + */ +static int +blr_write_special_event(ROUTER_INSTANCE *router, uint32_t file_offset, uint32_t event_size, REP_HEADER *hdr, int type) +{ + int n; + uint8_t *new_event; + char *new_event_desc; + + switch (type) + { + case BLRM_IGNORABLE: + new_event_desc = "IGNORABLE"; + MXS_INFO("Hole detected while writing in binlog '%s' @ %lu: an %s event " + "of %lu bytes will be written at pos %lu", + router->binlog_name, + router->current_pos, + new_event_desc, + (unsigned long)event_size, + (unsigned long)file_offset); + + /* Create the ignorable event */ + if ((new_event = blr_create_ignorable_event(event_size, + hdr, + file_offset, + router->master_chksum)) == NULL) + { + return 0; + } + break; + default: + new_event_desc = "UNKNOWN"; + MXS_ERROR("Cannot create special binlog event of %s type and size %lu " + "in binlog file '%s' @ %lu", + new_event_desc, + (unsigned long)event_size, + router->binlog_name, + router->current_pos); + new_event_desc = "UNKNOWN"; + return 0; + break; + } + + // Write the event + if ((n = pwrite(router->binlog_fd, new_event, event_size, file_offset)) != event_size) + { + char err_msg[STRERROR_BUFLEN]; + MXS_ERROR("%s: Failed to write %s special binlog record at %lu of %s, %s. " + "Truncating to previous record.", + router->service->name, new_event_desc, (unsigned long)file_offset, + router->binlog_name, + strerror_r(errno, err_msg, sizeof(err_msg))); + + /* Remove any partial event that was written */ + if (ftruncate(router->binlog_fd, router->last_written)) + { + MXS_ERROR("%s: Failed to truncate %s special binlog record at %lu of %s, %s. ", + router->service->name, new_event_desc, (unsigned long)file_offset, + router->binlog_name, + strerror_r(errno, err_msg, sizeof(err_msg))); + } + MXS_FREE(new_event); + return 0; + } + + MXS_FREE(new_event); + + // Increment offsets, next event will be written after this special one + spinlock_acquire(&router->binlog_lock); + + router->last_written += event_size; + router->current_pos = file_offset + event_size; + router->last_event_pos = file_offset; + + spinlock_release(&router->binlog_lock); + + // Force write + fsync(router->binlog_fd); + + return 1; +} diff --git a/server/modules/routing/binlog/blr_slave.c b/server/modules/routing/binlog/blr_slave.c index cbe4b3d07..84841c04e 100644 --- a/server/modules/routing/binlog/blr_slave.c +++ b/server/modules/routing/binlog/blr_slave.c @@ -60,6 +60,8 @@ * 23/10/2015 Markus Makela Added current_safe_event * 09/05/2016 Massimiliano Pinto Added SELECT USER() * 11/07/2016 Massimiliano Pinto Added SSL backend support + * 16/09/2016 Massimiliano Pinto IGNORABLE_EVENT created by MaxScale is not sent to slaves, + * Events with LOG_EVENT_IGNORABLE_F are also skipped. * * @endverbatim */ @@ -2309,6 +2311,15 @@ blr_slave_catchup(ROUTER_INSTANCE *router, ROUTER_SLAVE *slave, bool large) strcpy(binlog_name, slave->binlogfile); binlog_pos = slave->binlog_pos; + /* Don't sent special events generated by MaxScale */ + if (hdr.event_type == IGNORABLE_EVENT || (hdr.flags & LOG_EVENT_IGNORABLE_F)) + { + slave->binlog_pos = hdr.next_pos; + gwbuf_free(record); + record = NULL; + continue; + } + if (hdr.event_type == ROTATE_EVENT) { unsigned long beat1 = hkheartbeat; From 0800bf4a59ed93951bfba4cc8975f88145c7ee9b Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Fri, 14 Oct 2016 13:44:40 +0300 Subject: [PATCH 029/215] Rename skygw_debug.h to debug.h Also remove unneded includes in debug.h --- avro/maxavro_record.c | 2 +- avro/maxavro_schema.c | 2 +- avro/maxavrocheck.c | 2 +- include/maxscale/buffer.h | 2 +- include/maxscale/{skygw_debug.h => debug.h} | 15 ++++++--------- include/maxscale/hashtable.h | 2 +- include/maxscale/hint.h | 2 +- include/maxscale/listmanager.h | 2 +- include/maxscale/modules.h | 2 +- include/maxscale/skygw_utils.h | 2 +- include/maxscale/spinlock.h | 2 +- .../qc_mysqlembedded/qc_mysqlembedded.cc | 2 +- query_classifier/qc_sqlite/builtin_functions.c | 2 +- server/core/buffer.c | 2 +- server/core/log_manager.cc | 2 +- server/core/mysql_binlog.c | 2 +- server/core/mysql_utils.c | 2 +- server/core/queuemanager.c | 2 +- server/core/skygw_utils.cc | 2 +- server/core/spinlock.c | 2 +- server/core/test/testgwbitmask.c | 2 +- server/core/test/testmaxscalepcre2.c | 2 +- server/modules/filter/cache/cache_storage_api.h | 2 +- server/modules/filter/cache/test/testrules.c | 2 +- server/modules/filter/luafilter/luafilter.c | 2 +- server/modules/monitor/auroramon/auroramon.c | 2 +- server/modules/routing/avro/avro_index.c | 2 +- server/modules/routing/avro/avro_schema.c | 2 +- 28 files changed, 33 insertions(+), 36 deletions(-) rename include/maxscale/{skygw_debug.h => debug.h} (99%) diff --git a/avro/maxavro_record.c b/avro/maxavro_record.c index 961bc2add..1d0e70479 100644 --- a/avro/maxavro_record.c +++ b/avro/maxavro_record.c @@ -14,7 +14,7 @@ #include "maxavro.h" #include #include -#include +#include #include #include diff --git a/avro/maxavro_schema.c b/avro/maxavro_schema.c index e133b03c5..4b5b3fa60 100644 --- a/avro/maxavro_schema.c +++ b/avro/maxavro_schema.c @@ -14,7 +14,7 @@ #include "maxavro.h" #include #include -#include +#include #include static const MAXAVRO_SCHEMA_FIELD types[MAXAVRO_TYPE_MAX] = diff --git a/avro/maxavrocheck.c b/avro/maxavrocheck.c index 23655be56..9a55e1559 100644 --- a/avro/maxavrocheck.c +++ b/avro/maxavrocheck.c @@ -23,7 +23,7 @@ #include #include #include -#include +#include static int verbose = 0; static uint64_t seekto = 0; diff --git a/include/maxscale/buffer.h b/include/maxscale/buffer.h index 3c71ea223..f963167a0 100644 --- a/include/maxscale/buffer.h +++ b/include/maxscale/buffer.h @@ -46,7 +46,7 @@ #include #include -#include +#include #include #include #include diff --git a/include/maxscale/skygw_debug.h b/include/maxscale/debug.h similarity index 99% rename from include/maxscale/skygw_debug.h rename to include/maxscale/debug.h index cadc9f56b..69d0d7367 100644 --- a/include/maxscale/skygw_debug.h +++ b/include/maxscale/debug.h @@ -1,6 +1,6 @@ #pragma once -#ifndef _MAXSCALE_SKYGW_DEBUG_H -#define _MAXSCALE_SKYGW_DEBUG_H +#ifndef _MAXSCALE_DEBUG_H +#define _MAXSCALE_DEBUG_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -15,15 +15,12 @@ */ #include -#include -#include #include #include - -#define __USE_UNIX98 1 +#include +#include #include -#include -#include +#include MXS_BEGIN_DECLS @@ -572,4 +569,4 @@ static bool conn_open[10240]; MXS_END_DECLS -#endif /* SKYGW_DEBUG_H */ +#endif diff --git a/include/maxscale/hashtable.h b/include/maxscale/hashtable.h index 0f176d8ec..185b195ab 100644 --- a/include/maxscale/hashtable.h +++ b/include/maxscale/hashtable.h @@ -30,7 +30,7 @@ * @endverbatim */ #include -#include +#include #include MXS_BEGIN_DECLS diff --git a/include/maxscale/hint.h b/include/maxscale/hint.h index 4cd2b8bbc..d1943b900 100644 --- a/include/maxscale/hint.h +++ b/include/maxscale/hint.h @@ -27,7 +27,7 @@ */ #include -#include +#include MXS_BEGIN_DECLS diff --git a/include/maxscale/listmanager.h b/include/maxscale/listmanager.h index 10c3e6877..b35733847 100644 --- a/include/maxscale/listmanager.h +++ b/include/maxscale/listmanager.h @@ -29,7 +29,7 @@ #include #include -#include +#include MXS_BEGIN_DECLS diff --git a/include/maxscale/modules.h b/include/maxscale/modules.h index 07749f305..eb21810c9 100644 --- a/include/maxscale/modules.h +++ b/include/maxscale/modules.h @@ -37,7 +37,7 @@ #include #include #include -#include +#include MXS_BEGIN_DECLS diff --git a/include/maxscale/skygw_utils.h b/include/maxscale/skygw_utils.h index 9e3e38c56..e797621d5 100644 --- a/include/maxscale/skygw_utils.h +++ b/include/maxscale/skygw_utils.h @@ -34,7 +34,7 @@ MXS_BEGIN_DECLS #define FSYNCLIMIT 10 #include -#include +#include #define DISKWRITE_LATENCY (5*MSEC_USEC) diff --git a/include/maxscale/spinlock.h b/include/maxscale/spinlock.h index 8fcbc1918..72650f268 100644 --- a/include/maxscale/spinlock.h +++ b/include/maxscale/spinlock.h @@ -26,7 +26,7 @@ */ #include -#include +#include MXS_BEGIN_DECLS diff --git a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc index 23dd508fc..a5f0eb5f3 100644 --- a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc +++ b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc @@ -56,7 +56,7 @@ #include #include -#include +#include #include #include #include diff --git a/query_classifier/qc_sqlite/builtin_functions.c b/query_classifier/qc_sqlite/builtin_functions.c index 2cb2ed797..b719ef195 100644 --- a/query_classifier/qc_sqlite/builtin_functions.c +++ b/query_classifier/qc_sqlite/builtin_functions.c @@ -5,7 +5,7 @@ #include "builtin_functions.h" #include #include -#include +#include static struct { diff --git a/server/core/buffer.c b/server/core/buffer.c index 9d55aaff9..a8cd8a377 100644 --- a/server/core/buffer.c +++ b/server/core/buffer.c @@ -42,7 +42,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/server/core/log_manager.cc b/server/core/log_manager.cc index 75e855423..0997e3041 100644 --- a/server/core/log_manager.cc +++ b/server/core/log_manager.cc @@ -28,7 +28,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/server/core/mysql_binlog.c b/server/core/mysql_binlog.c index 6e52cc47b..15e2a1f1c 100644 --- a/server/core/mysql_binlog.c +++ b/server/core/mysql_binlog.c @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/server/core/mysql_utils.c b/server/core/mysql_utils.c index 7ae98160f..1ba15bf67 100644 --- a/server/core/mysql_utils.c +++ b/server/core/mysql_utils.c @@ -25,7 +25,7 @@ #include #include #include -#include +#include #include /** diff --git a/server/core/queuemanager.c b/server/core/queuemanager.c index 9ddafb487..75468563e 100644 --- a/server/core/queuemanager.c +++ b/server/core/queuemanager.c @@ -33,7 +33,7 @@ #include #include #include -#include +#include #if defined(SS_DEBUG) int debug_check_fail = 0; diff --git a/server/core/skygw_utils.cc b/server/core/skygw_utils.cc index c219d2521..498224c22 100644 --- a/server/core/skygw_utils.cc +++ b/server/core/skygw_utils.cc @@ -23,7 +23,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/server/core/spinlock.c b/server/core/spinlock.c index 9772eb65f..532e8c71b 100644 --- a/server/core/spinlock.c +++ b/server/core/spinlock.c @@ -26,7 +26,7 @@ #include #include #include -#include +#include /** * Initialise a spinlock. diff --git a/server/core/test/testgwbitmask.c b/server/core/test/testgwbitmask.c index 751920624..78bc51cbf 100644 --- a/server/core/test/testgwbitmask.c +++ b/server/core/test/testgwbitmask.c @@ -35,7 +35,7 @@ #include -#include +#include /** * test1 Create a bitmap and mess around with it diff --git a/server/core/test/testmaxscalepcre2.c b/server/core/test/testmaxscalepcre2.c index b3c1b7395..a2b69fd3b 100644 --- a/server/core/test/testmaxscalepcre2.c +++ b/server/core/test/testmaxscalepcre2.c @@ -35,7 +35,7 @@ #include #include #include -#include +#include #define test_assert(a, b) if(!(a)){fprintf(stderr, b);return 1;} diff --git a/server/modules/filter/cache/cache_storage_api.h b/server/modules/filter/cache/cache_storage_api.h index ffffa080f..a413afe54 100644 --- a/server/modules/filter/cache/cache_storage_api.h +++ b/server/modules/filter/cache/cache_storage_api.h @@ -18,7 +18,7 @@ #include #include #include -#include +#include MXS_BEGIN_DECLS diff --git a/server/modules/filter/cache/test/testrules.c b/server/modules/filter/cache/test/testrules.c index d717fd5a4..3039e6605 100644 --- a/server/modules/filter/cache/test/testrules.c +++ b/server/modules/filter/cache/test/testrules.c @@ -17,7 +17,7 @@ #if !defined(SS_DEBUG) #define SS_DEBUG #endif -#include +#include struct test_case { diff --git a/server/modules/filter/luafilter/luafilter.c b/server/modules/filter/luafilter/luafilter.c index 47f274326..589a4294c 100644 --- a/server/modules/filter/luafilter/luafilter.c +++ b/server/modules/filter/luafilter/luafilter.c @@ -38,7 +38,7 @@ #include #include -#include +#include #include #include #include diff --git a/server/modules/monitor/auroramon/auroramon.c b/server/modules/monitor/auroramon/auroramon.c index 784a7bd96..81196ff39 100644 --- a/server/modules/monitor/auroramon/auroramon.c +++ b/server/modules/monitor/auroramon/auroramon.c @@ -20,7 +20,7 @@ #include #include #include -#include +#include static char *version_str = (char*)"V1.0.0"; diff --git a/server/modules/routing/avro/avro_index.c b/server/modules/routing/avro/avro_index.c index 70acf2353..d4510fb03 100644 --- a/server/modules/routing/avro/avro_index.c +++ b/server/modules/routing/avro/avro_index.c @@ -33,7 +33,7 @@ */ #include "avrorouter.h" -#include +#include #include void* safe_key_free(void *data); diff --git a/server/modules/routing/avro/avro_schema.c b/server/modules/routing/avro/avro_schema.c index fdf48448a..f4172d689 100644 --- a/server/modules/routing/avro/avro_schema.c +++ b/server/modules/routing/avro/avro_schema.c @@ -28,7 +28,7 @@ #include #include #include -#include +#include #include #include #include From bff2cfb7e52f90b1e11dc34b03225497ddc9b414 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Fri, 14 Oct 2016 14:21:09 +0300 Subject: [PATCH 030/215] Remove skygw_types.h Some stuff moved to maxscale/utils.h. Further cleanup still needed. --- include/maxscale/skygw_types.h | 52 ------------------- include/maxscale/skygw_utils.h | 1 - include/maxscale/utils.h | 7 +++ .../qc_mysqlembedded/qc_mysqlembedded.cc | 1 - server/core/externcmd.c | 1 + server/core/log_manager.cc | 2 +- server/core/service.c | 5 +- server/core/skygw_utils.cc | 13 ++++- server/modules/filter/dbfwfilter/dbfwfilter.c | 1 - server/modules/filter/luafilter/luafilter.c | 1 - server/modules/filter/tpmfilter/tpmfilter.c | 1 + .../MySQL/MySQLBackend/mysql_backend.c | 1 - server/modules/protocol/MySQL/mysql_common.c | 1 - server/modules/routing/avro/avro.c | 1 - server/modules/routing/avro/avro_client.c | 1 - server/modules/routing/binlog/blr.c | 1 - server/modules/routing/binlog/blr_cache.c | 1 - server/modules/routing/binlog/blr_file.c | 1 - server/modules/routing/binlog/blr_master.c | 1 - server/modules/routing/binlog/blr_slave.c | 1 - .../modules/routing/binlog/test/testbinlog.c | 1 - .../modules/routing/maxinfo/maxinfo_parse.c | 2 + .../routing/readconnroute/readconnroute.c | 1 - 23 files changed, 27 insertions(+), 71 deletions(-) delete mode 100644 include/maxscale/skygw_types.h diff --git a/include/maxscale/skygw_types.h b/include/maxscale/skygw_types.h deleted file mode 100644 index 21ddd0027..000000000 --- a/include/maxscale/skygw_types.h +++ /dev/null @@ -1,52 +0,0 @@ -#pragma once -#ifndef _MAXSCALE_SKYGW_TYPES_H -#define _MAXSCALE_SKYGW_TYPES_H -/* - * Copyright (c) 2016 MariaDB Corporation Ab - * - * Use of this software is governed by the Business Source License included - * in the LICENSE.TXT file and at www.mariadb.com/bsl. - * - * Change Date: 2019-07-01 - * - * On the date above, in accordance with the Business Source License, use - * of this software will be governed by version 2 or later of the General - * Public License. - */ - -#include -#include -#include -#include - -MXS_BEGIN_DECLS - -#define SECOND_USEC (1024*1024L) -#define MSEC_USEC (1024L) - -#define KILOBYTE_BYTE (1024L) -#define MEGABYTE_BYTE (1024*1024L) -#define GIGABYTE_BYTE (1024*1024*1024L) - -#define KB KILOBYTE_BYTE -#define MB MEGABYTE_BYTE -#define GB GIGABYTE_BYTE - -#define CALCLEN(i) ((size_t)(floor(log10(abs(i))) + 1)) - -#define UINTLEN(i) (i<10 ? 1 : (i<100 ? 2 : (i<1000 ? 3 : CALCLEN(i)))) - -#if !defined(PATH_MAX) -# if defined(__USE_POSIX) -# define PATH_MAX _POSIX_PATH_MAX -# else -# define PATH_MAX 256 -# endif -#endif - -#define MAX_ERROR_MSG PATH_MAX -#define array_nelems(a) ((uint)(sizeof(a)/sizeof(a[0]))) - -MXS_END_DECLS - -#endif /* SKYGW_TYPES_H */ diff --git a/include/maxscale/skygw_utils.h b/include/maxscale/skygw_utils.h index e797621d5..50c238546 100644 --- a/include/maxscale/skygw_utils.h +++ b/include/maxscale/skygw_utils.h @@ -33,7 +33,6 @@ MXS_BEGIN_DECLS #endif #define FSYNCLIMIT 10 -#include #include #define DISKWRITE_LATENCY (5*MSEC_USEC) diff --git a/include/maxscale/utils.h b/include/maxscale/utils.h index eb6d29995..bca259700 100644 --- a/include/maxscale/utils.h +++ b/include/maxscale/utils.h @@ -27,9 +27,16 @@ */ #include +#include +#include MXS_BEGIN_DECLS +#define CALCLEN(i) ((size_t)(floor(log10(abs(i))) + 1)) +#define UINTLEN(i) (i<10 ? 1 : (i<100 ? 2 : (i<1000 ? 3 : CALCLEN(i)))) + +#define MXS_ARRAY_NELEMS(array) ((size_t)(sizeof(array)/sizeof(array[0]))) + int setnonblocking(int fd); char *gw_strend(register const char *s); static char gw_randomchar(); diff --git a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc index a5f0eb5f3..9baaee794 100644 --- a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc +++ b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc @@ -55,7 +55,6 @@ #include #include -#include #include #include #include diff --git a/server/core/externcmd.c b/server/core/externcmd.c index 9bcef24ae..d4fa6bcbb 100644 --- a/server/core/externcmd.c +++ b/server/core/externcmd.c @@ -12,6 +12,7 @@ */ #include +#include #include /** diff --git a/server/core/log_manager.cc b/server/core/log_manager.cc index 0997e3041..ebf893bc8 100644 --- a/server/core/log_manager.cc +++ b/server/core/log_manager.cc @@ -29,9 +29,9 @@ #include #include #include -#include #include #include +#include #define MAX_PREFIXLEN 250 #define MAX_SUFFIXLEN 250 diff --git a/server/core/service.c b/server/core/service.c index be80fc8c1..137324ea8 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -67,6 +67,7 @@ #include #include #include +#include /** To be used with configuration type checks */ typedef struct typelib_st @@ -78,13 +79,13 @@ typedef struct typelib_st /** Set of subsequent false,true pairs */ static const char* bool_strings[11] = {"FALSE", "TRUE", "OFF", "ON", "N", "Y", "0", "1", "NO", "YES", 0}; -typelib_t bool_type = {array_nelems(bool_strings) - 1, "bool_type", bool_strings}; +typelib_t bool_type = {MXS_ARRAY_NELEMS(bool_strings) - 1, "bool_type", bool_strings}; /** List of valid values */ static const char* sqlvar_target_strings[4] = {"MASTER", "ALL", 0}; typelib_t sqlvar_target_type = { - array_nelems(sqlvar_target_strings) - 1, + MXS_ARRAY_NELEMS(sqlvar_target_strings) - 1, "sqlvar_target_type", sqlvar_target_strings }; diff --git a/server/core/skygw_utils.cc b/server/core/skygw_utils.cc index 498224c22..6c6566c47 100644 --- a/server/core/skygw_utils.cc +++ b/server/core/skygw_utils.cc @@ -15,6 +15,8 @@ #define PCRE2_CODE_UNIT_WIDTH 8 #endif +#include +#include #include #include #include @@ -24,13 +26,22 @@ #include #include #include -#include #include #include #include #include #include +#if !defined(PATH_MAX) +# if defined(__USE_POSIX) +# define PATH_MAX _POSIX_PATH_MAX +# else +# define PATH_MAX 256 +# endif +#endif + +#define MAX_ERROR_MSG PATH_MAX + static void simple_mutex_free_memory(simple_mutex_t* sm); static void thread_free_memory(skygw_thread_t* th, char* name); /** End of static function declarations */ diff --git a/server/modules/filter/dbfwfilter/dbfwfilter.c b/server/modules/filter/dbfwfilter/dbfwfilter.c index 33e684bd1..8ba47775c 100644 --- a/server/modules/filter/dbfwfilter/dbfwfilter.c +++ b/server/modules/filter/dbfwfilter/dbfwfilter.c @@ -70,7 +70,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/filter/luafilter/luafilter.c b/server/modules/filter/luafilter/luafilter.c index 589a4294c..5da797e47 100644 --- a/server/modules/filter/luafilter/luafilter.c +++ b/server/modules/filter/luafilter/luafilter.c @@ -36,7 +36,6 @@ * or diagnostic being made for the session script. */ -#include #include #include #include diff --git a/server/modules/filter/tpmfilter/tpmfilter.c b/server/modules/filter/tpmfilter/tpmfilter.c index d803421a4..d082c89f6 100644 --- a/server/modules/filter/tpmfilter/tpmfilter.c +++ b/server/modules/filter/tpmfilter/tpmfilter.c @@ -39,6 +39,7 @@ * @endverbatim */ +#include #include #include #include diff --git a/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c b/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c index 30d763a12..fa1b05141 100644 --- a/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c +++ b/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c @@ -12,7 +12,6 @@ */ #include -#include #include #include #include diff --git a/server/modules/protocol/MySQL/mysql_common.c b/server/modules/protocol/MySQL/mysql_common.c index 08fb31b6e..77a827967 100644 --- a/server/modules/protocol/MySQL/mysql_common.c +++ b/server/modules/protocol/MySQL/mysql_common.c @@ -46,7 +46,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/routing/avro/avro.c b/server/modules/routing/avro/avro.c index 8c6cb1807..5e5da7926 100644 --- a/server/modules/routing/avro/avro.c +++ b/server/modules/routing/avro/avro.c @@ -39,7 +39,6 @@ #include #include -#include #include #include diff --git a/server/modules/routing/avro/avro_client.c b/server/modules/routing/avro/avro_client.c index 9a051728c..2c2936f20 100644 --- a/server/modules/routing/avro/avro_client.c +++ b/server/modules/routing/avro/avro_client.c @@ -35,7 +35,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/routing/binlog/blr.c b/server/modules/routing/binlog/blr.c index d4161b759..30b17dfb3 100644 --- a/server/modules/routing/binlog/blr.c +++ b/server/modules/routing/binlog/blr.c @@ -71,7 +71,6 @@ #include #include -#include #include #include diff --git a/server/modules/routing/binlog/blr_cache.c b/server/modules/routing/binlog/blr_cache.c index 3e6e5768d..9501f0f92 100644 --- a/server/modules/routing/binlog/blr_cache.c +++ b/server/modules/routing/binlog/blr_cache.c @@ -43,7 +43,6 @@ #include #include -#include #include #include diff --git a/server/modules/routing/binlog/blr_file.c b/server/modules/routing/binlog/blr_file.c index 6edafcb3c..342f64990 100644 --- a/server/modules/routing/binlog/blr_file.c +++ b/server/modules/routing/binlog/blr_file.c @@ -56,7 +56,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/routing/binlog/blr_master.c b/server/modules/routing/binlog/blr_master.c index c96c30a70..e3de00ebb 100644 --- a/server/modules/routing/binlog/blr_master.c +++ b/server/modules/routing/binlog/blr_master.c @@ -69,7 +69,6 @@ #include #include -#include #include #include diff --git a/server/modules/routing/binlog/blr_slave.c b/server/modules/routing/binlog/blr_slave.c index cbe4b3d07..4650053d7 100644 --- a/server/modules/routing/binlog/blr_slave.c +++ b/server/modules/routing/binlog/blr_slave.c @@ -79,7 +79,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/routing/binlog/test/testbinlog.c b/server/modules/routing/binlog/test/testbinlog.c index a9e3e2373..f55d038c9 100644 --- a/server/modules/routing/binlog/test/testbinlog.c +++ b/server/modules/routing/binlog/test/testbinlog.c @@ -36,7 +36,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/routing/maxinfo/maxinfo_parse.c b/server/modules/routing/maxinfo/maxinfo_parse.c index 5fd8bfe05..ccf3aea7a 100644 --- a/server/modules/routing/maxinfo/maxinfo_parse.c +++ b/server/modules/routing/maxinfo/maxinfo_parse.c @@ -23,6 +23,8 @@ * * @endverbatim */ + +#include #include #include #include diff --git a/server/modules/routing/readconnroute/readconnroute.c b/server/modules/routing/readconnroute/readconnroute.c index c55a82fe8..bbb7d55b6 100644 --- a/server/modules/routing/readconnroute/readconnroute.c +++ b/server/modules/routing/readconnroute/readconnroute.c @@ -84,7 +84,6 @@ #include #include -#include #include #include From 03dbc6df807197356abe4c5ad25627b5ff0cbfee Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Fri, 14 Oct 2016 15:38:16 +0300 Subject: [PATCH 031/215] Remove dependency on skygw_utils.h - STRERROR_BUFLEN moved to cdefs.h and renamed to MXS_STRERROR_BUFLEN. Better would be to provide a 'const char* mxs_strerror(int errno)' that would have a thread specific buffer for the error message. - MIN and MAX also moved to defs.h as MXS_MIN and MXS_MAX. - Now only mlist.h of the headers depend upon skygw_utils.h. --- avro/maxavro_record.c | 4 +- include/maxscale/cdefs.h | 38 ++++++++++++ include/maxscale/config.h | 1 - include/maxscale/dcb.h | 1 - include/maxscale/externcmd.h | 1 - include/maxscale/gwdirs.h.in | 1 - include/maxscale/log_manager.h | 7 --- include/maxscale/query_classifier.h | 1 - include/maxscale/session.h | 1 - include/maxscale/skygw_utils.h | 13 ---- include/maxscale/slist.h | 2 +- .../test/canonical_tests/canonizer.c | 1 + server/core/buffer.c | 4 +- server/core/config.c | 4 +- server/core/dcb.c | 48 +++++++-------- server/core/externcmd.c | 2 +- server/core/gateway.c | 28 ++++----- server/core/gw_utils.c | 4 +- server/core/log_manager.cc | 12 ++-- server/core/modutil.c | 5 +- server/core/poll.c | 12 ++-- server/core/query_classifier.c | 1 + server/core/secrets.c | 26 ++++---- server/core/service.c | 2 +- server/core/skygw_utils.cc | 60 +++++++++---------- server/core/test/testadminusers.c | 2 +- server/core/test/testpoll.c | 2 +- server/core/utils.c | 8 +-- server/modules/filter/cache/rules.c | 2 +- .../storage/storage_rocksdb/rocksdbstorage.cc | 2 +- server/modules/filter/dbfwfilter/dbfwfilter.c | 6 +- server/modules/filter/gatekeeper/gatekeeper.c | 12 ++-- server/modules/filter/qlafilter/qlafilter.c | 4 +- server/modules/monitor/galeramon/galeramon.c | 6 +- .../MySQL/MySQLBackend/mysql_backend.c | 18 +++--- server/modules/protocol/MySQL/mysql_common.c | 2 +- server/modules/protocol/maxscaled/maxscaled.c | 2 +- server/modules/routing/avro/avro.c | 6 +- server/modules/routing/avro/avro_client.c | 2 +- server/modules/routing/avro/avro_file.c | 5 +- server/modules/routing/avro/avro_rbr.c | 2 +- server/modules/routing/binlog/blr.c | 6 +- server/modules/routing/binlog/blr_file.c | 12 ++-- server/modules/routing/binlog/blr_master.c | 12 ++-- server/modules/routing/binlog/blr_slave.c | 2 +- server/modules/routing/cli/cli.c | 2 +- server/modules/routing/debugcli/debugcli.c | 2 +- server/modules/routing/debugcli/debugcmd.c | 6 +- .../routing/readwritesplit/readwritesplit.c | 6 +- .../routing/readwritesplit/rwsplit_mysql.c | 4 +- .../routing/schemarouter/schemarouter.c | 2 +- 51 files changed, 215 insertions(+), 199 deletions(-) diff --git a/avro/maxavro_record.c b/avro/maxavro_record.c index 1d0e70479..07a6de119 100644 --- a/avro/maxavro_record.c +++ b/avro/maxavro_record.c @@ -11,8 +11,8 @@ * Public License. */ +#include #include "maxavro.h" -#include #include #include #include @@ -324,7 +324,7 @@ GWBUF* maxavro_record_read_binary(MAXAVRO_FILE *file) { if (ferror(file->file)) { - char err[STRERROR_BUFLEN]; + char err[MXS_STRERROR_BUFLEN]; MXS_ERROR("Failed to read %ld bytes: %d, %s", data_size, errno, strerror_r(errno, err, sizeof(err))); file->last_error = MAXAVRO_ERR_IO; diff --git a/include/maxscale/cdefs.h b/include/maxscale/cdefs.h index ed7de8ba7..62c28b328 100644 --- a/include/maxscale/cdefs.h +++ b/include/maxscale/cdefs.h @@ -35,4 +35,42 @@ # define MXS_END_DECLS #endif +/** + * Define intended for use with strerror. + * + * char errbuf[MXS_STRERROR_BUFLEN]; + * strerror_r(errno, errbuf, sizeof(errbuf)) + */ +#define MXS_STRERROR_BUFLEN 512 + +/** + * Returns the smaller of two items. + * + * @param a A value. + * @param b Another value. + * + * @return a if a is smaller than b, b otherwise. + * + * @note This a macro, so the arguments will be evaluated more than once. + */ +#define MXS_MIN(a,b) ((a)<(b) ? (a) : (b)) + +/** + * Returns the larger of two items. + * + * @param a A value. + * @param b Another value. + * + * @return a if a is larger than b, b otherwise. + * + * @note This a macro, so the arguments will be evaluated more than once. + */ +#define MXS_MAX(a,b) ((a)>(b) ? (a) : (b)) + +/** + * COMMON INCLUDE FILES + */ +#include +#include + #endif diff --git a/include/maxscale/config.h b/include/maxscale/config.h index ba4c374a4..128a906eb 100644 --- a/include/maxscale/config.h +++ b/include/maxscale/config.h @@ -31,7 +31,6 @@ */ #include -#include #include #include #include diff --git a/include/maxscale/dcb.h b/include/maxscale/dcb.h index 8ef080523..5b08e377c 100644 --- a/include/maxscale/dcb.h +++ b/include/maxscale/dcb.h @@ -56,7 +56,6 @@ #include #include #include -#include #include MXS_BEGIN_DECLS diff --git a/include/maxscale/externcmd.h b/include/maxscale/externcmd.h index ed0a90f48..36efeb544 100644 --- a/include/maxscale/externcmd.h +++ b/include/maxscale/externcmd.h @@ -18,7 +18,6 @@ #include #include #include -#include #include #include diff --git a/include/maxscale/gwdirs.h.in b/include/maxscale/gwdirs.h.in index 39e55b0b2..a782c30d2 100644 --- a/include/maxscale/gwdirs.h.in +++ b/include/maxscale/gwdirs.h.in @@ -20,7 +20,6 @@ #include #include #include -#include MXS_BEGIN_DECLS diff --git a/include/maxscale/log_manager.h b/include/maxscale/log_manager.h index 357706c8f..e47d37509 100644 --- a/include/maxscale/log_manager.h +++ b/include/maxscale/log_manager.h @@ -21,13 +21,6 @@ MXS_BEGIN_DECLS -/* - * We need a common.h file that is included by every component. - */ -#if !defined(STRERROR_BUFLEN) -#define STRERROR_BUFLEN 512 -#endif - /** * If MXS_MODULE_NAME is defined before log_manager.h is included, then all * logged messages will be prefixed with that string enclosed in square brackets. diff --git a/include/maxscale/query_classifier.h b/include/maxscale/query_classifier.h index 4ea60e365..f53b6a1ca 100644 --- a/include/maxscale/query_classifier.h +++ b/include/maxscale/query_classifier.h @@ -15,7 +15,6 @@ */ #include -#include #include MXS_BEGIN_DECLS diff --git a/include/maxscale/session.h b/include/maxscale/session.h index 988fb9efd..c688e6149 100644 --- a/include/maxscale/session.h +++ b/include/maxscale/session.h @@ -42,7 +42,6 @@ #include #include #include -#include #include MXS_BEGIN_DECLS diff --git a/include/maxscale/skygw_utils.h b/include/maxscale/skygw_utils.h index 50c238546..e21a41afa 100644 --- a/include/maxscale/skygw_utils.h +++ b/include/maxscale/skygw_utils.h @@ -18,19 +18,6 @@ MXS_BEGIN_DECLS -/* - * We need a common.h file that is included by every component. - */ -#if !defined(STRERROR_BUFLEN) -#define STRERROR_BUFLEN 512 -#endif - -#ifndef MIN -#define MIN(a,b) (ab ? a : b) -#endif #define FSYNCLIMIT 10 #include diff --git a/include/maxscale/slist.h b/include/maxscale/slist.h index 93e43afd7..995e32cee 100644 --- a/include/maxscale/slist.h +++ b/include/maxscale/slist.h @@ -15,7 +15,7 @@ */ #include -#include +#include MXS_BEGIN_DECLS diff --git a/query_classifier/test/canonical_tests/canonizer.c b/query_classifier/test/canonical_tests/canonizer.c index 9dc4a5c49..fc95105e2 100644 --- a/query_classifier/test/canonical_tests/canonizer.c +++ b/query_classifier/test/canonical_tests/canonizer.c @@ -18,6 +18,7 @@ #include #include #include +#include int main(int argc, char** argv) { diff --git a/server/core/buffer.c b/server/core/buffer.c index a8cd8a377..cf25bdb08 100644 --- a/server/core/buffer.c +++ b/server/core/buffer.c @@ -121,7 +121,7 @@ gwbuf_alloc(unsigned int size) retblock: if (rval == NULL) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Memory allocation failed due to %s.", strerror_r(errno, errbuf, sizeof(errbuf))); } @@ -964,7 +964,7 @@ size_t gwbuf_copy_data(GWBUF *buffer, size_t offset, size_t bytes, uint8_t* dest if (buffer) { - bytes_left = MIN(GWBUF_LENGTH(buffer), bytes); + bytes_left = MXS_MIN(GWBUF_LENGTH(buffer), bytes); ptr = (uint8_t*) GWBUF_DATA(buffer); } } diff --git a/server/core/config.c b/server/core/config.c index e1cb5b747..7cda0ed0a 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -233,7 +233,7 @@ char* config_clean_string_list(char* str) PCRE2_ZERO_TERMINATED, 0, &re_err, &err_offset, NULL)) == NULL || (data = pcre2_match_data_create_from_pattern(re, NULL)) == NULL) { - PCRE2_UCHAR errbuf[STRERROR_BUFLEN]; + PCRE2_UCHAR errbuf[MXS_STRERROR_BUFLEN]; pcre2_get_error_message(re_err, errbuf, sizeof(errbuf)); MXS_ERROR("[%s] Regular expression compilation failed at %d: %s", __FUNCTION__, (int)err_offset, errbuf); @@ -2145,7 +2145,7 @@ bool config_has_duplicate_sections(const char* config) } else { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Failed to open file '%s': %s", config, strerror_r(errno, errbuf, sizeof(errbuf))); rval = true; diff --git a/server/core/dcb.c b/server/core/dcb.c index 27e324d37..acf811c7f 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -672,7 +672,7 @@ dcb_process_victim_queue(DCB *listofdcb) { int eno = errno; errno = 0; - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("%lu [dcb_process_victim_queue] Error : Failed to close " "socket %d on dcb %p due error %d, %s.", pthread_self(), @@ -1003,7 +1003,7 @@ dcb_bytes_readable(DCB *dcb) if (-1 == ioctl(dcb->fd, FIONREAD, &bytesavailable)) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; /* */ MXS_ERROR("%lu [dcb_read] Error : ioctl FIONREAD for dcb %p in " "state %s fd %d failed due error %d, %s.", @@ -1069,10 +1069,10 @@ dcb_basic_read(DCB *dcb, int bytesavailable, int maxbytes, int nreadtotal, int * { GWBUF *buffer; - int bufsize = MIN(bytesavailable, MAX_BUFFER_SIZE); + int bufsize = MXS_MIN(bytesavailable, MAX_BUFFER_SIZE); if (maxbytes) { - bufsize = MIN(bufsize, maxbytes - nreadtotal); + bufsize = MXS_MIN(bufsize, maxbytes - nreadtotal); } if ((buffer = gwbuf_alloc(bufsize)) == NULL) @@ -1081,7 +1081,7 @@ dcb_basic_read(DCB *dcb, int bytesavailable, int maxbytes, int nreadtotal, int * * This is a fatal error which should cause shutdown. * Todo shutdown if memory allocation fails. */ - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; /* */ MXS_ERROR("%lu [dcb_read] Error : Failed to allocate read buffer " "for dcb %p fd %d, due %d, %s.", @@ -1102,7 +1102,7 @@ dcb_basic_read(DCB *dcb, int bytesavailable, int maxbytes, int nreadtotal, int * { if (errno != 0 && errno != EAGAIN && errno != EWOULDBLOCK) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; /* */ MXS_ERROR("%lu [dcb_read] Error : Read failed, dcb %p in state " "%s fd %d, due %d, %s.", @@ -1211,7 +1211,7 @@ dcb_basic_read_SSL(DCB *dcb, int *nsingleread) * This is a fatal error which should cause shutdown. * Todo shutdown if memory allocation fails. */ - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; /* */ MXS_ERROR("%lu [dcb_read] Error : Failed to allocate read buffer " "for dcb %p fd %d, due %d, %s.", @@ -1297,7 +1297,7 @@ dcb_basic_read_SSL(DCB *dcb, int *nsingleread) static int dcb_log_errors_SSL (DCB *dcb, const char *called_by, int ret) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; unsigned long ssl_errno; ssl_errno = ERR_get_error(); @@ -1327,7 +1327,7 @@ dcb_log_errors_SSL (DCB *dcb, const char *called_by, int ret) { while (ssl_errno != 0) { - ERR_error_string_n(ssl_errno, errbuf, STRERROR_BUFLEN); + ERR_error_string_n(ssl_errno, errbuf, MXS_STRERROR_BUFLEN); MXS_ERROR("%s", errbuf); ssl_errno = ERR_get_error(); } @@ -1475,7 +1475,7 @@ dcb_log_write_failure(DCB *dcb, GWBUF *queue, int eno) { if (eno == EPIPE) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_DEBUG("%lu [dcb_write] Write to dcb " "%p in state %s fd %d failed " "due errno %d, %s", @@ -1494,7 +1494,7 @@ dcb_log_write_failure(DCB *dcb, GWBUF *queue, int eno) eno != EAGAIN && eno != EWOULDBLOCK) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Write to dcb %p in " "state %s fd %d failed due " "errno %d, %s", @@ -1528,7 +1528,7 @@ dcb_log_write_failure(DCB *dcb, GWBUF *queue, int eno) } if (dolog) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_DEBUG("%lu [dcb_write] Writing to %s socket failed due %d, %s.", pthread_self(), DCB_ROLE_CLIENT_HANDLER == dcb->dcb_role ? "client" : "backend server", @@ -2458,7 +2458,7 @@ gw_write(DCB *dcb, GWBUF *writeq, bool *stop_writing) saved_errno != EPIPE) #endif { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Write to %s %s in state %s failed due errno %d, %s", DCB_STRTYPE(dcb), dcb->remote, STRDCBSTATE(dcb->state), saved_errno, strerror_r(saved_errno, errbuf, sizeof(errbuf))); @@ -2802,7 +2802,7 @@ dcb_persistent_clean_count(DCB *dcb, bool cleanall) } persistentdcb = nextdcb; } - server->persistmax = MAX(server->persistmax, count); + server->persistmax = MXS_MAX(server->persistmax, count); spinlock_release(&server->persistlock); /** Call possible callback for this DCB in case of close */ while (disposals) @@ -3087,7 +3087,7 @@ dcb_accept(DCB *listener, GWPROTOCOL *protocol_funcs) int sendbuf; struct sockaddr_storage client_conn; socklen_t optlen = sizeof(sendbuf); - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; if ((c_sock = dcb_accept_one_connection(listener, (struct sockaddr *)&client_conn)) >= 0) { @@ -3256,7 +3256,7 @@ dcb_accept_one_connection(DCB *listener, struct sockaddr *client_conn) if (c_sock == -1) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; /* Did not get a file descriptor */ if (eno == EAGAIN || eno == EWOULDBLOCK) { @@ -3348,7 +3348,7 @@ dcb_listen(DCB *listener, const char *config, const char *protocol_name) if (listen(listener_socket, 10 * SOMAXCONN) != 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Failed to start listening on '%s' with protocol '%s': %d, %s", config, protocol_name, @@ -3402,7 +3402,7 @@ dcb_listen_create_socket_inet(const char *config_bind) /** Create the TCP socket */ if ((listener_socket = socket(AF_INET, SOCK_STREAM, 0)) < 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Can't create socket: %i, %s", errno, strerror_r(errno, errbuf, sizeof(errbuf))); @@ -3426,7 +3426,7 @@ dcb_listen_create_socket_inet(const char *config_bind) if (bind(listener_socket, (struct sockaddr *) &server_address, sizeof(server_address)) < 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Failed to bind on '%s': %i, %s", config_bind, errno, @@ -3469,7 +3469,7 @@ dcb_listen_create_socket_unix(const char *config_bind) // UNIX socket create if ((listener_socket = socket(AF_UNIX, SOCK_STREAM, 0)) < 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Can't create UNIX socket: %i, %s", errno, strerror_r(errno, errbuf, sizeof(errbuf))); @@ -3496,7 +3496,7 @@ dcb_listen_create_socket_unix(const char *config_bind) if ((-1 == unlink(config_bind)) && (errno != ENOENT)) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Failed to unlink Unix Socket %s: %d %s", config_bind, errno, strerror_r(errno, errbuf, sizeof(errbuf))); } @@ -3504,7 +3504,7 @@ dcb_listen_create_socket_unix(const char *config_bind) /* Bind the socket to the Unix domain socket */ if (bind(listener_socket, (struct sockaddr *) &local_addr, sizeof(local_addr)) < 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Failed to bind to UNIX Domain socket '%s': %i, %s", config_bind, errno, @@ -3516,7 +3516,7 @@ dcb_listen_create_socket_unix(const char *config_bind) /* set permission for all users */ if (chmod(config_bind, 0777) < 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Failed to change permissions on UNIX Domain socket '%s': %i, %s", config_bind, errno, @@ -3543,7 +3543,7 @@ dcb_set_socket_option(int sockfd, int level, int optname, void *optval, socklen_ { if (setsockopt(sockfd, level, optname, optval, optlen) != 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Failed to set socket options. Error %d: %s", errno, strerror_r(errno, errbuf, sizeof(errbuf))); diff --git a/server/core/externcmd.c b/server/core/externcmd.c index d4fa6bcbb..37264e19d 100644 --- a/server/core/externcmd.c +++ b/server/core/externcmd.c @@ -171,7 +171,7 @@ int externcmd_execute(EXTERNCMD* cmd) if (pid < 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Failed to execute command '%s', fork failed: [%d] %s", cmd->argv[0], errno, strerror_r(errno, errbuf, sizeof(errbuf))); rval = -1; diff --git a/server/core/gateway.c b/server/core/gateway.c index c2919e03e..0320ebf10 100644 --- a/server/core/gateway.c +++ b/server/core/gateway.c @@ -317,7 +317,7 @@ sigchld_handler (int i) if ((child = wait(&exit_status)) == -1) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Failed to wait child process: %d %s", errno, strerror_r(errno, errbuf, sizeof(errbuf))); } @@ -428,7 +428,7 @@ static int signal_set(int sig, void (*handler)(int)) { int eno = errno; errno = 0; - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Failed call sigaction() in %s due to %d, %s.", program_invocation_short_name, eno, @@ -463,7 +463,7 @@ static bool create_datadir(const char* base, char* datadir) } else { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Cannot create data directory '%s': %d %s\n", datadir, errno, strerror_r(errno, errbuf, sizeof(errbuf))); } @@ -473,7 +473,7 @@ static bool create_datadir(const char* base, char* datadir) { if (len < PATH_MAX) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; fprintf(stderr, "Error: Cannot create data directory '%s': %d %s\n", datadir, errno, strerror_r(errno, errbuf, sizeof(errbuf))); } @@ -501,7 +501,7 @@ int ntfw_cb(const char* filename, { int eno = errno; errno = 0; - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Failed to remove the data directory %s of MaxScale due to %d, %s.", datadir, eno, strerror_r(eno, errbuf, sizeof(errbuf))); } @@ -702,7 +702,7 @@ static void print_log_n_stderr( { if (mxs_log_init(NULL, get_logdir(), MXS_LOG_TARGET_FS)) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("%s%s%s%s", logstr, eno == 0 ? "" : " (", @@ -712,7 +712,7 @@ static void print_log_n_stderr( } if (do_stderr) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; fprintf(stderr, "* Error: %s%s%s%s\n", fprstr, @@ -1798,7 +1798,7 @@ int main(int argc, char **argv) } else { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Cannot create data directory '%s': %d %s\n", datadir, errno, strerror_r(errno, errbuf, sizeof(errbuf))); goto return_main; @@ -2121,7 +2121,7 @@ static void unlink_pidfile(void) { if (unlink(pidfile)) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; fprintf(stderr, "MaxScale failed to remove pidfile %s: error %d, %s\n", pidfile, @@ -2545,7 +2545,7 @@ static int set_user(const char* user) pwname = getpwnam(user); if (pwname == NULL) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; printf("Error: Failed to retrieve user information for '%s': %d %s\n", user, errno, errno == 0 ? "User not found" : strerror_r(errno, errbuf, sizeof(errbuf))); return -1; @@ -2554,7 +2554,7 @@ static int set_user(const char* user) rval = setgid(pwname->pw_gid); if (rval != 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; printf("Error: Failed to change group to '%d': %d %s\n", pwname->pw_gid, errno, strerror_r(errno, errbuf, sizeof(errbuf))); return rval; @@ -2563,7 +2563,7 @@ static int set_user(const char* user) rval = setuid(pwname->pw_uid); if (rval != 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; printf("Error: Failed to change user to '%s': %d %s\n", pwname->pw_name, errno, strerror_r(errno, errbuf, sizeof(errbuf))); return rval; @@ -2572,7 +2572,7 @@ static int set_user(const char* user) { if (prctl(PR_SET_DUMPABLE , 1) == -1) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; printf("Error: Failed to set dumpable flag on for the process '%s': %d %s\n", pwname->pw_name, errno, strerror_r(errno, errbuf, sizeof(errbuf))); return -1; @@ -2615,7 +2615,7 @@ static bool change_cwd() if (chdir(get_logdir()) != 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Failed to change working directory to '%s': %d, %s. " "Trying to change working directory to '/'.", get_logdir(), errno, strerror_r(errno, errbuf, sizeof (errbuf))); diff --git a/server/core/gw_utils.c b/server/core/gw_utils.c index d7489d49a..78729e46a 100644 --- a/server/core/gw_utils.c +++ b/server/core/gw_utils.c @@ -143,7 +143,7 @@ bool gw_daemonize(void) if (pid < 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; fprintf(stderr, "fork() error %s\n", strerror_r(errno, errbuf, sizeof(errbuf))); exit(1); } @@ -156,7 +156,7 @@ bool gw_daemonize(void) if (setsid() < 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; fprintf(stderr, "setsid() error %s\n", strerror_r(errno, errbuf, sizeof(errbuf))); exit(1); } diff --git a/server/core/log_manager.cc b/server/core/log_manager.cc index ebf893bc8..2a19fbf50 100644 --- a/server/core/log_manager.cc +++ b/server/core/log_manager.cc @@ -1643,7 +1643,7 @@ static bool logfile_write_header(skygw_file_t* file) if ((header_items != 1) || (line_items != 1)) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; LOG_ERROR("MaxScale Log: Writing header failed due to %d, %s\n", errno, strerror_r(errno, errbuf, sizeof(errbuf))); written = false; @@ -1820,13 +1820,13 @@ static bool check_file_and_path(const char* filename, bool* writable) { if (file_is_symlink(filename)) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; LOG_ERROR("MaxScale Log: Error, Can't access file pointed to by %s due to %d, %s.\n", filename, errno, strerror_r(errno, errbuf, sizeof(errbuf))); } else { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; LOG_ERROR("MaxScale Log: Error, Can't access %s due to %d, %s.\n", filename, errno, strerror_r(errno, errbuf, sizeof(errbuf))); } @@ -1925,7 +1925,7 @@ static bool logfile_init(logfile_t* logfile, if (mkdir(dir, S_IRWXU | S_IRWXG) != 0 && (errno != EEXIST)) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; LOG_ERROR("MaxScale Log: Error, creating directory %s failed due to %d, %s.\n", dir, errno, strerror_r(errno, errbuf, sizeof(errbuf))); @@ -2110,7 +2110,7 @@ static bool logfile_write_footer(skygw_file_t* file, const char* suffix) if ((header_items != 1) || (line_items != 1)) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; LOG_ERROR("MaxScale Log: Writing footer failed due to %d, %s\n", errno, strerror_r(errno, errbuf, sizeof(errbuf))); written = false; @@ -2238,7 +2238,7 @@ static bool thr_flush_file(logmanager_t *lm, filewriter_t *fwr) if (err) { // TODO: Log this to syslog. - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; LOG_ERROR("MaxScale Log: Error, writing to the log-file %s failed due to %d, %s. " "Disabling writing to the log.\n", lf->lf_full_file_name, err, strerror_r(err, errbuf, sizeof(errbuf))); diff --git a/server/core/modutil.c b/server/core/modutil.c index bb78c99e9..8f673d1a4 100644 --- a/server/core/modutil.c +++ b/server/core/modutil.c @@ -30,6 +30,7 @@ #include #include #include +#include #include /** These are used when converting MySQL wildcards to regular expressions */ @@ -1059,7 +1060,7 @@ void prepare_pcre2_patterns() { int err; size_t erroff; - PCRE2_UCHAR errbuf[STRERROR_BUFLEN]; + PCRE2_UCHAR errbuf[MXS_STRERROR_BUFLEN]; if ((re_percent = pcre2_compile(pattern_percent, PCRE2_ZERO_TERMINATED, 0, &err, &erroff, NULL)) && @@ -1137,7 +1138,7 @@ mxs_pcre2_result_t modutil_mysql_wildcard_match(const char* pattern, const char* { if (errcode != 0) { - PCRE2_UCHAR errbuf[STRERROR_BUFLEN]; + PCRE2_UCHAR errbuf[MXS_STRERROR_BUFLEN]; pcre2_get_error_message(errcode, errbuf, sizeof(errbuf)); MXS_ERROR("Failed to match pattern: %s", errbuf); } diff --git a/server/core/poll.c b/server/core/poll.c index e7a1fbb0f..683b8cbf3 100644 --- a/server/core/poll.c +++ b/server/core/poll.c @@ -215,7 +215,7 @@ poll_init() } if ((epoll_fd = epoll_create(MAX_EVENTS)) == -1) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("FATAL: Could not create epoll instance: %s", strerror_r(errno, errbuf, sizeof(errbuf))); exit(-1); } @@ -945,7 +945,7 @@ process_pollq(int thread_id) } else { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_DEBUG("%lu [poll_waitevents] " "EPOLLOUT due %d, %s. " "dcb %p, fd %i", @@ -1012,7 +1012,7 @@ process_pollq(int thread_id) if (eno == 0) { eno = dcb_fake_write_errno[dcb->fd]; - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_DEBUG("%lu [poll_waitevents] " "Added fake errno %d. " "%s", @@ -1024,7 +1024,7 @@ process_pollq(int thread_id) #endif /* FAKE_CODE */ if (eno != 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_DEBUG("%lu [poll_waitevents] " "EPOLLERR due %d, %s.", pthread_self(), @@ -1047,7 +1047,7 @@ process_pollq(int thread_id) { int eno = 0; eno = gw_getsockerrno(dcb->fd); - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_DEBUG("%lu [poll_waitevents] " "EPOLLHUP on dcb %p, fd %d. " "Errno %d, %s.", @@ -1083,7 +1083,7 @@ process_pollq(int thread_id) { int eno = 0; eno = gw_getsockerrno(dcb->fd); - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_DEBUG("%lu [poll_waitevents] " "EPOLLRDHUP on dcb %p, fd %d. " "Errno %d, %s.", diff --git a/server/core/query_classifier.c b/server/core/query_classifier.c index 0c984df43..7a0ab82cb 100644 --- a/server/core/query_classifier.c +++ b/server/core/query_classifier.c @@ -16,6 +16,7 @@ #include #include #include +#include //#define QC_TRACE_ENABLED #undef QC_TRACE_ENABLED diff --git a/server/core/secrets.c b/server/core/secrets.c index 4ab38da52..21e7318d1 100644 --- a/server/core/secrets.c +++ b/server/core/secrets.c @@ -89,7 +89,7 @@ secrets_readKeys(const char* path) } else { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("The provided path \"%s\" does not exist or cannot be accessed. " "Error: %d, %s.", path, errno, strerror_r(errno, errbuf, sizeof(errbuf))); return NULL; @@ -111,7 +111,7 @@ secrets_readKeys(const char* path) { if (!reported) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_NOTICE("Encrypted password file %s can't be accessed " "(%s). Password encryption is not used.", secret_file, @@ -121,7 +121,7 @@ secrets_readKeys(const char* path) } else { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Access for secrets file " "[%s] failed. Error %d, %s.", secret_file, @@ -137,7 +137,7 @@ secrets_readKeys(const char* path) { int eno = errno; errno = 0; - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Failed opening secret " "file [%s]. Error %d, %s.", secret_file, @@ -153,7 +153,7 @@ secrets_readKeys(const char* path) int eno = errno; errno = 0; close(fd); - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("fstat for secret file %s " "failed. Error %d, %s.", secret_file, @@ -167,7 +167,7 @@ secrets_readKeys(const char* path) int eno = errno; errno = 0; close(fd); - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Secrets file %s has " "incorrect size. Error %d, %s.", secret_file, @@ -202,7 +202,7 @@ secrets_readKeys(const char* path) errno = 0; close(fd); MXS_FREE(keys); - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Read from secrets file " "%s failed. Read %ld, expected %d bytes. Error %d, %s.", secret_file, @@ -219,7 +219,7 @@ secrets_readKeys(const char* path) int eno = errno; errno = 0; MXS_FREE(keys); - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Failed closing the " "secrets file %s. Error %d, %s.", secret_file, @@ -267,7 +267,7 @@ int secrets_writeKeys(const char *dir) /* Open for writing | Create | Truncate the file for writing */ if ((fd = open(secret_file, O_CREAT | O_WRONLY | O_TRUNC, S_IRUSR)) < 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("failed opening secret " "file [%s]. Error %d, %s.", secret_file, @@ -279,7 +279,7 @@ int secrets_writeKeys(const char *dir) /* Open for writing | Create | Truncate the file for writing */ if ((randfd = open("/dev/random", O_RDONLY)) < 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("failed opening /dev/random. Error %d, %s.", errno, strerror_r(errno, errbuf, sizeof(errbuf))); @@ -302,7 +302,7 @@ int secrets_writeKeys(const char *dir) /* Write data */ if (write(fd, &key, sizeof(key)) < 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("failed writing into " "secret file [%s]. Error %d, %s.", secret_file, @@ -315,7 +315,7 @@ int secrets_writeKeys(const char *dir) /* close file */ if (close(fd) < 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("failed closing the " "secret file [%s]. Error %d, %s.", secret_file, @@ -325,7 +325,7 @@ int secrets_writeKeys(const char *dir) if (chmod(secret_file, S_IRUSR) < 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("failed to change the permissions of the" "secret file [%s]. Error %d, %s.", secret_file, diff --git a/server/core/service.c b/server/core/service.c index 137324ea8..cc908b90f 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -383,7 +383,7 @@ int serviceStartAllPorts(SERVICE* service) service->stats.n_failed_starts++; char taskname[strlen(service->name) + strlen("_start_retry_") + (int) ceil(log10(INT_MAX)) + 1]; - int retry_after = MIN(service->stats.n_failed_starts * 10, SERVICE_MAX_RETRY_INTERVAL); + int retry_after = MXS_MIN(service->stats.n_failed_starts * 10, SERVICE_MAX_RETRY_INTERVAL); snprintf(taskname, sizeof(taskname), "%s_start_retry_%d", service->name, service->stats.n_failed_starts); hktask_oneshot(taskname, service_internal_restart, diff --git a/server/core/skygw_utils.cc b/server/core/skygw_utils.cc index 6c6566c47..8b1142edd 100644 --- a/server/core/skygw_utils.cc +++ b/server/core/skygw_utils.cc @@ -58,7 +58,7 @@ int skygw_rwlock_rdlock(skygw_rwlock_t* rwlock) else { rwlock->srw_rwlock_thr = 0; - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; ss_dfprintf(stderr, "* pthread_rwlock_rdlock : %s\n", strerror_r(err, errbuf, sizeof (errbuf))); @@ -77,7 +77,7 @@ int skygw_rwlock_wrlock(skygw_rwlock_t* rwlock) else { rwlock->srw_rwlock_thr = 0; - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; ss_dfprintf(stderr, "* pthread_rwlock_wrlock : %s\n", strerror_r(err, errbuf, sizeof (errbuf))); @@ -95,7 +95,7 @@ int skygw_rwlock_unlock(skygw_rwlock_t* rwlock) } else { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; ss_dfprintf(stderr, "* pthread_rwlock_unlock : %s\n", strerror_r(err, errbuf, sizeof (errbuf))); } @@ -108,7 +108,7 @@ int skygw_rwlock_destroy(skygw_rwlock_t* rwlock) /** Lock */ if ((err = pthread_rwlock_wrlock(rwlock->srw_rwlock)) != 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; fprintf(stderr, "* Error : pthread_rwlock_wrlock failed due to %d, %s.\n", err, strerror_r(err, errbuf, sizeof (errbuf))); goto retblock; @@ -120,7 +120,7 @@ int skygw_rwlock_destroy(skygw_rwlock_t* rwlock) /** Destroy */ if ((err = pthread_rwlock_destroy(rwlock->srw_rwlock)) != 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; fprintf(stderr, "* Error : pthread_rwlock_destroy failed due to %d,%s\n", err, strerror_r(err, errbuf, sizeof (errbuf))); } @@ -152,7 +152,7 @@ int skygw_rwlock_init(skygw_rwlock_t** rwlock) if (err != 0) { free(rwl); - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; ss_dfprintf(stderr, "* Creating pthread_rwlock failed : %s\n", strerror_r(err, errbuf, sizeof (errbuf))); goto return_err; @@ -204,7 +204,7 @@ size_t snprint_timestamp(char* p_ts, size_t tslen) t = time(NULL); localtime_r(&t, &tm); - snprintf(p_ts, MIN(tslen, timestamp_len), timestamp_formatstr, + snprintf(p_ts, MXS_MIN(tslen, timestamp_len), timestamp_formatstr, tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); rval = strlen(p_ts) * sizeof (char); @@ -245,7 +245,7 @@ size_t snprint_timestamp_hp(char* p_ts, size_t tslen) gettimeofday(&tv, NULL); localtime_r(&tv.tv_sec, &tm); usec = tv.tv_usec / 1000; - snprintf(p_ts, MIN(tslen, timestamp_len_hp), timestamp_formatstr_hp, + snprintf(p_ts, MXS_MIN(tslen, timestamp_len_hp), timestamp_formatstr_hp, tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, usec); rval = strlen(p_ts) * sizeof (char); @@ -350,7 +350,7 @@ int skygw_thread_start(skygw_thread_t* thr) if (err != 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; fprintf(stderr, "* Starting file writer thread failed due error, %d, %s\n", err, strerror_r(errno, errbuf, sizeof (errbuf))); goto return_err; @@ -526,7 +526,7 @@ simple_mutex_t* simple_mutex_init(simple_mutex_t* mutexptr, const char* name) if (err != 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; fprintf(stderr, "* Initializing simple mutex %s failed due error %d, %s\n", name, err, strerror_r(errno, errbuf, sizeof (errbuf))); perror("simple_mutex : "); @@ -566,7 +566,7 @@ int simple_mutex_done(simple_mutex_t* sm) if (err != 0) { perror("simple_mutex : "); - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; fprintf(stderr, "* Destroying simple mutex %s failed due %d, %s\n", sm->sm_name, err, strerror_r(errno, errbuf, sizeof (errbuf))); goto return_err; @@ -613,7 +613,7 @@ int simple_mutex_lock(simple_mutex_t* sm, bool block) if (err != 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; fprintf(stderr, "* Locking simple mutex %s failed due error, %d, %s\n", sm->sm_name, err, strerror_r(errno, errbuf, sizeof (errbuf))); perror("simple_mutex : "); @@ -642,7 +642,7 @@ int simple_mutex_unlock(simple_mutex_t* sm) if (err != 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; fprintf(stderr, "* Unlocking simple mutex %s failed due error %d, %s\n", sm->sm_name, err, strerror_r(errno, errbuf, sizeof (errbuf))); perror("simple_mutex : "); @@ -676,7 +676,7 @@ skygw_message_t* skygw_message_init(void) if (err != 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; fprintf(stderr, "* Initializing pthread mutex failed due error %d, %s\n", err, strerror_r(errno, errbuf, sizeof (errbuf))); free(mes); @@ -687,7 +687,7 @@ skygw_message_t* skygw_message_init(void) if (err != 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; fprintf(stderr, "* Initializing pthread cond var failed, due error %d, %s\n", err, strerror_r(errno, errbuf, sizeof (errbuf))); pthread_mutex_destroy(&mes->mes_mutex); @@ -716,7 +716,7 @@ void skygw_message_done(skygw_message_t* mes) if (err != 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; fprintf(stderr, "* Destroying cond var failed due error %d, %s\n", err, strerror_r(errno, errbuf, sizeof (errbuf))); } @@ -725,7 +725,7 @@ void skygw_message_done(skygw_message_t* mes) if (err != 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; fprintf(stderr, "* Destroying pthread mutex failed, due error %d, %s\n", err, strerror_r(errno, errbuf, sizeof (errbuf))); } @@ -743,7 +743,7 @@ skygw_mes_rc_t skygw_message_send(skygw_message_t* mes) if (err != 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; fprintf(stderr, "* Locking pthread mutex failed, due to error %d, %s\n", err, strerror_r(errno, errbuf, sizeof (errbuf))); goto return_mes_rc; @@ -757,7 +757,7 @@ skygw_mes_rc_t skygw_message_send(skygw_message_t* mes) } else { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; fprintf(stderr, "* Signaling pthread cond var failed, due to error %d, %s\n", err, strerror_r(errno, errbuf, sizeof (errbuf))); } @@ -765,7 +765,7 @@ skygw_mes_rc_t skygw_message_send(skygw_message_t* mes) if (err != 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; fprintf(stderr, "* Unlocking pthread mutex failed, due to error %d, %s\n", err, strerror_r(errno, errbuf, sizeof (errbuf))); } @@ -783,7 +783,7 @@ void skygw_message_wait(skygw_message_t* mes) if (err != 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; fprintf(stderr, "* Locking pthread mutex failed, due error %d, %s\n", err, strerror_r(errno, errbuf, sizeof (errbuf))); } @@ -795,7 +795,7 @@ void skygw_message_wait(skygw_message_t* mes) if (err != 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; fprintf(stderr, "* Locking pthread cond wait failed, due error %d, %s\n", err, strerror_r(errno, errbuf, sizeof (errbuf))); } @@ -805,7 +805,7 @@ void skygw_message_wait(skygw_message_t* mes) if (err != 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; fprintf(stderr, "* Unlocking pthread mutex failed, due error %d, %s\n", err, strerror_r(errno, errbuf, sizeof (errbuf))); } @@ -821,7 +821,7 @@ void skygw_message_reset(skygw_message_t* mes) if (err != 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; fprintf(stderr, "* Locking pthread mutex failed, due error %d, %s\n", err, strerror_r(errno, errbuf, sizeof (errbuf))); goto return_mes_rc; @@ -832,7 +832,7 @@ void skygw_message_reset(skygw_message_t* mes) if (err != 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; fprintf(stderr, "* Unlocking pthread mutex failed, due error %d, %s\n", err, strerror_r(errno, errbuf, sizeof (errbuf))); goto return_mes_rc; @@ -934,7 +934,7 @@ skygw_file_t* skygw_file_init(const char* fname, { int eno = errno; errno = 0; - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; fprintf(stderr, "* Opening file %s failed due %d, %s.\n", file->sf_fname, eno, strerror_r(eno, errbuf, sizeof (errbuf))); free(file); @@ -958,7 +958,7 @@ skygw_file_t* skygw_file_init(const char* fname, { int eno = errno; errno = 0; - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; fprintf(stderr, "failed to create symlink %s -> %s due %d, %s. Exiting.", fname, symlinkname, eno, strerror_r(eno, errbuf, sizeof (errbuf))); free(file); @@ -994,7 +994,7 @@ void skygw_file_close(skygw_file_t* file) if ((err = fclose(file->sf_file)) != 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; fprintf(stderr, "* Closing file %s failed due to %d, %s.\n", file->sf_fname, errno, strerror_r(errno, errbuf, sizeof (errbuf))); } @@ -1170,7 +1170,7 @@ char* replace_literal(char* haystack, const char* needle, const char* replacemen if (search_re == NULL) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; fprintf(stderr, "Regex memory allocation failed : %s\n", strerror_r(errno, errbuf, sizeof (errbuf))); newstr = haystack; @@ -1183,7 +1183,7 @@ char* replace_literal(char* haystack, const char* needle, const char* replacemen if (newstr == NULL) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; fprintf(stderr, "Regex memory allocation failed : %s\n", strerror_r(errno, errbuf, sizeof (errbuf))); free(search_re); diff --git a/server/core/test/testadminusers.c b/server/core/test/testadminusers.c index 4ea8f5e88..1164f6f0c 100644 --- a/server/core/test/testadminusers.c +++ b/server/core/test/testadminusers.c @@ -37,7 +37,7 @@ #include #include #include - +#include /** * test1 default user diff --git a/server/core/test/testpoll.c b/server/core/test/testpoll.c index aea0cd1a5..ecd5892a9 100644 --- a/server/core/test/testpoll.c +++ b/server/core/test/testpoll.c @@ -69,7 +69,7 @@ test1() if (dcb->fd < 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; ss_dfprintf(stderr, "\nError on function call: socket() returned %d: %s\n", errno, strerror_r(errno, errbuf, sizeof(errbuf))); return 1; diff --git a/server/core/utils.c b/server/core/utils.c index b439f4fca..e829b871a 100644 --- a/server/core/utils.c +++ b/server/core/utils.c @@ -60,7 +60,7 @@ int setnonblocking(int fd) if ((fl = fcntl(fd, F_GETFL, 0)) == -1) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Can't GET fcntl for %i, errno = %d, %s.", fd, errno, @@ -70,7 +70,7 @@ int setnonblocking(int fd) if (fcntl(fd, F_SETFL, fl | O_NONBLOCK) == -1) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Can't SET fcntl for %i, errno = %d, %s", fd, errno, @@ -346,7 +346,7 @@ static bool mkdir_all_internal(char *path, mode_t mask) } else { - char err[STRERROR_BUFLEN]; + char err[MXS_STRERROR_BUFLEN]; MXS_ERROR("Failed to create directory '%s': %d, %s", path, errno, strerror_r(errno, err, sizeof(err))); } @@ -355,7 +355,7 @@ static bool mkdir_all_internal(char *path, mode_t mask) } else { - char err[STRERROR_BUFLEN]; + char err[MXS_STRERROR_BUFLEN]; MXS_ERROR("Failed to create directory '%s': %d, %s", path, errno, strerror_r(errno, err, sizeof(err))); } diff --git a/server/modules/filter/cache/rules.c b/server/modules/filter/cache/rules.c index 8e5c0179a..cc2b3bdaa 100644 --- a/server/modules/filter/cache/rules.c +++ b/server/modules/filter/cache/rules.c @@ -246,7 +246,7 @@ CACHE_RULES *cache_rules_load(const char *path, uint32_t debug) } else { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Could not open rules file %s for reading: %s", path, strerror_r(errno, errbuf, sizeof(errbuf))); diff --git a/server/modules/filter/cache/storage/storage_rocksdb/rocksdbstorage.cc b/server/modules/filter/cache/storage/storage_rocksdb/rocksdbstorage.cc index 2fdbc5b1c..a57675594 100644 --- a/server/modules/filter/cache/storage/storage_rocksdb/rocksdbstorage.cc +++ b/server/modules/filter/cache/storage/storage_rocksdb/rocksdbstorage.cc @@ -118,7 +118,7 @@ bool RocksDBStorage::Initialize() else if (errno != EEXIST) { initialized = false; - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Failed to create storage directory %s: %s", u_storageDirectory.c_str(), diff --git a/server/modules/filter/dbfwfilter/dbfwfilter.c b/server/modules/filter/dbfwfilter/dbfwfilter.c index 8ba47775c..cebe94df1 100644 --- a/server/modules/filter/dbfwfilter/dbfwfilter.c +++ b/server/modules/filter/dbfwfilter/dbfwfilter.c @@ -1173,7 +1173,7 @@ bool define_regex_rule(void* scanner, char* pattern) } else { - PCRE2_UCHAR errbuf[STRERROR_BUFLEN]; + PCRE2_UCHAR errbuf[MXS_STRERROR_BUFLEN]; pcre2_get_error_message(err, errbuf, sizeof(errbuf)); MXS_ERROR("dbfwfilter: Invalid regular expression '%s': %s", start, errbuf); @@ -1343,7 +1343,7 @@ static bool process_rule_file(const char* filename, FW_INSTANCE* instance) } else { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Failed to open rule file '%s': %d, %s", filename, errno, strerror_r(errno, errbuf, sizeof(errbuf))); @@ -2240,7 +2240,7 @@ routeQuery(FILTER *instance, void *session, GWBUF *queue) int len; if (modutil_extract_SQL(queue, &sql, &len)) { - len = MIN(len, FW_MAX_SQL_LEN); + len = MXS_MIN(len, FW_MAX_SQL_LEN); if (match && my_instance->log_match & FW_LOG_MATCH) { ss_dassert(rname); diff --git a/server/modules/filter/gatekeeper/gatekeeper.c b/server/modules/filter/gatekeeper/gatekeeper.c index f51ffa0e6..e794c9a3b 100644 --- a/server/modules/filter/gatekeeper/gatekeeper.c +++ b/server/modules/filter/gatekeeper/gatekeeper.c @@ -183,7 +183,7 @@ static FILTER* createInstance(const char *name, char **options, FILTER_PARAMETER } else { - char err[STRERROR_BUFLEN]; + char err[MXS_STRERROR_BUFLEN]; MXS_ERROR(MODNAME"File is not accessible: %d, %s", errno, strerror_r(errno, err, sizeof(err))); ok = false; @@ -453,7 +453,7 @@ static bool write_stored_data(GK_INSTANCE *inst) if (write(fd, &len, sizeof(len)) != sizeof(len) || write(fd, key, len) != len) { - char err[STRERROR_BUFLEN]; + char err[MXS_STRERROR_BUFLEN]; MXS_ERROR(MODNAME"Failed to write key '%s' to disk (%d, %s). The datafile at '%s' was " "not updated but it will be updated when the next session closes. ", key, errno, strerror_r(errno, err, sizeof(err)), inst->datadir); @@ -471,7 +471,7 @@ static bool write_stored_data(GK_INSTANCE *inst) if (!rval) { - char err[STRERROR_BUFLEN]; + char err[MXS_STRERROR_BUFLEN]; MXS_ERROR(MODNAME"Failed to rename file '%s' to '%s' when writing data: %d, %s", filepath, newfilepath, errno, strerror_r(errno, err, sizeof(err))); } @@ -479,7 +479,7 @@ static bool write_stored_data(GK_INSTANCE *inst) } else if (fd == -1) { - char err[STRERROR_BUFLEN]; + char err[MXS_STRERROR_BUFLEN]; MXS_ERROR(MODNAME"Failed to open file '%s' when writing data: %d, %s", filepath, errno, strerror_r(errno, err, sizeof(err))); } @@ -497,7 +497,7 @@ static void report_failed_read(FILE *file, int nexpected, int nread) { if (ferror(file)) { - char err[STRERROR_BUFLEN]; + char err[MXS_STRERROR_BUFLEN]; MXS_ERROR(MODNAME"Failed to read %d bytes, only %d bytes read: %d, %s", nexpected, nread, errno, strerror_r(errno, err, sizeof(err))); } @@ -579,7 +579,7 @@ static bool read_stored_data(GK_INSTANCE *inst) } else { - char err[STRERROR_BUFLEN]; + char err[MXS_STRERROR_BUFLEN]; MXS_ERROR(MODNAME"Failed to open file '%s' when reading stored data: %d, %s", filepath, errno, strerror_r(errno, err, sizeof(err))); } diff --git a/server/modules/filter/qlafilter/qlafilter.c b/server/modules/filter/qlafilter/qlafilter.c index eece0cb0f..f1d7a9fc0 100644 --- a/server/modules/filter/qlafilter/qlafilter.c +++ b/server/modules/filter/qlafilter/qlafilter.c @@ -350,7 +350,7 @@ newSession(FILTER *instance, SESSION *session) if (my_session->fp == NULL) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Opening output file for qla " "fileter failed due to %d, %s", errno, @@ -363,7 +363,7 @@ newSession(FILTER *instance, SESSION *session) } else { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Memory allocation for qla filter failed due to " "%d, %s.", errno, diff --git a/server/modules/monitor/galeramon/galeramon.c b/server/modules/monitor/galeramon/galeramon.c index 781b7e891..c3b96a68e 100644 --- a/server/modules/monitor/galeramon/galeramon.c +++ b/server/modules/monitor/galeramon/galeramon.c @@ -559,12 +559,12 @@ monitorMain(void *arg) /* * Let's select a master server: - * it could be the candidate master following MIN(node_id) rule or + * it could be the candidate master following MXS_MIN(node_id) rule or * the server that was master in the previous monitor polling cycle * Decision depends on master_stickiness value set in configuration */ - /* get the candidate master, following MIN(node_id) rule */ + /* get the candidate master, following MXS_MIN(node_id) rule */ candidate_master = get_candidate_master(mon); /* Select the master, based on master_stickiness */ @@ -656,7 +656,7 @@ monitorMain(void *arg) /** * get candidate master from all nodes * - * The current available rule: get the server with MIN(node_id) + * The current available rule: get the server with MXS_MIN(node_id) * node_id comes from 'wsrep_local_index' variable * * @param servers The monitored servers list diff --git a/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c b/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c index fa1b05141..be8a20909 100644 --- a/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c +++ b/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c @@ -310,7 +310,7 @@ gw_do_connect_to_backend(char *host, int port, int *fd) if (so < 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Establishing connection to backend server " "%s:%d failed.\n\t\t Socket creation failed " "due %d, %s.", @@ -328,7 +328,7 @@ gw_do_connect_to_backend(char *host, int port, int *fd) if (setsockopt(so, SOL_SOCKET, SO_SNDBUF, &bufsize, sizeof(bufsize)) != 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Failed to set socket options " "%s:%d failed.\n\t\t Socket configuration failed " "due %d, %s.", @@ -345,7 +345,7 @@ gw_do_connect_to_backend(char *host, int port, int *fd) if (setsockopt(so, SOL_SOCKET, SO_RCVBUF, &bufsize, sizeof(bufsize)) != 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Failed to set socket options " "%s:%d failed.\n\t\t Socket configuration failed " "due %d, %s.", @@ -362,7 +362,7 @@ gw_do_connect_to_backend(char *host, int port, int *fd) int one = 1; if (setsockopt(so, IPPROTO_TCP, TCP_NODELAY, &one, sizeof(one)) != 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Failed to set socket options " "%s:%d failed.\n\t\t Socket configuration failed " "due %d, %s.", @@ -388,7 +388,7 @@ gw_do_connect_to_backend(char *host, int port, int *fd) } else { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Failed to connect backend server %s:%d, " "due %d, %s.", host, @@ -1057,7 +1057,7 @@ static int gw_error_backend_event(DCB *dcb) { if (error != 0) { - char errstring[STRERROR_BUFLEN]; + char errstring[MXS_STRERROR_BUFLEN]; MXS_ERROR("DCB in state %s got error '%s'.", STRDCBSTATE(dcb->state), strerror_r(error, errstring, sizeof(errstring))); @@ -1095,7 +1095,7 @@ static int gw_error_backend_event(DCB *dcb) { if (error != 0) { - char errstring[STRERROR_BUFLEN]; + char errstring[MXS_STRERROR_BUFLEN]; MXS_ERROR("Error '%s' in session that is not ready for routing.", strerror_r(error, errstring, sizeof(errstring))); } @@ -1199,7 +1199,7 @@ static int gw_backend_hangup(DCB *dcb) { if (error != 0 && ses_state != SESSION_STATE_STOPPING) { - char errstring[STRERROR_BUFLEN]; + char errstring[MXS_STRERROR_BUFLEN]; MXS_ERROR("Hangup in session that is not ready for routing, " "Error reported is '%s'.", strerror_r(error, errstring, sizeof(errstring))); @@ -1766,7 +1766,7 @@ close_socket(int sock) /*< Close newly created socket. */ if (close(sock) != 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Failed to close socket %d due %d, %s.", sock, errno, diff --git a/server/modules/protocol/MySQL/mysql_common.c b/server/modules/protocol/MySQL/mysql_common.c index 77a827967..16ea18d81 100644 --- a/server/modules/protocol/MySQL/mysql_common.c +++ b/server/modules/protocol/MySQL/mysql_common.c @@ -844,7 +844,7 @@ void init_response_status(GWBUF* buf, nparam = gw_mysql_get_byte2(readbuf); gwbuf_copy_data(buf, 11, 2, readbuf); nattr = gw_mysql_get_byte2(readbuf); - *npackets = 1 + nparam + MIN(1, nparam) + nattr + MIN(nattr, 1); + *npackets = 1 + nparam + MXS_MIN(1, nparam) + nattr + MXS_MIN(nattr, 1); break; case MYSQL_COM_QUIT: diff --git a/server/modules/protocol/maxscaled/maxscaled.c b/server/modules/protocol/maxscaled/maxscaled.c index 9e8e6d4d2..df8ef451c 100644 --- a/server/modules/protocol/maxscaled/maxscaled.c +++ b/server/modules/protocol/maxscaled/maxscaled.c @@ -162,7 +162,7 @@ static bool authenticate_socket(MAXSCALED *protocol, DCB *dcb) } else { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Could not get socket family of client connection: %s", strerror_r(errno, errbuf, sizeof(errbuf))); diff --git a/server/modules/routing/avro/avro.c b/server/modules/routing/avro/avro.c index 5e5da7926..794cb3d26 100644 --- a/server/modules/routing/avro/avro.c +++ b/server/modules/routing/avro/avro.c @@ -411,7 +411,7 @@ createInstance(SERVICE *service, char **options) } else if (strcmp(options[i], "start_index") == 0) { - first_file = MAX(1, atoi(value)); + first_file = MXS_MAX(1, atoi(value)); } else { @@ -1037,7 +1037,7 @@ void converter_func(void* data) if (binlog_end == AVRO_LAST_FILE) { - router->task_delay = MIN(router->task_delay + 1, AVRO_TASK_DELAY_MAX); + router->task_delay = MXS_MIN(router->task_delay + 1, AVRO_TASK_DELAY_MAX); add_conversion_task(router); MXS_INFO("Stopped processing file %s at position %lu. Waiting until" " more data is written before continuing. Next check in %d seconds.", @@ -1060,7 +1060,7 @@ static bool ensure_dir_ok(const char* path, int mode) if (path) { - char err[STRERROR_BUFLEN]; + char err[MXS_STRERROR_BUFLEN]; char resolved[PATH_MAX + 1]; const char *rp = realpath(path, resolved); diff --git a/server/modules/routing/avro/avro_client.c b/server/modules/routing/avro/avro_client.c index 2c2936f20..c72c34bce 100644 --- a/server/modules/routing/avro/avro_client.c +++ b/server/modules/routing/avro/avro_client.c @@ -857,7 +857,7 @@ GWBUF* read_avro_json_schema(const char *avrofile, const char* dir) } else { - char err[STRERROR_BUFLEN]; + char err[MXS_STRERROR_BUFLEN]; MXS_ERROR("Failed to open file '%s': %d, %s", buffer, errno, strerror_r(errno, err, sizeof(err))); } diff --git a/server/modules/routing/avro/avro_file.c b/server/modules/routing/avro/avro_file.c index bce83ffc8..fcaab652e 100644 --- a/server/modules/routing/avro/avro_file.c +++ b/server/modules/routing/avro/avro_file.c @@ -40,6 +40,7 @@ #include #include #include +#include static const char *statefile_section = "avro-conversion"; static const char *ddl_list_name = "table-ddl.list"; @@ -160,7 +161,7 @@ bool avro_save_conversion_state(AVRO_INSTANCE *router) { FILE *config_file; char filename[PATH_MAX + 1]; - char err_msg[STRERROR_BUFLEN]; + char err_msg[MXS_STRERROR_BUFLEN]; snprintf(filename, sizeof(filename), "%s/"AVRO_PROGRESS_FILE".tmp", router->avrodir); @@ -419,7 +420,7 @@ static GWBUF* read_event_data(AVRO_INSTANCE *router, REP_HEADER* hdr, uint64_t p { if (n == -1) { - char err_msg[STRERROR_BUFLEN]; + char err_msg[MXS_STRERROR_BUFLEN]; MXS_ERROR("Error reading the event at %lu in %s. " "%s, expected %d bytes.", pos, router->binlog_name, diff --git a/server/modules/routing/avro/avro_rbr.c b/server/modules/routing/avro/avro_rbr.c index a64927290..38dd47655 100644 --- a/server/modules/routing/avro/avro_rbr.c +++ b/server/modules/routing/avro/avro_rbr.c @@ -523,7 +523,7 @@ uint8_t* process_row_event_data(TABLE_MAP *map, TABLE_CREATE *create, avro_value { uint64_t value = 0; int width = metadata[metadata_offset] + metadata[metadata_offset + 1] * 8; - int bits_in_nullmap = MIN(width, extra_bits); + int bits_in_nullmap = MXS_MIN(width, extra_bits); extra_bits -= bits_in_nullmap; width -= bits_in_nullmap; size_t bytes = width / 8; diff --git a/server/modules/routing/binlog/blr.c b/server/modules/routing/binlog/blr.c index 30b17dfb3..a781cb9bb 100644 --- a/server/modules/routing/binlog/blr.c +++ b/server/modules/routing/binlog/blr.c @@ -568,7 +568,7 @@ createInstance(SERVICE *service, char **options) mkdir_rval = mkdir(inst->binlogdir, 0700); if (mkdir_rval == -1) { - char err_msg[STRERROR_BUFLEN]; + char err_msg[MXS_STRERROR_BUFLEN]; MXS_ERROR("Service %s, Failed to create binlog directory '%s': [%d] %s", service->name, inst->binlogdir, @@ -1628,7 +1628,7 @@ errorReply(ROUTER *instance, ROUTER_INSTANCE *router = (ROUTER_INSTANCE *)instance; int error; socklen_t len; - char msg[STRERROR_BUFLEN + 1 + 5] = ""; + char msg[MXS_STRERROR_BUFLEN + 1 + 5] = ""; char *errmsg; unsigned long mysql_errno; @@ -1688,7 +1688,7 @@ errorReply(ROUTER *instance, getsockopt(router->master->fd, SOL_SOCKET, SO_ERROR, &error, &len) == 0 && error != 0) { - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; sprintf(msg, "%s ", strerror_r(error, errbuf, sizeof(errbuf))); } else diff --git a/server/modules/routing/binlog/blr_file.c b/server/modules/routing/binlog/blr_file.c index 342f64990..0db05df59 100644 --- a/server/modules/routing/binlog/blr_file.c +++ b/server/modules/routing/binlog/blr_file.c @@ -244,7 +244,7 @@ blr_file_create(ROUTER_INSTANCE *router, char *file) } int created = 0; - char err_msg[STRERROR_BUFLEN]; + char err_msg[MXS_STRERROR_BUFLEN]; char path[PATH_MAX + 1] = ""; @@ -366,7 +366,7 @@ blr_write_binlog_record(ROUTER_INSTANCE *router, REP_HEADER *hdr, uint32_t size, if ((n = pwrite(router->binlog_fd, buf, size, router->last_written)) != size) { - char err_msg[STRERROR_BUFLEN]; + char err_msg[MXS_STRERROR_BUFLEN]; MXS_ERROR("%s: Failed to write binlog record at %lu of %s, %s. " "Truncating to previous record.", router->service->name, router->last_written, @@ -594,7 +594,7 @@ blr_read_binlog(ROUTER_INSTANCE *router, BLFILE *file, unsigned long pos, REP_HE break; case -1: { - char err_msg[STRERROR_BUFLEN]; + char err_msg[MXS_STRERROR_BUFLEN]; snprintf(errmsg, BINLOG_ERROR_MSG_LEN, "Failed to read binlog file '%s'; (%s), event at %lu", file->binlogname, strerror_r(errno, err_msg, sizeof(err_msg)), pos); @@ -677,7 +677,7 @@ blr_read_binlog(ROUTER_INSTANCE *router, BLFILE *file, unsigned long pos, REP_HE break; case -1: { - char err_msg[STRERROR_BUFLEN]; + char err_msg[MXS_STRERROR_BUFLEN]; snprintf(errmsg, BINLOG_ERROR_MSG_LEN, "Failed to reread header in binlog file '%s'; (%s), event at %lu", file->binlogname, strerror_r(errno, err_msg, sizeof(err_msg)), pos); @@ -737,7 +737,7 @@ blr_read_binlog(ROUTER_INSTANCE *router, BLFILE *file, unsigned long pos, REP_HE { if (n == -1) { - char err_msg[STRERROR_BUFLEN]; + char err_msg[MXS_STRERROR_BUFLEN]; snprintf(errmsg, BINLOG_ERROR_MSG_LEN, "Error reading the binlog event at %lu in binlog file '%s';" "(%s), expected %d bytes.", @@ -1906,7 +1906,7 @@ blr_file_write_master_config(ROUTER_INSTANCE *router, char *error) char filename[len + sizeof('/') + sizeof(MASTER_INI)]; // sizeof includes NULL char tmp_file[len + sizeof('/') + sizeof(MASTER_INI) + sizeof('.') + sizeof(TMP)]; - char err_msg[STRERROR_BUFLEN]; + char err_msg[MXS_STRERROR_BUFLEN]; char *ssl_ca; char *ssl_cert; char *ssl_key; diff --git a/server/modules/routing/binlog/blr_master.c b/server/modules/routing/binlog/blr_master.c index e3de00ebb..7b02a66e3 100644 --- a/server/modules/routing/binlog/blr_master.c +++ b/server/modules/routing/binlog/blr_master.c @@ -1291,7 +1291,7 @@ blr_handle_binlog_record(ROUTER_INSTANCE *router, GWBUF *pkt) if (router->master_chksum) { - uint32_t size = MIN(len - extra_bytes - semisync_bytes, + uint32_t size = MXS_MIN(len - extra_bytes - semisync_bytes, router->checksum_size); router->stored_checksum = crc32(router->stored_checksum, @@ -1342,7 +1342,7 @@ blr_handle_binlog_record(ROUTER_INSTANCE *router, GWBUF *pkt) size = len - (check_packet_len + MYSQL_CHECKSUM_LEN); } - size = MIN(size, router->checksum_size); + size = MXS_MIN(size, router->checksum_size); if (router->checksum_size > 0) { @@ -2454,7 +2454,7 @@ GWBUF break; case -1: { - char err_msg[STRERROR_BUFLEN]; + char err_msg[MXS_STRERROR_BUFLEN]; MXS_ERROR("Reading saved events: failed to read binlog " "file %s at position %llu" " (%s).", router->binlog_name, @@ -2514,7 +2514,7 @@ GWBUF { if (n == -1) { - char err_msg[STRERROR_BUFLEN]; + char err_msg[MXS_STRERROR_BUFLEN]; MXS_ERROR("Reading saved events: the event at %llu in %s. " "%s, expected %d bytes.", pos, router->binlog_name, @@ -2806,7 +2806,7 @@ blr_write_data_into_binlog(ROUTER_INSTANCE *router, uint32_t data_len, uint8_t * if ((n = pwrite(router->binlog_fd, buf, data_len, router->last_written)) != data_len) { - char err_msg[STRERROR_BUFLEN]; + char err_msg[MXS_STRERROR_BUFLEN]; MXS_ERROR("%s: Failed to write binlog record at %lu of %s, %s. " "Truncating to previous record.", router->service->name, router->last_written, @@ -2934,7 +2934,7 @@ bool blr_send_event(blr_thread_role_t role, while (rval && len > 0) { uint64_t payload_len = first ? MYSQL_PACKET_LENGTH_MAX - 1 : - MIN(MYSQL_PACKET_LENGTH_MAX, len); + MXS_MIN(MYSQL_PACKET_LENGTH_MAX, len); if (blr_send_packet(slave, buf, payload_len, first)) { diff --git a/server/modules/routing/binlog/blr_slave.c b/server/modules/routing/binlog/blr_slave.c index 4650053d7..6d4904b18 100644 --- a/server/modules/routing/binlog/blr_slave.c +++ b/server/modules/routing/binlog/blr_slave.c @@ -874,7 +874,7 @@ blr_slave_query(ROUTER_INSTANCE *router, ROUTER_SLAVE *slave, GWBUF *queue) if (removed_cfg == -1) { - char err_msg[STRERROR_BUFLEN]; + char err_msg[MXS_STRERROR_BUFLEN]; snprintf(error_string, BINLOG_ERROR_MSG_LEN, "Error removing %s, %s, errno %u", path, strerror_r(errno, err_msg, sizeof(err_msg)), errno); diff --git a/server/modules/routing/cli/cli.c b/server/modules/routing/cli/cli.c index e5a88e759..948cebc6a 100644 --- a/server/modules/routing/cli/cli.c +++ b/server/modules/routing/cli/cli.c @@ -272,7 +272,7 @@ execute(ROUTER *instance, void *router_session, GWBUF *queue) { const char* data = GWBUF_DATA(queue); int len = GWBUF_LENGTH(queue); - int n = MIN(len, CMDBUFLEN - cmdlen - 1); + int n = MXS_MIN(len, CMDBUFLEN - cmdlen - 1); if (n != len) { diff --git a/server/modules/routing/debugcli/debugcli.c b/server/modules/routing/debugcli/debugcli.c index 26208b328..c360f2da4 100644 --- a/server/modules/routing/debugcli/debugcli.c +++ b/server/modules/routing/debugcli/debugcli.c @@ -293,7 +293,7 @@ execute(ROUTER *instance, void *router_session, GWBUF *queue) { const char* data = GWBUF_DATA(queue); int len = GWBUF_LENGTH(queue); - int n = MIN(len, CMDBUFLEN - cmdlen - 1); + int n = MXS_MIN(len, CMDBUFLEN - cmdlen - 1); if (n != len) { diff --git a/server/modules/routing/debugcli/debugcmd.c b/server/modules/routing/debugcli/debugcmd.c index 247f69f87..b5d3d1d4c 100644 --- a/server/modules/routing/debugcli/debugcmd.c +++ b/server/modules/routing/debugcli/debugcmd.c @@ -1051,7 +1051,7 @@ execute_cmd(CLI_SESSION *cli) } } *lptr = 0; - args[MIN(MAXARGS - 1, i + 1)] = NULL; + args[MXS_MIN(MAXARGS - 1, i + 1)] = NULL; if (args[0] == NULL || *args[0] == 0) { @@ -1964,9 +1964,9 @@ static void fail_accept( char* arg1, char* arg2) { - int failcount = MIN(atoi(arg2), 100); + int failcount = MXS_MIN(atoi(arg2), 100); fail_accept_errno = atoi(arg1); - char errbuf[STRERROR_BUFLEN]; + char errbuf[MXS_STRERROR_BUFLEN]; switch(fail_accept_errno) { case EAGAIN: diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index 347ce8d15..7d9b8f3e4 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -559,7 +559,7 @@ static void *newSession(ROUTER *router_inst, SESSION *session) { int n_conn = 0; double pct = (double)client_rses->rses_config.rw_max_slave_conn_percent / 100.0; - n_conn = MAX(floor((double)client_rses->rses_nbackends * pct), 1); + n_conn = MXS_MAX(floor((double)client_rses->rses_nbackends * pct), 1); client_rses->rses_config.rw_max_slave_conn_count = n_conn; } @@ -1294,7 +1294,7 @@ int rses_get_max_slavecount(ROUTER_CLIENT_SES *rses, { conf_max_nslaves = (router_nservers * rses->rses_config.rw_max_slave_conn_percent) / 100; } - max_nslaves = MIN(router_nservers - 1, MAX(1, conf_max_nslaves)); + max_nslaves = MXS_MIN(router_nservers - 1, MXS_MAX(1, conf_max_nslaves)); return max_nslaves; } @@ -1897,7 +1897,7 @@ static bool have_enough_servers(ROUTER_CLIENT_SES **p_rses, const int min_nsrv, /** With too few servers session is not created */ if (router_nsrv < min_nsrv || - MAX((*p_rses)->rses_config.rw_max_slave_conn_count, + MXS_MAX((*p_rses)->rses_config.rw_max_slave_conn_count, (router_nsrv * (*p_rses)->rses_config.rw_max_slave_conn_percent) / 100) < min_nsrv) { diff --git a/server/modules/routing/readwritesplit/rwsplit_mysql.c b/server/modules/routing/readwritesplit/rwsplit_mysql.c index 4397f9a11..aa7050dcd 100644 --- a/server/modules/routing/readwritesplit/rwsplit_mysql.c +++ b/server/modules/routing/readwritesplit/rwsplit_mysql.c @@ -163,10 +163,10 @@ log_transaction_status(ROUTER_CLIENT_SES *rses, GWBUF *querybuf, qc_query_type_t { uint8_t *packet = GWBUF_DATA(querybuf); unsigned char ptype = packet[4]; - size_t len = MIN(GWBUF_LENGTH(querybuf), + size_t len = MXS_MIN(GWBUF_LENGTH(querybuf), MYSQL_GET_PACKET_LEN((unsigned char *)querybuf->start) - 1); char *data = (char *)&packet[5]; - char *contentstr = strndup(data, MIN(len, RWSPLIT_TRACE_MSG_LEN)); + char *contentstr = strndup(data, MXS_MIN(len, RWSPLIT_TRACE_MSG_LEN)); char *qtypestr = qc_get_qtype_str(qtype); MXS_INFO("> Autocommit: %s, trx is %s, cmd: %s, type: %s, stmt: %s%s %s", (rses->rses_autocommit_enabled ? "[enabled]" : "[disabled]"), diff --git a/server/modules/routing/schemarouter/schemarouter.c b/server/modules/routing/schemarouter/schemarouter.c index 3ed3ba191..551841b5a 100644 --- a/server/modules/routing/schemarouter/schemarouter.c +++ b/server/modules/routing/schemarouter/schemarouter.c @@ -2004,7 +2004,7 @@ static int routeQuery(ROUTER* instance, { uint8_t* packet = GWBUF_DATA(querybuf); unsigned char ptype = packet[4]; - size_t len = MIN(GWBUF_LENGTH(querybuf), + size_t len = MXS_MIN(GWBUF_LENGTH(querybuf), MYSQL_GET_PACKET_LEN((unsigned char *)querybuf->start) - 1); char* data = (char*)&packet[5]; char* contentstr = strndup(data, len); From cd26c69ffba39255de8710f95632505104cf7b85 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Fri, 14 Oct 2016 15:52:27 +0300 Subject: [PATCH 032/215] Remove slist Not used by anybody, so better to remove it. --- include/maxscale/slist.h | 68 ------- server/core/CMakeLists.txt | 2 +- server/core/slist.c | 364 ------------------------------------- 3 files changed, 1 insertion(+), 433 deletions(-) delete mode 100644 include/maxscale/slist.h delete mode 100644 server/core/slist.c diff --git a/include/maxscale/slist.h b/include/maxscale/slist.h deleted file mode 100644 index 995e32cee..000000000 --- a/include/maxscale/slist.h +++ /dev/null @@ -1,68 +0,0 @@ -#pragma once -#ifndef _MAXSCALE_SLIST_H -#define _MAXSCALE_SLIST_H -/* - * Copyright (c) 2016 MariaDB Corporation Ab - * - * Use of this software is governed by the Business Source License included - * in the LICENSE.TXT file and at www.mariadb.com/bsl. - * - * Change Date: 2019-07-01 - * - * On the date above, in accordance with the Business Source License, use - * of this software will be governed by version 2 or later of the General - * Public License. - */ - -#include -#include - -MXS_BEGIN_DECLS - -typedef struct slist_node_st slist_node_t; -typedef struct slist_st slist_t; -typedef struct slist_cursor_st slist_cursor_t; - -/** Single-linked list */ - -struct slist_node_st -{ - skygw_chk_t slnode_chk_top; - slist_t* slnode_list; - slist_node_t* slnode_next; - void* slnode_data; - size_t slnode_cursor_refcount; - skygw_chk_t slnode_chk_tail; -}; - -struct slist_st -{ - skygw_chk_t slist_chk_top; - slist_node_t* slist_head; - slist_node_t* slist_tail; - int slist_nelems; - slist_t* slist_cursors_list; - skygw_chk_t slist_chk_tail; -}; - -struct slist_cursor_st -{ - skygw_chk_t slcursor_chk_top; - slist_t* slcursor_list; - slist_node_t* slcursor_pos; - skygw_chk_t slcursor_chk_tail; -}; - -slist_cursor_t* slist_init(void); -void slist_done(slist_cursor_t* c); -size_t slist_size(slist_cursor_t* c); - -void slcursor_add_data(slist_cursor_t* c, void* data); -void* slcursor_get_data(slist_cursor_t* c); -void slcursor_remove_data(slist_cursor_t* c); -bool slcursor_move_to_begin(slist_cursor_t* c); -bool slcursor_step_ahead(slist_cursor_t* c); - -MXS_END_DECLS - -#endif diff --git a/server/core/CMakeLists.txt b/server/core/CMakeLists.txt index 06302df63..11a616a62 100644 --- a/server/core/CMakeLists.txt +++ b/server/core/CMakeLists.txt @@ -1,4 +1,4 @@ -add_library(maxscale-common SHARED adminusers.c alloc.c authenticator.c atomic.c buffer.c config.c dbusers.c dcb.c filter.c externcmd.c gwbitmask.c gwdirs.c gw_utils.c hashtable.c hint.c housekeeper.c listmanager.c load_utils.c log_manager.cc maxscale_pcre2.c memlog.c misc.c mlist.c modutil.c monitor.c queuemanager.c query_classifier.c poll.c random_jkiss.c resultset.c secrets.c server.c service.c session.c slist.c spinlock.c thread.c users.c utils.c skygw_utils.cc statistics.c listener.c gw_ssl.c mysql_utils.c mysql_binlog.c) +add_library(maxscale-common SHARED adminusers.c alloc.c authenticator.c atomic.c buffer.c config.c dbusers.c dcb.c filter.c externcmd.c gwbitmask.c gwdirs.c gw_utils.c hashtable.c hint.c housekeeper.c listmanager.c load_utils.c log_manager.cc maxscale_pcre2.c memlog.c misc.c mlist.c modutil.c monitor.c queuemanager.c query_classifier.c poll.c random_jkiss.c resultset.c secrets.c server.c service.c session.c spinlock.c thread.c users.c utils.c skygw_utils.cc statistics.c listener.c gw_ssl.c mysql_utils.c mysql_binlog.c) target_link_libraries(maxscale-common ${MARIADB_CONNECTOR_LIBRARIES} ${LZMA_LINK_FLAGS} ${PCRE2_LIBRARIES} ${CURL_LIBRARIES} ssl pthread crypt dl crypto inih z rt m stdc++) diff --git a/server/core/slist.c b/server/core/slist.c deleted file mode 100644 index ab5cdd7ca..000000000 --- a/server/core/slist.c +++ /dev/null @@ -1,364 +0,0 @@ -/* - * Copyright (c) 2016 MariaDB Corporation Ab - * - * Use of this software is governed by the Business Source License included - * in the LICENSE.TXT file and at www.mariadb.com/bsl. - * - * Change Date: 2019-07-01 - * - * On the date above, in accordance with the Business Source License, use - * of this software will be governed by version 2 or later of the General - * Public License. - */ - -#include -#include -#include - -static slist_cursor_t* slist_cursor_init(slist_t* list); -static slist_t* slist_init_ex(bool create_cursors); -static slist_node_t* slist_node_init(void* data, slist_cursor_t* cursor); -static void slist_add_node(slist_t* list, slist_node_t* node); - -#if defined(NOT_USED) -static slist_node_t* slist_node_get_next(slist_node_t* curr_node); -static slist_node_t* slist_get_first(slist_t* list); -static slist_cursor_t* slist_get_cursor(slist_t* list); -#endif /*< NOT_USED */ - -/** End of static function declarations */ - -static slist_t* slist_init_ex(bool create_cursors) -{ - slist_t* list; - - list = (slist_t*) MXS_CALLOC(1, sizeof (slist_t)); - MXS_ABORT_IF_NULL(list); - list->slist_chk_top = CHK_NUM_SLIST; - list->slist_chk_tail = CHK_NUM_SLIST; - - if (create_cursors) - { - list->slist_cursors_list = slist_init_ex(false); - } - - return list; -} - -static slist_node_t* slist_node_init(void* data, slist_cursor_t* cursor) -{ - slist_node_t* node; - - node = (slist_node_t*) MXS_CALLOC(1, sizeof (slist_node_t)); - MXS_ABORT_IF_NULL(node); - node->slnode_chk_top = CHK_NUM_SLIST_NODE; - node->slnode_chk_tail = CHK_NUM_SLIST_NODE; - node->slnode_data = data; - CHK_SLIST_NODE(node); - - if (cursor != NULL) - { - node->slnode_cursor_refcount += 1; - cursor->slcursor_pos = node; - } - - return node; -} - -static void slist_add_node(slist_t* list, slist_node_t* node) -{ - CHK_SLIST(list); - CHK_SLIST_NODE(node); - - if (list->slist_tail != NULL) - { - CHK_SLIST_NODE(list->slist_tail); - CHK_SLIST_NODE(list->slist_head); - ss_dassert(list->slist_tail->slnode_next == NULL); - list->slist_tail->slnode_next = node; - } - else - { - list->slist_head = node; - } - list->slist_tail = node; - node->slnode_list = list; - list->slist_nelems += 1; - CHK_SLIST(list); -} - - -#if defined(NOT_USED) - -static slist_node_t* slist_node_get_next(slist_node_t* curr_node) -{ - CHK_SLIST_NODE(curr_node); - - if (curr_node->slnode_next != NULL) - { - CHK_SLIST_NODE(curr_node->slnode_next); - return (curr_node->slnode_next); - } - - return NULL; -} - -static slist_node_t* slist_get_first(slist_t* list) -{ - CHK_SLIST(list); - - if (list->slist_head != NULL) - { - CHK_SLIST_NODE(list->slist_head); - return list->slist_head; - } - return NULL; -} - -static slist_cursor_t* slist_get_cursor(slist_t* list) -{ - CHK_SLIST(list); - - slist_cursor_t* c; - - c = slist_cursor_init(list); - return c; -} -#endif /*< NOT_USED */ - -static slist_cursor_t* slist_cursor_init(slist_t* list) -{ - CHK_SLIST(list); - slist_cursor_t* c; - - c = (slist_cursor_t *) MXS_CALLOC(1, sizeof (slist_cursor_t)); - MXS_ABORT_IF_NULL(c); - c->slcursor_chk_top = CHK_NUM_SLIST_CURSOR; - c->slcursor_chk_tail = CHK_NUM_SLIST_CURSOR; - c->slcursor_list = list; - /** Set cursor position is list is not empty */ - if (list->slist_head != NULL) - { - list->slist_head->slnode_cursor_refcount += 1; - c->slcursor_pos = list->slist_head; - } - /** Add cursor to cursor list */ - slist_add_node(list->slist_cursors_list, slist_node_init(c, NULL)); - - CHK_SLIST_CURSOR(c); - return c; -} - -/** - * @node Create a cursor and a list with cursors supported. 19.6.2013 : - * supports only cursor per list. - * - * Parameters: - * @param void - - * - * - * @return returns a pointer to cursor, which is not positioned - * because the list is empty. - * - * - * @details (write detailed description here) - * - */ -slist_cursor_t* slist_init(void) -{ - slist_t* list; - slist_cursor_t* slc; - - list = slist_init_ex(true); - CHK_SLIST(list); - slc = slist_cursor_init(list); - CHK_SLIST_CURSOR(slc); - - return slc; -} - -/** - * @node moves cursor to the first node of list. - * - * Parameters: - * @param c - - * - * - * @return true if there is first node in the list - * false is the list is empty. - * - * - * @details (write detailed description here) - * - */ -bool slcursor_move_to_begin(slist_cursor_t* c) -{ - bool succp = true; - slist_t* list; - - CHK_SLIST_CURSOR(c); - list = c->slcursor_list; - CHK_SLIST(list); - c->slcursor_pos = list->slist_head; - if (c->slcursor_pos == NULL) - { - succp = false; - } - return succp; -} - -/** - * @node moves cursor to next node - * - * Parameters: - * @param c - - * - * - * @return true in success, false is there is no next node on the list. - * - * - * @details (write detailed description here) - * - */ -bool slcursor_step_ahead(slist_cursor_t* c) -{ - bool succp = false; - slist_node_t* node; - CHK_SLIST_CURSOR(c); - CHK_SLIST_NODE(c->slcursor_pos); - - node = c->slcursor_pos->slnode_next; - - if (node != NULL) - { - CHK_SLIST_NODE(node); - c->slcursor_pos = node; - succp = true; - } - return succp; -} - -void* slcursor_get_data(slist_cursor_t* c) -{ - slist_node_t* node; - void* data = NULL; - - CHK_SLIST_CURSOR(c); - node = c->slcursor_pos; - - if (node != NULL) - { - CHK_SLIST_NODE(node); - data = node->slnode_data; - } - return data; -} - -/** - * @node Add data to the list by using cursor. - * - * Parameters: - * @param c - - * - * - * @param data - - * - * - * @return void - * - * - * @details (write detailed description here) - * - */ -void slcursor_add_data(slist_cursor_t* c, void* data) -{ - slist_t* list; - slist_node_t* pos; - - CHK_SLIST_CURSOR(c); - list = c->slcursor_list; - CHK_SLIST(list); - if (c->slcursor_pos != NULL) - { - CHK_SLIST_NODE(c->slcursor_pos); - } - ss_dassert(list->slist_tail->slnode_next == NULL); - pos = slist_node_init(data, c); - slist_add_node(list, pos); - CHK_SLIST(list); - CHK_SLIST_CURSOR(c); -} - -/** - * Remove the node currently pointed by the cursor from the slist. This does not delete the data in the - * node but will delete the structure pointing to that data. This is useful when - * the user wants to free the allocated memory. After node removal, the cursor - * will point to the node before the removed node. - * @param c Cursor pointing to the data node to be removed - */ -void slcursor_remove_data(slist_cursor_t* c) -{ - slist_node_t* node = c->slcursor_pos; - int havemore = slist_size(c); - slcursor_move_to_begin(c); - - if (node == c->slcursor_pos) - { - c->slcursor_list->slist_head = c->slcursor_list->slist_head->slnode_next; - slcursor_move_to_begin(c); - atomic_add((int*) &node->slnode_list->slist_nelems, -1); - atomic_add((int*) &node->slnode_cursor_refcount, -1); - if (node->slnode_cursor_refcount == 0) - { - MXS_FREE(node); - } - return; - } - - while (havemore) - { - if (c->slcursor_pos->slnode_next == node) - { - c->slcursor_pos->slnode_next = node->slnode_next; - atomic_add((int*) &node->slnode_list->slist_nelems, -1); - atomic_add((int*) &node->slnode_cursor_refcount, -1); - if (node->slnode_cursor_refcount == 0) - { - MXS_FREE(node); - } - return; - } - havemore = slcursor_step_ahead(c); - } -} - -/** - * Return the size of the slist. - * @param c slist cursor which refers to a list - * @return nummber of elements in the list - */ -size_t slist_size(slist_cursor_t* c) -{ - return c->slcursor_list->slist_nelems; -} - -void slist_done(slist_cursor_t* c) -{ - bool succp; - void* data; - - succp = slcursor_move_to_begin(c); - - while (succp) - { - data = slcursor_get_data(c); - MXS_FREE(data); - succp = slcursor_step_ahead(c); - } - MXS_FREE(c->slcursor_list); - MXS_FREE(c); -} - - -/** End of list implementation */ - From 68e53567a0bc0e3e174ea55cc7b27566521b43f3 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Fri, 14 Oct 2016 15:55:33 +0300 Subject: [PATCH 033/215] Make mlist.h private Mlist is only used by log_manager so for the time being it is moved under server/core/maxscale. Eventually it may be included in log_manager itself. --- server/core/log_manager.cc | 2 +- {include => server/core}/maxscale/mlist.h | 0 server/core/mlist.c | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename {include => server/core}/maxscale/mlist.h (100%) diff --git a/server/core/log_manager.cc b/server/core/log_manager.cc index 2a19fbf50..996e5e001 100644 --- a/server/core/log_manager.cc +++ b/server/core/log_manager.cc @@ -26,12 +26,12 @@ #include #include -#include #include #include #include #include #include +#include "maxscale/mlist.h" #define MAX_PREFIXLEN 250 #define MAX_SUFFIXLEN 250 diff --git a/include/maxscale/mlist.h b/server/core/maxscale/mlist.h similarity index 100% rename from include/maxscale/mlist.h rename to server/core/maxscale/mlist.h diff --git a/server/core/mlist.c b/server/core/mlist.c index c83df050b..a67822e2c 100644 --- a/server/core/mlist.c +++ b/server/core/mlist.c @@ -11,7 +11,7 @@ * Public License. */ -#include +#include "maxscale/mlist.h" #include static void mlist_free_memory(mlist_t* ml, char* name); From ba97361c02c915c613ef625adf7bfead83f2ab6b Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Fri, 14 Oct 2016 15:46:25 +0200 Subject: [PATCH 034/215] Hole detection code review update Hole detection code review update --- server/modules/routing/binlog/blr_file.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/server/modules/routing/binlog/blr_file.c b/server/modules/routing/binlog/blr_file.c index 94fffabc5..f5d61ca75 100644 --- a/server/modules/routing/binlog/blr_file.c +++ b/server/modules/routing/binlog/blr_file.c @@ -1581,8 +1581,7 @@ blr_read_events_all_events(ROUTER_INSTANCE *router, int fix, int debug) /* Detect possible Start Encryption Event */ if (hdr.event_type == MARIADB10_START_ENCRYPTION_EVENT) { - START_ENCRYPTION_EVENT ste_event; - memset(&ste_event, '\0', sizeof(START_ENCRYPTION_EVENT)); + START_ENCRYPTION_EVENT ste_event = {}; char nonce_hex[BLRM_NONCE_LENGTH * 2 + 1] = ""; /* The start encryption event data is 17 bytes long: * Scheme = 1 @@ -2289,7 +2288,6 @@ blr_write_special_event(ROUTER_INSTANCE *router, uint32_t file_offset, uint32_t (unsigned long)event_size, router->binlog_name, router->current_pos); - new_event_desc = "UNKNOWN"; return 0; break; } From 571919264fb74c6a4181c1d6f863d70335643abd Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 13 Oct 2016 23:10:59 +0300 Subject: [PATCH 035/215] MXS-862: Make packet sequence handling automatic Made the packet sequence number handling automatic so that it always uses the correct one. All functions now have documentation in them. Cleaned up code and added comments to GSSAPI code. --- include/maxscale/protocol/mysql.h | 17 ++- server/modules/authenticator/gssapi_auth.c | 16 ++- server/modules/authenticator/gssapi_auth.h | 7 +- .../authenticator/gssapi_auth_common.c | 1 + .../authenticator/gssapi_backend_auth.c | 115 ++++++++++++++---- .../protocol/MySQL/MySQLClient/mysql_client.c | 3 + 6 files changed, 122 insertions(+), 37 deletions(-) diff --git a/include/maxscale/protocol/mysql.h b/include/maxscale/protocol/mysql.h index 568e520c1..7a1c0af06 100644 --- a/include/maxscale/protocol/mysql.h +++ b/include/maxscale/protocol/mysql.h @@ -75,9 +75,15 @@ MXS_BEGIN_DECLS #define MYSQL_HEADER_LEN 4L #define MYSQL_CHECKSUM_LEN 4L -/** Offsets to various parts of the client packet */ -#define MYSQL_SEQ_OFFSET 3 -#define MYSQL_COM_OFFSET 4 +/** + * Offsets and sizes of various parts of the client packet. If the offset is + * defined but not the size, the size of the value is one byte. + */ +#define MYSQL_SEQ_OFFSET 3 +#define MYSQL_COM_OFFSET 4 +#define MYSQL_CHARSET_OFFSET 12 +#define MYSQL_CLIENT_CAP_OFFSET 4 +#define MYSQL_CLIENT_CAP_SIZE 4 #define GW_MYSQL_PROTOCOL_VERSION 10 // version is 10 #define GW_MYSQL_HANDSHAKE_FILLER 0x00 @@ -279,8 +285,9 @@ typedef struct } MySQLProtocol; /** Defines for response codes */ -#define MYSQL_REPLY_ERR 0xff -#define MYSQL_REPLY_OK 0x00 +#define MYSQL_REPLY_ERR 0xff +#define MYSQL_REPLY_OK 0x00 +#define MYSQL_REPLY_AUTHSWITCHREQUEST 0xfe /* * Let's try this with proper enums instead of numbers diff --git a/server/modules/authenticator/gssapi_auth.c b/server/modules/authenticator/gssapi_auth.c index 5f57d642a..e7264ff56 100644 --- a/server/modules/authenticator/gssapi_auth.c +++ b/server/modules/authenticator/gssapi_auth.c @@ -82,7 +82,7 @@ void* gssapi_auth_init(char **options) * @see https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchRequest * @see https://web.mit.edu/kerberos/krb5-1.5/krb5-1.5.4/doc/krb5-user/What-is-a-Kerberos-Principal_003f.html */ -static GWBUF* create_auth_change_packet(GSSAPI_INSTANCE *instance) +static GWBUF* create_auth_change_packet(GSSAPI_INSTANCE *instance, gssapi_auth_t *auth) { size_t principal_name_len = strlen(instance->principal_name); size_t plen = sizeof(auth_plugin_name) + 1 + principal_name_len; @@ -93,7 +93,7 @@ static GWBUF* create_auth_change_packet(GSSAPI_INSTANCE *instance) uint8_t *data = (uint8_t*)GWBUF_DATA(buffer); gw_mysql_set_byte3(data, plen); data += 3; - *data++ = 0x02; // Second packet + *data++ = ++auth->sequence; // Second packet *data++ = 0xfe; // AuthSwitchRequest command memcpy(data, auth_plugin_name, sizeof(auth_plugin_name)); // Plugin name data += sizeof(auth_plugin_name); @@ -143,10 +143,14 @@ static void copy_client_information(DCB *dcb, GWBUF *buffer) { size_t buflen = gwbuf_length(buffer); MySQLProtocol *protocol = (MySQLProtocol*)dcb->protocol; - /* Take data from fixed locations first */ - gwbuf_copy_data(buffer, 4, 4, (uint8_t*)&protocol->client_capabilities); + gssapi_auth_t *auth = (gssapi_auth_t*)dcb->authenticator_data; + + /* Store the connection characteristics and sequence number of the current packet */ protocol->charset = 0; - gwbuf_copy_data(buffer, 4 + 4 + 4, 1, (uint8_t*)&protocol->charset); + gwbuf_copy_data(buffer, MYSQL_CHARSET_OFFSET, 1, (uint8_t*)&protocol->charset); + gwbuf_copy_data(buffer, MYSQL_CLIENT_CAP_OFFSET, MYSQL_CLIENT_CAP_SIZE, + (uint8_t*)&protocol->client_capabilities); + gwbuf_copy_data(buffer, MYSQL_SEQ_OFFSET, 1, &auth->sequence); if (buflen > MYSQL_AUTH_PACKET_BASE_SIZE) { @@ -292,7 +296,7 @@ int gssapi_auth_authenticate(DCB *dcb) /** We need to send the authentication switch packet to change the * authentication to something other than the 'mysql_native_password' * method */ - GWBUF *buffer = create_auth_change_packet(instance); + GWBUF *buffer = create_auth_change_packet(instance, auth); if (buffer && dcb->func.write(dcb, buffer)) { diff --git a/server/modules/authenticator/gssapi_auth.h b/server/modules/authenticator/gssapi_auth.h index 317db08b7..be6bd7b0f 100644 --- a/server/modules/authenticator/gssapi_auth.h +++ b/server/modules/authenticator/gssapi_auth.h @@ -39,9 +39,10 @@ enum gssapi_auth_state /** Common state tracking structure */ typedef struct gssapi_auth { - enum gssapi_auth_state state; - uint8_t *principal_name; - size_t principal_name_len; + enum gssapi_auth_state state; /**< Authentication state*/ + uint8_t *principal_name; /**< Principal name */ + size_t principal_name_len; /**< Length of the principal name */ + uint8_t sequence; /**< The next packet seqence number */ } gssapi_auth_t; /** These functions can used for the `create` and `destroy` entry points */ diff --git a/server/modules/authenticator/gssapi_auth_common.c b/server/modules/authenticator/gssapi_auth_common.c index 8fa64627a..5ba0a35e5 100644 --- a/server/modules/authenticator/gssapi_auth_common.c +++ b/server/modules/authenticator/gssapi_auth_common.c @@ -24,6 +24,7 @@ void* gssapi_auth_alloc(void *instance) rval->state = GSSAPI_AUTH_INIT; rval->principal_name = NULL; rval->principal_name_len = 0; + rval->sequence = 0; } return rval; diff --git a/server/modules/authenticator/gssapi_backend_auth.c b/server/modules/authenticator/gssapi_backend_auth.c index d047b80f8..9b43a7f31 100644 --- a/server/modules/authenticator/gssapi_backend_auth.c +++ b/server/modules/authenticator/gssapi_backend_auth.c @@ -19,11 +19,14 @@ #include "gssapi_auth.h" /** - * @file gssapi_backend_auth.c GSSAPI backend authenticator + * @file gssapi_backend_auth.c - GSSAPI backend authenticator */ -/** TODO: Document functions and GSSAPI specific code */ - +/** + * @brief Create a new GSSAPI token + * @param dcb Backend DCB + * @return True on success, false on error + */ static bool send_new_auth_token(DCB *dcb) { bool rval = false; @@ -35,9 +38,11 @@ static bool send_new_auth_token(DCB *dcb) gss_name_t princ = GSS_C_NO_NAME; gssapi_auth_t *auth = (gssapi_auth_t*)dcb->authenticator_data; + /** The service principal name is sent by the backend server */ target.value = auth->principal_name; target.length = auth->principal_name_len + 1; + /** Convert the name into GSSAPI format */ major = gss_import_name(&minor, &target, GSS_C_NT_USER_NAME, &princ); if (GSS_ERROR(major)) @@ -45,6 +50,7 @@ static bool send_new_auth_token(DCB *dcb) report_error(major, minor); } + /** Request the token for the service */ major = gss_init_sec_context(&minor, GSS_C_NO_CREDENTIAL, &handle, princ, GSS_C_NO_OID, 0, 0, GSS_C_NO_CHANNEL_BINDINGS, &in, NULL, &out, 0, 0); @@ -54,6 +60,7 @@ static bool send_new_auth_token(DCB *dcb) } else { + /** We successfully requested the token, send it to the backend server */ GWBUF *buffer = gwbuf_alloc(MYSQL_HEADER_LEN + out.length); if (buffer) @@ -61,8 +68,9 @@ static bool send_new_auth_token(DCB *dcb) uint8_t *data = (uint8_t*)GWBUF_DATA(buffer); gw_mysql_set_byte3(data, out.length); data += 3; - *data++ = 0x03; + *data++ = ++auth->sequence; memcpy(data, out.value, out.length); + if (dcb_write(dcb, buffer)) { rval = true; @@ -77,6 +85,7 @@ static bool send_new_auth_token(DCB *dcb) } major = gss_release_name(&minor, &princ); + if (GSS_ERROR(major)) { report_error(major, minor); @@ -87,15 +96,48 @@ static bool send_new_auth_token(DCB *dcb) } +/** + * @brief Extract the principal name from the AuthSwitchRequest packet + * + * @param dcb Backend DCB + * @param buffer Buffer containing an AuthSwitchRequest packet + * @return True on success, false on error + */ bool extract_principal_name(DCB *dcb, GWBUF *buffer) { bool rval = false; size_t buflen = gwbuf_length(buffer) - MYSQL_HEADER_LEN; uint8_t databuf[buflen]; - gwbuf_copy_data(buffer, MYSQL_HEADER_LEN, buflen, databuf); uint8_t *data = databuf; + gssapi_auth_t *auth = (gssapi_auth_t*)dcb->authenticator_data; - while (*data) + /** Copy the payload and the current packet sequence number */ + gwbuf_copy_data(buffer, MYSQL_HEADER_LEN, buflen, databuf); + gwbuf_copy_data(buffer, MYSQL_SEQ_OFFSET, 1, &auth->sequence); + + if (databuf[0] != MYSQL_REPLY_AUTHSWITCHREQUEST) + { + /** Server responded with something we did not expect. If it's an OK packet, + * it's possible that the server authenticated us as the anonymous user. This + * means that the server is not secure. */ + MXS_ERROR("Server '%s' returned an unexpected authentication response.%s", + dcb->server->unique_name, databuf[0] == MYSQL_REPLY_OK ? + " Authentication was complete before it even started, " + "anonymous users might not be disabled." : ""); + return false; + } + + /** + * The AuthSwitchRequest packet + * + * 0xfe - Command byte + * string[NUL] - Auth plugin name + * string[EOF] - Auth plugin data + * + * Skip over the auth plugin name and copy the service principal name stored + * in the auth plugin data section. + */ + while (*data && data < databuf + buflen) { data++; } @@ -103,35 +145,51 @@ bool extract_principal_name(DCB *dcb, GWBUF *buffer) data++; buflen -= data - databuf; - uint8_t *principal = MXS_MALLOC(buflen); - - if (principal) + if (buflen > 0) { - memcpy(principal, data, buflen); - gssapi_auth_t *auth = (gssapi_auth_t*)dcb->authenticator_data; - auth->principal_name = principal; - auth->principal_name_len = buflen; - rval = true; + uint8_t *principal = MXS_MALLOC(buflen + 1); + + if (principal) + { + /** Store the principal name for later when we request the token + * from the GSSAPI server */ + memcpy(principal, data, buflen); + principal[buflen] = '\0'; + auth->principal_name = principal; + auth->principal_name_len = buflen; + rval = true; + } + } + else + { + MXS_ERROR("Backend server did not send any auth plugin data."); } return rval; } +/** + * @brief Extract data from a MySQL packet + * @param dcb Backend DCB + * @param buffer Buffer containing a complete packet + * @return MXS_AUTH_INCOMPLETE if authentication is ongoing, MXS_AUTH_SUCCEEDED + * if authentication is complete and MXS_AUTH_FAILED if authentication failed. + */ static int gssapi_backend_auth_extract(DCB *dcb, GWBUF *buffer) { int rval = MXS_AUTH_FAILED; - gssapi_auth_t *data = (gssapi_auth_t*)dcb->authenticator_data; + gssapi_auth_t *auth = (gssapi_auth_t*)dcb->authenticator_data; - if (data->state == GSSAPI_AUTH_INIT && extract_principal_name(dcb, buffer)) + if (auth->state == GSSAPI_AUTH_INIT && extract_principal_name(dcb, buffer)) { rval = MXS_AUTH_INCOMPLETE; } - else if (data->state == GSSAPI_AUTH_DATA_SENT) + else if (auth->state == GSSAPI_AUTH_DATA_SENT) { /** Read authentication response */ if (mxs_mysql_is_ok_packet(buffer)) { - data->state = GSSAPI_AUTH_OK; + auth->state = GSSAPI_AUTH_OK; rval = MXS_AUTH_SUCCEEDED; } } @@ -139,26 +197,37 @@ static int gssapi_backend_auth_extract(DCB *dcb, GWBUF *buffer) return rval; } +/** + * @brief Check whether the DCB supports SSL + * @param dcb Backend DCB + * @return True if DCB supports SSL + */ static bool gssapi_backend_auth_connectssl(DCB *dcb) { return dcb->server->server_ssl != NULL; } +/** + * @brief Authenticate the backend connection + * @param dcb Backend DCB + * @return MXS_AUTH_INCOMPLETE if authentication is ongoing, MXS_AUTH_SUCCEEDED + * if authentication is complete and MXS_AUTH_FAILED if authentication failed. + */ static int gssapi_backend_auth_authenticate(DCB *dcb) { int rval = MXS_AUTH_FAILED; - gssapi_auth_t *auth_data = (gssapi_auth_t*)dcb->authenticator_data; + gssapi_auth_t *auth = (gssapi_auth_t*)dcb->authenticator_data; - if (auth_data->state == GSSAPI_AUTH_INIT) + if (auth->state == GSSAPI_AUTH_INIT) { if (send_new_auth_token(dcb)) { rval = MXS_AUTH_INCOMPLETE; - auth_data->state = GSSAPI_AUTH_DATA_SENT; + auth->state = GSSAPI_AUTH_DATA_SENT; } } - else if (auth_data->state == GSSAPI_AUTH_OK) + else if (auth->state == GSSAPI_AUTH_OK) { rval = MXS_AUTH_SUCCEEDED; } @@ -171,7 +240,7 @@ static int gssapi_backend_auth_authenticate(DCB *dcb) */ static GWAUTHENTICATOR MyObject = { - NULL, /* TODO: implement initialize entry point */ + NULL, /* No initialize entry point */ gssapi_auth_alloc, /* Allocate authenticator data */ gssapi_backend_auth_extract, /* Extract data into structure */ gssapi_backend_auth_connectssl, /* Check if client supports SSL */ diff --git a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c index 53e240687..d57cbe677 100644 --- a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c +++ b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c @@ -479,6 +479,9 @@ int gw_read_client_event(DCB* dcb) static int gw_read_do_authentication(DCB *dcb, GWBUF *read_buffer, int nbytes_read) { + ss_debug(uint8_t hdr[MYSQL_HEADER_LEN]); + ss_dassert(gwbuf_copy_data(read_buffer, 0, MYSQL_HEADER_LEN, hdr) == MYSQL_HEADER_LEN && + MYSQL_GET_PACKET_LEN(hdr) + MYSQL_HEADER_LEN == gwbuf_length(read_buffer)); /** Allocate the shared session structure */ if (dcb->data == NULL && (dcb->data = mysql_session_alloc()) == NULL) { From 1333da071220b8eaf8c6920e107d780c4387b269 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Fri, 14 Oct 2016 19:41:50 +0300 Subject: [PATCH 036/215] Remove skygw_utils.h The general purpose stuff in skygw_utils.h was moved to utils.h and the corresponding implementation from skygw_utils.cc to utils.c. Includes updated accordingly. Skygw_utils.h is now only used by log_manager and by mlist, which is only used by log_manager. Consequently, skygw_utils.h was moved to server/maxscale. Utils.h needs a separate overhaul. --- include/maxscale/utils.h | 18 + query_classifier/qc_sqlite/qc_sqlite.c | 1 - .../test/canonical_tests/canonizer.c | 2 +- server/core/adminusers.c | 1 - server/core/buffer.c | 1 - server/core/config.c | 2 +- server/core/dbusers.c | 1 - server/core/dcb.c | 1 - server/core/filter.c | 1 - server/core/gateway.c | 2 +- server/core/gw_utils.c | 2 - server/core/load_utils.c | 1 - server/core/log_manager.cc | 1 - server/core/maxkeys.c | 1 - server/core/maxpasswd.c | 1 - server/core/maxscale/mlist.h | 2 +- .../core}/maxscale/skygw_utils.h | 16 - server/core/modutil.c | 1 - server/core/monitor.c | 1 - server/core/poll.c | 1 - server/core/query_classifier.c | 2 +- server/core/secrets.c | 1 - server/core/server.c | 1 - server/core/service.c | 1 - server/core/session.c | 1 - server/core/skygw_utils.cc | 485 +---------------- server/core/test/test_mysql_users.c | 1 - server/core/test/testadminusers.c | 2 +- server/core/test/testlog.c | 2 +- server/core/test/testlogorder.c | 2 +- server/core/utils.c | 494 +++++++++++++++++- server/modules/filter/ccrfilter/ccrfilter.c | 1 - server/modules/filter/hintfilter/hintparser.c | 1 - .../namedserverfilter/namedserverfilter.c | 1 - server/modules/filter/qlafilter/qlafilter.c | 2 +- .../modules/filter/regexfilter/regexfilter.c | 1 - server/modules/filter/tee/tee.c | 1 - server/modules/filter/topfilter/topfilter.c | 1 - server/modules/filter/tpmfilter/tpmfilter.c | 1 - server/modules/monitor/galeramon/galeramon.h | 1 - server/modules/monitor/mmmon/mmmon.h | 1 - server/modules/monitor/mysqlmon.h | 1 - .../monitor/ndbclustermon/ndbclustermon.h | 1 - .../MySQL/MySQLBackend/mysql_backend.c | 1 - .../protocol/MySQL/MySQLClient/mysql_client.c | 1 - server/modules/protocol/MySQL/mysql_common.c | 1 - server/modules/protocol/maxscaled/maxscaled.c | 1 - server/modules/protocol/telnetd/telnetd.c | 1 - server/modules/routing/avro/avro.c | 1 - server/modules/routing/avro/avro_client.c | 1 - server/modules/routing/avro/avro_file.c | 1 - server/modules/routing/avro/avro_schema.c | 1 - server/modules/routing/binlog/blr.c | 1 - server/modules/routing/binlog/blr_cache.c | 1 - server/modules/routing/binlog/blr_file.c | 1 - server/modules/routing/binlog/blr_master.c | 1 - server/modules/routing/binlog/blr_slave.c | 1 - .../modules/routing/binlog/test/testbinlog.c | 1 - server/modules/routing/cli/cli.c | 1 - server/modules/routing/debugcli/debugcli.c | 1 - server/modules/routing/debugcli/debugcmd.c | 1 - server/modules/routing/maxinfo/maxinfo.c | 1 - .../modules/routing/maxinfo/maxinfo_error.c | 1 - server/modules/routing/maxinfo/maxinfo_exec.c | 1 - .../modules/routing/maxinfo/maxinfo_parse.c | 1 - .../routing/readconnroute/readconnroute.c | 1 - .../routing/readwritesplit/readwritesplit.c | 1 - .../routing/readwritesplit/rwsplit_mysql.c | 1 - .../routing/schemarouter/schemarouter.c | 1 - 69 files changed, 521 insertions(+), 567 deletions(-) rename {include => server/core}/maxscale/skygw_utils.h (88%) diff --git a/include/maxscale/utils.h b/include/maxscale/utils.h index bca259700..1f608296d 100644 --- a/include/maxscale/utils.h +++ b/include/maxscale/utils.h @@ -37,6 +37,9 @@ MXS_BEGIN_DECLS #define MXS_ARRAY_NELEMS(array) ((size_t)(sizeof(array)/sizeof(array[0]))) +bool utils_init(); /*< Call this first before using any other function */ +void utils_end(); + int setnonblocking(int fd); char *gw_strend(register const char *s); static char gw_randomchar(); @@ -49,6 +52,21 @@ void gw_sha1_2_str(const uint8_t *in, int in_len, const uint8_t *in2, int in2_le int gw_getsockerrno(int fd); char *create_hex_sha1_sha1_passwd(char *passwd); +char* trim(char *str); +char* squeeze_whitespace(char* str); +bool strip_escape_chars(char*); + +bool is_valid_posix_path(char* path); + +char* remove_mysql_comments(const char** src, const size_t* srcsize, char** dest, + size_t* destsize); +char* replace_values(const char** src, const size_t* srcsize, char** dest, + size_t* destsize); +char* replace_literal(char* haystack, + const char* needle, + const char* replacement); +char* replace_quoted(const char** src, const size_t* srcsize, char** dest, size_t* destsize); + MXS_END_DECLS #endif diff --git a/query_classifier/qc_sqlite/qc_sqlite.c b/query_classifier/qc_sqlite/qc_sqlite.c index a5fbebd27..598bc3c2c 100644 --- a/query_classifier/qc_sqlite/qc_sqlite.c +++ b/query_classifier/qc_sqlite/qc_sqlite.c @@ -20,7 +20,6 @@ #include #include #include -#include #include #include #include "builtin_functions.h" diff --git a/query_classifier/test/canonical_tests/canonizer.c b/query_classifier/test/canonical_tests/canonizer.c index fc95105e2..56c59f1ea 100644 --- a/query_classifier/test/canonical_tests/canonizer.c +++ b/query_classifier/test/canonical_tests/canonizer.c @@ -18,7 +18,7 @@ #include #include #include -#include +#include int main(int argc, char** argv) { diff --git a/server/core/adminusers.c b/server/core/adminusers.c index 6b0f1693c..f0d1a1946 100644 --- a/server/core/adminusers.c +++ b/server/core/adminusers.c @@ -21,7 +21,6 @@ #include #include #include -#include #include #include #include diff --git a/server/core/buffer.c b/server/core/buffer.c index cf25bdb08..e91e79b58 100644 --- a/server/core/buffer.c +++ b/server/core/buffer.c @@ -43,7 +43,6 @@ #include #include #include -#include #include #include #include diff --git a/server/core/config.c b/server/core/config.c index 7cda0ed0a..4b9bbab2f 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -58,7 +58,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/server/core/dbusers.c b/server/core/dbusers.c index cd6491f77..4098474e2 100644 --- a/server/core/dbusers.c +++ b/server/core/dbusers.c @@ -43,7 +43,6 @@ #include #include #include -#include #include #include #include diff --git a/server/core/dcb.c b/server/core/dcb.c index acf811c7f..065213cec 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -81,7 +81,6 @@ #include #include #include -#include #include #include #include diff --git a/server/core/filter.c b/server/core/filter.c index 45fa3721a..c3f8812b1 100644 --- a/server/core/filter.c +++ b/server/core/filter.c @@ -30,7 +30,6 @@ #include #include #include -#include #include #include diff --git a/server/core/gateway.c b/server/core/gateway.c index 0320ebf10..6435e4b6d 100644 --- a/server/core/gateway.c +++ b/server/core/gateway.c @@ -74,7 +74,7 @@ #include #include -#include +#include #include #include diff --git a/server/core/gw_utils.c b/server/core/gw_utils.c index 78729e46a..2ef05638d 100644 --- a/server/core/gw_utils.c +++ b/server/core/gw_utils.c @@ -37,8 +37,6 @@ #include #include #include - -#include #include SPINLOCK tmplock = SPINLOCK_INIT; diff --git a/server/core/load_utils.c b/server/core/load_utils.c index ad83cfccf..c995ef085 100644 --- a/server/core/load_utils.c +++ b/server/core/load_utils.c @@ -36,7 +36,6 @@ #include #include #include -#include #include #include #include diff --git a/server/core/log_manager.cc b/server/core/log_manager.cc index 996e5e001..80263a5ef 100644 --- a/server/core/log_manager.cc +++ b/server/core/log_manager.cc @@ -28,7 +28,6 @@ #include #include #include -#include #include #include #include "maxscale/mlist.h" diff --git a/server/core/maxkeys.c b/server/core/maxkeys.c index 380338329..86c91d642 100644 --- a/server/core/maxkeys.c +++ b/server/core/maxkeys.c @@ -25,7 +25,6 @@ #include #include #include -#include #include #include diff --git a/server/core/maxpasswd.c b/server/core/maxpasswd.c index 4a7c77a2a..6fc98198d 100644 --- a/server/core/maxpasswd.c +++ b/server/core/maxpasswd.c @@ -26,7 +26,6 @@ #include #include #include -#include struct option options[] = { diff --git a/server/core/maxscale/mlist.h b/server/core/maxscale/mlist.h index c3ccc1695..be5a4fd55 100644 --- a/server/core/maxscale/mlist.h +++ b/server/core/maxscale/mlist.h @@ -15,7 +15,7 @@ */ #include -#include +#include "skygw_utils.h" MXS_BEGIN_DECLS diff --git a/include/maxscale/skygw_utils.h b/server/core/maxscale/skygw_utils.h similarity index 88% rename from include/maxscale/skygw_utils.h rename to server/core/maxscale/skygw_utils.h index e21a41afa..cbfad8a89 100644 --- a/include/maxscale/skygw_utils.h +++ b/server/core/maxscale/skygw_utils.h @@ -96,9 +96,6 @@ struct skygw_file_st skygw_chk_t sf_chk_tail; }; -bool utils_init(); /*< Call this first before using any other function */ -void utils_end(); - /** Skygw thread routines */ skygw_thread_t* skygw_thread_init(const char* name, void* (*sth_thrfun)(void* data), @@ -167,19 +164,6 @@ int skygw_rwlock_init(skygw_rwlock_t** rwlock); size_t get_decimal_len(size_t s); -char* remove_mysql_comments(const char** src, const size_t* srcsize, char** dest, - size_t* destsize); -char* replace_values(const char** src, const size_t* srcsize, char** dest, - size_t* destsize); -char* replace_literal(char* haystack, - const char* needle, - const char* replacement); -char* replace_quoted(const char** src, const size_t* srcsize, char** dest, size_t* destsize); -bool is_valid_posix_path(char* path); -bool strip_escape_chars(char*); -char* trim(char *str); -char* squeeze_whitespace(char* str); - MXS_END_DECLS #endif /* SKYGW_UTILS_H */ diff --git a/server/core/modutil.c b/server/core/modutil.c index 8f673d1a4..4e25999a6 100644 --- a/server/core/modutil.c +++ b/server/core/modutil.c @@ -30,7 +30,6 @@ #include #include #include -#include #include /** These are used when converting MySQL wildcards to regular expressions */ diff --git a/server/core/monitor.c b/server/core/monitor.c index 20f29323e..60bc4e238 100644 --- a/server/core/monitor.c +++ b/server/core/monitor.c @@ -33,7 +33,6 @@ #include #include #include -#include #include #include #include diff --git a/server/core/poll.c b/server/core/poll.c index 683b8cbf3..149b67fdb 100644 --- a/server/core/poll.c +++ b/server/core/poll.c @@ -24,7 +24,6 @@ #include #include #include -#include #include #include #include diff --git a/server/core/query_classifier.c b/server/core/query_classifier.c index 7a0ab82cb..071ddf0e9 100644 --- a/server/core/query_classifier.c +++ b/server/core/query_classifier.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include //#define QC_TRACE_ENABLED #undef QC_TRACE_ENABLED diff --git a/server/core/secrets.c b/server/core/secrets.c index 21e7318d1..30570044d 100644 --- a/server/core/secrets.c +++ b/server/core/secrets.c @@ -13,7 +13,6 @@ #include #include -#include #include #include #include diff --git a/server/core/server.c b/server/core/server.c index db3c49f94..101b5b5f0 100644 --- a/server/core/server.c +++ b/server/core/server.c @@ -40,7 +40,6 @@ #include #include #include -#include #include #include #include diff --git a/server/core/service.c b/server/core/service.c index cc908b90f..643011a31 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -55,7 +55,6 @@ #include #include #include -#include #include #include #include diff --git a/server/core/session.c b/server/core/session.c index f6ce149f6..a449c6998 100644 --- a/server/core/session.c +++ b/server/core/session.c @@ -40,7 +40,6 @@ #include #include #include -#include #include #include diff --git a/server/core/skygw_utils.cc b/server/core/skygw_utils.cc index 8b1142edd..fa0afb6fb 100644 --- a/server/core/skygw_utils.cc +++ b/server/core/skygw_utils.cc @@ -27,7 +27,7 @@ #include #include #include -#include +#include "maxscale/skygw_utils.h" #include #include #include @@ -40,8 +40,6 @@ # endif #endif -#define MAX_ERROR_MSG PATH_MAX - static void simple_mutex_free_memory(simple_mutex_t* sm); static void thread_free_memory(skygw_thread_t* th, char* name); /** End of static function declarations */ @@ -1006,298 +1004,6 @@ void skygw_file_close(skygw_file_t* file) } } -#define BUFFER_GROWTH_RATE 1.2 -static pcre2_code* remove_comments_re = NULL; -static const PCRE2_SPTR remove_comments_pattern = (PCRE2_SPTR) - "(?:`[^`]*`\\K)|(\\/[*](?!(M?!)).*?[*]\\/)|(?:#.*|--[[:space:]].*)"; - -/** - * Remove SQL comments from the end of a string - * - * The inline executable comments are not removed due to the fact that they can - * alter the behavior of the query. - * @param src Pointer to the string to modify. - * @param srcsize Pointer to a size_t variable which holds the length of the string to - * be modified. - * @param dest The address of the pointer where the result will be stored. If the - * value pointed by this parameter is NULL, new memory will be allocated as needed. - * @param Pointer to a size_t variable where the size of the result string is stored. - * @return Pointer to new modified string or NULL if memory allocation failed. - * If NULL is returned and the value pointed by @c dest was not NULL, no new - * memory will be allocated, the memory pointed by @dest will be freed and the - * contents of @c dest and @c destsize will be invalid. - */ -char* remove_mysql_comments(const char** src, const size_t* srcsize, char** dest, size_t* destsize) -{ - static const PCRE2_SPTR replace = (PCRE2_SPTR) ""; - pcre2_match_data* mdata; - char* output = *dest; - size_t orig_len = *srcsize; - size_t len = output ? *destsize : orig_len; - - if (orig_len > 0) - { - if ((output || (output = (char*) malloc(len * sizeof (char)))) && - (mdata = pcre2_match_data_create_from_pattern(remove_comments_re, NULL))) - { - while (pcre2_substitute(remove_comments_re, (PCRE2_SPTR) * src, orig_len, 0, - PCRE2_SUBSTITUTE_GLOBAL, mdata, NULL, - replace, PCRE2_ZERO_TERMINATED, - (PCRE2_UCHAR8*) output, &len) == PCRE2_ERROR_NOMEMORY) - { - char* tmp = (char*) realloc(output, (len = (size_t) (len * BUFFER_GROWTH_RATE + 1))); - if (tmp == NULL) - { - free(output); - output = NULL; - break; - } - output = tmp; - } - pcre2_match_data_free(mdata); - } - else - { - free(output); - output = NULL; - } - } - else if (output == NULL) - { - output = strdup(*src); - } - - if (output) - { - *destsize = strlen(output); - *dest = output; - } - - return output; -} - -static pcre2_code* replace_values_re = NULL; -static const PCRE2_SPTR replace_values_pattern = (PCRE2_SPTR) "(?i)([-=,+*/([:space:]]|\\b|[@])" - "(?:[0-9.-]+|(?<=[@])[a-z_0-9]+)([-=,+*/)[:space:];]|$)"; - -/** - * Replace literal numbers and user variables with a question mark. - * @param src Pointer to the string to modify. - * @param srcsize Pointer to a size_t variable which holds the length of the string to - * be modified. - * @param dest The address of the pointer where the result will be stored. If the - * value pointed by this parameter is NULL, new memory will be allocated as needed. - * @param Pointer to a size_t variable where the size of the result string is stored. - * @return Pointer to new modified string or NULL if memory allocation failed. - * If NULL is returned and the value pointed by @c dest was not NULL, no new - * memory will be allocated, the memory pointed by @dest will be freed and the - * contents of @c dest and @c destsize will be invalid. - */ -char* replace_values(const char** src, const size_t* srcsize, char** dest, size_t* destsize) -{ - static const PCRE2_SPTR replace = (PCRE2_SPTR) "$1?$2"; - pcre2_match_data* mdata; - char* output = *dest; - size_t orig_len = *srcsize; - size_t len = output ? *destsize : orig_len; - - if (orig_len > 0) - { - if ((output || (output = (char*) malloc(len * sizeof (char)))) && - (mdata = pcre2_match_data_create_from_pattern(replace_values_re, NULL))) - { - while (pcre2_substitute(replace_values_re, (PCRE2_SPTR) * src, orig_len, 0, - PCRE2_SUBSTITUTE_GLOBAL, mdata, NULL, - replace, PCRE2_ZERO_TERMINATED, - (PCRE2_UCHAR8*) output, &len) == PCRE2_ERROR_NOMEMORY) - { - char* tmp = (char*) realloc(output, (len = (size_t) (len * BUFFER_GROWTH_RATE + 1))); - if (tmp == NULL) - { - free(output); - output = NULL; - break; - } - output = tmp; - } - pcre2_match_data_free(mdata); - } - else - { - free(output); - output = NULL; - } - } - else if (output == NULL) - { - output = strdup(*src); - } - - if (output) - { - *destsize = strlen(output); - *dest = output; - } - - return output; -} - -/** - * Find the given needle - user-provided literal - and replace it with - * replacement string. Separate user-provided literals from matching table names - * etc. by searching only substrings preceded by non-letter and non-number. - * - * @param haystack Plain text query string, not to be freed - * @param needle Substring to be searched, not to be freed - * @param replacement Replacement text, not to be freed - * - * @return newly allocated string where needle is replaced - */ -char* replace_literal(char* haystack, const char* needle, const char* replacement) -{ - const char* prefix = "[ ='\",\\(]"; /*< ' ','=','(',''',''"',',' are allowed before needle */ - const char* suffix = "([^[:alnum:]]|$)"; /*< alpha-num chars aren't allowed after the needle */ - char* search_re; - char* newstr; - regex_t re; - regmatch_t match; - int rc; - size_t rlen = strlen(replacement); - size_t nlen = strlen(needle); - size_t hlen = strlen(haystack); - - search_re = (char *) malloc(strlen(prefix) + nlen + strlen(suffix) + 1); - - if (search_re == NULL) - { - char errbuf[MXS_STRERROR_BUFLEN]; - fprintf(stderr, "Regex memory allocation failed : %s\n", - strerror_r(errno, errbuf, sizeof (errbuf))); - newstr = haystack; - goto retblock; - } - - sprintf(search_re, "%s%s%s", prefix, needle, suffix); - /** Allocate memory for new string +1 for terminating byte */ - newstr = (char *) malloc(hlen - nlen + rlen + 1); - - if (newstr == NULL) - { - char errbuf[MXS_STRERROR_BUFLEN]; - fprintf(stderr, "Regex memory allocation failed : %s\n", - strerror_r(errno, errbuf, sizeof (errbuf))); - free(search_re); - free(newstr); - newstr = haystack; - goto retblock; - } - - rc = regcomp(&re, search_re, REG_EXTENDED | REG_ICASE); - ss_info_dassert(rc == 0, "Regex check"); - - if (rc != 0) - { - char error_message[MAX_ERROR_MSG]; - regerror(rc, &re, error_message, MAX_ERROR_MSG); - fprintf(stderr, "Regex error compiling '%s': %s\n", - search_re, error_message); - free(search_re); - free(newstr); - newstr = haystack; - goto retblock; - } - rc = regexec(&re, haystack, 1, &match, 0); - - if (rc != 0) - { - free(search_re); - free(newstr); - regfree(&re); - newstr = haystack; - goto retblock; - } - memcpy(newstr, haystack, match.rm_so + 1); - memcpy(newstr + match.rm_so + 1, replacement, rlen); - /** +1 is terminating byte */ - memcpy(newstr + match.rm_so + 1 + rlen, haystack + match.rm_so + 1 + nlen, hlen - (match.rm_so + 1) - nlen + 1); - - regfree(&re); - free(haystack); - free(search_re); -retblock: - return newstr; -} - -static pcre2_code* replace_quoted_re = NULL; -static const PCRE2_SPTR replace_quoted_pattern = (PCRE2_SPTR) - "(?>[^'\"]*)(?|(?:\"\\K(?:(?:(?<=\\\\)\")|[^\"])*(\"))|(?:'\\K(?:(?:(?<=\\\\)')|[^'])*(')))"; - -/** - * Replace contents of single or double quoted strings with question marks. - * @param src Pointer to the string to modify. - * @param srcsize Pointer to a size_t variable which holds the length of the string to - * be modified. - * @param dest The address of the pointer where the result will be stored. If the - * value pointed by this parameter is NULL, new memory will be allocated as needed. - * @param Pointer to a size_t variable where the size of the result string is stored. - * @return Pointer to new modified string or NULL if memory allocation failed. - * If NULL is returned and the value pointed by @c dest was not NULL, no new - * memory will be allocated, the memory pointed by @dest will be freed and the - * contents of @c dest and @c destsize will be invalid. - */ -char* replace_quoted(const char** src, const size_t* srcsize, char** dest, size_t* destsize) -{ - static const PCRE2_SPTR replace = (PCRE2_SPTR) "?$1"; - pcre2_match_data* mdata; - char* output = *dest; - size_t orig_len = *srcsize; - size_t len = output ? *destsize : orig_len; - - if (orig_len > 0) - { - if ((output || (output = (char*) malloc(len * sizeof (char)))) && - (mdata = pcre2_match_data_create_from_pattern(replace_quoted_re, NULL))) - { - while (pcre2_substitute(replace_quoted_re, (PCRE2_SPTR) * src, orig_len, 0, - PCRE2_SUBSTITUTE_GLOBAL, mdata, NULL, - replace, PCRE2_ZERO_TERMINATED, - (PCRE2_UCHAR8*) output, &len) == PCRE2_ERROR_NOMEMORY) - { - char* tmp = (char*) realloc(output, (len = (size_t) (len * BUFFER_GROWTH_RATE + 1))); - if (tmp == NULL) - { - free(output); - output = NULL; - break; - } - output = tmp; - } - pcre2_match_data_free(mdata); - } - else - { - free(output); - output = NULL; - } - } - else if (output == NULL) - { - output = strdup(*src); - } - - if (output) - { - *destsize = strlen(output); - *dest = output; - } - else - { - *dest = NULL; - } - - return output; -} - /** * Calculate the number of decimal numbers from a size_t value. * @@ -1312,192 +1018,3 @@ size_t get_decimal_len( { return value > 0 ? (size_t) log10((double) value) + 1 : 1; } - -/** - * Check if the provided pathname is POSIX-compliant. The valid characters - * are [a-z A-Z 0-9._-]. - * @param path A null-terminated string - * @return true if it is a POSIX-compliant pathname, otherwise false - */ -bool is_valid_posix_path(char* path) -{ - char* ptr = path; - while (*ptr != '\0') - { - if (isalnum(*ptr) || *ptr == '/' || *ptr == '.' || *ptr == '-' || *ptr == '_') - { - ptr++; - } - else - { - return false; - } - } - return true; -} - -/** - * Strip escape characters from a character string. - * @param String to parse. - * @return True if parsing was successful, false on errors. - */ -bool -strip_escape_chars(char* val) -{ - int cur, end; - - if (val == NULL) - { - return false; - } - - end = strlen(val) + 1; - cur = 0; - - while (cur < end) - { - if (val[cur] == '\\') - { - memmove(val + cur, val + cur + 1, end - cur - 1); - end--; - } - cur++; - } - return true; -} - -/** - * Trim leading and trailing whitespace from a string - * - * @param str String to trim - * @return Trimmed string - */ -char* trim(char *str) -{ - char* ptr = strchr(str, '\0') - 1; - - while (ptr > str && isspace(*ptr)) - { - ptr--; - } - - if (isspace(*(ptr + 1))) - { - *(ptr + 1) = '\0'; - } - - ptr = str; - - while (isspace(*ptr)) - { - ptr++; - } - - if (ptr != str) - { - memmove(str, ptr, strlen(ptr) + 1); - } - - return str; -} - -/** - * Replace all whitespace with spaces and squeeze repeating whitespace characters - * - * @param str String to squeeze - * @return Squeezed string - */ -char* squeeze_whitespace(char* str) -{ - char* store = str; - char* ptr = str; - - /** Remove leading whitespace */ - while (isspace(*ptr) && *ptr != '\0') - { - ptr++; - } - - /** Squeeze all repeating whitespace */ - while (*ptr != '\0') - { - while (isspace(*ptr) && isspace(*(ptr + 1))) - { - ptr++; - } - - if (isspace(*ptr)) - { - *store++ = ' '; - ptr++; - } - else - { - *store++ = *ptr++; - } - } - - *store = '\0'; - - /** Remove trailing whitespace */ - while (store > str && isspace(*(store - 1))) - { - store--; - *store = '\0'; - } - - return str; -} - -/** - * Initialize the utils library - * - * This function initializes structures used in various functions. - * @return true on success, false on error - */ -bool utils_init() -{ - bool rval = true; - - PCRE2_SIZE erroffset; - int errcode; - - ss_info_dassert(remove_comments_re == NULL, "utils_init called multiple times"); - remove_comments_re = pcre2_compile(remove_comments_pattern, PCRE2_ZERO_TERMINATED, 0, &errcode, - &erroffset, NULL); - if (remove_comments_re == NULL) - { - rval = false; - } - - ss_info_dassert(replace_quoted_re == NULL, "utils_init called multiple times"); - replace_quoted_re = pcre2_compile(replace_quoted_pattern, PCRE2_ZERO_TERMINATED, 0, &errcode, - &erroffset, NULL); - if (replace_quoted_re == NULL) - { - rval = false; - } - - ss_info_dassert(replace_values_re == NULL, "utils_init called multiple times"); - replace_values_re = pcre2_compile(replace_values_pattern, PCRE2_ZERO_TERMINATED, 0, &errcode, - &erroffset, NULL); - if (replace_values_re == NULL) - { - rval = false; - } - - return rval; -} - -/** - * Close the utils library. This should be the last call to this library. - */ -void utils_end() -{ - pcre2_code_free(remove_comments_re); - remove_comments_re = NULL; - pcre2_code_free(replace_quoted_re); - replace_quoted_re = NULL; - pcre2_code_free(replace_values_re); - replace_values_re = NULL; -} diff --git a/server/core/test/test_mysql_users.c b/server/core/test/test_mysql_users.c index c25b16205..49f980f53 100644 --- a/server/core/test/test_mysql_users.c +++ b/server/core/test/test_mysql_users.c @@ -31,7 +31,6 @@ #include #include #include -#include #include #include #include diff --git a/server/core/test/testadminusers.c b/server/core/test/testadminusers.c index 1164f6f0c..6fe85266e 100644 --- a/server/core/test/testadminusers.c +++ b/server/core/test/testadminusers.c @@ -37,7 +37,7 @@ #include #include #include -#include +#include /** * test1 default user diff --git a/server/core/test/testlog.c b/server/core/test/testlog.c index 2f48b71aa..cf409f0b4 100644 --- a/server/core/test/testlog.c +++ b/server/core/test/testlog.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include "../maxscale/skygw_utils.h" #include static void skygw_log_enable(int priority) diff --git a/server/core/test/testlogorder.c b/server/core/test/testlogorder.c index 42b9f78cb..c50e2ddbb 100644 --- a/server/core/test/testlogorder.c +++ b/server/core/test/testlogorder.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include static void skygw_log_enable(int priority) diff --git a/server/core/utils.c b/server/core/utils.c index e829b871a..ead727b8e 100644 --- a/server/core/utils.c +++ b/server/core/utils.c @@ -29,16 +29,27 @@ */ +#include #include #include #include #include #include #include -#include #include #include #include +#include + +#if !defined(PATH_MAX) +# if defined(__USE_POSIX) +# define PATH_MAX _POSIX_PATH_MAX +# else +# define PATH_MAX 256 +# endif +#endif + +#define MAX_ERROR_MSG PATH_MAX /* used in the hex2bin function */ #define char_val(X) (X >= '0' && X <= '9' ? X-'0' : \ @@ -50,6 +61,29 @@ char hex_upper[] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; char hex_lower[] = "0123456789abcdefghijklmnopqrstuvwxyz"; +/** + * Check if the provided pathname is POSIX-compliant. The valid characters + * are [a-z A-Z 0-9._-]. + * @param path A null-terminated string + * @return true if it is a POSIX-compliant pathname, otherwise false + */ +bool is_valid_posix_path(char* path) +{ + char* ptr = path; + while (*ptr != '\0') + { + if (isalnum(*ptr) || *ptr == '/' || *ptr == '.' || *ptr == '-' || *ptr == '_') + { + ptr++; + } + else + { + return false; + } + } + return true; +} + /***************************************** * backend read event triggered by EPOLLIN *****************************************/ @@ -388,3 +422,461 @@ bool mxs_mkdir_all(const char *path, int mask) return mkdir_all_internal(local_path, (mode_t)mask); } + +/** + * Trim leading and trailing whitespace from a string + * + * @param str String to trim + * @return Trimmed string + */ +char* trim(char *str) +{ + char* ptr = strchr(str, '\0') - 1; + + while (ptr > str && isspace(*ptr)) + { + ptr--; + } + + if (isspace(*(ptr + 1))) + { + *(ptr + 1) = '\0'; + } + + ptr = str; + + while (isspace(*ptr)) + { + ptr++; + } + + if (ptr != str) + { + memmove(str, ptr, strlen(ptr) + 1); + } + + return str; +} + +/** + * Replace all whitespace with spaces and squeeze repeating whitespace characters + * + * @param str String to squeeze + * @return Squeezed string + */ +char* squeeze_whitespace(char* str) +{ + char* store = str; + char* ptr = str; + + /** Remove leading whitespace */ + while (isspace(*ptr) && *ptr != '\0') + { + ptr++; + } + + /** Squeeze all repeating whitespace */ + while (*ptr != '\0') + { + while (isspace(*ptr) && isspace(*(ptr + 1))) + { + ptr++; + } + + if (isspace(*ptr)) + { + *store++ = ' '; + ptr++; + } + else + { + *store++ = *ptr++; + } + } + + *store = '\0'; + + /** Remove trailing whitespace */ + while (store > str && isspace(*(store - 1))) + { + store--; + *store = '\0'; + } + + return str; +} + +/** + * Strip escape characters from a character string. + * @param String to parse. + * @return True if parsing was successful, false on errors. + */ +bool +strip_escape_chars(char* val) +{ + int cur, end; + + if (val == NULL) + { + return false; + } + + end = strlen(val) + 1; + cur = 0; + + while (cur < end) + { + if (val[cur] == '\\') + { + memmove(val + cur, val + cur + 1, end - cur - 1); + end--; + } + cur++; + } + return true; +} + +#define BUFFER_GROWTH_RATE 1.2 +static pcre2_code* remove_comments_re = NULL; +static const PCRE2_SPTR remove_comments_pattern = (PCRE2_SPTR) + "(?:`[^`]*`\\K)|(\\/[*](?!(M?!)).*?[*]\\/)|(?:#.*|--[[:space:]].*)"; + +/** + * Remove SQL comments from the end of a string + * + * The inline executable comments are not removed due to the fact that they can + * alter the behavior of the query. + * @param src Pointer to the string to modify. + * @param srcsize Pointer to a size_t variable which holds the length of the string to + * be modified. + * @param dest The address of the pointer where the result will be stored. If the + * value pointed by this parameter is NULL, new memory will be allocated as needed. + * @param Pointer to a size_t variable where the size of the result string is stored. + * @return Pointer to new modified string or NULL if memory allocation failed. + * If NULL is returned and the value pointed by @c dest was not NULL, no new + * memory will be allocated, the memory pointed by @dest will be freed and the + * contents of @c dest and @c destsize will be invalid. + */ +char* remove_mysql_comments(const char** src, const size_t* srcsize, char** dest, size_t* destsize) +{ + static const PCRE2_SPTR replace = (PCRE2_SPTR) ""; + pcre2_match_data* mdata; + char* output = *dest; + size_t orig_len = *srcsize; + size_t len = output ? *destsize : orig_len; + + if (orig_len > 0) + { + if ((output || (output = (char*) malloc(len * sizeof (char)))) && + (mdata = pcre2_match_data_create_from_pattern(remove_comments_re, NULL))) + { + while (pcre2_substitute(remove_comments_re, (PCRE2_SPTR) * src, orig_len, 0, + PCRE2_SUBSTITUTE_GLOBAL, mdata, NULL, + replace, PCRE2_ZERO_TERMINATED, + (PCRE2_UCHAR8*) output, &len) == PCRE2_ERROR_NOMEMORY) + { + char* tmp = (char*) realloc(output, (len = (size_t) (len * BUFFER_GROWTH_RATE + 1))); + if (tmp == NULL) + { + free(output); + output = NULL; + break; + } + output = tmp; + } + pcre2_match_data_free(mdata); + } + else + { + free(output); + output = NULL; + } + } + else if (output == NULL) + { + output = strdup(*src); + } + + if (output) + { + *destsize = strlen(output); + *dest = output; + } + + return output; +} + +static pcre2_code* replace_values_re = NULL; +static const PCRE2_SPTR replace_values_pattern = (PCRE2_SPTR) "(?i)([-=,+*/([:space:]]|\\b|[@])" + "(?:[0-9.-]+|(?<=[@])[a-z_0-9]+)([-=,+*/)[:space:];]|$)"; + +/** + * Replace literal numbers and user variables with a question mark. + * @param src Pointer to the string to modify. + * @param srcsize Pointer to a size_t variable which holds the length of the string to + * be modified. + * @param dest The address of the pointer where the result will be stored. If the + * value pointed by this parameter is NULL, new memory will be allocated as needed. + * @param Pointer to a size_t variable where the size of the result string is stored. + * @return Pointer to new modified string or NULL if memory allocation failed. + * If NULL is returned and the value pointed by @c dest was not NULL, no new + * memory will be allocated, the memory pointed by @dest will be freed and the + * contents of @c dest and @c destsize will be invalid. + */ +char* replace_values(const char** src, const size_t* srcsize, char** dest, size_t* destsize) +{ + static const PCRE2_SPTR replace = (PCRE2_SPTR) "$1?$2"; + pcre2_match_data* mdata; + char* output = *dest; + size_t orig_len = *srcsize; + size_t len = output ? *destsize : orig_len; + + if (orig_len > 0) + { + if ((output || (output = (char*) malloc(len * sizeof (char)))) && + (mdata = pcre2_match_data_create_from_pattern(replace_values_re, NULL))) + { + while (pcre2_substitute(replace_values_re, (PCRE2_SPTR) * src, orig_len, 0, + PCRE2_SUBSTITUTE_GLOBAL, mdata, NULL, + replace, PCRE2_ZERO_TERMINATED, + (PCRE2_UCHAR8*) output, &len) == PCRE2_ERROR_NOMEMORY) + { + char* tmp = (char*) realloc(output, (len = (size_t) (len * BUFFER_GROWTH_RATE + 1))); + if (tmp == NULL) + { + free(output); + output = NULL; + break; + } + output = tmp; + } + pcre2_match_data_free(mdata); + } + else + { + free(output); + output = NULL; + } + } + else if (output == NULL) + { + output = strdup(*src); + } + + if (output) + { + *destsize = strlen(output); + *dest = output; + } + + return output; +} + +/** + * Find the given needle - user-provided literal - and replace it with + * replacement string. Separate user-provided literals from matching table names + * etc. by searching only substrings preceded by non-letter and non-number. + * + * @param haystack Plain text query string, not to be freed + * @param needle Substring to be searched, not to be freed + * @param replacement Replacement text, not to be freed + * + * @return newly allocated string where needle is replaced + */ +char* replace_literal(char* haystack, const char* needle, const char* replacement) +{ + const char* prefix = "[ ='\",\\(]"; /*< ' ','=','(',''',''"',',' are allowed before needle */ + const char* suffix = "([^[:alnum:]]|$)"; /*< alpha-num chars aren't allowed after the needle */ + char* search_re; + char* newstr; + regex_t re; + regmatch_t match; + int rc; + size_t rlen = strlen(replacement); + size_t nlen = strlen(needle); + size_t hlen = strlen(haystack); + + search_re = (char *) malloc(strlen(prefix) + nlen + strlen(suffix) + 1); + + if (search_re == NULL) + { + char errbuf[MXS_STRERROR_BUFLEN]; + fprintf(stderr, "Regex memory allocation failed : %s\n", + strerror_r(errno, errbuf, sizeof (errbuf))); + newstr = haystack; + goto retblock; + } + + sprintf(search_re, "%s%s%s", prefix, needle, suffix); + /** Allocate memory for new string +1 for terminating byte */ + newstr = (char *) malloc(hlen - nlen + rlen + 1); + + if (newstr == NULL) + { + char errbuf[MXS_STRERROR_BUFLEN]; + fprintf(stderr, "Regex memory allocation failed : %s\n", + strerror_r(errno, errbuf, sizeof (errbuf))); + free(search_re); + free(newstr); + newstr = haystack; + goto retblock; + } + + rc = regcomp(&re, search_re, REG_EXTENDED | REG_ICASE); + ss_info_dassert(rc == 0, "Regex check"); + + if (rc != 0) + { + char error_message[MAX_ERROR_MSG]; + regerror(rc, &re, error_message, MAX_ERROR_MSG); + fprintf(stderr, "Regex error compiling '%s': %s\n", + search_re, error_message); + free(search_re); + free(newstr); + newstr = haystack; + goto retblock; + } + rc = regexec(&re, haystack, 1, &match, 0); + + if (rc != 0) + { + free(search_re); + free(newstr); + regfree(&re); + newstr = haystack; + goto retblock; + } + memcpy(newstr, haystack, match.rm_so + 1); + memcpy(newstr + match.rm_so + 1, replacement, rlen); + /** +1 is terminating byte */ + memcpy(newstr + match.rm_so + 1 + rlen, haystack + match.rm_so + 1 + nlen, hlen - (match.rm_so + 1) - nlen + 1); + + regfree(&re); + free(haystack); + free(search_re); +retblock: + return newstr; +} + +static pcre2_code* replace_quoted_re = NULL; +static const PCRE2_SPTR replace_quoted_pattern = (PCRE2_SPTR) + "(?>[^'\"]*)(?|(?:\"\\K(?:(?:(?<=\\\\)\")|[^\"])*(\"))|(?:'\\K(?:(?:(?<=\\\\)')|[^'])*(')))"; + +/** + * Replace contents of single or double quoted strings with question marks. + * @param src Pointer to the string to modify. + * @param srcsize Pointer to a size_t variable which holds the length of the string to + * be modified. + * @param dest The address of the pointer where the result will be stored. If the + * value pointed by this parameter is NULL, new memory will be allocated as needed. + * @param Pointer to a size_t variable where the size of the result string is stored. + * @return Pointer to new modified string or NULL if memory allocation failed. + * If NULL is returned and the value pointed by @c dest was not NULL, no new + * memory will be allocated, the memory pointed by @dest will be freed and the + * contents of @c dest and @c destsize will be invalid. + */ +char* replace_quoted(const char** src, const size_t* srcsize, char** dest, size_t* destsize) +{ + static const PCRE2_SPTR replace = (PCRE2_SPTR) "?$1"; + pcre2_match_data* mdata; + char* output = *dest; + size_t orig_len = *srcsize; + size_t len = output ? *destsize : orig_len; + + if (orig_len > 0) + { + if ((output || (output = (char*) malloc(len * sizeof (char)))) && + (mdata = pcre2_match_data_create_from_pattern(replace_quoted_re, NULL))) + { + while (pcre2_substitute(replace_quoted_re, (PCRE2_SPTR) * src, orig_len, 0, + PCRE2_SUBSTITUTE_GLOBAL, mdata, NULL, + replace, PCRE2_ZERO_TERMINATED, + (PCRE2_UCHAR8*) output, &len) == PCRE2_ERROR_NOMEMORY) + { + char* tmp = (char*) realloc(output, (len = (size_t) (len * BUFFER_GROWTH_RATE + 1))); + if (tmp == NULL) + { + free(output); + output = NULL; + break; + } + output = tmp; + } + pcre2_match_data_free(mdata); + } + else + { + free(output); + output = NULL; + } + } + else if (output == NULL) + { + output = strdup(*src); + } + + if (output) + { + *destsize = strlen(output); + *dest = output; + } + else + { + *dest = NULL; + } + + return output; +} + +/** + * Initialize the utils library + * + * This function initializes structures used in various functions. + * @return true on success, false on error + */ +bool utils_init() +{ + bool rval = true; + + PCRE2_SIZE erroffset; + int errcode; + + ss_info_dassert(remove_comments_re == NULL, "utils_init called multiple times"); + remove_comments_re = pcre2_compile(remove_comments_pattern, PCRE2_ZERO_TERMINATED, 0, &errcode, + &erroffset, NULL); + if (remove_comments_re == NULL) + { + rval = false; + } + + ss_info_dassert(replace_quoted_re == NULL, "utils_init called multiple times"); + replace_quoted_re = pcre2_compile(replace_quoted_pattern, PCRE2_ZERO_TERMINATED, 0, &errcode, + &erroffset, NULL); + if (replace_quoted_re == NULL) + { + rval = false; + } + + ss_info_dassert(replace_values_re == NULL, "utils_init called multiple times"); + replace_values_re = pcre2_compile(replace_values_pattern, PCRE2_ZERO_TERMINATED, 0, &errcode, + &erroffset, NULL); + if (replace_values_re == NULL) + { + rval = false; + } + + return rval; +} + +/** + * Close the utils library. This should be the last call to this library. + */ +void utils_end() +{ + pcre2_code_free(remove_comments_re); + remove_comments_re = NULL; + pcre2_code_free(replace_quoted_re); + replace_quoted_re = NULL; + pcre2_code_free(replace_values_re); + replace_values_re = NULL; +} diff --git a/server/modules/filter/ccrfilter/ccrfilter.c b/server/modules/filter/ccrfilter/ccrfilter.c index e334e9f02..f9fe4f796 100644 --- a/server/modules/filter/ccrfilter/ccrfilter.c +++ b/server/modules/filter/ccrfilter/ccrfilter.c @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/filter/hintfilter/hintparser.c b/server/modules/filter/hintfilter/hintparser.c index bfc721a3a..1f3dbe79f 100644 --- a/server/modules/filter/hintfilter/hintparser.c +++ b/server/modules/filter/hintfilter/hintparser.c @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/filter/namedserverfilter/namedserverfilter.c b/server/modules/filter/namedserverfilter/namedserverfilter.c index 0dcbd662b..c30315a7f 100644 --- a/server/modules/filter/namedserverfilter/namedserverfilter.c +++ b/server/modules/filter/namedserverfilter/namedserverfilter.c @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/filter/qlafilter/qlafilter.c b/server/modules/filter/qlafilter/qlafilter.c index f1d7a9fc0..ec0690479 100644 --- a/server/modules/filter/qlafilter/qlafilter.c +++ b/server/modules/filter/qlafilter/qlafilter.c @@ -40,7 +40,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/server/modules/filter/regexfilter/regexfilter.c b/server/modules/filter/regexfilter/regexfilter.c index df84853e2..3266f129c 100644 --- a/server/modules/filter/regexfilter/regexfilter.c +++ b/server/modules/filter/regexfilter/regexfilter.c @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/filter/tee/tee.c b/server/modules/filter/tee/tee.c index 6ac656f49..304b8ae3d 100644 --- a/server/modules/filter/tee/tee.c +++ b/server/modules/filter/tee/tee.c @@ -46,7 +46,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/filter/topfilter/topfilter.c b/server/modules/filter/topfilter/topfilter.c index 4596f381c..5acbed3e8 100644 --- a/server/modules/filter/topfilter/topfilter.c +++ b/server/modules/filter/topfilter/topfilter.c @@ -37,7 +37,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/filter/tpmfilter/tpmfilter.c b/server/modules/filter/tpmfilter/tpmfilter.c index d082c89f6..97fddf10d 100644 --- a/server/modules/filter/tpmfilter/tpmfilter.c +++ b/server/modules/filter/tpmfilter/tpmfilter.c @@ -45,7 +45,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/monitor/galeramon/galeramon.h b/server/modules/monitor/galeramon/galeramon.h index 42a1c5198..967a2e52c 100644 --- a/server/modules/monitor/galeramon/galeramon.h +++ b/server/modules/monitor/galeramon/galeramon.h @@ -35,7 +35,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/monitor/mmmon/mmmon.h b/server/modules/monitor/mmmon/mmmon.h index f63aa221a..7c3a6fd9c 100644 --- a/server/modules/monitor/mmmon/mmmon.h +++ b/server/modules/monitor/mmmon/mmmon.h @@ -23,7 +23,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/monitor/mysqlmon.h b/server/modules/monitor/mysqlmon.h index 3d56ad4f6..92fc3584b 100644 --- a/server/modules/monitor/mysqlmon.h +++ b/server/modules/monitor/mysqlmon.h @@ -43,7 +43,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/monitor/ndbclustermon/ndbclustermon.h b/server/modules/monitor/ndbclustermon/ndbclustermon.h index 2b98613c4..efe557f4b 100644 --- a/server/modules/monitor/ndbclustermon/ndbclustermon.h +++ b/server/modules/monitor/ndbclustermon/ndbclustermon.h @@ -21,7 +21,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c b/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c index be8a20909..f99dfb605 100644 --- a/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c +++ b/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c @@ -12,7 +12,6 @@ */ #include -#include #include #include #include diff --git a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c index d57cbe677..519da671b 100644 --- a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c +++ b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c @@ -46,7 +46,6 @@ * 31/05/2016 Martin Brampton Implement connection throttling */ #include -#include #include #include #include diff --git a/server/modules/protocol/MySQL/mysql_common.c b/server/modules/protocol/MySQL/mysql_common.c index 16ea18d81..7d6958571 100644 --- a/server/modules/protocol/MySQL/mysql_common.c +++ b/server/modules/protocol/MySQL/mysql_common.c @@ -46,7 +46,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/protocol/maxscaled/maxscaled.c b/server/modules/protocol/maxscaled/maxscaled.c index df8ef451c..d6758604c 100644 --- a/server/modules/protocol/maxscaled/maxscaled.c +++ b/server/modules/protocol/maxscaled/maxscaled.c @@ -30,7 +30,6 @@ #include #include #include -#include #include #include #include "maxscaled.h" diff --git a/server/modules/protocol/telnetd/telnetd.c b/server/modules/protocol/telnetd/telnetd.c index f0f3a4ddd..95a3f96ea 100644 --- a/server/modules/protocol/telnetd/telnetd.c +++ b/server/modules/protocol/telnetd/telnetd.c @@ -31,7 +31,6 @@ #include #include #include -#include #include #include diff --git a/server/modules/routing/avro/avro.c b/server/modules/routing/avro/avro.c index 794cb3d26..122e3fa38 100644 --- a/server/modules/routing/avro/avro.c +++ b/server/modules/routing/avro/avro.c @@ -39,7 +39,6 @@ #include #include -#include #include #include diff --git a/server/modules/routing/avro/avro_client.c b/server/modules/routing/avro/avro_client.c index c72c34bce..cc0293b70 100644 --- a/server/modules/routing/avro/avro_client.c +++ b/server/modules/routing/avro/avro_client.c @@ -35,7 +35,6 @@ #include #include #include -#include #include #include #include "avrorouter.h" diff --git a/server/modules/routing/avro/avro_file.c b/server/modules/routing/avro/avro_file.c index fcaab652e..23ad1d904 100644 --- a/server/modules/routing/avro/avro_file.c +++ b/server/modules/routing/avro/avro_file.c @@ -40,7 +40,6 @@ #include #include #include -#include static const char *statefile_section = "avro-conversion"; static const char *ddl_list_name = "table-ddl.list"; diff --git a/server/modules/routing/avro/avro_schema.c b/server/modules/routing/avro/avro_schema.c index f4172d689..48739e4b0 100644 --- a/server/modules/routing/avro/avro_schema.c +++ b/server/modules/routing/avro/avro_schema.c @@ -27,7 +27,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/routing/binlog/blr.c b/server/modules/routing/binlog/blr.c index a781cb9bb..6d542e9a1 100644 --- a/server/modules/routing/binlog/blr.c +++ b/server/modules/routing/binlog/blr.c @@ -71,7 +71,6 @@ #include #include -#include #include #include diff --git a/server/modules/routing/binlog/blr_cache.c b/server/modules/routing/binlog/blr_cache.c index 9501f0f92..613bb35e2 100644 --- a/server/modules/routing/binlog/blr_cache.c +++ b/server/modules/routing/binlog/blr_cache.c @@ -43,7 +43,6 @@ #include #include -#include #include diff --git a/server/modules/routing/binlog/blr_file.c b/server/modules/routing/binlog/blr_file.c index 0db05df59..10a37989d 100644 --- a/server/modules/routing/binlog/blr_file.c +++ b/server/modules/routing/binlog/blr_file.c @@ -56,7 +56,6 @@ #include #include #include -#include #include #include diff --git a/server/modules/routing/binlog/blr_master.c b/server/modules/routing/binlog/blr_master.c index 7b02a66e3..c452dcefc 100644 --- a/server/modules/routing/binlog/blr_master.c +++ b/server/modules/routing/binlog/blr_master.c @@ -69,7 +69,6 @@ #include #include -#include #include #include diff --git a/server/modules/routing/binlog/blr_slave.c b/server/modules/routing/binlog/blr_slave.c index 6d4904b18..b41583c24 100644 --- a/server/modules/routing/binlog/blr_slave.c +++ b/server/modules/routing/binlog/blr_slave.c @@ -79,7 +79,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/routing/binlog/test/testbinlog.c b/server/modules/routing/binlog/test/testbinlog.c index f55d038c9..6f4b14d58 100644 --- a/server/modules/routing/binlog/test/testbinlog.c +++ b/server/modules/routing/binlog/test/testbinlog.c @@ -36,7 +36,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/routing/cli/cli.c b/server/modules/routing/cli/cli.c index 948cebc6a..c59f570a4 100644 --- a/server/modules/routing/cli/cli.c +++ b/server/modules/routing/cli/cli.c @@ -38,7 +38,6 @@ #include #include #include -#include #include diff --git a/server/modules/routing/debugcli/debugcli.c b/server/modules/routing/debugcli/debugcli.c index c360f2da4..eb1bd290f 100644 --- a/server/modules/routing/debugcli/debugcli.c +++ b/server/modules/routing/debugcli/debugcli.c @@ -37,7 +37,6 @@ #include #include #include -#include #include diff --git a/server/modules/routing/debugcli/debugcmd.c b/server/modules/routing/debugcli/debugcmd.c index b5d3d1d4c..14d25249d 100644 --- a/server/modules/routing/debugcli/debugcmd.c +++ b/server/modules/routing/debugcli/debugcmd.c @@ -72,7 +72,6 @@ #include #include -#include #include #include diff --git a/server/modules/routing/maxinfo/maxinfo.c b/server/modules/routing/maxinfo/maxinfo.c index ded66eff0..46114b2f7 100644 --- a/server/modules/routing/maxinfo/maxinfo.c +++ b/server/modules/routing/maxinfo/maxinfo.c @@ -44,7 +44,6 @@ #include #include #include "maxinfo.h" -#include #include #include #include diff --git a/server/modules/routing/maxinfo/maxinfo_error.c b/server/modules/routing/maxinfo/maxinfo_error.c index b16258a5b..aefd46f77 100644 --- a/server/modules/routing/maxinfo/maxinfo_error.c +++ b/server/modules/routing/maxinfo/maxinfo_error.c @@ -37,7 +37,6 @@ #include #include #include "maxinfo.h" -#include #include diff --git a/server/modules/routing/maxinfo/maxinfo_exec.c b/server/modules/routing/maxinfo/maxinfo_exec.c index d3c84ab0b..c5b031b94 100644 --- a/server/modules/routing/maxinfo/maxinfo_exec.c +++ b/server/modules/routing/maxinfo/maxinfo_exec.c @@ -41,7 +41,6 @@ #include #include #include "maxinfo.h" -#include #include #include #include diff --git a/server/modules/routing/maxinfo/maxinfo_parse.c b/server/modules/routing/maxinfo/maxinfo_parse.c index ccf3aea7a..e75d65e92 100644 --- a/server/modules/routing/maxinfo/maxinfo_parse.c +++ b/server/modules/routing/maxinfo/maxinfo_parse.c @@ -40,7 +40,6 @@ #include #include #include "maxinfo.h" -#include #include static MAXINFO_TREE *make_tree_node(MAXINFO_OPERATOR, char *, MAXINFO_TREE *, MAXINFO_TREE *); diff --git a/server/modules/routing/readconnroute/readconnroute.c b/server/modules/routing/readconnroute/readconnroute.c index bbb7d55b6..f95a24cf0 100644 --- a/server/modules/routing/readconnroute/readconnroute.c +++ b/server/modules/routing/readconnroute/readconnroute.c @@ -84,7 +84,6 @@ #include #include -#include #include #include diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index 7d9b8f3e4..8ad2c34d5 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -20,7 +20,6 @@ #include "readwritesplit.h" #include "rwsplit_internal.h" -#include #include #include #include diff --git a/server/modules/routing/readwritesplit/rwsplit_mysql.c b/server/modules/routing/readwritesplit/rwsplit_mysql.c index aa7050dcd..1025f44a9 100644 --- a/server/modules/routing/readwritesplit/rwsplit_mysql.c +++ b/server/modules/routing/readwritesplit/rwsplit_mysql.c @@ -22,7 +22,6 @@ #include "rwsplit_internal.h" #include -#include #include #include #include diff --git a/server/modules/routing/schemarouter/schemarouter.c b/server/modules/routing/schemarouter/schemarouter.c index 551841b5a..95e544a3c 100644 --- a/server/modules/routing/schemarouter/schemarouter.c +++ b/server/modules/routing/schemarouter/schemarouter.c @@ -21,7 +21,6 @@ #include "sharding_common.h" #include #include -#include #include #include #include From d5cf74bd24ed000df6aab90af44d9e432aee8e63 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Sat, 15 Oct 2016 11:14:11 +0300 Subject: [PATCH 037/215] Cleanup gw.h, part 1. Gw.h contained a fair amount of obsolete function declarations, duplicate declarations of functions declared in utils.h and declarations of functions that conceptually are similar to those in utils.h. The obsolete and duplicate ones were removed and all but one of the remaining moved to utils.h. Correspondingly the implementation was moved from gw_utils.c to utils.c. The one remaining function - gw_daemonize() - is not really worthy of a file of its own, so that is to be moved to gateway.c after which gw_utils.c can be removed. Gw.h still contains defines that are duplicated in maxscale/protocol/mysql.h. The ones in gw.h are to be removed. It appears that the entire gw.h will disappear. --- include/maxscale/cdefs.h | 1 + include/maxscale/gw.h | 16 ---- include/maxscale/utils.h | 11 +++ server/core/dcb.c | 1 + server/core/gw_utils.c | 167 ----------------------------------- server/core/gwdirs.c | 2 +- server/core/poll.c | 1 + server/core/utils.c | 185 +++++++++++++++++++++++++++++++++++++-- 8 files changed, 193 insertions(+), 191 deletions(-) diff --git a/include/maxscale/cdefs.h b/include/maxscale/cdefs.h index 62c28b328..a26342939 100644 --- a/include/maxscale/cdefs.h +++ b/include/maxscale/cdefs.h @@ -72,5 +72,6 @@ */ #include #include +#include #endif diff --git a/include/maxscale/gw.h b/include/maxscale/gw.h index 085c73e58..5e81ba2fe 100644 --- a/include/maxscale/gw.h +++ b/include/maxscale/gw.h @@ -69,23 +69,7 @@ MXS_BEGIN_DECLS #define MYSQL_CONN_DEBUG #undef MYSQL_CONN_DEBUG -#include "dcb.h" - bool gw_daemonize(void); -int do_read_dcb(DCB *dcb); -void MySQLListener(int epfd, char *config_bind); -int MySQLAccept(DCB *listener); -int do_read_dcb(DCB *dcb); -int do_read_10(DCB *dcb, uint8_t *buffer); -int MySQLWrite(DCB *dcb, GWBUF *queue); -int setnonblocking(int fd); -int gw_getsockerrno(int fd); -int parse_bindconfig(const char *, struct sockaddr_in *); -int setipaddress(struct in_addr *, char *); -char* get_libdir(); -long get_processor_count(); -void clean_up_pathname(char *path); -bool mxs_mkdir_all(const char *path, int mask); MXS_END_DECLS diff --git a/include/maxscale/utils.h b/include/maxscale/utils.h index 1f608296d..55ce9a34e 100644 --- a/include/maxscale/utils.h +++ b/include/maxscale/utils.h @@ -27,8 +27,10 @@ */ #include +#include #include #include +#include MXS_BEGIN_DECLS @@ -41,6 +43,9 @@ bool utils_init(); /*< Call this first before using any other function */ void utils_end(); int setnonblocking(int fd); +int parse_bindconfig(const char *, struct sockaddr_in *); +int setipaddress(struct in_addr *, char *); + char *gw_strend(register const char *s); static char gw_randomchar(); int gw_generate_random_str(char *output, int len); @@ -67,6 +72,12 @@ char* replace_literal(char* haystack, const char* replacement); char* replace_quoted(const char** src, const size_t* srcsize, char** dest, size_t* destsize); +void clean_up_pathname(char *path); + +bool mxs_mkdir_all(const char *path, int mask); + +long get_processor_count(); + MXS_END_DECLS #endif diff --git a/server/core/dcb.c b/server/core/dcb.c index 065213cec..675078798 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -90,6 +90,7 @@ #include #include #include +#include #if defined(FAKE_CODE) unsigned char dcb_fake_write_errno[10240]; diff --git a/server/core/gw_utils.c b/server/core/gw_utils.c index 2ef05638d..60cc9bed5 100644 --- a/server/core/gw_utils.c +++ b/server/core/gw_utils.c @@ -39,96 +39,6 @@ #include #include -SPINLOCK tmplock = SPINLOCK_INIT; - -/* - * Set IP address in socket structure in_addr - * - * @param a Pointer to a struct in_addr into which the address is written - * @param p The hostname to lookup - * @return 1 on success, 0 on failure - */ -int -setipaddress(struct in_addr *a, char *p) -{ -#ifdef __USE_POSIX - struct addrinfo *ai = NULL, hint; - int rc; - struct sockaddr_in *res_addr; - memset(&hint, 0, sizeof (hint)); - - hint.ai_socktype = SOCK_STREAM; - - /* - * This is for the listening socket, matching INADDR_ANY only for now. - * For future specific addresses bind, a dedicated routine woulbd be better - */ - - if (strcmp(p, "0.0.0.0") == 0) - { - hint.ai_flags = AI_PASSIVE; - hint.ai_family = AF_UNSPEC; - if ((rc = getaddrinfo(p, NULL, &hint, &ai)) != 0) - { - MXS_ERROR("Failed to obtain address for host %s, %s", - p, - gai_strerror(rc)); - - return 0; - } - } - else - { - hint.ai_flags = AI_CANONNAME; - hint.ai_family = AF_INET; - - if ((rc = getaddrinfo(p, NULL, &hint, &ai)) != 0) - { - MXS_ERROR("Failed to obtain address for host %s, %s", - p, - gai_strerror(rc)); - - return 0; - } - } - - /* take the first one */ - if (ai != NULL) - { - res_addr = (struct sockaddr_in *)(ai->ai_addr); - memcpy(a, &res_addr->sin_addr, sizeof(struct in_addr)); - - freeaddrinfo(ai); - - return 1; - } -#else - struct hostent *h; - - spinlock_acquire(&tmplock); - h = gethostbyname(p); - spinlock_release(&tmplock); - - if (h == NULL) - { - if ((a->s_addr = inet_addr(p)) == -1) - { - MXS_ERROR("gethostbyname failed for [%s]", p); - - return 0; - } - } - else - { - /* take the first one */ - memcpy(a, h->h_addr, h->h_length); - - return 1; - } -#endif - return 0; -} - /** * Daemonize the process by forking and putting the process into the * background. @@ -160,80 +70,3 @@ bool gw_daemonize(void) } return false; } - -/** - * Parse the bind config data. This is passed in a string as address:port. - * - * The address may be either a . separated IP address or a hostname to - * lookup. The address 0.0.0.0 is the wildcard address for SOCKADR_ANY. - * The ':' and port are required. - * - * @param config The bind address and port separated by a ':' - * @param addr The sockaddr_in in which the data is written - * @return 0 on failure - */ -int -parse_bindconfig(const char *config, struct sockaddr_in *addr) -{ - char buf[strlen(config) + 1]; - strcpy(buf, config); - - char *port = strrchr(buf, ':'); - short pnum; - if (port) - { - *port = 0; - port++; - pnum = atoi(port); - } - else - { - return 0; - } - - if (!strcmp(buf, "0.0.0.0")) - { - addr->sin_addr.s_addr = htonl(INADDR_ANY); - } - else - { - if (!inet_aton(buf, &addr->sin_addr)) - { - struct hostent *hp = gethostbyname(buf); - - if (hp) - { - bcopy(hp->h_addr, &(addr->sin_addr.s_addr), hp->h_length); - } - else - { - MXS_ERROR("Failed to lookup host '%s'.", buf); - return 0; - } - } - } - - addr->sin_family = AF_INET; - addr->sin_port = htons(pnum); - return 1; -} - -/** - * Return the number of processors available. - * @return Number of processors or 1 if the required definition of _SC_NPROCESSORS_CONF - * is not found - */ -long get_processor_count() -{ - long processors = 1; -#ifdef _SC_NPROCESSORS_ONLN - if ((processors = sysconf(_SC_NPROCESSORS_ONLN)) <= 0) - { - MXS_WARNING("Unable to establish the number of available cores. Defaulting to 1."); - processors = 1; - } -#else -#error _SC_NPROCESSORS_ONLN not available. -#endif - return processors; -} diff --git a/server/core/gwdirs.c b/server/core/gwdirs.c index d7c54be0f..eb8ca6394 100644 --- a/server/core/gwdirs.c +++ b/server/core/gwdirs.c @@ -13,7 +13,7 @@ #include #include -#include +#include /** * Set the configuration file directory diff --git a/server/core/poll.c b/server/core/poll.c index 149b67fdb..c32661f68 100644 --- a/server/core/poll.c +++ b/server/core/poll.c @@ -33,6 +33,7 @@ #include #include #include +#include #define PROFILE_POLL 0 diff --git a/server/core/utils.c b/server/core/utils.c index ead727b8e..0688da6c8 100644 --- a/server/core/utils.c +++ b/server/core/utils.c @@ -28,18 +28,21 @@ * @endverbatim */ - +#include +#include +#include #include -#include -#include -#include +#include +#include #include #include -#include +#include #include -#include -#include #include +#include +#include +#include +#include #if !defined(PATH_MAX) # if defined(__USE_POSIX) @@ -880,3 +883,171 @@ void utils_end() pcre2_code_free(replace_values_re); replace_values_re = NULL; } + +SPINLOCK tmplock = SPINLOCK_INIT; + +/* + * Set IP address in socket structure in_addr + * + * @param a Pointer to a struct in_addr into which the address is written + * @param p The hostname to lookup + * @return 1 on success, 0 on failure + */ +int +setipaddress(struct in_addr *a, char *p) +{ +#ifdef __USE_POSIX + struct addrinfo *ai = NULL, hint; + int rc; + struct sockaddr_in *res_addr; + memset(&hint, 0, sizeof (hint)); + + hint.ai_socktype = SOCK_STREAM; + + /* + * This is for the listening socket, matching INADDR_ANY only for now. + * For future specific addresses bind, a dedicated routine woulbd be better + */ + + if (strcmp(p, "0.0.0.0") == 0) + { + hint.ai_flags = AI_PASSIVE; + hint.ai_family = AF_UNSPEC; + if ((rc = getaddrinfo(p, NULL, &hint, &ai)) != 0) + { + MXS_ERROR("Failed to obtain address for host %s, %s", + p, + gai_strerror(rc)); + + return 0; + } + } + else + { + hint.ai_flags = AI_CANONNAME; + hint.ai_family = AF_INET; + + if ((rc = getaddrinfo(p, NULL, &hint, &ai)) != 0) + { + MXS_ERROR("Failed to obtain address for host %s, %s", + p, + gai_strerror(rc)); + + return 0; + } + } + + /* take the first one */ + if (ai != NULL) + { + res_addr = (struct sockaddr_in *)(ai->ai_addr); + memcpy(a, &res_addr->sin_addr, sizeof(struct in_addr)); + + freeaddrinfo(ai); + + return 1; + } +#else + struct hostent *h; + + spinlock_acquire(&tmplock); + h = gethostbyname(p); + spinlock_release(&tmplock); + + if (h == NULL) + { + if ((a->s_addr = inet_addr(p)) == -1) + { + MXS_ERROR("gethostbyname failed for [%s]", p); + + return 0; + } + } + else + { + /* take the first one */ + memcpy(a, h->h_addr, h->h_length); + + return 1; + } +#endif + return 0; +} + + +/** + * Parse the bind config data. This is passed in a string as address:port. + * + * The address may be either a . separated IP address or a hostname to + * lookup. The address 0.0.0.0 is the wildcard address for SOCKADR_ANY. + * The ':' and port are required. + * + * @param config The bind address and port separated by a ':' + * @param addr The sockaddr_in in which the data is written + * @return 0 on failure + */ +int +parse_bindconfig(const char *config, struct sockaddr_in *addr) +{ + char buf[strlen(config) + 1]; + strcpy(buf, config); + + char *port = strrchr(buf, ':'); + short pnum; + if (port) + { + *port = 0; + port++; + pnum = atoi(port); + } + else + { + return 0; + } + + if (!strcmp(buf, "0.0.0.0")) + { + addr->sin_addr.s_addr = htonl(INADDR_ANY); + } + else + { + if (!inet_aton(buf, &addr->sin_addr)) + { + struct hostent *hp = gethostbyname(buf); + + if (hp) + { + bcopy(hp->h_addr, &(addr->sin_addr.s_addr), hp->h_length); + } + else + { + MXS_ERROR("Failed to lookup host '%s'.", buf); + return 0; + } + } + } + + addr->sin_family = AF_INET; + addr->sin_port = htons(pnum); + return 1; +} + +/** + * Return the number of processors available. + * @return Number of processors or 1 if the required definition of _SC_NPROCESSORS_CONF + * is not found + */ +long get_processor_count() +{ + long processors = 1; +#ifdef _SC_NPROCESSORS_ONLN + if ((processors = sysconf(_SC_NPROCESSORS_ONLN)) <= 0) + { + MXS_WARNING("Unable to establish the number of available cores. Defaulting to 1."); + processors = 1; + } +#else +#error _SC_NPROCESSORS_ONLN not available. +#endif + return processors; +} From 4cb6d9227e1e95fa3991b11159a9b1ed0e0ac145 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Mon, 17 Oct 2016 07:20:35 +0300 Subject: [PATCH 038/215] MXS-778: Set read-only masters into Slave status When a master server is set into read-only mode, it can be treated as an always up-to-date slave. This gives us a somewhat graceful way to prevent writes to a master. Usually setting the master into read-only mode is done before a change in the replication topology to prevent stray writes arriving on the master. MaxScale should respect the read-only mode and not send any writes to a server that's in read-only mode. --- server/modules/monitor/mysqlmon/mysql_mon.c | 30 ++++++++++++++++----- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/server/modules/monitor/mysqlmon/mysql_mon.c b/server/modules/monitor/mysqlmon/mysql_mon.c index 901238f44..86967962d 100644 --- a/server/modules/monitor/mysqlmon/mysql_mon.c +++ b/server/modules/monitor/mysqlmon/mysql_mon.c @@ -49,6 +49,7 @@ #include #include #include +#include /** Column positions for SHOW SLAVE STATUS */ #define MYSQL55_STATUS_BINLOG_POS 5 @@ -1032,8 +1033,17 @@ void find_graph_cycles(MYSQL_MONITOR *handle, MONITOR_SERVERS *database, int nse * slave in this case can be either a normal slave or another * master. */ - monitor_set_pending_status(graph[i].db, SERVER_MASTER | SERVER_STALE_STATUS); - monitor_clear_pending_status(graph[i].db, SERVER_SLAVE); + if (graph[i].info->read_only) + { + /** The master is in read-only mode, set it into Slave state */ + monitor_set_pending_status(graph[i].db, SERVER_SLAVE); + monitor_clear_pending_status(graph[i].db, SERVER_MASTER | SERVER_STALE_STATUS); + } + else + { + monitor_set_pending_status(graph[i].db, SERVER_MASTER | SERVER_STALE_STATUS); + monitor_clear_pending_status(graph[i].db, SERVER_SLAVE); + } } } } @@ -1315,10 +1325,11 @@ monitorMain(void *arg) ptr = mon->databases; while (ptr) { - MYSQL_SERVER_INFO *serv_info = hashtable_fetch(handle->server_info, ptr->server->unique_name); - ss_dassert(serv_info); if (!SERVER_IN_MAINT(ptr->server)) { + MYSQL_SERVER_INFO *serv_info = hashtable_fetch(handle->server_info, ptr->server->unique_name); + ss_dassert(serv_info); + /** If "detect_stale_master" option is On, let's use the previous master. * * Multi-master mode detects the stale masters in find_graph_cycles(). @@ -1327,7 +1338,8 @@ monitorMain(void *arg) (strcmp(ptr->server->name, root_master->server->name) == 0 && ptr->server->port == root_master->server->port) && (ptr->server->status & SERVER_MASTER) && - !(ptr->pending_status & SERVER_MASTER)) + !(ptr->pending_status & SERVER_MASTER) && + !serv_info->read_only) { /** * In this case server->status will not be updated from pending_status @@ -1877,7 +1889,13 @@ static MONITOR_SERVERS *get_replication_tree(MONITOR *mon, int num_servers) monitor_clear_pending_status(handle->master, SERVER_MASTER); } - monitor_set_pending_status(master, SERVER_MASTER); + MYSQL_SERVER_INFO* info = hashtable_fetch(handle->server_info, + master->server->unique_name); + ss_dassert(info); + + /** Only set the Master status if read_only is disabled */ + monitor_set_pending_status(master, info->read_only ? SERVER_SLAVE : SERVER_MASTER); + handle->master = master; } else From 4faa7f8d21e25f8c2b7e38c547e501510e18c91f Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Mon, 17 Oct 2016 09:47:46 +0300 Subject: [PATCH 039/215] Obsolete defines removed --- include/maxscale/gw.h | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/include/maxscale/gw.h b/include/maxscale/gw.h index 5e81ba2fe..eb5ecb816 100644 --- a/include/maxscale/gw.h +++ b/include/maxscale/gw.h @@ -38,8 +38,6 @@ MXS_BEGIN_DECLS -#define EXIT_FAILURE 1 - // network buffer is 32K #define MAX_BUFFER_SIZE 32768 @@ -53,21 +51,6 @@ MXS_BEGIN_DECLS #define GW_CLIENT_SO_RCVBUF (128 * 1024) #define GW_NOINTR_CALL(A) do { errno = 0; A; } while (errno == EINTR) -#define GW_MYSQL_LOOP_TIMEOUT 300000000 -#define GW_MYSQL_READ 0 -#define GW_MYSQL_WRITE 1 - -#define GW_MYSQL_PROTOCOL_VERSION 10 // version is 10 -#define GW_MYSQL_HANDSHAKE_FILLER 0x00 -#define GW_MYSQL_SERVER_CAPABILITIES_BYTE1 0xff -#define GW_MYSQL_SERVER_CAPABILITIES_BYTE2 0xf7 -#define GW_MYSQL_SERVER_LANGUAGE 0x08 -#define GW_MYSQL_MAX_PACKET_LEN 0xffffffL; -#define GW_MYSQL_SCRAMBLE_SIZE 20 - -// debug for mysql_* functions -#define MYSQL_CONN_DEBUG -#undef MYSQL_CONN_DEBUG bool gw_daemonize(void); From 8aafa34d667be609619678d2c028aa0a35fcd341 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Mon, 17 Oct 2016 09:58:45 +0300 Subject: [PATCH 040/215] Hard limits moved to limits.h Limits used with SO_[RCV|SEND]BUF moved from gw.h to limits.h and renamed so that the prefix is MXS_. --- include/maxscale/gw.h | 9 ---- include/maxscale/limits.h | 47 +++++++++++++++++-- server/core/dcb.c | 4 +- .../MySQL/MySQLBackend/mysql_backend.c | 4 +- 4 files changed, 46 insertions(+), 18 deletions(-) diff --git a/include/maxscale/gw.h b/include/maxscale/gw.h index eb5ecb816..a560342fa 100644 --- a/include/maxscale/gw.h +++ b/include/maxscale/gw.h @@ -41,15 +41,6 @@ MXS_BEGIN_DECLS // network buffer is 32K #define MAX_BUFFER_SIZE 32768 -/** - * Configuration for send and receive socket buffer sizes for - * backend and cleint connections. - */ -#define GW_BACKEND_SO_SNDBUF (128 * 1024) -#define GW_BACKEND_SO_RCVBUF (128 * 1024) -#define GW_CLIENT_SO_SNDBUF (128 * 1024) -#define GW_CLIENT_SO_RCVBUF (128 * 1024) - #define GW_NOINTR_CALL(A) do { errno = 0; A; } while (errno == EINTR) bool gw_daemonize(void); diff --git a/include/maxscale/limits.h b/include/maxscale/limits.h index 6847683d9..ea3bc874c 100644 --- a/include/maxscale/limits.h +++ b/include/maxscale/limits.h @@ -18,12 +18,49 @@ MXS_BEGIN_DECLS -// This file defines hard limits of MaxScale. +/** + * @file lmits.h + * + * This file contains defines for hard limits of MaxScale. + */ -// Thread information is stored in a bitmask whose size must be a -// multiple of 8. The bitmask is indexed using the thread id that start -// from 1. Hence, the hard maximum number of threads must be a -// multiple of 8 minus 1. + +/** + * MXS_BACKEND_SO_RCVBUF + * + * The value used when setting SO_RCVBUF of backend sockets. + */ +#define MXS_BACKEND_SO_RCVBUF (128 * 1024) + +/** + * MSX_BACKEND_SO_SNDBUF + * + * The value used when setting SO_SNDBUF of backend sockets. + */ +#define MXS_BACKEND_SO_SNDBUF (128 * 1024) + +/** + * MXS_CLIENT_SO_RCVBUF + * + * The value used when setting SO_RCVBUF of client sockets. + */ +#define MXS_CLIENT_SO_RCVBUF (128 * 1024) + +/** + * MXS_CLIENT_SO_SNDBUF + * + * The value used when setting SO_SNDBUF of client sockets. + */ +#define MXS_CLIENT_SO_SNDBUF (128 * 1024) + +/** + * MXS_MAX_THREADS + * + * Thread information is stored in a bitmask whose size must be a + * multiple of 8. The bitmask is indexed using the thread id that start + * from 1. Hence, the hard maximum number of threads must be a + * multiple of 8 minus 1. + */ #define MXS_MAX_THREADS 255 MXS_END_DECLS diff --git a/server/core/dcb.c b/server/core/dcb.c index 675078798..9c9c74746 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -3101,7 +3101,7 @@ dcb_accept(DCB *listener, GWPROTOCOL *protocol_funcs) conn_open[c_sock] = true; #endif /* FAKE_CODE */ /* set nonblocking */ - sendbuf = GW_CLIENT_SO_SNDBUF; + sendbuf = MXS_CLIENT_SO_SNDBUF; if (setsockopt(c_sock, SOL_SOCKET, SO_SNDBUF, &sendbuf, optlen) != 0) { @@ -3109,7 +3109,7 @@ dcb_accept(DCB *listener, GWPROTOCOL *protocol_funcs) errno, strerror_r(errno, errbuf, sizeof(errbuf))); } - sendbuf = GW_CLIENT_SO_RCVBUF; + sendbuf = MXS_CLIENT_SO_RCVBUF; if (setsockopt(c_sock, SOL_SOCKET, SO_RCVBUF, &sendbuf, optlen) != 0) { diff --git a/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c b/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c index f99dfb605..c9ba4d89e 100644 --- a/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c +++ b/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c @@ -323,7 +323,7 @@ gw_do_connect_to_backend(char *host, int port, int *fd) /* prepare for connect */ setipaddress(&serv_addr.sin_addr, host); serv_addr.sin_port = htons(port); - bufsize = GW_BACKEND_SO_SNDBUF; + bufsize = MXS_BACKEND_SO_SNDBUF; if (setsockopt(so, SOL_SOCKET, SO_SNDBUF, &bufsize, sizeof(bufsize)) != 0) { @@ -340,7 +340,7 @@ gw_do_connect_to_backend(char *host, int port, int *fd) close_socket(so); goto return_rv; } - bufsize = GW_BACKEND_SO_RCVBUF; + bufsize = MXS_BACKEND_SO_RCVBUF; if (setsockopt(so, SOL_SOCKET, SO_RCVBUF, &bufsize, sizeof(bufsize)) != 0) { From b27774e67482cc3bf0e953c233f27659c03cc020 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Mon, 17 Oct 2016 10:24:32 +0300 Subject: [PATCH 041/215] Rename MAX_BUFFER_SIZE MAX_BUFFER_SIZE, which is used for limiting the amount of the data read from a socket renamed to MXS_MAX_NW_READ_BUFFER_SIZE and moved from gw.h to limits.h. Consider removing altogether. Difficult to justify since non-blocking reads are used and the amount of available data is known. --- include/maxscale/gw.h | 3 --- include/maxscale/limits.h | 12 +++++++++++- server/core/dcb.c | 6 +++--- server/modules/routing/avro/avrorouter.h | 2 +- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/include/maxscale/gw.h b/include/maxscale/gw.h index a560342fa..a9a5b601f 100644 --- a/include/maxscale/gw.h +++ b/include/maxscale/gw.h @@ -38,9 +38,6 @@ MXS_BEGIN_DECLS -// network buffer is 32K -#define MAX_BUFFER_SIZE 32768 - #define GW_NOINTR_CALL(A) do { errno = 0; A; } while (errno == EINTR) bool gw_daemonize(void); diff --git a/include/maxscale/limits.h b/include/maxscale/limits.h index ea3bc874c..43eab289f 100644 --- a/include/maxscale/limits.h +++ b/include/maxscale/limits.h @@ -33,7 +33,7 @@ MXS_BEGIN_DECLS #define MXS_BACKEND_SO_RCVBUF (128 * 1024) /** - * MSX_BACKEND_SO_SNDBUF + * MXS_BACKEND_SO_SNDBUF * * The value used when setting SO_SNDBUF of backend sockets. */ @@ -53,6 +53,16 @@ MXS_BEGIN_DECLS */ #define MXS_CLIENT_SO_SNDBUF (128 * 1024) +/** + * MXS_MAX_NW_READ_BUFFER_SIZE + * + * The maximum amount of data read in one gofrom a client DCB. + * + * TODO: Consider removing altogether so that we always read + * whatever is available in the socket. + */ +#define MXS_MAX_NW_READ_BUFFER_SIZE (32 * 1024) + /** * MXS_MAX_THREADS * diff --git a/server/core/dcb.c b/server/core/dcb.c index 9c9c74746..085eeb7f5 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -1069,7 +1069,7 @@ dcb_basic_read(DCB *dcb, int bytesavailable, int maxbytes, int nreadtotal, int * { GWBUF *buffer; - int bufsize = MXS_MIN(bytesavailable, MAX_BUFFER_SIZE); + int bufsize = MXS_MIN(bytesavailable, MXS_MAX_NW_READ_BUFFER_SIZE); if (maxbytes) { bufsize = MXS_MIN(bufsize, maxbytes - nreadtotal); @@ -1187,10 +1187,10 @@ dcb_read_SSL(DCB *dcb, GWBUF **head) static GWBUF * dcb_basic_read_SSL(DCB *dcb, int *nsingleread) { - unsigned char temp_buffer[MAX_BUFFER_SIZE]; + unsigned char temp_buffer[MXS_MAX_NW_READ_BUFFER_SIZE]; GWBUF *buffer = NULL; - *nsingleread = SSL_read(dcb->ssl, (void *)temp_buffer, MAX_BUFFER_SIZE); + *nsingleread = SSL_read(dcb->ssl, (void *)temp_buffer, MXS_MAX_NW_READ_BUFFER_SIZE); dcb->stats.n_reads++; switch (SSL_get_error(dcb->ssl, *nsingleread)) diff --git a/server/modules/routing/avro/avrorouter.h b/server/modules/routing/avro/avrorouter.h index 63147af7d..062e18450 100644 --- a/server/modules/routing/avro/avrorouter.h +++ b/server/modules/routing/avro/avrorouter.h @@ -108,7 +108,7 @@ typedef enum avro_binlog_end #define TABLE_MAP_MAX_NAME_LEN 64 /** How many bytes each thread tries to send */ -#define AVRO_DATA_BURST_SIZE MAX_BUFFER_SIZE +#define AVRO_DATA_BURST_SIZE (32 * 1024) /** A CREATE TABLE abstraction */ typedef struct table_create From e6d7ca4ed9a0de986d73f95948e7f92f6620ccdf Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Mon, 17 Oct 2016 10:41:25 +0300 Subject: [PATCH 042/215] GW_NOINTR_CALL removed. Used in just one place inside MaxScale so better to simply write it explcitly. --- include/maxscale/gw.h | 2 -- server/core/gateway.c | 21 +++++++++++++-------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/include/maxscale/gw.h b/include/maxscale/gw.h index a9a5b601f..22f4b5529 100644 --- a/include/maxscale/gw.h +++ b/include/maxscale/gw.h @@ -38,8 +38,6 @@ MXS_BEGIN_DECLS -#define GW_NOINTR_CALL(A) do { errno = 0; A; } while (errno == EINTR) - bool gw_daemonize(void); MXS_END_DECLS diff --git a/server/core/gateway.c b/server/core/gateway.c index 6435e4b6d..057fdd1b6 100644 --- a/server/core/gateway.c +++ b/server/core/gateway.c @@ -416,25 +416,30 @@ sigfatal_handler(int i) */ static int signal_set(int sig, void (*handler)(int)) { - static struct sigaction sigact; - static int err; int rc = 0; - memset(&sigact, 0, sizeof(struct sigaction)); + struct sigaction sigact = {}; sigact.sa_handler = handler; - GW_NOINTR_CALL(err = sigaction(sig, &sigact, NULL)); + + int err; + + do + { + errno = 0; + err = sigaction(sig, &sigact, NULL); + } + while (errno == EINTR); if (err < 0) { - int eno = errno; - errno = 0; char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Failed call sigaction() in %s due to %d, %s.", program_invocation_short_name, - eno, - strerror_r(eno, errbuf, sizeof(errbuf))); + errno, + strerror_r(errno, errbuf, sizeof(errbuf))); rc = 1; } + return rc; } From 63b5c10f310059320609bc63843e0b47e0534595 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Mon, 17 Oct 2016 10:53:38 +0300 Subject: [PATCH 043/215] Move daemonize() to gateway.c The one remaining function in gw_utils.c was moved to gateway.c, so gw_utils.c could be removed. --- include/maxscale/gw.h | 2 -- server/core/CMakeLists.txt | 2 +- server/core/gateway.c | 38 +++++++++++++++++++- server/core/gw_utils.c | 72 -------------------------------------- 4 files changed, 38 insertions(+), 76 deletions(-) delete mode 100644 server/core/gw_utils.c diff --git a/include/maxscale/gw.h b/include/maxscale/gw.h index 22f4b5529..3bcd8763f 100644 --- a/include/maxscale/gw.h +++ b/include/maxscale/gw.h @@ -38,8 +38,6 @@ MXS_BEGIN_DECLS -bool gw_daemonize(void); - MXS_END_DECLS #endif diff --git a/server/core/CMakeLists.txt b/server/core/CMakeLists.txt index 11a616a62..3c7c630b8 100644 --- a/server/core/CMakeLists.txt +++ b/server/core/CMakeLists.txt @@ -1,4 +1,4 @@ -add_library(maxscale-common SHARED adminusers.c alloc.c authenticator.c atomic.c buffer.c config.c dbusers.c dcb.c filter.c externcmd.c gwbitmask.c gwdirs.c gw_utils.c hashtable.c hint.c housekeeper.c listmanager.c load_utils.c log_manager.cc maxscale_pcre2.c memlog.c misc.c mlist.c modutil.c monitor.c queuemanager.c query_classifier.c poll.c random_jkiss.c resultset.c secrets.c server.c service.c session.c spinlock.c thread.c users.c utils.c skygw_utils.cc statistics.c listener.c gw_ssl.c mysql_utils.c mysql_binlog.c) +add_library(maxscale-common SHARED adminusers.c alloc.c authenticator.c atomic.c buffer.c config.c dbusers.c dcb.c filter.c externcmd.c gwbitmask.c gwdirs.c hashtable.c hint.c housekeeper.c listmanager.c load_utils.c log_manager.cc maxscale_pcre2.c memlog.c misc.c mlist.c modutil.c monitor.c queuemanager.c query_classifier.c poll.c random_jkiss.c resultset.c secrets.c server.c service.c session.c spinlock.c thread.c users.c utils.c skygw_utils.cc statistics.c listener.c gw_ssl.c mysql_utils.c mysql_binlog.c) target_link_libraries(maxscale-common ${MARIADB_CONNECTOR_LIBRARIES} ${LZMA_LINK_FLAGS} ${PCRE2_LIBRARIES} ${CURL_LIBRARIES} ssl pthread crypt dl crypto inih z rt m stdc++) diff --git a/server/core/gateway.c b/server/core/gateway.c index 057fdd1b6..64688565f 100644 --- a/server/core/gateway.c +++ b/server/core/gateway.c @@ -190,6 +190,7 @@ void write_child_exit_code(int fd, int code); static bool change_cwd(); void shutdown_server(); static void log_exit_status(); +static bool daemonize(); /** SSL multi-threading functions and structures */ @@ -1578,7 +1579,7 @@ int main(int argc, char **argv) /** Daemonize the process and wait for the child process to notify * the parent process of its exit status. */ - parent_process = gw_daemonize(); + parent_process = daemonize(); if (parent_process) { @@ -2663,3 +2664,38 @@ static void log_exit_status() break; } } + +/** + * Daemonize the process by forking and putting the process into the + * background. + * + * @return True if context is that of the parent process, false if that of the + * child process. + */ +static bool daemonize(void) +{ + pid_t pid; + + pid = fork(); + + if (pid < 0) + { + char errbuf[MXS_STRERROR_BUFLEN]; + fprintf(stderr, "fork() error %s\n", strerror_r(errno, errbuf, sizeof(errbuf))); + exit(1); + } + + if (pid != 0) + { + /* exit from main */ + return true; + } + + if (setsid() < 0) + { + char errbuf[MXS_STRERROR_BUFLEN]; + fprintf(stderr, "setsid() error %s\n", strerror_r(errno, errbuf, sizeof(errbuf))); + exit(1); + } + return false; +} diff --git a/server/core/gw_utils.c b/server/core/gw_utils.c deleted file mode 100644 index 60cc9bed5..000000000 --- a/server/core/gw_utils.c +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (c) 2016 MariaDB Corporation Ab - * - * Use of this software is governed by the Business Source License included - * in the LICENSE.TXT file and at www.mariadb.com/bsl. - * - * Change Date: 2019-07-01 - * - * On the date above, in accordance with the Business Source License, use - * of this software will be governed by version 2 or later of the General - * Public License. - */ - -/** - * @file gw_utils.c - A set if utility functions useful within the context - * of the gateway. - * - * @verbatim - * Revision History - * - * Date Who Description - * 03-06-2013 Massimiliano Pinto gateway utils - * 12-06-2013 Massimiliano Pinto gw_read_gwbuff - * with error detection - * and its handling - * 01-07-2013 Massimiliano Pinto Removed session->backends - * from gw_read_gwbuff() - * 25-09-2013 Massimiliano Pinto setipaddress uses getaddrinfo - * 06-02-2014 Mark Riddoch Added parse_bindconfig - * 10-02-2014 Massimiliano Pinto Added return code to setipaddress - * 02-09-2014 Martin Brampton Replace C++ comment with C comment - * 02-03-2016 Martin Brampton Remove default from parse_bindconfig - * - *@endverbatim - */ - -#include -#include -#include -#include - -/** - * Daemonize the process by forking and putting the process into the - * background. - */ -bool gw_daemonize(void) -{ - pid_t pid; - - pid = fork(); - - if (pid < 0) - { - char errbuf[MXS_STRERROR_BUFLEN]; - fprintf(stderr, "fork() error %s\n", strerror_r(errno, errbuf, sizeof(errbuf))); - exit(1); - } - - if (pid != 0) - { - /* exit from main */ - return true; - } - - if (setsid() < 0) - { - char errbuf[MXS_STRERROR_BUFLEN]; - fprintf(stderr, "setsid() error %s\n", strerror_r(errno, errbuf, sizeof(errbuf))); - exit(1); - } - return false; -} From f0cf391c1875305d89b97912f19ad120bed52d27 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Mon, 17 Oct 2016 11:23:14 +0300 Subject: [PATCH 044/215] include/maxscale/gw.h removed. In a subsequent change, the includes of server/core/*.c will be cleaned up, and if there is a common set of include files, needed by most, then a server/core/maxscale/core.h that includes those will be introduced. --- include/maxscale/gw.h | 43 ------------------- include/maxscale/protocol/mysql.h | 1 - server/core/config.c | 1 - server/core/dcb.c | 3 +- server/core/gateway.c | 3 +- server/core/load_utils.c | 1 - server/core/poll.c | 1 - server/core/secrets.c | 2 - server/core/service.c | 1 - server/modules/authenticator/cdc_plain_auth.c | 5 ++- server/modules/authenticator/mysql_auth.c | 1 - server/modules/include/cdc.h | 1 - server/modules/protocol/CDC/cdc.c | 1 - server/modules/protocol/HTTPD/httpd.c | 2 +- server/modules/protocol/HTTPD/httpd.h | 1 - .../MySQL/MySQLBackend/mysql_backend.c | 1 - .../protocol/MySQL/MySQLClient/mysql_client.c | 1 - server/modules/protocol/MySQL/mysql_common.c | 1 - server/modules/protocol/maxscaled/maxscaled.c | 3 +- server/modules/protocol/telnetd/telnetd.c | 1 - server/modules/routing/avro/avrorouter.h | 1 - server/modules/routing/binlog/blr.c | 1 - server/modules/routing/binlog/blr_slave.c | 1 - 23 files changed, 10 insertions(+), 67 deletions(-) delete mode 100644 include/maxscale/gw.h diff --git a/include/maxscale/gw.h b/include/maxscale/gw.h deleted file mode 100644 index 3bcd8763f..000000000 --- a/include/maxscale/gw.h +++ /dev/null @@ -1,43 +0,0 @@ -#pragma once -#ifndef _MAXSCALE_GW_HG -#define _MAXSCALE_GW_HG -/* - * Copyright (c) 2016 MariaDB Corporation Ab - * - * Use of this software is governed by the Business Source License included - * in the LICENSE.TXT file and at www.mariadb.com/bsl. - * - * Change Date: 2019-07-01 - * - * On the date above, in accordance with the Business Source License, use - * of this software will be governed by version 2 or later of the General - * Public License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -MXS_BEGIN_DECLS - -MXS_END_DECLS - -#endif diff --git a/include/maxscale/protocol/mysql.h b/include/maxscale/protocol/mysql.h index 7a1c0af06..254478e21 100644 --- a/include/maxscale/protocol/mysql.h +++ b/include/maxscale/protocol/mysql.h @@ -63,7 +63,6 @@ #include #include #include -#include #include MXS_BEGIN_DECLS diff --git a/server/core/config.c b/server/core/config.c index 4b9bbab2f..ce3fbf2f9 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -73,7 +73,6 @@ #include #include #include -#include #include #include #define PCRE2_CODE_UNIT_WIDTH 8 diff --git a/server/core/dcb.c b/server/core/dcb.c index 085eeb7f5..9d3c5cb87 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -78,7 +78,6 @@ #include #include #include -#include #include #include #include @@ -86,6 +85,8 @@ #include #include #include +#include +#include #include #include #include diff --git a/server/core/gateway.c b/server/core/gateway.c index 64688565f..107e68635 100644 --- a/server/core/gateway.c +++ b/server/core/gateway.c @@ -48,10 +48,10 @@ #include #include #include -#include #include #include #include +#include #include #include #include @@ -86,6 +86,7 @@ #include #include #include +#include #define STRING_BUFFER_SIZE 1024 #define PIDFD_CLOSED -1 diff --git a/server/core/load_utils.c b/server/core/load_utils.c index c995ef085..1c7e241bc 100644 --- a/server/core/load_utils.c +++ b/server/core/load_utils.c @@ -42,7 +42,6 @@ #include #include #include -#include #include #include diff --git a/server/core/poll.c b/server/core/poll.c index c32661f68..e6abc9856 100644 --- a/server/core/poll.c +++ b/server/core/poll.c @@ -25,7 +25,6 @@ #include #include #include -#include #include #include #include diff --git a/server/core/secrets.c b/server/core/secrets.c index 30570044d..b197ff1e0 100644 --- a/server/core/secrets.c +++ b/server/core/secrets.c @@ -20,8 +20,6 @@ #include #include -#include - /** * Generate a random printable character * diff --git a/server/core/service.c b/server/core/service.c index 643011a31..c6d7b871d 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -60,7 +60,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/authenticator/cdc_plain_auth.c b/server/modules/authenticator/cdc_plain_auth.c index 541d22441..1b19d9b20 100644 --- a/server/modules/authenticator/cdc_plain_auth.c +++ b/server/modules/authenticator/cdc_plain_auth.c @@ -26,11 +26,12 @@ */ #include +#include #include +#include +#include #include #include -#include -#include /* Allowed time interval (in seconds) after last update*/ #define CDC_USERS_REFRESH_TIME 30 diff --git a/server/modules/authenticator/mysql_auth.c b/server/modules/authenticator/mysql_auth.c index 90ac2aa7b..e4cc3ffba 100644 --- a/server/modules/authenticator/mysql_auth.c +++ b/server/modules/authenticator/mysql_auth.c @@ -32,7 +32,6 @@ #include #include #include -#include #include #include diff --git a/server/modules/include/cdc.h b/server/modules/include/cdc.h index 0785ccbd0..19a8ff477 100644 --- a/server/modules/include/cdc.h +++ b/server/modules/include/cdc.h @@ -37,7 +37,6 @@ #include #include #include -#include MXS_BEGIN_DECLS diff --git a/server/modules/protocol/CDC/cdc.c b/server/modules/protocol/CDC/cdc.c index 43471caad..e4612780e 100644 --- a/server/modules/protocol/CDC/cdc.c +++ b/server/modules/protocol/CDC/cdc.c @@ -32,7 +32,6 @@ #include #include -#include #include #include #include diff --git a/server/modules/protocol/HTTPD/httpd.c b/server/modules/protocol/HTTPD/httpd.c index d3ae66ff9..fc8220ec1 100644 --- a/server/modules/protocol/HTTPD/httpd.c +++ b/server/modules/protocol/HTTPD/httpd.c @@ -33,9 +33,9 @@ */ #include "httpd.h" +#include #include #include -#include #include #include #include diff --git a/server/modules/protocol/HTTPD/httpd.h b/server/modules/protocol/HTTPD/httpd.h index 4131f3ccf..c8ded44c4 100644 --- a/server/modules/protocol/HTTPD/httpd.h +++ b/server/modules/protocol/HTTPD/httpd.h @@ -37,7 +37,6 @@ #include #include #include -#include MXS_BEGIN_DECLS diff --git a/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c b/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c index c9ba4d89e..5cdf7e7eb 100644 --- a/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c +++ b/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c @@ -16,7 +16,6 @@ #include #include #include -#include #include /* diff --git a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c index 519da671b..066179748 100644 --- a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c +++ b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c @@ -52,7 +52,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/protocol/MySQL/mysql_common.c b/server/modules/protocol/MySQL/mysql_common.c index 7d6958571..9e3636579 100644 --- a/server/modules/protocol/MySQL/mysql_common.c +++ b/server/modules/protocol/MySQL/mysql_common.c @@ -42,7 +42,6 @@ * */ -#include #include #include #include diff --git a/server/modules/protocol/maxscaled/maxscaled.c b/server/modules/protocol/maxscaled/maxscaled.c index d6758604c..d3a7a7eeb 100644 --- a/server/modules/protocol/maxscaled/maxscaled.c +++ b/server/modules/protocol/maxscaled/maxscaled.c @@ -21,14 +21,15 @@ #include #include #include +#include #include +#include #include #include #include #include #include #include -#include #include #include #include diff --git a/server/modules/protocol/telnetd/telnetd.c b/server/modules/protocol/telnetd/telnetd.c index 95a3f96ea..df802bf8e 100644 --- a/server/modules/protocol/telnetd/telnetd.c +++ b/server/modules/protocol/telnetd/telnetd.c @@ -28,7 +28,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/routing/avro/avrorouter.h b/server/modules/routing/avro/avrorouter.h index 062e18450..1b9accbe9 100644 --- a/server/modules/routing/avro/avrorouter.h +++ b/server/modules/routing/avro/avrorouter.h @@ -22,7 +22,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/routing/binlog/blr.c b/server/modules/routing/binlog/blr.c index 6d542e9a1..5a978e925 100644 --- a/server/modules/routing/binlog/blr.c +++ b/server/modules/routing/binlog/blr.c @@ -78,7 +78,6 @@ #include #include #include -#include static char *version_str = "V2.1.0"; diff --git a/server/modules/routing/binlog/blr_slave.c b/server/modules/routing/binlog/blr_slave.c index b41583c24..b3fce8ec2 100644 --- a/server/modules/routing/binlog/blr_slave.c +++ b/server/modules/routing/binlog/blr_slave.c @@ -83,7 +83,6 @@ #include #include #include -#include extern int load_mysql_users(SERV_LISTENER *listener); extern void blr_master_close(ROUTER_INSTANCE* router); From 15de0a0fb3978b0660197a2d069e4043bc64435d Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Mon, 17 Oct 2016 12:16:47 +0200 Subject: [PATCH 045/215] COM_QUIT handler in binlog server now returns 1 COM_QUIT handler in binlog server now returns 1. --- server/modules/routing/binlog/blr_slave.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/modules/routing/binlog/blr_slave.c b/server/modules/routing/binlog/blr_slave.c index b3fce8ec2..b958f5cd3 100644 --- a/server/modules/routing/binlog/blr_slave.c +++ b/server/modules/routing/binlog/blr_slave.c @@ -260,7 +260,7 @@ blr_slave_request(ROUTER_INSTANCE *router, ROUTER_SLAVE *slave, GWBUF *queue) case COM_QUIT: MXS_DEBUG("COM_QUIT received from slave with server_id %d", slave->serverid); - break; + return 1; default: blr_send_custom_error(slave->dcb, 1, 0, "You have an error in your SQL syntax; Check the " From fe8ebda11e5f73a21f0a344740ad760ed9efc8f5 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Mon, 17 Oct 2016 13:59:59 +0300 Subject: [PATCH 046/215] MXS-828: Log warning if both syslog and maxlog disabled A warning is logged to stderr if both syslog and maxlog are disabled. Earlier a note was unconditionally written to stdout for each. --- server/core/gateway.c | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/server/core/gateway.c b/server/core/gateway.c index 107e68635..67eb2e9a9 100644 --- a/server/core/gateway.c +++ b/server/core/gateway.c @@ -1750,16 +1750,6 @@ int main(int argc, char **argv) goto return_main; } - if (!(*syslog_enabled)) - { - printf("Syslog logging is disabled.\n"); - } - - if (!(*maxlog_enabled)) - { - printf("MaxScale logging is disabled.\n"); - } - mxs_log_set_syslog_enabled(*syslog_enabled); mxs_log_set_maxlog_enabled(*maxlog_enabled); @@ -1826,6 +1816,11 @@ int main(int argc, char **argv) get_cachedir()); } + if (!(*syslog_enabled) && !(*maxlog_enabled)) + { + fprintf(stderr, "warning: Both MaxScale and Syslog logging disabled.\n"); + } + MXS_NOTICE("Configuration file: %s", cnf_file_path); MXS_NOTICE("Log directory: %s", get_logdir()); MXS_NOTICE("Data directory: %s", get_datadir()); From 8d92e295d8e692a179fb98fe1f21bc275cfa6df5 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Mon, 17 Oct 2016 15:29:20 +0200 Subject: [PATCH 047/215] Fixed wrong constant name Fixed wrong constant name --- server/modules/routing/binlog/blr_file.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/modules/routing/binlog/blr_file.c b/server/modules/routing/binlog/blr_file.c index dbcc71f2e..800f1aee5 100644 --- a/server/modules/routing/binlog/blr_file.c +++ b/server/modules/routing/binlog/blr_file.c @@ -2293,7 +2293,7 @@ blr_write_special_event(ROUTER_INSTANCE *router, uint32_t file_offset, uint32_t // Write the event if ((n = pwrite(router->binlog_fd, new_event, event_size, file_offset)) != event_size) { - char err_msg[STRERROR_BUFLEN]; + char err_msg[MXS_STRERROR_BUFLEN]; MXS_ERROR("%s: Failed to write %s special binlog record at %lu of %s, %s. " "Truncating to previous record.", router->service->name, new_event_desc, (unsigned long)file_offset, From c0ad2936a464ec1466dfeccfb4151b7c08ab54c4 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 14 Oct 2016 18:24:12 +0300 Subject: [PATCH 048/215] MXS-862: Implement loadusers entry point The GSSAPI client side authenticator now loads a list of users that are, for the time being, logged at info level. The next step is to store this information in the listener's user hashtable. --- server/modules/authenticator/gssapi_auth.c | 70 ++++++++++++++++++++-- 1 file changed, 65 insertions(+), 5 deletions(-) diff --git a/server/modules/authenticator/gssapi_auth.c b/server/modules/authenticator/gssapi_auth.c index e7264ff56..376d2029f 100644 --- a/server/modules/authenticator/gssapi_auth.c +++ b/server/modules/authenticator/gssapi_auth.c @@ -16,12 +16,26 @@ #include #include #include +#include +#include #include "gssapi_auth.h" +/** Query that gets all users that authenticate via the gssapi plugin */ +const char *gssapi_users_query = + "SELECT u.user, u.host, d.db FROM " + "mysql.user AS u JOIN mysql.db AS d " + "ON (u.user = d.user AND u.host = d.host) WHERE u.plugin = 'gssapi' " + "UNION " + "SELECT u.user, u.host, t.db FROM " + "mysql.user AS u JOIN mysql.tables_priv AS t " + "ON (u.user = t.user AND u.host = t.host) WHERE u.plugin = 'gssapi';"; + +#define GSSAPI_USERS_QUERY_NUM_FIELDS 3 + typedef struct gssapi_instance { char *principal_name; -}GSSAPI_INSTANCE; +} GSSAPI_INSTANCE; /** * @brief Initialize the GSSAPI authenticator @@ -337,14 +351,60 @@ void gssapi_auth_free_data(DCB *dcb) } /** - * @brief Dummy function for loadusers entry point + * @brief Load database users that use GSSAPI authentication + * + * Loading the list of database users that use the 'gssapi' plugin allows us to + * give more precise error messages to the clients when authentication fails. * * @param listener Listener definition - * @return Always MXS_AUTH_LOADUSERS_OK + * @return MXS_AUTH_LOADUSERS_OK on success, MXS_AUTH_LOADUSERS_ERROR on error */ int gssapi_auth_load_users(SERV_LISTENER *listener) { - return MXS_AUTH_LOADUSERS_OK; + char *user, *pw; + int rval = MXS_AUTH_LOADUSERS_ERROR; + + if (serviceGetUser(listener->service, &user, &pw) && (pw = decryptPassword(pw))) + { + for (SERVER_REF *servers = listener->service->dbref; servers; servers = servers->next) + { + MYSQL *mysql = mysql_init(NULL); + + if (mxs_mysql_real_connect(mysql, servers->server, user, pw)) + { + if (mysql_query(mysql, gssapi_users_query)) + { + MXS_ERROR("Failed to query server '%s' for GSSAPI users.", + servers->server->unique_name); + } + else + { + MYSQL_RES *res = mysql_store_result(mysql); + + if (res) + { + ss_dassert(mysql_num_fields(res) == GSSAPI_USERS_QUERY_NUM_FIELDS); + MYSQL_ROW row; + + while ((row = mysql_fetch_row(res))) + { + /** TODO: Store this information in the users table of the listener */ + MXS_INFO("Would add: '%s'@'%s' for '%s'", row[0], row[1], row[2]); + } + + rval = MXS_AUTH_LOADUSERS_OK; + mysql_free_result(res); + } + } + + mysql_close(mysql); + } + } + + MXS_FREE(pw); + } + + return rval; } /** @@ -359,7 +419,7 @@ static GWAUTHENTICATOR MyObject = gssapi_auth_authenticate, /* Authenticate user credentials */ gssapi_auth_free_data, /* Free the client data held in DCB */ gssapi_auth_free, /* Free authenticator data */ - gssapi_auth_load_users /* Dummy function */ + gssapi_auth_load_users /* Load database users */ }; MODULE_INFO info = From 891ce2d0e7ae03a45c555ec8632513241ed502a2 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Sat, 15 Oct 2016 10:53:53 +0300 Subject: [PATCH 049/215] MXS-862: Move client information extraction to client protocol The client protocol module now extracts the information that is relevant for all modules. Currently this information consists of the client capabilities, character set, username and default database. --- server/modules/authenticator/gssapi_auth.c | 23 ------- server/modules/authenticator/mysql_auth.c | 61 ++++--------------- .../protocol/MySQL/MySQLClient/mysql_client.c | 46 ++++++++++++++ 3 files changed, 57 insertions(+), 73 deletions(-) diff --git a/server/modules/authenticator/gssapi_auth.c b/server/modules/authenticator/gssapi_auth.c index 376d2029f..5e8063b63 100644 --- a/server/modules/authenticator/gssapi_auth.c +++ b/server/modules/authenticator/gssapi_auth.c @@ -155,31 +155,8 @@ bool store_client_token(DCB *dcb, GWBUF *buffer) */ static void copy_client_information(DCB *dcb, GWBUF *buffer) { - size_t buflen = gwbuf_length(buffer); - MySQLProtocol *protocol = (MySQLProtocol*)dcb->protocol; gssapi_auth_t *auth = (gssapi_auth_t*)dcb->authenticator_data; - - /* Store the connection characteristics and sequence number of the current packet */ - protocol->charset = 0; - gwbuf_copy_data(buffer, MYSQL_CHARSET_OFFSET, 1, (uint8_t*)&protocol->charset); - gwbuf_copy_data(buffer, MYSQL_CLIENT_CAP_OFFSET, MYSQL_CLIENT_CAP_SIZE, - (uint8_t*)&protocol->client_capabilities); gwbuf_copy_data(buffer, MYSQL_SEQ_OFFSET, 1, &auth->sequence); - - if (buflen > MYSQL_AUTH_PACKET_BASE_SIZE) - { - buflen -= MYSQL_AUTH_PACKET_BASE_SIZE; - - /** TODO: Implement something that can safely iterate bytes of a GWBUF - * so that we know where the terminating null character is. For the time - * being, we'll just copy everything. */ - uint8_t data[buflen]; - gwbuf_copy_data(buffer, MYSQL_AUTH_PACKET_BASE_SIZE, buflen, data); - - MYSQL_session *ses = (MYSQL_session*)dcb->data; - /** data is null-terminated so the strcpy is safe */ - strcpy(ses->user, (char*)data); - } } /** diff --git a/server/modules/authenticator/mysql_auth.c b/server/modules/authenticator/mysql_auth.c index e4cc3ffba..6e5710736 100644 --- a/server/modules/authenticator/mysql_auth.c +++ b/server/modules/authenticator/mysql_auth.c @@ -298,46 +298,30 @@ mysql_auth_set_client_data( uint8_t client_auth_packet[client_auth_packet_size]; gwbuf_copy_data(buffer, 0, client_auth_packet_size, client_auth_packet); - /* The numbers are the fixed elements in the client handshake packet */ - int auth_packet_base_size = MYSQL_AUTH_PACKET_BASE_SIZE; int packet_length_used = 0; - /* Take data from fixed locations first */ - memcpy(&protocol->client_capabilities, client_auth_packet + 4, 4); - protocol->charset = 0; - memcpy(&protocol->charset, client_auth_packet + 4 + 4 + 4, 1); - - /* Make username and database a null string in case none is provided */ - client_data->user[0] = 0; - client_data->db[0] = 0; /* Make authentication token length 0 and token null in case none is provided */ client_data->auth_token_len = 0; client_data->auth_token = NULL; - if (client_auth_packet_size > auth_packet_base_size) + if (client_auth_packet_size > MYSQL_AUTH_PACKET_BASE_SIZE) { /* Should have a username */ - char *first_letter_of_username = (char *)(client_auth_packet + auth_packet_base_size); + char *first_letter_of_username = (char *)(client_auth_packet + MYSQL_AUTH_PACKET_BASE_SIZE); int user_length = strlen(first_letter_of_username); - if (client_auth_packet_size > (auth_packet_base_size + user_length) - && user_length <= MYSQL_USER_MAXLEN) - { - strcpy(client_data->user, first_letter_of_username); - } - else - { - /* Packet has incomplete or too long username */ - return MXS_AUTH_FAILED; - } - if (client_auth_packet_size > (auth_packet_base_size + user_length + 1)) + + ss_dassert(client_auth_packet_size > (MYSQL_AUTH_PACKET_BASE_SIZE + user_length) + && user_length <= MYSQL_USER_MAXLEN); + + if (client_auth_packet_size > (MYSQL_AUTH_PACKET_BASE_SIZE + user_length + 1)) { /* Extra 1 is for the terminating null after user name */ - packet_length_used = auth_packet_base_size + user_length + 1; + packet_length_used = MYSQL_AUTH_PACKET_BASE_SIZE + user_length + 1; /* We should find an authentication token next */ /* One byte of packet is the length of authentication token */ memcpy(&client_data->auth_token_len, - client_auth_packet + packet_length_used, - 1); + client_auth_packet + packet_length_used, 1); + if (client_auth_packet_size > (packet_length_used + client_data->auth_token_len)) { @@ -346,7 +330,7 @@ mysql_auth_set_client_data( { /* The extra 1 is for the token length byte, just extracted*/ memcpy(client_data->auth_token, - client_auth_packet + auth_packet_base_size + user_length + 1 + 1, + client_auth_packet + MYSQL_AUTH_PACKET_BASE_SIZE + user_length + 1 + 1, client_data->auth_token_len); } else @@ -360,29 +344,6 @@ mysql_auth_set_client_data( /* Packet was too small to contain authentication token */ return MXS_AUTH_FAILED; } - packet_length_used += 1 + client_data->auth_token_len; - /* - * Note: some clients may pass empty database, CONNECT_WITH_DB !=0 but database ="" - */ - if ((uint32_t)GW_MYSQL_CAPABILITIES_CONNECT_WITH_DB & - gw_mysql_get_byte4((uint8_t *)&protocol->client_capabilities) - && client_auth_packet_size > packet_length_used) - { - char *database = (char *)(client_auth_packet + packet_length_used); - int database_length = strlen(database); - if (client_auth_packet_size > - (packet_length_used + database_length) - && strlen(database) <= MYSQL_DATABASE_MAXLEN) - { - strcpy(client_data->db, database); - } - else - { - /* Packet is too short to contain database string */ - /* or database string in packet is too long */ - return MXS_AUTH_FAILED; - } - } } } return MXS_AUTH_SUCCEEDED; diff --git a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c index 066179748..ead1971b4 100644 --- a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c +++ b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c @@ -466,6 +466,44 @@ int gw_read_client_event(DCB* dcb) return return_code; } +/** + * @brief Store client connection information into the DCB + * @param dcb Client DCB + * @param buffer Buffer containing the handshake response packet + */ +static void store_client_information(DCB *dcb, GWBUF *buffer) +{ + size_t len = gwbuf_length(buffer); + uint8_t data[len]; + MySQLProtocol *proto = (MySQLProtocol*)dcb->protocol; + MYSQL_session *ses = (MYSQL_session*)dcb->data; + + gwbuf_copy_data(buffer, 0, len, data); + ss_dassert(MYSQL_GET_PACKET_LEN(data) + MYSQL_HEADER_LEN == len); + + proto->client_capabilities = gw_mysql_get_byte4(data + MYSQL_CLIENT_CAP_OFFSET); + proto->charset = data[MYSQL_CHARSET_OFFSET]; + strcpy(ses->user, (char*)data + MYSQL_AUTH_PACKET_BASE_SIZE); + *ses->db = '\0'; + + if (proto->client_capabilities & GW_MYSQL_CAPABILITIES_CONNECT_WITH_DB) + { + /** Client supports default database on connect */ + size_t userlen = strlen(ses->user) + 1; + + /** Skip the authentication token, it is handled by the authenticators */ + uint8_t authlen = data[MYSQL_AUTH_PACKET_BASE_SIZE + userlen]; + + size_t dboffset = MYSQL_AUTH_PACKET_BASE_SIZE + userlen + authlen + 1; + + if (data[dboffset]) + { + /** Client is connecting with a default database */ + strcpy(ses->db, (char*)data + dboffset); + } + } +} + /** * @brief Client read event, process when client not yet authenticated * @@ -490,6 +528,14 @@ gw_read_do_authentication(DCB *dcb, GWBUF *read_buffer, int nbytes_read) /** Read the client's packet sequence and increment that by one */ uint8_t next_sequence; gwbuf_copy_data(read_buffer, MYSQL_SEQ_OFFSET, 1, &next_sequence); + + if (next_sequence == 1) + { + /** This is the first response from the client, read the connection + * information and store them in the shared structure */ + store_client_information(dcb, read_buffer); + } + next_sequence++; /** From 7d202eb084eda72ca8c4cee124ddc0956450be77 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 18 Oct 2016 21:35:33 +0300 Subject: [PATCH 050/215] Fix Travis builds The maxavro requires some special libraries which aren't always available. For this reason, the maxavro library shouldn't be build unless all libraries are available. --- .travis/build_maxscale.sh | 4 ++-- CMakeLists.txt | 3 ++- avro/CMakeLists.txt | 16 +++++++++------- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.travis/build_maxscale.sh b/.travis/build_maxscale.sh index dd1b89658..2aeda7c4f 100644 --- a/.travis/build_maxscale.sh +++ b/.travis/build_maxscale.sh @@ -12,9 +12,9 @@ echo TRAVIS_BUILD_DIR: ${TRAVIS_BUILD_DIR} mkdir build cd build -cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DBUILD_TESTS=Y +cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DBUILD_TESTS=Y -DBUILD_AVRO=N -make VERBOSE=1 +make make test sudo make install diff --git a/CMakeLists.txt b/CMakeLists.txt index c8f5a14a4..6f5e3f833 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -177,7 +177,6 @@ set(CMAKE_CXX_FLAGS_DEBUG "${DEBUG_FLAGS} -DSS_DEBUG -DLOG_ASSERT") set(CMAKE_CXX_FLAGS_RELEASE "") set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-ggdb") -include_directories(avro) include_directories(include) include_directories(server/inih) include_directories(server/modules/include) @@ -185,8 +184,10 @@ include_directories(${CMAKE_BINARY_DIR}/include) include_directories(${CURL_INCLUDE_DIRS}) if (BUILD_AVRO) + include_directories(avro) add_subdirectory(avro) endif() + add_subdirectory(plugins) add_subdirectory(query_classifier) add_subdirectory(server) diff --git a/avro/CMakeLists.txt b/avro/CMakeLists.txt index d76785788..1e86df41d 100644 --- a/avro/CMakeLists.txt +++ b/avro/CMakeLists.txt @@ -1,8 +1,10 @@ -include_directories(${CMAKE_CURRENT_SOURCE_DIR}) -add_library(maxavro maxavro.c maxavro_schema.c maxavro_record.c maxavro_file.c) -target_link_libraries(maxavro maxscale-common jansson) +if (AVRO_FOUND) + include_directories(${CMAKE_CURRENT_SOURCE_DIR}) + add_library(maxavro maxavro.c maxavro_schema.c maxavro_record.c maxavro_file.c) + target_link_libraries(maxavro maxscale-common jansson) -add_executable(maxavrocheck maxavrocheck.c) -target_link_libraries(maxavrocheck maxavro) -install_executable(maxavrocheck core) -add_subdirectory(test) + add_executable(maxavrocheck maxavrocheck.c) + target_link_libraries(maxavrocheck maxavro) + install_executable(maxavrocheck core) + add_subdirectory(test) +endif() From 623acfe1b525b4bed732730c19820eee19067786 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 18 Oct 2016 21:50:49 +0300 Subject: [PATCH 051/215] Build MaxScale in debug mode for Travis The internal test set depends on SS_DEBUG being defined and it is only defined for debug builds. --- .travis/build_maxscale.sh | 2 +- server/core/test/testlog.c | 4 ++++ server/core/test/testserver.c | 3 ++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.travis/build_maxscale.sh b/.travis/build_maxscale.sh index 2aeda7c4f..916b8d37c 100644 --- a/.travis/build_maxscale.sh +++ b/.travis/build_maxscale.sh @@ -12,7 +12,7 @@ echo TRAVIS_BUILD_DIR: ${TRAVIS_BUILD_DIR} mkdir build cd build -cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DBUILD_TESTS=Y -DBUILD_AVRO=N +cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=Y -DBUILD_AVRO=N make make test diff --git a/server/core/test/testlog.c b/server/core/test/testlog.c index cf409f0b4..5858b24d9 100644 --- a/server/core/test/testlog.c +++ b/server/core/test/testlog.c @@ -11,6 +11,10 @@ * Public License. */ +#ifndef SS_DEBUG +#define SS_DEBUG +#endif + #include #include #include diff --git a/server/core/test/testserver.c b/server/core/test/testserver.c index d1e214de2..19d33e20b 100644 --- a/server/core/test/testserver.c +++ b/server/core/test/testserver.c @@ -50,7 +50,8 @@ test1() /* Server tests */ ss_dfprintf(stderr, "testserver : creating server called MyServer"); - server = server_alloc("MyServer", "HTTPD", 9876, "NullAuthAccept", NULL); + server = server_alloc("MyServer", "HTTPD", 9876, "NullAuthAllow", NULL); + ss_info_dassert(server, "Allocating the server should not fail"); mxs_log_flush_sync(); //ss_info_dassert(NULL != service, "New server with valid protocol and port must not be null"); From e9f94dcd9d77487e83e8b1cdd497354cbdf40004 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Tue, 18 Oct 2016 20:42:24 +0300 Subject: [PATCH 052/215] mysql_common.c and mysql_client.c astyled --- .../protocol/MySQL/MySQLClient/mysql_client.c | 238 +++++++++--------- server/modules/protocol/MySQL/mysql_common.c | 90 +++---- 2 files changed, 164 insertions(+), 164 deletions(-) diff --git a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c index ead1971b4..4cf83d10e 100644 --- a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c +++ b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c @@ -59,9 +59,9 @@ #include - /* @see function load_module in load_utils.c for explanation of the following - * lint directives. - */ +/* @see function load_module in load_utils.c for explanation of the following + * lint directives. +*/ /*lint -e14 */ MODULE_INFO info = { @@ -90,7 +90,7 @@ static void mysql_client_auth_error_handling(DCB *dcb, int auth_val); static int gw_read_do_authentication(DCB *dcb, GWBUF *read_buffer, int nbytes_read); static int gw_read_normal_data(DCB *dcb, GWBUF *read_buffer, int nbytes_read); static int gw_read_finish_processing(DCB *dcb, GWBUF *read_buffer, uint8_t capabilities); -extern char* create_auth_fail_str(char *username, char *hostaddr, char *sha1, char *db,int); +extern char* create_auth_fail_str(char *username, char *hostaddr, char *sha1, char *db, int); static bool ensure_complete_packet(DCB *dcb, GWBUF **read_buffer, int nbytes_read); static void gw_process_one_new_client(DCB *client_dcb); @@ -185,7 +185,7 @@ int MySQLSendHandshake(DCB* dcb) uint8_t mysql_scramble_len = 21; uint8_t mysql_filler_ten[10]; /* uint8_t mysql_last_byte = 0x00; not needed */ - char server_scramble[GW_MYSQL_SCRAMBLE_SIZE + 1]=""; + char server_scramble[GW_MYSQL_SCRAMBLE_SIZE + 1] = ""; char *version_string; int len_version_string = 0; int id_num; @@ -249,7 +249,7 @@ int MySQLSendHandshake(DCB* dcb) gw_mysql_set_byte3(mysql_packet_header, mysql_payload_size); // write packet number, now is 0 - mysql_packet_header[3]= mysql_packet_id; + mysql_packet_header[3] = mysql_packet_id; memcpy(outbuf, mysql_packet_header, sizeof(mysql_packet_header)); // current buffer pointer @@ -427,19 +427,19 @@ int gw_read_client_event(DCB* dcb) * will be changed to MYSQL_IDLE (see below). * */ - case MXS_AUTH_STATE_MESSAGE_READ: - /* After this call read_buffer will point to freed data */ - if (nbytes_read < 3 || (0 == max_bytes && nbytes_read < - (MYSQL_GET_PACKET_LEN((uint8_t *) GWBUF_DATA(read_buffer)) + 4)) || - (0 != max_bytes && nbytes_read < max_bytes)) - { - spinlock_acquire(&dcb->authlock); - dcb->dcb_readqueue = read_buffer; - spinlock_release(&dcb->authlock); - return 0; - } - return_code = gw_read_do_authentication(dcb, read_buffer, nbytes_read); - break; + case MXS_AUTH_STATE_MESSAGE_READ: + /* After this call read_buffer will point to freed data */ + if (nbytes_read < 3 || (0 == max_bytes && nbytes_read < + (MYSQL_GET_PACKET_LEN((uint8_t *) GWBUF_DATA(read_buffer)) + 4)) || + (0 != max_bytes && nbytes_read < max_bytes)) + { + spinlock_acquire(&dcb->authlock); + dcb->dcb_readqueue = read_buffer; + spinlock_release(&dcb->authlock); + return 0; + } + return_code = gw_read_do_authentication(dcb, read_buffer, nbytes_read); + break; /** * @@ -448,19 +448,19 @@ int gw_read_client_event(DCB* dcb) * result in a call that comes to this section of code. * */ - case MXS_AUTH_STATE_COMPLETE: - /* After this call read_buffer will point to freed data */ - return_code = gw_read_normal_data(dcb, read_buffer, nbytes_read); - break; + case MXS_AUTH_STATE_COMPLETE: + /* After this call read_buffer will point to freed data */ + return_code = gw_read_normal_data(dcb, read_buffer, nbytes_read); + break; - case MXS_AUTH_STATE_FAILED: - gwbuf_free(read_buffer); - return_code = 1; - break; + case MXS_AUTH_STATE_FAILED: + gwbuf_free(read_buffer); + return_code = 1; + break; - default: - MXS_ERROR("In mysql_client.c unexpected protocol authentication state"); - break; + default: + MXS_ERROR("In mysql_client.c unexpected protocol authentication state"); + break; } return return_code; @@ -799,7 +799,7 @@ gw_read_normal_data(DCB *dcb, GWBUF *read_buffer, int nbytes_read) /** Ask what type of input the router expects */ capabilities = session->service->router->getCapabilities( - session->service->router_instance, session->router_session); + session->service->router_instance, session->router_session); /** Update the current protocol command being executed */ if (!process_client_commands(dcb, nbytes_read, &read_buffer)) @@ -933,84 +933,84 @@ mysql_client_auth_error_handling(DCB *dcb, int auth_val) switch (auth_val) { - case MXS_AUTH_NO_SESSION: - MXS_DEBUG("%lu [gw_read_client_event] session " - "creation failed. fd %d, " - "state = MYSQL_AUTH_NO_SESSION.", - pthread_self(), - dcb->fd); + case MXS_AUTH_NO_SESSION: + MXS_DEBUG("%lu [gw_read_client_event] session " + "creation failed. fd %d, " + "state = MYSQL_AUTH_NO_SESSION.", + pthread_self(), + dcb->fd); - /** Send ERR 1045 to client */ - mysql_send_auth_error(dcb, - packet_number, - 0, - "failed to create new session"); - break; - case MXS_AUTH_FAILED_DB: - MXS_DEBUG("%lu [gw_read_client_event] database " - "specified was not valid. fd %d, " - "state = MYSQL_FAILED_AUTH_DB.", - pthread_self(), - dcb->fd); - /** Send error 1049 to client */ - message_len = 25 + MYSQL_DATABASE_MAXLEN; + /** Send ERR 1045 to client */ + mysql_send_auth_error(dcb, + packet_number, + 0, + "failed to create new session"); + break; + case MXS_AUTH_FAILED_DB: + MXS_DEBUG("%lu [gw_read_client_event] database " + "specified was not valid. fd %d, " + "state = MYSQL_FAILED_AUTH_DB.", + pthread_self(), + dcb->fd); + /** Send error 1049 to client */ + message_len = 25 + MYSQL_DATABASE_MAXLEN; - fail_str = MXS_CALLOC(1, message_len+1); - MXS_ABORT_IF_NULL(fail_str); - snprintf(fail_str, message_len, "Unknown database '%s'", - (char*)((MYSQL_session *)dcb->data)->db); + fail_str = MXS_CALLOC(1, message_len + 1); + MXS_ABORT_IF_NULL(fail_str); + snprintf(fail_str, message_len, "Unknown database '%s'", + (char*)((MYSQL_session *)dcb->data)->db); - modutil_send_mysql_err_packet(dcb, packet_number, 0, 1049, "42000", fail_str); - break; - case MXS_AUTH_FAILED_SSL: - MXS_DEBUG("%lu [gw_read_client_event] client is " - "not SSL capable for SSL listener. fd %d, " - "state = MYSQL_FAILED_AUTH_SSL.", - pthread_self(), - dcb->fd); + modutil_send_mysql_err_packet(dcb, packet_number, 0, 1049, "42000", fail_str); + break; + case MXS_AUTH_FAILED_SSL: + MXS_DEBUG("%lu [gw_read_client_event] client is " + "not SSL capable for SSL listener. fd %d, " + "state = MYSQL_FAILED_AUTH_SSL.", + pthread_self(), + dcb->fd); - /** Send ERR 1045 to client */ - mysql_send_auth_error(dcb, - packet_number, - 0, - "failed to complete SSL authentication"); - break; - case MXS_AUTH_SSL_INCOMPLETE: - MXS_DEBUG("%lu [gw_read_client_event] unable to " - "complete SSL authentication. fd %d, " - "state = MYSQL_AUTH_SSL_INCOMPLETE.", - pthread_self(), - dcb->fd); + /** Send ERR 1045 to client */ + mysql_send_auth_error(dcb, + packet_number, + 0, + "failed to complete SSL authentication"); + break; + case MXS_AUTH_SSL_INCOMPLETE: + MXS_DEBUG("%lu [gw_read_client_event] unable to " + "complete SSL authentication. fd %d, " + "state = MYSQL_AUTH_SSL_INCOMPLETE.", + pthread_self(), + dcb->fd); - /** Send ERR 1045 to client */ - mysql_send_auth_error(dcb, - packet_number, - 0, - "failed to complete SSL authentication"); - break; - case MXS_AUTH_FAILED: - MXS_DEBUG("%lu [gw_read_client_event] authentication failed. fd %d, " - "state = MYSQL_FAILED_AUTH.", - pthread_self(), - dcb->fd); - /** Send error 1045 to client */ - fail_str = create_auth_fail_str((char *)((MYSQL_session *)dcb->data)->user, - dcb->remote, - (char*)((MYSQL_session *)dcb->data)->client_sha1, - (char*)((MYSQL_session *)dcb->data)->db, auth_val); - modutil_send_mysql_err_packet(dcb, packet_number, 0, 1045, "28000", fail_str); - break; - default: - MXS_DEBUG("%lu [gw_read_client_event] authentication failed. fd %d, " - "state unrecognized.", - pthread_self(), - dcb->fd); - /** Send error 1045 to client */ - fail_str = create_auth_fail_str((char *)((MYSQL_session *)dcb->data)->user, - dcb->remote, - (char*)((MYSQL_session *)dcb->data)->client_sha1, - (char*)((MYSQL_session *)dcb->data)->db, auth_val); - modutil_send_mysql_err_packet(dcb, packet_number, 0, 1045, "28000", fail_str); + /** Send ERR 1045 to client */ + mysql_send_auth_error(dcb, + packet_number, + 0, + "failed to complete SSL authentication"); + break; + case MXS_AUTH_FAILED: + MXS_DEBUG("%lu [gw_read_client_event] authentication failed. fd %d, " + "state = MYSQL_FAILED_AUTH.", + pthread_self(), + dcb->fd); + /** Send error 1045 to client */ + fail_str = create_auth_fail_str((char *)((MYSQL_session *)dcb->data)->user, + dcb->remote, + (char*)((MYSQL_session *)dcb->data)->client_sha1, + (char*)((MYSQL_session *)dcb->data)->db, auth_val); + modutil_send_mysql_err_packet(dcb, packet_number, 0, 1045, "28000", fail_str); + break; + default: + MXS_DEBUG("%lu [gw_read_client_event] authentication failed. fd %d, " + "state unrecognized.", + pthread_self(), + dcb->fd); + /** Send error 1045 to client */ + fail_str = create_auth_fail_str((char *)((MYSQL_session *)dcb->data)->user, + dcb->remote, + (char*)((MYSQL_session *)dcb->data)->client_sha1, + (char*)((MYSQL_session *)dcb->data)->db, auth_val); + modutil_send_mysql_err_packet(dcb, packet_number, 0, 1045, "28000", fail_str); } MXS_FREE(fail_str); } @@ -1145,8 +1145,8 @@ static void gw_process_one_new_client(DCB *client_dcb) /** delete client_dcb */ dcb_close(client_dcb); MXS_ERROR("%lu [gw_MySQLAccept] Failed to create " - "protocol object for client connection.", - pthread_self()); + "protocol object for client connection.", + pthread_self()); return; } CHK_PROTOCOL(protocol); @@ -1174,29 +1174,29 @@ static void gw_process_one_new_client(DCB *client_dcb) { /* Send a custom error as MySQL command reply */ mysql_send_custom_error(client_dcb, - 1, - 0, - "MaxScale encountered system limit while " - "attempting to register on an epoll instance."); + 1, + 0, + "MaxScale encountered system limit while " + "attempting to register on an epoll instance."); /** close client_dcb */ dcb_close(client_dcb); /** Previous state is recovered in poll_add_dcb. */ MXS_ERROR("%lu [gw_MySQLAccept] Failed to add dcb %p for " - "fd %d to epoll set.", - pthread_self(), - client_dcb, - client_dcb->fd); + "fd %d to epoll set.", + pthread_self(), + client_dcb, + client_dcb->fd); return; } else { MXS_DEBUG("%lu [gw_MySQLAccept] Added dcb %p for fd " - "%d to epoll set.", - pthread_self(), - client_dcb, - client_dcb->fd); + "%d to epoll set.", + pthread_self(), + client_dcb, + client_dcb->fd); } return; } @@ -1342,7 +1342,7 @@ static int route_by_statement(SESSION* session, GWBUF** p_readbuf) while (tmpbuf != NULL) { ss_dassert(GWBUF_IS_TYPE_MYSQL(tmpbuf)); - tmpbuf=tmpbuf->next; + tmpbuf = tmpbuf->next; } #endif do diff --git a/server/modules/protocol/MySQL/mysql_common.c b/server/modules/protocol/MySQL/mysql_common.c index 9e3636579..bd471a26a 100644 --- a/server/modules/protocol/MySQL/mysql_common.c +++ b/server/modules/protocol/MySQL/mysql_common.c @@ -160,24 +160,24 @@ retblock: */ const char* gw_mysql_protocol_state2string (int state) { - switch(state) + switch (state) { - case MXS_AUTH_STATE_INIT: - return "Authentication initialized"; - case MXS_AUTH_STATE_PENDING_CONNECT: - return "Network connection pending"; - case MXS_AUTH_STATE_CONNECTED: - return "Network connection created"; - case MXS_AUTH_STATE_MESSAGE_READ: - return "Read server handshake"; - case MXS_AUTH_STATE_RESPONSE_SENT: - return "Response to handshake sent"; - case MXS_AUTH_STATE_FAILED: - return "Authentication failed"; - case MXS_AUTH_STATE_COMPLETE: - return "Authentication is complete."; - default: - return "MySQL (unknown protocol state)"; + case MXS_AUTH_STATE_INIT: + return "Authentication initialized"; + case MXS_AUTH_STATE_PENDING_CONNECT: + return "Network connection pending"; + case MXS_AUTH_STATE_CONNECTED: + return "Network connection created"; + case MXS_AUTH_STATE_MESSAGE_READ: + return "Read server handshake"; + case MXS_AUTH_STATE_RESPONSE_SENT: + return "Response to handshake sent"; + case MXS_AUTH_STATE_FAILED: + return "Authentication failed"; + case MXS_AUTH_STATE_COMPLETE: + return "Authentication is complete."; + default: + return "MySQL (unknown protocol state)"; } } @@ -267,7 +267,7 @@ GWBUF* mysql_create_custom_error(int packet_number, field_count = 0xff; gw_mysql_set_byte2(mysql_err, /* mysql_errno */ 2003); - mysql_statemsg[0]='#'; + mysql_statemsg[0] = '#'; memcpy(mysql_statemsg + 1, mysql_state, 5); if (msg != NULL) @@ -333,8 +333,8 @@ GWBUF* mysql_create_custom_error(int packet_number, */ GWBUF * mysql_create_standard_error(int packet_number, - int error_number, - const char *error_message) + int error_number, + const char *error_message) { uint8_t *outbuf = NULL; uint32_t mysql_payload_size = 0; @@ -356,7 +356,7 @@ mysql_create_standard_error(int packet_number, gw_mysql_set_byte3(mysql_packet_header, mysql_payload_size); // write packet number, now is 0 - mysql_packet_header[3]= 0; + mysql_packet_header[3] = 0; memcpy(outbuf, mysql_packet_header, sizeof(mysql_packet_header)); // current buffer pointer @@ -392,9 +392,9 @@ mysql_create_standard_error(int packet_number, */ int mysql_send_standard_error(DCB *dcb, - int packet_number, - int error_number, - const char *error_message) + int packet_number, + int error_number, + const char *error_message) { GWBUF *buf; buf = mysql_create_standard_error(packet_number, error_number, error_message); @@ -469,7 +469,7 @@ int mysql_send_auth_error(DCB *dcb, field_count = 0xff; gw_mysql_set_byte2(mysql_err, /*mysql_errno */ 1045); - mysql_statemsg[0]='#'; + mysql_statemsg[0] = '#'; memcpy(mysql_statemsg + 1, mysql_state, 5); if (mysql_message != NULL) @@ -837,27 +837,27 @@ void init_response_status(GWBUF* buf, { switch (cmd) { - case MYSQL_COM_STMT_PREPARE: - gwbuf_copy_data(buf, 9, 2, readbuf); - nparam = gw_mysql_get_byte2(readbuf); - gwbuf_copy_data(buf, 11, 2, readbuf); - nattr = gw_mysql_get_byte2(readbuf); - *npackets = 1 + nparam + MXS_MIN(1, nparam) + nattr + MXS_MIN(nattr, 1); - break; + case MYSQL_COM_STMT_PREPARE: + gwbuf_copy_data(buf, 9, 2, readbuf); + nparam = gw_mysql_get_byte2(readbuf); + gwbuf_copy_data(buf, 11, 2, readbuf); + nattr = gw_mysql_get_byte2(readbuf); + *npackets = 1 + nparam + MXS_MIN(1, nparam) + nattr + MXS_MIN(nattr, 1); + break; - case MYSQL_COM_QUIT: - case MYSQL_COM_STMT_SEND_LONG_DATA: - case MYSQL_COM_STMT_CLOSE: - *npackets = 0; /*< these don't reply anything */ - break; + case MYSQL_COM_QUIT: + case MYSQL_COM_STMT_SEND_LONG_DATA: + case MYSQL_COM_STMT_CLOSE: + *npackets = 0; /*< these don't reply anything */ + break; - default: - /** - * assume that other session commands respond - * OK or ERR - */ - *npackets = 1; - break; + default: + /** + * assume that other session commands respond + * OK or ERR + */ + *npackets = 1; + break; } } @@ -923,7 +923,7 @@ char* create_auth_failed_msg(GWBUF*readbuf, uint8_t* sha1) { char* errstr; - char* uname=(char *)GWBUF_DATA(readbuf) + 5; + char* uname = (char *)GWBUF_DATA(readbuf) + 5; const char* ferrstr = "Access denied for user '%s'@'%s' (using password: %s)"; /** -4 comes from 2X'%s' minus terminating char */ From 48b1583830c11e5866422d4df82911029090f31e Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Sun, 9 Oct 2016 13:20:31 +0300 Subject: [PATCH 053/215] Add FindJansson.cmake Added find_package module for CMake which locates the Jansson libraries and headers. This will make dependency checking easier and prevents build failures due to missing dependencies. --- CMakeLists.txt | 1 + avro/CMakeLists.txt | 2 +- cmake/FindJansson.cmake | 16 ++++++++++++++++ server/modules/filter/cache/CMakeLists.txt | 20 ++++++++++++-------- server/modules/routing/avro/CMakeLists.txt | 7 ++++--- 5 files changed, 34 insertions(+), 12 deletions(-) create mode 100644 cmake/FindJansson.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 6f5e3f833..2ed799c0d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -41,6 +41,7 @@ find_package(Git) find_package(CURL) find_package(RabbitMQ) find_package(LibUUID) +find_package(Jansson) find_package(Avro) find_package(GSSAPI) diff --git a/avro/CMakeLists.txt b/avro/CMakeLists.txt index 1e86df41d..cb6df2eed 100644 --- a/avro/CMakeLists.txt +++ b/avro/CMakeLists.txt @@ -1,4 +1,4 @@ -if (AVRO_FOUND) +if (AVRO_FOUND AND JANSSON_FOUND) include_directories(${CMAKE_CURRENT_SOURCE_DIR}) add_library(maxavro maxavro.c maxavro_schema.c maxavro_record.c maxavro_file.c) target_link_libraries(maxavro maxscale-common jansson) diff --git a/cmake/FindJansson.cmake b/cmake/FindJansson.cmake new file mode 100644 index 000000000..c0322975e --- /dev/null +++ b/cmake/FindJansson.cmake @@ -0,0 +1,16 @@ +# This CMake file locates the Jansson libraries and headers +# +# The following variables are set: +# JANSSON_FOUND - If the Avro C library was found +# JANSSON_LIBRARIES - Path to the static library +# JANSSON_INCLUDE_DIR - Path to Avro headers + +find_path(JANSSON_INCLUDE_DIR jansson.h) +find_library(JANSSON_LIBRARIES NAMES libjansson.a libjansson.so) + +if (JANSSON_INCLUDE_DIR AND JANSSON_LIBRARIES) + message(STATUS "Found Jansson: ${JANSSON_LIBRARIES}") + set(JANSSON_FOUND TRUE) +else() + message(STATUS "Could not find Jansson") +endif() diff --git a/server/modules/filter/cache/CMakeLists.txt b/server/modules/filter/cache/CMakeLists.txt index 89db9c15b..3977cfbb0 100644 --- a/server/modules/filter/cache/CMakeLists.txt +++ b/server/modules/filter/cache/CMakeLists.txt @@ -1,11 +1,15 @@ -add_library(cache SHARED cache.c rules.c storage.c) -target_link_libraries(cache maxscale-common jansson) -set_target_properties(cache PROPERTIES VERSION "1.0.0") -set_target_properties(cache PROPERTIES LINK_FLAGS -Wl,-z,defs) -install_module(cache experimental) +if (JANSSON_FOUND) + add_library(cache SHARED cache.c rules.c storage.c) + target_link_libraries(cache maxscale-common jansson) + set_target_properties(cache PROPERTIES VERSION "1.0.0") + set_target_properties(cache PROPERTIES LINK_FLAGS -Wl,-z,defs) + install_module(cache experimental) -add_subdirectory(storage) + add_subdirectory(storage) -if(BUILD_TESTS) - add_subdirectory(test) + if(BUILD_TESTS) + add_subdirectory(test) + endif() +else() + message(STATUS "No Jansson libraries found, not building cache filter.") endif() diff --git a/server/modules/routing/avro/CMakeLists.txt b/server/modules/routing/avro/CMakeLists.txt index 4fcb896b6..0fed40b78 100644 --- a/server/modules/routing/avro/CMakeLists.txt +++ b/server/modules/routing/avro/CMakeLists.txt @@ -1,10 +1,11 @@ -if(AVRO_FOUND) +if(AVRO_FOUND AND JANSSON_FOUND) include_directories(${AVRO_INCLUDE_DIR}) + include_directories(${JANSSON_INCLUDE_DIR}) add_library(avrorouter SHARED avro.c ../binlog/binlog_common.c avro_client.c avro_schema.c avro_rbr.c avro_file.c avro_index.c) set_target_properties(avrorouter PROPERTIES VERSION "1.0.0") set_target_properties(avrorouter PROPERTIES LINK_FLAGS -Wl,-z,defs) - target_link_libraries(avrorouter maxscale-common jansson ${AVRO_LIBRARIES} maxavro sqlite3 lzma) + target_link_libraries(avrorouter maxscale-common ${JANSSON_LIBRARIES} ${AVRO_LIBRARIES} maxavro sqlite3 lzma) install_module(avrorouter core) else() - message(STATUS "Avro C libraries were not found, avrorouter will not be built.") + message(STATUS "No Avro C or Jansson libraries found, not building avrorouter.") endif() From 348143eb6c82043ad824e6dcc31ba1886176ce1c Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Wed, 19 Oct 2016 11:00:26 +0300 Subject: [PATCH 054/215] Remove too aggressive assert --- server/modules/filter/cache/cache.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/modules/filter/cache/cache.c b/server/modules/filter/cache/cache.c index d0d76c0cd..15e9aa9dc 100644 --- a/server/modules/filter/cache/cache.c +++ b/server/modules/filter/cache/cache.c @@ -621,7 +621,10 @@ static void cache_session_data_free(CACHE_SESSION_DATA* data) { if (data) { - ss_dassert(!data->use_db); + // In normal circumstances, only data->default_db may be non-NULL at + // this point. However, if the authentication with the backend fails + // and the session is closed, data->use_db may be non-NULL. + MXS_FREE(data->use_db); MXS_FREE(data->default_db); MXS_FREE(data); } From 21094fb0cb4faf2735cad8402981820a41f80f16 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Wed, 19 Oct 2016 11:32:02 +0300 Subject: [PATCH 055/215] Prefer so-version of Jansson library --- cmake/FindJansson.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/FindJansson.cmake b/cmake/FindJansson.cmake index c0322975e..7116d6f0f 100644 --- a/cmake/FindJansson.cmake +++ b/cmake/FindJansson.cmake @@ -6,7 +6,7 @@ # JANSSON_INCLUDE_DIR - Path to Avro headers find_path(JANSSON_INCLUDE_DIR jansson.h) -find_library(JANSSON_LIBRARIES NAMES libjansson.a libjansson.so) +find_library(JANSSON_LIBRARIES NAMES libjansson.so libjansson.a) if (JANSSON_INCLUDE_DIR AND JANSSON_LIBRARIES) message(STATUS "Found Jansson: ${JANSSON_LIBRARIES}") From d938f0e72fb9238afdcd7c2bb144a0c0e8e4c87e Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 18 Oct 2016 10:40:06 +0300 Subject: [PATCH 056/215] Add custom cache directory for MySQLAuth The MySQLAuth authenticator can now store the cached credentials in a custom location. This is intended for the binlogrouter, which currently uses a different location for the user cache. The MySQL authentication modules now have their own document in the Authenticators directory. Right now this document only describes the general details of the MySQL authentication and the new option. --- .../Authenticators/MySQL-Authenticator.md | 34 ++++++++ server/modules/authenticator/mysql_auth.c | 84 +++++++++++++++++-- 2 files changed, 109 insertions(+), 9 deletions(-) create mode 100644 Documentation/Authenticators/MySQL-Authenticator.md diff --git a/Documentation/Authenticators/MySQL-Authenticator.md b/Documentation/Authenticators/MySQL-Authenticator.md new file mode 100644 index 000000000..aee19d778 --- /dev/null +++ b/Documentation/Authenticators/MySQL-Authenticator.md @@ -0,0 +1,34 @@ +# MySQL Authenticator + +The _MySQLAuth_ and _MySQLBackendAuth_ modules implement the client and +backend authentication for the MySQL native password authentication. This +is the default authentication plugin used by both MariaDB and MySQL. + +These modules are the default authenticators for all MySQL connections and +needs no further configuration to work. + +## Authenticator options + +The client authentication module, _MySQLAuth_, supports authenticator options. + +### `cache_dir` + +The location where the user credential cache is stored. The default value +for this is `///cache/` where +`` by default is `/var/cache`. + +Each listener has its own user cache where the user credential information +queried from the backends is stored. This information is used to +authenticate users if a connection to the backend servers can't be made. + +#### Example configuration + +``` +[Read-Write Listener] +type=listener +service=Read-Write Service +protocol=MySQLClient +port=4006 +authenticator=MySQLAuth +authenticator_options=cache_dir=/tmp +``` diff --git a/server/modules/authenticator/mysql_auth.c b/server/modules/authenticator/mysql_auth.c index 6e5710736..9e0cb9214 100644 --- a/server/modules/authenticator/mysql_auth.c +++ b/server/modules/authenticator/mysql_auth.c @@ -35,6 +35,12 @@ #include #include +typedef struct mysql_auth +{ + char *cache_dir; /**< Custom cache directory location */ +} MYSQL_AUTH; + + /* @see function load_module in load_utils.c for explanation of the following * lint directives. */ @@ -50,6 +56,7 @@ MODULE_INFO info = static char *version_str = "V1.1.0"; +static void* mysql_auth_init(char **options); static int mysql_auth_set_protocol_data(DCB *dcb, GWBUF *buf); static bool mysql_auth_is_client_ssl_capable(DCB *dcb); static int mysql_auth_authenticate(DCB *dcb); @@ -61,7 +68,7 @@ static int mysql_auth_load_users(SERV_LISTENER *port); */ static GWAUTHENTICATOR MyObject = { - NULL, /* No initialize entry point */ + mysql_auth_init, /* Initialize the authenticator */ NULL, /* No create entry point */ mysql_auth_set_protocol_data, /* Extract data into structure */ mysql_auth_is_client_ssl_capable, /* Check if client supports SSL */ @@ -121,6 +128,60 @@ GWAUTHENTICATOR* GetModuleObject() } /*lint +e14 */ +/** + * @brief Initialize the authenticator instance + * + * @param options Authenticator options + * @return New MYSQL_AUTH instance or NULL on error + */ +static void* mysql_auth_init(char **options) +{ + MYSQL_AUTH *instance = MXS_MALLOC(sizeof(*instance)); + + if (instance) + { + bool error = false; + instance->cache_dir = NULL; + + for (int i = 0; options[i]; i++) + { + char *value = strchr(options[i], '='); + + if (value) + { + *value++ = '\0'; + + if (strcmp(options[i], "cache_dir") == 0) + { + if ((instance->cache_dir = MXS_STRDUP(value)) == NULL) + { + error = true; + } + } + else + { + MXS_ERROR("Unknown authenticator option: %s", options[i]); + error = true; + } + } + else + { + MXS_ERROR("Unknown authenticator option: %s", options[i]); + error = true; + } + } + + if (error) + { + MXS_FREE(instance->cache_dir); + MXS_FREE(instance); + instance = NULL; + } + } + + return instance; +} + /** * @brief Authenticates a MySQL user who is a client to MaxScale. * @@ -781,21 +842,29 @@ static int mysql_auth_load_users(SERV_LISTENER *port) { int rc = MXS_AUTH_LOADUSERS_OK; SERVICE *service = port->listener->service; + MYSQL_AUTH *instance = (MYSQL_AUTH*)port->auth_instance; int loaded = replace_mysql_users(port); + char path[PATH_MAX]; + + if (instance->cache_dir) + { + strcpy(path, instance->cache_dir); + } + else + { + sprintf(path, "%s/%s/%s/%s/", get_cachedir(), service->name, port->name, DBUSERS_DIR); + } if (loaded < 0) { MXS_ERROR("[%s] Unable to load users for listener %s listening at %s:%d.", service->name, port->name, port->address ? port->address : "0.0.0.0", port->port); - /* Try loading authentication data from file cache */ - char path[PATH_MAX]; - sprintf(path, "%s/%s/%s/%s/%s", get_cachedir(), service->name, port->name, - DBUSERS_DIR, DBUSERS_FILE); + strcat(path, DBUSERS_FILE); if ((loaded = dbusers_load(port->users, path)) == -1) { - MXS_ERROR("[%s] Failed to load cached users from '%s'.", service->name, path);; + MXS_ERROR("[%s] Failed to load cached users from '%s'.", service->name, path); rc = MXS_AUTH_LOADUSERS_ERROR; } else @@ -806,9 +875,6 @@ static int mysql_auth_load_users(SERV_LISTENER *port) else { /* Users loaded successfully, save authentication data to file cache */ - char path[PATH_MAX]; - sprintf(path, "%s/%s/%s/%s/", get_cachedir(), service->name, port->name, DBUSERS_DIR); - if (mxs_mkdir_all(path, 0777)) { strcat(path, DBUSERS_FILE); From 6bcb3ce4dc71de60c73d5e5bcd42e81e19932e9e Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Thu, 20 Oct 2016 17:47:00 +0300 Subject: [PATCH 057/215] Change capabilities to uint64_t - Since the capabilities are a bitmask, it is better if an unsigned integral type is used. - Since the function is part of an ABI, it is better if an explicit size is used. - 64-bits so that there also is room for independent filter capabilities. --- include/maxscale/router.h | 4 ++-- .../protocol/MySQL/MySQLClient/mysql_client.c | 12 ++++++------ server/modules/routing/avro/avro.c | 4 ++-- server/modules/routing/binlog/blr.c | 6 +++--- server/modules/routing/cli/cli.c | 4 ++-- server/modules/routing/debugcli/debugcli.c | 4 ++-- server/modules/routing/maxinfo/maxinfo.c | 4 ++-- .../modules/routing/readconnroute/readconnection.h | 1 - server/modules/routing/readconnroute/readconnroute.c | 6 ++---- .../modules/routing/readwritesplit/readwritesplit.c | 10 +++++----- server/modules/routing/schemarouter/schemarouter.c | 4 ++-- server/modules/routing/testroute/testroute.c | 4 ++-- server/modules/routing/webserver/webserver.c | 4 ++-- 13 files changed, 32 insertions(+), 35 deletions(-) diff --git a/include/maxscale/router.h b/include/maxscale/router.h index 9172074b4..91c363b9b 100644 --- a/include/maxscale/router.h +++ b/include/maxscale/router.h @@ -82,7 +82,7 @@ typedef struct router_object DCB* backend_dcb, error_action_t action, bool* succp); - int (*getCapabilities)(); + uint64_t (*getCapabilities)(); } ROUTER_OBJECT; /** @@ -90,7 +90,7 @@ typedef struct router_object * must update these versions numbers in accordance with the rules in * modinfo.h. */ -#define ROUTER_VERSION { 1, 0, 0 } +#define ROUTER_VERSION { 2, 0, 0 } /** * Router capability type. Indicates what kind of input router accepts. diff --git a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c index 4cf83d10e..7aa63f422 100644 --- a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c +++ b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c @@ -89,7 +89,7 @@ static int route_by_statement(SESSION *, GWBUF **); static void mysql_client_auth_error_handling(DCB *dcb, int auth_val); static int gw_read_do_authentication(DCB *dcb, GWBUF *read_buffer, int nbytes_read); static int gw_read_normal_data(DCB *dcb, GWBUF *read_buffer, int nbytes_read); -static int gw_read_finish_processing(DCB *dcb, GWBUF *read_buffer, uint8_t capabilities); +static int gw_read_finish_processing(DCB *dcb, GWBUF *read_buffer, uint64_t capabilities); extern char* create_auth_fail_str(char *username, char *hostaddr, char *sha1, char *db, int); static bool ensure_complete_packet(DCB *dcb, GWBUF **read_buffer, int nbytes_read); static void gw_process_one_new_client(DCB *client_dcb); @@ -780,7 +780,7 @@ gw_read_normal_data(DCB *dcb, GWBUF *read_buffer, int nbytes_read) { SESSION *session; session_state_t session_state_value; - uint8_t capabilities = 0; + uint64_t capabilities = 0; session = dcb->session; CHK_SESSION(session); @@ -809,7 +809,7 @@ gw_read_normal_data(DCB *dcb, GWBUF *read_buffer, int nbytes_read) /** If the router requires statement input or we are still authenticating * we need to make sure that a complete SQL packet is read before continuing */ - if (capabilities & (int)RCAP_TYPE_STMT_INPUT) + if (capabilities & RCAP_TYPE_STMT_INPUT) { uint8_t* data; int packet_size; @@ -836,7 +836,7 @@ gw_read_normal_data(DCB *dcb, GWBUF *read_buffer, int nbytes_read) * @return 0 if succeed, 1 otherwise */ static int -gw_read_finish_processing(DCB *dcb, GWBUF *read_buffer, uint8_t capabilities) +gw_read_finish_processing(DCB *dcb, GWBUF *read_buffer, uint64_t capabilities) { SESSION *session = dcb->session; uint8_t *payload = GWBUF_DATA(read_buffer); @@ -847,7 +847,7 @@ gw_read_finish_processing(DCB *dcb, GWBUF *read_buffer, uint8_t capabilities) /** Reset error handler when routing of the new query begins */ dcb->dcb_errhandle_called = false; - if (capabilities & (int)RCAP_TYPE_STMT_INPUT) + if (capabilities & RCAP_TYPE_STMT_INPUT) { /** * Feed each statement completely and separately @@ -865,7 +865,7 @@ gw_read_finish_processing(DCB *dcb, GWBUF *read_buffer, uint8_t capabilities) spinlock_release(&dcb->authlock); } } - else if (NULL != session->router_session || (capabilities & (int)RCAP_TYPE_NO_RSESSION)) + else if (NULL != session->router_session || (capabilities & RCAP_TYPE_NO_RSESSION)) { /** Feed whole packet to router, which will free it * and return 1 for success, 0 for failure diff --git a/server/modules/routing/avro/avro.c b/server/modules/routing/avro/avro.c index 122e3fa38..657fbcbec 100644 --- a/server/modules/routing/avro/avro.c +++ b/server/modules/routing/avro/avro.c @@ -79,7 +79,7 @@ static void clientReply(ROUTER *instance, void *router_session, GWBUF *queue, DCB *backend_dcb); static void errorReply(ROUTER *instance, void *router_session, GWBUF *message, DCB *backend_dcb, error_action_t action, bool *succp); -static int getCapabilities(); +static uint64_t getCapabilities(); extern int MaxScaleUptime(); extern void avro_get_used_tables(AVRO_INSTANCE *router, DCB *dcb); void converter_func(void* data); @@ -955,7 +955,7 @@ errorReply(ROUTER *instance, void *router_session, GWBUF *message, DCB *backend_ ss_dassert(false); } -static int getCapabilities() +static uint64_t getCapabilities() { return RCAP_TYPE_NO_RSESSION; } diff --git a/server/modules/routing/binlog/blr.c b/server/modules/routing/binlog/blr.c index 5a978e925..47909e625 100644 --- a/server/modules/routing/binlog/blr.c +++ b/server/modules/routing/binlog/blr.c @@ -100,7 +100,7 @@ static void errorReply(ROUTER *instance, error_action_t action, bool *succp); -static int getCapabilities(); +static uint64_t getCapabilities(); static int blr_handler_config(void *userdata, const char *section, const char *name, const char *value); static int blr_handle_config_item(const char *name, const char *value, ROUTER_INSTANCE *inst); static int blr_set_service_mysql_user(SERVICE *service); @@ -1788,9 +1788,9 @@ static void rses_end_locked_router_action(ROUTER_SLAVE *rses) } -static int getCapabilities() +static uint64_t getCapabilities() { - return (int)(RCAP_TYPE_NO_RSESSION | RCAP_TYPE_NO_USERS_INIT); + return (RCAP_TYPE_NO_RSESSION | RCAP_TYPE_NO_USERS_INIT); } /** diff --git a/server/modules/routing/cli/cli.c b/server/modules/routing/cli/cli.c index c59f570a4..ff7deff9f 100644 --- a/server/modules/routing/cli/cli.c +++ b/server/modules/routing/cli/cli.c @@ -58,7 +58,7 @@ static void closeSession(ROUTER *instance, void *router_session); static void freeSession(ROUTER *instance, void *router_session); static int execute(ROUTER *instance, void *router_session, GWBUF *queue); static void diagnostics(ROUTER *instance, DCB *dcb); -static int getCapabilities(); +static uint64_t getCapabilities(); /** The module object definition */ static ROUTER_OBJECT MyObject = @@ -302,7 +302,7 @@ diagnostics(ROUTER *instance, DCB *dcb) return; /* Nothing to do currently */ } -static int getCapabilities() +static uint64_t getCapabilities() { return 0; } diff --git a/server/modules/routing/debugcli/debugcli.c b/server/modules/routing/debugcli/debugcli.c index eb1bd290f..29e8a2fc3 100644 --- a/server/modules/routing/debugcli/debugcli.c +++ b/server/modules/routing/debugcli/debugcli.c @@ -57,7 +57,7 @@ static void closeSession(ROUTER *instance, void *router_session); static void freeSession(ROUTER *instance, void *router_session); static int execute(ROUTER *instance, void *router_session, GWBUF *queue); static void diagnostics(ROUTER *instance, DCB *dcb); -static int getCapabilities (); +static uint64_t getCapabilities (); /** The module object definition */ static ROUTER_OBJECT MyObject = @@ -333,7 +333,7 @@ diagnostics(ROUTER *instance, DCB *dcb) return; /* Nothing to do currently */ } -static int getCapabilities() +static uint64_t getCapabilities() { return 0; } diff --git a/server/modules/routing/maxinfo/maxinfo.c b/server/modules/routing/maxinfo/maxinfo.c index 46114b2f7..574cd8418 100644 --- a/server/modules/routing/maxinfo/maxinfo.c +++ b/server/modules/routing/maxinfo/maxinfo.c @@ -79,7 +79,7 @@ static void closeSession(ROUTER *instance, void *router_session); static void freeSession(ROUTER *instance, void *router_session); static int execute(ROUTER *instance, void *router_session, GWBUF *queue); static void diagnostics(ROUTER *instance, DCB *dcb); -static int getCapabilities(); +static uint64_t getCapabilities(); static void handleError(ROUTER *instance, void *router_session, GWBUF *errbuf, @@ -409,7 +409,7 @@ diagnostics(ROUTER *instance, DCB *dcb) * * Not used for the maxinfo router */ -static int +static uint64_t getCapabilities() { return 0; diff --git a/server/modules/routing/readconnroute/readconnection.h b/server/modules/routing/readconnroute/readconnection.h index e832b9c2e..13b4fa2ad 100644 --- a/server/modules/routing/readconnroute/readconnection.h +++ b/server/modules/routing/readconnroute/readconnection.h @@ -59,7 +59,6 @@ typedef struct router_client_session DCB *backend_dcb; /*< DCB Connection to the backend */ DCB *client_dcb; /**< Client DCB */ struct router_client_session *next; - int rses_capabilities; /*< input type, for example */ #if defined(SS_DEBUG) skygw_chk_t rses_chk_tail; #endif diff --git a/server/modules/routing/readconnroute/readconnroute.c b/server/modules/routing/readconnroute/readconnroute.c index f95a24cf0..a9f3a27c9 100644 --- a/server/modules/routing/readconnroute/readconnroute.c +++ b/server/modules/routing/readconnroute/readconnroute.c @@ -111,7 +111,7 @@ static void clientReply(ROUTER *instance, void *router_session, GWBUF *queue, DCB *backend_dcb); static void handleError(ROUTER *instance, void *router_session, GWBUF *errbuf, DCB *problem_dcb, error_action_t action, bool *succp); -static int getCapabilities(); +static uint64_t getCapabilities(); /** The module object definition */ @@ -561,8 +561,6 @@ newSession(ROUTER *instance, SESSION *session) } } - client_rses->rses_capabilities = RCAP_TYPE_PACKET_INPUT; - /* * We now have the server with the least connections. * Bump the connection count for this server @@ -987,7 +985,7 @@ static void rses_end_locked_router_action(ROUTER_CLIENT_SES* rses) spinlock_release(&rses->rses_lock); } -static int getCapabilities() +static uint64_t getCapabilities() { return RCAP_TYPE_PACKET_INPUT; } diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index 8ad2c34d5..67a566a13 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -82,7 +82,7 @@ static void clientReply(ROUTER *instance, void *router_session, GWBUF *queue, static void handleError(ROUTER *instance, void *router_session, GWBUF *errmsgbuf, DCB *backend_dcb, error_action_t action, bool *succp); -static int getCapabilities(); +static uint64_t getCapabilities(); /* * End of the API functions; now the module structure that links to them. @@ -1055,14 +1055,14 @@ lock_failed: /** * @brief Get router capabilities (API) - * + * * Return a bit map indicating the characteristics of this particular router. * In this case, the only bit set indicates that the router wants to receive * data for routing as whole SQL statements. - * - * @return int RCAP_TYPE_STMT_INPUT. + * + * @return RCAP_TYPE_STMT_INPUT. */ -static int getCapabilities() +static uint64_t getCapabilities() { return RCAP_TYPE_STMT_INPUT; } diff --git a/server/modules/routing/schemarouter/schemarouter.c b/server/modules/routing/schemarouter/schemarouter.c index 95e544a3c..bead571f8 100644 --- a/server/modules/routing/schemarouter/schemarouter.c +++ b/server/modules/routing/schemarouter/schemarouter.c @@ -92,7 +92,7 @@ static route_target_t get_shard_route_target(qc_query_type_t qtype, bool trx_active, HINT* hint); -static int getCapabilities(); +static uint64_t getCapabilities(); static bool connect_backend_servers(backend_ref_t* backend_ref, int router_nservers, @@ -3642,7 +3642,7 @@ static void tracelog_routed_query(ROUTER_CLIENT_SES* rses, /** * Return RCAP_TYPE_STMT_INPUT. */ -static int getCapabilities() +static uint64_t getCapabilities() { return RCAP_TYPE_STMT_INPUT; } diff --git a/server/modules/routing/testroute/testroute.c b/server/modules/routing/testroute/testroute.c index 3fb823ec1..3533686f6 100644 --- a/server/modules/routing/testroute/testroute.c +++ b/server/modules/routing/testroute/testroute.c @@ -32,7 +32,7 @@ static void freeSession(ROUTER *instance, void *session); static int routeQuery(ROUTER *instance, void *session, GWBUF *queue); static void clientReply(ROUTER *instance, void *session, GWBUF *queue, DCB*); static void diagnostic(ROUTER *instance, DCB *dcb); -static int getCapabilities (); +static uint64_t getCapabilities (); static void handleError(ROUTER *instance, void *router_session, GWBUF *errbuf, @@ -164,7 +164,7 @@ diagnostic(ROUTER *instance, DCB *dcb) { } -static int getCapabilities() +static uint64_t getCapabilities() { return 0; } diff --git a/server/modules/routing/webserver/webserver.c b/server/modules/routing/webserver/webserver.c index eceb4e94b..c9889ba77 100644 --- a/server/modules/routing/webserver/webserver.c +++ b/server/modules/routing/webserver/webserver.c @@ -51,7 +51,7 @@ static void closeSession(ROUTER *instance, void *session); static void freeSession(ROUTER *instance, void *session); static int routeQuery(ROUTER *instance, void *session, GWBUF *queue); static void diagnostic(ROUTER *instance, DCB *dcb); -static uint8_t getCapabilities(ROUTER* inst, void* router_session); +static uint64_t getCapabilities(ROUTER* inst, void* router_session); static ROUTER_OBJECT MyObject = @@ -257,7 +257,7 @@ diagnostic(ROUTER *instance, DCB *dcb) * @param router_session The router session * @return Router capabilities bitmask */ -static uint8_t +static uint64_t getCapabilities(ROUTER *inst, void *router_session) { return 0; From e1ccc8afe0ca964d06a38f554e55ec9971b1031f Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Wed, 19 Oct 2016 22:49:03 +0300 Subject: [PATCH 058/215] MXS-938: Recognize transaction start Now more information about a transaction start is provided. When a transaction start statement is parsed, the type of the statement with be QUERY_TYPE_BEGIN_TRX anded with QUERY_TYPE_READ or QUERY_TYPE_WRITE if the transaction was explicitly started as READ ONLY or READ WRITE. Now also BEGIN WORK and [COMMIT|ROLLBACK] WORK are recognized. "AND CHAIN" will still need to be recognized. --- .../qc_mysqlembedded/qc_mysqlembedded.cc | 8 +++ query_classifier/qc_sqlite/qc_sqlite.c | 2 +- .../qc_sqlite/sqlite-src-3110100/src/parse.y | 57 ++++++++++++++++--- query_classifier/test/maxscale.test | 18 +++++- 4 files changed, 76 insertions(+), 9 deletions(-) diff --git a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc index 9baaee794..285692879 100644 --- a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc +++ b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc @@ -630,6 +630,14 @@ static uint32_t resolve_query_type(THD* thd) case SQLCOM_BEGIN: type |= QUERY_TYPE_BEGIN_TRX; + if (lex->start_transaction_opt & MYSQL_START_TRANS_OPT_READ_WRITE) + { + type |= QUERY_TYPE_WRITE; + } + else if (lex->start_transaction_opt & MYSQL_START_TRANS_OPT_READ_ONLY) + { + type |= QUERY_TYPE_READ; + } goto return_qtype; break; diff --git a/query_classifier/qc_sqlite/qc_sqlite.c b/query_classifier/qc_sqlite/qc_sqlite.c index 598bc3c2c..942e01a6f 100644 --- a/query_classifier/qc_sqlite/qc_sqlite.c +++ b/query_classifier/qc_sqlite/qc_sqlite.c @@ -968,7 +968,7 @@ void mxs_sqlite3BeginTransaction(Parse* pParse, int type) ss_dassert(info); info->status = QC_QUERY_PARSED; - info->types = QUERY_TYPE_BEGIN_TRX; + info->types = QUERY_TYPE_BEGIN_TRX | type; } void mxs_sqlite3BeginTrigger(Parse *pParse, /* The parse context of the CREATE TRIGGER statement */ diff --git a/query_classifier/qc_sqlite/sqlite-src-3110100/src/parse.y b/query_classifier/qc_sqlite/sqlite-src-3110100/src/parse.y index 2bff7038f..8abf935f8 100644 --- a/query_classifier/qc_sqlite/sqlite-src-3110100/src/parse.y +++ b/query_classifier/qc_sqlite/sqlite-src-3110100/src/parse.y @@ -59,6 +59,13 @@ %include { #include "sqliteInt.h" +// Copied from query_classifier.h +enum +{ + QUERY_TYPE_READ = 0x000002, /*< Read database data:any */ + QUERY_TYPE_WRITE = 0x000004, /*< Master data will be modified:master */ +}; + // MaxScale naming convention: // // - A function that "overloads" a sqlite3 function has the same name @@ -294,11 +301,13 @@ cmdx ::= cmd. { sqlite3FinishCoding(pParse); } // %ifdef MAXSCALE -cmd ::= BEGIN transtype(Y) trans_opt. {mxs_sqlite3BeginTransaction(pParse, Y);} +id_opt ::= . +id_opt ::= deferred_id. + +cmd ::= BEGIN id_opt. {mxs_sqlite3BeginTransaction(pParse, 0);} // BEGIN [WORK] %endif %ifndef MAXSCALE cmd ::= BEGIN transtype(Y) trans_opt. {sqlite3BeginTransaction(pParse, Y);} -%endif trans_opt ::= . trans_opt ::= TRANSACTION. trans_opt ::= TRANSACTION nm. @@ -307,10 +316,11 @@ transtype(A) ::= . {A = TK_DEFERRED;} transtype(A) ::= DEFERRED(X). {A = @X;} transtype(A) ::= IMMEDIATE(X). {A = @X;} transtype(A) ::= EXCLUSIVE(X). {A = @X;} +%endif %ifdef MAXSCALE -cmd ::= COMMIT trans_opt. {mxs_sqlite3CommitTransaction(pParse);} -cmd ::= END trans_opt. {mxs_sqlite3CommitTransaction(pParse);} -cmd ::= ROLLBACK trans_opt. {mxs_sqlite3RollbackTransaction(pParse);} +cmd ::= COMMIT id_opt. {mxs_sqlite3CommitTransaction(pParse);} +cmd ::= END id_opt. {mxs_sqlite3CommitTransaction(pParse);} +cmd ::= ROLLBACK id_opt. {mxs_sqlite3RollbackTransaction(pParse);} %endif %ifndef MAXSCALE cmd ::= COMMIT trans_opt. {sqlite3CommitTransaction(pParse);} @@ -318,6 +328,7 @@ cmd ::= END trans_opt. {sqlite3CommitTransaction(pParse);} cmd ::= ROLLBACK trans_opt. {sqlite3RollbackTransaction(pParse);} %endif +%ifndef MAXSCALE savepoint_opt ::= SAVEPOINT. savepoint_opt ::= . cmd ::= SAVEPOINT nm(X). { @@ -329,6 +340,7 @@ cmd ::= RELEASE savepoint_opt nm(X). { cmd ::= ROLLBACK trans_opt TO savepoint_opt nm(X). { sqlite3Savepoint(pParse, SAVEPOINT_ROLLBACK, &X); } +%endif ///////////////////// The CREATE TABLE statement //////////////////////////// // @@ -3144,8 +3156,39 @@ show(A) ::= SHOW WARNINGS show_warnings_options. { //////////////////////// The START TRANSACTION statement //////////////////////////////////// // -cmd ::= START TRANSACTION. { - mxs_sqlite3BeginTransaction(pParse, 0); +%type start_transaction_characteristic {int} + +start_transaction_characteristic(A) ::= READ WRITE. { + A = QUERY_TYPE_WRITE; +} + +start_transaction_characteristic(A) ::= READ id. { // READ ONLY + A = QUERY_TYPE_READ; +} + +start_transaction_characteristic(A) ::= WITH id id. { // WITH CONSISTENT SNAPSHOT + A = 0; +} + +%type start_transaction_characteristics {int} + +start_transaction_characteristics(A) ::= . +{ + A = 0; +} + +start_transaction_characteristics(A) ::= start_transaction_characteristic(X). +{ + A = X; +} + +start_transaction_characteristics(A) ::= + start_transaction_characteristics(X) COMMA start_transaction_characteristic(Y). { + A = X | Y; +} + +cmd ::= START TRANSACTION start_transaction_characteristics(X). { + mxs_sqlite3BeginTransaction(pParse, X); } //////////////////////// The TRUNCATE statement //////////////////////////////////// diff --git a/query_classifier/test/maxscale.test b/query_classifier/test/maxscale.test index 48fb14660..6b3fe828e 100644 --- a/query_classifier/test/maxscale.test +++ b/query_classifier/test/maxscale.test @@ -54,4 +54,20 @@ SET autocommit=true; SET autocommit=FALSE; SET autocommit=Off; -LOAD DATA LOCAL INFILE '/tmp/data.csv' INTO TABLE test.t1; \ No newline at end of file +LOAD DATA LOCAL INFILE '/tmp/data.csv' INTO TABLE test.t1; + +START TRANSACTION; +START TRANSACTION READ ONLY; +START TRANSACTION READ WRITE; +START TRANSACTION READ ONLY, WITH CONSISTENT SNAPSHOT; +START TRANSACTION READ WRITE, WITH CONSISTENT SNAPSHOT; +START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY; +START TRANSACTION WITH CONSISTENT SNAPSHOT, READ WRITE; + +BEGIN; +BEGIN WORK; +COMMIT; +COMMIT WORK; +ROLLBACK; +ROLLBACK WORK; + From 311eae8cfe73340629ddfb3c6a5e6062c7a9b9b4 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Thu, 20 Oct 2016 13:50:01 +0300 Subject: [PATCH 059/215] MXS-983: Add transaction state to session object --- include/maxscale/session.h | 83 ++++++++++++++++++++++++++++++++++++++ server/core/session.c | 34 ++++++++++++++++ 2 files changed, 117 insertions(+) diff --git a/include/maxscale/session.h b/include/maxscale/session.h index c688e6149..0b1cd1d97 100644 --- a/include/maxscale/session.h +++ b/include/maxscale/session.h @@ -73,6 +73,36 @@ typedef enum SESSION_STATE_DUMMY /*< dummy session for consistency */ } session_state_t; +typedef enum +{ + SESSION_TRX_INACTIVE_BIT = 1, /* 0b0001 */ + SESSION_TRX_ACTIVE_BIT = 2, /* 0b0010 */ + SESSION_TRX_READ_ONLY_BIT = 4, /* 0b0100 */ + SESSION_TRX_READ_WRITE_BIT = 8, /* 0b1000 */ +} session_trx_state_bit_t; + +typedef enum +{ + /*< The current transaction state is not known. */ + SESSION_TRX_UNKNOWN = 0, + /*< There is no on-going transaction. */ + SESSION_TRX_INACTIVE = SESSION_TRX_INACTIVE_BIT, + /*< A transaction is active. */ + SESSION_TRX_ACTIVE = SESSION_TRX_ACTIVE_BIT, + /*< An explicit READ ONLY transaction is active. */ + SESSION_TRX_READ_ONLY = (SESSION_TRX_ACTIVE_BIT | SESSION_TRX_READ_ONLY_BIT), + /*< An explicit READ WRITE transaction is active. */ + SESSION_TRX_READ_WRITE = (SESSION_TRX_ACTIVE_BIT | SESSION_TRX_READ_WRITE_BIT) +} session_trx_state_t; + +/** + * Convert transaction state to string representation. + * + * @param state A transaction state. + * @return String representation of the state. + */ +const char* session_trx_state_to_string(session_trx_state_t state); + /** * The downstream element in the filter chain. This may refer to * another filter or to a router. @@ -150,6 +180,7 @@ typedef struct session UPSTREAM tail; /*< The tail of the filter chain */ int refcount; /*< Reference count on the session */ bool ses_is_child; /*< this is a child session */ + session_trx_state_t trx_state; /*< The current transaction state. */ skygw_chk_t ses_chk_tail; } SESSION; @@ -206,6 +237,58 @@ RESULTSET *sessionGetList(SESSIONLISTFILTER); void process_idle_sessions(); void enable_session_timeouts(); +/** + * Get the transaction state of the session. + * + * @param ses The SESSION object. + * @return The transaction state. + */ +session_trx_state_t session_get_trx_state(const SESSION* ses); + +/** + * Set the transaction state of the session. + * + * NOTE: Only the protocol object may call this. + * + * @param ses The SESSION object. + * @param new_state The new transaction state. + * + * @return The previous transaction state. + */ +session_trx_state_t session_set_trx_state(SESSION* ses, session_trx_state_t new_state); + +/** + * Tells whether a transaction is active. + * + * @return True if a transaction is active, false otherwise. + */ +static inline bool session_trx_is_active(const SESSION* ses) +{ + return ses->trx_state & SESSION_TRX_ACTIVE_BIT; +} + +/** + * Tells whether an explicit READ ONLY transaction is active. + * + * @return True if an explicit READ ONLY transaction is active, + * false otherwise. + */ +static inline bool session_trx_is_read_only(const SESSION* ses) +{ + return ses->trx_state == SESSION_TRX_READ_ONLY; +} + +/** + * Tells whether an explicit READ WRITE transaction is active. + * + * @return True if an explicit READ WRITE transaction is active, + * false otherwise. + */ +static inline bool session_trx_is_read_write(const SESSION* ses) +{ + return ses->trx_state == SESSION_TRX_READ_WRITE; +} + MXS_END_DECLS #endif diff --git a/server/core/session.c b/server/core/session.c index a449c6998..f14def029 100644 --- a/server/core/session.c +++ b/server/core/session.c @@ -142,6 +142,7 @@ session_alloc(SERVICE *service, DCB *client_dcb) */ session->state = SESSION_STATE_READY; + session->trx_state = SESSION_TRX_UNKNOWN; /* * Only create a router session if we are not the listening * DCB or an internal DCB. Creating a router session may create a connection to a @@ -1061,3 +1062,36 @@ sessionGetList(SESSIONLISTFILTER filter) } /*lint +e429 */ +session_trx_state_t session_get_trx_state(const SESSION* ses) +{ + return ses->trx_state; +} + +session_trx_state_t session_set_trx_state(SESSION* ses, session_trx_state_t new_state) +{ + session_trx_state_t prev_state = ses->trx_state; + + ses->trx_state = new_state; + + return prev_state; +} + +const char* session_trx_state_to_string(session_trx_state_t state) +{ + switch (state) + { + case SESSION_TRX_UNKNOWN: + return "SESSION_TRX_UNKNOWN"; + case SESSION_TRX_INACTIVE: + return "SESSION_TRX_INACTIVE"; + case SESSION_TRX_ACTIVE: + return "SESSION_TRX_ACTIVE"; + case SESSION_TRX_READ_ONLY: + return "SESSION_TRX_READ_ONLY"; + case SESSION_TRX_READ_WRITE: + return "SESSION_TRX_READ_WRITE"; + } + + MXS_ERROR("Unknown session_trx_state_t value: %d", (int)state); + return "UNKNOWN"; +} From d397533e7fe90c1b45c11563c9ce10f419a95c9e Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Thu, 20 Oct 2016 17:25:50 +0300 Subject: [PATCH 060/215] MXS-983: Update session transaction state Not final implementation, we may need to track what response the server returns. --- .../protocol/MySQL/MySQLClient/mysql_client.c | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c index 7aa63f422..893da5c15 100644 --- a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c +++ b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c @@ -56,8 +56,9 @@ #include #include #include - +#include #include +#include /* @see function load_module in load_utils.c for explanation of the following * lint directives. @@ -849,6 +850,34 @@ gw_read_finish_processing(DCB *dcb, GWBUF *read_buffer, uint64_t capabilities) if (capabilities & RCAP_TYPE_STMT_INPUT) { + SESSION *ses = dcb->session; + ss_dassert(ses); + + uint32_t type = qc_get_type(read_buffer); + + if (type & QUERY_TYPE_BEGIN_TRX) + { + session_trx_state_t trx_state; + if (type & QUERY_TYPE_WRITE) + { + trx_state = SESSION_TRX_READ_WRITE; + } + else if (type & QUERY_TYPE_READ) + { + trx_state = SESSION_TRX_READ_ONLY; + } + else + { + trx_state = SESSION_TRX_ACTIVE; + } + + session_set_trx_state(ses, trx_state); + } + else if ((type & QUERY_TYPE_COMMIT) || (type & QUERY_TYPE_ROLLBACK)) + { + session_set_trx_state(ses, SESSION_TRX_INACTIVE); + } + /** * Feed each statement completely and separately * to router. The routing functions return 1 for From 9ef06c7d2f1aed9b2629a98c67019bcc95c93670 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 21 Oct 2016 11:04:29 +0300 Subject: [PATCH 061/215] MXS-650: Use correct package number for error messages When a client without SSL connected to a listener which requires SSL, the error message always used the package sequence number 3 even if the response was the second packet. Also fixed debug assertions which failed when non-SSL connections were created to SSL listeners. --- .../protocol/MySQL/MySQLClient/mysql_client.c | 51 +++++++++++++++---- 1 file changed, 40 insertions(+), 11 deletions(-) diff --git a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c index 893da5c15..2629853b3 100644 --- a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c +++ b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c @@ -87,7 +87,7 @@ static char *gw_default_auth(); static int gw_connection_limit(DCB *dcb, int limit); static int MySQLSendHandshake(DCB* dcb); static int route_by_statement(SESSION *, GWBUF **); -static void mysql_client_auth_error_handling(DCB *dcb, int auth_val); +static void mysql_client_auth_error_handling(DCB *dcb, int auth_val, int packet_number); static int gw_read_do_authentication(DCB *dcb, GWBUF *read_buffer, int nbytes_read); static int gw_read_normal_data(DCB *dcb, GWBUF *read_buffer, int nbytes_read); static int gw_read_finish_processing(DCB *dcb, GWBUF *read_buffer, uint64_t capabilities); @@ -480,7 +480,8 @@ static void store_client_information(DCB *dcb, GWBUF *buffer) MYSQL_session *ses = (MYSQL_session*)dcb->data; gwbuf_copy_data(buffer, 0, len, data); - ss_dassert(MYSQL_GET_PACKET_LEN(data) + MYSQL_HEADER_LEN == len); + ss_dassert(MYSQL_GET_PACKET_LEN(data) + MYSQL_HEADER_LEN == len || + len == MYSQL_AUTH_PACKET_BASE_SIZE); // For SSL request packet proto->client_capabilities = gw_mysql_get_byte4(data + MYSQL_CLIENT_CAP_OFFSET); proto->charset = data[MYSQL_CHARSET_OFFSET]; @@ -505,6 +506,37 @@ static void store_client_information(DCB *dcb, GWBUF *buffer) } } +/** + * @brief Debug check function for authentication packets + * + * Check that the packet is consistent with how the protocol works and that no + * unexpected data is processed. + * + * @param dcb Client DCB + * @param buf Buffer containing packet + * @param bytes Number of bytes available + */ +static void check_packet(DCB *dcb, GWBUF *buf, int bytes) +{ + uint8_t hdr[MYSQL_HEADER_LEN]; + ss_dassert(gwbuf_copy_data(buf, 0, MYSQL_HEADER_LEN, hdr) == MYSQL_HEADER_LEN); + + int buflen = gwbuf_length(buf); + int pktlen = MYSQL_GET_PACKET_LEN(hdr) + MYSQL_HEADER_LEN; + + if (bytes == MYSQL_AUTH_PACKET_BASE_SIZE) + { + /** This is an SSL request packet */ + ss_dassert(dcb->listener->ssl); + ss_dassert(buflen == bytes && pktlen >= buflen); + } + else + { + /** Normal packet */ + ss_dassert(buflen == pktlen); + } +} + /** * @brief Client read event, process when client not yet authenticated * @@ -516,9 +548,8 @@ static void store_client_information(DCB *dcb, GWBUF *buffer) static int gw_read_do_authentication(DCB *dcb, GWBUF *read_buffer, int nbytes_read) { - ss_debug(uint8_t hdr[MYSQL_HEADER_LEN]); - ss_dassert(gwbuf_copy_data(read_buffer, 0, MYSQL_HEADER_LEN, hdr) == MYSQL_HEADER_LEN && - MYSQL_GET_PACKET_LEN(hdr) + MYSQL_HEADER_LEN == gwbuf_length(read_buffer)); + ss_debug(check_packet(dcb, read_buffer, nbytes_read)); + /** Allocate the shared session structure */ if (dcb->data == NULL && (dcb->data = mysql_session_alloc()) == NULL) { @@ -612,7 +643,7 @@ gw_read_do_authentication(DCB *dcb, GWBUF *read_buffer, int nbytes_read) MXS_AUTH_SSL_INCOMPLETE != auth_val) { protocol->protocol_auth_state = MXS_AUTH_STATE_FAILED; - mysql_client_auth_error_handling(dcb, auth_val); + mysql_client_auth_error_handling(dcb, auth_val, next_sequence); /** * Close DCB and which will release MYSQL_session */ @@ -953,13 +984,11 @@ gw_read_finish_processing(DCB *dcb, GWBUF *read_buffer, uint64_t capabilities) * @note Authentication status codes are defined in maxscale/protocol/mysql.h */ static void -mysql_client_auth_error_handling(DCB *dcb, int auth_val) +mysql_client_auth_error_handling(DCB *dcb, int auth_val, int packet_number) { - int packet_number, message_len; + int message_len; char *fail_str = NULL; - packet_number = ssl_required_by_dcb(dcb) ? 3 : 2; - switch (auth_val) { case MXS_AUTH_NO_SESSION: @@ -1002,7 +1031,7 @@ mysql_client_auth_error_handling(DCB *dcb, int auth_val) mysql_send_auth_error(dcb, packet_number, 0, - "failed to complete SSL authentication"); + "Access without SSL denied"); break; case MXS_AUTH_SSL_INCOMPLETE: MXS_DEBUG("%lu [gw_read_client_event] unable to " From 3484b34747a601e45faf49595e449ff4274c579b Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 21 Oct 2016 11:47:27 +0300 Subject: [PATCH 062/215] Copy client information also for SSL connection When a MySQL SSL connection is made, the first packet from the client will contain the SSLRequest packet. This packet is a truncated normal auth packet and the real auth packet will be sent after SSL has been established. The MySQLClient protocol should read the client information for SSL connections after SSL has been established. --- .../protocol/MySQL/MySQLClient/mysql_client.c | 38 +++++++++++-------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c index 2629853b3..bb8ab60b7 100644 --- a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c +++ b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c @@ -485,23 +485,26 @@ static void store_client_information(DCB *dcb, GWBUF *buffer) proto->client_capabilities = gw_mysql_get_byte4(data + MYSQL_CLIENT_CAP_OFFSET); proto->charset = data[MYSQL_CHARSET_OFFSET]; - strcpy(ses->user, (char*)data + MYSQL_AUTH_PACKET_BASE_SIZE); - *ses->db = '\0'; - if (proto->client_capabilities & GW_MYSQL_CAPABILITIES_CONNECT_WITH_DB) + if (len > MYSQL_AUTH_PACKET_BASE_SIZE) { - /** Client supports default database on connect */ - size_t userlen = strlen(ses->user) + 1; + strcpy(ses->user, (char*)data + MYSQL_AUTH_PACKET_BASE_SIZE); - /** Skip the authentication token, it is handled by the authenticators */ - uint8_t authlen = data[MYSQL_AUTH_PACKET_BASE_SIZE + userlen]; - - size_t dboffset = MYSQL_AUTH_PACKET_BASE_SIZE + userlen + authlen + 1; - - if (data[dboffset]) + if (proto->client_capabilities & GW_MYSQL_CAPABILITIES_CONNECT_WITH_DB) { - /** Client is connecting with a default database */ - strcpy(ses->db, (char*)data + dboffset); + /** Client supports default database on connect */ + size_t userlen = strlen(ses->user) + 1; + + /** Skip the authentication token, it is handled by the authenticators */ + uint8_t authlen = data[MYSQL_AUTH_PACKET_BASE_SIZE + userlen]; + + size_t dboffset = MYSQL_AUTH_PACKET_BASE_SIZE + userlen + authlen + 1; + + if (data[dboffset]) + { + /** Client is connecting with a default database */ + strcpy(ses->db, (char*)data + dboffset); + } } } } @@ -561,10 +564,15 @@ gw_read_do_authentication(DCB *dcb, GWBUF *read_buffer, int nbytes_read) uint8_t next_sequence; gwbuf_copy_data(read_buffer, MYSQL_SEQ_OFFSET, 1, &next_sequence); - if (next_sequence == 1) + if (next_sequence == 1 || (ssl_required_by_dcb(dcb) && next_sequence == 2)) { /** This is the first response from the client, read the connection - * information and store them in the shared structure */ + * information and store them in the shared structure. For SSL connections, + * this will be packet number two since the first packet will be the + * Protocol::SSLRequest packet. + * + * @see https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::SSLRequest + */ store_client_information(dcb, read_buffer); } From 5cc436c37c047de481966b79e75fadf20d33ace2 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Fri, 21 Oct 2016 10:05:07 +0300 Subject: [PATCH 063/215] MXS-955: Put back proper .maxadmin handling Now MaxScale again supports having 'hostname', 'user', 'port' and 'passwd' in the .maxadmin file in addition to 'socket'. --- Documentation/Reference/MaxAdmin.md | 15 ++++- client/maxadmin.c | 98 +++++++++++++++++++---------- 2 files changed, 78 insertions(+), 35 deletions(-) diff --git a/Documentation/Reference/MaxAdmin.md b/Documentation/Reference/MaxAdmin.md index af0d094ad..c3f0a7099 100644 --- a/Documentation/Reference/MaxAdmin.md +++ b/Documentation/Reference/MaxAdmin.md @@ -287,13 +287,22 @@ Then simply set this file to have execute permissions and it may be run like any ## The .maxadmin file -MaxAdmin supports a mechanism to set defaults for all the command line switches via a file in the home directory of the user. If a file named .maxadmin exists it will be read and parameters set according to the lies in this files. The parameter that can be set is: socket. An example of a .maxadmin file that will alter the default password and user name arguments would be +MaxAdmin supports a mechanism to set defaults for the command line switches via a file in the home directory of the user. If a file named `.maxadmin` exists, it will be read and parameters set according to the entries in that file. + +This mechanism can be used to provide defaults to the command line options. If a command line option is provided, it will still override the value in the `.maxadmin` file. + +The parameters than can be set are: + * `1.4`: _hostname_, _port_, _user_ and _passwd_ + * `2.0.0` and `2.0.1`: _socket_ + * `2.0.2` onwards: _socket_, _hostname_, _port_, _user_ and _passwd_ (and as synonym _password_) + +An example of a `.maxadmin` file that will alter the default socket path is: socket=/somepath/maxadmin.socket -This mechanism can be used to provide a means of passwords entry into maxadmin or to override any of the command line option defaults. If a command line option is given that will still override the value in the .maxadmin file. +Note that if in `2.0.2` or later, a value for _socket_ as well as any of the internet socket related options, such as _hostname_, is provided in `.maxadmin`, then _socket_ takes precedense. In that case, provide at least one internet socket related option on the command line to force MaxAdmin to use an internet socket and thus the internet socket related options from `.maxadmin`. -The .maxadmin file may be made read only to protect any passwords written to that file. +The `.maxadmin` file may be made read only to protect any passwords written to that file. # Getting Help diff --git a/client/maxadmin.c b/client/maxadmin.c index 13c92de7e..dc057b2f6 100644 --- a/client/maxadmin.c +++ b/client/maxadmin.c @@ -75,7 +75,9 @@ static void DoSource(int so, char *cmd); static void DoUsage(const char*); static int isquit(char *buf); static void PrintVersion(const char *progname); -static void read_inifile(char **, int*); +static void read_inifile(char **socket, + char **hostname, char **port, char **user, char **passwd, + int *editor); static bool getPassword(char *password, size_t length); #ifdef HISTORY @@ -116,10 +118,6 @@ static struct option long_options[] = int main(int argc, char **argv) { - const char* vi = "vi"; - const char* emacs = "emacs"; - - int i, num, rv; #ifdef HISTORY char *buf; EditLine *el = NULL; @@ -133,39 +131,45 @@ main(int argc, char **argv) char *port = NULL; char *user = NULL; char *passwd = NULL; - char *conn_socket = NULL; - char *default_socket = MAXADMIN_DEFAULT_SOCKET; + char *socket_path = NULL; int use_emacs = 0; - int so; + + read_inifile(&socket_path, &hostname, &port, &user, &passwd, &use_emacs); + + bool use_inet_socket = false; + bool use_unix_socket = false; + int option_index = 0; char c; - - read_inifile(&conn_socket, &use_emacs); - while ((c = getopt_long(argc, argv, "h:p:P:u:S:v?e", long_options, &option_index)) >= 0) { switch (c) { case 'h': + use_inet_socket = true; hostname = strdup(optarg); break; case 'p': + use_inet_socket = true; passwd = strdup(optarg); memset(optarg, '\0', strlen(optarg)); break; case 'P': + use_inet_socket = true; port = strdup(optarg); break; case 'u': + use_inet_socket = true; user = strdup(optarg); break; case 'S': - conn_socket = strdup(optarg); + use_unix_socket = true; + socket_path = strdup(optarg); break; case 'v': @@ -182,16 +186,20 @@ main(int argc, char **argv) } } - if ((hostname || port || user || passwd) && (conn_socket)) + if (use_inet_socket && use_unix_socket) { - // Either socket or any parameters related to hostname/port. + // Both unix socket path and at least of the internet socket + // options have been provided. DoUsage(argv[0]); exit(EXIT_FAILURE); } - if (hostname || port || user || passwd) + if (use_inet_socket || (!socket_path && (hostname || port || user || passwd))) { - assert(!conn_socket); + // If any of the internet socket options have explicitly been provided, or + // .maxadmin does not contain "socket" but does contain at least one of + // the internet socket options, we use an internet socket. Note that if + // -S is provided, then socket_path will be non-NULL. if (!hostname) { @@ -210,23 +218,29 @@ main(int argc, char **argv) } else { - if (!conn_socket) + use_unix_socket = true; + + if (!socket_path) { - conn_socket = MAXADMIN_DEFAULT_SOCKET; + socket_path = MAXADMIN_DEFAULT_SOCKET; } } - assert(!((hostname || port) && conn_socket)); + int so; - if (conn_socket) + if (use_unix_socket) { - if ((so = connectUsingUnixSocket(conn_socket)) == -1) + assert(socket_path); + + if ((so = connectUsingUnixSocket(socket_path)) == -1) { exit(EXIT_FAILURE); } } else { + assert(hostname && user && port); + char password[MAX_PASSWORD_LEN]; if (passwd == NULL) @@ -301,11 +315,11 @@ main(int argc, char **argv) if (use_emacs) { - el_set(el, EL_EDITOR, emacs); /** Editor is emacs */ + el_set(el, EL_EDITOR, "emacs"); /** Editor is emacs */ } else { - el_set(el, EL_EDITOR, vi); /* Default editor is vi */ + el_set(el, EL_EDITOR, "vi"); /* Default editor is vi */ } el_set(el, EL_SIGNAL, 1); /* Handle signals gracefully */ el_set(el, EL_PROMPT, prompt); /* Set the prompt function */ @@ -325,6 +339,7 @@ main(int argc, char **argv) */ el_source(el, NULL); + int num; while ((buf = (char *) el_gets(el, &num)) != NULL && num != 0) { #else @@ -333,7 +348,7 @@ main(int argc, char **argv) num = strlen(buf); #endif /* Strip trailing \n\r */ - for (i = num - 1; buf[i] == '\r' || buf[i] == '\n'; i--) + for (int i = num - 1; buf[i] == '\r' || buf[i] == '\n'; i--) { buf[i] = 0; } @@ -350,6 +365,7 @@ main(int argc, char **argv) else if (!strcasecmp(buf, "history")) { #ifdef HISTORY + int rv; for (rv = history(hist, &ev, H_LAST); rv != -1; rv = history(hist, &ev, H_PREV)) { @@ -394,11 +410,11 @@ main(int argc, char **argv) /** * Connect to the MaxScale server * - * @param conn_socket The UNIX socket to connect to + * @param socket_path The UNIX socket to connect to * @return The connected socket or -1 on error */ static int -connectUsingUnixSocket(const char *conn_socket) +connectUsingUnixSocket(const char *socket_path) { int so = -1; @@ -408,7 +424,7 @@ connectUsingUnixSocket(const char *conn_socket) memset(&local_addr, 0, sizeof local_addr); local_addr.sun_family = AF_UNIX; - strncpy(local_addr.sun_path, conn_socket, sizeof(local_addr.sun_path) - 1); + strncpy(local_addr.sun_path, socket_path, sizeof(local_addr.sun_path) - 1); if (connect(so, (struct sockaddr *) &local_addr, sizeof(local_addr)) == 0) { @@ -441,7 +457,7 @@ connectUsingUnixSocket(const char *conn_socket) { char errbuf[STRERROR_BUFLEN]; fprintf(stderr, "Unable to connect to MaxScale at %s: %s\n", - conn_socket, strerror_r(errno, errbuf, sizeof(errbuf))); + socket_path, strerror_r(errno, errbuf, sizeof(errbuf))); close(so); so = -1; } @@ -853,13 +869,16 @@ rtrim(char *str) * Read defaults for hostname, port, user and password from * the .maxadmin file in the users home directory. * - * @param hostname Pointer the hostname to be updated + * @param socket Pointer to the socket to be updated. + * @param hostname Pointer to the hostname to be updated * @param port Pointer to the port to be updated * @param user Pointer to the user to be updated * @param passwd Pointer to the password to be updated */ static void -read_inifile(char **conn_socket, int* editor) +read_inifile(char **socket, + char **hostname, char** port, char **user, char **passwd, + int* editor) { char pathname[400]; char *home, *brkt; @@ -893,11 +912,26 @@ read_inifile(char **conn_socket, int* editor) { if (strcmp(name, "socket") == 0) { - *conn_socket = strdup(value); + *socket = strdup(value); + } + else if (strcmp(name, "hostname") == 0) + { + *hostname = strdup(value); + } + else if (strcmp(name, "port") == 0) + { + *port = strdup(value); + } + else if (strcmp(name, "user") == 0) + { + *user = strdup(value); + } + else if ((strcmp(name, "passwd") == 0) || (strcmp(name, "password") == 0)) + { + *passwd = strdup(value); } else if (strcmp(name, "editor") == 0) { - if (strcmp(value, "vi") == 0) { *editor = 0; From 693d8dcbb4fdd457a7ec76c632353fcb827af0af Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Fri, 21 Oct 2016 13:08:18 +0300 Subject: [PATCH 064/215] Track session autocommit state This tracks only what is explicitly set. That is, if autocommit has been set true then, even if a transaction is started, autocommit will not be set false. That is, a user of the session autocommit and transaction states need to be aware of their semantics. If a transaction is active, then the state of autocommit is irrelevant. --- include/maxscale/session.h | 36 +++++++++++++++++++ server/core/session.c | 1 + .../protocol/MySQL/MySQLClient/mysql_client.c | 8 +++++ 3 files changed, 45 insertions(+) diff --git a/include/maxscale/session.h b/include/maxscale/session.h index 0b1cd1d97..cba1456a8 100644 --- a/include/maxscale/session.h +++ b/include/maxscale/session.h @@ -181,6 +181,7 @@ typedef struct session int refcount; /*< Reference count on the session */ bool ses_is_child; /*< this is a child session */ session_trx_state_t trx_state; /*< The current transaction state. */ + bool autocommit; /*< Whether autocommit is on. */ skygw_chk_t ses_chk_tail; } SESSION; @@ -289,6 +290,41 @@ static inline bool session_trx_is_read_write(const SESSION* ses) return ses->trx_state == SESSION_TRX_READ_WRITE; } +/** + * Tells whether autocommit is ON or not. + * + * Note that the returned value effectively only tells the last value + * of the statement "set autocommit=...". + * + * That is, if the statement "set autocommit=1" has been executed, then + * even if a transaction has been started, which implicitly will cause + * autocommit to be set to 0 for the duration of the transaction, this + * function will still return true. + * + * Note also that by default autocommit is ON. + * + * @return True if autocommit has been set ON, false otherwise. + */ +static inline bool session_is_autocommit(const SESSION* ses) +{ + return ses->autocommit; +} + +/** + * Sets the autocommit state of the session. + * + * NOTE: Only the protocol object may call this. + * + * @param enable True if autocommit is enabled, false otherwise. + * @return The previous state. + */ +static inline bool session_set_autocommit(SESSION* ses, bool autocommit) +{ + bool prev_autocommit = ses->autocommit; + ses->autocommit = autocommit; + return prev_autocommit; +} + MXS_END_DECLS #endif diff --git a/server/core/session.c b/server/core/session.c index f14def029..5e51cc48a 100644 --- a/server/core/session.c +++ b/server/core/session.c @@ -143,6 +143,7 @@ session_alloc(SERVICE *service, DCB *client_dcb) session->state = SESSION_STATE_READY; session->trx_state = SESSION_TRX_UNKNOWN; + session->autocommit = true; /* * Only create a router session if we are not the listening * DCB or an internal DCB. Creating a router session may create a connection to a diff --git a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c index bb8ab60b7..655b4e53c 100644 --- a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c +++ b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c @@ -916,6 +916,14 @@ gw_read_finish_processing(DCB *dcb, GWBUF *read_buffer, uint64_t capabilities) { session_set_trx_state(ses, SESSION_TRX_INACTIVE); } + else if (type & QUERY_TYPE_ENABLE_AUTOCOMMIT) + { + session_set_autocommit(ses, true); + } + else if (type & QUERY_TYPE_DISABLE_AUTOCOMMIT) + { + session_set_autocommit(ses, false); + } /** * Feed each statement completely and separately From fe689504b0f0344cbcb0d0299e9ecf5eeefaeb64 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 18 Oct 2016 14:06:32 +0300 Subject: [PATCH 065/215] Move service user injection into MySQL authenticator The MySQL authenticator now injects the service user into the list of allowed users if loading of database users fails. This allows the removal of common code in the binlogrouter and maxinfo modules. --- .../Authenticators/MySQL-Authenticator.md | 18 ++++++ include/maxscale/router.h | 4 +- server/core/service.c | 1 - server/modules/authenticator/mysql_auth.c | 60 ++++++++++++++++++- server/modules/routing/binlog/blr.c | 2 +- 5 files changed, 79 insertions(+), 6 deletions(-) diff --git a/Documentation/Authenticators/MySQL-Authenticator.md b/Documentation/Authenticators/MySQL-Authenticator.md index aee19d778..85a1adaca 100644 --- a/Documentation/Authenticators/MySQL-Authenticator.md +++ b/Documentation/Authenticators/MySQL-Authenticator.md @@ -32,3 +32,21 @@ port=4006 authenticator=MySQLAuth authenticator_options=cache_dir=/tmp ``` + +### `inject_service_user` + +Inject service credentials into the list of database users if loading of +users fails. This option takes a boolean value and it is enabled by +default. + +When a connection to the backend database cannot be made, the service user +can be injected into the list of allowed users. This allows administrative +operations to be done via the SQL interface with modules that support it +e.g. the Binlogrouter and Maxinfo modules. + +If users are loaded successfully, the service user credentials are _not_ +injected into the list of users. + +``` +authenticator_options=inject_service_user=false +``` diff --git a/include/maxscale/router.h b/include/maxscale/router.h index 91c363b9b..40b27038a 100644 --- a/include/maxscale/router.h +++ b/include/maxscale/router.h @@ -100,9 +100,7 @@ typedef enum router_capability_t RCAP_TYPE_UNDEFINED = 0x00, RCAP_TYPE_STMT_INPUT = 0x01, /**< Statement per buffer */ RCAP_TYPE_PACKET_INPUT = 0x02, /**< Data as it was read from DCB */ - RCAP_TYPE_NO_RSESSION = 0x04, /**< Router does not use router sessions */ - RCAP_TYPE_NO_USERS_INIT = 0x08 /**< Prevent the loading of authenticator - users when the service is started */ + RCAP_TYPE_NO_RSESSION = 0x04 /**< Router does not use router sessions */ } router_capability_t; MXS_END_DECLS diff --git a/server/core/service.c b/server/core/service.c index c6d7b871d..a6a5e167c 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -303,7 +303,6 @@ serviceStartPort(SERVICE *service, SERV_LISTENER *port) /** Load the authentication users before before starting the listener */ if (port->listener->authfunc.loadusers && - (service->router->getCapabilities() & RCAP_TYPE_NO_USERS_INIT) == 0 && port->listener->authfunc.loadusers(port) != MXS_AUTH_LOADUSERS_OK) { MXS_ERROR("[%s] Failed to load users for listener '%s', authentication might not work.", diff --git a/server/modules/authenticator/mysql_auth.c b/server/modules/authenticator/mysql_auth.c index 9e0cb9214..4c002ae53 100644 --- a/server/modules/authenticator/mysql_auth.c +++ b/server/modules/authenticator/mysql_auth.c @@ -37,7 +37,8 @@ typedef struct mysql_auth { - char *cache_dir; /**< Custom cache directory location */ + char *cache_dir; /**< Custom cache directory location */ + bool inject_service_user; /**< Inject the service user into the list of users */ } MYSQL_AUTH; @@ -142,6 +143,7 @@ static void* mysql_auth_init(char **options) { bool error = false; instance->cache_dir = NULL; + instance->inject_service_user = true; for (int i = 0; options[i]; i++) { @@ -158,6 +160,10 @@ static void* mysql_auth_init(char **options) error = true; } } + else if (strcmp(options[i], "inject_service_user") == 0) + { + instance->inject_service_user = config_truth_value(value); + } else { MXS_ERROR("Unknown authenticator option: %s", options[i]); @@ -830,6 +836,48 @@ mysql_auth_free_client_data(DCB *dcb) MXS_FREE(dcb->data); } +/** + * @brief Inject the service user into the cache + * + * @param port Service listener + * @return True on success, false on error + */ +static bool add_service_user(SERV_LISTENER *port) +{ + char *user = NULL; + char *pw = NULL; + bool rval = false; + + if (serviceGetUser(port->service, &user, &pw)) + { + pw = decryptPassword(pw); + + if (pw) + { + char *newpw = create_hex_sha1_sha1_passwd(pw); + + if (newpw) + { + add_mysql_users_with_host_ipv4(port->users, user, "%", newpw, "Y", ""); + add_mysql_users_with_host_ipv4(port->users, user, "localhost", newpw, "Y", ""); + MXS_FREE(newpw); + rval = true; + } + MXS_FREE(pw); + } + else + { + MXS_ERROR("[%s] Failed to decrypt service user password.", port->service->name); + } + } + else + { + MXS_ERROR("[%s] Failed to retrieve service credentials.", port->service->name); + } + + return rval; +} + /** * @brief Load MySQL authentication users * @@ -871,6 +919,16 @@ static int mysql_auth_load_users(SERV_LISTENER *port) { MXS_WARNING("Using cached credential information."); } + + if (instance->inject_service_user) + { + /** Inject the service user as a 'backup' user that's available + * if loading of the users fails */ + if (!add_service_user(port)) + { + MXS_ERROR("[%s] Failed to inject service user.", port->service->name); + } + } } else { diff --git a/server/modules/routing/binlog/blr.c b/server/modules/routing/binlog/blr.c index 47909e625..213b9eb87 100644 --- a/server/modules/routing/binlog/blr.c +++ b/server/modules/routing/binlog/blr.c @@ -1790,7 +1790,7 @@ static void rses_end_locked_router_action(ROUTER_SLAVE *rses) static uint64_t getCapabilities() { - return (RCAP_TYPE_NO_RSESSION | RCAP_TYPE_NO_USERS_INIT); + return RCAP_TYPE_NO_RSESSION; } /** From 4e07c3313c9c09236d0a7089499863ebf6493b4a Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 20 Oct 2016 21:26:06 +0300 Subject: [PATCH 066/215] Move dbusers.c out of the core The dbusers.c was a MySQL protocol specific file which was used directly by some of the modules. Added a new return value for the loadusers authenticator entry point which allows fatal failures to occur when users are loaded. Currently this is only taken into notice when the service is first started. If a listener later returns a fatal error, it is only logged but the service stays in operation. Moved the MySQLAuth authenticator sources and the tests that relate to this module into a subdirectory in the authenticator directory. Eventually, all authenticators could have a subdirectory of their own. --- include/maxscale/gw_authenticator.h | 7 +- include/maxscale/protocol/mysql.h | 9 +- include/maxscale/service.h | 9 + server/core/CMakeLists.txt | 2 +- server/core/authenticator.c | 4 +- server/core/config.c | 1 - server/core/listener.c | 6 +- server/core/mysql_binlog.c | 1 - server/core/server.c | 3 +- server/core/service.c | 122 ++++++++------ server/core/test/CMakeLists.txt | 3 - server/modules/authenticator/CMakeLists.txt | 5 +- .../authenticator/MySQLAuth/CMakeLists.txt | 10 ++ .../authenticator/MySQLAuth}/dbusers.c | 2 +- .../authenticator/MySQLAuth}/dbusers.h | 16 +- .../{ => MySQLAuth}/mysql_auth.c | 11 +- .../MySQLAuth}/test_mysql_users.c | 2 +- server/modules/routing/avro/avro_client.c | 1 - server/modules/routing/avro/avrorouter.h | 1 - server/modules/routing/binlog/blr.c | 156 ------------------ server/modules/routing/binlog/blr_slave.c | 27 +-- server/modules/routing/debugcli/debugcmd.c | 1 - server/modules/routing/maxinfo/maxinfo.c | 80 --------- 23 files changed, 127 insertions(+), 352 deletions(-) create mode 100644 server/modules/authenticator/MySQLAuth/CMakeLists.txt rename server/{core => modules/authenticator/MySQLAuth}/dbusers.c (99%) rename {include/maxscale => server/modules/authenticator/MySQLAuth}/dbusers.h (76%) rename server/modules/authenticator/{ => MySQLAuth}/mysql_auth.c (99%) rename server/{core/test => modules/authenticator/MySQLAuth}/test_mysql_users.c (99%) diff --git a/include/maxscale/gw_authenticator.h b/include/maxscale/gw_authenticator.h index a78523ff3..deafd14b7 100644 --- a/include/maxscale/gw_authenticator.h +++ b/include/maxscale/gw_authenticator.h @@ -72,7 +72,7 @@ struct servlistener; * destroy Destroy the unique DCB data returned by the `create` * entry point. * - * loadUsers Load or update authenticator user data + * loadusers Load or update authenticator user data * @endverbatim * * This forms the "module object" for authenticator modules within the gateway. @@ -102,7 +102,8 @@ typedef struct gw_authenticator /** Return values for the loadusers entry point */ #define MXS_AUTH_LOADUSERS_OK 0 /**< Users loaded successfully */ -#define MXS_AUTH_LOADUSERS_ERROR 1 /**< Failed to load users */ +#define MXS_AUTH_LOADUSERS_ERROR 1 /**< Temporary error, service is started */ +#define MXS_AUTH_LOADUSERS_FATAL 2 /**< Fatal error, service is not started */ /** * Authentication states @@ -136,7 +137,7 @@ typedef enum bool authenticator_init(void **instance, const char *authenticator, const char *options); -char* get_default_authenticator(const char *protocol); +const char* get_default_authenticator(const char *protocol); MXS_END_DECLS diff --git a/include/maxscale/protocol/mysql.h b/include/maxscale/protocol/mysql.h index 254478e21..3fc66cc49 100644 --- a/include/maxscale/protocol/mysql.h +++ b/include/maxscale/protocol/mysql.h @@ -59,7 +59,6 @@ #include #include #include -#include #include #include #include @@ -93,6 +92,7 @@ MXS_BEGIN_DECLS #define GW_MYSQL_SCRAMBLE_SIZE 20 #define GW_SCRAMBLE_LENGTH_323 8 +/** Name of the default server side authentication plugin */ #define DEFAULT_MYSQL_AUTH_PLUGIN "mysql_native_password" /** All authentication responses are at least this many bytes long */ @@ -105,7 +105,12 @@ MXS_BEGIN_DECLS # define MYSQL_SCRAMBLE_LEN GW_MYSQL_SCRAMBLE_SIZE #endif -#define MYSQL_HOSTNAME_MAXLEN 60 +/* Max length of fields in the mysql.user table */ +#define MYSQL_USER_MAXLEN 128 +#define MYSQL_PASSWORD_LEN 41 +#define MYSQL_HOST_MAXLEN 60 +#define MYSQL_DATABASE_MAXLEN 128 +#define MYSQL_TABLE_MAXLEN 64 #define GW_NOINTR_CALL(A) do { errno = 0; A; } while (errno == EINTR) #define SMALL_CHUNK 1024 diff --git a/include/maxscale/service.h b/include/maxscale/service.h index fa401628d..6cb80793b 100644 --- a/include/maxscale/service.h +++ b/include/maxscale/service.h @@ -114,6 +114,15 @@ typedef struct server_ref_t */ #define SERVICE_PARAM_UNINIT -1 +/* Refresh rate limits for load users from database */ +#define USERS_REFRESH_TIME 30 /* Allowed time interval (in seconds) after last update*/ +#define USERS_REFRESH_MAX_PER_TIME 4 /* Max number of load calls within the time interval */ + +/** Default timeout values used by the connections which fetch user authentication data */ +#define DEFAULT_AUTH_CONNECT_TIMEOUT 3 +#define DEFAULT_AUTH_READ_TIMEOUT 1 +#define DEFAULT_AUTH_WRITE_TIMEOUT 2 + /** * Defines a service within the gateway. * diff --git a/server/core/CMakeLists.txt b/server/core/CMakeLists.txt index 3c7c630b8..6131c33b6 100644 --- a/server/core/CMakeLists.txt +++ b/server/core/CMakeLists.txt @@ -1,4 +1,4 @@ -add_library(maxscale-common SHARED adminusers.c alloc.c authenticator.c atomic.c buffer.c config.c dbusers.c dcb.c filter.c externcmd.c gwbitmask.c gwdirs.c hashtable.c hint.c housekeeper.c listmanager.c load_utils.c log_manager.cc maxscale_pcre2.c memlog.c misc.c mlist.c modutil.c monitor.c queuemanager.c query_classifier.c poll.c random_jkiss.c resultset.c secrets.c server.c service.c session.c spinlock.c thread.c users.c utils.c skygw_utils.cc statistics.c listener.c gw_ssl.c mysql_utils.c mysql_binlog.c) +add_library(maxscale-common SHARED adminusers.c alloc.c authenticator.c atomic.c buffer.c config.c dcb.c filter.c externcmd.c gwbitmask.c gwdirs.c hashtable.c hint.c housekeeper.c listmanager.c load_utils.c log_manager.cc maxscale_pcre2.c memlog.c misc.c mlist.c modutil.c monitor.c queuemanager.c query_classifier.c poll.c random_jkiss.c resultset.c secrets.c server.c service.c session.c spinlock.c thread.c users.c utils.c skygw_utils.cc statistics.c listener.c gw_ssl.c mysql_utils.c mysql_binlog.c) target_link_libraries(maxscale-common ${MARIADB_CONNECTOR_LIBRARIES} ${LZMA_LINK_FLAGS} ${PCRE2_LIBRARIES} ${CURL_LIBRARIES} ssl pthread crypt dl crypto inih z rt m stdc++) diff --git a/server/core/authenticator.c b/server/core/authenticator.c index 30b766e29..761103a1d 100644 --- a/server/core/authenticator.c +++ b/server/core/authenticator.c @@ -89,14 +89,14 @@ bool authenticator_init(void** dest, const char *authenticator, const char *opti * @return The default authenticator for the protocol or NULL if the protocol * does not provide one */ -char* get_default_authenticator(const char *protocol) +const char* get_default_authenticator(const char *protocol) { char *rval = NULL; GWPROTOCOL *protofuncs = (GWPROTOCOL*)load_module(protocol, MODULE_PROTOCOL); if (protofuncs && protofuncs->auth_default) { - rval = MXS_STRDUP(protofuncs->auth_default()); + rval = protofuncs->auth_default(); } return rval; diff --git a/server/core/config.c b/server/core/config.c index ce3fbf2f9..e8c4ef03d 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -72,7 +72,6 @@ #include #include #include -#include #include #include #define PCRE2_CODE_UNIT_WIDTH 8 diff --git a/server/core/listener.c b/server/core/listener.c index a7de9403e..075ae8eff 100644 --- a/server/core/listener.c +++ b/server/core/listener.c @@ -71,7 +71,8 @@ listener_alloc(struct service* service, char* name, char *protocol, char *addres { authenticator = MXS_STRDUP(authenticator); } - else if ((authenticator = get_default_authenticator(protocol)) == NULL) + else if ((authenticator = (char*)get_default_authenticator(protocol)) == NULL || + (authenticator = MXS_STRDUP(authenticator)) == NULL) { MXS_ERROR("No authenticator defined for listener '%s' and could not get " "default authenticator for protocol '%s'.", name, protocol); @@ -83,6 +84,9 @@ listener_alloc(struct service* service, char* name, char *protocol, char *addres { MXS_ERROR("Failed to initialize authenticator module '%s' for " "listener '%s'.", authenticator, name); + MXS_FREE(address); + MXS_FREE(authenticator); + return NULL; } protocol = MXS_STRDUP(protocol); diff --git a/server/core/mysql_binlog.c b/server/core/mysql_binlog.c index 15e2a1f1c..0087bf19b 100644 --- a/server/core/mysql_binlog.c +++ b/server/core/mysql_binlog.c @@ -22,7 +22,6 @@ #include #include #include -#include #include /** diff --git a/server/core/server.c b/server/core/server.c index 101b5b5f0..632aa4a94 100644 --- a/server/core/server.c +++ b/server/core/server.c @@ -69,7 +69,8 @@ server_alloc(char *servname, char *protocol, unsigned short port, char *authenti { authenticator = MXS_STRDUP(authenticator); } - else if ((authenticator = get_default_authenticator(protocol)) == NULL) + else if ((authenticator = (char*)get_default_authenticator(protocol)) == NULL || + (authenticator = MXS_STRDUP(authenticator)) == NULL) { MXS_ERROR("No authenticator defined for server at %s:%u and no default " "authenticator for protocol '%s'.", servname, port, protocol); diff --git a/server/core/service.c b/server/core/service.c index a6a5e167c..ceaeac0cb 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -53,7 +53,6 @@ #include #include #include -#include #include #include #include @@ -210,6 +209,16 @@ service_isvalid(SERVICE *service) return rval; } +static inline void close_port(SERV_LISTENER *port) +{ + port->service->state = SERVICE_STATE_FAILED; + if (port->listener) + { + dcb_close(port->listener); + port->listener = NULL; + } +} + /** * Start an individual port/protocol pair * @@ -232,7 +241,9 @@ serviceStartPort(SERVICE *service, SERV_LISTENER *port) { /* Should never happen, this guarantees it can't */ MXS_ERROR("Attempt to start port with null or incomplete service"); - goto retblock; + close_port(port); + ss_dassert(false); + return 0; } port->listener = dcb_alloc(DCB_ROLE_SERVICE_LISTENER, port); @@ -240,7 +251,8 @@ serviceStartPort(SERVICE *service, SERV_LISTENER *port) if (port->listener == NULL) { MXS_ERROR("Failed to create listener for service %s.", service->name); - goto retblock; + close_port(port); + return 0; } port->listener->service = service; @@ -252,13 +264,10 @@ serviceStartPort(SERVICE *service, SERV_LISTENER *port) if ((funcs = (GWPROTOCOL *)load_module(port->protocol, MODULE_PROTOCOL)) == NULL) { - dcb_close(port->listener); - port->listener = NULL; - MXS_ERROR("Unable to load protocol module %s. Listener " - "for service %s not started.", - port->protocol, - service->name); - goto retblock; + MXS_ERROR("Unable to load protocol module %s. Listener for service %s not started.", + port->protocol, service->name); + close_port(port); + return 0; } memcpy(&(port->listener->func), funcs, sizeof(GWPROTOCOL)); @@ -280,8 +289,7 @@ serviceStartPort(SERVICE *service, SERV_LISTENER *port) { MXS_ERROR("Failed to load authenticator module '%s' for listener '%s'", authenticator_name, port->name); - dcb_close(port->listener); - port->listener = NULL; + close_port(port); return 0; } @@ -302,11 +310,24 @@ serviceStartPort(SERVICE *service, SERV_LISTENER *port) } /** Load the authentication users before before starting the listener */ - if (port->listener->authfunc.loadusers && - port->listener->authfunc.loadusers(port) != MXS_AUTH_LOADUSERS_OK) + if (port->listener->authfunc.loadusers) { - MXS_ERROR("[%s] Failed to load users for listener '%s', authentication might not work.", - service->name, port->name); + switch (port->listener->authfunc.loadusers(port)) + { + case MXS_AUTH_LOADUSERS_FATAL: + MXS_ERROR("[%s] Fatal error when loading users for listener '%s', " + "service is not started.", service->name, port->name); + close_port(port); + return 0; + + case MXS_AUTH_LOADUSERS_ERROR: + MXS_WARNING("[%s] Failed to load users for listener '%s', authentication" + " might not work.", service->name, port->name); + break; + + default: + break; + } } /** @@ -327,24 +348,16 @@ serviceStartPort(SERVICE *service, SERV_LISTENER *port) } else { - MXS_ERROR("Failed to create session to service %s.", - service->name); - dcb_close(port->listener); - port->listener = NULL; - goto retblock; + MXS_ERROR("[%s] Failed to create listener session.", service->name); + close_port(port); } } else { - MXS_ERROR("Unable to start to listen port %d for %s %s.", - port->port, - port->protocol, - service->name); - dcb_close(port->listener); - port->listener = NULL; + MXS_ERROR("[%s] Failed to listen on %s", service->name, config_bind); + close_port(port); } -retblock: return listeners; } @@ -369,7 +382,11 @@ int serviceStartAllPorts(SERVICE* service) port = port->next; } - if (listeners) + if (service->state == SERVICE_STATE_FAILED) + { + listeners = 0; + } + else if (listeners) { service->state = SERVICE_STATE_STARTED; service->stats.started = time(0); @@ -457,29 +474,20 @@ int serviceStart(SERVICE *service) { int listeners = 0; + char **router_options = copy_string_array(service->routerOptions); - if (check_service_permissions(service)) + if ((service->router_instance = service->router->createInstance(service, router_options))) { - char **router_options = copy_string_array(service->routerOptions); - if ((service->router_instance = service->router->createInstance( - service, router_options))) - { - listeners += serviceStartAllPorts(service); - } - else - { - MXS_ERROR("%s: Failed to create router instance for service. Service not started.", - service->name); - service->state = SERVICE_STATE_FAILED; - } - free_string_array(router_options); + listeners = serviceStartAllPorts(service); } else { - MXS_ERROR("%s: Inadequate user permissions for service. Service not started.", - service->name); + MXS_ERROR("%s: Failed to create router instance. Service not started.", service->name); service->state = SERVICE_STATE_FAILED; } + + free_string_array(router_options); + return listeners; } @@ -1461,12 +1469,26 @@ int service_refresh_users(SERVICE *service) for (SERV_LISTENER *port = service->ports; port; port = port->next) { - if (port->listener->authfunc.loadusers && - port->listener->authfunc.loadusers(port) != MXS_AUTH_LOADUSERS_OK) + /** Load the authentication users before before starting the listener */ + if (port->listener->authfunc.loadusers) { - MXS_ERROR("[%s] Failed to load users for listener '%s', authentication might not work.", - service->name, port->name); - ret = 1; + switch (port->listener->authfunc.loadusers(port)) + { + case MXS_AUTH_LOADUSERS_FATAL: + MXS_ERROR("[%s] Fatal error when loading users for listener '%s'," + " authentication will not work.", service->name, port->name); + ret = 1; + break; + + case MXS_AUTH_LOADUSERS_ERROR: + MXS_WARNING("[%s] Failed to load users for listener '%s', authentication" + " might not work.", service->name, port->name); + ret = 1; + break; + + default: + break; + } } } } diff --git a/server/core/test/CMakeLists.txt b/server/core/test/CMakeLists.txt index b16fd2542..f12f22ca5 100644 --- a/server/core/test/CMakeLists.txt +++ b/server/core/test/CMakeLists.txt @@ -9,7 +9,6 @@ add_executable(test_log testlog.c) add_executable(test_logorder testlogorder.c) add_executable(test_logthrottling testlogthrottling.cc) add_executable(test_modutil testmodutil.c) -add_executable(test_mysql_users test_mysql_users.c) add_executable(test_poll testpoll.c) add_executable(test_queuemanager testqueuemanager.c) add_executable(test_server testserver.c) @@ -30,7 +29,6 @@ target_link_libraries(test_log maxscale-common) target_link_libraries(test_logorder maxscale-common) target_link_libraries(test_logthrottling maxscale-common) target_link_libraries(test_modutil maxscale-common) -target_link_libraries(test_mysql_users MySQLAuth MySQLCommon maxscale-common) target_link_libraries(test_poll maxscale-common) target_link_libraries(test_queuemanager maxscale-common) target_link_libraries(test_server maxscale-common) @@ -53,7 +51,6 @@ add_test(TestLogThrottling test_logthrottling) add_test(TestMaxScalePCRE2 testmaxscalepcre2) add_test(TestMemlog testmemlog) add_test(TestModutil test_modutil) -add_test(TestMySQLUsers test_mysql_users) add_test(NAME TestMaxPasswd COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/testmaxpasswd.sh) add_test(TestPoll test_poll) add_test(TestQueueManager test_queuemanager) diff --git a/server/modules/authenticator/CMakeLists.txt b/server/modules/authenticator/CMakeLists.txt index a7f6da3e1..1f339f7ce 100644 --- a/server/modules/authenticator/CMakeLists.txt +++ b/server/modules/authenticator/CMakeLists.txt @@ -1,7 +1,4 @@ -add_library(MySQLAuth SHARED mysql_auth.c) -target_link_libraries(MySQLAuth maxscale-common MySQLCommon) -set_target_properties(MySQLAuth PROPERTIES VERSION "1.0.0") -install_module(MySQLAuth core) +add_subdirectory(MySQLAuth) add_library(MySQLBackendAuth SHARED mysql_backend_auth.c) target_link_libraries(MySQLBackendAuth maxscale-common MySQLCommon) diff --git a/server/modules/authenticator/MySQLAuth/CMakeLists.txt b/server/modules/authenticator/MySQLAuth/CMakeLists.txt new file mode 100644 index 000000000..b27f39766 --- /dev/null +++ b/server/modules/authenticator/MySQLAuth/CMakeLists.txt @@ -0,0 +1,10 @@ +add_library(MySQLAuth SHARED mysql_auth.c dbusers.c) +target_link_libraries(MySQLAuth maxscale-common MySQLCommon) +set_target_properties(MySQLAuth PROPERTIES VERSION "1.0.0") +install_module(MySQLAuth core) + +if (BUILD_TESTS) + add_executable(test_mysql_users test_mysql_users.c) + target_link_libraries(test_mysql_users MySQLAuth MySQLCommon maxscale-common) + add_test(TestMySQLUsers test_mysql_users) +endif() diff --git a/server/core/dbusers.c b/server/modules/authenticator/MySQLAuth/dbusers.c similarity index 99% rename from server/core/dbusers.c rename to server/modules/authenticator/MySQLAuth/dbusers.c index 4098474e2..0859deaaf 100644 --- a/server/core/dbusers.c +++ b/server/modules/authenticator/MySQLAuth/dbusers.c @@ -42,7 +42,7 @@ #include #include #include -#include +#include "dbusers.h" #include #include #include diff --git a/include/maxscale/dbusers.h b/server/modules/authenticator/MySQLAuth/dbusers.h similarity index 76% rename from include/maxscale/dbusers.h rename to server/modules/authenticator/MySQLAuth/dbusers.h index 22b7bc2d0..35d6c998f 100644 --- a/include/maxscale/dbusers.h +++ b/server/modules/authenticator/MySQLAuth/dbusers.h @@ -32,25 +32,11 @@ #include #include +#include #include MXS_BEGIN_DECLS -/* Refresh rate limits for load users from database */ -#define USERS_REFRESH_TIME 30 /* Allowed time interval (in seconds) after last update*/ -#define USERS_REFRESH_MAX_PER_TIME 4 /* Max number of load calls within the time interval */ - -/** Default timeout values used by the connections which fetch user authentication data */ -#define DEFAULT_AUTH_CONNECT_TIMEOUT 3 -#define DEFAULT_AUTH_READ_TIMEOUT 1 -#define DEFAULT_AUTH_WRITE_TIMEOUT 2 - -/* Max length of fields in the mysql.user table */ -#define MYSQL_USER_MAXLEN 128 -#define MYSQL_PASSWORD_LEN 41 -#define MYSQL_HOST_MAXLEN 60 -#define MYSQL_DATABASE_MAXLEN 128 -#define MYSQL_TABLE_MAXLEN 64 /** Cache directory and file names */ static const char DBUSERS_DIR[] = "cache"; diff --git a/server/modules/authenticator/mysql_auth.c b/server/modules/authenticator/MySQLAuth/mysql_auth.c similarity index 99% rename from server/modules/authenticator/mysql_auth.c rename to server/modules/authenticator/MySQLAuth/mysql_auth.c index 4c002ae53..a7942783d 100644 --- a/server/modules/authenticator/mysql_auth.c +++ b/server/modules/authenticator/MySQLAuth/mysql_auth.c @@ -30,7 +30,7 @@ #include #include #include -#include +#include "dbusers.h" #include #include #include @@ -884,13 +884,20 @@ static bool add_service_user(SERV_LISTENER *port) * This function loads MySQL users from the backend database. * * @param port Listener definition - * @return AUTH_LOADUSERS_OK on success, AUTH_LOADUSERS_ERROR on error + * @return MXS_AUTH_LOADUSERS_OK on success, MXS_AUTH_LOADUSERS_ERROR and + * MXS_AUTH_LOADUSERS_FATAL on fatal error */ static int mysql_auth_load_users(SERV_LISTENER *port) { int rc = MXS_AUTH_LOADUSERS_OK; SERVICE *service = port->listener->service; MYSQL_AUTH *instance = (MYSQL_AUTH*)port->auth_instance; + + if (port->users == NULL && !check_service_permissions(port->service)) + { + return MXS_AUTH_LOADUSERS_FATAL; + } + int loaded = replace_mysql_users(port); char path[PATH_MAX]; diff --git a/server/core/test/test_mysql_users.c b/server/modules/authenticator/MySQLAuth/test_mysql_users.c similarity index 99% rename from server/core/test/test_mysql_users.c rename to server/modules/authenticator/MySQLAuth/test_mysql_users.c index 49f980f53..1a30b5e03 100644 --- a/server/core/test/test_mysql_users.c +++ b/server/modules/authenticator/MySQLAuth/test_mysql_users.c @@ -33,7 +33,7 @@ #include #include #include -#include +#include "dbusers.h" #include #include #include diff --git a/server/modules/routing/avro/avro_client.c b/server/modules/routing/avro/avro_client.c index cc0293b70..1caa06221 100644 --- a/server/modules/routing/avro/avro_client.c +++ b/server/modules/routing/avro/avro_client.c @@ -40,7 +40,6 @@ #include "avrorouter.h" #include #include -#include extern char *blr_extract_column(GWBUF *buf, int col); extern uint32_t extract_field(uint8_t *src, int bits); diff --git a/server/modules/routing/avro/avrorouter.h b/server/modules/routing/avro/avrorouter.h index 1b9accbe9..6bdeecceb 100644 --- a/server/modules/routing/avro/avrorouter.h +++ b/server/modules/routing/avro/avrorouter.h @@ -27,7 +27,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/routing/binlog/blr.c b/server/modules/routing/binlog/blr.c index 213b9eb87..350c04609 100644 --- a/server/modules/routing/binlog/blr.c +++ b/server/modules/routing/binlog/blr.c @@ -103,7 +103,6 @@ static void errorReply(ROUTER *instance, static uint64_t getCapabilities(); static int blr_handler_config(void *userdata, const char *section, const char *name, const char *value); static int blr_handle_config_item(const char *name, const char *value, ROUTER_INSTANCE *inst); -static int blr_set_service_mysql_user(SERVICE *service); static int blr_load_dbusers(const ROUTER_INSTANCE *router); static int blr_check_binlog(ROUTER_INSTANCE *router); int blr_read_events_all_events(ROUTER_INSTANCE *router, int fix, int debug); @@ -578,18 +577,6 @@ createInstance(SERVICE *service, char **options) } } - /* Allocate dbusers for this router here instead of serviceStartPort() */ - for (SERV_LISTENER *port = service->ports; port; port = port->next) - { - if ((port->users = mysql_users_alloc()) == NULL) - { - MXS_ERROR("%s: Error allocating dbusers in createInstance", - inst->service->name); - free_instance(inst); - return NULL; - } - } - /* Dynamically allocate master_host server struct, not written in any cnf file */ if (service->dbref == NULL) { @@ -674,19 +661,10 @@ createInstance(SERVICE *service, char **options) inst->service->name, inst->binlogdir); } - /* Set service user or load db users */ - blr_set_service_mysql_user(inst->service); - } else { inst->master_state = BLRM_UNCONNECTED; - - /* Try loading dbusers */ - if (inst->service->ports) - { - blr_load_dbusers(inst); - } } /** @@ -2134,140 +2112,6 @@ blr_handle_config_item(const char *name, const char *value, ROUTER_INSTANCE *ins return 1; } -/** - * Add the service user to mysql dbusers (service->users) - * via mysql_users_alloc and add_mysql_users_with_host_ipv4 - * User is added for '%' and 'localhost' hosts - * - * @param service The current service - * @return 0 on success, 1 on failure - */ -static int -blr_set_service_mysql_user(SERVICE *service) -{ - char *dpwd = NULL; - char *newpasswd = NULL; - char *service_user = NULL; - char *service_passwd = NULL; - - if (serviceGetUser(service, &service_user, &service_passwd) == 0) - { - MXS_ERROR("failed to get service user details for service %s", - service->name); - - return 1; - } - - dpwd = decryptPassword(service->credentials.authdata); - - if (!dpwd) - { - MXS_ERROR("decrypt password failed for service user %s, service %s", - service_user, - service->name); - - return 1; - } - - newpasswd = create_hex_sha1_sha1_passwd(dpwd); - - if (!newpasswd) - { - MXS_ERROR("create hex_sha1_sha1_password failed for service user %s", - service_user); - - MXS_FREE(dpwd); - return 1; - } - - /** Add the service user for % and localhost to all listeners so that - * it can always be used. */ - for (SERV_LISTENER *port = service->ports; port; port = port->next) - { - add_mysql_users_with_host_ipv4(port->users, service->credentials.name, - "%", newpasswd, "Y", ""); - add_mysql_users_with_host_ipv4(port->users, service->credentials.name, - "localhost", newpasswd, "Y", ""); - } - - MXS_FREE(newpasswd); - MXS_FREE(dpwd); - - return 0; -} - -/** - * Load mysql dbusers into (service->users) - * - * @param router The router instance - * @return -1 on failure, 0 for no users found, > 0 for found users - */ -static int -blr_load_dbusers(const ROUTER_INSTANCE *router) -{ - int loaded_total = 0; - SERVICE *service; - char path[PATH_MAX]; - service = router->service; - - for (SERV_LISTENER *port = service->ports; port; port = port->next) - { - sprintf(path, "%s/%s/%s/", router->binlogdir, BLR_DBUSERS_DIR, port->name); - - if (mxs_mkdir_all(path, 0775)) - { - strcat(path, BLR_DBUSERS_FILE); - } - - /* Try loading dbusers from configured backends */ - int loaded = load_mysql_users(port); - - if (loaded < 0) - { - MXS_ERROR("Unable to load users for service %s", service->name); - - /* Try loading authentication data from file cache */ - loaded = dbusers_load(port->users, path); - - if (loaded != -1) - { - MXS_ERROR("Service %s, Listener %s, Using cached credential information file %s.", - service->name, port->name, path); - } - else - { - MXS_ERROR("Service %s, Listener %s, Unable to read cache credential" - " information from %s. No database user added to service users table.", - service->name, port->name, path); - } - } - else - { - /* don't update cache if no user was loaded */ - if (loaded == 0) - { - MXS_ERROR("Service %s, Listener %s: failed to load any user information." - " Authentication will probably fail as a result.", - service->name, port->name); - } - else - { - /* update cached data */ - dbusers_save(port->users, path); - } - } - loaded_total += loaded; - } - - /* At service start last update is set to USERS_REFRESH_TIME seconds earlier. - * This way MaxScale could try reloading users' just after startup - */ - service->rate_limit.last = time(NULL) - USERS_REFRESH_TIME; - service->rate_limit.nloads = 1; - - return loaded_total; -} - /** * Extract a numeric field from a packet of the specified number of bits * diff --git a/server/modules/routing/binlog/blr_slave.c b/server/modules/routing/binlog/blr_slave.c index b958f5cd3..09613fcba 100644 --- a/server/modules/routing/binlog/blr_slave.c +++ b/server/modules/routing/binlog/blr_slave.c @@ -432,7 +432,7 @@ blr_slave_query(ROUTER_INSTANCE *router, ROUTER_SLAVE *slave, GWBUF *queue) else if (strcasecmp(word, "USER()") == 0) { /* Return user@host */ - char user_host[MYSQL_USER_MAXLEN + 1 + MYSQL_HOSTNAME_MAXLEN + 1] = ""; + char user_host[MYSQL_USER_MAXLEN + 1 + MYSQL_HOST_MAXLEN + 1] = ""; MXS_FREE(query_text); snprintf(user_host, sizeof(user_host), @@ -3401,8 +3401,6 @@ blr_stop_slave(ROUTER_INSTANCE* router, ROUTER_SLAVE* slave) static int blr_start_slave(ROUTER_INSTANCE* router, ROUTER_SLAVE* slave) { - int loaded; - /* if unconfigured return an error */ if (router->master_state == BLRM_UNCONFIGURED) { @@ -3532,28 +3530,7 @@ blr_start_slave(ROUTER_INSTANCE* router, ROUTER_SLAVE* slave) router->current_pos, router->binlog_position); /* Try reloading new users and update cached credentials */ - loaded = service_refresh_users(router->service); - - if (loaded == 0) - { - for (SERV_LISTENER *port = router->service->ports; port; port = port->next) - { - char path[PATH_MAX]; - sprintf(path, "%s/%s/%s/", router->binlogdir, BLR_DBUSERS_DIR, port->name); - - if (mxs_mkdir_all(path, 0775)) - { - strcat(path, BLR_DBUSERS_FILE); - dbusers_save(port->users, path); - } - } - } - else - { - MXS_NOTICE("Service %s: user credentials could not be refreshed. " - "Will use existing cached credentials (%s/%s) if possible.", - router->service->name, router->binlogdir, BLR_DBUSERS_DIR); - } + service_refresh_users(router->service); return blr_slave_send_ok(router, slave); } diff --git a/server/modules/routing/debugcli/debugcmd.c b/server/modules/routing/debugcli/debugcmd.c index 14d25249d..950f0882a 100644 --- a/server/modules/routing/debugcli/debugcmd.c +++ b/server/modules/routing/debugcli/debugcmd.c @@ -63,7 +63,6 @@ #include #include #include -#include #include #include #include diff --git a/server/modules/routing/maxinfo/maxinfo.c b/server/modules/routing/maxinfo/maxinfo.c index 574cd8418..d0fb26388 100644 --- a/server/modules/routing/maxinfo/maxinfo.c +++ b/server/modules/routing/maxinfo/maxinfo.c @@ -50,7 +50,6 @@ #include #include #include -#include MODULE_INFO info = @@ -69,7 +68,6 @@ static int maxinfo_statistics(INFO_INSTANCE *, INFO_SESSION *, GWBUF *); static int maxinfo_ping(INFO_INSTANCE *, INFO_SESSION *, GWBUF *); static int maxinfo_execute_query(INFO_INSTANCE *, INFO_SESSION *, char *); static int handle_url(INFO_INSTANCE *instance, INFO_SESSION *router_session, GWBUF *queue); -static int maxinfo_add_mysql_user(SERVICE *service); /* The router entry points */ @@ -182,13 +180,6 @@ createInstance(SERVICE *service, char **options) instances = inst; spinlock_release(&instlock); - /* - * The following add the service user to service->users via mysql_users_alloc() - * password to be used. - */ - - maxinfo_add_mysql_user(service); - return (ROUTER *)inst; } @@ -736,74 +727,3 @@ handle_url(INFO_INSTANCE *instance, INFO_SESSION *session, GWBUF *queue) gwbuf_free(queue); return 1; } - -/** - * Add the service user to the service->users - * via mysql_users_alloc and add_mysql_users_with_host_ipv4 - * User is added for '%' and 'localhost' hosts - * - * @param service The service for this router - * @return 0 on success, 1 on failure - */ -static int -maxinfo_add_mysql_user(SERVICE *service) -{ - int rval = 1; - char *service_user = NULL; - char *service_passwd = NULL; - - if (serviceGetUser(service, &service_user, &service_passwd) == 0) - { - MXS_ERROR("maxinfo: failed to get service user details"); - return 1; - } - - char *dpwd = decryptPassword(service->credentials.authdata); - - if (!dpwd) - { - MXS_ERROR("maxinfo: decrypt password failed for service user %s", service_user); - return 1; - } - - SERV_LISTENER *port = service->ports; - - while (port) - { - while (port && strcmp(port->protocol, "MySQLClient")) - { - port = port->next; - } - - if (port) - { - port->users = (void *)mysql_users_alloc(); - - char *newpasswd = create_hex_sha1_sha1_passwd(dpwd); - - if (!newpasswd) - { - MXS_ERROR("maxinfo: create hex_sha1_sha1_password failed for " - "service user %s", service_user); - users_free(port->users); - break; - } - - /* add service user for % and localhost */ - add_mysql_users_with_host_ipv4(port->users, service->credentials.name, - "%", newpasswd, "Y", ""); - add_mysql_users_with_host_ipv4(port->users, service->credentials.name, - "localhost", newpasswd, "Y", ""); - rval = 0; - MXS_FREE(newpasswd); - - /** Continue processing listeners in case there are multiple - * MySQLClient listeners*/ - port = port->next; - } - } - - MXS_FREE(dpwd); - - return rval; -} From 8f55cfbc16c84628f7122d4e74619b0df44ca165 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 21 Oct 2016 12:36:38 +0300 Subject: [PATCH 067/215] Fix cache_dir path handling in MySQLAuth The path that was given as the option for the cache directory wasn't properly checked for terminating forward slashes. Due to this, the cache file was created with the wrong name. --- include/maxscale/utils.h | 2 +- server/core/utils.c | 7 +++++-- server/modules/authenticator/MySQLAuth/mysql_auth.c | 5 +++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/include/maxscale/utils.h b/include/maxscale/utils.h index 55ce9a34e..67f699af2 100644 --- a/include/maxscale/utils.h +++ b/include/maxscale/utils.h @@ -72,7 +72,7 @@ char* replace_literal(char* haystack, const char* replacement); char* replace_quoted(const char** src, const size_t* srcsize, char** dest, size_t* destsize); -void clean_up_pathname(char *path); +bool clean_up_pathname(char *path); bool mxs_mkdir_all(const char *path, int mask); diff --git a/server/core/utils.c b/server/core/utils.c index 0688da6c8..9a930def9 100644 --- a/server/core/utils.c +++ b/server/core/utils.c @@ -315,14 +315,15 @@ char *create_hex_sha1_sha1_passwd(char *passwd) * Remove duplicate and trailing forward slashes from a path. * @param path Path to clean up */ -void clean_up_pathname(char *path) +bool clean_up_pathname(char *path) { char *data = path; size_t len = strlen(path); if (len > PATH_MAX) { - MXS_WARNING("Pathname too long: %s", path); + MXS_ERROR("Pathname too long: %s", path); + return false; } while (*data != '\0') @@ -350,6 +351,8 @@ void clean_up_pathname(char *path) len--; } } + + return true; } /** diff --git a/server/modules/authenticator/MySQLAuth/mysql_auth.c b/server/modules/authenticator/MySQLAuth/mysql_auth.c index a7942783d..a2fe41894 100644 --- a/server/modules/authenticator/MySQLAuth/mysql_auth.c +++ b/server/modules/authenticator/MySQLAuth/mysql_auth.c @@ -155,7 +155,8 @@ static void* mysql_auth_init(char **options) if (strcmp(options[i], "cache_dir") == 0) { - if ((instance->cache_dir = MXS_STRDUP(value)) == NULL) + if ((instance->cache_dir = MXS_STRDUP(value)) == NULL || + !clean_up_pathname(instance->cache_dir)) { error = true; } @@ -903,7 +904,7 @@ static int mysql_auth_load_users(SERV_LISTENER *port) if (instance->cache_dir) { - strcpy(path, instance->cache_dir); + snprintf(path, sizeof(path) - sizeof(DBUSERS_FILE) - 1, "%s/", instance->cache_dir); } else { From a4a7e806d0d69a0e21fd390859b806bf8229cf81 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 21 Oct 2016 13:07:18 +0300 Subject: [PATCH 068/215] Always replace MySQL users in MySQLAuth Doing the checksum matching after memory is allocated and all the work is done is not very efficient. A simpler solution is to always replace the users when we reload them. Replacing the users every time the service users are reloaded will not cause a degradation in performance because the previous implementation already does all the extra work but then just discards it. A faster solution would be to first query the server and request some sort of a checksum based on the result set the users query would create. Currently, this can be done inside a stored procedure but it is not very convenient for the average user. Another option would be to generate a long string with GROUP_CONCAT but it is highly likely that some internal buffer limit is hit before the complete value is calculated. --- .../modules/authenticator/MySQLAuth/dbusers.c | 53 +++---------------- .../modules/authenticator/MySQLAuth/dbusers.h | 2 - 2 files changed, 7 insertions(+), 48 deletions(-) diff --git a/server/modules/authenticator/MySQLAuth/dbusers.c b/server/modules/authenticator/MySQLAuth/dbusers.c index 0859deaaf..de4b5f2ec 100644 --- a/server/modules/authenticator/MySQLAuth/dbusers.c +++ b/server/modules/authenticator/MySQLAuth/dbusers.c @@ -235,19 +235,6 @@ static bool host_matches_singlechar_wildcard(const char* user, const char* wild) return true; } -/** - * Load the user/passwd form mysql.user table into the service users' hashtable - * environment. - * - * @param service The current service - * @return -1 on any error or the number of users inserted (0 means no users at all) - */ -int -load_mysql_users(SERV_LISTENER *listener) -{ - return get_users(listener, listener->users); -} - /** * Replace the user/passwd form mysql.user table into the service users' hashtable * environment. @@ -292,45 +279,19 @@ replace_mysql_users(SERV_LISTENER *listener) return i; } - USERS *oldusers = listener->users; - - /** - * TODO: Comparing the checksum after loading users is not necessary. We - * have already queried the server, allocated memory and done the processing - * so comparing if a change was made is pointless since the end result is - * always the same. We end up with either the same users or a new set of - * users. If the new users would always be taken into use, we'd avoid - * the costly task of calculating the diff. - * - * An improvement to the diff calculation would be to push the calculation - * to the backend server. This way the bandwidth usage would be minimized - * and the backend server would tell us if we need to query for more data. - */ - if (oldusers != NULL && memcmp(oldusers->cksum, newusers->cksum, - SHA_DIGEST_LENGTH) == 0) - { - /* same data, nothing to do */ - MXS_DEBUG("%lu [replace_mysql_users] users' tables not switched, checksum is the same", - pthread_self()); - - /* free the new table */ - users_free(newusers); - i = 0; - } - else - { - /* replace the service with effective new data */ - MXS_DEBUG("%lu [replace_mysql_users] users' tables replaced, checksum differs", - pthread_self()); - listener->users = newusers; - } + /** TODO: Figure out a way to create a checksum function in the backend server + * so that we can avoid querying the complete list of users every time we + * need to refresh the users */ + MXS_DEBUG("%lu [replace_mysql_users] users' tables replaced", pthread_self()); + USERS *oldusers = listener->users; + listener->users = newusers; spinlock_release(&listener->lock); /* free old resources */ resource_free(oldresources); - if (i && oldusers) + if (oldusers) { /* free the old table */ users_free(oldusers); diff --git a/server/modules/authenticator/MySQLAuth/dbusers.h b/server/modules/authenticator/MySQLAuth/dbusers.h index 35d6c998f..095df8b34 100644 --- a/server/modules/authenticator/MySQLAuth/dbusers.h +++ b/server/modules/authenticator/MySQLAuth/dbusers.h @@ -59,11 +59,9 @@ extern int add_mysql_users_with_host_ipv4(USERS *users, const char *user, const extern bool check_service_permissions(SERVICE* service); extern int dbusers_load(USERS *, const char *filename); extern int dbusers_save(USERS *, const char *filename); -extern int load_mysql_users(SERV_LISTENER *listener); extern int mysql_users_add(USERS *users, MYSQL_USER_HOST *key, char *auth); extern USERS *mysql_users_alloc(); extern char *mysql_users_fetch(USERS *users, MYSQL_USER_HOST *key); -extern int reload_mysql_users(SERV_LISTENER *listener); extern int replace_mysql_users(SERV_LISTENER *listener); MXS_END_DECLS From aa20385347659693a674b676cade8bb17630b788 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 21 Oct 2016 14:54:45 +0300 Subject: [PATCH 069/215] Log cached credentials path Logging the path to the cached credentials should help to the user if stale cache files are used. --- server/modules/authenticator/MySQLAuth/mysql_auth.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/modules/authenticator/MySQLAuth/mysql_auth.c b/server/modules/authenticator/MySQLAuth/mysql_auth.c index a2fe41894..238b54a9e 100644 --- a/server/modules/authenticator/MySQLAuth/mysql_auth.c +++ b/server/modules/authenticator/MySQLAuth/mysql_auth.c @@ -925,7 +925,7 @@ static int mysql_auth_load_users(SERV_LISTENER *port) } else { - MXS_WARNING("Using cached credential information."); + MXS_WARNING("[%s] Using cached credential information from '%s'.", service->name, path); } if (instance->inject_service_user) @@ -945,6 +945,7 @@ static int mysql_auth_load_users(SERV_LISTENER *port) { strcat(path, DBUSERS_FILE); dbusers_save(port->users, path); + MXS_INFO("[%s] Storing cached credential information at '%s'.", service->name, path); } } From c915ea71aa7a7455d71430c557bba9b6de64a250 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 21 Oct 2016 16:26:11 +0300 Subject: [PATCH 070/215] Update the user credential cache documentation The release notes now mention that the caching is done by the authenticators, not by the core. It also mentions that only MySQLAuth implements this functionality. Added notes to binlogrouter documentation about the changes to the loading and storing of users. Cleaned up MySQL-Authenticator document to make it a bit easier to read. --- .../Authenticators/MySQL-Authenticator.md | 16 +++++++------- .../MaxScale-2.1.0-Release-Notes.md | 5 +++++ Documentation/Routers/Binlogrouter.md | 21 +++++++++++++++++-- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/Documentation/Authenticators/MySQL-Authenticator.md b/Documentation/Authenticators/MySQL-Authenticator.md index 85a1adaca..82db5d52d 100644 --- a/Documentation/Authenticators/MySQL-Authenticator.md +++ b/Documentation/Authenticators/MySQL-Authenticator.md @@ -9,7 +9,10 @@ needs no further configuration to work. ## Authenticator options -The client authentication module, _MySQLAuth_, supports authenticator options. +The client authentication module, _MySQLAuth_, supports authenticator +options. The `authenticator_options` parameter is supported by listeners +and servers and expects a comma-separated list of key-value pairs. The +following options contain examples on how to define it. ### `cache_dir` @@ -17,19 +20,14 @@ The location where the user credential cache is stored. The default value for this is `///cache/` where `` by default is `/var/cache`. +If _cache_dir_ is defined, the user cache file is stored in `/`. No additional directories are appended to the _cache_dir_ value. + Each listener has its own user cache where the user credential information queried from the backends is stored. This information is used to authenticate users if a connection to the backend servers can't be made. -#### Example configuration - ``` -[Read-Write Listener] -type=listener -service=Read-Write Service -protocol=MySQLClient -port=4006 -authenticator=MySQLAuth authenticator_options=cache_dir=/tmp ``` diff --git a/Documentation/Release-Notes/MaxScale-2.1.0-Release-Notes.md b/Documentation/Release-Notes/MaxScale-2.1.0-Release-Notes.md index fc42d8c7d..ddd62b464 100644 --- a/Documentation/Release-Notes/MaxScale-2.1.0-Release-Notes.md +++ b/Documentation/Release-Notes/MaxScale-2.1.0-Release-Notes.md @@ -47,6 +47,11 @@ For more information about this configuration entry, please see ### User data cache +The user data cache stores the cached credentials that are used by some router +modules. In 2.1.0, the authenticator modules are responsible for the persisting +of the user data cache. Currently, only the MySQLAuth module implements user +data caching. + The user data loaded from the backend databases is now stored on a per listener basis instead of a per service basis. In earlier versions, each service had its own cache directory in `/var/cache/maxscale`. This directory contains cached user diff --git a/Documentation/Routers/Binlogrouter.md b/Documentation/Routers/Binlogrouter.md index a9e9b86aa..5d1bed760 100644 --- a/Documentation/Routers/Binlogrouter.md +++ b/Documentation/Routers/Binlogrouter.md @@ -16,8 +16,21 @@ Binlogrouter is configured with a comma-separated list of key-value pairs. The f ### `binlogdir` -This parameter allows the location that MariaDB MaxScale uses to store binlog files to be set. If this parameter is not set to a directory name then MariaDB MaxScale will store the binlog files in the directory /var/cache/maxscale/. -In the binlog dir there is also the 'cache' directory that contains data retrieved from the master during registration phase and the master.ini file which contains the configuration of current configured master. +This parameter allows the location that MariaDB MaxScale uses to store binlog +files to be set. If this parameter is not set to a directory name then MariaDB +MaxScale will store the binlog files in the directory +/var/cache/maxscale/. In the binlog dir there is also the 'cache' +directory that contains data retrieved from the master during registration phase +and the master.ini file which contains the configuration of current configured +master. + +From 2.1 onwards, the 'cache' directory is stored in the same location as other +user credential caches. This means that with the default options, the user +credential cache is stored in /var/cache/maxscale///cache/. + +Read the [MySQL Authenticator](../Authenticators/MySQL-Authenticator.md) +documentation for instructions on how to define a custom location for the user +cache. ### `uuid` @@ -52,6 +65,10 @@ This is the user name that MariaDB MaxScale uses when it connects to the master. This user is the only one available for MySQL connection to MaxScale Binlog Server for administration when master connection is not done yet. +In MaxScale 2.1, the service user injection is done by the MySQLAuth +authenticator module. Read the [MySQL Authenticator](../Authenticators/MySQL-Authenticator.md) +documentation for more details. + The user that is used for replication, either defined using the user= option in the router options or using the username and password defined of the service must be granted replication privileges on the database server. ``` From 0a9662c5282710a103451702d9a62664e8961c06 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 21 Oct 2016 18:13:31 +0300 Subject: [PATCH 071/215] Log authentication failures on warning instead of info level Logging authentication failures on warning level seems more appropriate and makes them easier to spot. --- server/modules/authenticator/MySQLAuth/mysql_auth.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/modules/authenticator/MySQLAuth/mysql_auth.c b/server/modules/authenticator/MySQLAuth/mysql_auth.c index 238b54a9e..17ade6b30 100644 --- a/server/modules/authenticator/MySQLAuth/mysql_auth.c +++ b/server/modules/authenticator/MySQLAuth/mysql_auth.c @@ -264,7 +264,7 @@ mysql_auth_authenticate(DCB *dcb) } else if (dcb->service->log_auth_warnings) { - MXS_NOTICE("%s: login attempt for user '%s'@%s:%d, authentication failed.", + MXS_WARNING("%s: login attempt for user '%s'@%s:%d, authentication failed.", dcb->service->name, client_data->user, dcb->remote, ntohs(dcb->ipv4.sin_port)); if (dcb->ipv4.sin_addr.s_addr == 0x0100007F && !dcb->service->localhost_match_wildcard_host) From ab9f8ebfacef7cc575a9dc1b7f60329611389e40 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Sat, 22 Oct 2016 05:06:42 +0300 Subject: [PATCH 072/215] Fix non-libedit builds of maxadmin A variable was not declared if libedit-devel wasn't installed. --- client/maxadmin.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/maxadmin.c b/client/maxadmin.c index dc057b2f6..96602b95e 100644 --- a/client/maxadmin.c +++ b/client/maxadmin.c @@ -345,7 +345,7 @@ main(int argc, char **argv) #else while (printf("MaxScale> ") && fgets(buf, 1024, stdin) != NULL) { - num = strlen(buf); + int num = strlen(buf); #endif /* Strip trailing \n\r */ for (int i = num - 1; buf[i] == '\r' || buf[i] == '\n'; i--) From d50acd02e4cee29b5a7749af0705b613e6a3c910 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Sat, 22 Oct 2016 05:57:36 +0300 Subject: [PATCH 073/215] Disable `make test` in Travis builds The internal test suite does not work in all cases. It should be fixed before being enabled again. --- .travis/build_maxscale.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis/build_maxscale.sh b/.travis/build_maxscale.sh index 916b8d37c..11c2c5ba6 100644 --- a/.travis/build_maxscale.sh +++ b/.travis/build_maxscale.sh @@ -15,7 +15,6 @@ cd build cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=Y -DBUILD_AVRO=N make -make test sudo make install sudo ./postinst From 59a4152d8acf0c373ff56ffab901911110cdd6fa Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Fri, 21 Oct 2016 14:14:50 +0300 Subject: [PATCH 074/215] Add getCapabilities to filters Common capabilities are now defined in routing.h. The common capabilities can be defined using bits 0 - 15. Router capabilities are defined using bits 16-31 and filter capabilities (should there ever be such) using bits 32-47. So, to find out the capabilities of a service you only need to OR the capabilities of the router and all filters together. For instance, if a single filter needs statement based routing, then that is what is done. --- include/maxscale/filter.h | 20 +++++++++- include/maxscale/router.h | 18 ++++++--- include/maxscale/routing.h | 38 +++++++++++++++++++ server/modules/filter/cache/cache.c | 13 +++++++ server/modules/filter/ccrfilter/ccrfilter.c | 12 ++++++ server/modules/filter/dbfwfilter/dbfwfilter.c | 12 ++++++ server/modules/filter/gatekeeper/gatekeeper.c | 13 +++++++ server/modules/filter/hintfilter/hintfilter.c | 12 ++++++ server/modules/filter/luafilter/luafilter.c | 12 ++++++ server/modules/filter/mqfilter/mqfilter.c | 12 ++++++ .../namedserverfilter/namedserverfilter.c | 12 ++++++ server/modules/filter/qlafilter/qlafilter.c | 12 ++++++ .../modules/filter/regexfilter/regexfilter.c | 12 ++++++ server/modules/filter/tee/tee.c | 12 ++++++ server/modules/filter/testfilter/testfilter.c | 12 ++++++ server/modules/filter/topfilter/topfilter.c | 12 ++++++ server/modules/filter/tpmfilter/tpmfilter.c | 12 ++++++ .../routing/readconnroute/readconnroute.c | 2 +- 18 files changed, 240 insertions(+), 8 deletions(-) create mode 100644 include/maxscale/routing.h diff --git a/include/maxscale/filter.h b/include/maxscale/filter.h index 780ed06f2..995901a0a 100644 --- a/include/maxscale/filter.h +++ b/include/maxscale/filter.h @@ -25,6 +25,7 @@ */ #include +#include #include #include #include @@ -83,6 +84,7 @@ typedef struct filter_object int (*routeQuery)(FILTER *instance, void *fsession, GWBUF *queue); int (*clientReply)(FILTER *instance, void *fsession, GWBUF *queue); void (*diagnostics)(FILTER *instance, void *fsession, DCB *dcb); + uint64_t (*getCapabilities)(); } FILTER_OBJECT; /** @@ -90,7 +92,7 @@ typedef struct filter_object * is changed these values must be updated in line with the rules in the * file modinfo.h. */ -#define FILTER_VERSION {2, 1, 0} +#define FILTER_VERSION {2, 2, 0} /** * The definition of a filter from the configuration file. * This is basically the link between a plugin to load and the @@ -121,6 +123,22 @@ void dprintAllFilters(DCB *); void dprintFilter(DCB *, FILTER_DEF *); void dListFilters(DCB *); +/** + * Specifies capabilities specific for filters. Common capabilities + * are defined by @c routing_capability_t. + * + * @see routing_capability_t + * + * @note The values of the capabilities here *must* be between 0x000100000000 + * and 0x800000000000, that is, bits 32 to 47. + */ + +/* +typedef enum filter_capability +{ +} filter_capability_t; +*/ + MXS_END_DECLS #endif diff --git a/include/maxscale/router.h b/include/maxscale/router.h index 40b27038a..28ed196fe 100644 --- a/include/maxscale/router.h +++ b/include/maxscale/router.h @@ -30,6 +30,7 @@ */ #include +#include #include #include #include @@ -93,14 +94,19 @@ typedef struct router_object #define ROUTER_VERSION { 2, 0, 0 } /** - * Router capability type. Indicates what kind of input router accepts. + * Specifies capabilities specific for routers. Common capabilities + * are defined by @c routing_capability_t. + * + * @see routing_capability_t + * + * @note The values of the capabilities here *must* be between 0x00010000 + * and 0x80000000, that is, bits 16 to 31. */ -typedef enum router_capability_t +typedef enum router_capability { - RCAP_TYPE_UNDEFINED = 0x00, - RCAP_TYPE_STMT_INPUT = 0x01, /**< Statement per buffer */ - RCAP_TYPE_PACKET_INPUT = 0x02, /**< Data as it was read from DCB */ - RCAP_TYPE_NO_RSESSION = 0x04 /**< Router does not use router sessions */ + RCAP_TYPE_NO_RSESSION = 0x00010000, /**< Router does not use router sessions */ + RCAP_TYPE_NO_USERS_INIT = 0x00020000, /**< Prevent the loading of authenticator + users when the service is started */ } router_capability_t; MXS_END_DECLS diff --git a/include/maxscale/routing.h b/include/maxscale/routing.h new file mode 100644 index 000000000..447d1146d --- /dev/null +++ b/include/maxscale/routing.h @@ -0,0 +1,38 @@ +#pragma once +/* + * Copyright (c) 2016 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl. + * + * Change Date: 2019-07-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ + +/** + * @file routing.h - Common definitions and declarations for routers and filters. + */ + +#include + +MXS_BEGIN_DECLS + +/** + * Routing capability type. Indicates what kind of input a router or + * a filter accepts. + * + * @note The values of the capabilities here *must* be between 0x0000 + * and 0x8000, that is, bits 0 to 15. + */ +typedef enum routing_capability +{ + RCAP_TYPE_STMT_INPUT = 0x0001, /**< Statement per buffer. */ +} routing_capability_t; + +#define RCAP_TYPE_NONE 0 + +MXS_END_DECLS + diff --git a/server/modules/filter/cache/cache.c b/server/modules/filter/cache/cache.c index 15e9aa9dc..9342a57f0 100644 --- a/server/modules/filter/cache/cache.c +++ b/server/modules/filter/cache/cache.c @@ -35,6 +35,7 @@ static void setUpstream(FILTER *instance, void *sdata, UPSTREAM *upstream); static int routeQuery(FILTER *instance, void *sdata, GWBUF *queue); static int clientReply(FILTER *instance, void *sdata, GWBUF *queue); static void diagnostics(FILTER *instance, void *sdata, DCB *dcb); +static uint64_t getCapabilities(); #define C_DEBUG(format, ...) MXS_LOG_MESSAGE(LOG_NOTICE, format, ##__VA_ARGS__) @@ -81,6 +82,7 @@ FILTER_OBJECT *GetModuleObject() routeQuery, clientReply, diagnostics, + getCapabilities }; return &object; @@ -550,6 +552,17 @@ static void diagnostics(FILTER *instance, void *sdata, DCB *dcb) dcb_printf(dcb, "Hello World from Cache!\n"); } + +/** + * Capability routine. + * + * @return The capabilities of the filter. + */ +static uint64_t getCapabilities() +{ + return RCAP_TYPE_STMT_INPUT; +} + // // API END // diff --git a/server/modules/filter/ccrfilter/ccrfilter.c b/server/modules/filter/ccrfilter/ccrfilter.c index f9fe4f796..a144f2295 100644 --- a/server/modules/filter/ccrfilter/ccrfilter.c +++ b/server/modules/filter/ccrfilter/ccrfilter.c @@ -63,6 +63,7 @@ static void freeSession(FILTER *instance, void *session); static void setDownstream(FILTER *instance, void *fsession, DOWNSTREAM *downstream); static int routeQuery(FILTER *instance, void *fsession, GWBUF *queue); static void diagnostic(FILTER *instance, void *fsession, DCB *dcb); +static uint64_t getCapabilities(); static FILTER_OBJECT MyObject = @@ -76,6 +77,7 @@ static FILTER_OBJECT MyObject = routeQuery, NULL, diagnostic, + getCapabilities, }; #define CCR_DEFAULT_TIME 60 @@ -412,3 +414,13 @@ diagnostic(FILTER *instance, void *fsession, DCB *dcb) dcb_printf(dcb, "\tNo. of hints added based on count: %d\n", my_instance->stats.n_add_count); dcb_printf(dcb, "\tNo. of hints added based on time: %d\n", my_instance->stats.n_add_time); } + +/** + * Capability routine. + * + * @return The capabilities of the filter. + */ +static uint64_t getCapabilities() +{ + return RCAP_TYPE_STMT_INPUT; +} diff --git a/server/modules/filter/dbfwfilter/dbfwfilter.c b/server/modules/filter/dbfwfilter/dbfwfilter.c index cebe94df1..66ad835ed 100644 --- a/server/modules/filter/dbfwfilter/dbfwfilter.c +++ b/server/modules/filter/dbfwfilter/dbfwfilter.c @@ -105,6 +105,7 @@ static void freeSession(FILTER *instance, void *session); static void setDownstream(FILTER *instance, void *fsession, DOWNSTREAM *downstream); static int routeQuery(FILTER *instance, void *fsession, GWBUF *queue); static void diagnostic(FILTER *instance, void *fsession, DCB *dcb); +static uint64_t getCapabilities(); static FILTER_OBJECT MyObject = { @@ -117,6 +118,7 @@ static FILTER_OBJECT MyObject = routeQuery, NULL, diagnostic, + getCapabilities, }; /** @@ -2328,6 +2330,16 @@ diagnostic(FILTER *instance, void *fsession, DCB *dcb) } } +/** + * Capability routine. + * + * @return The capabilities of the filter. + */ +static uint64_t getCapabilities() +{ + return RCAP_TYPE_STMT_INPUT; +} + #ifdef BUILD_RULE_PARSER // TODO: Not ok to include file from other component's test directory. #include "../../../core/test/test_utils.h" diff --git a/server/modules/filter/gatekeeper/gatekeeper.c b/server/modules/filter/gatekeeper/gatekeeper.c index e794c9a3b..d49153df0 100644 --- a/server/modules/filter/gatekeeper/gatekeeper.c +++ b/server/modules/filter/gatekeeper/gatekeeper.c @@ -89,6 +89,8 @@ static void freeSession(FILTER *instance, void *session); static void setDownstream(FILTER *instance, void *fsession, DOWNSTREAM *downstream); static int routeQuery(FILTER *instance, void *fsession, GWBUF *queue); static void diagnostic(FILTER *instance, void *fsession, DCB *dcb); +static uint64_t getCapabilities(); + static bool read_stored_data(GK_INSTANCE *inst); static bool write_stored_data(GK_INSTANCE *inst); @@ -103,6 +105,7 @@ static FILTER_OBJECT MyObject = routeQuery, NULL, diagnostic, + getCapabilities, }; /** @@ -422,6 +425,16 @@ static void diagnostic(FILTER *instance, void *fsession, DCB *dcb) dcb_printf(dcb, "\t\tQueryhash misses: %u\n", inst->stats.miss); } +/** + * @brief Capability routine. + * + * @return The capabilities of the filter. + */ +static uint64_t getCapabilities() +{ + return RCAP_TYPE_STMT_INPUT; +} + /** * @brief Write query patterns from memory to disk * diff --git a/server/modules/filter/hintfilter/hintfilter.c b/server/modules/filter/hintfilter/hintfilter.c index c7efaabaa..7d0bfc258 100644 --- a/server/modules/filter/hintfilter/hintfilter.c +++ b/server/modules/filter/hintfilter/hintfilter.c @@ -40,6 +40,7 @@ static void freeSession(FILTER *instance, void *session); static void setDownstream(FILTER *instance, void *fsession, DOWNSTREAM *downstream); static int routeQuery(FILTER *instance, void *fsession, GWBUF *queue); static void diagnostic(FILTER *instance, void *fsession, DCB *dcb); +static uint64_t getCapabilities(); static FILTER_OBJECT MyObject = @@ -53,6 +54,7 @@ static FILTER_OBJECT MyObject = routeQuery, NULL, diagnostic, + getCapabilities, }; /** @@ -278,3 +280,13 @@ diagnostic(FILTER *instance, void *fsession, DCB *dcb) HINT_SESSION *my_session = (HINT_SESSION *)fsession; } + +/** + * Capability routine. + * + * @return The capabilities of the filter. + */ +static uint64_t getCapabilities() +{ + return RCAP_TYPE_STMT_INPUT; +} diff --git a/server/modules/filter/luafilter/luafilter.c b/server/modules/filter/luafilter/luafilter.c index 5da797e47..fc523519d 100644 --- a/server/modules/filter/luafilter/luafilter.c +++ b/server/modules/filter/luafilter/luafilter.c @@ -81,6 +81,7 @@ static void setUpstream(FILTER *instance, void *fsession, UPSTREAM *upstream); static int routeQuery(FILTER *instance, void *fsession, GWBUF *queue); static int clientReply(FILTER *instance, void *fsession, GWBUF *queue); static void diagnostic(FILTER *instance, void *fsession, DCB *dcb); +static uint64_t getCapabilities(); static FILTER_OBJECT MyObject = @@ -94,6 +95,7 @@ static FILTER_OBJECT MyObject = routeQuery, clientReply, diagnostic, + getCapabilities, }; /** @@ -589,3 +591,13 @@ static void diagnostic(FILTER *instance, void *fsession, DCB *dcb) } } } + +/** + * Capability routine. + * + * @return The capabilities of the filter. + */ +static uint64_t getCapabilities() +{ + return RCAP_TYPE_STMT_INPUT; +} diff --git a/server/modules/filter/mqfilter/mqfilter.c b/server/modules/filter/mqfilter/mqfilter.c index 3642eb3f3..d4279ca79 100644 --- a/server/modules/filter/mqfilter/mqfilter.c +++ b/server/modules/filter/mqfilter/mqfilter.c @@ -101,6 +101,7 @@ static void setUpstream(FILTER *instance, void *fsession, UPSTREAM *upstream); static int routeQuery(FILTER *instance, void *fsession, GWBUF *queue); static int clientReply(FILTER *instance, void *fsession, GWBUF *queue); static void diagnostic(FILTER *instance, void *fsession, DCB *dcb); +static uint64_t getCapabilities(); static FILTER_OBJECT MyObject = @@ -114,6 +115,7 @@ static FILTER_OBJECT MyObject = routeQuery, clientReply, diagnostic, + getCapabilities, }; /** @@ -1703,3 +1705,13 @@ diagnostic(FILTER *instance, void *fsession, DCB *dcb) my_instance->stats.n_sent); } } + +/** + * Capability routine. + * + * @return The capabilities of the filter. + */ +static uint64_t getCapabilities() +{ + return RCAP_TYPE_STMT_INPUT; +} diff --git a/server/modules/filter/namedserverfilter/namedserverfilter.c b/server/modules/filter/namedserverfilter/namedserverfilter.c index c30315a7f..246ebb7db 100644 --- a/server/modules/filter/namedserverfilter/namedserverfilter.c +++ b/server/modules/filter/namedserverfilter/namedserverfilter.c @@ -57,6 +57,7 @@ static void freeSession(FILTER *instance, void *session); static void setDownstream(FILTER *instance, void *fsession, DOWNSTREAM *downstream); static int routeQuery(FILTER *instance, void *fsession, GWBUF *queue); static void diagnostic(FILTER *instance, void *fsession, DCB *dcb); +static uint64_t getCapabilities(); static FILTER_OBJECT MyObject = @@ -70,6 +71,7 @@ static FILTER_OBJECT MyObject = routeQuery, NULL, diagnostic, + getCapabilities, }; /** @@ -410,3 +412,13 @@ diagnostic(FILTER *instance, void *fsession, DCB *dcb) my_instance->user); } } + +/** + * Capability routine. + * + * @return The capabilities of the filter. + */ +static uint64_t getCapabilities() +{ + return RCAP_TYPE_STMT_INPUT; +} diff --git a/server/modules/filter/qlafilter/qlafilter.c b/server/modules/filter/qlafilter/qlafilter.c index ec0690479..1f9ebe7e0 100644 --- a/server/modules/filter/qlafilter/qlafilter.c +++ b/server/modules/filter/qlafilter/qlafilter.c @@ -72,6 +72,7 @@ static void freeSession(FILTER *instance, void *session); static void setDownstream(FILTER *instance, void *fsession, DOWNSTREAM *downstream); static int routeQuery(FILTER *instance, void *fsession, GWBUF *queue); static void diagnostic(FILTER *instance, void *fsession, DCB *dcb); +static uint64_t getCapabilities(); static FILTER_OBJECT MyObject = @@ -85,6 +86,7 @@ static FILTER_OBJECT MyObject = routeQuery, NULL, // No client reply diagnostic, + getCapabilities, }; /** @@ -514,3 +516,13 @@ diagnostic(FILTER *instance, void *fsession, DCB *dcb) my_instance->nomatch); } } + +/** + * Capability routine. + * + * @return The capabilities of the filter. + */ +static uint64_t getCapabilities() +{ + return RCAP_TYPE_STMT_INPUT; +} diff --git a/server/modules/filter/regexfilter/regexfilter.c b/server/modules/filter/regexfilter/regexfilter.c index 3266f129c..921c6d030 100644 --- a/server/modules/filter/regexfilter/regexfilter.c +++ b/server/modules/filter/regexfilter/regexfilter.c @@ -57,6 +57,7 @@ static void freeSession(FILTER *instance, void *session); static void setDownstream(FILTER *instance, void *fsession, DOWNSTREAM *downstream); static int routeQuery(FILTER *instance, void *fsession, GWBUF *queue); static void diagnostic(FILTER *instance, void *fsession, DCB *dcb); +static uint64_t getCapabilities(); static char *regex_replace(const char *sql, pcre2_code *re, pcre2_match_data *study, const char *replace); @@ -72,6 +73,7 @@ static FILTER_OBJECT MyObject = routeQuery, NULL, diagnostic, + getCapabilities, }; /** @@ -545,3 +547,13 @@ void log_nomatch(REGEX_INSTANCE* inst, char* re, char* old) MXS_INFO("No match %s: [%s]", re, old); } } + +/** + * Capability routine. + * + * @return The capabilities of the filter. + */ +static uint64_t getCapabilities() +{ + return RCAP_TYPE_STMT_INPUT; +} diff --git a/server/modules/filter/tee/tee.c b/server/modules/filter/tee/tee.c index 304b8ae3d..69b02c7ee 100644 --- a/server/modules/filter/tee/tee.c +++ b/server/modules/filter/tee/tee.c @@ -116,6 +116,7 @@ static void setUpstream(FILTER *instance, void *fsession, UPSTREAM *upstream); static int routeQuery(FILTER *instance, void *fsession, GWBUF *queue); static int clientReply(FILTER *instance, void *fsession, GWBUF *queue); static void diagnostic(FILTER *instance, void *fsession, DCB *dcb); +static uint64_t getCapabilities(); static FILTER_OBJECT MyObject = { @@ -128,6 +129,7 @@ static FILTER_OBJECT MyObject = routeQuery, clientReply, diagnostic, + getCapabilities, }; /** @@ -1124,6 +1126,16 @@ diagnostic(FILTER *instance, void *fsession, DCB *dcb) } } +/** + * Capability routine. + * + * @return The capabilities of the filter. + */ +static uint64_t getCapabilities() +{ + return RCAP_TYPE_STMT_INPUT; +} + /** * Determine if the packet is a command that must be sent to the branch * to maintain the session consistancy. These are COM_INIT_DB, diff --git a/server/modules/filter/testfilter/testfilter.c b/server/modules/filter/testfilter/testfilter.c index 8caa25b6e..e2ce7bbd0 100644 --- a/server/modules/filter/testfilter/testfilter.c +++ b/server/modules/filter/testfilter/testfilter.c @@ -46,6 +46,7 @@ static void freeSession(FILTER *instance, void *session); static void setDownstream(FILTER *instance, void *fsession, DOWNSTREAM *downstream); static int routeQuery(FILTER *instance, void *fsession, GWBUF *queue); static void diagnostic(FILTER *instance, void *fsession, DCB *dcb); +static uint64_t getCapabilities(); static FILTER_OBJECT MyObject = @@ -59,6 +60,7 @@ static FILTER_OBJECT MyObject = routeQuery, NULL, diagnostic, + getCapabilities, }; /** @@ -246,3 +248,13 @@ diagnostic(FILTER *instance, void *fsession, DCB *dcb) dcb_printf(dcb, "\t\tNo. of sessions created: %d\n", my_instance->sessions); } + +/** + * Capability routine. + * + * @return The capabilities of the filter. + */ +static uint64_t getCapabilities() +{ + return RCAP_TYPE_NONE; +} diff --git a/server/modules/filter/topfilter/topfilter.c b/server/modules/filter/topfilter/topfilter.c index 5acbed3e8..2f69868e1 100644 --- a/server/modules/filter/topfilter/topfilter.c +++ b/server/modules/filter/topfilter/topfilter.c @@ -67,6 +67,7 @@ static void setUpstream(FILTER *instance, void *fsession, UPSTREAM *upstream); static int routeQuery(FILTER *instance, void *fsession, GWBUF *queue); static int clientReply(FILTER *instance, void *fsession, GWBUF *queue); static void diagnostic(FILTER *instance, void *fsession, DCB *dcb); +static uint64_t getCapabilities(); static FILTER_OBJECT MyObject = @@ -80,6 +81,7 @@ static FILTER_OBJECT MyObject = routeQuery, clientReply, diagnostic, + getCapabilities, }; /** @@ -682,3 +684,13 @@ diagnostic(FILTER *instance, void *fsession, DCB *dcb) } } } + +/** + * Capability routine. + * + * @return The capabilities of the filter. + */ +static uint64_t getCapabilities() +{ + return RCAP_TYPE_STMT_INPUT; +} diff --git a/server/modules/filter/tpmfilter/tpmfilter.c b/server/modules/filter/tpmfilter/tpmfilter.c index 97fddf10d..e57063826 100644 --- a/server/modules/filter/tpmfilter/tpmfilter.c +++ b/server/modules/filter/tpmfilter/tpmfilter.c @@ -81,6 +81,7 @@ static void setUpstream(FILTER *instance, void *fsession, UPSTREAM *upstream static int routeQuery(FILTER *instance, void *fsession, GWBUF *queue); static int clientReply(FILTER *instance, void *fsession, GWBUF *queue); static void diagnostic(FILTER *instance, void *fsession, DCB *dcb); +static uint64_t getCapabilities(); static FILTER_OBJECT MyObject = { @@ -93,6 +94,7 @@ static FILTER_OBJECT MyObject = routeQuery, clientReply, diagnostic, + getCapabilities, }; /** @@ -557,3 +559,13 @@ diagnostic(FILTER *instance, void *fsession, DCB *dcb) dcb_printf(dcb, "\t\tLogging with query delimiter %s.\n", my_instance->query_delimiter); } + +/** + * Capability routine. + * + * @return The capabilities of the filter. + */ +static uint64_t getCapabilities() +{ + return RCAP_TYPE_STMT_INPUT; +} diff --git a/server/modules/routing/readconnroute/readconnroute.c b/server/modules/routing/readconnroute/readconnroute.c index a9f3a27c9..704b1613b 100644 --- a/server/modules/routing/readconnroute/readconnroute.c +++ b/server/modules/routing/readconnroute/readconnroute.c @@ -987,7 +987,7 @@ static void rses_end_locked_router_action(ROUTER_CLIENT_SES* rses) static uint64_t getCapabilities() { - return RCAP_TYPE_PACKET_INPUT; + return RCAP_TYPE_NONE; } /******************************** From db2cccbd8f550a0178d116ec850e0baf502ae7c0 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Fri, 21 Oct 2016 20:59:30 +0300 Subject: [PATCH 075/215] Replace getCapabilities() with getCapabilities(void) The latter means no args, the former means any args. --- include/maxscale/filter.h | 2 +- include/maxscale/router.h | 2 +- server/modules/filter/cache/cache.c | 4 ++-- server/modules/filter/ccrfilter/ccrfilter.c | 4 ++-- server/modules/filter/dbfwfilter/dbfwfilter.c | 4 ++-- server/modules/filter/gatekeeper/gatekeeper.c | 4 ++-- server/modules/filter/hintfilter/hintfilter.c | 4 ++-- server/modules/filter/luafilter/luafilter.c | 4 ++-- server/modules/filter/mqfilter/mqfilter.c | 4 ++-- server/modules/filter/namedserverfilter/namedserverfilter.c | 4 ++-- server/modules/filter/qlafilter/qlafilter.c | 4 ++-- server/modules/filter/regexfilter/regexfilter.c | 4 ++-- server/modules/filter/tee/tee.c | 4 ++-- server/modules/filter/testfilter/testfilter.c | 4 ++-- server/modules/filter/topfilter/topfilter.c | 4 ++-- server/modules/filter/tpmfilter/tpmfilter.c | 4 ++-- server/modules/protocol/MySQL/MySQLClient/mysql_client.c | 3 +-- server/modules/routing/avro/avro.c | 4 ++-- server/modules/routing/binlog/blr.c | 4 ++-- server/modules/routing/cli/cli.c | 4 ++-- server/modules/routing/debugcli/debugcli.c | 2 +- server/modules/routing/maxinfo/maxinfo.c | 4 ++-- server/modules/routing/readconnroute/readconnroute.c | 4 ++-- server/modules/routing/readwritesplit/readwritesplit.c | 4 ++-- server/modules/routing/schemarouter/schemarouter.c | 4 ++-- server/modules/routing/testroute/testroute.c | 2 +- 26 files changed, 47 insertions(+), 48 deletions(-) diff --git a/include/maxscale/filter.h b/include/maxscale/filter.h index 995901a0a..ec837d8e9 100644 --- a/include/maxscale/filter.h +++ b/include/maxscale/filter.h @@ -84,7 +84,7 @@ typedef struct filter_object int (*routeQuery)(FILTER *instance, void *fsession, GWBUF *queue); int (*clientReply)(FILTER *instance, void *fsession, GWBUF *queue); void (*diagnostics)(FILTER *instance, void *fsession, DCB *dcb); - uint64_t (*getCapabilities)(); + uint64_t (*getCapabilities)(void); } FILTER_OBJECT; /** diff --git a/include/maxscale/router.h b/include/maxscale/router.h index 28ed196fe..7ae6e3f2a 100644 --- a/include/maxscale/router.h +++ b/include/maxscale/router.h @@ -83,7 +83,7 @@ typedef struct router_object DCB* backend_dcb, error_action_t action, bool* succp); - uint64_t (*getCapabilities)(); + uint64_t (*getCapabilities)(void); } ROUTER_OBJECT; /** diff --git a/server/modules/filter/cache/cache.c b/server/modules/filter/cache/cache.c index 9342a57f0..fa6ef74d0 100644 --- a/server/modules/filter/cache/cache.c +++ b/server/modules/filter/cache/cache.c @@ -35,7 +35,7 @@ static void setUpstream(FILTER *instance, void *sdata, UPSTREAM *upstream); static int routeQuery(FILTER *instance, void *sdata, GWBUF *queue); static int clientReply(FILTER *instance, void *sdata, GWBUF *queue); static void diagnostics(FILTER *instance, void *sdata, DCB *dcb); -static uint64_t getCapabilities(); +static uint64_t getCapabilities(void); #define C_DEBUG(format, ...) MXS_LOG_MESSAGE(LOG_NOTICE, format, ##__VA_ARGS__) @@ -558,7 +558,7 @@ static void diagnostics(FILTER *instance, void *sdata, DCB *dcb) * * @return The capabilities of the filter. */ -static uint64_t getCapabilities() +static uint64_t getCapabilities(void) { return RCAP_TYPE_STMT_INPUT; } diff --git a/server/modules/filter/ccrfilter/ccrfilter.c b/server/modules/filter/ccrfilter/ccrfilter.c index a144f2295..a24391b21 100644 --- a/server/modules/filter/ccrfilter/ccrfilter.c +++ b/server/modules/filter/ccrfilter/ccrfilter.c @@ -63,7 +63,7 @@ static void freeSession(FILTER *instance, void *session); static void setDownstream(FILTER *instance, void *fsession, DOWNSTREAM *downstream); static int routeQuery(FILTER *instance, void *fsession, GWBUF *queue); static void diagnostic(FILTER *instance, void *fsession, DCB *dcb); -static uint64_t getCapabilities(); +static uint64_t getCapabilities(void); static FILTER_OBJECT MyObject = @@ -420,7 +420,7 @@ diagnostic(FILTER *instance, void *fsession, DCB *dcb) * * @return The capabilities of the filter. */ -static uint64_t getCapabilities() +static uint64_t getCapabilities(void) { return RCAP_TYPE_STMT_INPUT; } diff --git a/server/modules/filter/dbfwfilter/dbfwfilter.c b/server/modules/filter/dbfwfilter/dbfwfilter.c index 66ad835ed..5008bfad2 100644 --- a/server/modules/filter/dbfwfilter/dbfwfilter.c +++ b/server/modules/filter/dbfwfilter/dbfwfilter.c @@ -105,7 +105,7 @@ static void freeSession(FILTER *instance, void *session); static void setDownstream(FILTER *instance, void *fsession, DOWNSTREAM *downstream); static int routeQuery(FILTER *instance, void *fsession, GWBUF *queue); static void diagnostic(FILTER *instance, void *fsession, DCB *dcb); -static uint64_t getCapabilities(); +static uint64_t getCapabilities(void); static FILTER_OBJECT MyObject = { @@ -2335,7 +2335,7 @@ diagnostic(FILTER *instance, void *fsession, DCB *dcb) * * @return The capabilities of the filter. */ -static uint64_t getCapabilities() +static uint64_t getCapabilities(void) { return RCAP_TYPE_STMT_INPUT; } diff --git a/server/modules/filter/gatekeeper/gatekeeper.c b/server/modules/filter/gatekeeper/gatekeeper.c index d49153df0..58e1d0fd2 100644 --- a/server/modules/filter/gatekeeper/gatekeeper.c +++ b/server/modules/filter/gatekeeper/gatekeeper.c @@ -89,7 +89,7 @@ static void freeSession(FILTER *instance, void *session); static void setDownstream(FILTER *instance, void *fsession, DOWNSTREAM *downstream); static int routeQuery(FILTER *instance, void *fsession, GWBUF *queue); static void diagnostic(FILTER *instance, void *fsession, DCB *dcb); -static uint64_t getCapabilities(); +static uint64_t getCapabilities(void); static bool read_stored_data(GK_INSTANCE *inst); static bool write_stored_data(GK_INSTANCE *inst); @@ -430,7 +430,7 @@ static void diagnostic(FILTER *instance, void *fsession, DCB *dcb) * * @return The capabilities of the filter. */ -static uint64_t getCapabilities() +static uint64_t getCapabilities(void) { return RCAP_TYPE_STMT_INPUT; } diff --git a/server/modules/filter/hintfilter/hintfilter.c b/server/modules/filter/hintfilter/hintfilter.c index 7d0bfc258..b6a0e7dd3 100644 --- a/server/modules/filter/hintfilter/hintfilter.c +++ b/server/modules/filter/hintfilter/hintfilter.c @@ -40,7 +40,7 @@ static void freeSession(FILTER *instance, void *session); static void setDownstream(FILTER *instance, void *fsession, DOWNSTREAM *downstream); static int routeQuery(FILTER *instance, void *fsession, GWBUF *queue); static void diagnostic(FILTER *instance, void *fsession, DCB *dcb); -static uint64_t getCapabilities(); +static uint64_t getCapabilities(void); static FILTER_OBJECT MyObject = @@ -286,7 +286,7 @@ diagnostic(FILTER *instance, void *fsession, DCB *dcb) * * @return The capabilities of the filter. */ -static uint64_t getCapabilities() +static uint64_t getCapabilities(void) { return RCAP_TYPE_STMT_INPUT; } diff --git a/server/modules/filter/luafilter/luafilter.c b/server/modules/filter/luafilter/luafilter.c index fc523519d..137446f8c 100644 --- a/server/modules/filter/luafilter/luafilter.c +++ b/server/modules/filter/luafilter/luafilter.c @@ -81,7 +81,7 @@ static void setUpstream(FILTER *instance, void *fsession, UPSTREAM *upstream); static int routeQuery(FILTER *instance, void *fsession, GWBUF *queue); static int clientReply(FILTER *instance, void *fsession, GWBUF *queue); static void diagnostic(FILTER *instance, void *fsession, DCB *dcb); -static uint64_t getCapabilities(); +static uint64_t getCapabilities(void); static FILTER_OBJECT MyObject = @@ -597,7 +597,7 @@ static void diagnostic(FILTER *instance, void *fsession, DCB *dcb) * * @return The capabilities of the filter. */ -static uint64_t getCapabilities() +static uint64_t getCapabilities(void) { return RCAP_TYPE_STMT_INPUT; } diff --git a/server/modules/filter/mqfilter/mqfilter.c b/server/modules/filter/mqfilter/mqfilter.c index d4279ca79..f56c30669 100644 --- a/server/modules/filter/mqfilter/mqfilter.c +++ b/server/modules/filter/mqfilter/mqfilter.c @@ -101,7 +101,7 @@ static void setUpstream(FILTER *instance, void *fsession, UPSTREAM *upstream); static int routeQuery(FILTER *instance, void *fsession, GWBUF *queue); static int clientReply(FILTER *instance, void *fsession, GWBUF *queue); static void diagnostic(FILTER *instance, void *fsession, DCB *dcb); -static uint64_t getCapabilities(); +static uint64_t getCapabilities(void); static FILTER_OBJECT MyObject = @@ -1711,7 +1711,7 @@ diagnostic(FILTER *instance, void *fsession, DCB *dcb) * * @return The capabilities of the filter. */ -static uint64_t getCapabilities() +static uint64_t getCapabilities(void) { return RCAP_TYPE_STMT_INPUT; } diff --git a/server/modules/filter/namedserverfilter/namedserverfilter.c b/server/modules/filter/namedserverfilter/namedserverfilter.c index 246ebb7db..c8737befe 100644 --- a/server/modules/filter/namedserverfilter/namedserverfilter.c +++ b/server/modules/filter/namedserverfilter/namedserverfilter.c @@ -57,7 +57,7 @@ static void freeSession(FILTER *instance, void *session); static void setDownstream(FILTER *instance, void *fsession, DOWNSTREAM *downstream); static int routeQuery(FILTER *instance, void *fsession, GWBUF *queue); static void diagnostic(FILTER *instance, void *fsession, DCB *dcb); -static uint64_t getCapabilities(); +static uint64_t getCapabilities(void); static FILTER_OBJECT MyObject = @@ -418,7 +418,7 @@ diagnostic(FILTER *instance, void *fsession, DCB *dcb) * * @return The capabilities of the filter. */ -static uint64_t getCapabilities() +static uint64_t getCapabilities(void) { return RCAP_TYPE_STMT_INPUT; } diff --git a/server/modules/filter/qlafilter/qlafilter.c b/server/modules/filter/qlafilter/qlafilter.c index 1f9ebe7e0..eab3bae54 100644 --- a/server/modules/filter/qlafilter/qlafilter.c +++ b/server/modules/filter/qlafilter/qlafilter.c @@ -72,7 +72,7 @@ static void freeSession(FILTER *instance, void *session); static void setDownstream(FILTER *instance, void *fsession, DOWNSTREAM *downstream); static int routeQuery(FILTER *instance, void *fsession, GWBUF *queue); static void diagnostic(FILTER *instance, void *fsession, DCB *dcb); -static uint64_t getCapabilities(); +static uint64_t getCapabilities(void); static FILTER_OBJECT MyObject = @@ -522,7 +522,7 @@ diagnostic(FILTER *instance, void *fsession, DCB *dcb) * * @return The capabilities of the filter. */ -static uint64_t getCapabilities() +static uint64_t getCapabilities(void) { return RCAP_TYPE_STMT_INPUT; } diff --git a/server/modules/filter/regexfilter/regexfilter.c b/server/modules/filter/regexfilter/regexfilter.c index 921c6d030..21fe8e433 100644 --- a/server/modules/filter/regexfilter/regexfilter.c +++ b/server/modules/filter/regexfilter/regexfilter.c @@ -57,7 +57,7 @@ static void freeSession(FILTER *instance, void *session); static void setDownstream(FILTER *instance, void *fsession, DOWNSTREAM *downstream); static int routeQuery(FILTER *instance, void *fsession, GWBUF *queue); static void diagnostic(FILTER *instance, void *fsession, DCB *dcb); -static uint64_t getCapabilities(); +static uint64_t getCapabilities(void); static char *regex_replace(const char *sql, pcre2_code *re, pcre2_match_data *study, const char *replace); @@ -553,7 +553,7 @@ void log_nomatch(REGEX_INSTANCE* inst, char* re, char* old) * * @return The capabilities of the filter. */ -static uint64_t getCapabilities() +static uint64_t getCapabilities(void) { return RCAP_TYPE_STMT_INPUT; } diff --git a/server/modules/filter/tee/tee.c b/server/modules/filter/tee/tee.c index 69b02c7ee..355c420b0 100644 --- a/server/modules/filter/tee/tee.c +++ b/server/modules/filter/tee/tee.c @@ -116,7 +116,7 @@ static void setUpstream(FILTER *instance, void *fsession, UPSTREAM *upstream); static int routeQuery(FILTER *instance, void *fsession, GWBUF *queue); static int clientReply(FILTER *instance, void *fsession, GWBUF *queue); static void diagnostic(FILTER *instance, void *fsession, DCB *dcb); -static uint64_t getCapabilities(); +static uint64_t getCapabilities(void); static FILTER_OBJECT MyObject = { @@ -1131,7 +1131,7 @@ diagnostic(FILTER *instance, void *fsession, DCB *dcb) * * @return The capabilities of the filter. */ -static uint64_t getCapabilities() +static uint64_t getCapabilities(void) { return RCAP_TYPE_STMT_INPUT; } diff --git a/server/modules/filter/testfilter/testfilter.c b/server/modules/filter/testfilter/testfilter.c index e2ce7bbd0..5c3cb7739 100644 --- a/server/modules/filter/testfilter/testfilter.c +++ b/server/modules/filter/testfilter/testfilter.c @@ -46,7 +46,7 @@ static void freeSession(FILTER *instance, void *session); static void setDownstream(FILTER *instance, void *fsession, DOWNSTREAM *downstream); static int routeQuery(FILTER *instance, void *fsession, GWBUF *queue); static void diagnostic(FILTER *instance, void *fsession, DCB *dcb); -static uint64_t getCapabilities(); +static uint64_t getCapabilities(void); static FILTER_OBJECT MyObject = @@ -254,7 +254,7 @@ diagnostic(FILTER *instance, void *fsession, DCB *dcb) * * @return The capabilities of the filter. */ -static uint64_t getCapabilities() +static uint64_t getCapabilities(void) { return RCAP_TYPE_NONE; } diff --git a/server/modules/filter/topfilter/topfilter.c b/server/modules/filter/topfilter/topfilter.c index 2f69868e1..21f865345 100644 --- a/server/modules/filter/topfilter/topfilter.c +++ b/server/modules/filter/topfilter/topfilter.c @@ -67,7 +67,7 @@ static void setUpstream(FILTER *instance, void *fsession, UPSTREAM *upstream); static int routeQuery(FILTER *instance, void *fsession, GWBUF *queue); static int clientReply(FILTER *instance, void *fsession, GWBUF *queue); static void diagnostic(FILTER *instance, void *fsession, DCB *dcb); -static uint64_t getCapabilities(); +static uint64_t getCapabilities(void); static FILTER_OBJECT MyObject = @@ -690,7 +690,7 @@ diagnostic(FILTER *instance, void *fsession, DCB *dcb) * * @return The capabilities of the filter. */ -static uint64_t getCapabilities() +static uint64_t getCapabilities(void) { return RCAP_TYPE_STMT_INPUT; } diff --git a/server/modules/filter/tpmfilter/tpmfilter.c b/server/modules/filter/tpmfilter/tpmfilter.c index e57063826..5811b74fe 100644 --- a/server/modules/filter/tpmfilter/tpmfilter.c +++ b/server/modules/filter/tpmfilter/tpmfilter.c @@ -81,7 +81,7 @@ static void setUpstream(FILTER *instance, void *fsession, UPSTREAM *upstream static int routeQuery(FILTER *instance, void *fsession, GWBUF *queue); static int clientReply(FILTER *instance, void *fsession, GWBUF *queue); static void diagnostic(FILTER *instance, void *fsession, DCB *dcb); -static uint64_t getCapabilities(); +static uint64_t getCapabilities(void); static FILTER_OBJECT MyObject = { @@ -565,7 +565,7 @@ diagnostic(FILTER *instance, void *fsession, DCB *dcb) * * @return The capabilities of the filter. */ -static uint64_t getCapabilities() +static uint64_t getCapabilities(void) { return RCAP_TYPE_STMT_INPUT; } diff --git a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c index 655b4e53c..21591c91c 100644 --- a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c +++ b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c @@ -838,8 +838,7 @@ gw_read_normal_data(DCB *dcb, GWBUF *read_buffer, int nbytes_read) } /** Ask what type of input the router expects */ - capabilities = session->service->router->getCapabilities( - session->service->router_instance, session->router_session); + capabilities = session->service->router->getCapabilities(); /** Update the current protocol command being executed */ if (!process_client_commands(dcb, nbytes_read, &read_buffer)) diff --git a/server/modules/routing/avro/avro.c b/server/modules/routing/avro/avro.c index 657fbcbec..54f256d3c 100644 --- a/server/modules/routing/avro/avro.c +++ b/server/modules/routing/avro/avro.c @@ -79,7 +79,7 @@ static void clientReply(ROUTER *instance, void *router_session, GWBUF *queue, DCB *backend_dcb); static void errorReply(ROUTER *instance, void *router_session, GWBUF *message, DCB *backend_dcb, error_action_t action, bool *succp); -static uint64_t getCapabilities(); +static uint64_t getCapabilities(void); extern int MaxScaleUptime(); extern void avro_get_used_tables(AVRO_INSTANCE *router, DCB *dcb); void converter_func(void* data); @@ -955,7 +955,7 @@ errorReply(ROUTER *instance, void *router_session, GWBUF *message, DCB *backend_ ss_dassert(false); } -static uint64_t getCapabilities() +static uint64_t getCapabilities(void) { return RCAP_TYPE_NO_RSESSION; } diff --git a/server/modules/routing/binlog/blr.c b/server/modules/routing/binlog/blr.c index 350c04609..4b0f5ba93 100644 --- a/server/modules/routing/binlog/blr.c +++ b/server/modules/routing/binlog/blr.c @@ -100,7 +100,7 @@ static void errorReply(ROUTER *instance, error_action_t action, bool *succp); -static uint64_t getCapabilities(); +static uint64_t getCapabilities(void); static int blr_handler_config(void *userdata, const char *section, const char *name, const char *value); static int blr_handle_config_item(const char *name, const char *value, ROUTER_INSTANCE *inst); static int blr_load_dbusers(const ROUTER_INSTANCE *router); @@ -1766,7 +1766,7 @@ static void rses_end_locked_router_action(ROUTER_SLAVE *rses) } -static uint64_t getCapabilities() +static uint64_t getCapabilities(void) { return RCAP_TYPE_NO_RSESSION; } diff --git a/server/modules/routing/cli/cli.c b/server/modules/routing/cli/cli.c index ff7deff9f..ee263009e 100644 --- a/server/modules/routing/cli/cli.c +++ b/server/modules/routing/cli/cli.c @@ -58,7 +58,7 @@ static void closeSession(ROUTER *instance, void *router_session); static void freeSession(ROUTER *instance, void *router_session); static int execute(ROUTER *instance, void *router_session, GWBUF *queue); static void diagnostics(ROUTER *instance, DCB *dcb); -static uint64_t getCapabilities(); +static uint64_t getCapabilities(void); /** The module object definition */ static ROUTER_OBJECT MyObject = @@ -302,7 +302,7 @@ diagnostics(ROUTER *instance, DCB *dcb) return; /* Nothing to do currently */ } -static uint64_t getCapabilities() +static uint64_t getCapabilities(void) { return 0; } diff --git a/server/modules/routing/debugcli/debugcli.c b/server/modules/routing/debugcli/debugcli.c index 29e8a2fc3..962fbe1b4 100644 --- a/server/modules/routing/debugcli/debugcli.c +++ b/server/modules/routing/debugcli/debugcli.c @@ -333,7 +333,7 @@ diagnostics(ROUTER *instance, DCB *dcb) return; /* Nothing to do currently */ } -static uint64_t getCapabilities() +static uint64_t getCapabilities(void) { return 0; } diff --git a/server/modules/routing/maxinfo/maxinfo.c b/server/modules/routing/maxinfo/maxinfo.c index d0fb26388..135de44f0 100644 --- a/server/modules/routing/maxinfo/maxinfo.c +++ b/server/modules/routing/maxinfo/maxinfo.c @@ -77,7 +77,7 @@ static void closeSession(ROUTER *instance, void *router_session); static void freeSession(ROUTER *instance, void *router_session); static int execute(ROUTER *instance, void *router_session, GWBUF *queue); static void diagnostics(ROUTER *instance, DCB *dcb); -static uint64_t getCapabilities(); +static uint64_t getCapabilities(void); static void handleError(ROUTER *instance, void *router_session, GWBUF *errbuf, @@ -401,7 +401,7 @@ diagnostics(ROUTER *instance, DCB *dcb) * Not used for the maxinfo router */ static uint64_t -getCapabilities() +getCapabilities(void) { return 0; } diff --git a/server/modules/routing/readconnroute/readconnroute.c b/server/modules/routing/readconnroute/readconnroute.c index 704b1613b..540ef431a 100644 --- a/server/modules/routing/readconnroute/readconnroute.c +++ b/server/modules/routing/readconnroute/readconnroute.c @@ -111,7 +111,7 @@ static void clientReply(ROUTER *instance, void *router_session, GWBUF *queue, DCB *backend_dcb); static void handleError(ROUTER *instance, void *router_session, GWBUF *errbuf, DCB *problem_dcb, error_action_t action, bool *succp); -static uint64_t getCapabilities(); +static uint64_t getCapabilities(void); /** The module object definition */ @@ -985,7 +985,7 @@ static void rses_end_locked_router_action(ROUTER_CLIENT_SES* rses) spinlock_release(&rses->rses_lock); } -static uint64_t getCapabilities() +static uint64_t getCapabilities(void) { return RCAP_TYPE_NONE; } diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index 67a566a13..9ddf0f4f5 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -82,7 +82,7 @@ static void clientReply(ROUTER *instance, void *router_session, GWBUF *queue, static void handleError(ROUTER *instance, void *router_session, GWBUF *errmsgbuf, DCB *backend_dcb, error_action_t action, bool *succp); -static uint64_t getCapabilities(); +static uint64_t getCapabilities(void); /* * End of the API functions; now the module structure that links to them. @@ -1062,7 +1062,7 @@ lock_failed: * * @return RCAP_TYPE_STMT_INPUT. */ -static uint64_t getCapabilities() +static uint64_t getCapabilities(void) { return RCAP_TYPE_STMT_INPUT; } diff --git a/server/modules/routing/schemarouter/schemarouter.c b/server/modules/routing/schemarouter/schemarouter.c index bead571f8..fbf5229e5 100644 --- a/server/modules/routing/schemarouter/schemarouter.c +++ b/server/modules/routing/schemarouter/schemarouter.c @@ -92,7 +92,7 @@ static route_target_t get_shard_route_target(qc_query_type_t qtype, bool trx_active, HINT* hint); -static uint64_t getCapabilities(); +static uint64_t getCapabilities(void); static bool connect_backend_servers(backend_ref_t* backend_ref, int router_nservers, @@ -3642,7 +3642,7 @@ static void tracelog_routed_query(ROUTER_CLIENT_SES* rses, /** * Return RCAP_TYPE_STMT_INPUT. */ -static uint64_t getCapabilities() +static uint64_t getCapabilities(void) { return RCAP_TYPE_STMT_INPUT; } diff --git a/server/modules/routing/testroute/testroute.c b/server/modules/routing/testroute/testroute.c index 3533686f6..502e7cfab 100644 --- a/server/modules/routing/testroute/testroute.c +++ b/server/modules/routing/testroute/testroute.c @@ -164,7 +164,7 @@ diagnostic(ROUTER *instance, DCB *dcb) { } -static uint64_t getCapabilities() +static uint64_t getCapabilities(void) { return 0; } From cff90ffee8cb9892332a8a1be2b39572a73985f3 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Mon, 24 Oct 2016 11:04:26 +0300 Subject: [PATCH 076/215] Fix HTTPD protocol authentication The HTTPD protocol mistakenly assumed that the `authenticator` parameter of a listener would be NULL if the default authenticator is used. Recent changes modified it so that the value is never NULL and `NullAuthDeny` would be used for protocols which did not implement the auth_default entry point. --- server/core/service.c | 2 +- server/modules/protocol/HTTPD/httpd.c | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/server/core/service.c b/server/core/service.c index ceaeac0cb..96792230a 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -272,7 +272,7 @@ serviceStartPort(SERVICE *service, SERV_LISTENER *port) memcpy(&(port->listener->func), funcs, sizeof(GWPROTOCOL)); - const char *authenticator_name = "NullAuth"; + const char *authenticator_name = "NullAuthDeny"; if (port->authenticator) { diff --git a/server/modules/protocol/HTTPD/httpd.c b/server/modules/protocol/HTTPD/httpd.c index fc8220ec1..3393c0ec0 100644 --- a/server/modules/protocol/HTTPD/httpd.c +++ b/server/modules/protocol/HTTPD/httpd.c @@ -215,9 +215,10 @@ static int httpd_read_event(DCB* dcb) } } - /** If listener->authenticator is NULL, it means we're using the default - * authenticator and we don't need to check the user credentials. */ - bool auth_ok = dcb->listener->authenticator == 0; + /** If listener->authenticator is the default authenticator, it means that + * we don't need to check the user credentials. All other authenticators + * cause a 401 Unauthorized to be returned on the first try. */ + bool auth_ok = strcmp(httpd_default_auth(), dcb->listener->authenticator) == 0; /** * Get the request headers @@ -254,7 +255,7 @@ static int httpd_read_event(DCB* dcb) /** The freeing entry point is called automatically when * the client DCB is closed */ dcb->authfunc.extract(dcb, auth_data); - auth_ok = dcb->authfunc.authenticate(dcb) == 0; + auth_ok = dcb->authfunc.authenticate(dcb) == MXS_AUTH_SUCCEEDED; gwbuf_free(auth_data); } } From d9618646350bc4ff98998aecaa0821fdb574b754 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Mon, 24 Oct 2016 06:46:46 +0300 Subject: [PATCH 077/215] Fix tests and limit internal test set size Some of the tests depended on a working installation where modules are all located at the default paths. These tests now explicitly set the module directory which fixes the immediate problem. Disabled the starting of services in the service test as this will fail with real modules. The dummy internal modules aren't build and should be removed in a later commit. In general, it might be better to do service level testing outside the internal test suite. --- .travis/build_maxscale.sh | 1 + server/core/test/testserver.c | 5 ++-- server/core/test/testservice.c | 28 ++++--------------- .../modules/routing/binlog/test/testbinlog.c | 2 +- 4 files changed, 11 insertions(+), 25 deletions(-) diff --git a/.travis/build_maxscale.sh b/.travis/build_maxscale.sh index 11c2c5ba6..916b8d37c 100644 --- a/.travis/build_maxscale.sh +++ b/.travis/build_maxscale.sh @@ -15,6 +15,7 @@ cd build cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=Y -DBUILD_AVRO=N make +make test sudo make install sudo ./postinst diff --git a/server/core/test/testserver.c b/server/core/test/testserver.c index 19d33e20b..9c10e6edf 100644 --- a/server/core/test/testserver.c +++ b/server/core/test/testserver.c @@ -36,6 +36,7 @@ #include #include #include +#include /** * test1 Allocate a server and do lots of other things * @@ -48,8 +49,8 @@ test1() char *status; /* Server tests */ - ss_dfprintf(stderr, - "testserver : creating server called MyServer"); + ss_dfprintf(stderr, "testserver : creating server called MyServer"); + set_libdir(MXS_STRDUP_A("../../modules/authenticator/")); server = server_alloc("MyServer", "HTTPD", 9876, "NullAuthAllow", NULL); ss_info_dassert(server, "Allocating the server should not fail"); mxs_log_flush_sync(); diff --git a/server/core/test/testservice.c b/server/core/test/testservice.c index b3101f1f7..e6aa0bd5b 100644 --- a/server/core/test/testservice.c +++ b/server/core/test/testservice.c @@ -51,45 +51,29 @@ test1() int result; int argc = 3; + mxs_log_init(NULL, "/tmp", MXS_LOG_TARGET_FS); init_test_env(NULL); /* Service tests */ ss_dfprintf(stderr, "testservice : creating service called MyService with router nonexistent"); service = service_alloc("MyService", "non-existent"); - mxs_log_flush_sync(); ss_info_dassert(NULL == service, "New service with invalid router should be null"); ss_info_dassert(0 == service_isvalid(service), "Service must not be valid after incorrect creation"); ss_dfprintf(stderr, "\t..done\nValid service creation, router testroute."); - set_libdir(MXS_STRDUP_A("../../modules/routing/")); - service = service_alloc("MyService", "testroute"); - mxs_log_flush_sync(); + set_libdir(MXS_STRDUP_A("../../modules/routing/readconnroute/")); + service = service_alloc("MyService", "readconnroute"); + ss_info_dassert(NULL != service, "New service with valid router must not be null"); ss_info_dassert(0 != service_isvalid(service), "Service must be valid after creation"); ss_info_dassert(0 == strcmp("MyService", service_get_name(service)), "Service must have given name"); ss_dfprintf(stderr, "\t..done\nAdding protocol testprotocol."); + set_libdir(MXS_STRDUP_A("../../modules/authenticator/MySQLAuth/")); ss_info_dassert(0 != serviceAddProtocol(service, "TestProtocol", "testprotocol", - "localhost", 9876, "MySQLClient", "MySQLAuth", NULL), + "localhost", 9876, "MySQLAuth", NULL, NULL), "Add Protocol should succeed"); ss_info_dassert(0 != serviceHasProtocol(service, "testprotocol", "localhost", 9876), "Service should have new protocol as requested"); - set_libdir(MXS_STRDUP_A("../../modules/protocol/")); - serviceStartProtocol(service, "testprotocol", 9876); - mxs_log_flush_sync(); - ss_dfprintf(stderr, "\t..done\nStarting Service."); - result = serviceStart(service); - mxs_log_flush_sync(); - ss_info_dassert(0 != result, "Start should succeed"); - serviceStop(service); - mxs_log_flush_sync(); - ss_info_dassert(service->state == SERVICE_STATE_STOPPED, "Stop should succeed"); - result = serviceStartAll(); - mxs_log_flush_sync(); - ss_info_dassert(0 != result, "Start all should succeed"); - ss_dfprintf(stderr, "\t..done\nStopping Service."); - serviceStop(service); - ss_info_dassert(service->state == SERVICE_STATE_STOPPED, "Stop should succeed"); - ss_dfprintf(stderr, "\t..done\n"); return 0; diff --git a/server/modules/routing/binlog/test/testbinlog.c b/server/modules/routing/binlog/test/testbinlog.c index 6f4b14d58..2bb630e54 100644 --- a/server/modules/routing/binlog/test/testbinlog.c +++ b/server/modules/routing/binlog/test/testbinlog.c @@ -102,7 +102,7 @@ int main(int argc, char **argv) { serviceAddRouterOption(service, s); s = strtok_r(NULL, ",", &lasts); } - + set_libdir(MXS_STRDUP_A("../../../authenticator/")); server = server_alloc("_none_", "MySQLBackend", 3306, "MySQLBackendAuth", NULL); if (server == NULL) { return 1; From bd4542139c7c31de711a68c306e8102045c8385f Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Mon, 24 Oct 2016 05:58:57 +0300 Subject: [PATCH 078/215] Install third-party license file Some of the libraries used by MaxScale are distributed under BSD licenses. The BSD license requires that the license text is included with the binary distributions. --- CMakeLists.txt | 1 + LICENSE-THIRDPARTY.TXT | 86 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 LICENSE-THIRDPARTY.TXT diff --git a/CMakeLists.txt b/CMakeLists.txt index 2ed799c0d..78d70c26e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -224,6 +224,7 @@ endif() install_file(${CMAKE_SOURCE_DIR}/COPYRIGHT core) install_file(${CMAKE_SOURCE_DIR}/README core) install_file(${CMAKE_SOURCE_DIR}/LICENSE.TXT core) +install_file(${CMAKE_SOURCE_DIR}/LICENSE-THIRDPARTY.TXT core) install_file(etc/lsyncd_example.conf core) install_manual(Documentation/maxscale.1 1 core) install_file(${CMAKE_SOURCE_DIR}/server/maxscale_binlogserver_template.cnf core) diff --git a/LICENSE-THIRDPARTY.TXT b/LICENSE-THIRDPARTY.TXT new file mode 100644 index 000000000..45f5071f5 --- /dev/null +++ b/LICENSE-THIRDPARTY.TXT @@ -0,0 +1,86 @@ +The following software may be included by this product: + +FindGSSAPI.cmake + +Copyright (c) 2006, Pino Toscano, + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The "inih" library is distributed under the New BSD license: + +Copyright (c) 2009, Brush Technology +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Brush Technology nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY BRUSH TECHNOLOGY ''AS IS'' AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL BRUSH TECHNOLOGY BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Release 10 of PCRE2 is distributed under the terms of the "BSD" licence. + +THE "BSD" LICENCE +----------------- + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of any + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. From 1a2e0ba3ee17c669de5716557a7e05c94b1831b8 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Mon, 24 Oct 2016 13:01:11 +0300 Subject: [PATCH 079/215] Check log priority before logging With this change, whether something should be logged or, that is, whether the used log priority is enabled or not, is checked before the logging function is called. That should save a few cycles. Now mxs_log_message() always logs a message, without consideration for whether the priority is enabled or not. The inline function mxs_log_is_priority_enabled() returns true if a particular priority is enabled and the MXS_LOG_MESSAGE() macro (that MXS_ERROR etc. are expanded into) call that before calling the actual logging function. --- include/maxscale/log_manager.h | 11 +- server/core/log_manager.cc | 197 ++++++++++++++++----------------- 2 files changed, 107 insertions(+), 101 deletions(-) diff --git a/include/maxscale/log_manager.h b/include/maxscale/log_manager.h index e47d37509..fcd6f8c94 100644 --- a/include/maxscale/log_manager.h +++ b/include/maxscale/log_manager.h @@ -15,6 +15,7 @@ */ #include +#include #include #include #include @@ -111,6 +112,12 @@ void mxs_log_set_throttling(const MXS_LOG_THROTTLING* throttling); void mxs_log_get_throttling(MXS_LOG_THROTTLING* throttling); +static inline bool mxs_log_priority_is_enabled(int priority) +{ + assert((priority & ~LOG_PRIMASK) == 0); + return MXS_LOG_PRIORITY_IS_ENABLED(priority); +} + int mxs_log_message(int priority, const char* modname, const char* file, int line, const char* function, @@ -126,7 +133,9 @@ int mxs_log_message(int priority, * MXS_ERROR, MXS_WARNING, etc. macros instead. */ #define MXS_LOG_MESSAGE(priority, format, ...)\ - mxs_log_message(priority, MXS_MODULE_NAME, __FILE__, __LINE__, __func__, format, ##__VA_ARGS__) + (mxs_log_priority_is_enabled(priority) ? \ + mxs_log_message(priority, MXS_MODULE_NAME, __FILE__, __LINE__, __func__, format, ##__VA_ARGS__) :\ + 0) /** * Log an alert, error, warning, notice, info, or debug message. diff --git a/server/core/log_manager.cc b/server/core/log_manager.cc index 80263a5ef..b15f253b8 100644 --- a/server/core/log_manager.cc +++ b/server/core/log_manager.cc @@ -2913,131 +2913,128 @@ int mxs_log_message(int priority, if ((priority & ~LOG_PRIMASK) == 0) // Check that the priority is ok, { - if (MXS_LOG_PRIORITY_IS_ENABLED(priority)) - { - message_suppression_t status = MESSAGE_NOT_SUPPRESSED; + message_suppression_t status = MESSAGE_NOT_SUPPRESSED; - // We only throttle errors and warnings. Info and debug messages - // are never on during normal operation, so if they are enabled, - // we are presumably debugging something. Notice messages are - // assumed to be logged for a reason and always in a context where - // flooding cannot be caused. - if ((priority == LOG_ERR) || (priority == LOG_WARNING)) + // We only throttle errors and warnings. Info and debug messages + // are never on during normal operation, so if they are enabled, + // we are presumably debugging something. Notice messages are + // assumed to be logged for a reason and always in a context where + // flooding cannot be caused. + if ((priority == LOG_ERR) || (priority == LOG_WARNING)) + { + status = message_status(file, line); + } + + if (status != MESSAGE_STILL_SUPPRESSED) + { + va_list valist; + + int modname_len = modname ? strlen(modname) + 3 : 0; // +3 due to "[...] " + + static const char SUPPRESSION[] = + " (subsequent similar messages suppressed for %lu milliseconds)"; + int suppression_len = 0; + size_t suppress_ms = log_config.throttling.suppress_ms; + + if (status == MESSAGE_SUPPRESSED) { - status = message_status(file, line); + suppression_len += sizeof(SUPPRESSION) - 1; // Remove trailing NULL + suppression_len -= 3; // Remove the %lu + suppression_len += UINTLEN(suppress_ms); } - if (status != MESSAGE_STILL_SUPPRESSED) + /** + * Find out the length of log string (to be formatted str). + */ + va_start(valist, format); + int message_len = vsnprintf(NULL, 0, format, valist); + va_end(valist); + + if (message_len >= 0) { - va_list valist; + log_prefix_t prefix = priority_to_prefix(priority); - int modname_len = modname ? strlen(modname) + 3 : 0; // +3 due to "[...] " + static const char FORMAT_FUNCTION[] = "(%s): "; - static const char SUPPRESSION[] = - " (subsequent similar messages suppressed for %lu milliseconds)"; - int suppression_len = 0; - size_t suppress_ms = log_config.throttling.suppress_ms; + // Other thread might change log_config.augmentation. + int augmentation = log_config.augmentation; + int augmentation_len = 0; - if (status == MESSAGE_SUPPRESSED) + switch (augmentation) { - suppression_len += sizeof(SUPPRESSION) - 1; // Remove trailing NULL - suppression_len -= 3; // Remove the %lu - suppression_len += UINTLEN(suppress_ms); + case MXS_LOG_AUGMENT_WITH_FUNCTION: + augmentation_len = sizeof(FORMAT_FUNCTION) - 1; // Remove trailing 0 + augmentation_len -= 2; // Remove the %s + augmentation_len += strlen(function); + break; + + default: + break; } - /** - * Find out the length of log string (to be formatted str). - */ - va_start(valist, format); - int message_len = vsnprintf(NULL, 0, format, valist); - va_end(valist); + int buffer_len = 0; + buffer_len += prefix.len; + buffer_len += modname_len; + buffer_len += augmentation_len; + buffer_len += message_len; + buffer_len += suppression_len; + buffer_len += 1; // Trailing NULL - if (message_len >= 0) + if (buffer_len > MAX_LOGSTRLEN) { - log_prefix_t prefix = priority_to_prefix(priority); + message_len -= (buffer_len - MAX_LOGSTRLEN); + buffer_len = MAX_LOGSTRLEN; - static const char FORMAT_FUNCTION[] = "(%s): "; + ss_dassert(prefix.len + modname_len + + augmentation_len + message_len + suppression_len + 1 == buffer_len); + } - // Other thread might change log_config.augmentation. - int augmentation = log_config.augmentation; - int augmentation_len = 0; + char buffer[buffer_len]; + + char *prefix_text = buffer; + char *modname_text = prefix_text + prefix.len; + char *augmentation_text = modname_text + modname_len; + char *message_text = augmentation_text + augmentation_len; + char *suppression_text = message_text + message_len; + + strcpy(prefix_text, prefix.text); + + if (modname_len) + { + strcpy(modname_text, "["); + strcat(modname_text, modname); + strcat(modname_text, "] "); + } + + if (augmentation_len) + { + int len = 0; switch (augmentation) { case MXS_LOG_AUGMENT_WITH_FUNCTION: - augmentation_len = sizeof(FORMAT_FUNCTION) - 1; // Remove trailing 0 - augmentation_len -= 2; // Remove the %s - augmentation_len += strlen(function); + len = sprintf(augmentation_text, FORMAT_FUNCTION, function); break; default: - break; + assert(!true); } - int buffer_len = 0; - buffer_len += prefix.len; - buffer_len += modname_len; - buffer_len += augmentation_len; - buffer_len += message_len; - buffer_len += suppression_len; - buffer_len += 1; // Trailing NULL - - if (buffer_len > MAX_LOGSTRLEN) - { - message_len -= (buffer_len - MAX_LOGSTRLEN); - buffer_len = MAX_LOGSTRLEN; - - ss_dassert(prefix.len + modname_len + - augmentation_len + message_len + suppression_len + 1 == buffer_len); - } - - char buffer[buffer_len]; - - char *prefix_text = buffer; - char *modname_text = prefix_text + prefix.len; - char *augmentation_text = modname_text + modname_len; - char *message_text = augmentation_text + augmentation_len; - char *suppression_text = message_text + message_len; - - strcpy(prefix_text, prefix.text); - - if (modname_len) - { - strcpy(modname_text, "["); - strcat(modname_text, modname); - strcat(modname_text, "] "); - } - - if (augmentation_len) - { - int len = 0; - - switch (augmentation) - { - case MXS_LOG_AUGMENT_WITH_FUNCTION: - len = sprintf(augmentation_text, FORMAT_FUNCTION, function); - break; - - default: - assert(!true); - } - - assert(len == augmentation_len); - } - - va_start(valist, format); - vsnprintf(message_text, message_len + 1, format, valist); - va_end(valist); - - if (suppression_len) - { - sprintf(suppression_text, SUPPRESSION, suppress_ms); - } - - enum log_flush flush = priority_to_flush(priority); - - err = log_write(priority, file, line, function, prefix.len, buffer_len, buffer, flush); + assert(len == augmentation_len); } + + va_start(valist, format); + vsnprintf(message_text, message_len + 1, format, valist); + va_end(valist); + + if (suppression_len) + { + sprintf(suppression_text, SUPPRESSION, suppress_ms); + } + + enum log_flush flush = priority_to_flush(priority); + + err = log_write(priority, file, line, function, prefix.len, buffer_len, buffer, flush); } } } From 3915b4e7c7e10c9d5ef75145fead5c4c36a9d8c9 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Mon, 24 Oct 2016 13:41:00 +0300 Subject: [PATCH 080/215] Exclude MXS_DEBUG in release builds Now, MXS_DEBUG turns into a nop in release builds. --- Documentation/Getting-Started/Configuration-Guide.md | 6 +++++- include/maxscale/log_manager.h | 5 +++++ server/core/dcb.c | 10 +++------- server/core/poll.c | 10 ++++------ server/modules/authenticator/MySQLAuth/dbusers.c | 3 +-- server/modules/routing/binlog/blr.c | 3 +-- server/modules/routing/readconnroute/readconnroute.c | 3 +-- server/modules/routing/schemarouter/schemarouter.c | 9 +++------ 8 files changed, 23 insertions(+), 26 deletions(-) diff --git a/Documentation/Getting-Started/Configuration-Guide.md b/Documentation/Getting-Started/Configuration-Guide.md index ea7fc03a4..007f96110 100644 --- a/Documentation/Getting-Started/Configuration-Guide.md +++ b/Documentation/Getting-Started/Configuration-Guide.md @@ -247,7 +247,11 @@ To disable these messages use the value 0 and to enable them use the value 1. #### `log_debug` -Enable or disable the logging of messages whose syslog priority is *debug*. This kind of messages are intended for development purposes and are disabled by default. +Enable or disable the logging of messages whose syslog priority is *debug*. +This kind of messages are intended for development purposes and are disabled +by default. Note that if MariaDB MaxScale has been built in release mode, then +debug messages are excluded from the build and this setting will not have any +effect. ``` # Valid options are: diff --git a/include/maxscale/log_manager.h b/include/maxscale/log_manager.h index fcd6f8c94..e46369f5e 100644 --- a/include/maxscale/log_manager.h +++ b/include/maxscale/log_manager.h @@ -156,7 +156,12 @@ int mxs_log_message(int priority, #define MXS_WARNING(format, ...) MXS_LOG_MESSAGE(LOG_WARNING, format, ##__VA_ARGS__) #define MXS_NOTICE(format, ...) MXS_LOG_MESSAGE(LOG_NOTICE, format, ##__VA_ARGS__) #define MXS_INFO(format, ...) MXS_LOG_MESSAGE(LOG_INFO, format, ##__VA_ARGS__) + +#if defined(SS_DEBUG) #define MXS_DEBUG(format, ...) MXS_LOG_MESSAGE(LOG_DEBUG, format, ##__VA_ARGS__) +#else +#define MXS_DEBUG(format, ...) +#endif /** * Log an out of memory error using custom message. diff --git a/server/core/dcb.c b/server/core/dcb.c index 9d3c5cb87..24e252e91 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -2919,20 +2919,16 @@ dcb_create_SSL(DCB* dcb, SSL_LISTENER *ssl) */ int dcb_accept_SSL(DCB* dcb) { - int ssl_rval; - char *remote; - char *user; - if ((NULL == dcb->listener || NULL == dcb->listener->ssl) || (NULL == dcb->ssl && dcb_create_SSL(dcb, dcb->listener->ssl) != 0)) { return -1; } - remote = dcb->remote ? dcb->remote : ""; - user = dcb->user ? dcb->user : ""; + ss_debug(char *remote = dcb->remote ? dcb->remote : ""); + ss_debug(char *user = dcb->user ? dcb->user : ""); - ssl_rval = SSL_accept(dcb->ssl); + int ssl_rval = SSL_accept(dcb->ssl); switch (SSL_get_error(dcb->ssl, ssl_rval)) { diff --git a/server/core/poll.c b/server/core/poll.c index e6abc9856..ea7fa657c 100644 --- a/server/core/poll.c +++ b/server/core/poll.c @@ -1044,9 +1044,8 @@ process_pollq(int thread_id) if (ev & EPOLLHUP) { - int eno = 0; - eno = gw_getsockerrno(dcb->fd); - char errbuf[MXS_STRERROR_BUFLEN]; + ss_debug(int eno = gw_getsockerrno(dcb->fd)); + ss_debug(char errbuf[MXS_STRERROR_BUFLEN]); MXS_DEBUG("%lu [poll_waitevents] " "EPOLLHUP on dcb %p, fd %d. " "Errno %d, %s.", @@ -1080,9 +1079,8 @@ process_pollq(int thread_id) #ifdef EPOLLRDHUP if (ev & EPOLLRDHUP) { - int eno = 0; - eno = gw_getsockerrno(dcb->fd); - char errbuf[MXS_STRERROR_BUFLEN]; + ss_debug(int eno = gw_getsockerrno(dcb->fd)); + ss_debug(char errbuf[MXS_STRERROR_BUFLEN]); MXS_DEBUG("%lu [poll_waitevents] " "EPOLLRDHUP on dcb %p, fd %d. " "Errno %d, %s.", diff --git a/server/modules/authenticator/MySQLAuth/dbusers.c b/server/modules/authenticator/MySQLAuth/dbusers.c index de4b5f2ec..ec4cbad6f 100644 --- a/server/modules/authenticator/MySQLAuth/dbusers.c +++ b/server/modules/authenticator/MySQLAuth/dbusers.c @@ -1199,7 +1199,6 @@ get_users(SERV_LISTENER *listener, USERS *users) MYSQL_PASSWORD_LEN + sizeof(char) + MYSQL_DATABASE_MAXLEN; - int dbnames = 0; int db_grants = 0; char dbnm[MYSQL_DATABASE_MAXLEN + 1]; bool anon_user = false; @@ -1454,7 +1453,7 @@ get_users(SERV_LISTENER *listener, USERS *users) if (db_grants) { /* load all mysql database names */ - dbnames = get_databases(listener, con); + ss_debug(int dbnames =) get_databases(listener, con); MXS_DEBUG("Loaded %d MySQL Database Names for service [%s]", dbnames, service->name); } diff --git a/server/modules/routing/binlog/blr.c b/server/modules/routing/binlog/blr.c index 4b0f5ba93..eec12b98f 100644 --- a/server/modules/routing/binlog/blr.c +++ b/server/modules/routing/binlog/blr.c @@ -914,9 +914,8 @@ static void freeSession(ROUTER* router_instance, { ROUTER_INSTANCE *router = (ROUTER_INSTANCE *)router_instance; ROUTER_SLAVE *slave = (ROUTER_SLAVE *)router_client_ses; - int prev_val; - prev_val = atomic_add(&router->stats.n_slaves, -1); + ss_debug(int prev_val = ) atomic_add(&router->stats.n_slaves, -1); ss_dassert(prev_val > 0); /* diff --git a/server/modules/routing/readconnroute/readconnroute.c b/server/modules/routing/readconnroute/readconnroute.c index 540ef431a..6f78663a6 100644 --- a/server/modules/routing/readconnroute/readconnroute.c +++ b/server/modules/routing/readconnroute/readconnroute.c @@ -633,9 +633,8 @@ static void freeSession(ROUTER* router_instance, void* router_client_ses) ROUTER_INSTANCE* router = (ROUTER_INSTANCE *) router_instance; ROUTER_CLIENT_SES* router_cli_ses = (ROUTER_CLIENT_SES *) router_client_ses; - int prev_val; - prev_val = atomic_add(&router_cli_ses->backend->current_connection_count, -1); + ss_debug(int prev_val = ) atomic_add(&router_cli_ses->backend->current_connection_count, -1); ss_dassert(prev_val > 0); spinlock_acquire(&router->lock); diff --git a/server/modules/routing/schemarouter/schemarouter.c b/server/modules/routing/schemarouter/schemarouter.c index fbf5229e5..5bc82eb06 100644 --- a/server/modules/routing/schemarouter/schemarouter.c +++ b/server/modules/routing/schemarouter/schemarouter.c @@ -3578,17 +3578,14 @@ static void tracelog_routed_query(ROUTER_CLIENT_SES* rses, size_t buflen = GWBUF_LENGTH(buf); char* querystr; char* startpos = (char *)&packet[5]; - BACKEND* b; - backend_type_t be_type; - DCB* dcb; CHK_BACKEND_REF(bref); - b = bref->bref_backend; + ss_debug(BACKEND *b = bref->bref_backend); CHK_BACKEND(b); - dcb = bref->bref_dcb; + ss_debug(DCB *dcb = bref->bref_dcb); CHK_DCB(dcb); - be_type = BACKEND_TYPE(b); + ss_debug(backend_type_t be_type = BACKEND_TYPE(b)); if (GWBUF_IS_TYPE_MYSQL(buf)) { From bd18a7d8ed009ac9bd0a7790e6c01d28a23164be Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Fri, 21 Oct 2016 14:36:36 +0300 Subject: [PATCH 081/215] Add service capabilities The service capabilities are the union of the capabilities of the router and all filters. --- include/maxscale/service.h | 14 ++++++++++++++ server/core/service.c | 9 ++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/include/maxscale/service.h b/include/maxscale/service.h index 6cb80793b..dfbdb5b8a 100644 --- a/include/maxscale/service.h +++ b/include/maxscale/service.h @@ -167,6 +167,7 @@ typedef struct service struct service *next; /**< The next service in the linked list */ bool retry_start; /*< If starting of the service should be retried later */ bool log_auth_warnings; /*< Log authentication failures and warnings */ + uint64_t capabilities; /*< The capabilities of the service. */ } SERVICE; typedef enum count_spec_t @@ -237,6 +238,19 @@ extern RESULTSET *serviceGetList(); extern RESULTSET *serviceGetListenerList(); extern bool service_all_services_have_listeners(); +/** + * Get the capabilities of the servive. + * + * The capabilities of a service are the union of the capabilities of + * its router and all filters. + * + * @return The service capabilities. + */ +static inline uint64_t service_get_capabilities(const SERVICE *service) +{ + return service->capabilities; +} + MXS_END_DECLS #endif diff --git a/server/core/service.c b/server/core/service.c index 96792230a..c936d91bd 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -143,6 +143,7 @@ service_alloc(const char *servname, const char *router) return NULL; } + service->capabilities = service->router->getCapabilities(); service->client_count = 0; service->name = (char*)servname; service->routerModule = (char*)router; @@ -1066,6 +1067,7 @@ serviceSetFilters(SERVICE *service, char *filters) char *ptr, *brkt; int n = 0; bool rval = true; + uint64_t capabilities = 0; if ((flist = (FILTER_DEF **) MXS_MALLOC(sizeof(FILTER_DEF *))) == NULL) { @@ -1088,7 +1090,11 @@ serviceSetFilters(SERVICE *service, char *filters) if ((flist[n - 1] = filter_find(filter_name))) { - if (!filter_load(flist[n - 1])) + if (filter_load(flist[n - 1])) + { + capabilities |= flist[n - 1]->obj->getCapabilities(); + } + else { MXS_ERROR("Failed to load filter '%s' for service '%s'.", filter_name, service->name); @@ -1112,6 +1118,7 @@ serviceSetFilters(SERVICE *service, char *filters) { service->filters = flist; service->n_filters = n; + service->capabilities |= capabilities; } else { From 28ecba4022ba2a07700125da05be60f369000732 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Fri, 21 Oct 2016 21:24:53 +0300 Subject: [PATCH 082/215] Use service capabilities Now the capabilities of routers and filters alike will be honoured. --- server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c | 2 +- server/modules/protocol/MySQL/MySQLClient/mysql_client.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c b/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c index 5cdf7e7eb..17f52cb37 100644 --- a/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c +++ b/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c @@ -794,7 +794,7 @@ gw_read_and_write(DCB *dcb) dcb->session->client_dcb != NULL && dcb->session->client_dcb->state == DCB_STATE_POLLING && (session->router_session || - session->service->router->getCapabilities() & (int)RCAP_TYPE_NO_RSESSION)) + service_get_capabilities(session->service) & RCAP_TYPE_NO_RSESSION)) { MySQLProtocol *client_protocol = (MySQLProtocol *)dcb->session->client_dcb->protocol; if (client_protocol != NULL) diff --git a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c index 21591c91c..bad382043 100644 --- a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c +++ b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c @@ -837,8 +837,8 @@ gw_read_normal_data(DCB *dcb, GWBUF *read_buffer, int nbytes_read) return 1; } - /** Ask what type of input the router expects */ - capabilities = session->service->router->getCapabilities(); + /** Ask what type of input the router/filter chain expects */ + capabilities = service_get_capabilities(session->service); /** Update the current protocol command being executed */ if (!process_client_commands(dcb, nbytes_read, &read_buffer)) From 4c286b85e61a65cd6db4a84e5f5cc4f413608094 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Mon, 24 Oct 2016 15:32:19 +0300 Subject: [PATCH 083/215] Add new capability types --- include/maxscale/routing.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/include/maxscale/routing.h b/include/maxscale/routing.h index 447d1146d..7d7ac8158 100644 --- a/include/maxscale/routing.h +++ b/include/maxscale/routing.h @@ -29,7 +29,13 @@ MXS_BEGIN_DECLS */ typedef enum routing_capability { - RCAP_TYPE_STMT_INPUT = 0x0001, /**< Statement per buffer. */ + /**< Statements are delivered one per buffer. */ + RCAP_TYPE_STMT_INPUT = 0x0001, /* 0b0000000000000001 */ + /**< Each delivered buffer is contiguous; implies RCAP_TYPE_STMT_INPUT. */ + RCAP_TYPE_CONTIGUOUS_INPUT = 0x0003, /* 0b0000000000000011 */ + /**< The transaction state and autocommit mode of the session are tracked; + implies RCAP_TYPE_CONTIGUOUS_INPUT and RCAP_TYPE_STMT_INPUT. */ + RCAP_TYPE_TRANSACTION_TRACKING = 0x0007, /* 0b0000000000000111 */ } routing_capability_t; #define RCAP_TYPE_NONE 0 From 7e822aed4dfa2deb95b6ca1b611331c130405f42 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 21 Oct 2016 22:59:01 +0300 Subject: [PATCH 084/215] MXS-862: Add SQLite based authentication checks The authentication checks make sure that a user has all the required grants to access the database. This prevents the creation of unnecessary backend connections reducing the overall load on the database. Doing preliminary authentication in MaxScale enables the creation of more informative error messages. --- server/modules/authenticator/CMakeLists.txt | 2 +- server/modules/authenticator/gssapi_auth.c | 239 +++++++++++++++++- server/modules/authenticator/gssapi_auth.h | 8 +- .../authenticator/gssapi_auth_common.c | 25 -- .../authenticator/gssapi_backend_auth.c | 29 ++- 5 files changed, 257 insertions(+), 46 deletions(-) diff --git a/server/modules/authenticator/CMakeLists.txt b/server/modules/authenticator/CMakeLists.txt index 1f339f7ce..8244eef4d 100644 --- a/server/modules/authenticator/CMakeLists.txt +++ b/server/modules/authenticator/CMakeLists.txt @@ -9,7 +9,7 @@ if (GSSAPI_FOUND) include_directories(${GSSAPI_INCS}) add_library(GSSAPIAuth SHARED gssapi_auth.c gssapi_auth_common.c) - target_link_libraries(GSSAPIAuth maxscale-common ${GSSAPI_LIBS} MySQLCommon) + target_link_libraries(GSSAPIAuth maxscale-common ${GSSAPI_LIBS} sqlite3 MySQLCommon) set_target_properties(GSSAPIAuth PROPERTIES VERSION "1.0.0") install_module(GSSAPIAuth core) diff --git a/server/modules/authenticator/gssapi_auth.c b/server/modules/authenticator/gssapi_auth.c index 5e8063b63..3d1f8b095 100644 --- a/server/modules/authenticator/gssapi_auth.c +++ b/server/modules/authenticator/gssapi_auth.c @@ -18,23 +18,71 @@ #include #include #include +#include #include "gssapi_auth.h" +/** Default timeout is one minute */ +#define MXS_SQLITE_BUSY_TIMEOUT 60000 + +/** + * MySQL queries for retrieving the list of users + */ + /** Query that gets all users that authenticate via the gssapi plugin */ const char *gssapi_users_query = - "SELECT u.user, u.host, d.db FROM " - "mysql.user AS u JOIN mysql.db AS d " + "SELECT u.user, u.host, d.db, u.select_priv FROM " + "mysql.user AS u LEFT JOIN mysql.db AS d " "ON (u.user = d.user AND u.host = d.host) WHERE u.plugin = 'gssapi' " "UNION " - "SELECT u.user, u.host, t.db FROM " - "mysql.user AS u JOIN mysql.tables_priv AS t " - "ON (u.user = t.user AND u.host = t.host) WHERE u.plugin = 'gssapi';"; + "SELECT u.user, u.host, t.db, u.select_priv FROM " + "mysql.user AS u LEFT JOIN mysql.tables_priv AS t " + "ON (u.user = t.user AND u.host = t.host) WHERE u.plugin = 'gssapi' " + "ORDER BY user"; -#define GSSAPI_USERS_QUERY_NUM_FIELDS 3 +#define GSSAPI_USERS_QUERY_NUM_FIELDS 4 +/** + * SQLite queries for authenticating users + */ + +/** Name of the in-memory database */ +#define GSSAPI_DATABASE_NAME "file:gssapi.db?mode=memory&cache=shared" + +/** The table name where we store the users */ +#define GSSAPI_TABLE_NAME "gssapi_users" + +/** CREATE TABLE statement for the in-memory table */ +const char create_sql[] = + "CREATE TABLE IF NOT EXISTS " GSSAPI_TABLE_NAME + "(user varchar(255), host varchar(255), db varchar(255), anydb boolean)"; + +/** The query that is executed when a user is authenticated */ +static const char gssapi_auth_query[] = + "SELECT * FROM " GSSAPI_TABLE_NAME + " WHERE user = '%s' AND '%s' LIKE host AND (anydb = '1' OR '%s' = '' OR '%s' LIKE db) LIMIT 1"; + +/** Delete query used to clean up the database before loading new users */ +static const char delete_query[] = "DELETE FROM " GSSAPI_TABLE_NAME; + +/** The insert query template which adds users to the gssapi_users table */ +static const char insert_sql_pattern[] = + "INSERT INTO " GSSAPI_TABLE_NAME " VALUES ('%s', '%s', %s, %s)"; + +/** Used for NULL value creation in the INSERT query */ +static const char null_token[] = "NULL"; + +/** Flags for sqlite3_open_v2() */ +static int db_flags = SQLITE_OPEN_READWRITE | + SQLITE_OPEN_CREATE | + SQLITE_OPEN_URI | + SQLITE_OPEN_SHAREDCACHE; + +/** The instance structure for the client side GSSAPI authenticator, created in + * gssapi_auth_init() */ typedef struct gssapi_instance { - char *principal_name; + char *principal_name; /**< Service principal name given to the client */ + sqlite3 *handle; /**< SQLite3 database handle */ } GSSAPI_INSTANCE; /** @@ -54,6 +102,24 @@ void* gssapi_auth_init(char **options) { instance->principal_name = NULL; + if (sqlite3_open_v2(GSSAPI_DATABASE_NAME, &instance->handle, db_flags, NULL) != SQLITE_OK) + { + MXS_ERROR("Failed to open SQLite3 handle."); + MXS_FREE(instance); + return NULL; + } + + char *err; + + if (sqlite3_exec(instance->handle, create_sql, NULL, NULL, &err) != SQLITE_OK) + { + MXS_ERROR("Failed to create database: %s", err); + sqlite3_free(err); + sqlite3_close_v2(instance->handle); + MXS_FREE(instance); + return NULL; + } + for (int i = 0; options[i]; i++) { if (strstr(options[i], "principal_name")) @@ -84,6 +150,43 @@ void* gssapi_auth_init(char **options) return instance; } +void* gssapi_auth_alloc(void *instance) +{ + gssapi_auth_t* rval = MXS_MALLOC(sizeof(gssapi_auth_t)); + + if (rval) + { + rval->state = GSSAPI_AUTH_INIT; + rval->principal_name = NULL; + rval->principal_name_len = 0; + rval->sequence = 0; + + if (sqlite3_open_v2(GSSAPI_DATABASE_NAME, &rval->handle, db_flags, NULL) == SQLITE_OK) + { + sqlite3_busy_timeout(rval->handle, MXS_SQLITE_BUSY_TIMEOUT); + } + else + { + MXS_ERROR("Failed to open SQLite3 handle."); + MXS_FREE(rval); + rval = NULL; + } + } + + return rval; +} + +void gssapi_auth_free(void *data) +{ + if (data) + { + gssapi_auth_t *auth = (gssapi_auth_t*)data; + sqlite3_close_v2(auth->handle); + MXS_FREE(auth->principal_name); + MXS_FREE(auth); + } +} + /** * @brief Create a AuthSwitchRequest packet * @@ -268,6 +371,54 @@ static bool validate_gssapi_token(uint8_t* token, size_t len) return true; } +/** @brief Callback for sqlite3_exec() */ +static int auth_cb(void *data, int columns, char** rows, char** row_names) +{ + bool *rv = (bool*)data; + *rv = true; + return 0; +} + +/** + * @brief Verify the user has access to the database + * + * @param auth Authenticator session + * @param dcb Client DCB + * @param session MySQL session + * @return True if the user has access to the database + */ +static bool validate_user(gssapi_auth_t *auth, DCB *dcb, MYSQL_session *session) +{ + size_t len = sizeof(gssapi_auth_query) + strlen(session->user) + + strlen(session->db) + strlen(dcb->remote); + char sql[len + 1]; + bool rval = false; + char *err; + + sprintf(sql, gssapi_auth_query, session->user, dcb->remote, session->db, session->db); + + /** + * Try authentication twice; first time with the current users, second + * time with fresh users + */ + for (int i = 0; i < 2 && !rval; i++) + { + if (sqlite3_exec(auth->handle, sql, auth_cb, &rval, &err) != SQLITE_OK) + { + MXS_ERROR("Failed to execute auth query: %s", err); + sqlite3_free(err); + rval = false; + } + + if (!rval) + { + service_refresh_users(dcb->service); + } + } + + return rval; +} + /** * @brief Authenticate the client * @@ -302,7 +453,8 @@ int gssapi_auth_authenticate(DCB *dcb) MYSQL_session *ses = (MYSQL_session*)dcb->data; - if (validate_gssapi_token(ses->auth_token, ses->auth_token_len)) + if (validate_gssapi_token(ses->auth_token, ses->auth_token_len) && + validate_user(auth, dcb, ses)) { rval = MXS_AUTH_SUCCEEDED; } @@ -327,6 +479,59 @@ void gssapi_auth_free_data(DCB *dcb) } } +/** + * @brief Delete old users from the database + * @param handle Database handle + */ +static void delete_old_users(sqlite3 *handle) +{ + char *err; + + if (sqlite3_exec(handle, delete_query, NULL, NULL, &err) != SQLITE_OK) + { + MXS_ERROR("Failed to delete old users: %s", err); + sqlite3_free(err); + } +} + +/** + * @brief Add new GSSAPI user to the internal user database + * + * @param handle Database handle + * @param user Username + * @param host Host + * @param db Database + * @param anydb Global access to databases + */ +static void add_gssapi_user(sqlite3 *handle, const char *user, const char *host, + const char *db, bool anydb) +{ + size_t dblen = db ? strlen(db) + 2 : sizeof(null_token); /** +2 for single quotes */ + char dbstr[dblen + 1]; + + if (db) + { + sprintf(dbstr, "'%s'", db); + } + else + { + strcpy(dbstr, null_token); + } + + size_t len = sizeof(insert_sql_pattern) + strlen(user) + strlen(host) + dblen + 1; + char insert_sql[len + 1]; + sprintf(insert_sql, insert_sql_pattern, user, host, dbstr, anydb ? "1" : "0"); + + char *err; + if (sqlite3_exec(handle, insert_sql, NULL, NULL, &err) != SQLITE_OK) + { + MXS_ERROR("Failed to insert user: %s", err); + sqlite3_free(err); + } + + MXS_INFO("Added user: %s", insert_sql); +} + /** * @brief Load database users that use GSSAPI authentication * @@ -340,6 +545,7 @@ int gssapi_auth_load_users(SERV_LISTENER *listener) { char *user, *pw; int rval = MXS_AUTH_LOADUSERS_ERROR; + GSSAPI_INSTANCE *inst = (GSSAPI_INSTANCE*)listener->auth_instance; if (serviceGetUser(listener->service, &user, &pw) && (pw = decryptPassword(pw))) { @@ -351,13 +557,15 @@ int gssapi_auth_load_users(SERV_LISTENER *listener) { if (mysql_query(mysql, gssapi_users_query)) { - MXS_ERROR("Failed to query server '%s' for GSSAPI users.", - servers->server->unique_name); + MXS_ERROR("Failed to query server '%s' for GSSAPI users: %s", + servers->server->unique_name, mysql_error(mysql)); } else { MYSQL_RES *res = mysql_store_result(mysql); + delete_old_users(inst->handle); + if (res) { ss_dassert(mysql_num_fields(res) == GSSAPI_USERS_QUERY_NUM_FIELDS); @@ -365,8 +573,8 @@ int gssapi_auth_load_users(SERV_LISTENER *listener) while ((row = mysql_fetch_row(res))) { - /** TODO: Store this information in the users table of the listener */ - MXS_INFO("Would add: '%s'@'%s' for '%s'", row[0], row[1], row[2]); + add_gssapi_user(inst->handle, row[0], row[1], row[2], + row[3] && strcasecmp(row[3], "Y") == 0); } rval = MXS_AUTH_LOADUSERS_OK; @@ -375,6 +583,11 @@ int gssapi_auth_load_users(SERV_LISTENER *listener) } mysql_close(mysql); + + if (rval == MXS_AUTH_LOADUSERS_OK) + { + break; + } } } @@ -407,7 +620,7 @@ MODULE_INFO info = "GSSAPI authenticator" }; -static char *version_str = "V1.0.0"; +static char version_str[] = "V1.0.0"; /** * Version string entry point diff --git a/server/modules/authenticator/gssapi_auth.h b/server/modules/authenticator/gssapi_auth.h index be6bd7b0f..cbb8335da 100644 --- a/server/modules/authenticator/gssapi_auth.h +++ b/server/modules/authenticator/gssapi_auth.h @@ -18,6 +18,7 @@ #include #include #include +#include MXS_BEGIN_DECLS @@ -36,19 +37,16 @@ enum gssapi_auth_state GSSAPI_AUTH_FAILED }; -/** Common state tracking structure */ +/** Common structure for both backend and client authenticators */ typedef struct gssapi_auth { enum gssapi_auth_state state; /**< Authentication state*/ uint8_t *principal_name; /**< Principal name */ size_t principal_name_len; /**< Length of the principal name */ uint8_t sequence; /**< The next packet seqence number */ + sqlite3 *handle; /**< SQLite3 database handle */ } gssapi_auth_t; -/** These functions can used for the `create` and `destroy` entry points */ -void* gssapi_auth_alloc(void *instance); -void gssapi_auth_free(void *data); - /** Report GSSAPI errors */ void report_error(OM_uint32 major, OM_uint32 minor); diff --git a/server/modules/authenticator/gssapi_auth_common.c b/server/modules/authenticator/gssapi_auth_common.c index 5ba0a35e5..5b2c9cc33 100644 --- a/server/modules/authenticator/gssapi_auth_common.c +++ b/server/modules/authenticator/gssapi_auth_common.c @@ -15,31 +15,6 @@ #include #include -void* gssapi_auth_alloc(void *instance) -{ - gssapi_auth_t* rval = MXS_MALLOC(sizeof(gssapi_auth_t)); - - if (rval) - { - rval->state = GSSAPI_AUTH_INIT; - rval->principal_name = NULL; - rval->principal_name_len = 0; - rval->sequence = 0; - } - - return rval; -} - -void gssapi_auth_free(void *data) -{ - if (data) - { - gssapi_auth_t *auth = (gssapi_auth_t*)data; - MXS_FREE(auth->principal_name); - MXS_FREE(auth); - } -} - /** * @brief Report GSSAPI errors * diff --git a/server/modules/authenticator/gssapi_backend_auth.c b/server/modules/authenticator/gssapi_backend_auth.c index 9b43a7f31..44325881a 100644 --- a/server/modules/authenticator/gssapi_backend_auth.c +++ b/server/modules/authenticator/gssapi_backend_auth.c @@ -22,6 +22,31 @@ * @file gssapi_backend_auth.c - GSSAPI backend authenticator */ +void* gssapi_backend_auth_alloc(void *instance) +{ + gssapi_auth_t* rval = MXS_MALLOC(sizeof(gssapi_auth_t)); + + if (rval) + { + rval->state = GSSAPI_AUTH_INIT; + rval->principal_name = NULL; + rval->principal_name_len = 0; + rval->sequence = 0; + } + + return rval; +} + +void gssapi_backend_auth_free(void *data) +{ + if (data) + { + gssapi_auth_t *auth = (gssapi_auth_t*)data; + MXS_FREE(auth->principal_name); + MXS_FREE(auth); + } +} + /** * @brief Create a new GSSAPI token * @param dcb Backend DCB @@ -241,12 +266,12 @@ static int gssapi_backend_auth_authenticate(DCB *dcb) static GWAUTHENTICATOR MyObject = { NULL, /* No initialize entry point */ - gssapi_auth_alloc, /* Allocate authenticator data */ + gssapi_backend_auth_alloc, /* Allocate authenticator data */ gssapi_backend_auth_extract, /* Extract data into structure */ gssapi_backend_auth_connectssl, /* Check if client supports SSL */ gssapi_backend_auth_authenticate, /* Authenticate user credentials */ NULL, /* Client plugin will free shared data */ - gssapi_auth_free, /* Free authenticator data */ + gssapi_backend_auth_free, /* Free authenticator data */ NULL /* Load users from backend databases */ }; From 041df398197321710785f179f31939bf35133bf7 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Tue, 25 Oct 2016 10:58:49 +0300 Subject: [PATCH 085/215] Move transaction state management to the right place The transaction state must be updated after a buffer has been split into buffer containing individual packets. NOTE: The actual updating of the transaction state and the autocommit mode is currently wrong, but will be updated in a subsequent change. --- include/maxscale/routing.h | 13 ++ .../protocol/MySQL/MySQLClient/mysql_client.c | 111 +++++++++++------- 2 files changed, 82 insertions(+), 42 deletions(-) diff --git a/include/maxscale/routing.h b/include/maxscale/routing.h index 7d7ac8158..1544414a2 100644 --- a/include/maxscale/routing.h +++ b/include/maxscale/routing.h @@ -40,5 +40,18 @@ typedef enum routing_capability #define RCAP_TYPE_NONE 0 +/** + * Determines whether a particular capability type is required. + * + * @param capabilites The capability bits to be tested. + * @param type A particular capability type or a bitmask of types. + * + * @return True, if @c type is present in @c capabilities. + */ +static inline bool rcap_type_required(uint64_t capabilities, uint64_t type) +{ + return (capabilities & type) == type; +} + MXS_END_DECLS diff --git a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c index bad382043..b28191aa7 100644 --- a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c +++ b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c @@ -86,7 +86,7 @@ static int gw_client_hangup_event(DCB *dcb); static char *gw_default_auth(); static int gw_connection_limit(DCB *dcb, int limit); static int MySQLSendHandshake(DCB* dcb); -static int route_by_statement(SESSION *, GWBUF **); +static int route_by_statement(SESSION *, uint64_t, GWBUF **); static void mysql_client_auth_error_handling(DCB *dcb, int auth_val, int packet_number); static int gw_read_do_authentication(DCB *dcb, GWBUF *read_buffer, int nbytes_read); static int gw_read_normal_data(DCB *dcb, GWBUF *read_buffer, int nbytes_read); @@ -848,7 +848,7 @@ gw_read_normal_data(DCB *dcb, GWBUF *read_buffer, int nbytes_read) /** If the router requires statement input or we are still authenticating * we need to make sure that a complete SQL packet is read before continuing */ - if (capabilities & RCAP_TYPE_STMT_INPUT) + if (rcap_type_required(capabilities, RCAP_TYPE_STMT_INPUT)) { uint8_t* data; int packet_size; @@ -886,50 +886,14 @@ gw_read_finish_processing(DCB *dcb, GWBUF *read_buffer, uint64_t capabilities) /** Reset error handler when routing of the new query begins */ dcb->dcb_errhandle_called = false; - if (capabilities & RCAP_TYPE_STMT_INPUT) + if (rcap_type_required(capabilities, RCAP_TYPE_STMT_INPUT)) { - SESSION *ses = dcb->session; - ss_dassert(ses); - - uint32_t type = qc_get_type(read_buffer); - - if (type & QUERY_TYPE_BEGIN_TRX) - { - session_trx_state_t trx_state; - if (type & QUERY_TYPE_WRITE) - { - trx_state = SESSION_TRX_READ_WRITE; - } - else if (type & QUERY_TYPE_READ) - { - trx_state = SESSION_TRX_READ_ONLY; - } - else - { - trx_state = SESSION_TRX_ACTIVE; - } - - session_set_trx_state(ses, trx_state); - } - else if ((type & QUERY_TYPE_COMMIT) || (type & QUERY_TYPE_ROLLBACK)) - { - session_set_trx_state(ses, SESSION_TRX_INACTIVE); - } - else if (type & QUERY_TYPE_ENABLE_AUTOCOMMIT) - { - session_set_autocommit(ses, true); - } - else if (type & QUERY_TYPE_DISABLE_AUTOCOMMIT) - { - session_set_autocommit(ses, false); - } - /** * Feed each statement completely and separately * to router. The routing functions return 1 for * success or 0 for failure. */ - return_code = route_by_statement(session, &read_buffer) ? 0 : 1; + return_code = route_by_statement(session, capabilities, &read_buffer) ? 0 : 1; if (read_buffer != NULL) { @@ -940,7 +904,7 @@ gw_read_finish_processing(DCB *dcb, GWBUF *read_buffer, uint64_t capabilities) spinlock_release(&dcb->authlock); } } - else if (NULL != session->router_session || (capabilities & RCAP_TYPE_NO_RSESSION)) + else if (NULL != session->router_session || (rcap_type_required(capabilities, RCAP_TYPE_NO_RSESSION))) { /** Feed whole packet to router, which will free it * and return 1 for success, 0 for failure @@ -1400,11 +1364,12 @@ retblock: * leave incomplete packet to readbuf. * * @param session Session pointer + * @param capabilities The capabilities of the service. * @param p_readbuf Pointer to the address of GWBUF including the query * * @return 1 if succeed, */ -static int route_by_statement(SESSION* session, GWBUF** p_readbuf) +static int route_by_statement(SESSION* session, uint64_t capabilities, GWBUF** p_readbuf) { int rc; GWBUF* packetbuf; @@ -1426,6 +1391,7 @@ static int route_by_statement(SESSION* session, GWBUF** p_readbuf) * Collect incoming bytes to a buffer until complete packet has * arrived and then return the buffer. */ + // TODO: This should be replaced with modutil_get_next_MySQL_packet. packetbuf = gw_MySQL_get_next_packet(p_readbuf); if (packetbuf != NULL) @@ -1445,6 +1411,67 @@ static int route_by_statement(SESSION* session, GWBUF** p_readbuf) * sure it is set to each (MySQL) packet. */ gwbuf_set_type(packetbuf, GWBUF_TYPE_SINGLE_STMT); + + if (rcap_type_required(capabilities, RCAP_TYPE_CONTIGUOUS_INPUT)) + { + if (!GWBUF_IS_CONTIGUOUS(packetbuf)) + { + // TODO: As long as gw_MySQL_get_next_packet is used above, the buffer + // TODO: will be contiguous. That function should be replaced with + // TODO: modutil_get_next_MySQL_packet. + GWBUF* tmp = gwbuf_make_contiguous(packetbuf); + if (tmp) + { + packetbuf = tmp; + } + else + { + // TODO: A memory allocation failure. We should close the dcb + // TODO: and terminate the session. + rc = 0; + goto return_rc; + } + } + + if (rcap_type_required(capabilities, RCAP_TYPE_TRANSACTION_TRACKING)) + { + uint32_t type = qc_get_type(packetbuf); + + if (type & QUERY_TYPE_BEGIN_TRX) + { + session_trx_state_t trx_state; + if (type & QUERY_TYPE_WRITE) + { + trx_state = SESSION_TRX_READ_WRITE; + } + else if (type & QUERY_TYPE_READ) + { + trx_state = SESSION_TRX_READ_ONLY; + } + else + { + trx_state = SESSION_TRX_ACTIVE; + } + + session_set_trx_state(session, trx_state); + + if (type & QUERY_TYPE_DISABLE_AUTOCOMMIT) + { + session_set_autocommit(session, false); + } + } + else if ((type & QUERY_TYPE_COMMIT) || (type & QUERY_TYPE_ROLLBACK)) + { + session_set_trx_state(session, SESSION_TRX_INACTIVE); + + if (type & QUERY_TYPE_ENABLE_AUTOCOMMIT) + { + session_set_autocommit(session, true); + } + } + } + } + /** Route query */ rc = SESSION_ROUTE_QUERY(session, packetbuf); } From f961f87e5ee77e866c17704738a61daeb764fb14 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Tue, 25 Oct 2016 12:36:07 +0300 Subject: [PATCH 086/215] Update autocommit mode and transaction state The transaction state only reflects explicitly started transactions. Thus, by looking at the autocommit mode and the transaction state a component can figure out whether the current statement will be committed or not. --- include/maxscale/session.h | 35 +++++++++++++++++-- server/core/session.c | 4 +-- .../protocol/MySQL/MySQLClient/mysql_client.c | 35 ++++++++++--------- 3 files changed, 52 insertions(+), 22 deletions(-) diff --git a/include/maxscale/session.h b/include/maxscale/session.h index cba1456a8..1be219640 100644 --- a/include/maxscale/session.h +++ b/include/maxscale/session.h @@ -83,8 +83,6 @@ typedef enum typedef enum { - /*< The current transaction state is not known. */ - SESSION_TRX_UNKNOWN = 0, /*< There is no on-going transaction. */ SESSION_TRX_INACTIVE = SESSION_TRX_INACTIVE_BIT, /*< A transaction is active. */ @@ -241,6 +239,20 @@ void enable_session_timeouts(); /** * Get the transaction state of the session. * + * Note that this tells only the state of @e explicitly started transactions. + * That is, if @e autocommit is OFF, which means that there is always an + * active transaction that is ended with an explicit COMMIT or ROLLBACK, + * at which point a new transaction is started, this function will still + * return SESSION_TRX_INACTIVE, unless a transaction has explicitly been + * started with START TRANSACTION. + * + * Likewise, if @e autocommit is ON, which means that every statement is + * executed in a transaction of its own, this will return false, unless a + * transaction has explicitly been started with START TRANSACTION. + * + * @note The return value is valid only if either a router or a filter + * has declared that it needs RCAP_TYPE_TRANSACTION_TRACKING. + * * @param ses The SESSION object. * @return The transaction state. */ @@ -259,7 +271,12 @@ session_trx_state_t session_get_trx_state(const SESSION* ses); session_trx_state_t session_set_trx_state(SESSION* ses, session_trx_state_t new_state); /** - * Tells whether a transaction is active. + * Tells whether an explicit transaction is active. + * + * @see session_get_trx_state + * + * @note The return value is valid only if either a router or a filter + * has declared that it needs RCAP_TYPE_TRANSACTION_TRACKING. * * @return True if a transaction is active, false otherwise. */ @@ -271,6 +288,11 @@ static inline bool session_trx_is_active(const SESSION* ses) /** * Tells whether an explicit READ ONLY transaction is active. * + * @see session_get_trx_state + * + * @note The return value is valid only if either a router or a filter + * has declared that it needs RCAP_TYPE_TRANSACTION_TRACKING. + * * @return True if an explicit READ ONLY transaction is active, * false otherwise. */ @@ -282,6 +304,11 @@ static inline bool session_trx_is_read_only(const SESSION* ses) /** * Tells whether an explicit READ WRITE transaction is active. * + * @see session_get_trx_state + * + * @note The return value is valid only if either a router or a filter + * has declared that it needs RCAP_TYPE_TRANSACTION_TRACKING. + * * @return True if an explicit READ WRITE transaction is active, * false otherwise. */ @@ -303,6 +330,8 @@ static inline bool session_trx_is_read_write(const SESSION* ses) * * Note also that by default autocommit is ON. * + * @see session_get_trx_state + * * @return True if autocommit has been set ON, false otherwise. */ static inline bool session_is_autocommit(const SESSION* ses) diff --git a/server/core/session.c b/server/core/session.c index 5e51cc48a..8d62f23b8 100644 --- a/server/core/session.c +++ b/server/core/session.c @@ -142,7 +142,7 @@ session_alloc(SERVICE *service, DCB *client_dcb) */ session->state = SESSION_STATE_READY; - session->trx_state = SESSION_TRX_UNKNOWN; + session->trx_state = SESSION_TRX_INACTIVE; session->autocommit = true; /* * Only create a router session if we are not the listening @@ -1081,8 +1081,6 @@ const char* session_trx_state_to_string(session_trx_state_t state) { switch (state) { - case SESSION_TRX_UNKNOWN: - return "SESSION_TRX_UNKNOWN"; case SESSION_TRX_INACTIVE: return "SESSION_TRX_INACTIVE"; case SESSION_TRX_ACTIVE: diff --git a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c index b28191aa7..93b2f907b 100644 --- a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c +++ b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c @@ -1439,25 +1439,28 @@ static int route_by_statement(SESSION* session, uint64_t capabilities, GWBUF** p if (type & QUERY_TYPE_BEGIN_TRX) { - session_trx_state_t trx_state; - if (type & QUERY_TYPE_WRITE) - { - trx_state = SESSION_TRX_READ_WRITE; - } - else if (type & QUERY_TYPE_READ) - { - trx_state = SESSION_TRX_READ_ONLY; - } - else - { - trx_state = SESSION_TRX_ACTIVE; - } - - session_set_trx_state(session, trx_state); - if (type & QUERY_TYPE_DISABLE_AUTOCOMMIT) { session_set_autocommit(session, false); + session_set_trx_state(session, SESSION_TRX_INACTIVE); + } + else + { + session_trx_state_t trx_state; + if (type & QUERY_TYPE_WRITE) + { + trx_state = SESSION_TRX_READ_WRITE; + } + else if (type & QUERY_TYPE_READ) + { + trx_state = SESSION_TRX_READ_ONLY; + } + else + { + trx_state = SESSION_TRX_ACTIVE; + } + + session_set_trx_state(session, trx_state); } } else if ((type & QUERY_TYPE_COMMIT) || (type & QUERY_TYPE_ROLLBACK)) From 8391579206768590ee38057d0fbc55741c0e48ea Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Tue, 25 Oct 2016 13:53:10 +0300 Subject: [PATCH 087/215] Update cache with new capabilities Now that a filter can express that the transaction state is tracked, the cache implementation can be simplified. We do not need to cater for the case that a "too short" or "too long" a packet would be delivered. Further, since the autocommit mode and transaction state of the session are tracked, the filter can cache data when it is safe to do so. In practice that means when either AUTOCOMMIT is ON and no explicit transaction is active or when a READ ONLY transaction is active, irrespective of the autocommit state. In principle it would be possible to tentatively cache data during a transaction, and if the transaction is committed successfully flush the tentatively cached data to the actual cache, but that will be for another day. --- server/modules/filter/cache/cache.c | 180 ++++++++++++---------------- 1 file changed, 79 insertions(+), 101 deletions(-) diff --git a/server/modules/filter/cache/cache.c b/server/modules/filter/cache/cache.c index fa6ef74d0..de830d7cd 100644 --- a/server/modules/filter/cache/cache.c +++ b/server/modules/filter/cache/cache.c @@ -133,11 +133,6 @@ typedef enum cache_session_state CACHE_IGNORING_RESPONSE, // We are not interested in the data received from the server. } cache_session_state_t; -typedef struct cache_request_state -{ - GWBUF* data; /**< Request data, possibly incomplete. */ -} CACHE_REQUEST_STATE; - typedef struct cache_response_state { GWBUF* data; /**< Response data, possibly incomplete. */ @@ -156,7 +151,6 @@ typedef struct cache_session_data CACHE_STORAGE *storage; /**< The storage to be used with this session data. */ DOWNSTREAM down; /**< The previous filter or equivalent. */ UPSTREAM up; /**< The next filter or equivalent. */ - CACHE_REQUEST_STATE req; /**< The request state. */ CACHE_RESPONSE_STATE res; /**< The response state. */ SESSION *session; /**< The session this data is associated with. */ char key[CACHE_KEY_MAXLEN]; /**< Key storage. */ @@ -333,129 +327,113 @@ static void setUpstream(FILTER *instance, void *sdata, UPSTREAM *up) * * @param instance The filter instance data * @param sdata The filter session data - * @param packets The query data + * @param buffer Buffer containing an MySQL protocol packet. */ -static int routeQuery(FILTER *instance, void *sdata, GWBUF *data) +static int routeQuery(FILTER *instance, void *sdata, GWBUF *packet) { CACHE_INSTANCE *cinstance = (CACHE_INSTANCE*)instance; CACHE_SESSION_DATA *csdata = (CACHE_SESSION_DATA*)sdata; - if (csdata->req.data) - { - gwbuf_append(csdata->req.data, data); - } - else - { - csdata->req.data = data; - } + uint8_t *data = GWBUF_DATA(packet); - GWBUF *packet = modutil_get_next_MySQL_packet(&csdata->req.data); + // All of these should be guaranteed by RCAP_TYPE_TRANSACTION_TRACKING + ss_dassert(GWBUF_IS_CONTIGUOUS(packet)); + ss_dassert(GWBUF_LENGTH(packet) >= MYSQL_HEADER_LEN + 1); + ss_dassert(MYSQL_GET_PACKET_LEN(data) + MYSQL_HEADER_LEN == GWBUF_LENGTH(packet)); + + bool use_default = true; + + cache_response_state_reset(&csdata->res); + csdata->state = CACHE_IGNORING_RESPONSE; int rv; - if (packet) + switch ((int)MYSQL_GET_COMMAND(data)) { - bool use_default = true; - - cache_response_state_reset(&csdata->res); - csdata->state = CACHE_IGNORING_RESPONSE; - - if (gwbuf_length(packet) > MYSQL_HEADER_LEN + 1) // We need at least a packet with a type. + case MYSQL_COM_INIT_DB: { - uint8_t header[MYSQL_HEADER_LEN + 1]; + ss_dassert(!csdata->use_db); + size_t len = MYSQL_GET_PACKET_LEN(data) - 1; // Remove the command byte. + csdata->use_db = MXS_MALLOC(len + 1); - gwbuf_copy_data(packet, 0, sizeof(header), header); - - switch ((int)MYSQL_GET_COMMAND(header)) + if (csdata->use_db) { - case MYSQL_COM_INIT_DB: + memcpy(csdata->use_db, data + MYSQL_HEADER_LEN + 1, len); + csdata->use_db[len] = 0; + csdata->state = CACHE_EXPECTING_USE_RESPONSE; + } + else + { + // Memory allocation failed. We need to remove the default database to + // prevent incorrect cache entries, since we won't know what the + // default db is. But we only need to do that if "USE " really + // succeeds. The right thing will happen by itself in + // handle_expecting_use_response(); if OK is returned, default_db will + // become NULL, if ERR, default_db will not be changed. + } + } + break; + + case MYSQL_COM_QUERY: + { + // We do not care whether the query was fully parsed or not. + // If a query cannot be fully parsed, the worst thing that can + // happen is that caching is not used, even though it would be + // possible. + if (qc_get_operation(packet) == QUERY_OP_SELECT) + { + SESSION *session = csdata->session; + + if ((session_is_autocommit(session) && !session_trx_is_active(session)) || + session_trx_is_read_only(session)) { - ss_dassert(!csdata->use_db); - size_t len = MYSQL_GET_PACKET_LEN(header) - 1; // Remove the command byte. - csdata->use_db = MXS_MALLOC(len + 1); - - if (csdata->use_db) + if (cache_rules_should_store(cinstance->rules, csdata->default_db, packet)) { - uint8_t *use_db = (uint8_t*)csdata->use_db; - gwbuf_copy_data(packet, MYSQL_HEADER_LEN + 1, len, use_db); - csdata->use_db[len] = 0; - csdata->state = CACHE_EXPECTING_USE_RESPONSE; - } - else - { - // Memory allocation failed. We need to remove the default database to - // prevent incorrect cache entries, since we won't know what the - // default db is. But we only need to do that if "USE " really - // succeeds. The right thing will happen by itself in - // handle_expecting_use_response(); if OK is returned, default_db will - // become NULL, if ERR, default_db will not be changed. - } - } - break; - - case MYSQL_COM_QUERY: - { - GWBUF *tmp = gwbuf_make_contiguous(packet); - - if (tmp) - { - packet = tmp; - - // We do not care whether the query was fully parsed or not. - // If a query cannot be fully parsed, the worst thing that can - // happen is that caching is not used, even though it would be - // possible. - - if (qc_get_operation(packet) == QUERY_OP_SELECT) + if (cache_rules_should_use(cinstance->rules, csdata->session)) { - if (cache_rules_should_store(cinstance->rules, csdata->default_db, packet)) + GWBUF *result; + use_default = !route_using_cache(csdata, packet, &result); + + if (use_default) { - if (cache_rules_should_use(cinstance->rules, csdata->session)) - { - GWBUF *result; - use_default = !route_using_cache(csdata, packet, &result); - - if (use_default) - { - csdata->state = CACHE_EXPECTING_RESPONSE; - } - else - { - csdata->state = CACHE_EXPECTING_NOTHING; - C_DEBUG("Using data from cache."); - gwbuf_free(packet); - DCB *dcb = csdata->session->client_dcb; - - // TODO: This is not ok. Any filters before this filter, will not - // TODO: see this data. - rv = dcb->func.write(dcb, result); - } - } + csdata->state = CACHE_EXPECTING_RESPONSE; } else { - csdata->state = CACHE_IGNORING_RESPONSE; + csdata->state = CACHE_EXPECTING_NOTHING; + C_DEBUG("Using data from cache."); + gwbuf_free(packet); + DCB *dcb = csdata->session->client_dcb; + + // TODO: This is not ok. Any filters before this filter, will not + // TODO: see this data. + rv = dcb->func.write(dcb, result); } } } + else + { + csdata->state = CACHE_IGNORING_RESPONSE; + } + } + else + { + C_DEBUG("autocommit = %s and transaction state %s => Not using or storing to cache.", + session_is_autocommit(csdata->session) ? "ON" : "OFF", + session_trx_state_to_string(session_get_trx_state(csdata->session))); } - break; - - default: - break; } - } + break; - if (use_default) - { - C_DEBUG("Using default processing."); - rv = csdata->down.routeQuery(csdata->down.instance, csdata->down.session, packet); + default: + break; } } - else + + if (use_default) { - // We need more data before we can do something. - rv = 1; + C_DEBUG("Using default processing."); + rv = csdata->down.routeQuery(csdata->down.instance, csdata->down.session, packet); } return rv; @@ -560,7 +538,7 @@ static void diagnostics(FILTER *instance, void *sdata, DCB *dcb) */ static uint64_t getCapabilities(void) { - return RCAP_TYPE_STMT_INPUT; + return RCAP_TYPE_TRANSACTION_TRACKING; } // From 46c2b6e76a1ed55406b5edd40525b07d58daf52e Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Tue, 25 Oct 2016 20:28:50 +0300 Subject: [PATCH 088/215] Only parse COM_QUERY packets --- .../protocol/MySQL/MySQLClient/mysql_client.c | 55 ++++++++++--------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c index 93b2f907b..9346eb32e 100644 --- a/server/modules/protocol/MySQL/MySQLClient/mysql_client.c +++ b/server/modules/protocol/MySQL/MySQLClient/mysql_client.c @@ -1435,41 +1435,46 @@ static int route_by_statement(SESSION* session, uint64_t capabilities, GWBUF** p if (rcap_type_required(capabilities, RCAP_TYPE_TRANSACTION_TRACKING)) { - uint32_t type = qc_get_type(packetbuf); + uint32_t *data = GWBUF_DATA(packetbuf); - if (type & QUERY_TYPE_BEGIN_TRX) + if (MYSQL_GET_COMMAND(data) == MYSQL_COM_QUERY) { - if (type & QUERY_TYPE_DISABLE_AUTOCOMMIT) + uint32_t type = qc_get_type(packetbuf); + + if (type & QUERY_TYPE_BEGIN_TRX) { - session_set_autocommit(session, false); - session_set_trx_state(session, SESSION_TRX_INACTIVE); - } - else - { - session_trx_state_t trx_state; - if (type & QUERY_TYPE_WRITE) + if (type & QUERY_TYPE_DISABLE_AUTOCOMMIT) { - trx_state = SESSION_TRX_READ_WRITE; - } - else if (type & QUERY_TYPE_READ) - { - trx_state = SESSION_TRX_READ_ONLY; + session_set_autocommit(session, false); + session_set_trx_state(session, SESSION_TRX_INACTIVE); } else { - trx_state = SESSION_TRX_ACTIVE; + session_trx_state_t trx_state; + if (type & QUERY_TYPE_WRITE) + { + trx_state = SESSION_TRX_READ_WRITE; + } + else if (type & QUERY_TYPE_READ) + { + trx_state = SESSION_TRX_READ_ONLY; + } + else + { + trx_state = SESSION_TRX_ACTIVE; + } + + session_set_trx_state(session, trx_state); } - - session_set_trx_state(session, trx_state); } - } - else if ((type & QUERY_TYPE_COMMIT) || (type & QUERY_TYPE_ROLLBACK)) - { - session_set_trx_state(session, SESSION_TRX_INACTIVE); - - if (type & QUERY_TYPE_ENABLE_AUTOCOMMIT) + else if ((type & QUERY_TYPE_COMMIT) || (type & QUERY_TYPE_ROLLBACK)) { - session_set_autocommit(session, true); + session_set_trx_state(session, SESSION_TRX_INACTIVE); + + if (type & QUERY_TYPE_ENABLE_AUTOCOMMIT) + { + session_set_autocommit(session, true); + } } } } From 8f8823dc416d988119112d27bc2ecfa4ed34bbb2 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Tue, 25 Oct 2016 17:59:46 +0300 Subject: [PATCH 089/215] qc_sqlite: Protect against misuse - Ensure contiguous buffer of expected size. - Ensure COM_QUERY content. --- query_classifier/qc_sqlite/qc_sqlite.c | 66 ++++++++++++++++++-------- query_classifier/test/compare.cc | 13 ++--- 2 files changed, 52 insertions(+), 27 deletions(-) diff --git a/query_classifier/qc_sqlite/qc_sqlite.c b/query_classifier/qc_sqlite/qc_sqlite.c index 942e01a6f..a78238111 100644 --- a/query_classifier/qc_sqlite/qc_sqlite.c +++ b/query_classifier/qc_sqlite/qc_sqlite.c @@ -433,36 +433,60 @@ static bool parse_query(GWBUF* query) bool parsed = false; ss_dassert(!query_is_parsed(query)); - QC_SQLITE_INFO* info = info_alloc(); - - if (info) + if (GWBUF_IS_CONTIGUOUS(query)) { - this_thread.info = info; - - // TODO: Somewhere it needs to be ensured that this buffer is contiguous. - // TODO: Where is it checked that the GWBUF really contains a query? uint8_t* data = (uint8_t*) GWBUF_DATA(query); - size_t len = MYSQL_GET_PACKET_LEN(data) - 1; // Subtract 1 for packet type byte. - const char* s = (const char*) &data[5]; // TODO: Are there symbolic constants somewhere? + if ((GWBUF_LENGTH(query) >= MYSQL_HEADER_LEN + 1) && + (GWBUF_LENGTH(query) == MYSQL_HEADER_LEN + MYSQL_GET_PACKET_LEN(data))) + { + if (MYSQL_GET_COMMAND(data) == MYSQL_COM_QUERY) + { + QC_SQLITE_INFO* info = info_alloc(); - this_thread.info->query = s; - this_thread.info->query_len = len; - parse_query_string(s, len); - this_thread.info->query = NULL; - this_thread.info->query_len = 0; + if (info) + { + this_thread.info = info; - // TODO: Add return value to gwbuf_add_buffer_object. - // Always added; also when it was not recognized. If it was not recognized now, - // it won't be if we try a second time. - gwbuf_add_buffer_object(query, GWBUF_PARSING_INFO, info, buffer_object_free); - parsed = true; + size_t len = MYSQL_GET_PACKET_LEN(data) - 1; // Subtract 1 for packet type byte. - this_thread.info = NULL; + const char* s = (const char*) &data[MYSQL_HEADER_LEN + 1]; + + this_thread.info->query = s; + this_thread.info->query_len = len; + parse_query_string(s, len); + this_thread.info->query = NULL; + this_thread.info->query_len = 0; + + // TODO: Add return value to gwbuf_add_buffer_object. + // Always added; also when it was not recognized. If it was not recognized now, + // it won't be if we try a second time. + gwbuf_add_buffer_object(query, GWBUF_PARSING_INFO, info, buffer_object_free); + parsed = true; + + this_thread.info = NULL; + } + else + { + MXS_ERROR("qc_sqlite: Could not allocate structure for containing parse data."); + } + } + else + { + MXS_ERROR("qc_sqlite: The provided buffer does not contain a COM_QUERY, but a %s.", + STRPACKETTYPE(MYSQL_GET_COMMAND(data))); + } + } + else + { + MXS_ERROR("qc_sqlite: Packet size %ld, provided buffer is %ld.", + MYSQL_HEADER_LEN + MYSQL_GET_PACKET_LEN(data), + GWBUF_LENGTH(query)); + } } else { - MXS_ERROR("qc_sqlite: Could not allocate structure for containing parse data."); + MXS_ERROR("qc_sqlite: Provided buffer is not contiguous."); } return parsed; diff --git a/query_classifier/test/compare.cc b/query_classifier/test/compare.cc index 964a41dd0..5b9d2cac7 100644 --- a/query_classifier/test/compare.cc +++ b/query_classifier/test/compare.cc @@ -117,17 +117,18 @@ ostream& operator << (ostream& out, qc_parse_result_t x) GWBUF* create_gwbuf(const string& s) { - size_t len = s.length() + 1; - size_t gwbuf_len = len + MYSQL_HEADER_LEN + 1; + size_t len = s.length(); + size_t payload_len = len + 1; + size_t gwbuf_len = MYSQL_HEADER_LEN + payload_len; GWBUF* gwbuf = gwbuf_alloc(gwbuf_len); - *((unsigned char*)((char*)GWBUF_DATA(gwbuf))) = len; - *((unsigned char*)((char*)GWBUF_DATA(gwbuf) + 1)) = (len >> 8); - *((unsigned char*)((char*)GWBUF_DATA(gwbuf) + 2)) = (len >> 16); + *((unsigned char*)((char*)GWBUF_DATA(gwbuf))) = payload_len; + *((unsigned char*)((char*)GWBUF_DATA(gwbuf) + 1)) = (payload_len >> 8); + *((unsigned char*)((char*)GWBUF_DATA(gwbuf) + 2)) = (payload_len >> 16); *((unsigned char*)((char*)GWBUF_DATA(gwbuf) + 3)) = 0x00; *((unsigned char*)((char*)GWBUF_DATA(gwbuf) + 4)) = 0x03; - memcpy((char*)GWBUF_DATA(gwbuf) + 5, s.c_str(), s.length() + 1); + memcpy((char*)GWBUF_DATA(gwbuf) + 5, s.c_str(), len); return gwbuf; } From f2bee763f87f9af16f5a6b8bb26628647cd3ef79 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Tue, 25 Oct 2016 21:11:58 +0300 Subject: [PATCH 090/215] qc_sqlite: Take MXS_MODULE_NAME into use --- query_classifier/qc_sqlite/qc_sqlite.c | 67 +++++++++++++------------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/query_classifier/qc_sqlite/qc_sqlite.c b/query_classifier/qc_sqlite/qc_sqlite.c index a78238111..0a41ea240 100644 --- a/query_classifier/qc_sqlite/qc_sqlite.c +++ b/query_classifier/qc_sqlite/qc_sqlite.c @@ -11,17 +11,18 @@ * Public License. */ +#define MXS_MODULE_NAME "qc_sqlite" #include #include #include +#include #include #include -#include -#include -#include #include -#include +#include +#include +#include #include "builtin_functions.h" //#define QC_TRACE_ENABLED @@ -348,7 +349,7 @@ static void parse_query_string(const char* query, size_t len) if (qc_info_was_tokenized(this_thread.info->status)) { format = - "qc_sqlite: Statement was classified only based on keywords " + "Statement was classified only based on keywords " "(Sqlite3 error: %s, %s): \"%.*s%s\""; } else @@ -356,7 +357,7 @@ static void parse_query_string(const char* query, size_t len) if (qc_info_was_parsed(this_thread.info->status)) { format = - "qc_sqlite: Statement was only partially parsed " + "Statement was only partially parsed " "(Sqlite3 error: %s, %s): \"%.*s%s\""; // The status was set to QC_QUERY_PARSED, but sqlite3 returned an @@ -366,7 +367,7 @@ static void parse_query_string(const char* query, size_t len) else { format = - "qc_sqlite: Statement was neither parsed nor recognized from keywords " + "Statement was neither parsed nor recognized from keywords " "(Sqlite3 error: %s, %s): \"%.*s%s\""; } } @@ -406,7 +407,7 @@ static void parse_query_string(const char* query, size_t len) { // This suggests a callback from the parser into this module is not made. format = - "qc_sqlite: Statement was classified only based on keywords, " + "Statement was classified only based on keywords, " "even though the statement was parsed: \"%.*s%s\""; MXS_WARNING(format, l, query, suffix); @@ -416,7 +417,7 @@ static void parse_query_string(const char* query, size_t len) // This suggests there are keywords that should be recognized but are not, // a tentative classification cannot be (or is not) made using the keywords // seen and/or a callback from the parser into this module is not made. - format = "qc_sqlite: Statement was parsed, but not classified: \"%.*s%s\""; + format = "Statement was parsed, but not classified: \"%.*s%s\""; MXS_ERROR(format, l, query, suffix); } @@ -468,25 +469,25 @@ static bool parse_query(GWBUF* query) } else { - MXS_ERROR("qc_sqlite: Could not allocate structure for containing parse data."); + MXS_ERROR("Could not allocate structure for containing parse data."); } } else { - MXS_ERROR("qc_sqlite: The provided buffer does not contain a COM_QUERY, but a %s.", + MXS_ERROR("The provided buffer does not contain a COM_QUERY, but a %s.", STRPACKETTYPE(MYSQL_GET_COMMAND(data))); } } else { - MXS_ERROR("qc_sqlite: Packet size %ld, provided buffer is %ld.", + MXS_ERROR("Packet size %ld, provided buffer is %ld.", MYSQL_HEADER_LEN + MYSQL_GET_PACKET_LEN(data), GWBUF_LENGTH(query)); } } else { - MXS_ERROR("qc_sqlite: Provided buffer is not contiguous."); + MXS_ERROR("Provided buffer is not contiguous."); } return parsed; @@ -589,7 +590,7 @@ static void log_invalid_data(GWBUF* query, const char* message) length = GWBUF_LENGTH(query) - MYSQL_HEADER_LEN - 1; } - MXS_INFO("qc_sqlite: Parsing the query failed, %s: %*s", message, length, sql); + MXS_INFO("Parsing the query failed, %s: %*s", message, length, sql); } } } @@ -727,13 +728,13 @@ static void update_affected_fields(QC_SQLITE_INFO* info, } else if (zToken[0] != '?') { - MXS_WARNING("qc_sqlite: %s reported as VARIABLE.", zToken); + MXS_WARNING("%s reported as VARIABLE.", zToken); } } break; default: - MXS_DEBUG("qc_sqlite: Token %d not handled explicitly.", pExpr->op); + MXS_DEBUG("Token %d not handled explicitly.", pExpr->op); // Fallthrough intended. case TK_BETWEEN: case TK_CASE: @@ -2407,18 +2408,18 @@ static bool qc_sqlite_init(const char* args) } else { - MXS_WARNING("qc_sqlite: '%s' is not a number between %d and %d.", + MXS_WARNING("'%s' is not a number between %d and %d.", value, QC_LOG_NOTHING, QC_LOG_NON_TOKENIZED); } } else { - MXS_WARNING("qc_sqlite: '%s' is not a recognized argument.", key); + MXS_WARNING("'%s' is not a recognized argument.", key); } } else { - MXS_WARNING("qc_sqlite: '%s' is not a recognized argument string.", args); + MXS_WARNING("'%s' is not a recognized argument string.", args); } } @@ -2454,7 +2455,7 @@ static bool qc_sqlite_init(const char* args) ss_dassert(!true); } - MXS_NOTICE("qc_sqlite: %s", message); + MXS_NOTICE("%s", message); } } else @@ -2497,12 +2498,12 @@ static bool qc_sqlite_thread_init(void) { this_thread.initialized = true; - MXS_INFO("qc_sqlite: In-memory sqlite database successfully opened for thread %lu.", + MXS_INFO("In-memory sqlite database successfully opened for thread %lu.", (unsigned long) pthread_self()); } else { - MXS_ERROR("qc_sqlite: Failed to open in-memory sqlite database for thread %lu: %d, %s", + MXS_ERROR("Failed to open in-memory sqlite database for thread %lu: %d, %s", (unsigned long) pthread_self(), rc, sqlite3_errstr(rc)); } @@ -2520,7 +2521,7 @@ static void qc_sqlite_thread_end(void) if (rc != SQLITE_OK) { - MXS_WARNING("qc_sqlite: The closing of the thread specific sqlite database failed: %d, %s", + MXS_WARNING("The closing of the thread specific sqlite database failed: %d, %s", rc, sqlite3_errstr(rc)); } @@ -2561,7 +2562,7 @@ static uint32_t qc_sqlite_get_type(GWBUF* query) } else { - MXS_ERROR("qc_sqlite: The query could not be parsed. Response not valid."); + MXS_ERROR("The query could not be parsed. Response not valid."); } return types; @@ -2589,7 +2590,7 @@ static qc_query_op_t qc_sqlite_get_operation(GWBUF* query) } else { - MXS_ERROR("qc_sqlite: The query could not be parsed. Response not valid."); + MXS_ERROR("The query could not be parsed. Response not valid."); } return op; @@ -2621,7 +2622,7 @@ static char* qc_sqlite_get_created_table_name(GWBUF* query) } else { - MXS_ERROR("qc_sqlite: The query could not be parsed. Response not valid."); + MXS_ERROR("The query could not be parsed. Response not valid."); } return created_table_name; @@ -2649,7 +2650,7 @@ static bool qc_sqlite_is_drop_table_query(GWBUF* query) } else { - MXS_ERROR("qc_sqlite: The query could not be parsed. Response not valid."); + MXS_ERROR("The query could not be parsed. Response not valid."); } return is_drop_table; @@ -2677,7 +2678,7 @@ static bool qc_sqlite_is_real_query(GWBUF* query) } else { - MXS_ERROR("qc_sqlite: The query could not be parsed. Response not valid."); + MXS_ERROR("The query could not be parsed. Response not valid."); } return is_real_query; @@ -2721,7 +2722,7 @@ static char** qc_sqlite_get_table_names(GWBUF* query, int* tblsize, bool fullnam } else { - MXS_ERROR("qc_sqlite: The query could not be parsed. Response not valid."); + MXS_ERROR("The query could not be parsed. Response not valid."); } return table_names; @@ -2733,7 +2734,7 @@ static char* qc_sqlite_get_canonical(GWBUF* query) ss_dassert(this_unit.initialized); ss_dassert(this_thread.initialized); - MXS_ERROR("qc_sqlite: qc_get_canonical not implemented yet."); + MXS_ERROR("qc_get_canonical not implemented yet."); return NULL; } @@ -2760,7 +2761,7 @@ static bool qc_sqlite_query_has_clause(GWBUF* query) } else { - MXS_ERROR("qc_sqlite: The query could not be parsed. Response not valid."); + MXS_ERROR("The query could not be parsed. Response not valid."); } return has_clause; @@ -2788,7 +2789,7 @@ static char* qc_sqlite_get_affected_fields(GWBUF* query) } else { - MXS_ERROR("qc_sqlite: The query could not be parsed. Response not valid."); + MXS_ERROR("The query could not be parsed. Response not valid."); } if (!affected_fields) @@ -2827,7 +2828,7 @@ static char** qc_sqlite_get_database_names(GWBUF* query, int* sizep) } else { - MXS_ERROR("qc_sqlite: The query could not be parsed. Response not valid."); + MXS_ERROR("The query could not be parsed. Response not valid."); } return database_names; From d7ce3ad75beccb15575d0de6e034dec6a0b63dc4 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Wed, 26 Oct 2016 08:23:15 +0300 Subject: [PATCH 091/215] Replace include guards with #pragma once --- include/maxscale/adminusers.h.in | 4 ---- include/maxscale/alloc.h | 4 ---- include/maxscale/atomic.h | 4 ---- include/maxscale/buffer.h | 4 ---- include/maxscale/cdefs.h | 4 ---- include/maxscale/config.h | 4 ---- include/maxscale/dcb.h | 4 ---- include/maxscale/debug.h | 4 ---- include/maxscale/externcmd.h | 4 ---- include/maxscale/filter.h | 4 ---- include/maxscale/gw_authenticator.h | 5 ----- include/maxscale/gw_protocol.h | 5 ----- include/maxscale/gw_ssl.h | 4 ---- include/maxscale/gwbitmask.h | 4 ---- include/maxscale/gwdirs.h.in | 4 ---- include/maxscale/hashtable.h | 4 ---- include/maxscale/hint.h | 4 ---- include/maxscale/hk_heartbeat.h | 5 ----- include/maxscale/housekeeper.h | 4 ---- include/maxscale/limits.h | 4 ---- include/maxscale/listener.h | 4 ---- include/maxscale/listmanager.h | 4 ---- include/maxscale/log_manager.h | 4 ---- include/maxscale/maxadmin.h | 4 ---- include/maxscale/maxscale.h | 4 ---- include/maxscale/memlog.h | 4 ---- include/maxscale/modinfo.h | 4 ---- include/maxscale/modules.h | 4 ---- include/maxscale/modutil.h | 4 ---- include/maxscale/monitor.h | 4 ---- include/maxscale/mysql_binlog.h | 4 ---- include/maxscale/mysql_utils.h | 4 ---- include/maxscale/notification.h | 4 ---- include/maxscale/pcre2.h | 4 ---- include/maxscale/platform.h | 4 ---- include/maxscale/poll.h | 4 ---- include/maxscale/protocol/mysql.h | 5 +---- include/maxscale/query_classifier.h | 4 ---- include/maxscale/queuemanager.h | 4 ---- include/maxscale/random_jkiss.h | 4 ---- include/maxscale/rdtsc.h | 4 ---- include/maxscale/resultset.h | 4 ---- include/maxscale/router.h | 4 ---- include/maxscale/secrets.h | 4 ---- include/maxscale/server.h | 4 ---- include/maxscale/service.h | 4 ---- include/maxscale/session.h | 4 ---- include/maxscale/spinlock.h | 4 ---- include/maxscale/statistics.h | 4 ---- include/maxscale/thread.h | 4 ---- include/maxscale/users.h | 4 ---- include/maxscale/utils.h | 4 ---- 52 files changed, 1 insertion(+), 211 deletions(-) diff --git a/include/maxscale/adminusers.h.in b/include/maxscale/adminusers.h.in index bc9302b13..5526d1ee7 100644 --- a/include/maxscale/adminusers.h.in +++ b/include/maxscale/adminusers.h.in @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_ADMINUSERS_H -#define _MAXSCALE_ADMINUSERS_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -69,5 +67,3 @@ extern bool admin_verify_inet_user(const char *uname, const char *passwor extern void dcb_PrintAdminUsers(DCB *dcb); MXS_END_DECLS - -#endif diff --git a/include/maxscale/alloc.h b/include/maxscale/alloc.h index 49cd04a29..8c2dad478 100644 --- a/include/maxscale/alloc.h +++ b/include/maxscale/alloc.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_ALLOC_H -#define _MAXSCALE_ALLOC_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -79,5 +77,3 @@ char *mxs_strndup_a(const char *s, size_t n/*, const char *caller*/); #define MXS_ABORT_IF_FALSE(b) do { if (!b) { abort(); } } while (false) MXS_END_DECLS - -#endif diff --git a/include/maxscale/atomic.h b/include/maxscale/atomic.h index 71127ee7c..f6d694e0c 100644 --- a/include/maxscale/atomic.h +++ b/include/maxscale/atomic.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_ATOMIC_H -#define _MAXSCALE_ATOMIC_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -35,5 +33,3 @@ int atomic_add(int *variable, int value); int atomic_add(int *variable, int value); MXS_END_DECLS - -#endif diff --git a/include/maxscale/buffer.h b/include/maxscale/buffer.h index f963167a0..3e9995faf 100644 --- a/include/maxscale/buffer.h +++ b/include/maxscale/buffer.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_BUFFER_H -#define _MAXSCALE_BUFFER_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -219,5 +217,3 @@ extern void dprintAllBuffers(void *pdcb); #endif MXS_END_DECLS - -#endif diff --git a/include/maxscale/cdefs.h b/include/maxscale/cdefs.h index a26342939..3b24b55c9 100644 --- a/include/maxscale/cdefs.h +++ b/include/maxscale/cdefs.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_CDEFS_H -#define _MAXSCALE_CDEFS_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -73,5 +71,3 @@ #include #include #include - -#endif diff --git a/include/maxscale/config.h b/include/maxscale/config.h index 128a906eb..b765c33fb 100644 --- a/include/maxscale/config.h +++ b/include/maxscale/config.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_CONFIG_H -#define _MAXSCALE_CONFIG_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -166,5 +164,3 @@ void free_config_parameter(CONFIG_PARAMETER* p1); bool is_internal_service(const char *router); MXS_END_DECLS - -#endif diff --git a/include/maxscale/dcb.h b/include/maxscale/dcb.h index 5b08e377c..617d124e1 100644 --- a/include/maxscale/dcb.h +++ b/include/maxscale/dcb.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_DCB_H -#define _MAXSCALE_DCB_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -375,5 +373,3 @@ void dcb_append_readqueue(DCB *dcb, GWBUF *buffer); #define DCB_REPLIED(d) ((d)->flags & DCBF_REPLIED) MXS_END_DECLS - -#endif /* _DCB_H */ diff --git a/include/maxscale/debug.h b/include/maxscale/debug.h index 69d0d7367..dde1f3032 100644 --- a/include/maxscale/debug.h +++ b/include/maxscale/debug.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_DEBUG_H -#define _MAXSCALE_DEBUG_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -568,5 +566,3 @@ static bool conn_open[10240]; #endif /* FAKE_CODE */ MXS_END_DECLS - -#endif diff --git a/include/maxscale/externcmd.h b/include/maxscale/externcmd.h index 36efeb544..284c7e8dd 100644 --- a/include/maxscale/externcmd.h +++ b/include/maxscale/externcmd.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_EXTERN_CMD_HG -#define _MAXSCALE_EXTERN_CMD_HG /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -42,5 +40,3 @@ bool externcmd_can_execute(const char* argstr); bool externcmd_matches(const EXTERNCMD* cmd, const char* match); MXS_END_DECLS - -#endif diff --git a/include/maxscale/filter.h b/include/maxscale/filter.h index ec837d8e9..1eabbe326 100644 --- a/include/maxscale/filter.h +++ b/include/maxscale/filter.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_FILTER_H -#define _MAXSCALE_FILTER_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -140,5 +138,3 @@ typedef enum filter_capability */ MXS_END_DECLS - -#endif diff --git a/include/maxscale/gw_authenticator.h b/include/maxscale/gw_authenticator.h index deafd14b7..b76ea0c59 100644 --- a/include/maxscale/gw_authenticator.h +++ b/include/maxscale/gw_authenticator.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_GW_AUTHENTICATOR_H -#define _MAXSCALE_GW_AUTHENTICATOR_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -140,6 +138,3 @@ bool authenticator_init(void **instance, const char *authenticator, const char * const char* get_default_authenticator(const char *protocol); MXS_END_DECLS - -#endif /* GW_AUTHENTICATOR_H */ - diff --git a/include/maxscale/gw_protocol.h b/include/maxscale/gw_protocol.h index aa5ad2abf..e0f8b774e 100644 --- a/include/maxscale/gw_protocol.h +++ b/include/maxscale/gw_protocol.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_GW_PROTOCOL_H -#define _MAXSCALE_GW_PROTOCOL_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -86,6 +84,3 @@ typedef struct gw_protocol #define GWPROTOCOL_VERSION {1, 1, 0} MXS_END_DECLS - -#endif /* GW_PROTOCOL_H */ - diff --git a/include/maxscale/gw_ssl.h b/include/maxscale/gw_ssl.h index 65ac9ca82..d15733e79 100644 --- a/include/maxscale/gw_ssl.h +++ b/include/maxscale/gw_ssl.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_GW_SSL_H -#define _MAXSCALE_GW_SSL_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -83,5 +81,3 @@ bool ssl_required_but_not_negotiated(struct dcb *dcb); const char* ssl_method_type_to_string(ssl_method_type_t method_type); MXS_END_DECLS - -#endif /* _GW_SSL_H */ diff --git a/include/maxscale/gwbitmask.h b/include/maxscale/gwbitmask.h index d5d041eea..1a47c26fd 100644 --- a/include/maxscale/gwbitmask.h +++ b/include/maxscale/gwbitmask.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_GWBITMASK_H -#define _MAXSCALE_GWBITMASK_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -60,5 +58,3 @@ extern void bitmask_copy(GWBITMASK *, GWBITMASK *); extern char *bitmask_render_readable(GWBITMASK *); MXS_END_DECLS - -#endif diff --git a/include/maxscale/gwdirs.h.in b/include/maxscale/gwdirs.h.in index a782c30d2..909d324f0 100644 --- a/include/maxscale/gwdirs.h.in +++ b/include/maxscale/gwdirs.h.in @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_GW_DIRS_HG -#define _MAXSCALE_GW_DIRS_HG /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -79,5 +77,3 @@ char* get_langdir(); char* get_execdir(); MXS_END_DECLS - -#endif diff --git a/include/maxscale/hashtable.h b/include/maxscale/hashtable.h index 185b195ab..6eabfa87b 100644 --- a/include/maxscale/hashtable.h +++ b/include/maxscale/hashtable.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_HASTABLE_H -#define _MAXSCALE_HASTABLE_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -153,5 +151,3 @@ extern void* hashtable_item_strdup(const void *str); extern int hashtable_item_strhash(const void *str); MXS_END_DECLS - -#endif diff --git a/include/maxscale/hint.h b/include/maxscale/hint.h index d1943b900..d79ba731b 100644 --- a/include/maxscale/hint.h +++ b/include/maxscale/hint.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_HINT_H -#define _MAXSCALE_HINT_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -68,5 +66,3 @@ extern HINT *hint_dup(HINT *); bool hint_exists(HINT **, HINT_TYPE); MXS_END_DECLS - -#endif diff --git a/include/maxscale/hk_heartbeat.h b/include/maxscale/hk_heartbeat.h index bdc2567ef..6ad90161d 100644 --- a/include/maxscale/hk_heartbeat.h +++ b/include/maxscale/hk_heartbeat.h @@ -1,7 +1,4 @@ #pragma once -#ifndef _MAXSCALE_HK_HEARTBEAT_H -#define _MAXSCALE_HK_HEARTBEAT_H - /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -27,5 +24,3 @@ MXS_BEGIN_DECLS extern long hkheartbeat; MXS_END_DECLS - -#endif diff --git a/include/maxscale/housekeeper.h b/include/maxscale/housekeeper.h index 131a0fbae..3a7922ceb 100644 --- a/include/maxscale/housekeeper.h +++ b/include/maxscale/housekeeper.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_HOUSEKEEPER_H -#define _MAXSCALE_HOUSEKEEPER_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -61,5 +59,3 @@ extern void hkshutdown(); extern void hkshow_tasks(DCB *pdcb); MXS_END_DECLS - -#endif diff --git a/include/maxscale/limits.h b/include/maxscale/limits.h index 43eab289f..548f7896f 100644 --- a/include/maxscale/limits.h +++ b/include/maxscale/limits.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_LIMITS_H -#define _MAXSCALE_LIMITS_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -74,5 +72,3 @@ MXS_BEGIN_DECLS #define MXS_MAX_THREADS 255 MXS_END_DECLS - -#endif diff --git a/include/maxscale/listener.h b/include/maxscale/listener.h index 0e8cb6097..0df0f5d8d 100644 --- a/include/maxscale/listener.h +++ b/include/maxscale/listener.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_LISTENER_H -#define _MAXSCALE_LISTENER_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -70,5 +68,3 @@ void listener_set_certificates(SSL_LISTENER *ssl_listener, char* cert, char* key int listener_init_SSL(SSL_LISTENER *ssl_listener); MXS_END_DECLS - -#endif diff --git a/include/maxscale/listmanager.h b/include/maxscale/listmanager.h index b35733847..eb582efed 100644 --- a/include/maxscale/listmanager.h +++ b/include/maxscale/listmanager.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_LISTMANAGER_H -#define _MAXSCALE_LISTMANAGER_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -120,5 +118,3 @@ list_entry_t *list_remove_first(LIST_CONFIG *list_config); list_entry_t *list_remove_last(LIST_CONFIG *list_config); MXS_END_DECLS - -#endif /* LISTMANAGER_H */ diff --git a/include/maxscale/log_manager.h b/include/maxscale/log_manager.h index e46369f5e..307fc6e89 100644 --- a/include/maxscale/log_manager.h +++ b/include/maxscale/log_manager.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_LOG_MANAGER_H -#define _MAXSCALE_LOG_MANAGER_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -203,5 +201,3 @@ enum }; MXS_END_DECLS - -#endif /** LOG_MANAGER_H */ diff --git a/include/maxscale/maxadmin.h b/include/maxscale/maxadmin.h index 8eb577b6d..ab8c0f4c6 100644 --- a/include/maxscale/maxadmin.h +++ b/include/maxscale/maxadmin.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_MAXADMIN_H -#define _MAXSCALE_MAXADMIN_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -34,5 +32,3 @@ MXS_BEGIN_DECLS #define MAXADMIN_AUTH_PASSWORD_PROMPT_LEN 8 MXS_END_DECLS - -#endif diff --git a/include/maxscale/maxscale.h b/include/maxscale/maxscale.h index 5263409cf..93d43bd19 100644 --- a/include/maxscale/maxscale.h +++ b/include/maxscale/maxscale.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_MAXSCALE_H -#define _MAXSCALE_MAXSCALE_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -47,5 +45,3 @@ time_t maxscale_started(void); int maxscale_uptime(void); MXS_END_DECLS - -#endif diff --git a/include/maxscale/memlog.h b/include/maxscale/memlog.h index 5c6fdfa6c..8d609af3d 100644 --- a/include/maxscale/memlog.h +++ b/include/maxscale/memlog.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_MEMLOG_H -#define _MAXSCALE_MEMLOG_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -65,5 +63,3 @@ extern void memlog_flush_all(); extern void memlog_flush(MEMLOG *); MXS_END_DECLS - -#endif diff --git a/include/maxscale/modinfo.h b/include/maxscale/modinfo.h index 99d68a37f..09e413bd0 100644 --- a/include/maxscale/modinfo.h +++ b/include/maxscale/modinfo.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_MODINFO_H -#define _MAXSCALE_MODINFO_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -90,5 +88,3 @@ typedef struct } MODULE_INFO; MXS_END_DECLS - -#endif diff --git a/include/maxscale/modules.h b/include/maxscale/modules.h index eb21810c9..1c8f5787d 100644 --- a/include/maxscale/modules.h +++ b/include/maxscale/modules.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_MODULES_H -#define _MAXSCALE_MODULES_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -75,5 +73,3 @@ extern void module_feedback_send(void*); extern void moduleShowFeedbackReport(DCB *dcb); MXS_END_DECLS - -#endif diff --git a/include/maxscale/modutil.h b/include/maxscale/modutil.h index 9f609f795..ad888894f 100644 --- a/include/maxscale/modutil.h +++ b/include/maxscale/modutil.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_MODUTIL_H -#define _MAXSCALE_MODUTIL_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -76,5 +74,3 @@ bool is_mysql_sp_end(const char* start, int len); char* modutil_get_canonical(GWBUF* querybuf); MXS_END_DECLS - -#endif diff --git a/include/maxscale/monitor.h b/include/maxscale/monitor.h index 9d0b9269b..106b17113 100644 --- a/include/maxscale/monitor.h +++ b/include/maxscale/monitor.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_MONITOR_H -#define _MAXSCALE_MONITOR_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -232,5 +230,3 @@ void mon_log_connect_error(MONITOR_SERVERS* database, connect_result_t rval); void mon_log_state_change(MONITOR_SERVERS *ptr); MXS_END_DECLS - -#endif diff --git a/include/maxscale/mysql_binlog.h b/include/maxscale/mysql_binlog.h index 2e7f4c5de..ecf2f3068 100644 --- a/include/maxscale/mysql_binlog.h +++ b/include/maxscale/mysql_binlog.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_MYSQL_BINLOG_H -#define _MAXSCALE_MYSQL_BINLOG_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -96,5 +94,3 @@ uint64_t unpack_bit(uint8_t *ptr, uint8_t *null_mask, uint32_t col_count, void format_temporal_value(char *str, size_t size, uint8_t type, struct tm *tm); MXS_END_DECLS - -#endif /* MYSQL_BINLOG_H */ diff --git a/include/maxscale/mysql_utils.h b/include/maxscale/mysql_utils.h index f5b4d79ce..35748bf1e 100644 --- a/include/maxscale/mysql_utils.h +++ b/include/maxscale/mysql_utils.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_MYSQL_UTILS_H -#define _MAXSCALE_MYSQL_UTILS_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -34,5 +32,3 @@ char* lestr_consume(uint8_t** c, size_t *size); MYSQL *mxs_mysql_real_connect(MYSQL *mysql, SERVER *server, const char *user, const char *passwd); MXS_END_DECLS - -#endif diff --git a/include/maxscale/notification.h b/include/maxscale/notification.h index db410910f..af695ebb4 100644 --- a/include/maxscale/notification.h +++ b/include/maxscale/notification.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_NOTIFICATION_H -#define _MAXSCALE_NOTIFICATION_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -64,5 +62,3 @@ extern void gw_sha1_str(const uint8_t *in, int in_len, uint8_t *out); extern FEEDBACK_CONF * config_get_feedback_data(); MXS_END_DECLS - -#endif diff --git a/include/maxscale/pcre2.h b/include/maxscale/pcre2.h index e2373cca4..77d7d6976 100644 --- a/include/maxscale/pcre2.h +++ b/include/maxscale/pcre2.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_PCRE2_H -#define _MAXSCALE_PCRE2_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -50,5 +48,3 @@ mxs_pcre2_result_t mxs_pcre2_simple_match(const char* pattern, const char* subje int options, int* error); MXS_END_DECLS - -#endif diff --git a/include/maxscale/platform.h b/include/maxscale/platform.h index 486898d0b..8223d4d57 100644 --- a/include/maxscale/platform.h +++ b/include/maxscale/platform.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_PLATFORM_H -#define _MAXSCALE_PLATFORM_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -54,5 +52,3 @@ MXS_BEGIN_DECLS #endif // __cplusplus MXS_END_DECLS - -#endif // _PLATFORM_H diff --git a/include/maxscale/poll.h b/include/maxscale/poll.h index ab0e6e385..315423cb2 100644 --- a/include/maxscale/poll.h +++ b/include/maxscale/poll.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_POLL_H -#define _MAXSCALE_POLL_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -75,5 +73,3 @@ extern void poll_fake_write_event(DCB *dcb); extern void poll_fake_read_event(DCB *dcb); MXS_END_DECLS - -#endif diff --git a/include/maxscale/protocol/mysql.h b/include/maxscale/protocol/mysql.h index 3fc66cc49..7731c3c30 100644 --- a/include/maxscale/protocol/mysql.h +++ b/include/maxscale/protocol/mysql.h @@ -1,5 +1,4 @@ -#ifndef _MAXSCALE_PROTOCOL_MYSQL_H -#define _MAXSCALE_PROTOCOL_MYSQL_H +#pragma once /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -388,5 +387,3 @@ int mxs_mysql_send_ok(DCB *dcb, int sequence, int affected_rows, const char* mes bool mxs_mysql_is_ok_packet(GWBUF *buffer); MXS_END_DECLS - -#endif /** _MYSQL_PROTOCOL_H */ diff --git a/include/maxscale/query_classifier.h b/include/maxscale/query_classifier.h index f53b6a1ca..2c26a5970 100644 --- a/include/maxscale/query_classifier.h +++ b/include/maxscale/query_classifier.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_QUERY_CLASSIFIER_HG -#define _MAXSCALE_QUERY_CLASSIFIER_HG /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -132,5 +130,3 @@ struct query_classifier #define QUERY_CLASSIFIER_VERSION {1, 0, 0} MXS_END_DECLS - -#endif diff --git a/include/maxscale/queuemanager.h b/include/maxscale/queuemanager.h index 15f9ac39f..f8f1ee118 100644 --- a/include/maxscale/queuemanager.h +++ b/include/maxscale/queuemanager.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_QUEUEMANAGER_H -#define _MAXSCALE_QUEUEMANAGER_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -72,5 +70,3 @@ mxs_queue_count(QUEUE_CONFIG *queue_config) } MXS_END_DECLS - -#endif /* QUEUEMANAGER_H */ diff --git a/include/maxscale/random_jkiss.h b/include/maxscale/random_jkiss.h index c1041ea8f..5f1e2bee9 100644 --- a/include/maxscale/random_jkiss.h +++ b/include/maxscale/random_jkiss.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_RANDOM_JKISS_H -#define _MAXSCALE_RANDOM_JKISS_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -28,5 +26,3 @@ MXS_BEGIN_DECLS extern unsigned int random_jkiss(void); MXS_END_DECLS - -#endif /* RANDOM_H */ diff --git a/include/maxscale/rdtsc.h b/include/maxscale/rdtsc.h index f928fd85b..c87f96129 100644 --- a/include/maxscale/rdtsc.h +++ b/include/maxscale/rdtsc.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_RDTSC_H -#define _MAXSCALE_RDTSC_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -60,5 +58,3 @@ static __inline__ CYCLES rdtsc(void) } MXS_END_DECLS - -#endif diff --git a/include/maxscale/resultset.h b/include/maxscale/resultset.h index 90e9c8042..52d533bbb 100644 --- a/include/maxscale/resultset.h +++ b/include/maxscale/resultset.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_RESULTSET_H -#define _MAXSCALE_RESULTSET_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -90,5 +88,3 @@ extern void resultset_stream_mysql(RESULTSET *, DCB *); extern void resultset_stream_json(RESULTSET *, DCB *); MXS_END_DECLS - -#endif diff --git a/include/maxscale/router.h b/include/maxscale/router.h index 7ae6e3f2a..90e8083d0 100644 --- a/include/maxscale/router.h +++ b/include/maxscale/router.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_ROUTER_H -#define _MAXSCALE_ROUTER_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -110,5 +108,3 @@ typedef enum router_capability } router_capability_t; MXS_END_DECLS - -#endif diff --git a/include/maxscale/secrets.h b/include/maxscale/secrets.h index 257718c6a..655687ea3 100644 --- a/include/maxscale/secrets.h +++ b/include/maxscale/secrets.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_SECRETS_H -#define _MAXSCALE_SECRETS_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -60,5 +58,3 @@ extern char *decryptPassword(const char *); extern char *encryptPassword(const char*, const char *); MXS_END_DECLS - -#endif diff --git a/include/maxscale/server.h b/include/maxscale/server.h index f06167568..8b85c2b13 100644 --- a/include/maxscale/server.h +++ b/include/maxscale/server.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_SERVER_H -#define _MAXSCALE_SERVER_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -228,5 +226,3 @@ extern unsigned int server_map_status(char *str); extern bool server_set_version_string(SERVER* server, const char* string); MXS_END_DECLS - -#endif diff --git a/include/maxscale/service.h b/include/maxscale/service.h index dfbdb5b8a..815e8cf62 100644 --- a/include/maxscale/service.h +++ b/include/maxscale/service.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_SERVICE_H -#define _MAXSCALE_SERVICE_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -252,5 +250,3 @@ static inline uint64_t service_get_capabilities(const SERVICE *service) } MXS_END_DECLS - -#endif diff --git a/include/maxscale/session.h b/include/maxscale/session.h index 1be219640..855e922ca 100644 --- a/include/maxscale/session.h +++ b/include/maxscale/session.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_SESSION_H -#define _MAXSCALE_SESSION_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -355,5 +353,3 @@ static inline bool session_set_autocommit(SESSION* ses, bool autocommit) } MXS_END_DECLS - -#endif diff --git a/include/maxscale/spinlock.h b/include/maxscale/spinlock.h index 72650f268..f7c844b06 100644 --- a/include/maxscale/spinlock.h +++ b/include/maxscale/spinlock.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_SPINLOCK_H -#define _MAXSCALE_SPINLOCK_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -78,5 +76,3 @@ extern void spinlock_release(const SPINLOCK *lock); extern void spinlock_stats(const SPINLOCK *lock, void (*reporter)(void *, char *, int), void *hdl); MXS_END_DECLS - -#endif diff --git a/include/maxscale/statistics.h b/include/maxscale/statistics.h index ffb70e808..851eb1c46 100644 --- a/include/maxscale/statistics.h +++ b/include/maxscale/statistics.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_STATISTICS_HG -#define _MAXSCALE_STATISTICS_HG /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -73,5 +71,3 @@ ts_stats_set(ts_stats_t stats, int value, int thread_id) } MXS_END_DECLS - -#endif diff --git a/include/maxscale/thread.h b/include/maxscale/thread.h index d775cfec2..166685bc1 100644 --- a/include/maxscale/thread.h +++ b/include/maxscale/thread.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_THREAD_H -#define _MAXSCALE_THREAD_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -39,5 +37,3 @@ extern void thread_wait(THREAD thd); extern void thread_millisleep(int ms); MXS_END_DECLS - -#endif diff --git a/include/maxscale/users.h b/include/maxscale/users.h index 1d95016cd..cdd919de4 100644 --- a/include/maxscale/users.h +++ b/include/maxscale/users.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_USERS_H -#define _MAXSCALE_USERS_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -76,5 +74,3 @@ extern void usersPrint(USERS *); /**< Print data about the user extern void dcb_usersPrint(DCB *, USERS *); /**< Print data about the users loaded */ MXS_END_DECLS - -#endif diff --git a/include/maxscale/utils.h b/include/maxscale/utils.h index 67f699af2..70fc97ab9 100644 --- a/include/maxscale/utils.h +++ b/include/maxscale/utils.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_UTILS_H -#define _MAXSCALE_UTILS_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -79,5 +77,3 @@ bool mxs_mkdir_all(const char *path, int mask); long get_processor_count(); MXS_END_DECLS - -#endif From dc97de57c2d5f114b4d4170ec112911d2fa52472 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Wed, 26 Oct 2016 09:19:02 +0300 Subject: [PATCH 092/215] qc: Some minor cleanup - qc_types_to_string renamed to qc_typemask_to_string - qc_get_qtype_str removed, duplicate of qc_typemask_to_string --- include/maxscale/query_classifier.h | 3 +- query_classifier/test/compare.cc | 6 +-- server/core/query_classifier.c | 52 +------------------ .../routing/readwritesplit/rwsplit_mysql.c | 4 +- .../routing/schemarouter/schemarouter.c | 2 +- 5 files changed, 8 insertions(+), 59 deletions(-) diff --git a/include/maxscale/query_classifier.h b/include/maxscale/query_classifier.h index 2c26a5970..b2e01d533 100644 --- a/include/maxscale/query_classifier.h +++ b/include/maxscale/query_classifier.h @@ -96,13 +96,12 @@ bool qc_is_real_query(GWBUF* querybuf); char** qc_get_table_names(GWBUF* querybuf, int* tblsize, bool fullnames); char* qc_get_canonical(GWBUF* querybuf); bool qc_query_has_clause(GWBUF* buf); -char* qc_get_qtype_str(qc_query_type_t qtype); char* qc_get_affected_fields(GWBUF* buf); char** qc_get_database_names(GWBUF* querybuf, int* size); const char* qc_op_to_string(qc_query_op_t op); const char* qc_type_to_string(qc_query_type_t type); -char* qc_types_to_string(uint32_t types); +char* qc_typemask_to_string(uint32_t typemask); struct query_classifier { diff --git a/query_classifier/test/compare.cc b/query_classifier/test/compare.cc index 5b9d2cac7..5d403406e 100644 --- a/query_classifier/test/compare.cc +++ b/query_classifier/test/compare.cc @@ -354,7 +354,7 @@ bool compare_get_type(QUERY_CLASSIFIER* pClassifier1, GWBUF* pCopy1, if (rv1 == rv2) { - char* types = qc_types_to_string(rv1); + char* types = qc_typemask_to_string(rv1); ss << "Ok : " << types; free(types); success = true; @@ -385,8 +385,8 @@ bool compare_get_type(QUERY_CLASSIFIER* pClassifier1, GWBUF* pCopy1, rv2b &= ~(uint32_t)QUERY_TYPE_LOCAL_READ; } - char* types1 = qc_types_to_string(rv1); - char* types2 = qc_types_to_string(rv2); + char* types1 = qc_typemask_to_string(rv1); + char* types2 = qc_typemask_to_string(rv2); if (rv1b == rv2b) { diff --git a/server/core/query_classifier.c b/server/core/query_classifier.c index 071ddf0e9..36e0a9262 100644 --- a/server/core/query_classifier.c +++ b/server/core/query_classifier.c @@ -212,56 +212,6 @@ bool qc_query_has_clause(GWBUF* query) return classifier->qc_query_has_clause(query); } -/** - * Generate a string of query type value. - * Caller must free the memory of the resulting string. - * - * @param qtype Query type value, combination of values listed in - * query_classifier.h - * - * @return string representing the query type value - */ -char* qc_get_qtype_str(qc_query_type_t qtype) -{ - QC_TRACE(); - int t1 = (int) qtype; - int t2 = 1; - qc_query_type_t t = QUERY_TYPE_UNKNOWN; - char* qtype_str = NULL; - - /** - * Test values (bits) and clear matching bits from t1 one by one until - * t1 is completely cleared. - */ - while (t1 != 0) - { - if (t1 & t2) - { - t = (qc_query_type_t) t2; - - if (qtype_str == NULL) - { - qtype_str = MXS_STRDUP_A(STRQTYPE(t)); - } - else - { - size_t len = strlen(STRQTYPE(t)); - /** reallocate space for delimiter, new string and termination */ - qtype_str = (char *) MXS_REALLOC(qtype_str, strlen(qtype_str) + 1 + len + 1); - MXS_ABORT_IF_NULL(qtype_str); - snprintf(qtype_str + strlen(qtype_str), 1 + len + 1, "|%s", STRQTYPE(t)); - } - - /** Remove found value from t1 */ - t1 &= ~t2; - } - - t2 <<= 1; - } - - return qtype_str; -} - char* qc_get_affected_fields(GWBUF* query) { QC_TRACE(); @@ -595,7 +545,7 @@ static const int QUERY_TYPE_MAX_LEN = 29; // strlen("QUERY_TYPE_PREPARE_NAMED_ST * NOTE: The returned string is dynamically allocated * and *must* be freed by the caller. */ -char* qc_types_to_string(uint32_t types) +char* qc_typemask_to_string(uint32_t types) { int len = 0; diff --git a/server/modules/routing/readwritesplit/rwsplit_mysql.c b/server/modules/routing/readwritesplit/rwsplit_mysql.c index 1025f44a9..dbad88573 100644 --- a/server/modules/routing/readwritesplit/rwsplit_mysql.c +++ b/server/modules/routing/readwritesplit/rwsplit_mysql.c @@ -166,7 +166,7 @@ log_transaction_status(ROUTER_CLIENT_SES *rses, GWBUF *querybuf, qc_query_type_t MYSQL_GET_PACKET_LEN((unsigned char *)querybuf->start) - 1); char *data = (char *)&packet[5]; char *contentstr = strndup(data, MXS_MIN(len, RWSPLIT_TRACE_MSG_LEN)); - char *qtypestr = qc_get_qtype_str(qtype); + char *qtypestr = qc_typemask_to_string(qtype); MXS_INFO("> Autocommit: %s, trx is %s, cmd: %s, type: %s, stmt: %s%s %s", (rses->rses_autocommit_enabled ? "[enabled]" : "[disabled]"), (rses->rses_transaction_active ? "[open]" : "[not open]"), @@ -221,7 +221,7 @@ handle_target_is_all(route_target_t route_target, /* NOTE: modutil_get_query is MySQL specific */ char *query_str = modutil_get_query(querybuf); - char *qtype_str = qc_get_qtype_str(qtype); + char *qtype_str = qc_typemask_to_string(qtype); /* NOTE: packet_type is MySQL specific */ MXS_ERROR("Can't route %s:%s:\"%s\". SELECT with session data " diff --git a/server/modules/routing/schemarouter/schemarouter.c b/server/modules/routing/schemarouter/schemarouter.c index 5bc82eb06..7ef6d8681 100644 --- a/server/modules/routing/schemarouter/schemarouter.c +++ b/server/modules/routing/schemarouter/schemarouter.c @@ -2007,7 +2007,7 @@ static int routeQuery(ROUTER* instance, MYSQL_GET_PACKET_LEN((unsigned char *)querybuf->start) - 1); char* data = (char*)&packet[5]; char* contentstr = strndup(data, len); - char* qtypestr = qc_get_qtype_str(qtype); + char* qtypestr = qc_typemask_to_string(qtype); MXS_INFO("> Cmd: %s, type: %s, " "stmt: %s%s %s", From 19e017e4995a78ed6d0bbbe487f485f549e133bf Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Wed, 26 Oct 2016 11:31:46 +0300 Subject: [PATCH 093/215] QC: Documentation cleaned up and moved to header --- include/maxscale/query_classifier.h | 349 +++++++++++++++--- .../qc_mysqlembedded/qc_mysqlembedded.cc | 45 --- server/core/query_classifier.c | 48 --- 3 files changed, 303 insertions(+), 139 deletions(-) diff --git a/include/maxscale/query_classifier.h b/include/maxscale/query_classifier.h index b2e01d533..94c9b8d19 100644 --- a/include/maxscale/query_classifier.h +++ b/include/maxscale/query_classifier.h @@ -17,7 +17,15 @@ MXS_BEGIN_DECLS -typedef enum +#define QUERY_CLASSIFIER_VERSION {1, 0, 0} + +/** + * qc_query_type_t defines bits that provide information about a + * particular statement. + * + * Note that more than one bit may be set for a single statement. + */ +typedef enum qc_query_type { QUERY_TYPE_UNKNOWN = 0x000000, /*< Initial value, can't be tested bitwisely */ QUERY_TYPE_LOCAL_READ = 0x000001, /*< Read non-database data, execute in MaxScale:any */ @@ -47,7 +55,10 @@ typedef enum QUERY_TYPE_SHOW_TABLES = 0x400000 /*< Show list of tables */ } qc_query_type_t; -typedef enum +/** + * qc_query_op_t defines the operations a particular statement can perform. + */ +typedef enum qc_query_op { QUERY_OP_UNDEFINED = 0, QUERY_OP_SELECT = (1 << 0), @@ -64,6 +75,9 @@ typedef enum QUERY_OP_REVOKE = (1 << 11) } qc_query_op_t; +/** + * qc_parse_result_t defines the possible outcomes when a statement is parsed. + */ typedef enum qc_parse_result { QC_QUERY_INVALID = 0, /*< The query was not recognized or could not be parsed. */ @@ -72,38 +86,15 @@ typedef enum qc_parse_result QC_QUERY_PARSED = 3 /*< The query was fully parsed; completely classified. */ } qc_parse_result_t; -#define QUERY_IS_TYPE(mask,type) ((mask & type) == type) -bool qc_init(const char* plugin_name, const char* plugin_args); -void qc_end(void); - -typedef struct query_classifier QUERY_CLASSIFIER; - -QUERY_CLASSIFIER* qc_load(const char* plugin_name); -void qc_unload(QUERY_CLASSIFIER* classifier); - -bool qc_thread_init(void); -void qc_thread_end(void); - -qc_parse_result_t qc_parse(GWBUF* querybuf); - -uint32_t qc_get_type(GWBUF* querybuf); -qc_query_op_t qc_get_operation(GWBUF* querybuf); - -char* qc_get_created_table_name(GWBUF* querybuf); -bool qc_is_drop_table_query(GWBUF* querybuf); -bool qc_is_real_query(GWBUF* querybuf); -char** qc_get_table_names(GWBUF* querybuf, int* tblsize, bool fullnames); -char* qc_get_canonical(GWBUF* querybuf); -bool qc_query_has_clause(GWBUF* buf); -char* qc_get_affected_fields(GWBUF* buf); -char** qc_get_database_names(GWBUF* querybuf, int* size); - -const char* qc_op_to_string(qc_query_op_t op); -const char* qc_type_to_string(qc_query_type_t type); -char* qc_typemask_to_string(uint32_t typemask); - -struct query_classifier +/** + * QUERY_CLASSIFIER defines the object a query classifier plugin must + * implement and return. + * + * To a user of the query classifier functionality, it can in general + * be ignored. + */ +typedef struct query_classifier { bool (*qc_init)(const char* args); void (*qc_end)(void); @@ -111,21 +102,287 @@ struct query_classifier bool (*qc_thread_init)(void); void (*qc_thread_end)(void); - qc_parse_result_t (*qc_parse)(GWBUF* querybuf); + qc_parse_result_t (*qc_parse)(GWBUF* stmt); - uint32_t (*qc_get_type)(GWBUF* querybuf); - qc_query_op_t (*qc_get_operation)(GWBUF* querybuf); + uint32_t (*qc_get_type)(GWBUF* stmt); + qc_query_op_t (*qc_get_operation)(GWBUF* stmt); - char* (*qc_get_created_table_name)(GWBUF* querybuf); - bool (*qc_is_drop_table_query)(GWBUF* querybuf); - bool (*qc_is_real_query)(GWBUF* querybuf); - char** (*qc_get_table_names)(GWBUF* querybuf, int* tblsize, bool fullnames); - char* (*qc_get_canonical)(GWBUF* querybuf); - bool (*qc_query_has_clause)(GWBUF* buf); - char* (*qc_get_affected_fields)(GWBUF* buf); - char** (*qc_get_database_names)(GWBUF* querybuf, int* size); -}; + char* (*qc_get_created_table_name)(GWBUF* stmt); + bool (*qc_is_drop_table_query)(GWBUF* stmt); + bool (*qc_is_real_query)(GWBUF* stmt); + char** (*qc_get_table_names)(GWBUF* stmt, int* tblsize, bool fullnames); + char* (*qc_get_canonical)(GWBUF* stmt); + bool (*qc_query_has_clause)(GWBUF* stmt); + char* (*qc_get_affected_fields)(GWBUF* stmt); + char** (*qc_get_database_names)(GWBUF* stmt, int* size); +} QUERY_CLASSIFIER; -#define QUERY_CLASSIFIER_VERSION {1, 0, 0} +/** + * Loads and initializes the default query classifier. + * + * This must be called once during the execution of a process. The query + * classifier functions can only be used if this function returns true. + * MaxScale calls this function, so plugins should not do that. + * + * @param plugin_name The name of the plugin from which the query classifier + * should be loaded. + * @param plugin_args The arguments to be provided to the query classifier. + * + * @return True if the query classifier could be loaded and initialized, + * false otherwise. + * + * @see qc_end qc_thread_init + */ +bool qc_init(const char* plugin_name, const char* plugin_args); + +/** + * Finalizes and unloads the query classifier. + * + * A successful call of qc_init() should before program exit be followed + * by a call to this function. MaxScale calls this function, so plugins + * should not do that. + * + * @see qc_init qc_thread_end + */ +void qc_end(void); + +/** + * Loads a particular query classifier. + * + * In general there is no need to use this function, but rely upon qc_init(). + * However, if there is a need to use multiple query classifiers concurrently + * then this function provides the means for that. Note that after a query + * classifier has been loaded, it must explicitly be initialized before it + * can be used. + * + * @param plugin_name The name of the plugin from which the query classifier + * should be loaded. + * + * @return A QUERY_CLASSIFIER object if successful, NULL otherwise. + * + * @see qc_unload + */ +QUERY_CLASSIFIER* qc_load(const char* plugin_name); + +/** + * Unloads an explicitly loaded query classifier. + * + * @see qc_load + */ +void qc_unload(QUERY_CLASSIFIER* classifier); + +/** + * Performs thread initialization needed by the query classifier. + * Should be called in every thread, except the one where qc_init() + * was called. MaxScale calls this function, so plugins should not + * do that. + * + * @return True if the initialization succeeded, false otherwise. + * + * @see qc_thread_end + */ +bool qc_thread_init(void); + +/** + * Performs thread finalization needed by the query classifier. + * A successful call to qc_thread_init() should at some point be followed + * by a call to this function. MaxScale calls this function, so plugins + * should not do that. + * + * @see qc_thread_init + */ +void qc_thread_end(void); + +/** + * Parses the statement in the provided buffer and returns a value specifying + * to what extent the statement could be parsed. + * + * There is no need to call this function explicitly before calling any of + * the other functions; e.g. qc_get_type(). When some particular property of + * a statement is asked for, the statement will be parsed if it has not been + * parsed yet. Also, if the statement in the provided buffer has been parsed + * already then this function will only return the result of that parsing; + * the statement will not be parsed again. + * + * @param stmt A buffer containing an COM_QUERY packet. + * + * @return To what extent the statement could be parsed. + */ +qc_parse_result_t qc_parse(GWBUF* stmt); + +/** + * Returns the fields the statement affects, as a string of names separated + * by spaces. Note that the fields do not contain any table information. + * + * @param stmt A buffer containing a COM_QUERY packet. + * + * @return A string containing the fields or NULL if a memory allocation + * failure occurs. The string must be freed by the caller. + */ +char* qc_get_affected_fields(GWBUF* stmt); + +/** + * Returns the statement, with literals replaced with question marks. + * + * @param stmt A buffer containing a COM_QUERY packet. + * + * @return A statement in its canonical form, or NULL if a memory + * allocation fails. The string must be freed by the caller. + */ +char* qc_get_canonical(GWBUF* stmt); + +/** + * Returns the name of the created table. + * + * @param stmt A buffer containing a COM_QUERY packet. + * + * @return The name of the created table or NULL if the statement + * does not create a table or a memory allocation failed. + * The string must be freed by the caller. + */ +char* qc_get_created_table_name(GWBUF* stmt); + +/** + * Returns the databases accessed by the statement. Note that a + * possible default database is not returned. + * + * @param stmt A buffer containing a COM_QUERY packet. + * @param size Pointer to integer where the number of databases + * is stored. + * + * @return Array of strings or NULL if a memory allocation fails. + * + * @note The returned array and the strings pointed to @b must be freed + * by the caller. + */ +char** qc_get_database_names(GWBUF* stmt, int* size); + +/** + * Returns the operation of the statement. + * + * @param stmt A buffer containing a COM_QUERY packet. + * + * @return The operation of the statement. + */ +qc_query_op_t qc_get_operation(GWBUF* stmt); + +/** + * Returns the tables accessed by the statement. + * + * @param stmt A buffer containing a COM_QUERY packet. + * @param tblsize Pointer to integer where the number of tables is stored. + * @param fullnames If true, a table names will include the database name + * as well (if explicitly referred to in the statement). + * + * @return Array of strings or NULL if a memory allocation fails. + * + * @note The returned array and the strings pointed to @b must be freed + * by the caller. + */ +char** qc_get_table_names(GWBUF* stmt, int* size, bool fullnames); + + +/** + * Returns a bitmask specifying the type(s) of the statement. The result + * should be tested against specific qc_query_type_t values* using the + * bitwise & operator, never using the == operator. + * + * @param stmt A buffer containing a COM_QUERY packet. + * + * @return A bitmask with the type(s) the query. + * + * @see qc_query_is_type + */ +uint32_t qc_get_type(GWBUF* stmt); + +/** + * Returns whether the statement is a DROP TABLE statement. + * + * @param stmt A buffer containing a COM_QUERY packet. + * + * @return True if the statement is a DROP TABLE statement, false otherwise. + * + * @todo This function is far too specific. + */ +bool qc_is_drop_table_query(GWBUF* stmt); + +/** + * Returns whether the statement is a "real" statement. Statements that affect + * the underlying database are considered real statements, while statements that + * target specific rows or variable data are regarded as real statement. That is, + * a real statement is SELECT, UPDATE, INSERT, DELETE or a variation thereof. + * + * @param stmt A buffer containing a COM_QUERY. + * + * @return True if the statement is a real query, false otherwise. + * + * @todo Consider whether the function name should be changed or the function + * removed altogether. + */ +bool qc_is_real_query(GWBUF* stmt); + +/** + * Returns the string representation of a query operation. + * + * @param op A query operation. + * + * @return The corresponding string. + * + * @note The returned string is statically allocated and must *not* be freed. + */ +const char* qc_op_to_string(qc_query_op_t op); + +/** + * Returns whether the typemask contains a particular type. + * + * @param typemask A bitmask of query types. + * @param type A particular qc_query_type_t value. + * + * @return True, if the type is in the mask. + */ +static inline bool qc_query_is_type(uint32_t typemask, qc_query_type_t type) +{ + return (typemask & type) == type; +} + +/** + * Returns whether the statement has a WHERE or a USING clause. + * + * @param stmt A buffer containing a COM_QUERY. + * + * @return True, if the statement has a WHERE or USING clause, false + * otherwise. + */ +bool qc_query_has_clause(GWBUF* stmt); + +/** + * Returns the string representation of a query type. + * + * @param type A query type (not a bitmask of several). + * + * @return The corresponding string. + * + * @note The returned string is statically allocated and must @b not be freed. + */ +const char* qc_type_to_string(qc_query_type_t type); + +/** + * Returns a string representation of a type bitmask. + * + * @param typemask A bit mask of query types. + * + * @return The corresponding string or NULL if the allocation fails. + * + * @note The returned string is dynamically allocated and @b must be freed. + */ +char* qc_typemask_to_string(uint32_t typemask); + +/** + * @deprecated + * Synonym for qc_query_is_type(). + * + * @see qc_query_is_type + */ +#define QUERY_IS_TYPE(typemask, type) qc_query_is_type(typemask, type) MXS_END_DECLS diff --git a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc index 285692879..e8391d9b1 100644 --- a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc +++ b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc @@ -1083,15 +1083,6 @@ static void* skygw_get_affected_tables(void* lexptr) return (void*) lex->current_select->table_list.first; } -/** - * Reads the parsetree and lists all the affected tables and views in the query. - * In the case of an error, the size of the table is set to zero and no memory - * is allocated. The caller must free the allocated memory. - * - * @param querybuf GWBUF where the table names are extracted from - * @param tblsize Pointer where the number of tables is written - * @return Array of null-terminated strings with the table names - */ char** qc_get_table_names(GWBUF* querybuf, int* tblsize, bool fullnames) { LEX* lex; @@ -1193,11 +1184,6 @@ retblock: return tables; } -/** - * Extract, allocate memory and copy the name of the created table. - * @param querybuf Buffer to use. - * @return A pointer to the name if a table was created, otherwise NULL - */ char* qc_get_created_table_name(GWBUF* querybuf) { if (querybuf == NULL) @@ -1226,16 +1212,6 @@ char* qc_get_created_table_name(GWBUF* querybuf) return table_name; } -/** - * Checks whether the query is a "real" query ie. SELECT,UPDATE,INSERT,DELETE or - * any variation of these. Queries that affect the underlying database are not - * considered as real queries and the queries that target specific row or - * variable data are regarded as the real queries. - * - * @param GWBUF to analyze - * - * @return true if the query is a real query, otherwise false - */ bool qc_is_real_query(GWBUF* querybuf) { bool succp; @@ -1291,11 +1267,6 @@ retblock: return succp; } -/** - * Checks whether the buffer contains a DROP TABLE... query. - * @param querybuf Buffer to inspect - * @return true if it contains the query otherwise false - */ bool qc_is_drop_table_query(GWBUF* querybuf) { bool answer = false; @@ -1522,11 +1493,6 @@ static void collect_affected_fields(collect_source_t source, } } -/** - * Returns all the fields that the query affects. - * @param buf Buffer to parse - * @return Pointer to newly allocated string or NULL if nothing was found - */ char* qc_get_affected_fields(GWBUF* buf) { LEX* lex; @@ -1773,17 +1739,6 @@ static void parsing_info_set_plain_str(void* ptr, char* str) pi->pi_query_plain_str = str; } -/** - * Returns an array of strings of databases that this query uses. - * If the database isn't defined in the query, it is assumed that this query - * only targets the current database. - * The value of @p size is set to the number of allocated strings. The caller is - * responsible for freeing all the allocated memory. - * @param querybuf GWBUF containing the query - * @param size Size of the resulting array - * @return A new array of strings containing the database names or NULL if no - * databases were found. - */ char** qc_get_database_names(GWBUF* querybuf, int* size) { LEX* lex; diff --git a/server/core/query_classifier.c b/server/core/query_classifier.c index 36e0a9262..40c5ef4e9 100644 --- a/server/core/query_classifier.c +++ b/server/core/query_classifier.c @@ -101,20 +101,6 @@ void qc_thread_end(void) return classifier->qc_thread_end(); } -/** - * Parses the query in the provided buffer and returns a value specifying - * to what extent the query could be parsed. - * - * There is no need to call this function explicitly before calling any of - * the other functions; e.g. qc_get_type. When some particular property of - * a query is asked for, the query will be parsed if it has not been parsed - * yet. Also, if the query in the provided buffer has been parsed already - * then this function will only return the result of that parsing; the query - * will not be parsed again. - * - * @param query A GWBUF containing an SQL statement. - * @result To what extent the query could be parsed. - */ qc_parse_result_t qc_parse(GWBUF* query) { QC_TRACE(); @@ -123,15 +109,6 @@ qc_parse_result_t qc_parse(GWBUF* query) return classifier->qc_parse(query); } -/** - * Returns a bitmask specifying the type(s) of the query. - * The result should be tested against specific qc_query_type_t values - * using the bitwise & operator, never using the == operator. - * - * @param query A buffer containing a query. - * - * @return A bitmask of type bits. - */ uint32_t qc_get_type(GWBUF* query) { QC_TRACE(); @@ -228,14 +205,6 @@ char** qc_get_database_names(GWBUF* query, int* sizep) return classifier->qc_get_database_names(query, sizep); } -/** - * Returns the string representation of a query operation. - * - * @param op An operation. - * @return The corresponding string. - * NOTE: The returned string is statically allocated - * and must *not* be freed. - */ const char* qc_op_to_string(qc_query_op_t op) { switch (op) @@ -489,15 +458,6 @@ struct type_name_info type_to_type_name_info(qc_query_type_t type) } - -/** - * Returns the string representation of a query type. - * - * @param type A specific type (not a bitmask of several). - * @return The corresponding string. - * NOTE: The returned string is statically allocated - * and must *not* be freed. - */ const char* qc_type_to_string(qc_query_type_t type) { return type_to_type_name_info(type).name; @@ -537,14 +497,6 @@ static const qc_query_type_t QUERY_TYPES[] = static const int N_QUERY_TYPES = sizeof(QUERY_TYPES) / sizeof(QUERY_TYPES[0]); static const int QUERY_TYPE_MAX_LEN = 29; // strlen("QUERY_TYPE_PREPARE_NAMED_STMT"); -/** - * Returns the string representation of a bitmask of query types. - * - * @param type Bitmask of several qc_query_type_t values. - * @return The corresponding string. - * NOTE: The returned string is dynamically allocated - * and *must* be freed by the caller. - */ char* qc_typemask_to_string(uint32_t types) { int len = 0; From 7499c1e1bfb78251c5f1f728809728d53ece1c53 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Wed, 26 Oct 2016 12:48:48 +0300 Subject: [PATCH 094/215] qc: Add minimal support for qc_get_prepare_name If the query is a PREPARE or EXECUTE statement, then qc_get_prepare_name will return the name of the prepared statement. --- include/maxscale/query_classifier.h | 21 ++++++++++++- .../qc_mysqlembedded/qc_mysqlembedded.cc | 16 ++++++++++ query_classifier/qc_sqlite/qc_sqlite.c | 29 +++++++++++++++++ query_classifier/test/compare.cc | 31 +++++++++++++++++++ server/core/query_classifier.c | 8 +++++ 5 files changed, 104 insertions(+), 1 deletion(-) diff --git a/include/maxscale/query_classifier.h b/include/maxscale/query_classifier.h index 94c9b8d19..ccef8d2fc 100644 --- a/include/maxscale/query_classifier.h +++ b/include/maxscale/query_classifier.h @@ -17,7 +17,7 @@ MXS_BEGIN_DECLS -#define QUERY_CLASSIFIER_VERSION {1, 0, 0} +#define QUERY_CLASSIFIER_VERSION {1, 1, 0} /** * qc_query_type_t defines bits that provide information about a @@ -115,6 +115,7 @@ typedef struct query_classifier bool (*qc_query_has_clause)(GWBUF* stmt); char* (*qc_get_affected_fields)(GWBUF* stmt); char** (*qc_get_database_names)(GWBUF* stmt, int* size); + char* (*qc_get_prepare_name)(GWBUF* stmt); } QUERY_CLASSIFIER; /** @@ -266,6 +267,24 @@ char** qc_get_database_names(GWBUF* stmt, int* size); */ qc_query_op_t qc_get_operation(GWBUF* stmt); +/** + * Returns the name of the prepared statement, if the statement + * is a PREPARE or EXECUTE statement. + * + * @param stmt A buffer containing a COM_QUERY packet. + * + * @return The name of the prepared statement, if the statement + * is a PREPARE or EXECUTE statement; otherwise NULL. + * + * @note The returned string @b must be freed by the caller. + * + * @note Even though a COM_STMT_PREPARE can be given to the query + * classifier for parsing, this function will in that case + * return NULL since the id of the statement is provided by + * the server. + */ +char* qc_get_prepare_name(GWBUF* stmt); + /** * Returns the tables accessed by the statement. * diff --git a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc index e8391d9b1..3adb8ff16 100644 --- a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc +++ b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc @@ -1904,6 +1904,21 @@ qc_query_op_t qc_get_operation(GWBUF* querybuf) return operation; } +char* qc_get_prepare_name(GWBUF* stmt) +{ + char* name = NULL; + + if (stmt) + { + if (ensure_query_is_parsed(stmt)) + { + MXS_WARNING("qc_get_prepare_name not implemented yet."); + } + } + + return name; +} + namespace { @@ -2048,6 +2063,7 @@ static QUERY_CLASSIFIER qc = qc_query_has_clause, qc_get_affected_fields, qc_get_database_names, + qc_get_prepare_name, }; /* @see function load_module in load_utils.c for explanation of the following diff --git a/query_classifier/qc_sqlite/qc_sqlite.c b/query_classifier/qc_sqlite/qc_sqlite.c index 0a41ea240..29d3cd00d 100644 --- a/query_classifier/qc_sqlite/qc_sqlite.c +++ b/query_classifier/qc_sqlite/qc_sqlite.c @@ -2834,6 +2834,34 @@ static char** qc_sqlite_get_database_names(GWBUF* query, int* sizep) return database_names; } +static char* qc_sqlite_get_prepare_name(GWBUF* query) +{ + QC_TRACE(); + ss_dassert(this_unit.initialized); + ss_dassert(this_thread.initialized); + + char* name = NULL; + QC_SQLITE_INFO* info = get_query_info(query); + + if (info) + { + if (qc_info_is_valid(info->status)) + { + MXS_WARNING("qc_get_prepare_name not implemented yet."); + } + else if (MXS_LOG_PRIORITY_IS_ENABLED(LOG_INFO)) + { + log_invalid_data(query, "cannot report the name of a prepared statement"); + } + } + else + { + MXS_ERROR("The query could not be parsed. Response not valid."); + } + + return name; +} + /** * EXPORTS */ @@ -2857,6 +2885,7 @@ static QUERY_CLASSIFIER qc = qc_sqlite_query_has_clause, qc_sqlite_get_affected_fields, qc_sqlite_get_database_names, + qc_sqlite_get_prepare_name, }; diff --git a/query_classifier/test/compare.cc b/query_classifier/test/compare.cc index 5d403406e..066f086ba 100644 --- a/query_classifier/test/compare.cc +++ b/query_classifier/test/compare.cc @@ -814,6 +814,36 @@ bool compare_get_database_names(QUERY_CLASSIFIER* pClassifier1, GWBUF* pCopy1, return success; } +bool compare_get_prepare_name(QUERY_CLASSIFIER* pClassifier1, GWBUF* pCopy1, + QUERY_CLASSIFIER* pClassifier2, GWBUF* pCopy2) +{ + bool success = false; + const char HEADING[] = "qc_get_prepare_name : "; + + char* rv1 = pClassifier1->qc_get_prepare_name(pCopy1); + char* rv2 = pClassifier2->qc_get_prepare_name(pCopy2); + + stringstream ss; + ss << HEADING; + + if ((!rv1 && !rv2) || (rv1 && rv2 && (strcmp(rv1, rv2) == 0))) + { + ss << "Ok : " << (rv1 ? rv1 : "NULL"); + success = true; + } + else + { + ss << "ERR: " << (rv1 ? rv1 : "NULL") << " != " << (rv2 ? rv2 : "NULL"); + } + + report(success, ss.str()); + + free(rv1); + free(rv2); + + return success; +} + bool compare(QUERY_CLASSIFIER* pClassifier1, QUERY_CLASSIFIER* pClassifier2, const string& s) { GWBUF* pCopy1 = create_gwbuf(s); @@ -832,6 +862,7 @@ bool compare(QUERY_CLASSIFIER* pClassifier1, QUERY_CLASSIFIER* pClassifier2, con errors += !compare_query_has_clause(pClassifier1, pCopy1, pClassifier2, pCopy2); errors += !compare_get_affected_fields(pClassifier1, pCopy1, pClassifier2, pCopy2); errors += !compare_get_database_names(pClassifier1, pCopy1, pClassifier2, pCopy2); + errors += !compare_get_prepare_name(pClassifier1, pCopy1, pClassifier2, pCopy2); gwbuf_free(pCopy1); gwbuf_free(pCopy2); diff --git a/server/core/query_classifier.c b/server/core/query_classifier.c index 40c5ef4e9..5800f38e1 100644 --- a/server/core/query_classifier.c +++ b/server/core/query_classifier.c @@ -205,6 +205,14 @@ char** qc_get_database_names(GWBUF* query, int* sizep) return classifier->qc_get_database_names(query, sizep); } +char* qc_get_prepare_name(GWBUF* query) +{ + QC_TRACE(); + ss_dassert(classifier); + + return classifier->qc_get_prepare_name(query); +} + const char* qc_op_to_string(qc_query_op_t op) { switch (op) From 23fdf1776c5d6011799589252e2f75a5e12a8182 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Wed, 26 Oct 2016 13:32:32 +0300 Subject: [PATCH 095/215] qc: Pick out the name in a PREPARE statement --- .../qc_mysqlembedded/qc_mysqlembedded.cc | 12 +++++++++++- query_classifier/qc_sqlite/qc_sqlite.c | 13 ++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc index 3adb8ff16..d8b45ca97 100644 --- a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc +++ b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc @@ -1912,7 +1912,17 @@ char* qc_get_prepare_name(GWBUF* stmt) { if (ensure_query_is_parsed(stmt)) { - MXS_WARNING("qc_get_prepare_name not implemented yet."); + LEX* lex = get_lex(stmt); + + if (lex->sql_command == SQLCOM_PREPARE) + { + name = (char*)malloc(lex->prepared_stmt_name.length + 1); + if (name) + { + memcpy(name, lex->prepared_stmt_name.str, lex->prepared_stmt_name.length); + name[lex->prepared_stmt_name.length] = 0; + } + } } } diff --git a/query_classifier/qc_sqlite/qc_sqlite.c b/query_classifier/qc_sqlite/qc_sqlite.c index 29d3cd00d..3789fce2e 100644 --- a/query_classifier/qc_sqlite/qc_sqlite.c +++ b/query_classifier/qc_sqlite/qc_sqlite.c @@ -78,6 +78,7 @@ typedef struct qc_sqlite_info size_t database_names_capacity; // The capacity of database_names. int keyword_1; // The first encountered keyword. int keyword_2; // The second encountered keyword. + char* prepare_name; // The name of a prepared statement. } QC_SQLITE_INFO; typedef enum qc_log_level @@ -1961,6 +1962,13 @@ void maxscalePrepare(Parse* pParse, Token* pName, Token* pStmt) info->status = QC_QUERY_PARSED; info->types = QUERY_TYPE_PREPARE_NAMED_STMT; info->is_real_query = true; + + info->prepare_name = MXS_MALLOC(pName->n + 1); + if (info->prepare_name) + { + memcpy(info->prepare_name, pName->z, pName->n); + info->prepare_name[pName->n] = 0; + } } void maxscalePrivileges(Parse* pParse, int kind) @@ -2847,7 +2855,10 @@ static char* qc_sqlite_get_prepare_name(GWBUF* query) { if (qc_info_is_valid(info->status)) { - MXS_WARNING("qc_get_prepare_name not implemented yet."); + if (info->prepare_name) + { + name = MXS_STRDUP(info->prepare_name); + } } else if (MXS_LOG_PRIORITY_IS_ENABLED(LOG_INFO)) { From 48af3d21b6e1f622e8840b49bfd1a95219c4b959 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Wed, 26 Oct 2016 15:58:06 +0200 Subject: [PATCH 096/215] max rows filter first implementation max rows filter first implementation Resultsets with more than max_resultset_rows will be skipped and empty result set is returned to client. Not yet tested with multi_statements [MaxRows] type=filter module=maxrows max_resultset_rows=10 debug=15 --- server/modules/filter/CMakeLists.txt | 1 + server/modules/filter/maxrows/CMakeLists.txt | 9 + server/modules/filter/maxrows/maxrows.c | 937 +++++++++++++++++++ server/modules/filter/maxrows/maxrows.h | 43 + 4 files changed, 990 insertions(+) create mode 100644 server/modules/filter/maxrows/CMakeLists.txt create mode 100644 server/modules/filter/maxrows/maxrows.c create mode 100644 server/modules/filter/maxrows/maxrows.h diff --git a/server/modules/filter/CMakeLists.txt b/server/modules/filter/CMakeLists.txt index 39d7775eb..bd8b7a1b9 100644 --- a/server/modules/filter/CMakeLists.txt +++ b/server/modules/filter/CMakeLists.txt @@ -1,4 +1,5 @@ add_subdirectory(cache) +add_subdirectory(maxrows) add_subdirectory(ccrfilter) add_subdirectory(dbfwfilter) add_subdirectory(gatekeeper) diff --git a/server/modules/filter/maxrows/CMakeLists.txt b/server/modules/filter/maxrows/CMakeLists.txt new file mode 100644 index 000000000..8ad47a642 --- /dev/null +++ b/server/modules/filter/maxrows/CMakeLists.txt @@ -0,0 +1,9 @@ +add_library(maxrows SHARED maxrows.c) +target_link_libraries(maxrows maxscale-common) +set_target_properties(maxrows PROPERTIES VERSION "1.0.0") +set_target_properties(maxrows PROPERTIES LINK_FLAGS -Wl,-z,defs) +install_module(maxrows experimental) + +if(BUILD_TESTS) + add_subdirectory(test) +endif() diff --git a/server/modules/filter/maxrows/maxrows.c b/server/modules/filter/maxrows/maxrows.c new file mode 100644 index 000000000..e1415f1ea --- /dev/null +++ b/server/modules/filter/maxrows/maxrows.c @@ -0,0 +1,937 @@ +/* + * Copyright (c) 2016 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl. + * + * Change Date: 2019-07-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ + +/** + * @file maxrows.c - Result set limit Filter + * @verbatim + * + * + * The filter returns a void result set if the number of rows in the result set + * from backend exceeds the max_rows parameter. + * + * Date Who Description + * 26/10/2016 Massimiliano Pinto Initial implementation + * + * @endverbatim + */ + +#define MXS_MODULE_NAME "cache" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "maxrows.h" + +static char VERSION_STRING[] = "V1.0.0"; + +static FILTER *createInstance(const char *name, char **options, FILTER_PARAMETER **); +static void *newSession(FILTER *instance, SESSION *session); +static void closeSession(FILTER *instance, void *sdata); +static void freeSession(FILTER *instance, void *sdata); +static void setDownstream(FILTER *instance, void *sdata, DOWNSTREAM *downstream); +static void setUpstream(FILTER *instance, void *sdata, UPSTREAM *upstream); +static int routeQuery(FILTER *instance, void *sdata, GWBUF *queue); +static int clientReply(FILTER *instance, void *sdata, GWBUF *queue); +static void diagnostics(FILTER *instance, void *sdata, DCB *dcb); +static uint64_t getCapabilities(void); + +#define C_DEBUG(format, ...) MXS_LOG_MESSAGE(LOG_NOTICE, format, ##__VA_ARGS__) + +/* Global symbols of the Module */ + +MODULE_INFO info = +{ + MODULE_API_FILTER, + MODULE_IN_DEVELOPMENT, + FILTER_VERSION, + "A filter that is capable of limiting the resultset number of rows." +}; + +char *version() +{ + return VERSION_STRING; +} + +/** + * The module initialization functions, called when the module has + * been loaded. + */ +void ModuleInit() +{ +} + +/** + * The module entry point function, called when the module is loaded. + * + * @return The module object. + */ +FILTER_OBJECT *GetModuleObject() +{ + static FILTER_OBJECT object = + { + createInstance, + newSession, + closeSession, + freeSession, + setDownstream, + setUpstream, + routeQuery, + clientReply, + diagnostics, + getCapabilities + }; + + return &object; +}; + +/* Implementation */ + +typedef struct maxrows_config +{ + uint32_t max_resultset_rows; + uint32_t max_resultset_size; + uint32_t debug; +} MAXROWS_CONFIG; + +static const MAXROWS_CONFIG DEFAULT_CONFIG = +{ + MAXROWS_DEFAULT_MAX_RESULTSET_ROWS, + MAXROWS_DEFAULT_MAX_RESULTSET_SIZE, + MAXROWS_DEFAULT_DEBUG +}; + +typedef struct maxrows_instance +{ + const char *name; + MAXROWS_CONFIG config; +} MAXROWS_INSTANCE; + +typedef enum maxrows_session_state +{ + MAXROWS_EXPECTING_RESPONSE, // A select has been sent, and we are waiting for the response. + MAXROWS_EXPECTING_FIELDS, // A select has been sent, and we want more fields. + MAXROWS_EXPECTING_ROWS, // A select has been sent, and we want more rows. + MAXROWS_EXPECTING_NOTHING, // We are not expecting anything from the server. + MAXROWS_EXPECTING_USE_RESPONSE, // A "USE DB" was issued. + MAXROWS_IGNORING_RESPONSE, // We are not interested in the data received from the server. +} maxrows_session_state_t; + +typedef struct maxrows_response_state +{ + GWBUF* data; /**< Response data, possibly incomplete. */ + size_t n_totalfields; /**< The number of fields a resultset contains. */ + size_t n_fields; /**< How many fields we have received, <= n_totalfields. */ + size_t n_rows; /**< How many rows we have received. */ + size_t offset; /**< Where we are in the response buffer. */ +} MAXROWS_RESPONSE_STATE; + +static void maxrows_response_state_reset(MAXROWS_RESPONSE_STATE *state); + +typedef struct cache_session_data +{ + MAXROWS_INSTANCE *instance; /**< The cache instance the session is associated with. */ + DOWNSTREAM down; /**< The previous filter or equivalent. */ + UPSTREAM up; /**< The next filter or equivalent. */ + MAXROWS_RESPONSE_STATE res; /**< The response state. */ + SESSION *session; /**< The session this data is associated with. */ + char *default_db; /**< The default database. */ + char *use_db; /**< Pending default database. Needs server response. */ + maxrows_session_state_t state; +} MAXROWS_SESSION_DATA; + +static MAXROWS_SESSION_DATA *maxrows_session_data_create(MAXROWS_INSTANCE *instance, SESSION *session); +static void maxrows_session_data_free(MAXROWS_SESSION_DATA *data); + +static int handle_expecting_fields(MAXROWS_SESSION_DATA *csdata); +static int handle_expecting_nothing(MAXROWS_SESSION_DATA *csdata); +static int handle_expecting_response(MAXROWS_SESSION_DATA *csdata); +static int handle_expecting_rows(MAXROWS_SESSION_DATA *csdata); +static int handle_expecting_use_response(MAXROWS_SESSION_DATA *csdata); +static int handle_ignoring_response(MAXROWS_SESSION_DATA *csdata); +static bool process_params(char **options, FILTER_PARAMETER **params, MAXROWS_CONFIG* config); + +static int send_upstream(MAXROWS_SESSION_DATA *csdata); +static int send_ok_upstream(MAXROWS_SESSION_DATA *csdata); + +/* API BEGIN */ + +/** + * Create an instance of the cache filter for a particular service + * within MaxScale. + * + * @param name The name of the instance (as defined in the config file). + * @param options The options for this filter + * @param params The array of name/value pair parameters for the filter + * + * @return The instance data for this new instance + */ +static FILTER *createInstance(const char *name, char **options, FILTER_PARAMETER **params) +{ + MAXROWS_INSTANCE *cinstance = NULL; + MAXROWS_CONFIG config = DEFAULT_CONFIG; + + if (process_params(options, params, &config)) + { + cinstance = MXS_CALLOC(1, sizeof(MAXROWS_INSTANCE)); + if (cinstance) + { + cinstance->name = name; + cinstance->config = config; + } + } + + return (FILTER*)cinstance; +} + +/** + * Associate a new session with this instance of the filter. + * + * @param instance The cache instance data + * @param session The session itself + * + * @return Session specific data for this session + */ +static void *newSession(FILTER *instance, SESSION *session) +{ + MAXROWS_INSTANCE *cinstance = (MAXROWS_INSTANCE*)instance; + MAXROWS_SESSION_DATA *csdata = maxrows_session_data_create(cinstance, session); + + return csdata; +} + +/** + * A session has been closed. + * + * @param instance The cache instance data + * @param sdata The session data of the session being closed + */ +static void closeSession(FILTER *instance, void *sdata) +{ + MAXROWS_INSTANCE *cinstance = (MAXROWS_INSTANCE*)instance; + MAXROWS_SESSION_DATA *csdata = (MAXROWS_SESSION_DATA*)sdata; +} + +/** + * Free the session data. + * + * @param instance The cache instance data + * @param sdata The session data of the session being closed + */ +static void freeSession(FILTER *instance, void *sdata) +{ + MAXROWS_INSTANCE *cinstance = (MAXROWS_INSTANCE*)instance; + MAXROWS_SESSION_DATA *csdata = (MAXROWS_SESSION_DATA*)sdata; + + maxrows_session_data_free(csdata); +} + +/** + * Set the downstream component for this filter. + * + * @param instance The cache instance data + * @param sdata The session data of the session + * @param down The downstream filter or router + */ +static void setDownstream(FILTER *instance, void *sdata, DOWNSTREAM *down) +{ + MAXROWS_INSTANCE *cinstance = (MAXROWS_INSTANCE*)instance; + MAXROWS_SESSION_DATA *csdata = (MAXROWS_SESSION_DATA*)sdata; + + csdata->down = *down; +} + +/** + * Set the upstream component for this filter. + * + * @param instance The cache instance data + * @param sdata The session data of the session + * @param up The upstream filter or router + */ +static void setUpstream(FILTER *instance, void *sdata, UPSTREAM *up) +{ + MAXROWS_INSTANCE *cinstance = (MAXROWS_INSTANCE*)instance; + MAXROWS_SESSION_DATA *csdata = (MAXROWS_SESSION_DATA*)sdata; + + csdata->up = *up; +} + +/** + * A request on its way to a backend is delivered to this function. + * + * @param instance The filter instance data + * @param sdata The filter session data + * @param buffer Buffer containing an MySQL protocol packet. + */ +static int routeQuery(FILTER *instance, void *sdata, GWBUF *packet) +{ + MAXROWS_INSTANCE *cinstance = (MAXROWS_INSTANCE*)instance; + MAXROWS_SESSION_DATA *csdata = (MAXROWS_SESSION_DATA*)sdata; + + uint8_t *data = GWBUF_DATA(packet); + + // All of these should be guaranteed by RCAP_TYPE_TRANSACTION_TRACKING + ss_dassert(GWBUF_IS_CONTIGUOUS(packet)); + ss_dassert(GWBUF_LENGTH(packet) >= MYSQL_HEADER_LEN + 1); + ss_dassert(MYSQL_GET_PACKET_LEN(data) + MYSQL_HEADER_LEN == GWBUF_LENGTH(packet)); + + bool use_default = true; + + maxrows_response_state_reset(&csdata->res); + csdata->state = MAXROWS_IGNORING_RESPONSE; + + int rv; + + switch ((int)MYSQL_GET_COMMAND(data)) + { + case MYSQL_COM_INIT_DB: + { + ss_dassert(!csdata->use_db); + size_t len = MYSQL_GET_PACKET_LEN(data) - 1; // Remove the command byte. + csdata->use_db = MXS_MALLOC(len + 1); + + if (csdata->use_db) + { + memcpy(csdata->use_db, data + MYSQL_HEADER_LEN + 1, len); + csdata->use_db[len] = 0; + csdata->state = MAXROWS_EXPECTING_USE_RESPONSE; + } + else + { + /* Do we need to handle it? */ + } + } + break; + + case MYSQL_COM_QUERY: + { + /* Detect the SELECT statement only */ + if (qc_get_operation(packet) == QUERY_OP_SELECT) + { + SESSION *session = csdata->session; + + if ((session_is_autocommit(session) && !session_trx_is_active(session)) || + session_trx_is_read_only(session)) + { + /* Waiting for a reply: + * Data will be stored in csdata->res->data via + * clientReply routine + */ + csdata->state = MAXROWS_EXPECTING_RESPONSE; + } + else + { + C_DEBUG("autocommit = %s and transaction state %s => Not using memory buffer for resultset.", + session_is_autocommit(csdata->session) ? "ON" : "OFF", + session_trx_state_to_string(session_get_trx_state(csdata->session))); + } + } + break; + + default: + break; + } + } + + if (use_default) + { + C_DEBUG("Maxrows filter is sends data."); + rv = csdata->down.routeQuery(csdata->down.instance, csdata->down.session, packet); + } + + return rv; +} + +/** + * A response on its way to the client is delivered to this function. + * + * @param instance The filter instance data + * @param sdata The filter session data + * @param queue The query data + */ +static int clientReply(FILTER *instance, void *sdata, GWBUF *data) +{ + MAXROWS_INSTANCE *cinstance = (MAXROWS_INSTANCE*)instance; + MAXROWS_SESSION_DATA *csdata = (MAXROWS_SESSION_DATA*)sdata; + + int rv; + + if (csdata->res.data) + { + gwbuf_append(csdata->res.data, data); + } + else + { + csdata->res.data = data; + } + + if (csdata->state != MAXROWS_IGNORING_RESPONSE) + { + if (gwbuf_length(csdata->res.data) > csdata->instance->config.max_resultset_size) + { + C_DEBUG("Current size %uB of resultset, at least as much " + "as maximum allowed size %uKiB. Not caching.", + gwbuf_length(csdata->res.data), + csdata->instance->config.max_resultset_size / 1024); + + csdata->state = MAXROWS_IGNORING_RESPONSE; + } + } + + switch (csdata->state) + { + case MAXROWS_EXPECTING_FIELDS: + rv = handle_expecting_fields(csdata); + break; + + case MAXROWS_EXPECTING_NOTHING: + rv = handle_expecting_nothing(csdata); + break; + + case MAXROWS_EXPECTING_RESPONSE: + rv = handle_expecting_response(csdata); + break; + + case MAXROWS_EXPECTING_ROWS: + rv = handle_expecting_rows(csdata); + break; + + case MAXROWS_EXPECTING_USE_RESPONSE: + rv = handle_expecting_use_response(csdata); + break; + + case MAXROWS_IGNORING_RESPONSE: + rv = handle_ignoring_response(csdata); + break; + + default: + MXS_ERROR("Internal filter logic broken, unexpected state: %d", csdata->state); + ss_dassert(!true); + rv = send_upstream(csdata); + maxrows_response_state_reset(&csdata->res); + csdata->state = MAXROWS_IGNORING_RESPONSE; + } + + return rv; +} + +/** + * Diagnostics routine + * + * If csdata is NULL then print diagnostics on the instance as a whole, + * otherwise print diagnostics for the particular session. + * + * @param instance The filter instance + * @param fsession Filter session, may be NULL + * @param dcb The DCB for diagnostic output + */ +static void diagnostics(FILTER *instance, void *sdata, DCB *dcb) +{ + MAXROWS_INSTANCE *cinstance = (MAXROWS_INSTANCE*)instance; + MAXROWS_SESSION_DATA *csdata = (MAXROWS_SESSION_DATA*)sdata; + + dcb_printf(dcb, "Maxrows filter is working\n"); +} + + +/** + * Capability routine. + * + * @return The capabilities of the filter. + */ +static uint64_t getCapabilities(void) +{ + return RCAP_TYPE_TRANSACTION_TRACKING; +} + +// +// API END +// + +/** + * Reset cache response state + * + * @param state Pointer to object. + */ +static void maxrows_response_state_reset(MAXROWS_RESPONSE_STATE *state) +{ + state->data = NULL; + state->n_totalfields = 0; + state->n_fields = 0; + state->n_rows = 0; + state->offset = 0; +} + +/** + * Create cache session data + * + * @param instance The cache instance this data is associated with. + * + * @return Session data or NULL if creation fails. + */ +static MAXROWS_SESSION_DATA *maxrows_session_data_create(MAXROWS_INSTANCE *instance, + SESSION* session) +{ + MAXROWS_SESSION_DATA *data = (MAXROWS_SESSION_DATA*)MXS_CALLOC(1, sizeof(MAXROWS_SESSION_DATA)); + + if (data) + { + char *default_db = NULL; + + ss_dassert(session->client_dcb); + ss_dassert(session->client_dcb->data); + MYSQL_session *mysql_session = (MYSQL_session*)session->client_dcb->data; + + if (mysql_session->db[0] != 0) + { + default_db = MXS_STRDUP(mysql_session->db); + } + + if ((mysql_session->db[0] == 0) || default_db) + { + data->instance = instance; + data->session = session; + data->state = MAXROWS_EXPECTING_NOTHING; + data->default_db = default_db; + } + else + { + MXS_FREE(data); + data = NULL; + } + } + + return data; +} + +/** + * Free cache session data. + * + * @param A cache session data previously allocated using session_data_create(). + */ +static void maxrows_session_data_free(MAXROWS_SESSION_DATA* data) +{ + if (data) + { + // In normal circumstances, only data->default_db may be non-NULL at + // this point. However, if the authentication with the backend fails + // and the session is closed, data->use_db may be non-NULL. + MXS_FREE(data->use_db); + MXS_FREE(data->default_db); + MXS_FREE(data); + } +} + +/** + * Called when resultset field information is handled. + * + * @param csdata The cache session data. + */ +static int handle_expecting_fields(MAXROWS_SESSION_DATA *csdata) +{ + ss_dassert(csdata->state == MAXROWS_EXPECTING_FIELDS); + ss_dassert(csdata->res.data); + + int rv = 1; + + bool insufficient = false; + + size_t buflen = gwbuf_length(csdata->res.data); + + while (!insufficient && (buflen - csdata->res.offset >= MYSQL_HEADER_LEN)) + { + uint8_t header[MYSQL_HEADER_LEN + 1]; + gwbuf_copy_data(csdata->res.data, csdata->res.offset, MYSQL_HEADER_LEN + 1, header); + + size_t packetlen = MYSQL_HEADER_LEN + MYSQL_GET_PACKET_LEN(header); + + if (csdata->res.offset + packetlen <= buflen) + { + // We have at least one complete packet. + int command = (int)MYSQL_GET_COMMAND(header); + + switch (command) + { + case 0xfe: // EOF, the one after the fields. + csdata->res.offset += packetlen; + csdata->state = MAXROWS_EXPECTING_ROWS; + rv = handle_expecting_rows(csdata); + break; + + default: // Field information. + csdata->res.offset += packetlen; + ++csdata->res.n_fields; + ss_dassert(csdata->res.n_fields <= csdata->res.n_totalfields); + break; + } + } + else + { + // We need more data + insufficient = true; + } + } + + return rv; +} + +/** + * Called when data is received (even if nothing is expected) from the server. + * + * @param csdata The cache session data. + */ +static int handle_expecting_nothing(MAXROWS_SESSION_DATA *csdata) +{ + ss_dassert(csdata->state == MAXROWS_EXPECTING_NOTHING); + ss_dassert(csdata->res.data); + MXS_ERROR("Received data from the backend althoug we were expecting nothing."); + ss_dassert(!true); + + return send_upstream(csdata); +} + +/** + * Called when a response is received from the server. + * + * @param csdata The cache session data. + */ +static int handle_expecting_response(MAXROWS_SESSION_DATA *csdata) +{ + ss_dassert(csdata->state == MAXROWS_EXPECTING_RESPONSE); + ss_dassert(csdata->res.data); + + int rv = 1; + + size_t buflen = gwbuf_length(csdata->res.data); + + if (buflen >= MYSQL_HEADER_LEN + 1) // We need the command byte. + { + // Reserve enough space to accomodate for the largest length encoded integer, + // which is type field + 8 bytes. + uint8_t header[MYSQL_HEADER_LEN + 1 + 8]; + gwbuf_copy_data(csdata->res.data, 0, MYSQL_HEADER_LEN + 1, header); + + switch ((int)MYSQL_GET_COMMAND(header)) + { + case 0x00: // OK + case 0xff: // ERR + C_DEBUG("OK or ERR"); + rv = send_upstream(csdata); + csdata->state = MAXROWS_IGNORING_RESPONSE; + break; + + case 0xfb: // GET_MORE_CLIENT_DATA/SEND_MORE_CLIENT_DATA + C_DEBUG("GET_MORE_CLIENT_DATA"); + rv = send_upstream(csdata); + csdata->state = MAXROWS_IGNORING_RESPONSE; + break; + + default: + C_DEBUG("RESULTSET"); + + if (csdata->res.n_totalfields != 0) + { + // We've seen the header and have figured out how many fields there are. + csdata->state = MAXROWS_EXPECTING_FIELDS; + rv = handle_expecting_fields(csdata); + } + else + { + // leint_bytes() returns the length of the int type field + the size of the + // integer. + size_t n_bytes = leint_bytes(&header[4]); + + if (MYSQL_HEADER_LEN + n_bytes <= buflen) + { + // Now we can figure out how many fields there are, but first we + // need to copy some more data. + gwbuf_copy_data(csdata->res.data, + MYSQL_HEADER_LEN + 1, n_bytes - 1, &header[MYSQL_HEADER_LEN + 1]); + + csdata->res.n_totalfields = leint_value(&header[4]); + csdata->res.offset = MYSQL_HEADER_LEN + n_bytes; + + csdata->state = MAXROWS_EXPECTING_FIELDS; + rv = handle_expecting_fields(csdata); + } + else + { + // We need more data. We will be called again, when data is available. + } + } + break; + } + } + + return rv; +} + +/** + * Called when resultset rows are handled. + * + * @param csdata The cache session data. + */ +static int handle_expecting_rows(MAXROWS_SESSION_DATA *csdata) +{ + ss_dassert(csdata->state == MAXROWS_EXPECTING_ROWS); + ss_dassert(csdata->res.data); + + int rv = 1; + + bool insufficient = false; + + size_t buflen = gwbuf_length(csdata->res.data); + + while (!insufficient && (buflen - csdata->res.offset >= MYSQL_HEADER_LEN)) + { + uint8_t header[MYSQL_HEADER_LEN + 1]; + gwbuf_copy_data(csdata->res.data, csdata->res.offset, MYSQL_HEADER_LEN + 1, header); + + size_t packetlen = MYSQL_HEADER_LEN + MYSQL_GET_PACKET_LEN(header); + + if (csdata->res.offset + packetlen <= buflen) + { + // We have at least one complete packet. + int command = (int)MYSQL_GET_COMMAND(header); + + switch (command) + { + case 0xfe: // EOF, the one after the rows. + csdata->res.offset += packetlen; + ss_dassert(csdata->res.offset == buflen); + + /* Send data only if number of rows is below the limit */ + if (csdata->state != MAXROWS_IGNORING_RESPONSE) + { + rv = send_upstream(csdata); + } + + csdata->state = MAXROWS_EXPECTING_NOTHING; + break; + + case 0xfb: // NULL + default: // length-encoded-string + csdata->res.offset += packetlen; + ++csdata->res.n_rows; + + if (csdata->res.n_rows > csdata->instance->config.max_resultset_rows) + { + C_DEBUG("Max rows %lu reached, not caching result.", csdata->res.n_rows); + /* Just return 0 result set */ + rv = send_ok_upstream(csdata); + csdata->res.offset = buflen; // To abort the loop. + csdata->state = MAXROWS_IGNORING_RESPONSE; + } + break; + } + } + else + { + // We need more data + insufficient = true; + } + } + + return rv; +} + +/** + * Called when a response to a "USE db" is received from the server. + * + * @param csdata The cache session data. + */ +static int handle_expecting_use_response(MAXROWS_SESSION_DATA *csdata) +{ + ss_dassert(csdata->state == MAXROWS_EXPECTING_USE_RESPONSE); + ss_dassert(csdata->res.data); + + int rv = 1; + + size_t buflen = gwbuf_length(csdata->res.data); + + if (buflen >= MYSQL_HEADER_LEN + 1) // We need the command byte. + { + uint8_t command; + + gwbuf_copy_data(csdata->res.data, MYSQL_HEADER_LEN, 1, &command); + + switch (command) + { + case 0x00: // OK + // In case csdata->use_db could not be allocated in routeQuery(), we will + // in fact reset the default db here. That's ok as it will prevent broken + // entries in the cache. + MXS_FREE(csdata->default_db); + csdata->default_db = csdata->use_db; + csdata->use_db = NULL; + break; + + case 0xff: // ERR + MXS_FREE(csdata->use_db); + csdata->use_db = NULL; + break; + + default: + MXS_ERROR("\"USE %s\" received unexpected server response %d.", + csdata->use_db ? csdata->use_db : "", command); + MXS_FREE(csdata->default_db); + MXS_FREE(csdata->use_db); + csdata->default_db = NULL; + csdata->use_db = NULL; + } + + rv = send_upstream(csdata); + csdata->state = MAXROWS_IGNORING_RESPONSE; + } + + return rv; +} + +/** + * Called when all data from the server is ignored. + * + * @param csdata The cache session data. + */ +static int handle_ignoring_response(MAXROWS_SESSION_DATA *csdata) +{ + ss_dassert(csdata->state == MAXROWS_IGNORING_RESPONSE); + ss_dassert(csdata->res.data); + + return send_upstream(csdata); +} + +/** + * Processes the cache params + * + * @param options Options as passed to the filter. + * @param params Parameters as passed to the filter. + * @param config Pointer to config instance where params will be stored. + * + * @return True if all parameters could be processed, false otherwise. + */ +static bool process_params(char **options, FILTER_PARAMETER **params, MAXROWS_CONFIG* config) +{ + bool error = false; + + for (int i = 0; params[i]; ++i) + { + const FILTER_PARAMETER *param = params[i]; + + /* We could add a new parameter, max_resultset_columns: + * This way if result has more than max_resultset_columns + * we return 0 result + */ + + if (strcmp(param->name, "max_resultset_rows") == 0) + { + int v = atoi(param->value); + + if (v > 0) + { + config->max_resultset_rows = v; + } + else + { + config->max_resultset_rows = MAXROWS_DEFAULT_MAX_RESULTSET_ROWS; + } + } + else if (strcmp(param->name, "max_resultset_size") == 0) + { + int v = atoi(param->value); + + if (v > 0) + { + config->max_resultset_size = v * 1024; + } + else + { + MXS_ERROR("The value of the configuration entry '%s' must " + "be an integer larger than 0.", param->name); + error = true; + } + } + else if (strcmp(param->name, "debug") == 0) + { + int v = atoi(param->value); + + if ((v >= MAXROWS_DEBUG_MIN) && (v <= MAXROWS_DEBUG_MAX)) + { + config->debug = v; + } + else + { + MXS_ERROR("The value of the configuration entry '%s' must " + "be between %d and %d, inclusive.", + param->name, MAXROWS_DEBUG_MIN, MAXROWS_DEBUG_MAX); + error = true; + } + } + else if (!filter_standard_parameter(params[i]->name)) + { + MXS_ERROR("Unknown configuration entry '%s'.", param->name); + error = true; + } + } + + return !error; +} + +/** + * Send data upstream. + * + * @param csdata Session data + * + * @return Whatever the upstream returns. + */ +static int send_upstream(MAXROWS_SESSION_DATA *csdata) +{ + ss_dassert(csdata->res.data != NULL); + + int rv = csdata->up.clientReply(csdata->up.instance, csdata->up.session, csdata->res.data); + csdata->res.data = NULL; + + return rv; +} + +/** + * Send OK packet data upstream. + * + * @param csdata Session data + * + * @return Whatever the upstream returns. + */ +static int send_ok_upstream(MAXROWS_SESSION_DATA *csdata) +{ + /* Note: sequence id is always 01 (4th byte) */ + uint8_t ok[MAXROWS_OK_PACKET_LEN] = {07,00,00,01,00,00,00,02,00,00,00}; + GWBUF *packet = gwbuf_alloc(MAXROWS_OK_PACKET_LEN); + uint8_t *ptr = GWBUF_DATA(packet); + memcpy(ptr, &ok, MAXROWS_OK_PACKET_LEN); + + ss_dassert(csdata->res.data != NULL); + + int rv = csdata->up.clientReply(csdata->up.instance, csdata->up.session, packet); + gwbuf_free(csdata->res.data); + csdata->res.data = NULL; + + return rv; +} + diff --git a/server/modules/filter/maxrows/maxrows.h b/server/modules/filter/maxrows/maxrows.h new file mode 100644 index 000000000..7aad82c1c --- /dev/null +++ b/server/modules/filter/maxrows/maxrows.h @@ -0,0 +1,43 @@ +#pragma once +#ifndef _MAXSCALE_FILTER_MAXROWS_MAXROWS_H +#define _MAXSCALE_FILTER_MAXROWS_MAXROWS_H +/* + * Copyright (c) 2016 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl. + * + * Change Date: 2019-07-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ + +#include + +MXS_BEGIN_DECLS + +#define MAXROWS_OK_PACKET_LEN 11 + +#define MAXROWS_DEBUG_NONE 0 +#define MAXROWS_DEBUG_MATCHING 1 +#define MAXROWS_DEBUG_NON_MATCHING 2 +#define MAXROWS_DEBUG_USE 4 +#define MAXROWS_DEBUG_NON_USE 8 + +#define MAXROWS_DEBUG_RULES (MAXROWS_DEBUG_MATCHING | MAXROWS_DEBUG_NON_MATCHING) +#define MAXROWS_DEBUG_USAGE (MAXROWS_DEBUG_USE | MAXROWS_DEBUG_NON_USE) +#define MAXROWS_DEBUG_MIN MAXROWS_DEBUG_NONE +#define MAXROWS_DEBUG_MAX (MAXROWS_DEBUG_RULES | MAXROWS_DEBUG_USAGE) + +// Count +#define MAXROWS_DEFAULT_MAX_RESULTSET_ROWS UINT_MAX +// Bytes +#define MAXROWS_DEFAULT_MAX_RESULTSET_SIZE 64 * 1024 +// Integer value +#define MAXROWS_DEFAULT_DEBUG 0 + +MXS_END_DECLS + +#endif From 5aa0c32e8f96db1db1f20a16a4b37ab113652fc3 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Wed, 26 Oct 2016 18:58:07 +0200 Subject: [PATCH 097/215] Minor changes Minor changes --- server/modules/filter/maxrows/maxrows.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/server/modules/filter/maxrows/maxrows.c b/server/modules/filter/maxrows/maxrows.c index e1415f1ea..8785c67ba 100644 --- a/server/modules/filter/maxrows/maxrows.c +++ b/server/modules/filter/maxrows/maxrows.c @@ -25,7 +25,7 @@ * @endverbatim */ -#define MXS_MODULE_NAME "cache" +#define MXS_MODULE_NAME "maxrows" #include #include #include @@ -459,12 +459,10 @@ static void diagnostics(FILTER *instance, void *sdata, DCB *dcb) */ static uint64_t getCapabilities(void) { - return RCAP_TYPE_TRANSACTION_TRACKING; + return RCAP_TYPE_STMT_INPUT; } -// -// API END -// +/* API END */ /** * Reset cache response state From 50db0db3168c73850c8795016a42e2ea8fe33763 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Wed, 26 Oct 2016 18:33:57 +0300 Subject: [PATCH 098/215] MXS-934: Set operation of PREPARE statement The operation of a PREPARE statement will be that of the preparable statement. That will make it possible to know whether an EXECUTEd prepared statement e.g. is a SELECT or an UPDATE. --- query_classifier/qc_sqlite/qc_sqlite.c | 36 ++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/query_classifier/qc_sqlite/qc_sqlite.c b/query_classifier/qc_sqlite/qc_sqlite.c index 3789fce2e..ca3f02639 100644 --- a/query_classifier/qc_sqlite/qc_sqlite.c +++ b/query_classifier/qc_sqlite/qc_sqlite.c @@ -79,6 +79,8 @@ typedef struct qc_sqlite_info int keyword_1; // The first encountered keyword. int keyword_2; // The second encountered keyword. char* prepare_name; // The name of a prepared statement. + size_t preparable_stmt_offset; // The start of the preparable statement. + size_t preparable_stmt_length; // The length of the preparable statement. } QC_SQLITE_INFO; typedef enum qc_log_level @@ -328,6 +330,9 @@ static QC_SQLITE_INFO* info_init(QC_SQLITE_INFO* info) info->database_names_capacity = 0; info->keyword_1 = 0; // Sqlite3 starts numbering tokens from 1, so 0 means info->keyword_2 = 0; // that we have not seen a keyword. + info->prepare_name = NULL; + info->preparable_stmt_offset = 0; + info->preparable_stmt_length = 0; return info; } @@ -460,6 +465,31 @@ static bool parse_query(GWBUF* query) this_thread.info->query = NULL; this_thread.info->query_len = 0; + if (info->types & QUERY_TYPE_PREPARE_NAMED_STMT) + { + QC_SQLITE_INFO* preparable_info = info_alloc(); + + if (preparable_info) + { + this_thread.info = preparable_info; + + const char *preparable_s = s + info->preparable_stmt_offset; + size_t preparable_len = info->preparable_stmt_length; + + this_thread.info->query = preparable_s; + this_thread.info->query_len = preparable_len; + parse_query_string(preparable_s, preparable_len); + this_thread.info->query = NULL; + this_thread.info->query_len = 0; + + // TODO: Perhaps the rest of the stuff should be + // TODO: copied as well. + info->operation = preparable_info->operation; + + info_free(preparable_info); + } + } + // TODO: Add return value to gwbuf_add_buffer_object. // Always added; also when it was not recognized. If it was not recognized now, // it won't be if we try a second time. @@ -1969,6 +1999,12 @@ void maxscalePrepare(Parse* pParse, Token* pName, Token* pStmt) memcpy(info->prepare_name, pName->z, pName->n); info->prepare_name[pName->n] = 0; } + + // We store the position of the preparable statement inside the original + // statement. That will allow us to later create a new GWBUF of the + // parsable statment and parse that. + info->preparable_stmt_offset = pParse->sLastToken.z - pParse->zTail + 1; // Ignore starting quote. + info->preparable_stmt_length = pStmt->n - 2; // Remove starting and ending quotes. } void maxscalePrivileges(Parse* pParse, int kind) From 7419ddcecac7f3438b07ffc6f01ee4d7ca23b12d Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Wed, 26 Oct 2016 19:12:56 +0200 Subject: [PATCH 099/215] Code review update Code review update --- server/modules/filter/maxrows/maxrows.c | 99 ++----------------------- server/modules/filter/maxrows/maxrows.h | 4 - 2 files changed, 5 insertions(+), 98 deletions(-) diff --git a/server/modules/filter/maxrows/maxrows.c b/server/modules/filter/maxrows/maxrows.c index 8785c67ba..c5e520439 100644 --- a/server/modules/filter/maxrows/maxrows.c +++ b/server/modules/filter/maxrows/maxrows.c @@ -131,7 +131,6 @@ typedef enum maxrows_session_state MAXROWS_EXPECTING_FIELDS, // A select has been sent, and we want more fields. MAXROWS_EXPECTING_ROWS, // A select has been sent, and we want more rows. MAXROWS_EXPECTING_NOTHING, // We are not expecting anything from the server. - MAXROWS_EXPECTING_USE_RESPONSE, // A "USE DB" was issued. MAXROWS_IGNORING_RESPONSE, // We are not interested in the data received from the server. } maxrows_session_state_t; @@ -165,7 +164,6 @@ static int handle_expecting_fields(MAXROWS_SESSION_DATA *csdata); static int handle_expecting_nothing(MAXROWS_SESSION_DATA *csdata); static int handle_expecting_response(MAXROWS_SESSION_DATA *csdata); static int handle_expecting_rows(MAXROWS_SESSION_DATA *csdata); -static int handle_expecting_use_response(MAXROWS_SESSION_DATA *csdata); static int handle_ignoring_response(MAXROWS_SESSION_DATA *csdata); static bool process_params(char **options, FILTER_PARAMETER **params, MAXROWS_CONFIG* config); @@ -302,47 +300,16 @@ static int routeQuery(FILTER *instance, void *sdata, GWBUF *packet) switch ((int)MYSQL_GET_COMMAND(data)) { - case MYSQL_COM_INIT_DB: - { - ss_dassert(!csdata->use_db); - size_t len = MYSQL_GET_PACKET_LEN(data) - 1; // Remove the command byte. - csdata->use_db = MXS_MALLOC(len + 1); - - if (csdata->use_db) - { - memcpy(csdata->use_db, data + MYSQL_HEADER_LEN + 1, len); - csdata->use_db[len] = 0; - csdata->state = MAXROWS_EXPECTING_USE_RESPONSE; - } - else - { - /* Do we need to handle it? */ - } - } - break; - case MYSQL_COM_QUERY: { /* Detect the SELECT statement only */ if (qc_get_operation(packet) == QUERY_OP_SELECT) { - SESSION *session = csdata->session; - - if ((session_is_autocommit(session) && !session_trx_is_active(session)) || - session_trx_is_read_only(session)) - { - /* Waiting for a reply: - * Data will be stored in csdata->res->data via - * clientReply routine - */ - csdata->state = MAXROWS_EXPECTING_RESPONSE; - } - else - { - C_DEBUG("autocommit = %s and transaction state %s => Not using memory buffer for resultset.", - session_is_autocommit(csdata->session) ? "ON" : "OFF", - session_trx_state_to_string(session_get_trx_state(csdata->session))); - } + /* Waiting for a reply: + * Data will be stored in csdata->res->data via + * clientReply routine + */ + csdata->state = MAXROWS_EXPECTING_RESPONSE; } break; @@ -414,10 +381,6 @@ static int clientReply(FILTER *instance, void *sdata, GWBUF *data) rv = handle_expecting_rows(csdata); break; - case MAXROWS_EXPECTING_USE_RESPONSE: - rv = handle_expecting_use_response(csdata); - break; - case MAXROWS_IGNORING_RESPONSE: rv = handle_ignoring_response(csdata); break; @@ -751,58 +714,6 @@ static int handle_expecting_rows(MAXROWS_SESSION_DATA *csdata) return rv; } -/** - * Called when a response to a "USE db" is received from the server. - * - * @param csdata The cache session data. - */ -static int handle_expecting_use_response(MAXROWS_SESSION_DATA *csdata) -{ - ss_dassert(csdata->state == MAXROWS_EXPECTING_USE_RESPONSE); - ss_dassert(csdata->res.data); - - int rv = 1; - - size_t buflen = gwbuf_length(csdata->res.data); - - if (buflen >= MYSQL_HEADER_LEN + 1) // We need the command byte. - { - uint8_t command; - - gwbuf_copy_data(csdata->res.data, MYSQL_HEADER_LEN, 1, &command); - - switch (command) - { - case 0x00: // OK - // In case csdata->use_db could not be allocated in routeQuery(), we will - // in fact reset the default db here. That's ok as it will prevent broken - // entries in the cache. - MXS_FREE(csdata->default_db); - csdata->default_db = csdata->use_db; - csdata->use_db = NULL; - break; - - case 0xff: // ERR - MXS_FREE(csdata->use_db); - csdata->use_db = NULL; - break; - - default: - MXS_ERROR("\"USE %s\" received unexpected server response %d.", - csdata->use_db ? csdata->use_db : "", command); - MXS_FREE(csdata->default_db); - MXS_FREE(csdata->use_db); - csdata->default_db = NULL; - csdata->use_db = NULL; - } - - rv = send_upstream(csdata); - csdata->state = MAXROWS_IGNORING_RESPONSE; - } - - return rv; -} - /** * Called when all data from the server is ignored. * diff --git a/server/modules/filter/maxrows/maxrows.h b/server/modules/filter/maxrows/maxrows.h index 7aad82c1c..45d354048 100644 --- a/server/modules/filter/maxrows/maxrows.h +++ b/server/modules/filter/maxrows/maxrows.h @@ -1,6 +1,4 @@ #pragma once -#ifndef _MAXSCALE_FILTER_MAXROWS_MAXROWS_H -#define _MAXSCALE_FILTER_MAXROWS_MAXROWS_H /* * Copyright (c) 2016 MariaDB Corporation Ab * @@ -39,5 +37,3 @@ MXS_BEGIN_DECLS #define MAXROWS_DEFAULT_DEBUG 0 MXS_END_DECLS - -#endif From 59b3a1fe36184263ea676954b4225550b80fb4d0 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Wed, 26 Oct 2016 19:20:25 +0200 Subject: [PATCH 100/215] Handle max_resultset_size in maxrows Handle max_resultset_size in maxrows --- server/modules/filter/maxrows/maxrows.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/modules/filter/maxrows/maxrows.c b/server/modules/filter/maxrows/maxrows.c index c5e520439..3949a1180 100644 --- a/server/modules/filter/maxrows/maxrows.c +++ b/server/modules/filter/maxrows/maxrows.c @@ -360,6 +360,9 @@ static int clientReply(FILTER *instance, void *sdata, GWBUF *data) csdata->instance->config.max_resultset_size / 1024); csdata->state = MAXROWS_IGNORING_RESPONSE; + + /* Send empty result set */ + return send_ok_upstream(csdata); } } From cbef9b944da4bd27bee62b103ab7590494126739 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Wed, 26 Oct 2016 20:47:40 +0300 Subject: [PATCH 101/215] qc_sqlite: Pick out the name in an EXECUTE stmt --- query_classifier/qc_sqlite/qc_sqlite.c | 7 +++++++ query_classifier/qc_sqlite/sqlite-src-3110100/src/parse.y | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/query_classifier/qc_sqlite/qc_sqlite.c b/query_classifier/qc_sqlite/qc_sqlite.c index ca3f02639..d68a6afe4 100644 --- a/query_classifier/qc_sqlite/qc_sqlite.c +++ b/query_classifier/qc_sqlite/qc_sqlite.c @@ -1625,6 +1625,13 @@ void maxscaleExecute(Parse* pParse, Token* pName) info->status = QC_QUERY_PARSED; info->types = QUERY_TYPE_WRITE; info->is_real_query = true; + + info->prepare_name = MXS_MALLOC(pName->n + 1); + if (info->prepare_name) + { + memcpy(info->prepare_name, pName->z, pName->n); + info->prepare_name[pName->n] = 0; + } } void maxscaleExplain(Parse* pParse, SrcList* pName) diff --git a/query_classifier/qc_sqlite/sqlite-src-3110100/src/parse.y b/query_classifier/qc_sqlite/sqlite-src-3110100/src/parse.y index 8abf935f8..b015bc5c7 100644 --- a/query_classifier/qc_sqlite/sqlite-src-3110100/src/parse.y +++ b/query_classifier/qc_sqlite/sqlite-src-3110100/src/parse.y @@ -2827,7 +2827,7 @@ execute_variables ::= VARIABLE. execute_variables ::= execute_variables COMMA VARIABLE. execute_variables_opt ::= . -execute_variables_opt ::= execute_variables. +execute_variables_opt ::= USING execute_variables. execute ::= EXECUTE nm(X) execute_variables_opt. { maxscaleExecute(pParse, &X); From b76cdfd36753dfc7f3fe29228291fe1a7d5aaa85 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Wed, 26 Oct 2016 21:00:54 +0300 Subject: [PATCH 102/215] qc_sqlite: Pick out prepare name in DEALLOCATE stmt --- query_classifier/qc_sqlite/qc_sqlite.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/query_classifier/qc_sqlite/qc_sqlite.c b/query_classifier/qc_sqlite/qc_sqlite.c index d68a6afe4..bb180ac64 100644 --- a/query_classifier/qc_sqlite/qc_sqlite.c +++ b/query_classifier/qc_sqlite/qc_sqlite.c @@ -1588,6 +1588,13 @@ void maxscaleDeallocate(Parse* pParse, Token* pName) info->status = QC_QUERY_PARSED; info->types = QUERY_TYPE_WRITE; + + info->prepare_name = MXS_MALLOC(pName->n + 1); + if (info->prepare_name) + { + memcpy(info->prepare_name, pName->z, pName->n); + info->prepare_name[pName->n] = 0; + } } void maxscaleDo(Parse* pParse, ExprList* pEList) From c73c3b390a3fa59ea8d80f53e649fcb8cb21e959 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Thu, 27 Oct 2016 09:25:08 +0200 Subject: [PATCH 103/215] Addition of MAXROWS_DISCARDING_RESPONSE Addition of MAXROWS_DISCARDING_RESPONSE Packets received after sent OK are consumed and not sent --- server/modules/filter/maxrows/maxrows.c | 83 ++++++++++++++++--------- 1 file changed, 54 insertions(+), 29 deletions(-) diff --git a/server/modules/filter/maxrows/maxrows.c b/server/modules/filter/maxrows/maxrows.c index 3949a1180..954e2d8e0 100644 --- a/server/modules/filter/maxrows/maxrows.c +++ b/server/modules/filter/maxrows/maxrows.c @@ -132,6 +132,7 @@ typedef enum maxrows_session_state MAXROWS_EXPECTING_ROWS, // A select has been sent, and we want more rows. MAXROWS_EXPECTING_NOTHING, // We are not expecting anything from the server. MAXROWS_IGNORING_RESPONSE, // We are not interested in the data received from the server. + MAXROWS_DISCARDING_RESPONSE, // We have returned empty result set and we discard any new data. } maxrows_session_state_t; typedef struct maxrows_response_state @@ -145,9 +146,9 @@ typedef struct maxrows_response_state static void maxrows_response_state_reset(MAXROWS_RESPONSE_STATE *state); -typedef struct cache_session_data +typedef struct maxrows_session_data { - MAXROWS_INSTANCE *instance; /**< The cache instance the session is associated with. */ + MAXROWS_INSTANCE *instance; /**< The maxrows instance the session is associated with. */ DOWNSTREAM down; /**< The previous filter or equivalent. */ UPSTREAM up; /**< The next filter or equivalent. */ MAXROWS_RESPONSE_STATE res; /**< The response state. */ @@ -169,11 +170,12 @@ static bool process_params(char **options, FILTER_PARAMETER **params, MAXROWS_CO static int send_upstream(MAXROWS_SESSION_DATA *csdata); static int send_ok_upstream(MAXROWS_SESSION_DATA *csdata); +static int handle_discarding_response(MAXROWS_SESSION_DATA *csdata); /* API BEGIN */ /** - * Create an instance of the cache filter for a particular service + * Create an instance of the maxrows filter for a particular service * within MaxScale. * * @param name The name of the instance (as defined in the config file). @@ -203,7 +205,7 @@ static FILTER *createInstance(const char *name, char **options, FILTER_PARAMETER /** * Associate a new session with this instance of the filter. * - * @param instance The cache instance data + * @param instance The maxrows instance data * @param session The session itself * * @return Session specific data for this session @@ -219,7 +221,7 @@ static void *newSession(FILTER *instance, SESSION *session) /** * A session has been closed. * - * @param instance The cache instance data + * @param instance The maxrows instance data * @param sdata The session data of the session being closed */ static void closeSession(FILTER *instance, void *sdata) @@ -231,7 +233,7 @@ static void closeSession(FILTER *instance, void *sdata) /** * Free the session data. * - * @param instance The cache instance data + * @param instance The maxrows instance data * @param sdata The session data of the session being closed */ static void freeSession(FILTER *instance, void *sdata) @@ -245,7 +247,7 @@ static void freeSession(FILTER *instance, void *sdata) /** * Set the downstream component for this filter. * - * @param instance The cache instance data + * @param instance The maxrowsinstance data * @param sdata The session data of the session * @param down The downstream filter or router */ @@ -260,7 +262,7 @@ static void setDownstream(FILTER *instance, void *sdata, DOWNSTREAM *down) /** * Set the upstream component for this filter. * - * @param instance The cache instance data + * @param instance The maxrows instance data * @param sdata The session data of the session * @param up The upstream filter or router */ @@ -320,7 +322,7 @@ static int routeQuery(FILTER *instance, void *sdata, GWBUF *packet) if (use_default) { - C_DEBUG("Maxrows filter is sends data."); + C_DEBUG("Maxrows filter is sending data."); rv = csdata->down.routeQuery(csdata->down.instance, csdata->down.session, packet); } @@ -355,11 +357,11 @@ static int clientReply(FILTER *instance, void *sdata, GWBUF *data) if (gwbuf_length(csdata->res.data) > csdata->instance->config.max_resultset_size) { C_DEBUG("Current size %uB of resultset, at least as much " - "as maximum allowed size %uKiB. Not caching.", + "as maximum allowed size %uKiB. Not returning data.", gwbuf_length(csdata->res.data), csdata->instance->config.max_resultset_size / 1024); - csdata->state = MAXROWS_IGNORING_RESPONSE; + csdata->state = MAXROWS_DISCARDING_RESPONSE; /* Send empty result set */ return send_ok_upstream(csdata); @@ -388,6 +390,10 @@ static int clientReply(FILTER *instance, void *sdata, GWBUF *data) rv = handle_ignoring_response(csdata); break; + case MAXROWS_DISCARDING_RESPONSE: + rv = handle_discarding_response(csdata); + break; + default: MXS_ERROR("Internal filter logic broken, unexpected state: %d", csdata->state); ss_dassert(!true); @@ -431,7 +437,7 @@ static uint64_t getCapabilities(void) /* API END */ /** - * Reset cache response state + * Reset maxrows response state * * @param state Pointer to object. */ @@ -445,9 +451,9 @@ static void maxrows_response_state_reset(MAXROWS_RESPONSE_STATE *state) } /** - * Create cache session data + * Create maxrows session data * - * @param instance The cache instance this data is associated with. + * @param instance The maxrows instance this data is associated with. * * @return Session data or NULL if creation fails. */ @@ -487,9 +493,9 @@ static MAXROWS_SESSION_DATA *maxrows_session_data_create(MAXROWS_INSTANCE *insta } /** - * Free cache session data. + * Free maxrows session data. * - * @param A cache session data previously allocated using session_data_create(). + * @param A maxrows session data previously allocated using session_data_create(). */ static void maxrows_session_data_free(MAXROWS_SESSION_DATA* data) { @@ -507,7 +513,7 @@ static void maxrows_session_data_free(MAXROWS_SESSION_DATA* data) /** * Called when resultset field information is handled. * - * @param csdata The cache session data. + * @param csdata The maxrows session data. */ static int handle_expecting_fields(MAXROWS_SESSION_DATA *csdata) { @@ -560,7 +566,7 @@ static int handle_expecting_fields(MAXROWS_SESSION_DATA *csdata) /** * Called when data is received (even if nothing is expected) from the server. * - * @param csdata The cache session data. + * @param csdata The maxrows session data. */ static int handle_expecting_nothing(MAXROWS_SESSION_DATA *csdata) { @@ -575,7 +581,7 @@ static int handle_expecting_nothing(MAXROWS_SESSION_DATA *csdata) /** * Called when a response is received from the server. * - * @param csdata The cache session data. + * @param csdata The maxrows session data. */ static int handle_expecting_response(MAXROWS_SESSION_DATA *csdata) { @@ -651,7 +657,7 @@ static int handle_expecting_response(MAXROWS_SESSION_DATA *csdata) /** * Called when resultset rows are handled. * - * @param csdata The cache session data. + * @param csdata The maxrows session data. */ static int handle_expecting_rows(MAXROWS_SESSION_DATA *csdata) { @@ -682,11 +688,7 @@ static int handle_expecting_rows(MAXROWS_SESSION_DATA *csdata) csdata->res.offset += packetlen; ss_dassert(csdata->res.offset == buflen); - /* Send data only if number of rows is below the limit */ - if (csdata->state != MAXROWS_IGNORING_RESPONSE) - { - rv = send_upstream(csdata); - } + rv = send_upstream(csdata); csdata->state = MAXROWS_EXPECTING_NOTHING; break; @@ -698,11 +700,16 @@ static int handle_expecting_rows(MAXROWS_SESSION_DATA *csdata) if (csdata->res.n_rows > csdata->instance->config.max_resultset_rows) { - C_DEBUG("Max rows %lu reached, not caching result.", csdata->res.n_rows); + C_DEBUG("max_resultset_rows %lu reached, not returning the result.", csdata->res.n_rows); + /* Just return 0 result set */ rv = send_ok_upstream(csdata); csdata->res.offset = buflen; // To abort the loop. - csdata->state = MAXROWS_IGNORING_RESPONSE; + + /* If additional packets from server will come + * they will be "consumed" and not sent to client + */ + csdata->state = MAXROWS_DISCARDING_RESPONSE; } break; } @@ -720,7 +727,7 @@ static int handle_expecting_rows(MAXROWS_SESSION_DATA *csdata) /** * Called when all data from the server is ignored. * - * @param csdata The cache session data. + * @param csdata The maxrows session data. */ static int handle_ignoring_response(MAXROWS_SESSION_DATA *csdata) { @@ -731,7 +738,7 @@ static int handle_ignoring_response(MAXROWS_SESSION_DATA *csdata) } /** - * Processes the cache params + * Processes the maxrows params * * @param options Options as passed to the filter. * @param params Parameters as passed to the filter. @@ -847,3 +854,21 @@ static int send_ok_upstream(MAXROWS_SESSION_DATA *csdata) return rv; } +/** + * Called when all data from the server should be consumed and not sent. + * + * @param csdata The maxrows session data. + */ +static int handle_discarding_response(MAXROWS_SESSION_DATA *csdata) +{ + ss_dassert(csdata->state == MAXROWS_DISCARDING_RESPONSE); + ss_dassert(csdata->res.data); + + gwbuf_free(csdata->res.data); + csdata->res.data = NULL; + + MXS_INFO("maxrows filter is receiving response packets from the backend in 'discarding_response' state."); + + return 1; +} + From 199dd5131203fca677a528dcd2d8aa030baa533f Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Thu, 27 Oct 2016 11:10:36 +0300 Subject: [PATCH 104/215] Cache: C_DEBUG removed and debug option extended The development time C_DEBUG is now removed, and replaced with a debug option that allows top-level decisions to be logged. --- Documentation/Filters/Cache.md | 15 ++++++------ server/modules/filter/cache/cache.c | 38 +++++++++++++++++------------ server/modules/filter/cache/cache.h | 11 +++++---- 3 files changed, 36 insertions(+), 28 deletions(-) diff --git a/Documentation/Filters/Cache.md b/Documentation/Filters/Cache.md index c983165e1..53372dcfc 100644 --- a/Documentation/Filters/Cache.md +++ b/Documentation/Filters/Cache.md @@ -106,16 +106,17 @@ An integer value, using which the level of debug logging made by the cache can be controlled. The value is actually a bitfield with different bits denoting different logging. - * `0` (`0b0000`) No logging is made. - * `1` (`0b0001`) A matching rule is logged. - * `2` (`0b0010`) A non-matching rule is logged. - * `4` (`0b0100`) A decision to use data from the cache is logged. - * `8` (`0b1000`) A decision not to use data from the cache is logged. + * ` 0` (`0b00000`) No logging is made. + * ` 1` (`0b00001`) A matching rule is logged. + * ` 2` (`0b00010`) A non-matching rule is logged. + * ` 4` (`0b00100`) A decision to use data from the cache is logged. + * ` 8` (`0b01000`) A decision not to use data from the cache is logged. + * '16' (`0b10000`) Higher level decisions are logged. -Default is `0`. To log everything, give `debug` a value of `15`. +Default is `0`. To log everything, give `debug` a value of `31`. ``` -debug=2 +debug=31 ``` # Rules diff --git a/server/modules/filter/cache/cache.c b/server/modules/filter/cache/cache.c index de830d7cd..2e869cf06 100644 --- a/server/modules/filter/cache/cache.c +++ b/server/modules/filter/cache/cache.c @@ -37,8 +37,6 @@ static int clientReply(FILTER *instance, void *sdata, GWBUF *queue); static void diagnostics(FILTER *instance, void *sdata, DCB *dcb); static uint64_t getCapabilities(void); -#define C_DEBUG(format, ...) MXS_LOG_MESSAGE(LOG_NOTICE, format, ##__VA_ARGS__) - // // Global symbols of the Module // @@ -401,7 +399,10 @@ static int routeQuery(FILTER *instance, void *sdata, GWBUF *packet) else { csdata->state = CACHE_EXPECTING_NOTHING; - C_DEBUG("Using data from cache."); + if (csdata->instance.config & CACHE_DEBUG_DECISIONS) + { + MXS_NOTICE("Using data from cache."); + } gwbuf_free(packet); DCB *dcb = csdata->session->client_dcb; @@ -418,9 +419,13 @@ static int routeQuery(FILTER *instance, void *sdata, GWBUF *packet) } else { - C_DEBUG("autocommit = %s and transaction state %s => Not using or storing to cache.", - session_is_autocommit(csdata->session) ? "ON" : "OFF", - session_trx_state_to_string(session_get_trx_state(csdata->session))); + if (csdata->instance.config & CACHE_DEBUG_DECISIONS) + { + MXS_NOTICE("autocommit = %s and transaction state %s => Not using or " + "storing to cache.", + session_is_autocommit(csdata->session) ? "ON" : "OFF", + session_trx_state_to_string(session_get_trx_state(csdata->session))); + } } } break; @@ -432,7 +437,6 @@ static int routeQuery(FILTER *instance, void *sdata, GWBUF *packet) if (use_default) { - C_DEBUG("Using default processing."); rv = csdata->down.routeQuery(csdata->down.instance, csdata->down.session, packet); } @@ -466,10 +470,13 @@ static int clientReply(FILTER *instance, void *sdata, GWBUF *data) { if (gwbuf_length(csdata->res.data) > csdata->instance->config.max_resultset_size) { - C_DEBUG("Current size %uB of resultset, at least as much " - "as maximum allowed size %uKiB. Not caching.", - gwbuf_length(csdata->res.data), - csdata->instance->config.max_resultset_size / 1024); + if (csdata->instance.config & CACHE_DEBUG_DECISIONS) + { + MXS_NOTICE("Current size %uB of resultset, at least as much " + "as maximum allowed size %uKiB. Not caching.", + gwbuf_length(csdata->res.data), + csdata->instance->config.max_resultset_size / 1024); + } csdata->state = CACHE_IGNORING_RESPONSE; } @@ -714,7 +721,6 @@ static int handle_expecting_response(CACHE_SESSION_DATA *csdata) { case 0x00: // OK case 0xff: // ERR - C_DEBUG("OK or ERR"); store_result(csdata); rv = send_upstream(csdata); @@ -722,14 +728,11 @@ static int handle_expecting_response(CACHE_SESSION_DATA *csdata) break; case 0xfb: // GET_MORE_CLIENT_DATA/SEND_MORE_CLIENT_DATA - C_DEBUG("GET_MORE_CLIENT_DATA"); rv = send_upstream(csdata); csdata->state = CACHE_IGNORING_RESPONSE; break; default: - C_DEBUG("RESULTSET"); - if (csdata->res.n_totalfields != 0) { // We've seen the header and have figured out how many fields there are. @@ -814,7 +817,10 @@ static int handle_expecting_rows(CACHE_SESSION_DATA *csdata) if (csdata->res.n_rows > csdata->instance->config.max_resultset_rows) { - C_DEBUG("Max rows %lu reached, not caching result.", csdata->res.n_rows); + if (csdata->instance.config & CACHE_DEBUG_DECISIONS) + { + MXS_NOTICE("Max rows %lu reached, not caching result.", csdata->res.n_rows); + } rv = send_upstream(csdata); csdata->res.offset = buflen; // To abort the loop. csdata->state = CACHE_IGNORING_RESPONSE; diff --git a/server/modules/filter/cache/cache.h b/server/modules/filter/cache/cache.h index a0fc8bf2b..93c1261e3 100644 --- a/server/modules/filter/cache/cache.h +++ b/server/modules/filter/cache/cache.h @@ -18,11 +18,12 @@ MXS_BEGIN_DECLS -#define CACHE_DEBUG_NONE 0 -#define CACHE_DEBUG_MATCHING 1 -#define CACHE_DEBUG_NON_MATCHING 2 -#define CACHE_DEBUG_USE 4 -#define CACHE_DEBUG_NON_USE 8 +#define CACHE_DEBUG_NONE 0 /* 0b00000 */ +#define CACHE_DEBUG_MATCHING 1 /* 0b00001 */ +#define CACHE_DEBUG_NON_MATCHING 2 /* 0b00010 */ +#define CACHE_DEBUG_USE 4 /* 0b00100 */ +#define CACHE_DEBUG_NON_USE 8 /* 0b01000 */ +#define CACHE_DEBUG_DECISIONS 16 /* 0b10000 */ #define CACHE_DEBUG_RULES (CACHE_DEBUG_MATCHING | CACHE_DEBUG_NON_MATCHING) #define CACHE_DEBUG_USAGE (CACHE_DEBUG_USE | CACHE_DEBUG_NON_USE) From 0cfb82f1464592f579a8b377af7d8c80ed7f1c59 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Thu, 27 Oct 2016 10:53:07 +0200 Subject: [PATCH 105/215] The MAXROWS_DISCARDING_RESPONSE is handled differently The MAXROWS_DISCARDING_RESPONSE is handled differently: the OK packet is sent only after an EOF is seen in a reply even with multiple packet transmission from the backend --- server/modules/filter/maxrows/maxrows.c | 69 +++++++++---------------- 1 file changed, 25 insertions(+), 44 deletions(-) diff --git a/server/modules/filter/maxrows/maxrows.c b/server/modules/filter/maxrows/maxrows.c index 954e2d8e0..5a4eec817 100644 --- a/server/modules/filter/maxrows/maxrows.c +++ b/server/modules/filter/maxrows/maxrows.c @@ -170,7 +170,6 @@ static bool process_params(char **options, FILTER_PARAMETER **params, MAXROWS_CO static int send_upstream(MAXROWS_SESSION_DATA *csdata); static int send_ok_upstream(MAXROWS_SESSION_DATA *csdata); -static int handle_discarding_response(MAXROWS_SESSION_DATA *csdata); /* API BEGIN */ @@ -354,17 +353,17 @@ static int clientReply(FILTER *instance, void *sdata, GWBUF *data) if (csdata->state != MAXROWS_IGNORING_RESPONSE) { - if (gwbuf_length(csdata->res.data) > csdata->instance->config.max_resultset_size) + if (csdata->state != MAXROWS_DISCARDING_RESPONSE) { - C_DEBUG("Current size %uB of resultset, at least as much " - "as maximum allowed size %uKiB. Not returning data.", - gwbuf_length(csdata->res.data), - csdata->instance->config.max_resultset_size / 1024); + if (gwbuf_length(csdata->res.data) > csdata->instance->config.max_resultset_size) + { + C_DEBUG("Current size %uB of resultset, at least as much " + "as maximum allowed size %uKiB. Not returning data.", + gwbuf_length(csdata->res.data), + csdata->instance->config.max_resultset_size / 1024); - csdata->state = MAXROWS_DISCARDING_RESPONSE; - - /* Send empty result set */ - return send_ok_upstream(csdata); + csdata->state = MAXROWS_DISCARDING_RESPONSE; + } } } @@ -382,6 +381,7 @@ static int clientReply(FILTER *instance, void *sdata, GWBUF *data) rv = handle_expecting_response(csdata); break; + case MAXROWS_DISCARDING_RESPONSE: case MAXROWS_EXPECTING_ROWS: rv = handle_expecting_rows(csdata); break; @@ -390,10 +390,6 @@ static int clientReply(FILTER *instance, void *sdata, GWBUF *data) rv = handle_ignoring_response(csdata); break; - case MAXROWS_DISCARDING_RESPONSE: - rv = handle_discarding_response(csdata); - break; - default: MXS_ERROR("Internal filter logic broken, unexpected state: %d", csdata->state); ss_dassert(!true); @@ -661,7 +657,7 @@ static int handle_expecting_response(MAXROWS_SESSION_DATA *csdata) */ static int handle_expecting_rows(MAXROWS_SESSION_DATA *csdata) { - ss_dassert(csdata->state == MAXROWS_EXPECTING_ROWS); + ss_dassert(csdata->state == MAXROWS_EXPECTING_ROWS || csdata->state == MAXROWS_DISCARDING_RESPONSE); ss_dassert(csdata->res.data); int rv = 1; @@ -688,7 +684,14 @@ static int handle_expecting_rows(MAXROWS_SESSION_DATA *csdata) csdata->res.offset += packetlen; ss_dassert(csdata->res.offset == buflen); - rv = send_upstream(csdata); + if (csdata->state == MAXROWS_DISCARDING_RESPONSE) + { + rv = send_ok_upstream(csdata); + } + else + { + rv = send_upstream(csdata); + } csdata->state = MAXROWS_EXPECTING_NOTHING; break; @@ -698,18 +701,14 @@ static int handle_expecting_rows(MAXROWS_SESSION_DATA *csdata) csdata->res.offset += packetlen; ++csdata->res.n_rows; - if (csdata->res.n_rows > csdata->instance->config.max_resultset_rows) + if (csdata->state != MAXROWS_DISCARDING_RESPONSE) { - C_DEBUG("max_resultset_rows %lu reached, not returning the result.", csdata->res.n_rows); + if (csdata->res.n_rows > csdata->instance->config.max_resultset_rows) + { + C_DEBUG("max_resultset_rows %lu reached, not returning the result.", csdata->res.n_rows); - /* Just return 0 result set */ - rv = send_ok_upstream(csdata); - csdata->res.offset = buflen; // To abort the loop. - - /* If additional packets from server will come - * they will be "consumed" and not sent to client - */ - csdata->state = MAXROWS_DISCARDING_RESPONSE; + csdata->state = MAXROWS_DISCARDING_RESPONSE; + } } break; } @@ -854,21 +853,3 @@ static int send_ok_upstream(MAXROWS_SESSION_DATA *csdata) return rv; } -/** - * Called when all data from the server should be consumed and not sent. - * - * @param csdata The maxrows session data. - */ -static int handle_discarding_response(MAXROWS_SESSION_DATA *csdata) -{ - ss_dassert(csdata->state == MAXROWS_DISCARDING_RESPONSE); - ss_dassert(csdata->res.data); - - gwbuf_free(csdata->res.data); - csdata->res.data = NULL; - - MXS_INFO("maxrows filter is receiving response packets from the backend in 'discarding_response' state."); - - return 1; -} - From dcfd5b76013ede8a5c2e1b28ac3d0a0e15d1df8e Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Thu, 27 Oct 2016 11:34:01 +0200 Subject: [PATCH 106/215] maxrows now handles SHOW statements and SELECT in prepared statements maxrows now handles SHOW statements and SELECT in prepared statements --- server/modules/filter/maxrows/maxrows.c | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/server/modules/filter/maxrows/maxrows.c b/server/modules/filter/maxrows/maxrows.c index 5a4eec817..2f8748848 100644 --- a/server/modules/filter/maxrows/maxrows.c +++ b/server/modules/filter/maxrows/maxrows.c @@ -292,26 +292,14 @@ static int routeQuery(FILTER *instance, void *sdata, GWBUF *packet) ss_dassert(GWBUF_LENGTH(packet) >= MYSQL_HEADER_LEN + 1); ss_dassert(MYSQL_GET_PACKET_LEN(data) + MYSQL_HEADER_LEN == GWBUF_LENGTH(packet)); - bool use_default = true; - maxrows_response_state_reset(&csdata->res); csdata->state = MAXROWS_IGNORING_RESPONSE; - int rv; - switch ((int)MYSQL_GET_COMMAND(data)) { case MYSQL_COM_QUERY: { - /* Detect the SELECT statement only */ - if (qc_get_operation(packet) == QUERY_OP_SELECT) - { - /* Waiting for a reply: - * Data will be stored in csdata->res->data via - * clientReply routine - */ - csdata->state = MAXROWS_EXPECTING_RESPONSE; - } + csdata->state = MAXROWS_EXPECTING_RESPONSE; break; default: @@ -319,13 +307,8 @@ static int routeQuery(FILTER *instance, void *sdata, GWBUF *packet) } } - if (use_default) - { - C_DEBUG("Maxrows filter is sending data."); - rv = csdata->down.routeQuery(csdata->down.instance, csdata->down.session, packet); - } - - return rv; + C_DEBUG("Maxrows filter is sending data."); + return csdata->down.routeQuery(csdata->down.instance, csdata->down.session, packet); } /** From 6cc7a26582b735ed8545c64258f73d86b6728793 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Thu, 27 Oct 2016 13:54:22 +0300 Subject: [PATCH 107/215] Cache: Add forgotten change --- server/modules/filter/cache/cache.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/modules/filter/cache/cache.c b/server/modules/filter/cache/cache.c index 2e869cf06..8ddb27e8a 100644 --- a/server/modules/filter/cache/cache.c +++ b/server/modules/filter/cache/cache.c @@ -399,7 +399,7 @@ static int routeQuery(FILTER *instance, void *sdata, GWBUF *packet) else { csdata->state = CACHE_EXPECTING_NOTHING; - if (csdata->instance.config & CACHE_DEBUG_DECISIONS) + if (csdata->instance->config.debug & CACHE_DEBUG_DECISIONS) { MXS_NOTICE("Using data from cache."); } @@ -419,7 +419,7 @@ static int routeQuery(FILTER *instance, void *sdata, GWBUF *packet) } else { - if (csdata->instance.config & CACHE_DEBUG_DECISIONS) + if (csdata->instance->config.debug & CACHE_DEBUG_DECISIONS) { MXS_NOTICE("autocommit = %s and transaction state %s => Not using or " "storing to cache.", @@ -470,7 +470,7 @@ static int clientReply(FILTER *instance, void *sdata, GWBUF *data) { if (gwbuf_length(csdata->res.data) > csdata->instance->config.max_resultset_size) { - if (csdata->instance.config & CACHE_DEBUG_DECISIONS) + if (csdata->instance->config.debug & CACHE_DEBUG_DECISIONS) { MXS_NOTICE("Current size %uB of resultset, at least as much " "as maximum allowed size %uKiB. Not caching.", @@ -817,7 +817,7 @@ static int handle_expecting_rows(CACHE_SESSION_DATA *csdata) if (csdata->res.n_rows > csdata->instance->config.max_resultset_rows) { - if (csdata->instance.config & CACHE_DEBUG_DECISIONS) + if (csdata->instance->config.debug & CACHE_DEBUG_DECISIONS) { MXS_NOTICE("Max rows %lu reached, not caching result.", csdata->res.n_rows); } From a390331e421b43a8129804271deaa99e1322276a Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Thu, 27 Oct 2016 14:08:58 +0300 Subject: [PATCH 108/215] Feature defines moved to Feature defines that affect the available functionality or the compile time environment should be defined in a common way for all source. --- include/maxscale/cdefs.h | 3 +++ server/core/gateway.c | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/include/maxscale/cdefs.h b/include/maxscale/cdefs.h index 3b24b55c9..4f163ee1e 100644 --- a/include/maxscale/cdefs.h +++ b/include/maxscale/cdefs.h @@ -33,6 +33,9 @@ # define MXS_END_DECLS #endif +#define _XOPEN_SOURCE 700 +#define OPENSSL_THREAD_DEFINES + /** * Define intended for use with strerror. * diff --git a/server/core/gateway.c b/server/core/gateway.c index 67eb2e9a9..309565bcb 100644 --- a/server/core/gateway.c +++ b/server/core/gateway.c @@ -34,8 +34,8 @@ * 19/01/16 Markus Makela Set cwd to log directory * @endverbatim */ -#define _XOPEN_SOURCE 700 -#define OPENSSL_THREAD_DEFINES + +#include #include #include From 6c80f2bc0a43bf5ada6db2fdeda8a32fbc79cace Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Thu, 27 Oct 2016 14:13:31 +0300 Subject: [PATCH 109/215] Cleanup gateway.c include files - First - Then all system, c-runtime, OS include files in alphabetical order. - Then include files for "3rd-party" software in a loose order of importance. - Then maxscale headers ordered alphabetically. --- server/core/gateway.c | 76 +++++++++++++++++++------------------------ 1 file changed, 34 insertions(+), 42 deletions(-) diff --git a/server/core/gateway.c b/server/core/gateway.c index 309565bcb..b226e00c7 100644 --- a/server/core/gateway.c +++ b/server/core/gateway.c @@ -36,57 +36,43 @@ */ #include -#include - -#include -#if defined(OPENSSL_THREADS) -#define HAVE_OPENSSL_THREADS 1 -#else -#define HAVE_OPENSSL_THREADS 0 -#endif - +#include #include -#include -#include -#include -#include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - #include -#include -#include -#include -#include -#include - +#include +#include +#include +#include #include #include - -#include -#include -#include - -#include - -#include #include -#include -#include -#include +#include +#include +#include +#include +#include #include +#include +#include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #define STRING_BUFFER_SIZE 1024 #define PIDFD_CLOSED -1 @@ -96,6 +82,12 @@ # define _GNU_SOURCE #endif +#if defined(OPENSSL_THREADS) +#define HAVE_OPENSSL_THREADS 1 +#else +#define HAVE_OPENSSL_THREADS 0 +#endif + extern char *program_invocation_name; extern char *program_invocation_short_name; From 1728e2dc2e03e605d843411e1a919b3b20f3af4e Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Thu, 27 Oct 2016 14:50:22 +0300 Subject: [PATCH 110/215] Minor cleanup of gateway.c Pre-parsing of configuration file moved to separate function. To make it slightly easier to see the forest for the trees. --- server/core/gateway.c | 66 +++++++++++++++++++++++++++++-------------- 1 file changed, 45 insertions(+), 21 deletions(-) diff --git a/server/core/gateway.c b/server/core/gateway.c index b226e00c7..2099bdb7b 100644 --- a/server/core/gateway.c +++ b/server/core/gateway.c @@ -184,6 +184,7 @@ static bool change_cwd(); void shutdown_server(); static void log_exit_status(); static bool daemonize(); +static bool sniff_configuration(const char* filepath); /** SSL multi-threading functions and structures */ @@ -1696,32 +1697,12 @@ int main(int argc, char **argv) goto return_main; } - if ((ini_rval = ini_parse(cnf_file_path, cnf_preparser, NULL)) != 0) + if (!sniff_configuration(cnf_file_path)) { - char errorbuffer[STRING_BUFFER_SIZE]; - - if (ini_rval > 0) - { - snprintf(errorbuffer, sizeof(errorbuffer), - "Error: Failed to pre-parse configuration file. Error on line %d.", ini_rval); - } - else if (ini_rval == -1) - { - snprintf(errorbuffer, sizeof(errorbuffer), - "Error: Failed to pre-parse configuration file. Failed to open file."); - } - else - { - snprintf(errorbuffer, sizeof(errorbuffer), - "Error: Failed to pre-parse configuration file. Memory allocation failed."); - } - - print_log_n_stderr(true, true, errorbuffer, errorbuffer, 0); rc = MAXSCALE_BADCONFIG; goto return_main; } - /** Use the cache dir for the mysql folder of the embedded library */ snprintf(mysql_home, PATH_MAX, "%s/mysql", get_cachedir()); mysql_home[PATH_MAX] = '\0'; @@ -2687,3 +2668,46 @@ static bool daemonize(void) } return false; } + +/** + * Sniffs the configuration file, primarily for various directory paths, + * so that certain settings take effect immediately. + * + * @param filepath The path of the configuration file. + * + * @return True, if the sniffing succeeded, false otherwise. + */ +static bool sniff_configuration(const char* filepath) +{ + int rv = ini_parse(filepath, cnf_preparser, NULL); + + if (rv != 0) + { + const char FORMAT_SYNTAX[] = + "Error: Failed to pre-parse configuration file %s. Error on line %d."; + const char FORMAT_OPEN[] = + "Error: Failed to pre-parse configuration file %s. Failed to open file."; + const char FORMAT_MALLOC[] = + "Error: Failed to pre-parse configuration file %s. Memory allocation failed."; + + // We just use the largest one. + char errorbuffer[sizeof(FORMAT_MALLOC) + strlen(filepath) + UINTLEN(abs(rv))]; + + if (rv > 0) + { + snprintf(errorbuffer, sizeof(errorbuffer), FORMAT_SYNTAX, filepath, rv); + } + else if (rv == -1) + { + snprintf(errorbuffer, sizeof(errorbuffer), FORMAT_OPEN, filepath); + } + else + { + snprintf(errorbuffer, sizeof(errorbuffer), FORMAT_MALLOC, filepath); + } + + print_log_n_stderr(true, true, errorbuffer, errorbuffer, 0); + } + + return rv == 0; +} From bbade8b7366e5e54fff4584fce41ce898920d017 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Thu, 27 Oct 2016 15:25:00 +0300 Subject: [PATCH 111/215] PCRE2_CODE_UNIT_WIDTH must not be defined. --- include/maxscale/pcre2.h | 8 +++++--- server/modules/filter/regexfilter/regexfilter.c | 14 +++++++------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/include/maxscale/pcre2.h b/include/maxscale/pcre2.h index 77d7d6976..de28c74aa 100644 --- a/include/maxscale/pcre2.h +++ b/include/maxscale/pcre2.h @@ -27,14 +27,16 @@ #include -#ifndef PCRE2_CODE_UNIT_WIDTH +MXS_BEGIN_DECLS + +#if defined(PCRE2_CODE_UNIT_WIDTH) +#error PCRE2_CODE_UNIT_WIDTH already defined. Do not define, and include . +#else #define PCRE2_CODE_UNIT_WIDTH 8 #endif #include -MXS_BEGIN_DECLS - typedef enum { MXS_PCRE2_MATCH, diff --git a/server/modules/filter/regexfilter/regexfilter.c b/server/modules/filter/regexfilter/regexfilter.c index 21fe8e433..918cc95f0 100644 --- a/server/modules/filter/regexfilter/regexfilter.c +++ b/server/modules/filter/regexfilter/regexfilter.c @@ -11,17 +11,17 @@ * Public License. */ -#define PCRE2_CODE_UNIT_WIDTH 8 +#include +#include #include +#include +#include +#include #include +#include #include #include -#include -#include -#include -#include -#include -#include +#include /** * @file regexfilter.c - a very simple regular expression rewrite filter. From 1be3bb9b8df6e4b68c7391ce3020c4e0ba3ae588 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Thu, 27 Oct 2016 15:44:42 +0300 Subject: [PATCH 112/215] Minor cleanup of config.[h|c] - Add const where applicable. - Cleanup header files. Fix collateral damage (statistics.c) as well. --- include/maxscale/config.h | 31 +++++++-------- server/core/config.c | 84 ++++++++++++++++----------------------- server/core/statistics.c | 3 +- 3 files changed, 52 insertions(+), 66 deletions(-) diff --git a/include/maxscale/config.h b/include/maxscale/config.h index b765c33fb..65c1f8c28 100644 --- a/include/maxscale/config.h +++ b/include/maxscale/config.h @@ -29,10 +29,9 @@ */ #include +#include #include -#include #include -#include MXS_BEGIN_DECLS @@ -130,29 +129,29 @@ typedef struct } GATEWAY_CONF; -char* config_clean_string_list(char* str); -CONFIG_PARAMETER* config_clone_param(CONFIG_PARAMETER* param); +char* config_clean_string_list(const char* str); +CONFIG_PARAMETER* config_clone_param(const CONFIG_PARAMETER* param); void config_enable_feedback_task(void); void config_disable_feedback_task(void); unsigned long config_get_gateway_id(void); GATEWAY_CONF* config_get_global_options(); CONFIG_PARAMETER* config_get_param(CONFIG_PARAMETER* params, const char* name); -config_param_type_t config_get_paramtype(CONFIG_PARAMETER* param); -bool config_get_valint(int* val, - CONFIG_PARAMETER* param, - const char* name, /*< if NULL examine current param only */ +config_param_type_t config_get_paramtype(const CONFIG_PARAMETER* param); +bool config_get_valint(int* val, + const CONFIG_PARAMETER* param, + const char* name, /*< if NULL examine current param only */ config_param_type_t ptype); -bool config_get_valbool(bool* val, - CONFIG_PARAMETER* param, - const char* name, /*< if NULL examine current param only */ +bool config_get_valbool(bool* val, + const CONFIG_PARAMETER* param, + const char* name, /*< if NULL examine current param only */ config_param_type_t ptype); -bool config_get_valtarget(target_t* val, - CONFIG_PARAMETER* param, - const char* name, /*< if NULL examine current param only */ +bool config_get_valtarget(target_t* val, + const CONFIG_PARAMETER* param, + const char* name, /*< if NULL examine current param only */ config_param_type_t ptype); -bool config_load(char *); +bool config_load(const char *); unsigned int config_nbpolls(); -double config_percentage_value(char *str); +double config_percentage_value(const char *str); unsigned int config_pollsleep(); int config_reload(); bool config_set_qualified_param(CONFIG_PARAMETER* param, diff --git a/server/core/config.c b/server/core/config.c index e8c4ef03d..d490c5f75 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -44,38 +44,25 @@ * * @endverbatim */ -#include -#include -#include -#include -#include -#include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include #include -#include #include -#include -#include -#include -#include +#include +#include #include -#include +#include +#include #include +#include #include -#define PCRE2_CODE_UNIT_WIDTH 8 -#include +#include +#include +#include +#include +#include +#include +#include extern int setipaddress(struct in_addr *, char *); static bool process_config_context(CONFIG_CONTEXT *); @@ -105,7 +92,7 @@ int create_new_listener(CONFIG_CONTEXT *obj, bool startnow); int create_new_filter(CONFIG_CONTEXT *obj); int configure_new_service(CONFIG_CONTEXT *context, CONFIG_CONTEXT *obj); -static char *config_file = NULL; +static const char *config_file = NULL; static GATEWAY_CONF gateway; static FEEDBACK_CONF feedback; char *version_string = NULL; @@ -215,7 +202,7 @@ static char *server_params[] = * @param strptr String to clean * @return pointer to a new string or NULL if an error occurred */ -char* config_clean_string_list(char* str) +char* config_clean_string_list(const char* str) { size_t destsize = strlen(str) + 1; char *dest = MXS_MALLOC(destsize); @@ -375,7 +362,7 @@ handler(void *userdata, const char *section, const char *name, const char *value * @return True on success, false on fatal error */ bool -config_load(char *file) +config_load(const char *file) { CONFIG_CONTEXT config = {.object = ""}; int ini_rval; @@ -665,16 +652,16 @@ CONFIG_PARAMETER* config_get_param( } config_param_type_t config_get_paramtype( - CONFIG_PARAMETER* param) + const CONFIG_PARAMETER* param) { return param->qfd_param_type; } bool config_get_valint( - int* val, - CONFIG_PARAMETER* param, - const char* name, /*< if NULL examine current param only */ - config_param_type_t ptype) + int* val, + const CONFIG_PARAMETER* param, + const char* name, /*< if NULL examine current param only */ + config_param_type_t ptype) { bool succp = false;; @@ -708,10 +695,10 @@ return_succp: bool config_get_valbool( - bool* val, - CONFIG_PARAMETER* param, - const char* name, - config_param_type_t ptype) + bool* val, + const CONFIG_PARAMETER* param, + const char* name, + config_param_type_t ptype) { bool succp; @@ -742,10 +729,10 @@ return_succp: bool config_get_valtarget( - target_t* val, - CONFIG_PARAMETER* param, - const char* name, - config_param_type_t ptype) + target_t* val, + const CONFIG_PARAMETER* param, + const char* name, + config_param_type_t ptype) { bool succp; @@ -774,8 +761,7 @@ return_succp: return succp; } -CONFIG_PARAMETER* config_clone_param( - CONFIG_PARAMETER* param) +CONFIG_PARAMETER* config_clone_param(const CONFIG_PARAMETER* param) { CONFIG_PARAMETER* p2; @@ -786,12 +772,12 @@ CONFIG_PARAMETER* config_clone_param( goto return_p2; } memcpy(p2, param, sizeof(CONFIG_PARAMETER)); - p2->name = strndup(param->name, MAX_PARAM_LEN); - p2->value = strndup(param->value, MAX_PARAM_LEN); + p2->name = MXS_STRNDUP_A(param->name, MAX_PARAM_LEN); + p2->value = MXS_STRNDUP_A(param->value, MAX_PARAM_LEN); if (param->qfd_param_type == STRING_TYPE) { - p2->qfd.valstr = strndup(param->qfd.valstr, MAX_PARAM_LEN); + p2->qfd.valstr = MXS_STRNDUP_A(param->qfd.valstr, MAX_PARAM_LEN); } return_p2: @@ -1693,7 +1679,7 @@ bool config_set_qualified_param(CONFIG_PARAMETER* param, switch (type) { case STRING_TYPE: - param->qfd.valstr = strndup((const char *)val, MAX_PARAM_LEN); + param->qfd.valstr = MXS_STRNDUP_A((const char *)val, MAX_PARAM_LEN); succp = true; break; @@ -1760,7 +1746,7 @@ config_truth_value(char *str) * @return String converted to a floating point percentage */ double -config_percentage_value(char *str) +config_percentage_value(const char *str) { double value = 0; diff --git a/server/core/statistics.c b/server/core/statistics.c index c0e1dadfb..1730629a2 100644 --- a/server/core/statistics.c +++ b/server/core/statistics.c @@ -24,9 +24,10 @@ */ #include +#include #include #include -#include +#include #include static int thread_count = 0; From 3c26ea16e280932d741c1d0841f1d586810da5ce Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 28 Oct 2016 12:23:47 +0300 Subject: [PATCH 113/215] Add and use FindSQLite.cmake Added a find_package module for SQLite development libraries. --- CMakeLists.txt | 1 + cmake/FindJansson.cmake | 4 ++-- cmake/FindSQLite.cmake | 23 ++++++++++++++++++ server/modules/authenticator/CMakeLists.txt | 26 +++++++++++++-------- 4 files changed, 42 insertions(+), 12 deletions(-) create mode 100644 cmake/FindSQLite.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 78d70c26e..218964698 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -44,6 +44,7 @@ find_package(LibUUID) find_package(Jansson) find_package(Avro) find_package(GSSAPI) +find_package(SQLite) # Find or build PCRE2 # Read BuildPCRE2 for details about how to add pcre2 as a dependency to a target diff --git a/cmake/FindJansson.cmake b/cmake/FindJansson.cmake index 7116d6f0f..8c2216cac 100644 --- a/cmake/FindJansson.cmake +++ b/cmake/FindJansson.cmake @@ -1,9 +1,9 @@ # This CMake file locates the Jansson libraries and headers # # The following variables are set: -# JANSSON_FOUND - If the Avro C library was found +# JANSSON_FOUND - If the Jansson library was found # JANSSON_LIBRARIES - Path to the static library -# JANSSON_INCLUDE_DIR - Path to Avro headers +# JANSSON_INCLUDE_DIR - Path to Jansson headers find_path(JANSSON_INCLUDE_DIR jansson.h) find_library(JANSSON_LIBRARIES NAMES libjansson.so libjansson.a) diff --git a/cmake/FindSQLite.cmake b/cmake/FindSQLite.cmake new file mode 100644 index 000000000..e5feb5300 --- /dev/null +++ b/cmake/FindSQLite.cmake @@ -0,0 +1,23 @@ +# This CMake file locates the SQLite3 development libraries +# +# The following variables are set: +# SQLITE_FOUND - If the SQLite library was found +# SQLITE_LIBRARIES - Path to the static library +# SQLITE_INCLUDE_DIR - Path to SQLite headers +# SQLITE_VERSION - Library version + +find_path(SQLITE_INCLUDE_DIR sqlite3.h) +find_library(SQLITE_LIBRARIES NAMES libsqlite3.so) + +if (SQLITE_INCLUDE_DIR AND SQLITE_LIBRARIES) + + execute_process(COMMAND grep ".*#define.*SQLITE_VERSION " ${SQLITE_INCLUDE_DIR}/sqlite3.h + COMMAND sed "s/.*\"\\(.*\\)\".*/\\1/" + OUTPUT_VARIABLE SQLITE_VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE) + + message(STATUS "Found SQLite version ${SQLITE_VERSION}: ${SQLITE_LIBRARIES}") + set(SQLITE_FOUND TRUE) +else() + message(STATUS "Could not find SQLite") +endif() diff --git a/server/modules/authenticator/CMakeLists.txt b/server/modules/authenticator/CMakeLists.txt index 8244eef4d..b8294d01a 100644 --- a/server/modules/authenticator/CMakeLists.txt +++ b/server/modules/authenticator/CMakeLists.txt @@ -5,18 +5,24 @@ target_link_libraries(MySQLBackendAuth maxscale-common MySQLCommon) set_target_properties(MySQLBackendAuth PROPERTIES VERSION "1.0.0") install_module(MySQLBackendAuth core) -if (GSSAPI_FOUND) - include_directories(${GSSAPI_INCS}) +if (GSSAPI_FOUND AND SQLITE_FOUND) + if (NOT SQLITE_VERSION VERSION_LESS "3.7.7") + include_directories(${GSSAPI_INCS}) + include_directories(${SQLITE_INCLUDE_DIR}) - add_library(GSSAPIAuth SHARED gssapi_auth.c gssapi_auth_common.c) - target_link_libraries(GSSAPIAuth maxscale-common ${GSSAPI_LIBS} sqlite3 MySQLCommon) - set_target_properties(GSSAPIAuth PROPERTIES VERSION "1.0.0") - install_module(GSSAPIAuth core) + add_library(GSSAPIAuth SHARED gssapi_auth.c gssapi_auth_common.c) + target_link_libraries(GSSAPIAuth maxscale-common ${GSSAPI_LIBS} ${SQLITE_LIBRARIES} MySQLCommon) + set_target_properties(GSSAPIAuth PROPERTIES VERSION "1.0.0") + install_module(GSSAPIAuth core) - add_library(GSSAPIBackendAuth SHARED gssapi_backend_auth.c gssapi_auth_common.c) - target_link_libraries(GSSAPIBackendAuth maxscale-common ${GSSAPI_LIBS} MySQLCommon) - set_target_properties(GSSAPIBackendAuth PROPERTIES VERSION "1.0.0") - install_module(GSSAPIBackendAuth core) + add_library(GSSAPIBackendAuth SHARED gssapi_backend_auth.c gssapi_auth_common.c) + target_link_libraries(GSSAPIBackendAuth maxscale-common ${GSSAPI_LIBS} MySQLCommon) + set_target_properties(GSSAPIBackendAuth PROPERTIES VERSION "1.0.0") + install_module(GSSAPIBackendAuth core) + + else() + message(STATUS "Minimum requires SQLite version for GSSAPIAuth is 3.7.7, current SQLite version is ${SQLITE_VERSION}") + endif() endif() add_library(NullAuthAllow SHARED null_auth_allow.c) From 282ed9f8c292a4c798b1aa4e3f42f40f18a7f428 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 28 Oct 2016 17:08:06 +0300 Subject: [PATCH 114/215] Start MaxScale as the maxscale user with systemd Systemd allows MaxScale to start as the 'maxscale' user and it should be done to completely prevent root access for the maxscale process. --- etc/maxscale.service.in | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/etc/maxscale.service.in b/etc/maxscale.service.in index 84b76bcc3..5e04afa08 100644 --- a/etc/maxscale.service.in +++ b/etc/maxscale.service.in @@ -5,9 +5,18 @@ After=network.target [Service] Type=forking Restart=on-abort -PIDFile=@MAXSCALE_VARDIR@/run/maxscale/maxscale.pid + +# Make sure /var/run/maxscale exists +PermissionsStartOnly=true ExecStartPre=/usr/bin/install -d @MAXSCALE_VARDIR@/run/maxscale -o maxscale -g maxscale -ExecStart=@CMAKE_INSTALL_PREFIX@/@MAXSCALE_BINDIR@/maxscale --user=maxscale + +PIDFile=@MAXSCALE_VARDIR@/run/maxscale/maxscale.pid + +# Use the default user and group +User=maxscale +Group=maxscale + +ExecStart=@CMAKE_INSTALL_PREFIX@/@MAXSCALE_BINDIR@/maxscale TimeoutStartSec=120 LimitNOFILE=65535 From 5a1af968d2ba3416216d44acee69d118f07bec83 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Mon, 31 Oct 2016 08:38:05 +0200 Subject: [PATCH 115/215] Take contiguous input capability into use in filters The filters which require contiguous input now return the RCAP_TYPE_CONTIGUOUS_INPUT capability flag. --- server/modules/filter/ccrfilter/ccrfilter.c | 7 +------ server/modules/filter/mqfilter/mqfilter.c | 7 +------ .../modules/filter/namedserverfilter/namedserverfilter.c | 6 +----- server/modules/filter/qlafilter/qlafilter.c | 6 +----- server/modules/filter/regexfilter/regexfilter.c | 6 +----- server/modules/filter/tee/tee.c | 2 +- server/modules/filter/topfilter/topfilter.c | 6 +----- server/modules/filter/tpmfilter/tpmfilter.c | 6 +----- 8 files changed, 8 insertions(+), 38 deletions(-) diff --git a/server/modules/filter/ccrfilter/ccrfilter.c b/server/modules/filter/ccrfilter/ccrfilter.c index a24391b21..3abda9137 100644 --- a/server/modules/filter/ccrfilter/ccrfilter.c +++ b/server/modules/filter/ccrfilter/ccrfilter.c @@ -333,11 +333,6 @@ routeQuery(FILTER *instance, void *session, GWBUF *queue) if (modutil_is_SQL(queue)) { - if (queue->next) - { - queue = gwbuf_make_contiguous(queue); - } - /** * Not a simple SELECT statement, possibly modifies data. If we're processing a statement * with unknown query type, the safest thing to do is to treat it as a data modifying statement. @@ -422,5 +417,5 @@ diagnostic(FILTER *instance, void *fsession, DCB *dcb) */ static uint64_t getCapabilities(void) { - return RCAP_TYPE_STMT_INPUT; + return RCAP_TYPE_CONTIGUOUS_INPUT; } diff --git a/server/modules/filter/mqfilter/mqfilter.c b/server/modules/filter/mqfilter/mqfilter.c index f56c30669..263626c1b 100644 --- a/server/modules/filter/mqfilter/mqfilter.c +++ b/server/modules/filter/mqfilter/mqfilter.c @@ -1373,11 +1373,6 @@ routeQuery(FILTER *instance, void *session, GWBUF *queue) } } - if (queue->next != NULL) - { - queue = gwbuf_make_contiguous(queue); - } - if (modutil_extract_SQL(queue, &ptr, &length)) { @@ -1713,5 +1708,5 @@ diagnostic(FILTER *instance, void *fsession, DCB *dcb) */ static uint64_t getCapabilities(void) { - return RCAP_TYPE_STMT_INPUT; + return RCAP_TYPE_CONTIGUOUS_INPUT; } diff --git a/server/modules/filter/namedserverfilter/namedserverfilter.c b/server/modules/filter/namedserverfilter/namedserverfilter.c index c8737befe..788a560d7 100644 --- a/server/modules/filter/namedserverfilter/namedserverfilter.c +++ b/server/modules/filter/namedserverfilter/namedserverfilter.c @@ -349,10 +349,6 @@ routeQuery(FILTER *instance, void *session, GWBUF *queue) if (modutil_is_SQL(queue) && my_session->active) { - if (queue->next != NULL) - { - queue = gwbuf_make_contiguous(queue); - } if ((sql = modutil_get_SQL(queue)) != NULL) { if (regexec(&my_instance->re, sql, 0, NULL, 0) == 0) @@ -420,5 +416,5 @@ diagnostic(FILTER *instance, void *fsession, DCB *dcb) */ static uint64_t getCapabilities(void) { - return RCAP_TYPE_STMT_INPUT; + return RCAP_TYPE_CONTIGUOUS_INPUT; } diff --git a/server/modules/filter/qlafilter/qlafilter.c b/server/modules/filter/qlafilter/qlafilter.c index eab3bae54..420e63328 100644 --- a/server/modules/filter/qlafilter/qlafilter.c +++ b/server/modules/filter/qlafilter/qlafilter.c @@ -447,10 +447,6 @@ routeQuery(FILTER *instance, void *session, GWBUF *queue) if (my_session->active) { - if (queue->next != NULL) - { - queue = gwbuf_make_contiguous(queue); - } if ((ptr = modutil_get_SQL(queue)) != NULL) { if ((my_instance->match == NULL || @@ -524,5 +520,5 @@ diagnostic(FILTER *instance, void *fsession, DCB *dcb) */ static uint64_t getCapabilities(void) { - return RCAP_TYPE_STMT_INPUT; + return RCAP_TYPE_CONTIGUOUS_INPUT; } diff --git a/server/modules/filter/regexfilter/regexfilter.c b/server/modules/filter/regexfilter/regexfilter.c index 918cc95f0..21100fa55 100644 --- a/server/modules/filter/regexfilter/regexfilter.c +++ b/server/modules/filter/regexfilter/regexfilter.c @@ -394,10 +394,6 @@ routeQuery(FILTER *instance, void *session, GWBUF *queue) if (my_session->active && modutil_is_SQL(queue)) { - if (queue->next != NULL) - { - queue = gwbuf_make_contiguous(queue); - } if ((sql = modutil_get_SQL(queue)) != NULL) { newsql = regex_replace(sql, @@ -555,5 +551,5 @@ void log_nomatch(REGEX_INSTANCE* inst, char* re, char* old) */ static uint64_t getCapabilities(void) { - return RCAP_TYPE_STMT_INPUT; + return RCAP_TYPE_CONTIGUOUS_INPUT; } diff --git a/server/modules/filter/tee/tee.c b/server/modules/filter/tee/tee.c index 355c420b0..c3dd9db19 100644 --- a/server/modules/filter/tee/tee.c +++ b/server/modules/filter/tee/tee.c @@ -1133,7 +1133,7 @@ diagnostic(FILTER *instance, void *fsession, DCB *dcb) */ static uint64_t getCapabilities(void) { - return RCAP_TYPE_STMT_INPUT; + return RCAP_TYPE_CONTIGUOUS_INPUT; } /** diff --git a/server/modules/filter/topfilter/topfilter.c b/server/modules/filter/topfilter/topfilter.c index 21f865345..9b5d38e4f 100644 --- a/server/modules/filter/topfilter/topfilter.c +++ b/server/modules/filter/topfilter/topfilter.c @@ -528,10 +528,6 @@ routeQuery(FILTER *instance, void *session, GWBUF *queue) if (my_session->active) { - if (queue->next != NULL) - { - queue = gwbuf_make_contiguous(queue); - } if ((ptr = modutil_get_SQL(queue)) != NULL) { if ((my_instance->match == NULL || @@ -692,5 +688,5 @@ diagnostic(FILTER *instance, void *fsession, DCB *dcb) */ static uint64_t getCapabilities(void) { - return RCAP_TYPE_STMT_INPUT; + return RCAP_TYPE_CONTIGUOUS_INPUT; } diff --git a/server/modules/filter/tpmfilter/tpmfilter.c b/server/modules/filter/tpmfilter/tpmfilter.c index 5811b74fe..da2c42811 100644 --- a/server/modules/filter/tpmfilter/tpmfilter.c +++ b/server/modules/filter/tpmfilter/tpmfilter.c @@ -394,10 +394,6 @@ routeQuery(FILTER *instance, void *session, GWBUF *queue) if (my_session->active) { - if (queue->next != NULL) - { - queue = gwbuf_make_contiguous(queue); - } if ((ptr = modutil_get_SQL(queue)) != NULL) { my_session->query_end = false; @@ -567,5 +563,5 @@ diagnostic(FILTER *instance, void *fsession, DCB *dcb) */ static uint64_t getCapabilities(void) { - return RCAP_TYPE_STMT_INPUT; + return RCAP_TYPE_CONTIGUOUS_INPUT; } From ab372b291b73a12d51d957508bb97f5ab2c79a2a Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Fri, 28 Oct 2016 15:05:06 +0300 Subject: [PATCH 116/215] MXS-304: Minor refactoring of config.c The loading of a single file refactored out into a separate function. To be used later when multiple files are loaded and processed. --- server/core/config.c | 83 ++++++++++++++++++++++++++------------------ 1 file changed, 49 insertions(+), 34 deletions(-) diff --git a/server/core/config.c b/server/core/config.c index d490c5f75..94320f8d1 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -268,7 +268,7 @@ char* config_clean_string_list(const char* str) * @return zero on error */ static int -handler(void *userdata, const char *section, const char *name, const char *value) +ini_handler(void *userdata, const char *section, const char *name, const char *value) { CONFIG_CONTEXT *cntxt = (CONFIG_CONTEXT *)userdata; CONFIG_CONTEXT *ptr = cntxt; @@ -352,6 +352,47 @@ handler(void *userdata, const char *section, const char *name, const char *value return 1; } +/** + * Load single configuration file. + * + * @param file The file to load. + * @param context The context used when parsing. + * + * @return True if the file could be parsed, false otherwise. + */ +static bool config_load_single_file(const char* file, CONFIG_CONTEXT* context) +{ + int rval = -1; + + if (!config_has_duplicate_sections(file)) + { + if ((rval = ini_parse(file, ini_handler, context)) != 0) + { + char errorbuffer[1024 + 1]; + + if (rval > 0) + { + snprintf(errorbuffer, sizeof(errorbuffer), + "Failed to parse configuration file. Error on line %d.", rval); + } + else if (rval == -1) + { + snprintf(errorbuffer, sizeof(errorbuffer), + "Failed to parse configuration file. Failed to open file."); + } + else + { + snprintf(errorbuffer, sizeof(errorbuffer), + "Failed to parse configuration file. Memory allocation failed."); + } + + MXS_ERROR("%s", errorbuffer); + } + } + + return rval == 0; +} + /** * @brief Load the configuration file for the MaxScale * @@ -364,15 +405,8 @@ handler(void *userdata, const char *section, const char *name, const char *value bool config_load(const char *file) { - CONFIG_CONTEXT config = {.object = ""}; - int ini_rval; bool rval = false; - if (config_has_duplicate_sections(file)) - { - return false; - } - /* Temporary - should use configuration values and test return value (bool) */ dcb_pre_alloc(1000); session_pre_alloc(250); @@ -380,35 +414,16 @@ config_load(const char *file) global_defaults(); feedback_defaults(); - if ((ini_rval = ini_parse(file, handler, &config)) != 0) + CONFIG_CONTEXT config = {.object = ""}; + + if (config_load_single_file(file, &config)) { - char errorbuffer[1024 + 1]; + config_file = file; - if (ini_rval > 0) + if (check_config_objects(config.next) && process_config_context(config.next)) { - snprintf(errorbuffer, sizeof(errorbuffer), - "Error: Failed to parse configuration file. Error on line %d.", ini_rval); + rval = true; } - else if (ini_rval == -1) - { - snprintf(errorbuffer, sizeof(errorbuffer), - "Error: Failed to parse configuration file. Failed to open file."); - } - else - { - snprintf(errorbuffer, sizeof(errorbuffer), - "Error: Failed to parse configuration file. Memory allocation failed."); - } - - MXS_ERROR("%s", errorbuffer); - return 0; - } - - config_file = file; - - if (check_config_objects(config.next) && process_config_context(config.next)) - { - rval = true; } free_config_context(config.next); @@ -446,7 +461,7 @@ config_reload() config.object = ""; config.next = NULL; - if (ini_parse(config_file, handler, &config) < 0) + if (ini_parse(config_file, ini_handler, &config) < 0) { return 0; } From daf6f06138591621172d77611ad7ae22f6ff4464 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Sun, 30 Oct 2016 19:23:03 +0200 Subject: [PATCH 117/215] MXS-304: Support nested configuration files Given a config file "config.cnf", we look for the directory "config.cnf.d" and recursively in that hierarhcy load all files whose suffix is ".cnf"; other files are ignored. Currently duplicate sections are checked on a file by file basis. That will be changed so that duplicate sections are not allowed across all the files. --- server/core/config.c | 127 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 118 insertions(+), 9 deletions(-) diff --git a/server/core/config.c b/server/core/config.c index 94320f8d1..fa080df3d 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -46,6 +46,7 @@ */ #include #include +#include #include #include #include @@ -62,6 +63,7 @@ #include #include #include +#include #include extern int setipaddress(struct in_addr *, char *); @@ -373,37 +375,108 @@ static bool config_load_single_file(const char* file, CONFIG_CONTEXT* context) if (rval > 0) { snprintf(errorbuffer, sizeof(errorbuffer), - "Failed to parse configuration file. Error on line %d.", rval); + "Failed to parse configuration file %s. Error on line %d.", file, rval); } else if (rval == -1) { snprintf(errorbuffer, sizeof(errorbuffer), - "Failed to parse configuration file. Failed to open file."); + "Failed to parse configuration file %s. Could not open file.", file); } else { snprintf(errorbuffer, sizeof(errorbuffer), - "Failed to parse configuration file. Memory allocation failed."); + "Failed to parse configuration file %s. Memory allocation failed.", file); } MXS_ERROR("%s", errorbuffer); } } + if (rval == 0) + { + MXS_NOTICE("Loaded %s.", file); + } + return rval == 0; } +/** + * The current parsing contex, must be managed explicitly since the ftw callback + * can not have user data. + */ +static CONFIG_CONTEXT *current_context; + +/** + * The nftw callback. + * + * @see man ftw + */ +int config_cb(const char* fpath, const struct stat *sb, int typeflag, struct FTW *ftwbuf) +{ + int rval = 0; + + if (typeflag == FTW_F) // We are only interested in files, + { + const char* filename = fpath + ftwbuf->base; + const char* dot = strrchr(filename, '.'); + + if (dot) // that must have a suffix, + { + const char* suffix = dot + 1; + + if (strcmp(suffix, "cnf") == 0) // that is ".cnf". + { + if (!config_load_single_file(fpath, current_context)) + { + rval = -1; + } + } + } + } + + return rval; +} + +/** + * Loads all configuration files in a directory hierarchy. + * + * Only files with the suffix ".cnf" are considered to be configuration files. + * + * @param dir The directory. + * @param context The configuration context. + * + * @return True, if all configuration files in the directory hierarchy could be loaded, + * otherwise false. + */ +static bool config_load_dir(const char *dir, CONFIG_CONTEXT *context) +{ + // Since there is no way to pass userdata to the callback, we need to store + // the current context into a static variable. Consequently, we need lock. + // Should not matter since config_load() is called once at startup. + static SPINLOCK lock = SPINLOCK_INIT; + + int nopenfd = 5; // Maximum concurrently opened directory descriptors + + spinlock_acquire(&lock); + current_context = context; + int rv = nftw(dir, config_cb, nopenfd, FTW_PHYS); + current_context = NULL; + spinlock_release(&lock); + + return rv == 0; +} + /** * @brief Load the configuration file for the MaxScale * * This function will parse the configuration file, check for duplicate sections, * validate the module parameters and finally turn it into a set of objects. * - * @param file The filename of the configuration file + * @param filename The filename of the configuration file * @return True on success, false on fatal error */ bool -config_load(const char *file) +config_load(const char *filename) { bool rval = false; @@ -416,13 +489,49 @@ config_load(const char *file) CONFIG_CONTEXT config = {.object = ""}; - if (config_load_single_file(file, &config)) + if (config_load_single_file(filename, &config)) { - config_file = file; + const char DIR_SUFFIX[] = ".d"; - if (check_config_objects(config.next) && process_config_context(config.next)) + char dir[strlen(filename) + sizeof(DIR_SUFFIX)]; + strcpy(dir, filename); + strcat(dir, DIR_SUFFIX); + + rval = true; + + struct stat st; + if (stat(dir, &st) == -1) { - rval = true; + if (errno == ENOENT) + { + MXS_NOTICE("%s does not exist, not reading.", dir); + } + else + { + char errbuf[MXS_STRERROR_BUFLEN]; + MXS_WARNING("Could not access %s, not reading: %s", + dir, strerror_r(errno, errbuf, sizeof(errbuf))); + } + } + else + { + if (S_ISDIR(st.st_mode)) + { + rval = config_load_dir(dir, &config); + } + else + { + MXS_WARNING("%s exists, but it is not a directory. Ignoring.", dir); + } + } + + if (rval) + { + if (check_config_objects(config.next) && process_config_context(config.next)) + { + config_file = filename; + rval = true; + } } } From 2116c1549c0b30cdb1eeb740869279f1dcb81892 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Sun, 30 Oct 2016 21:29:03 +0200 Subject: [PATCH 118/215] MXS-304: Refactor duplicate section detection Preparatory work for being able to check duplicate sections across multiple files. Data needed for the checking is held in a separate object. --- server/core/config.c | 92 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 75 insertions(+), 17 deletions(-) diff --git a/server/core/config.c b/server/core/config.c index fa080df3d..3346d4886 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -66,6 +66,16 @@ #include #include +typedef struct duplicate_context +{ + HASHTABLE *hash; + pcre2_code *re; + pcre2_match_data *mdata; +} DUPLICATE_CONTEXT; + +static bool duplicate_context_init(DUPLICATE_CONTEXT* context); +static void duplicate_context_finish(DUPLICATE_CONTEXT* context); + extern int setipaddress(struct in_addr *, char *); static bool process_config_context(CONFIG_CONTEXT *); static int process_config_update(CONFIG_CONTEXT *); @@ -198,6 +208,61 @@ static char *server_params[] = NULL }; +/** + * Initialize the context object used for tracking duplicate sections. + * + * @param context The context object to be initialized. + * + * @return True, if the object could be initialized. + */ +static bool duplicate_context_init(DUPLICATE_CONTEXT* context) +{ + bool rv = false; + + const int table_size = 10; + HASHTABLE *hash = hashtable_alloc(table_size, hashtable_item_strhash, hashtable_item_strcmp); + int errcode; + PCRE2_SIZE erroffset; + pcre2_code *re = pcre2_compile((PCRE2_SPTR) "^\\s*\\[(.+)\\]\\s*$", PCRE2_ZERO_TERMINATED, + 0, &errcode, &erroffset, NULL); + pcre2_match_data *mdata = NULL; + + if (hash && re && (mdata = pcre2_match_data_create_from_pattern(re, NULL))) + { + hashtable_memory_fns(hash, hashtable_item_strdup, NULL, hashtable_item_free, NULL); + + context->hash = hash; + context->re = re; + context->mdata = mdata; + rv = true; + } + else + { + pcre2_match_data_free(mdata); + pcre2_code_free(re); + hashtable_free(hash); + } + + return rv; +} + +/** + * Finalize the context object used for tracking duplicate sections. + * + * @param context The context object to be initialized. + */ +static void duplicate_context_finish(DUPLICATE_CONTEXT* context) +{ + pcre2_match_data_free(context->mdata); + pcre2_code_free(context->re); + hashtable_free(context->hash); + + context->mdata = NULL; + context->re = NULL; + context->hash = NULL; +} + + /** * Remove extra commas and whitespace from a string. This string is interpreted * as a list of string values separated by commas. @@ -2209,40 +2274,34 @@ GATEWAY_CONF* config_get_global_options() bool config_has_duplicate_sections(const char* config) { bool rval = false; - const int table_size = 10; - int errcode; - PCRE2_SIZE erroffset; - HASHTABLE *hash = hashtable_alloc(table_size, hashtable_item_strhash, hashtable_item_strcmp); - pcre2_code *re = pcre2_compile((PCRE2_SPTR) "^\\s*\\[(.+)\\]\\s*$", PCRE2_ZERO_TERMINATED, - 0, &errcode, &erroffset, NULL); - pcre2_match_data *mdata = NULL; + DUPLICATE_CONTEXT context; + int size = 1024; char *buffer = MXS_MALLOC(size * sizeof(char)); - if (buffer && hash && re && (mdata = pcre2_match_data_create_from_pattern(re, NULL))) + if (buffer && duplicate_context_init(&context)) { - hashtable_memory_fns(hash, hashtable_item_strdup, NULL, hashtable_item_free, NULL); FILE* file = fopen(config, "r"); if (file) { while (maxscale_getline(&buffer, &size, file) > 0) { - if (pcre2_match(re, (PCRE2_SPTR) buffer, + if (pcre2_match(context.re, (PCRE2_SPTR) buffer, PCRE2_ZERO_TERMINATED, 0, 0, - mdata, NULL) > 0) + context.mdata, NULL) > 0) { /** * Neither of the PCRE2 calls will fail since we know the pattern * beforehand and we allocate enough memory from the stack */ PCRE2_SIZE len; - pcre2_substring_length_bynumber(mdata, 1, &len); + pcre2_substring_length_bynumber(context.mdata, 1, &len); len += 1; /** one for the null terminator */ PCRE2_UCHAR section[len]; - pcre2_substring_copy_bynumber(mdata, 1, section, &len); + pcre2_substring_copy_bynumber(context.mdata, 1, section, &len); - if (hashtable_add(hash, section, "") == 0) + if (hashtable_add(context.hash, section, "") == 0) { MXS_ERROR("Duplicate section found: %s", section); rval = true; @@ -2258,6 +2317,8 @@ bool config_has_duplicate_sections(const char* config) strerror_r(errno, errbuf, sizeof(errbuf))); rval = true; } + + duplicate_context_finish(&context); } else { @@ -2266,9 +2327,6 @@ bool config_has_duplicate_sections(const char* config) rval = true; } - hashtable_free(hash); - pcre2_code_free(re); - pcre2_match_data_free(mdata); MXS_FREE(buffer); return rval; } From f71f41b0b87c00b387581adb86167458bb9b47b1 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Sun, 30 Oct 2016 21:50:36 +0200 Subject: [PATCH 119/215] MXS-304: Check duplicate sections across all files --- server/core/config.c | 167 +++++++++++++++++++++++++------------------ 1 file changed, 97 insertions(+), 70 deletions(-) diff --git a/server/core/config.c b/server/core/config.c index 3346d4886..54de05b16 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -96,7 +96,7 @@ int config_get_ifaddr(unsigned char *output); static int config_get_release_string(char* release); FEEDBACK_CONF *config_get_feedback_data(); void config_add_param(CONFIG_CONTEXT*, char*, char*); -bool config_has_duplicate_sections(const char* config); +bool config_has_duplicate_sections(const char* config, DUPLICATE_CONTEXT* context); int create_new_service(CONFIG_CONTEXT *obj); int create_new_server(CONFIG_CONTEXT *obj); int create_new_monitor(CONFIG_CONTEXT *context, CONFIG_CONTEXT *obj, HASHTABLE* monitorhash); @@ -423,17 +423,24 @@ ini_handler(void *userdata, const char *section, const char *name, const char *v * Load single configuration file. * * @param file The file to load. - * @param context The context used when parsing. + * @param dcontext The context object used when tracking duplicate sections. + * @param ccontext The context object used when parsing. * * @return True if the file could be parsed, false otherwise. */ -static bool config_load_single_file(const char* file, CONFIG_CONTEXT* context) +static bool config_load_single_file(const char* file, + DUPLICATE_CONTEXT* dcontext, + CONFIG_CONTEXT* ccontext) { int rval = -1; - if (!config_has_duplicate_sections(file)) + // With multiple configuration files being loaded, we need to log the file + // currently being loaded so that the context is clear in case of errors. + MXS_NOTICE("Loading %s.", file); + + if (!config_has_duplicate_sections(file, dcontext)) { - if ((rval = ini_parse(file, ini_handler, context)) != 0) + if ((rval = ini_parse(file, ini_handler, ccontext)) != 0) { char errorbuffer[1024 + 1]; @@ -457,19 +464,15 @@ static bool config_load_single_file(const char* file, CONFIG_CONTEXT* context) } } - if (rval == 0) - { - MXS_NOTICE("Loaded %s.", file); - } - return rval == 0; } /** - * The current parsing contex, must be managed explicitly since the ftw callback + * The current parsing contexts must be managed explicitly since the ftw callback * can not have user data. */ -static CONFIG_CONTEXT *current_context; +static CONFIG_CONTEXT *current_ccontext; +static DUPLICATE_CONTEXT *current_dcontext; /** * The nftw callback. @@ -491,7 +494,10 @@ int config_cb(const char* fpath, const struct stat *sb, int typeflag, struct FTW if (strcmp(suffix, "cnf") == 0) // that is ".cnf". { - if (!config_load_single_file(fpath, current_context)) + ss_dassert(current_dcontext); + ss_dassert(current_ccontext); + + if (!config_load_single_file(fpath, current_dcontext, current_ccontext)) { rval = -1; } @@ -507,13 +513,14 @@ int config_cb(const char* fpath, const struct stat *sb, int typeflag, struct FTW * * Only files with the suffix ".cnf" are considered to be configuration files. * - * @param dir The directory. - * @param context The configuration context. + * @param dir The directory. + * @param dcontext The duplicate section context. + * @param ccontext The configuration context. * * @return True, if all configuration files in the directory hierarchy could be loaded, * otherwise false. */ -static bool config_load_dir(const char *dir, CONFIG_CONTEXT *context) +static bool config_load_dir(const char *dir, DUPLICATE_CONTEXT *dcontext, CONFIG_CONTEXT *ccontext) { // Since there is no way to pass userdata to the callback, we need to store // the current context into a static variable. Consequently, we need lock. @@ -523,9 +530,11 @@ static bool config_load_dir(const char *dir, CONFIG_CONTEXT *context) int nopenfd = 5; // Maximum concurrently opened directory descriptors spinlock_acquire(&lock); - current_context = context; + current_dcontext = dcontext; + current_ccontext = ccontext; int rv = nftw(dir, config_cb, nopenfd, FTW_PHYS); - current_context = NULL; + current_ccontext = NULL; + current_dcontext = NULL; spinlock_release(&lock); return rv == 0; @@ -552,55 +561,62 @@ config_load(const char *filename) global_defaults(); feedback_defaults(); - CONFIG_CONTEXT config = {.object = ""}; + DUPLICATE_CONTEXT dcontext; - if (config_load_single_file(filename, &config)) + if (duplicate_context_init(&dcontext)) { - const char DIR_SUFFIX[] = ".d"; + CONFIG_CONTEXT ccontext = {.object = ""}; - char dir[strlen(filename) + sizeof(DIR_SUFFIX)]; - strcpy(dir, filename); - strcat(dir, DIR_SUFFIX); - - rval = true; - - struct stat st; - if (stat(dir, &st) == -1) + if (config_load_single_file(filename, &dcontext, &ccontext)) { - if (errno == ENOENT) + const char DIR_SUFFIX[] = ".d"; + + char dir[strlen(filename) + sizeof(DIR_SUFFIX)]; + strcpy(dir, filename); + strcat(dir, DIR_SUFFIX); + + rval = true; + + struct stat st; + if (stat(dir, &st) == -1) { - MXS_NOTICE("%s does not exist, not reading.", dir); + if (errno == ENOENT) + { + MXS_NOTICE("%s does not exist, not reading.", dir); + } + else + { + char errbuf[MXS_STRERROR_BUFLEN]; + MXS_WARNING("Could not access %s, not reading: %s", + dir, strerror_r(errno, errbuf, sizeof(errbuf))); + } } else { - char errbuf[MXS_STRERROR_BUFLEN]; - MXS_WARNING("Could not access %s, not reading: %s", - dir, strerror_r(errno, errbuf, sizeof(errbuf))); + if (S_ISDIR(st.st_mode)) + { + rval = config_load_dir(dir, &dcontext, &ccontext); + } + else + { + MXS_WARNING("%s exists, but it is not a directory. Ignoring.", dir); + } } - } - else - { - if (S_ISDIR(st.st_mode)) + + if (rval) { - rval = config_load_dir(dir, &config); - } - else - { - MXS_WARNING("%s exists, but it is not a directory. Ignoring.", dir); + if (check_config_objects(ccontext.next) && process_config_context(ccontext.next)) + { + config_file = filename; + rval = true; + } } } - if (rval) - { - if (check_config_objects(config.next) && process_config_context(config.next)) - { - config_file = filename; - rval = true; - } - } + free_config_context(ccontext.next); + + duplicate_context_finish(&dcontext); } - - free_config_context(config.next); return rval; } @@ -612,15 +628,24 @@ config_load(const char *filename) int config_reload() { - CONFIG_CONTEXT config; - int rval; + int rval = 0; - if (!config_file) + if (config_file) { return 0; } - if (config_has_duplicate_sections(config_file)) + DUPLICATE_CONTEXT dcontext; + + if (!duplicate_context_init(&dcontext)) + { + return 0; + } + + bool duplicates = config_has_duplicate_sections(config_file, &dcontext); + duplicate_context_finish(&dcontext); + + if (duplicates) { return 0; } @@ -632,6 +657,7 @@ config_reload() global_defaults(); + CONFIG_CONTEXT config; config.object = ""; config.next = NULL; @@ -2268,40 +2294,43 @@ GATEWAY_CONF* config_get_global_options() /** * Check if sections are defined multiple times in the configuration file. - * @param config Path to the configuration file + * + * @param filename Path to the configuration file + * @param context The context object used for tracking the duplication + * section information. + * * @return True if duplicate sections were found or an error occurred */ -bool config_has_duplicate_sections(const char* config) +bool config_has_duplicate_sections(const char* filename, DUPLICATE_CONTEXT* context) { bool rval = false; - DUPLICATE_CONTEXT context; int size = 1024; char *buffer = MXS_MALLOC(size * sizeof(char)); - if (buffer && duplicate_context_init(&context)) + if (buffer) { - FILE* file = fopen(config, "r"); + FILE* file = fopen(filename, "r"); if (file) { while (maxscale_getline(&buffer, &size, file) > 0) { - if (pcre2_match(context.re, (PCRE2_SPTR) buffer, + if (pcre2_match(context->re, (PCRE2_SPTR) buffer, PCRE2_ZERO_TERMINATED, 0, 0, - context.mdata, NULL) > 0) + context->mdata, NULL) > 0) { /** * Neither of the PCRE2 calls will fail since we know the pattern * beforehand and we allocate enough memory from the stack */ PCRE2_SIZE len; - pcre2_substring_length_bynumber(context.mdata, 1, &len); + pcre2_substring_length_bynumber(context->mdata, 1, &len); len += 1; /** one for the null terminator */ PCRE2_UCHAR section[len]; - pcre2_substring_copy_bynumber(context.mdata, 1, section, &len); + pcre2_substring_copy_bynumber(context->mdata, 1, section, &len); - if (hashtable_add(context.hash, section, "") == 0) + if (hashtable_add(context->hash, section, "") == 0) { MXS_ERROR("Duplicate section found: %s", section); rval = true; @@ -2313,12 +2342,10 @@ bool config_has_duplicate_sections(const char* config) else { char errbuf[MXS_STRERROR_BUFLEN]; - MXS_ERROR("Failed to open file '%s': %s", config, + MXS_ERROR("Failed to open file '%s': %s", filename, strerror_r(errno, errbuf, sizeof(errbuf))); rval = true; } - - duplicate_context_finish(&context); } else { From 60f0c2ecaa0722a4f3e3da30a08a348c246eba83 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Mon, 31 Oct 2016 09:53:21 +0200 Subject: [PATCH 120/215] MXS-304: Update documentation. --- .../Getting-Started/Configuration-Guide.md | 24 ++++++++++++++----- .../MaxScale-2.1.0-Release-Notes.md | 14 ++++++++++- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/Documentation/Getting-Started/Configuration-Guide.md b/Documentation/Getting-Started/Configuration-Guide.md index 007f96110..bf0b33fde 100644 --- a/Documentation/Getting-Started/Configuration-Guide.md +++ b/Documentation/Getting-Started/Configuration-Guide.md @@ -41,16 +41,28 @@ connection failover| When a connection currently being used between MariaDB MaxS ## Configuration -The MariaDB MaxScale configuration is read from a file that MariaDB MaxScale will look for -in a number of places. +The MariaDB MaxScale configuration is read from a file that MariaDB MaxScale +will look for in the following places: -1. Location given with the --configdir= command line argument +1. By default, the file `maxscale.cnf` in the directory `/etc` +1. The location given with the `--configdir=` command line argument. -2. MariaDB MaxScale will look for a configuration file called `maxscale.cnf` in the directory `/etc/maxscale.cnf` +MariaDB MaxScale will further look for a directory with the same name as the +configuration file, followed by `.d` (for instance `/etc/maxscale.cnf.d`) and +recursively read all files, having a `.cnf` suffix, it finds in the directory +hierarchy. All other files will be ignored. -An explicit path to a configuration file can be passed by using the `-f` option to MariaDB MaxScale. +There are no restrictions on how different configuration sections are arranged, +but the strong suggestion is to place global settings into the configuration +file MariaDB MaxScale is invoked with, and then, if deemed necessary, create +separate configuration files for _servers_, _filters_, etc. -The configuration file itself is based on the ".ini" file format and consists of various sections that are used to build the configuration; these sections define services, servers, listeners, monitors and global settings. Parameters, which expect a comma-separated list of values can be defined on multiple lines. The following is an example of a multi-line definition. +The configuration file itself is based on the [.ini](https://en.wikipedia.org/wiki/INI_file) +file format and consists of various sections that are used to build the +configuration; these sections define services, servers, listeners, monitors and +global settings. Parameters, which expect a comma-separated list of values can +be defined on multiple lines. The following is an example of a multi-line +definition. ``` [MyService] diff --git a/Documentation/Release-Notes/MaxScale-2.1.0-Release-Notes.md b/Documentation/Release-Notes/MaxScale-2.1.0-Release-Notes.md index ddd62b464..7ffdace44 100644 --- a/Documentation/Release-Notes/MaxScale-2.1.0-Release-Notes.md +++ b/Documentation/Release-Notes/MaxScale-2.1.0-Release-Notes.md @@ -8,7 +8,19 @@ release 2.0.X. For any problems you encounter, please consider submitting a bug report at [Jira](https://jira.mariadb.org). -## Changes Features +## Changed Features + +### Configuration Files + +From 2.1.0 onwards MariaDB MaxScale supports hierarchical configuration +files. When invoked with a configuration file, e.g. `maxscale.cnf`, MariaDB +MaxScale looks for a directory `maxscale.cnf.d` in the same directory as the +configuration file, and reads all `.cnf` files it finds in that directory +hierarchy. All other files will be ignored. + +Please see the +[Configuration Guide](../Getting-Started/Configuration-Guide.md#configuration) +for details. ### Logging From 8c20ad5cd08c1ef0863543da2b41c996e8edd365 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Mon, 31 Oct 2016 12:39:30 +0200 Subject: [PATCH 121/215] MXS-304: Support hierarchical reload 'reload config' now also processes all configuration files of a hierarchy. --- include/maxscale/config.h | 2 +- server/core/config.c | 108 +++++++++++++++++++------------------- 2 files changed, 56 insertions(+), 54 deletions(-) diff --git a/include/maxscale/config.h b/include/maxscale/config.h index 65c1f8c28..8e616aae4 100644 --- a/include/maxscale/config.h +++ b/include/maxscale/config.h @@ -153,7 +153,7 @@ bool config_load(const char *); unsigned int config_nbpolls(); double config_percentage_value(const char *str); unsigned int config_pollsleep(); -int config_reload(); +bool config_reload(); bool config_set_qualified_param(CONFIG_PARAMETER* param, void* val, config_param_type_t type); diff --git a/server/core/config.c b/server/core/config.c index 54de05b16..f85d685d6 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -77,8 +77,8 @@ static bool duplicate_context_init(DUPLICATE_CONTEXT* context); static void duplicate_context_finish(DUPLICATE_CONTEXT* context); extern int setipaddress(struct in_addr *, char *); -static bool process_config_context(CONFIG_CONTEXT *); -static int process_config_update(CONFIG_CONTEXT *); +static bool process_config_context(CONFIG_CONTEXT *); +static bool process_config_update(CONFIG_CONTEXT *); static void free_config_context(CONFIG_CONTEXT *); static char *config_get_value(CONFIG_PARAMETER *, const char *); static char *config_get_password(CONFIG_PARAMETER *); @@ -541,26 +541,22 @@ static bool config_load_dir(const char *dir, DUPLICATE_CONTEXT *dcontext, CONFIG } /** - * @brief Load the configuration file for the MaxScale + * @brief Load the specified configuration file for MaxScale * * This function will parse the configuration file, check for duplicate sections, * validate the module parameters and finally turn it into a set of objects. * - * @param filename The filename of the configuration file + * @param filename The filename of the configuration file + * @param process_config The function using which the successfully loaded + * configuration should be processed. + * * @return True on success, false on fatal error */ -bool -config_load(const char *filename) +static bool +config_load_and_process(const char* filename, bool (*process_config)(CONFIG_CONTEXT*)) { bool rval = false; - /* Temporary - should use configuration values and test return value (bool) */ - dcb_pre_alloc(1000); - session_pre_alloc(250); - - global_defaults(); - feedback_defaults(); - DUPLICATE_CONTEXT dcontext; if (duplicate_context_init(&dcontext)) @@ -605,9 +601,8 @@ config_load(const char *filename) if (rval) { - if (check_config_objects(ccontext.next) && process_config_context(ccontext.next)) + if (check_config_objects(ccontext.next) && process_config(ccontext.next)) { - config_file = filename; rval = true; } } @@ -620,55 +615,62 @@ config_load(const char *filename) return rval; } +/** + * @brief Load the configuration file for the MaxScale + * + * @param filename The filename of the configuration file + * @return True on success, false on fatal error + */ +bool +config_load(const char *filename) +{ + ss_dassert(!config_file); + + /* Temporary - should use configuration values and test return value (bool) */ + dcb_pre_alloc(1000); + session_pre_alloc(250); + + global_defaults(); + feedback_defaults(); + + bool rval = config_load_and_process(filename, process_config_context); + + if (rval) + { + config_file = filename; + } + + return rval; +} + /** * Reload the configuration file for the MaxScale * - * @return A zero return indicates a fatal error reading the configuration + * @return True on success, false on fatal error. */ -int +bool config_reload() { - int rval = 0; + bool rval = false; if (config_file) { - return 0; + if (gateway.version_string) + { + MXS_FREE(gateway.version_string); + } + + global_defaults(); + feedback_defaults(); + + rval = config_load_and_process(config_file, process_config_update); } - - DUPLICATE_CONTEXT dcontext; - - if (!duplicate_context_init(&dcontext)) + else { - return 0; + MXS_ERROR("config_reload() called without the configuration having " + "been loaded first."); } - bool duplicates = config_has_duplicate_sections(config_file, &dcontext); - duplicate_context_finish(&dcontext); - - if (duplicates) - { - return 0; - } - - if (gateway.version_string) - { - MXS_FREE(gateway.version_string); - } - - global_defaults(); - - CONFIG_CONTEXT config; - config.object = ""; - config.next = NULL; - - if (ini_parse(config_file, ini_handler, &config) < 0) - { - return 0; - } - - rval = process_config_update(config.next); - free_config_context(config.next); - return rval; } @@ -1574,7 +1576,7 @@ feedback_defaults() * * @param context The configuration data */ -static int +static bool process_config_update(CONFIG_CONTEXT *context) { CONFIG_CONTEXT *obj; @@ -1816,7 +1818,7 @@ process_config_update(CONFIG_CONTEXT *context) obj = obj->next; } - return 1; + return true; } /** From 91e36ecaed51577144570816fc86f56c486dd8b1 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Mon, 31 Oct 2016 13:38:18 +0200 Subject: [PATCH 122/215] Make PREPARE operation explicit The operation of the statement to be prepared is no longer reported as the operation of the PREPARE statement. Instead, when the type of the statement is QUERY_TYPE_PREPARE_NAMED_STMT, the operation can be obtained using qc_get_prepare_operation(). The qc_mysqlembedded implementation will be provided in a subsequent commit. --- include/maxscale/query_classifier.h | 12 +++++++ .../qc_mysqlembedded/qc_mysqlembedded.cc | 21 +++++++++++ query_classifier/qc_sqlite/qc_sqlite.c | 35 +++++++++++++++++-- query_classifier/test/compare.cc | 28 +++++++++++++++ 4 files changed, 93 insertions(+), 3 deletions(-) diff --git a/include/maxscale/query_classifier.h b/include/maxscale/query_classifier.h index ccef8d2fc..b1b4a9f18 100644 --- a/include/maxscale/query_classifier.h +++ b/include/maxscale/query_classifier.h @@ -116,6 +116,7 @@ typedef struct query_classifier char* (*qc_get_affected_fields)(GWBUF* stmt); char** (*qc_get_database_names)(GWBUF* stmt, int* size); char* (*qc_get_prepare_name)(GWBUF* stmt); + qc_query_op_t (*qc_get_prepare_operation)(GWBUF* stmt); } QUERY_CLASSIFIER; /** @@ -285,6 +286,17 @@ qc_query_op_t qc_get_operation(GWBUF* stmt); */ char* qc_get_prepare_name(GWBUF* stmt); +/** + * Returns the operator of the prepared statement, if the statement + * is a PREPARE statement. + * + * @param stmt A buffer containing a COM_QUERY packet. + * + * @return The operator of the prepared statement, if the statement + * is a PREPARE statement; otherwise QUERY_OP_UNDEFINED. + */ +qc_query_op_t qc_get_prepare_operation(GWBUF* stmt); + /** * Returns the tables accessed by the statement. * diff --git a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc index d8b45ca97..11848a89e 100644 --- a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc +++ b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc @@ -1929,6 +1929,26 @@ char* qc_get_prepare_name(GWBUF* stmt) return name; } +qc_query_op_t qc_get_prepare_operation(GWBUF* stmt) +{ + qc_query_op_t operation = QUERY_OP_UNDEFINED; + + if (stmt) + { + if (ensure_query_is_parsed(stmt)) + { + LEX* lex = get_lex(stmt); + + if (lex->sql_command == SQLCOM_PREPARE) + { + MXS_WARNING("qc_get_prepare_operation not implemented yet."); + } + } + } + + return operation; +} + namespace { @@ -2074,6 +2094,7 @@ static QUERY_CLASSIFIER qc = qc_get_affected_fields, qc_get_database_names, qc_get_prepare_name, + qc_get_prepare_operation, }; /* @see function load_module in load_utils.c for explanation of the following diff --git a/query_classifier/qc_sqlite/qc_sqlite.c b/query_classifier/qc_sqlite/qc_sqlite.c index bb180ac64..f78efebc8 100644 --- a/query_classifier/qc_sqlite/qc_sqlite.c +++ b/query_classifier/qc_sqlite/qc_sqlite.c @@ -79,6 +79,7 @@ typedef struct qc_sqlite_info int keyword_1; // The first encountered keyword. int keyword_2; // The second encountered keyword. char* prepare_name; // The name of a prepared statement. + qc_query_op_t prepare_operation; // The operation of a prepared statement. size_t preparable_stmt_offset; // The start of the preparable statement. size_t preparable_stmt_length; // The length of the preparable statement. } QC_SQLITE_INFO; @@ -331,6 +332,7 @@ static QC_SQLITE_INFO* info_init(QC_SQLITE_INFO* info) info->keyword_1 = 0; // Sqlite3 starts numbering tokens from 1, so 0 means info->keyword_2 = 0; // that we have not seen a keyword. info->prepare_name = NULL; + info->prepare_operation = QUERY_OP_UNDEFINED; info->preparable_stmt_offset = 0; info->preparable_stmt_length = 0; @@ -482,9 +484,7 @@ static bool parse_query(GWBUF* query) this_thread.info->query = NULL; this_thread.info->query_len = 0; - // TODO: Perhaps the rest of the stuff should be - // TODO: copied as well. - info->operation = preparable_info->operation; + info->prepare_operation = preparable_info->operation; info_free(preparable_info); } @@ -2923,6 +2923,34 @@ static char* qc_sqlite_get_prepare_name(GWBUF* query) return name; } +static qc_query_op_t qc_sqlite_get_prepare_operation(GWBUF* query) +{ + QC_TRACE(); + ss_dassert(this_unit.initialized); + ss_dassert(this_thread.initialized); + + qc_query_op_t op = QUERY_OP_UNDEFINED; + QC_SQLITE_INFO* info = get_query_info(query); + + if (info) + { + if (qc_info_is_valid(info->status)) + { + op = info->prepare_operation; + } + else if (MXS_LOG_PRIORITY_IS_ENABLED(LOG_INFO)) + { + log_invalid_data(query, "cannot report the operation of a prepared statement"); + } + } + else + { + MXS_ERROR("The query could not be parsed. Response not valid."); + } + + return op; +} + /** * EXPORTS */ @@ -2947,6 +2975,7 @@ static QUERY_CLASSIFIER qc = qc_sqlite_get_affected_fields, qc_sqlite_get_database_names, qc_sqlite_get_prepare_name, + qc_sqlite_get_prepare_operation, }; diff --git a/query_classifier/test/compare.cc b/query_classifier/test/compare.cc index 066f086ba..99fd1f7ba 100644 --- a/query_classifier/test/compare.cc +++ b/query_classifier/test/compare.cc @@ -844,6 +844,33 @@ bool compare_get_prepare_name(QUERY_CLASSIFIER* pClassifier1, GWBUF* pCopy1, return success; } +bool compare_get_prepare_operation(QUERY_CLASSIFIER* pClassifier1, GWBUF* pCopy1, + QUERY_CLASSIFIER* pClassifier2, GWBUF* pCopy2) +{ + bool success = false; + const char HEADING[] = "qc_get_prepare_operation : "; + + qc_query_op_t rv1 = pClassifier1->qc_get_prepare_operation(pCopy1); + qc_query_op_t rv2 = pClassifier2->qc_get_prepare_operation(pCopy2); + + stringstream ss; + ss << HEADING; + + if (rv1 == rv2) + { + ss << "Ok : " << qc_op_to_string(rv1); + success = true; + } + else + { + ss << "ERR: " << qc_op_to_string(rv1) << " != " << qc_op_to_string(rv2); + } + + report(success, ss.str()); + + return success; +} + bool compare(QUERY_CLASSIFIER* pClassifier1, QUERY_CLASSIFIER* pClassifier2, const string& s) { GWBUF* pCopy1 = create_gwbuf(s); @@ -863,6 +890,7 @@ bool compare(QUERY_CLASSIFIER* pClassifier1, QUERY_CLASSIFIER* pClassifier2, con errors += !compare_get_affected_fields(pClassifier1, pCopy1, pClassifier2, pCopy2); errors += !compare_get_database_names(pClassifier1, pCopy1, pClassifier2, pCopy2); errors += !compare_get_prepare_name(pClassifier1, pCopy1, pClassifier2, pCopy2); + errors += !compare_get_prepare_operation(pClassifier1, pCopy1, pClassifier2, pCopy2); gwbuf_free(pCopy1); gwbuf_free(pCopy2); From 8662221ec1f49a63ee921ec56f196dcc60721572 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Mon, 31 Oct 2016 15:47:54 +0200 Subject: [PATCH 123/215] qc_mysqlembedded: Add qc_get_prepare_operation --- .../qc_mysqlembedded/qc_mysqlembedded.cc | 60 ++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc index 11848a89e..f6113ede3 100644 --- a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc +++ b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc @@ -1941,7 +1941,65 @@ qc_query_op_t qc_get_prepare_operation(GWBUF* stmt) if (lex->sql_command == SQLCOM_PREPARE) { - MXS_WARNING("qc_get_prepare_operation not implemented yet."); + // This is terriby inefficient, but as qc_mysqlembedded is not used + // for anything else but comparisons it is ok. + const char* prepare_str = lex->prepared_stmt_code.str; + size_t prepare_str_len = lex->prepared_stmt_code.length; + + // MySQL does not parse e.g. "select * from x where ?=5". To work + // around that we'll replace all "?":s with "@a":s. We might replace + // something unnecessarily, but that won't hurt the classification. + size_t n_questions = 0; + const char* p = prepare_str; + while (p < prepare_str + prepare_str_len) + { + if (*p == '?') + { + ++n_questions; + } + + ++p; + } + + size_t payload_len = prepare_str_len + n_questions * 2 + 1; + size_t prepare_stmt_len = MYSQL_HEADER_LEN + payload_len; + + GWBUF* prepare_stmt = gwbuf_alloc(prepare_stmt_len); + + if (prepare_stmt) + { + // Encode the length of the payload in the 3 first bytes. + *((unsigned char*)GWBUF_DATA(prepare_stmt) + 0) = payload_len; + *((unsigned char*)GWBUF_DATA(prepare_stmt) + 1) = (payload_len >> 8); + *((unsigned char*)GWBUF_DATA(prepare_stmt) + 2) = (payload_len >> 16); + // Sequence id + *((unsigned char*)GWBUF_DATA(prepare_stmt) + 3) = 0x00; + // Payload, starts with command. + *((unsigned char*)GWBUF_DATA(prepare_stmt) + 4) = COM_QUERY; + // Is followed by the statement. + char *s = (char*)GWBUF_DATA(prepare_stmt) + 5; + p = prepare_str; + + while (p < prepare_str + prepare_str_len) + { + switch (*p) + { + case '?': + *s++ = '@'; + *s = 'a'; + break; + + default: + *s = *p; + } + + ++p; + ++s; + } + + operation = qc_get_operation(prepare_stmt); + gwbuf_free(prepare_stmt); + } } } } From e8e2933b569c48418fed83584abc580fe6698eb3 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Mon, 31 Oct 2016 22:45:48 +0200 Subject: [PATCH 124/215] Add authenticator section to Documentation Contents The authenticator documents are now linked in the main document. --- Documentation/Documentation-Contents.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Documentation/Documentation-Contents.md b/Documentation/Documentation-Contents.md index 4904c92a5..69070b10b 100644 --- a/Documentation/Documentation-Contents.md +++ b/Documentation/Documentation-Contents.md @@ -106,6 +106,15 @@ Documentation for MaxScale protocol modules. - [Change Data Capture (CDC) Protocol](Protocols/CDC.md) - [Change Data Capture (CDC) Users](Protocols/CDC_users.md) +## Authenticators + +A short description of the authentication module type can be found in the +[Authentication Modules](Authenticators/Authentication-Modules.md) +document. + + - [MySQL Authenticator](Authenticators/MySQL-Authenticator.md) + - [GSSAPI Authenticator](Authenticators/GSSAPI-Authenticator.md) + ## Utilities - [RabbitMQ Consumer Client](Filters/RabbitMQ-Consumer-Client.md) From 5085b41d202b6b1209ccb4909b76fdc083a135fe Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 28 Oct 2016 11:57:52 +0300 Subject: [PATCH 125/215] Reset persistent MySQL connections When a persistent connection is taken from the pool, the state is reset with a COM_CHANGE_USER on the next write. This allows reuse of persistent connections without having to worry about the state of the MySQL session. --- include/maxscale/dcb.h | 1 + include/maxscale/protocol/mysql.h | 37 ++++----- server/core/dcb.c | 3 + .../MySQL/MySQLBackend/mysql_backend.c | 77 +++++++++++++++++++ server/modules/protocol/MySQL/mysql_common.c | 4 + 5 files changed, 102 insertions(+), 20 deletions(-) diff --git a/include/maxscale/dcb.h b/include/maxscale/dcb.h index 617d124e1..b79dbeee5 100644 --- a/include/maxscale/dcb.h +++ b/include/maxscale/dcb.h @@ -276,6 +276,7 @@ typedef struct dcb bool ssl_write_want_read; /*< Flag */ bool ssl_write_want_write; /*< Flag */ int dcb_port; /**< port of target server */ + bool was_persistent; /**< Whether this DCB was in the persistent pool */ skygw_chk_t dcb_chk_tail; } DCB; diff --git a/include/maxscale/protocol/mysql.h b/include/maxscale/protocol/mysql.h index 7731c3c30..49c175615 100644 --- a/include/maxscale/protocol/mysql.h +++ b/include/maxscale/protocol/mysql.h @@ -262,28 +262,25 @@ typedef struct server_command_st typedef struct { #if defined(SS_DEBUG) - skygw_chk_t protocol_chk_top; + skygw_chk_t protocol_chk_top; #endif - int fd; /*< The socket descriptor */ - struct dcb *owner_dcb; /*< The DCB of the socket - * we are running on */ - SPINLOCK protocol_lock; - mysql_server_cmd_t current_command; /**< Current command being executed */ - server_command_t protocol_command; /*< session command list */ - server_command_t* protocol_cmd_history; /*< session command history */ - mxs_auth_state_t protocol_auth_state; /*< Authentication status */ - mysql_protocol_state_t protocol_state; /*< Protocol struct status */ - uint8_t scramble[MYSQL_SCRAMBLE_LEN]; /*< server scramble, - * created or received */ - uint32_t server_capabilities; /*< server capabilities, - * created or received */ - uint32_t client_capabilities; /*< client capabilities, - * created or received */ - unsigned long tid; /*< MySQL Thread ID, in - * handshake */ - unsigned int charset; /*< MySQL character set at connect time */ + int fd; /*< The socket descriptor */ + struct dcb* owner_dcb; /*< The DCB of the socket we are running on */ + SPINLOCK protocol_lock; /*< Protocol lock */ + mysql_server_cmd_t current_command; /*< Current command being executed */ + server_command_t protocol_command; /*< session command list */ + server_command_t* protocol_cmd_history; /*< session command history */ + mxs_auth_state_t protocol_auth_state; /*< Authentication status */ + mysql_protocol_state_t protocol_state; /*< Protocol struct status */ + uint8_t scramble[MYSQL_SCRAMBLE_LEN]; /*< server scramble, created or received */ + uint32_t server_capabilities; /*< server capabilities, created or received */ + uint32_t client_capabilities; /*< client capabilities, created or received */ + unsigned long tid; /*< MySQL Thread ID, in handshake */ + unsigned int charset; /*< MySQL character set at connect time */ + bool ignore_reply; /*< If the reply should be discarded */ + GWBUF* stored_query; /*< Temporarily stored queries */ #if defined(SS_DEBUG) - skygw_chk_t protocol_chk_tail; + skygw_chk_t protocol_chk_tail; #endif } MySQLProtocol; diff --git a/server/core/dcb.c b/server/core/dcb.c index 24e252e91..939fe6a2b 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -772,6 +772,7 @@ dcb_connect(SERVER *server, SESSION *session, const char *protocol) MXS_DEBUG("%lu [dcb_connect] Reusing a persistent connection, dcb %p\n", pthread_self(), dcb); dcb->persistentstart = 0; + dcb->was_persistent = true; return dcb; } else @@ -867,6 +868,8 @@ dcb_connect(SERVER *server, SESSION *session, const char *protocol) dcb->dcb_server_status = server->status; dcb->dcb_port = server->port; + dcb->was_persistent = false; + /** * backend_dcb is connected to backend server, and once backend_dcb * is added to poll set, authentication takes place as part of diff --git a/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c b/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c index 17f52cb37..4253b1c45 100644 --- a/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c +++ b/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c @@ -759,6 +759,41 @@ gw_read_and_write(DCB *dcb) } } + MySQLProtocol *proto = (MySQLProtocol *)dcb->protocol; + + spinlock_acquire(&dcb->authlock); + + if (proto->ignore_reply) + { + + /** The reply to a COM_CHANGE_USER is in packet */ + GWBUF *query = proto->stored_query; + uint8_t result = *((uint8_t*)GWBUF_DATA(read_buffer) + 4); + proto->stored_query = NULL; + proto->ignore_reply = false; + gwbuf_free(read_buffer); + + spinlock_release(&dcb->authlock); + + int rval = 0; + + if (result == MYSQL_REPLY_OK) + { + rval = query ? dcb->func.write(dcb, query) : 1; + } + else if (query) + { + /** The COM_CHANGE USER failed, generate a fake hangup event to + * close the DCB and send an error to the client. */ + gwbuf_free(query); + poll_fake_hangup_event(dcb); + } + + return rval; + } + + spinlock_release(&dcb->authlock); + /** * If protocol has session command set, concatenate whole * response into one buffer. @@ -911,6 +946,48 @@ static int gw_MySQLWrite_backend(DCB *dcb, GWBUF *queue) CHK_DCB(dcb); spinlock_acquire(&dcb->authlock); + + if (dcb->was_persistent) + { + /** + * This is a DCB that was just taken out of the persistent connection pool. + * We need to sent a COM_CHANGE_USER query to the backend to reset the + * session state. + */ + if (backend_protocol->stored_query) + { + /** It is possible that the client DCB is closed before the COM_CHANGE_USER + * response is received. */ + gwbuf_free(backend_protocol->stored_query); + } + dcb->was_persistent = false; + backend_protocol->ignore_reply = true; + backend_protocol->stored_query = queue; + + spinlock_release(&dcb->authlock); + + GWBUF *buf = gw_create_change_user_packet(dcb->session->client_dcb->data, dcb->protocol); + return dcb_write(dcb, buf) ? 1 : 0; + } + else if (backend_protocol->ignore_reply) + { + if (MYSQL_IS_COM_QUIT((uint8_t*)GWBUF_DATA(queue))) + { + gwbuf_free(queue); + } + else + { + /** + * We're still waiting on the reply to the COM_CHANGE_USER, append the + * buffer to the stored query. This is possible if the client sends + * BLOB data on the first command. + */ + backend_protocol->stored_query = gwbuf_append(backend_protocol->stored_query, queue); + } + spinlock_release(&dcb->authlock); + return 1; + } + /** * Pick action according to state of protocol. * If auth failed, return value is 0, write and buffered write diff --git a/server/modules/protocol/MySQL/mysql_common.c b/server/modules/protocol/MySQL/mysql_common.c index bd471a26a..5104720be 100644 --- a/server/modules/protocol/MySQL/mysql_common.c +++ b/server/modules/protocol/MySQL/mysql_common.c @@ -102,6 +102,7 @@ MySQLProtocol* mysql_protocol_init(DCB* dcb, int fd) p->protocol_command.scom_cmd = MYSQL_COM_UNDEFINED; p->protocol_command.scom_nresponse_packets = 0; p->protocol_command.scom_nbytes_to_read = 0; + p->stored_query = NULL; #if defined(SS_DEBUG) p->protocol_chk_top = CHK_NUM_PROTOCOL; p->protocol_chk_tail = CHK_NUM_PROTOCOL; @@ -145,6 +146,9 @@ void mysql_protocol_done(DCB* dcb) MXS_FREE(scmd); scmd = scmd2; } + + gwbuf_free(p->stored_query); + p->protocol_state = MYSQL_PROTOCOL_DONE; retblock: From c0e29691e08674780c8e611ea5f08176e319b7f5 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Mon, 31 Oct 2016 18:02:50 +0200 Subject: [PATCH 126/215] qc_mysqlembedded: Return prepare name for EXECUTE --- query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc index f6113ede3..e60e2f8e3 100644 --- a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc +++ b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc @@ -1914,7 +1914,7 @@ char* qc_get_prepare_name(GWBUF* stmt) { LEX* lex = get_lex(stmt); - if (lex->sql_command == SQLCOM_PREPARE) + if ((lex->sql_command == SQLCOM_PREPARE) || (lex->sql_command == SQLCOM_EXECUTE)) { name = (char*)malloc(lex->prepared_stmt_name.length + 1); if (name) From f7c31504621174c8d2db4e269683a7557b9bd825 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Mon, 31 Oct 2016 19:11:39 +0200 Subject: [PATCH 127/215] qc_sqlite: Copy preparable statement Peeking into the token does not always work. --- query_classifier/qc_sqlite/qc_sqlite.c | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/query_classifier/qc_sqlite/qc_sqlite.c b/query_classifier/qc_sqlite/qc_sqlite.c index f78efebc8..ccec8c165 100644 --- a/query_classifier/qc_sqlite/qc_sqlite.c +++ b/query_classifier/qc_sqlite/qc_sqlite.c @@ -80,7 +80,7 @@ typedef struct qc_sqlite_info int keyword_2; // The second encountered keyword. char* prepare_name; // The name of a prepared statement. qc_query_op_t prepare_operation; // The operation of a prepared statement. - size_t preparable_stmt_offset; // The start of the preparable statement. + char* preparable_stmt; // The preparable statement. size_t preparable_stmt_length; // The length of the preparable statement. } QC_SQLITE_INFO; @@ -294,6 +294,8 @@ static void info_finish(QC_SQLITE_INFO* info) free_string_array(info->table_fullnames); free(info->created_table_name); free_string_array(info->database_names); + free(info->prepare_name); + free(info->preparable_stmt); } static void info_free(QC_SQLITE_INFO* info) @@ -333,7 +335,7 @@ static QC_SQLITE_INFO* info_init(QC_SQLITE_INFO* info) info->keyword_2 = 0; // that we have not seen a keyword. info->prepare_name = NULL; info->prepare_operation = QUERY_OP_UNDEFINED; - info->preparable_stmt_offset = 0; + info->preparable_stmt = NULL; info->preparable_stmt_length = 0; return info; @@ -467,7 +469,7 @@ static bool parse_query(GWBUF* query) this_thread.info->query = NULL; this_thread.info->query_len = 0; - if (info->types & QUERY_TYPE_PREPARE_NAMED_STMT) + if ((info->types & QUERY_TYPE_PREPARE_NAMED_STMT) && info->preparable_stmt) { QC_SQLITE_INFO* preparable_info = info_alloc(); @@ -475,7 +477,7 @@ static bool parse_query(GWBUF* query) { this_thread.info = preparable_info; - const char *preparable_s = s + info->preparable_stmt_offset; + const char *preparable_s = info->preparable_stmt; size_t preparable_len = info->preparable_stmt_length; this_thread.info->query = preparable_s; @@ -2014,11 +2016,12 @@ void maxscalePrepare(Parse* pParse, Token* pName, Token* pStmt) info->prepare_name[pName->n] = 0; } - // We store the position of the preparable statement inside the original - // statement. That will allow us to later create a new GWBUF of the - // parsable statment and parse that. - info->preparable_stmt_offset = pParse->sLastToken.z - pParse->zTail + 1; // Ignore starting quote. - info->preparable_stmt_length = pStmt->n - 2; // Remove starting and ending quotes. + info->preparable_stmt_length = pStmt->n - 2; + info->preparable_stmt = MXS_MALLOC(info->preparable_stmt_length); + if (info->preparable_stmt) + { + memcpy(info->preparable_stmt, pStmt->z + 1, pStmt->n - 2); + } } void maxscalePrivileges(Parse* pParse, int kind) From 57eb599769fae076cbbb9f47e69737ec16142323 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Tue, 1 Nov 2016 12:05:35 +0200 Subject: [PATCH 128/215] qc_mysqlembedded: Fix missing functionality --- query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc index e60e2f8e3..dd5789b59 100644 --- a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc +++ b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc @@ -1852,6 +1852,7 @@ qc_query_op_t qc_get_operation(GWBUF* querybuf) case SQLCOM_INSERT: case SQLCOM_INSERT_SELECT: case SQLCOM_REPLACE: + case SQLCOM_REPLACE_SELECT: operation = QUERY_OP_INSERT; break; @@ -1914,7 +1915,9 @@ char* qc_get_prepare_name(GWBUF* stmt) { LEX* lex = get_lex(stmt); - if ((lex->sql_command == SQLCOM_PREPARE) || (lex->sql_command == SQLCOM_EXECUTE)) + if ((lex->sql_command == SQLCOM_PREPARE) || + (lex->sql_command == SQLCOM_EXECUTE) || + (lex->sql_command == SQLCOM_DEALLOCATE_PREPARE)) { name = (char*)malloc(lex->prepared_stmt_name.length + 1); if (name) From d69025bfa691e96ddfdac40b9f581f7c784117f1 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 1 Nov 2016 11:20:34 +0200 Subject: [PATCH 129/215] Add changes to persistent connections to documentation The documentation now states that, starting with 2.1, the persistent connections will reset the MySQL session state when they are taken into use. --- .../MaxScale-2.1.0-Release-Notes.md | 12 ++++++++++ .../Tutorials/Administration-Tutorial.md | 23 +++++++++++++------ 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/Documentation/Release-Notes/MaxScale-2.1.0-Release-Notes.md b/Documentation/Release-Notes/MaxScale-2.1.0-Release-Notes.md index 7ffdace44..d5617474f 100644 --- a/Documentation/Release-Notes/MaxScale-2.1.0-Release-Notes.md +++ b/Documentation/Release-Notes/MaxScale-2.1.0-Release-Notes.md @@ -57,6 +57,18 @@ by default, is configured using the new global configuration entry `log_throttli For more information about this configuration entry, please see [Global Settings](../Getting-Started/Configuration-Guide.md#global-settings). +### Persistent Connections + +Starting with the 2.1 version of MariaDB MaxScale, when a MySQL protocol +persistent connection is taken from the persistent connection pool, the +state of the MySQL session will be reset when the the connection is used +for the first time. This allows persistent connections to be used with no +functional limitations and makes them behave like normal MySQL +connections. + +For more information about persistent connections, please read the +[Administration Tutorial](../Tutorials/Administration-Tutorial.md). + ### User data cache The user data cache stores the cached credentials that are used by some router diff --git a/Documentation/Tutorials/Administration-Tutorial.md b/Documentation/Tutorials/Administration-Tutorial.md index e9f4e5929..14d64b807 100644 --- a/Documentation/Tutorials/Administration-Tutorial.md +++ b/Documentation/Tutorials/Administration-Tutorial.md @@ -106,13 +106,22 @@ than `persistmaxtime` seconds. It was also be discarded if it has been disconne by the back end server. Connections will be selected that match the user name and protocol for the new request. -**Please note** that because persistent connections have previously been in use, they -may give a different environment from a fresh connection. For example, if the -previous use of the connection issued "use mydatabase" then this setting will be -carried over into the reuse of the same connection. For many applications this will -not be noticeable, since each request will assume that nothing has been set and -will issue fresh requests such as "use" to establish the desired configuration. In -exceptional cases this feature could be a problem. +Starting with the 2.1 version of MaxScale, when a MySQL protocol connection is +taken from the pool the backend protocol module resets the session state. This +allows persistent connections to be used with no functional limitations. + +The session state is reset when the first outgoing network transmission is +done. This _lazy initialization_ of the persistent connections allows +MaxScale to take multiple new connections into use but only initialize the +ones that it actually needs. + +**Please note** that in versions before 2.1 the persistent connections may give +a different environment when compared to a fresh connection. For example, if the +previous use of the connection issued a "USE mydatabase;" statement then this +setting will be carried over into the reuse of the same connection. For many +applications this will not be noticeable, since each request will assume that +nothing has been set and will issue fresh requests such as "USE" to establish +the desired configuration. In exceptional cases this feature could be a problem. It is possible to have pools for as many servers as you wish, with configuration values in each server section. From ab487e687f79facbb61c23556824dcadffb8ddca Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Tue, 1 Nov 2016 16:06:51 +0200 Subject: [PATCH 130/215] qc: Remove QUERY_IS_TYPE macro Explicit call to qc_query_is_type() instead. --- include/maxscale/query_classifier.h | 8 -- .../qc_mysqlembedded/qc_mysqlembedded.cc | 12 +- .../readwritesplit/rwsplit_route_stmt.c | 104 +++++++++--------- .../readwritesplit/rwsplit_tmp_table_multi.c | 12 +- .../routing/schemarouter/schemarouter.c | 32 +++--- 5 files changed, 80 insertions(+), 88 deletions(-) diff --git a/include/maxscale/query_classifier.h b/include/maxscale/query_classifier.h index b1b4a9f18..438613eac 100644 --- a/include/maxscale/query_classifier.h +++ b/include/maxscale/query_classifier.h @@ -408,12 +408,4 @@ const char* qc_type_to_string(qc_query_type_t type); */ char* qc_typemask_to_string(uint32_t typemask); -/** - * @deprecated - * Synonym for qc_query_is_type(). - * - * @see qc_query_is_type - */ -#define QUERY_IS_TYPE(typemask, type) qc_query_is_type(typemask, type) - MXS_END_DECLS diff --git a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc index dd5789b59..48330808e 100644 --- a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc +++ b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc @@ -681,12 +681,12 @@ static uint32_t resolve_query_type(THD* thd) #endif // TODO: This test is meaningless, since at this point // TODO: qtype (not type) is QUERY_TYPE_UNKNOWN. - if (QUERY_IS_TYPE(qtype, QUERY_TYPE_UNKNOWN) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_LOCAL_READ) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_READ) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_USERVAR_READ) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_SYSVAR_READ) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_GSYSVAR_READ)) + if (qc_query_is_type(qtype, QUERY_TYPE_UNKNOWN) || + qc_query_is_type(qtype, QUERY_TYPE_LOCAL_READ) || + qc_query_is_type(qtype, QUERY_TYPE_READ) || + qc_query_is_type(qtype, QUERY_TYPE_USERVAR_READ) || + qc_query_is_type(qtype, QUERY_TYPE_SYSVAR_READ) || + qc_query_is_type(qtype, QUERY_TYPE_GSYSVAR_READ)) { /** * These values won't change qtype more restrictive than write. diff --git a/server/modules/routing/readwritesplit/rwsplit_route_stmt.c b/server/modules/routing/readwritesplit/rwsplit_route_stmt.c index 73f0120f0..cdaddc5ca 100644 --- a/server/modules/routing/readwritesplit/rwsplit_route_stmt.c +++ b/server/modules/routing/readwritesplit/rwsplit_route_stmt.c @@ -84,7 +84,7 @@ bool route_single_stmt(ROUTER_INSTANCE *inst, ROUTER_CLIENT_SES *rses, * transaction is committed and autocommit is enabled again. */ if (rses->rses_autocommit_enabled && - QUERY_IS_TYPE(qtype, QUERY_TYPE_DISABLE_AUTOCOMMIT)) + qc_query_is_type(qtype, QUERY_TYPE_DISABLE_AUTOCOMMIT)) { rses->rses_autocommit_enabled = false; @@ -94,7 +94,7 @@ bool route_single_stmt(ROUTER_INSTANCE *inst, ROUTER_CLIENT_SES *rses, } } else if (!rses->rses_transaction_active && - QUERY_IS_TYPE(qtype, QUERY_TYPE_BEGIN_TRX)) + qc_query_is_type(qtype, QUERY_TYPE_BEGIN_TRX)) { rses->rses_transaction_active = true; } @@ -102,13 +102,13 @@ bool route_single_stmt(ROUTER_INSTANCE *inst, ROUTER_CLIENT_SES *rses, * Explicit COMMIT and ROLLBACK, implicit COMMIT. */ if (rses->rses_autocommit_enabled && rses->rses_transaction_active && - (QUERY_IS_TYPE(qtype, QUERY_TYPE_COMMIT) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_ROLLBACK))) + (qc_query_is_type(qtype, QUERY_TYPE_COMMIT) || + qc_query_is_type(qtype, QUERY_TYPE_ROLLBACK))) { rses->rses_transaction_active = false; } else if (!rses->rses_autocommit_enabled && - QUERY_IS_TYPE(qtype, QUERY_TYPE_ENABLE_AUTOCOMMIT)) + qc_query_is_type(qtype, QUERY_TYPE_ENABLE_AUTOCOMMIT)) { rses->rses_autocommit_enabled = true; rses->rses_transaction_active = false; @@ -727,13 +727,13 @@ route_target_t get_route_target(ROUTER_CLIENT_SES *rses, * These queries are not affected by hints */ else if (!load_active && - (QUERY_IS_TYPE(qtype, QUERY_TYPE_SESSION_WRITE) || + (qc_query_is_type(qtype, QUERY_TYPE_SESSION_WRITE) || /** Configured to allow writing variables to all nodes */ (use_sql_variables_in == TYPE_ALL && - QUERY_IS_TYPE(qtype, QUERY_TYPE_GSYSVAR_WRITE)) || + qc_query_is_type(qtype, QUERY_TYPE_GSYSVAR_WRITE)) || /** enable or disable autocommit are always routed to all */ - QUERY_IS_TYPE(qtype, QUERY_TYPE_ENABLE_AUTOCOMMIT) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_DISABLE_AUTOCOMMIT))) + qc_query_is_type(qtype, QUERY_TYPE_ENABLE_AUTOCOMMIT) || + qc_query_is_type(qtype, QUERY_TYPE_DISABLE_AUTOCOMMIT))) { /** * This is problematic query because it would be routed to all @@ -750,9 +750,9 @@ route_target_t get_route_target(ROUTER_CLIENT_SES *rses, * the execution of the prepared statements to the right server would be * an easy one. Currently this is not supported. */ - if (QUERY_IS_TYPE(qtype, QUERY_TYPE_READ) && - !(QUERY_IS_TYPE(qtype, QUERY_TYPE_PREPARE_STMT) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_PREPARE_NAMED_STMT))) + if (qc_query_is_type(qtype, QUERY_TYPE_READ) && + !(qc_query_is_type(qtype, QUERY_TYPE_PREPARE_STMT) || + qc_query_is_type(qtype, QUERY_TYPE_PREPARE_NAMED_STMT))) { MXS_WARNING("The query can't be routed to all " "backend servers because it includes SELECT and " @@ -770,40 +770,40 @@ route_target_t get_route_target(ROUTER_CLIENT_SES *rses, * Hints may affect on routing of the following queries */ else if (!trx_active && !load_active && - !QUERY_IS_TYPE(qtype, QUERY_TYPE_WRITE) && - (QUERY_IS_TYPE(qtype, QUERY_TYPE_READ) || /*< any SELECT */ - QUERY_IS_TYPE(qtype, QUERY_TYPE_SHOW_TABLES) || /*< 'SHOW TABLES' */ - QUERY_IS_TYPE(qtype, - QUERY_TYPE_USERVAR_READ) || /*< read user var */ - QUERY_IS_TYPE(qtype, QUERY_TYPE_SYSVAR_READ) || /*< read sys var */ - QUERY_IS_TYPE(qtype, - QUERY_TYPE_EXEC_STMT) || /*< prepared stmt exec */ - QUERY_IS_TYPE(qtype, QUERY_TYPE_PREPARE_STMT) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_PREPARE_NAMED_STMT) || - QUERY_IS_TYPE(qtype, - QUERY_TYPE_GSYSVAR_READ))) /*< read global sys var */ + !qc_query_is_type(qtype, QUERY_TYPE_WRITE) && + (qc_query_is_type(qtype, QUERY_TYPE_READ) || /*< any SELECT */ + qc_query_is_type(qtype, QUERY_TYPE_SHOW_TABLES) || /*< 'SHOW TABLES' */ + qc_query_is_type(qtype, + QUERY_TYPE_USERVAR_READ) || /*< read user var */ + qc_query_is_type(qtype, QUERY_TYPE_SYSVAR_READ) || /*< read sys var */ + qc_query_is_type(qtype, + QUERY_TYPE_EXEC_STMT) || /*< prepared stmt exec */ + qc_query_is_type(qtype, QUERY_TYPE_PREPARE_STMT) || + qc_query_is_type(qtype, QUERY_TYPE_PREPARE_NAMED_STMT) || + qc_query_is_type(qtype, + QUERY_TYPE_GSYSVAR_READ))) /*< read global sys var */ { /** First set expected targets before evaluating hints */ - if (!QUERY_IS_TYPE(qtype, QUERY_TYPE_MASTER_READ) && - (QUERY_IS_TYPE(qtype, QUERY_TYPE_READ) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_SHOW_TABLES) || /*< 'SHOW TABLES' */ + if (!qc_query_is_type(qtype, QUERY_TYPE_MASTER_READ) && + (qc_query_is_type(qtype, QUERY_TYPE_READ) || + qc_query_is_type(qtype, QUERY_TYPE_SHOW_TABLES) || /*< 'SHOW TABLES' */ /** Configured to allow reading variables from slaves */ (use_sql_variables_in == TYPE_ALL && - (QUERY_IS_TYPE(qtype, QUERY_TYPE_USERVAR_READ) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_SYSVAR_READ) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_GSYSVAR_READ))))) + (qc_query_is_type(qtype, QUERY_TYPE_USERVAR_READ) || + qc_query_is_type(qtype, QUERY_TYPE_SYSVAR_READ) || + qc_query_is_type(qtype, QUERY_TYPE_GSYSVAR_READ))))) { target = TARGET_SLAVE; } - if (QUERY_IS_TYPE(qtype, QUERY_TYPE_MASTER_READ) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_EXEC_STMT) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_PREPARE_STMT) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_PREPARE_NAMED_STMT) || + if (qc_query_is_type(qtype, QUERY_TYPE_MASTER_READ) || + qc_query_is_type(qtype, QUERY_TYPE_EXEC_STMT) || + qc_query_is_type(qtype, QUERY_TYPE_PREPARE_STMT) || + qc_query_is_type(qtype, QUERY_TYPE_PREPARE_NAMED_STMT) || /** Configured not to allow reading variables from slaves */ (use_sql_variables_in == TYPE_MASTER && - (QUERY_IS_TYPE(qtype, QUERY_TYPE_USERVAR_READ) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_SYSVAR_READ)))) + (qc_query_is_type(qtype, QUERY_TYPE_USERVAR_READ) || + qc_query_is_type(qtype, QUERY_TYPE_SYSVAR_READ)))) { target = TARGET_MASTER; } @@ -818,26 +818,26 @@ route_target_t get_route_target(ROUTER_CLIENT_SES *rses, { /** hints don't affect on routing */ ss_dassert(trx_active || - (QUERY_IS_TYPE(qtype, QUERY_TYPE_WRITE) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_MASTER_READ) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_SESSION_WRITE) || - (QUERY_IS_TYPE(qtype, QUERY_TYPE_USERVAR_READ) && + (qc_query_is_type(qtype, QUERY_TYPE_WRITE) || + qc_query_is_type(qtype, QUERY_TYPE_MASTER_READ) || + qc_query_is_type(qtype, QUERY_TYPE_SESSION_WRITE) || + (qc_query_is_type(qtype, QUERY_TYPE_USERVAR_READ) && use_sql_variables_in == TYPE_MASTER) || - (QUERY_IS_TYPE(qtype, QUERY_TYPE_SYSVAR_READ) && + (qc_query_is_type(qtype, QUERY_TYPE_SYSVAR_READ) && use_sql_variables_in == TYPE_MASTER) || - (QUERY_IS_TYPE(qtype, QUERY_TYPE_GSYSVAR_READ) && + (qc_query_is_type(qtype, QUERY_TYPE_GSYSVAR_READ) && use_sql_variables_in == TYPE_MASTER) || - (QUERY_IS_TYPE(qtype, QUERY_TYPE_GSYSVAR_WRITE) && + (qc_query_is_type(qtype, QUERY_TYPE_GSYSVAR_WRITE) && use_sql_variables_in == TYPE_MASTER) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_BEGIN_TRX) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_ENABLE_AUTOCOMMIT) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_DISABLE_AUTOCOMMIT) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_ROLLBACK) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_COMMIT) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_EXEC_STMT) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_CREATE_TMP_TABLE) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_READ_TMP_TABLE) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_UNKNOWN))); + qc_query_is_type(qtype, QUERY_TYPE_BEGIN_TRX) || + qc_query_is_type(qtype, QUERY_TYPE_ENABLE_AUTOCOMMIT) || + qc_query_is_type(qtype, QUERY_TYPE_DISABLE_AUTOCOMMIT) || + qc_query_is_type(qtype, QUERY_TYPE_ROLLBACK) || + qc_query_is_type(qtype, QUERY_TYPE_COMMIT) || + qc_query_is_type(qtype, QUERY_TYPE_EXEC_STMT) || + qc_query_is_type(qtype, QUERY_TYPE_CREATE_TMP_TABLE) || + qc_query_is_type(qtype, QUERY_TYPE_READ_TMP_TABLE) || + qc_query_is_type(qtype, QUERY_TYPE_UNKNOWN))); target = TARGET_MASTER; } diff --git a/server/modules/routing/readwritesplit/rwsplit_tmp_table_multi.c b/server/modules/routing/readwritesplit/rwsplit_tmp_table_multi.c index a82945982..e008ed031 100644 --- a/server/modules/routing/readwritesplit/rwsplit_tmp_table_multi.c +++ b/server/modules/routing/readwritesplit/rwsplit_tmp_table_multi.c @@ -147,11 +147,11 @@ qc_query_type_t is_read_tmp_table(ROUTER_CLIENT_SES *router_cli_ses, dbname = (char *)data->db; - if (QUERY_IS_TYPE(qtype, QUERY_TYPE_READ) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_LOCAL_READ) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_USERVAR_READ) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_SYSVAR_READ) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_GSYSVAR_READ)) + if (qc_query_is_type(qtype, QUERY_TYPE_READ) || + qc_query_is_type(qtype, QUERY_TYPE_LOCAL_READ) || + qc_query_is_type(qtype, QUERY_TYPE_USERVAR_READ) || + qc_query_is_type(qtype, QUERY_TYPE_SYSVAR_READ) || + qc_query_is_type(qtype, QUERY_TYPE_GSYSVAR_READ)) { tbl = qc_get_table_names(querybuf, &tsize, false); @@ -199,7 +199,7 @@ qc_query_type_t is_read_tmp_table(ROUTER_CLIENT_SES *router_cli_ses, void check_create_tmp_table(ROUTER_CLIENT_SES *router_cli_ses, GWBUF *querybuf, qc_query_type_t type) { - if (!QUERY_IS_TYPE(type, QUERY_TYPE_CREATE_TMP_TABLE)) + if (!qc_query_is_type(type, QUERY_TYPE_CREATE_TMP_TABLE)) { return; } diff --git a/server/modules/routing/schemarouter/schemarouter.c b/server/modules/routing/schemarouter/schemarouter.c index 7ef6d8681..ce6bcecec 100644 --- a/server/modules/routing/schemarouter/schemarouter.c +++ b/server/modules/routing/schemarouter/schemarouter.c @@ -524,7 +524,7 @@ char* get_shard_target_name(ROUTER_INSTANCE* router, /* Check if the query is a show tables query with a specific database */ - if (QUERY_IS_TYPE(qtype, QUERY_TYPE_SHOW_TABLES)) + if (qc_query_is_type(qtype, QUERY_TYPE_SHOW_TABLES)) { query = modutil_get_SQL(buffer); if ((tmp = strcasestr(query, "from"))) @@ -1457,19 +1457,19 @@ static route_target_t get_shard_route_target(qc_query_type_t qtype, /** * These queries are not affected by hints */ - if (QUERY_IS_TYPE(qtype, QUERY_TYPE_SESSION_WRITE) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_PREPARE_STMT) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_PREPARE_NAMED_STMT) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_GSYSVAR_WRITE) || + if (qc_query_is_type(qtype, QUERY_TYPE_SESSION_WRITE) || + qc_query_is_type(qtype, QUERY_TYPE_PREPARE_STMT) || + qc_query_is_type(qtype, QUERY_TYPE_PREPARE_NAMED_STMT) || + qc_query_is_type(qtype, QUERY_TYPE_GSYSVAR_WRITE) || /** enable or disable autocommit are always routed to all */ - QUERY_IS_TYPE(qtype, QUERY_TYPE_ENABLE_AUTOCOMMIT) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_DISABLE_AUTOCOMMIT)) + qc_query_is_type(qtype, QUERY_TYPE_ENABLE_AUTOCOMMIT) || + qc_query_is_type(qtype, QUERY_TYPE_DISABLE_AUTOCOMMIT)) { /** hints don't affect on routing */ target = TARGET_ALL; } - else if (QUERY_IS_TYPE(qtype, QUERY_TYPE_SYSVAR_READ) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_GSYSVAR_READ)) + else if (qc_query_is_type(qtype, QUERY_TYPE_SYSVAR_READ) || + qc_query_is_type(qtype, QUERY_TYPE_GSYSVAR_READ)) { target = TARGET_ANY; } @@ -1559,11 +1559,11 @@ qc_query_type_t is_read_tmp_table(ROUTER* instance, rses_prop_tmp = router_cli_ses->rses_properties[RSES_PROP_TYPE_TMPTABLES]; dbname = router_cli_ses->current_db; - if (QUERY_IS_TYPE(qtype, QUERY_TYPE_READ) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_LOCAL_READ) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_USERVAR_READ) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_SYSVAR_READ) || - QUERY_IS_TYPE(qtype, QUERY_TYPE_GSYSVAR_READ)) + if (qc_query_is_type(qtype, QUERY_TYPE_READ) || + qc_query_is_type(qtype, QUERY_TYPE_LOCAL_READ) || + qc_query_is_type(qtype, QUERY_TYPE_USERVAR_READ) || + qc_query_is_type(qtype, QUERY_TYPE_SYSVAR_READ) || + qc_query_is_type(qtype, QUERY_TYPE_GSYSVAR_READ)) { tbl = qc_get_table_names(querybuf, &tsize, false); @@ -1633,7 +1633,7 @@ void check_create_tmp_table(ROUTER* instance, rses_prop_tmp = router_cli_ses->rses_properties[RSES_PROP_TYPE_TMPTABLES]; dbname = router_cli_ses->current_db; - if (QUERY_IS_TYPE(type, QUERY_TYPE_CREATE_TMP_TABLE)) + if (qc_query_is_type(type, QUERY_TYPE_CREATE_TMP_TABLE)) { bool is_temp = true; char* tblname = NULL; @@ -2081,7 +2081,7 @@ static int routeQuery(ROUTER* instance, } /** Create the response to the SHOW DATABASES from the mapped databases */ - if (QUERY_IS_TYPE(qtype, QUERY_TYPE_SHOW_DATABASES)) + if (qc_query_is_type(qtype, QUERY_TYPE_SHOW_DATABASES)) { if (send_database_list(inst, router_cli_ses)) { From c652f1330a4626a73fa96434110c37c612befedd Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Tue, 1 Nov 2016 16:34:28 +0200 Subject: [PATCH 131/215] qc: Add qc_get_field_info This function returns more detailed information about the fields of a statement. Supersedes qc_get_affected_fields() that will be deprecated and removed. Note that this function now introduced new kind of behaviour; the returned data belongs to the GWBUF and remains valid for as long as the GWBUF is alive. That means that unnecessary copying need not be done. --- include/maxscale/query_classifier.h | 28 +++++++++++++++++++ .../qc_mysqlembedded/qc_mysqlembedded.cc | 9 ++++++ query_classifier/qc_sqlite/qc_sqlite.c | 28 +++++++++++++++++++ server/core/query_classifier.c | 8 ++++++ 4 files changed, 73 insertions(+) diff --git a/include/maxscale/query_classifier.h b/include/maxscale/query_classifier.h index 438613eac..02db99d5d 100644 --- a/include/maxscale/query_classifier.h +++ b/include/maxscale/query_classifier.h @@ -87,6 +87,18 @@ typedef enum qc_parse_result } qc_parse_result_t; +/** + * QC_FIELD_INFO contains information about a field used in a statement. + */ +typedef struct qc_field_info +{ + char* database; /** Present if the field is of the form "a.b.c", NULL otherwise. */ + char* table; /** Present if the field is of the form "a.b", NULL otherwise. */ + char* column; /** Always present. */ + // TODO: Possibly add bits telling where the field is used; e.g. in the select + // TODO: part or the where part, or both. +} QC_FIELD_INFO; + /** * QUERY_CLASSIFIER defines the object a query classifier plugin must * implement and return. @@ -117,6 +129,7 @@ typedef struct query_classifier char** (*qc_get_database_names)(GWBUF* stmt, int* size); char* (*qc_get_prepare_name)(GWBUF* stmt); qc_query_op_t (*qc_get_prepare_operation)(GWBUF* stmt); + void (*qc_get_field_info)(GWBUF* stmt, const QC_FIELD_INFO** infos, size_t* n_infos); } QUERY_CLASSIFIER; /** @@ -223,6 +236,21 @@ qc_parse_result_t qc_parse(GWBUF* stmt); */ char* qc_get_affected_fields(GWBUF* stmt); +/** + * Returns information about affected fields. + * + * @param stmt A buffer containing a COM_QUERY packet. + * @param infos Pointer to pointer that after the call will point to an + * array of QC_FIELD_INFO:s. + * @param n_infos Pointer to size_t variable where the number of items + * in @c infos will be returned. + * + * @note The returned array belongs to the GWBUF and remains valid for as + * long as the GWBUF is valid. If the data is needed for longer than + * that, it must be copied. + */ +void qc_get_field_info(GWBUF* stmt, const QC_FIELD_INFO** infos, size_t* n_infos); + /** * Returns the statement, with literals replaced with question marks. * diff --git a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc index 48330808e..1041b6d44 100644 --- a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc +++ b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc @@ -2010,6 +2010,14 @@ qc_query_op_t qc_get_prepare_operation(GWBUF* stmt) return operation; } +void qc_get_field_info(GWBUF* stmt, const QC_FIELD_INFO** infos, size_t* n_infos) +{ + MXS_ERROR("qc_get_field_info not implemented yet."); + + *infos = NULL; + *n_infos = 0; +} + namespace { @@ -2156,6 +2164,7 @@ static QUERY_CLASSIFIER qc = qc_get_database_names, qc_get_prepare_name, qc_get_prepare_operation, + qc_get_field_info, }; /* @see function load_module in load_utils.c for explanation of the following diff --git a/query_classifier/qc_sqlite/qc_sqlite.c b/query_classifier/qc_sqlite/qc_sqlite.c index ccec8c165..d73d06bea 100644 --- a/query_classifier/qc_sqlite/qc_sqlite.c +++ b/query_classifier/qc_sqlite/qc_sqlite.c @@ -248,6 +248,21 @@ static bool ensure_query_is_parsed(GWBUF* query) return parsed; } +void free_field_infos(QC_FIELD_INFO* infos, size_t n_infos) +{ + if (infos) + { + for (int i = 0; i < n_infos; ++i) + { + MXS_FREE(infos[i].database); + MXS_FREE(infos[i].table); + MXS_FREE(infos[i].column); + } + + MXS_FREE(infos); + } +} + static void free_string_array(char** sa) { if (sa) @@ -2954,6 +2969,18 @@ static qc_query_op_t qc_sqlite_get_prepare_operation(GWBUF* query) return op; } +void qc_sqlite_get_field_info(GWBUF* stmt, const QC_FIELD_INFO** infos, size_t* n_infos) +{ + QC_TRACE(); + ss_dassert(this_unit.initialized); + ss_dassert(this_thread.initialized); + + MXS_ERROR("qc_get_field_info not implemented yet."); + + *infos = NULL; + *n_infos = 0; +} + /** * EXPORTS */ @@ -2979,6 +3006,7 @@ static QUERY_CLASSIFIER qc = qc_sqlite_get_database_names, qc_sqlite_get_prepare_name, qc_sqlite_get_prepare_operation, + qc_sqlite_get_field_info, }; diff --git a/server/core/query_classifier.c b/server/core/query_classifier.c index 5800f38e1..70d7f3cfe 100644 --- a/server/core/query_classifier.c +++ b/server/core/query_classifier.c @@ -197,6 +197,14 @@ char* qc_get_affected_fields(GWBUF* query) return classifier->qc_get_affected_fields(query); } +void qc_get_field_info(GWBUF* query, const QC_FIELD_INFO** infos, size_t* n_infos) +{ + QC_TRACE(); + ss_dassert(classifier); + + classifier->qc_get_field_info(query, infos, n_infos); +} + char** qc_get_database_names(GWBUF* query, int* sizep) { QC_TRACE(); From af65ee0ef99ee7ce5cbacddd007d7e22222bd535 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Thu, 3 Nov 2016 08:30:45 +0200 Subject: [PATCH 132/215] qc: Ensure type is uint32_t Some C++ compiler complains about signed being compared with unsigned. --- include/maxscale/query_classifier.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/maxscale/query_classifier.h b/include/maxscale/query_classifier.h index 02db99d5d..00a7688c5 100644 --- a/include/maxscale/query_classifier.h +++ b/include/maxscale/query_classifier.h @@ -401,7 +401,7 @@ const char* qc_op_to_string(qc_query_op_t op); */ static inline bool qc_query_is_type(uint32_t typemask, qc_query_type_t type) { - return (typemask & type) == type; + return (typemask & (uint32_t)type) == (uint32_t)type; } /** From 02d28a8d8d242b2dc7b6cc98790727205a3d1798 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 3 Nov 2016 08:37:24 +0200 Subject: [PATCH 133/215] Always use format strings in dcb_printf The luafilter didn't use a format string with dcb_printf which can lead to unexpected results if the returned string contains printf special characters. --- server/modules/filter/luafilter/luafilter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/modules/filter/luafilter/luafilter.c b/server/modules/filter/luafilter/luafilter.c index 137446f8c..dbfe5b2ae 100644 --- a/server/modules/filter/luafilter/luafilter.c +++ b/server/modules/filter/luafilter/luafilter.c @@ -570,7 +570,7 @@ static void diagnostic(FILTER *instance, void *fsession, DCB *dcb) lua_gettop(my_instance->global_lua_state); if (lua_isstring(my_instance->global_lua_state, -1)) { - dcb_printf(dcb, lua_tostring(my_instance->global_lua_state, -1)); + dcb_printf(dcb, "%s", lua_tostring(my_instance->global_lua_state, -1)); dcb_printf(dcb, "\n"); } } From 45f463c438271097a0c112d42b31f1198f0f307b Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Thu, 3 Nov 2016 09:19:28 +0200 Subject: [PATCH 134/215] Require GCC 4.8 for RocksDB GCC 4.8 is the first GCC version that fully implements C++11 --- .../filter/cache/storage/storage_rocksdb/BuildRocksDB.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/modules/filter/cache/storage/storage_rocksdb/BuildRocksDB.cmake b/server/modules/filter/cache/storage/storage_rocksdb/BuildRocksDB.cmake index bfe2aa37e..bdc630b4e 100644 --- a/server/modules/filter/cache/storage/storage_rocksdb/BuildRocksDB.cmake +++ b/server/modules/filter/cache/storage/storage_rocksdb/BuildRocksDB.cmake @@ -1,7 +1,7 @@ # Build RocksDB -if ((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND (NOT (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.7))) - message(STATUS "GCC >= 4.7, RocksDB is built.") +if ((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND (NOT (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.8))) + message(STATUS "GCC >= 4.8, RocksDB is built.") set(ROCKSDB_REPO "https://github.com/facebook/rocksdb.git" CACHE STRING "RocksDB Git repository") From 8778e0c81e5f771ac2af730085e81f54d834da0e Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Thu, 3 Nov 2016 09:35:49 +0200 Subject: [PATCH 135/215] Make Users const correct --- include/maxscale/users.h | 22 ++++++++-------- server/core/test/testusers.c | 6 ++--- server/core/users.c | 25 ++++++++++--------- server/modules/authenticator/cdc_plain_auth.c | 2 +- 4 files changed, 28 insertions(+), 27 deletions(-) diff --git a/include/maxscale/users.h b/include/maxscale/users.h index cdd919de4..d037d6cf4 100644 --- a/include/maxscale/users.h +++ b/include/maxscale/users.h @@ -61,16 +61,16 @@ typedef struct users unsigned char cksum[SHA_DIGEST_LENGTH]; /**< The users' table ckecksum */ } USERS; -extern USERS *users_alloc(); /**< Allocate a users table */ -extern void users_free(USERS *); /**< Free a users table */ -extern int users_add(USERS *, char *, char *); /**< Add a user to the users table */ -extern int users_delete(USERS *, char *); /**< Delete a user from the users table */ -extern char *users_fetch(USERS *, char *); /**< Fetch the authentication data for a user */ -extern int users_update(USERS *, char *, char *); /**< Change the password data for a user in - the users table */ -extern int users_default_loadusers(SERV_LISTENER *port); /**< A generic implementation of the authenticator - * loadusers entry point */ -extern void usersPrint(USERS *); /**< Print data about the users loaded */ -extern void dcb_usersPrint(DCB *, USERS *); /**< Print data about the users loaded */ +extern USERS *users_alloc(); /**< Allocate a users table */ +extern void users_free(USERS *); /**< Free a users table */ +extern int users_add(USERS *, const char *, const char *); /**< Add a user to the users table */ +extern int users_delete(USERS *, const char *); /**< Delete a user from the users table */ +extern const char *users_fetch(USERS *, const char *); /**< Fetch the authentication data for a user*/ +extern int users_update(USERS *, const char *, const char *); /**< Change the password data for a user in + the users table */ +extern int users_default_loadusers(SERV_LISTENER *port); /**< A generic implementation of the + authenticator loadusers entry point */ +extern void usersPrint(const USERS *); /**< Print data about the users loaded */ +extern void dcb_usersPrint(DCB *, const USERS *); /**< Print data about the users loaded */ MXS_END_DECLS diff --git a/server/core/test/testusers.c b/server/core/test/testusers.c index f6bea1ef6..5986e5a00 100644 --- a/server/core/test/testusers.c +++ b/server/core/test/testusers.c @@ -45,9 +45,9 @@ static int test1() { - USERS *users; - char *authdata; - int result, count; + USERS *users; + const char *authdata; + int result, count; /* Poll tests */ ss_dfprintf(stderr, diff --git a/server/core/users.c b/server/core/users.c index 2c2a7ca56..c136a7a6d 100644 --- a/server/core/users.c +++ b/server/core/users.c @@ -87,12 +87,12 @@ users_free(USERS *users) * @return The number of users added to the table */ int -users_add(USERS *users, char *user, char *auth) +users_add(USERS *users, const char *user, const char *auth) { int add; atomic_add(&users->stats.n_adds, 1); - add = hashtable_add(users->data, user, auth); + add = hashtable_add(users->data, (char*)user, (char*)auth); atomic_add(&users->stats.n_entries, add); return add; } @@ -105,12 +105,12 @@ users_add(USERS *users, char *user, char *auth) * @return The number of users deleted from the table */ int -users_delete(USERS *users, char *user) +users_delete(USERS *users, const char *user) { int del; atomic_add(&users->stats.n_deletes, 1); - del = hashtable_delete(users->data, user); + del = hashtable_delete(users->data, (char*)user); atomic_add(&users->stats.n_entries, -del); return del; } @@ -122,11 +122,12 @@ users_delete(USERS *users, char *user) * @param user The user name * @return The authentication data or NULL on error */ -char -*users_fetch(USERS *users, char *user) +const char +*users_fetch(USERS *users, const char *user) { atomic_add(&users->stats.n_fetches, 1); - return hashtable_fetch(users->data, user); + // TODO: Returning data from the hashtable is not threadsafe. + return hashtable_fetch(users->data, (char*)user); } /** @@ -139,13 +140,13 @@ char * @return Number of users updated */ int -users_update(USERS *users, char *user, char *auth) +users_update(USERS *users, const char *user, const char *auth) { - if (hashtable_delete(users->data, user) == 0) + if (hashtable_delete(users->data, (char*)user) == 0) { return 0; } - return hashtable_add(users->data, user, auth); + return hashtable_add(users->data, (char*)user, (char*)auth); } /** @@ -154,7 +155,7 @@ users_update(USERS *users, char *user, char *auth) * @param users The users table */ void -usersPrint(USERS *users) +usersPrint(const USERS *users) { printf("Users table data\n"); hashtable_stats(users->data); @@ -167,7 +168,7 @@ usersPrint(USERS *users) * @param users The users table */ void -dcb_usersPrint(DCB *dcb, USERS *users) +dcb_usersPrint(DCB *dcb, const USERS *users) { if (users == NULL || users->data == NULL) { diff --git a/server/modules/authenticator/cdc_plain_auth.c b/server/modules/authenticator/cdc_plain_auth.c index 1b19d9b20..19c60a29f 100644 --- a/server/modules/authenticator/cdc_plain_auth.c +++ b/server/modules/authenticator/cdc_plain_auth.c @@ -139,7 +139,7 @@ static int cdc_auth_check(DCB *dcb, CDC_protocol *protocol, char *username, uint { if (dcb->listener->users) { - char *user_password = users_fetch(dcb->listener->users, username); + const char *user_password = users_fetch(dcb->listener->users, username); if (user_password) { From f38b510d2b0c72f470882add0e73364290983795 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 1 Nov 2016 23:16:25 +0200 Subject: [PATCH 136/215] MXS-962: Add nonpositive priority for Galera nodes If a Galera node has a nonpositive priority, the node will never be chosen as the master. This gives the user more control over how the master is chosen. --- Documentation/Monitors/Galera-Monitor.md | 28 +++++++++++++++++--- server/modules/monitor/galeramon/galeramon.c | 12 ++++++--- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/Documentation/Monitors/Galera-Monitor.md b/Documentation/Monitors/Galera-Monitor.md index a74545ad7..d4d65488e 100644 --- a/Documentation/Monitors/Galera-Monitor.md +++ b/Documentation/Monitors/Galera-Monitor.md @@ -64,7 +64,16 @@ use_priority=true ## Interaction with Server Priorities -If the `use_priority` option is set and a server is configured with the `priority=` parameter, galeramon will use that as the basis on which the master node is chosen. This requires the `disable_master_role_setting` to be undefined or disabled. The server with the lowest value in `priority` will be chosen as the master node when a replacement Galera node is promoted to a master server inside MaxScale. +If the `use_priority` option is set and a server is configured with the +`priority=` parameter, galeramon will use that as the basis on which the +master node is chosen. This requires the `disable_master_role_setting` to be +undefined or disabled. The server with the lowest positive value in _priority_ +will be chosen as the master node when a replacement Galera node is promoted to +a master server inside MaxScale. + +Nodes with a non-positive value (_priority_ <= 0) will never be chosen as the master. This allows +you to mark some servers as permanent slaves by assigning a non-positive value +into _priority_. Here is an example with two servers. @@ -86,8 +95,21 @@ type=server address=192.168.122.103 port=3306 priority=2 + +[node-4] +type=server +address=192.168.122.104 +port=3306 +priority=0 ``` -In this example `node-1` is always used as the master if available. If `node-1` is not available, then the next node with the highest priority rank is used. In this case it would be `node-3`. If both `node-1` and `node-3` were down, then `node-2` would be used. Nodes without priority are considered as having the lowest priority rank and will be used only if all nodes with priority ranks are not available. +In this example `node-1` is always used as the master if available. If `node-1` +is not available, then the next node with the highest priority rank is used. In +this case it would be `node-3`. If both `node-1` and `node-3` were down, then +`node-2` would be used. Because `node-4` has a value of 0 in _priority_, it will +never be the master. Nodes without priority are considered as having the lowest +priority rank and will be used only if all nodes with priority ranks are not +available. -With priority ranks you can control the order in which MaxScale chooses the master node. This will allow for a controlled failure and replacement of nodes. +With priority ranks you can control the order in which MaxScale chooses the +master node. This will allow for a controlled failure and replacement of nodes. diff --git a/server/modules/monitor/galeramon/galeramon.c b/server/modules/monitor/galeramon/galeramon.c index c3b96a68e..db28a6123 100644 --- a/server/modules/monitor/galeramon/galeramon.c +++ b/server/modules/monitor/galeramon/galeramon.c @@ -681,11 +681,15 @@ static MONITOR_SERVERS *get_candidate_master(MONITOR* mon) if (handle->use_priority && (value = serverGetParameter(moitor_servers->server, "priority")) != NULL) { - currval = atoi(value); - if (currval < minval && currval > 0) + /** The server has a priority */ + if ((currval = atoi(value)) > 0) { - minval = currval; - candidate_master = moitor_servers; + /** The priority is valid */ + if (currval < minval && currval > 0) + { + minval = currval; + candidate_master = moitor_servers; + } } } else if (moitor_servers->server->node_id >= 0 && From 96547a1c0db15423da4dee47c1ef003b9c62f2f2 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 4 Nov 2016 10:48:02 +0200 Subject: [PATCH 137/215] Fix failure to exit on configuration error The check for the success of the configuration file always resulted in a successful return value even if the loading failed. In addition to this, a log message referred to the active configuration when the active configuration was set only after the processing was complete. Since configuration failures are always fatal, there's no harm in preemptively setting the active configuration to the one currently being processed. --- server/core/config.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/server/core/config.c b/server/core/config.c index f85d685d6..7958cb175 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -601,9 +601,9 @@ config_load_and_process(const char* filename, bool (*process_config)(CONFIG_CONT if (rval) { - if (check_config_objects(ccontext.next) && process_config(ccontext.next)) + if (!check_config_objects(ccontext.next) || !process_config(ccontext.next)) { - rval = true; + rval = false; } } } @@ -633,13 +633,9 @@ config_load(const char *filename) global_defaults(); feedback_defaults(); + config_file = filename; bool rval = config_load_and_process(filename, process_config_context); - if (rval) - { - config_file = filename; - } - return rval; } From 5b7e5aa36f7413cef13ab614427c6f6896826577 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Fri, 4 Nov 2016 15:48:14 +0100 Subject: [PATCH 138/215] SERVER_MORE_RESULTS_EXIST is now detected The filter can detect SERVER_MORE_RESULTS_EXIST which means the server is sending more result sets: example: DROP PROCEDURE IF EXISTS multi; DELIMITER $$ CREATE PROCEDURE multi() BEGIN SELECT 1; SELECT id FROM t2 limit 40; set @a=4; SELECT 2; END$$ DELIMITER ; MySQL> call multi() --- server/modules/filter/maxrows/maxrows.c | 56 ++++++++++++++++++++++--- server/modules/filter/maxrows/maxrows.h | 7 ++++ 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/server/modules/filter/maxrows/maxrows.c b/server/modules/filter/maxrows/maxrows.c index 2f8748848..738eee875 100644 --- a/server/modules/filter/maxrows/maxrows.c +++ b/server/modules/filter/maxrows/maxrows.c @@ -21,6 +21,8 @@ * * Date Who Description * 26/10/2016 Massimiliano Pinto Initial implementation + * 04/11/2016 Massimiliano Pinto Addition of SERVER_MORE_RESULTS_EXIST flag (0x0008) + * detection in handle_expecting_rows(). * * @endverbatim */ @@ -551,7 +553,7 @@ static int handle_expecting_nothing(MAXROWS_SESSION_DATA *csdata) { ss_dassert(csdata->state == MAXROWS_EXPECTING_NOTHING); ss_dassert(csdata->res.data); - MXS_ERROR("Received data from the backend althoug we were expecting nothing."); + MXS_ERROR("Received data from the backend although we were expecting nothing."); ss_dassert(!true); return send_upstream(csdata); @@ -651,8 +653,8 @@ static int handle_expecting_rows(MAXROWS_SESSION_DATA *csdata) while (!insufficient && (buflen - csdata->res.offset >= MYSQL_HEADER_LEN)) { - uint8_t header[MYSQL_HEADER_LEN + 1]; - gwbuf_copy_data(csdata->res.data, csdata->res.offset, MYSQL_HEADER_LEN + 1, header); + uint8_t header[MAXROWS_EOF_PACKET_LEN]; //it holds a full EOF packet + gwbuf_copy_data(csdata->res.data, csdata->res.offset, MAXROWS_EOF_PACKET_LEN, header); size_t packetlen = MYSQL_HEADER_LEN + MYSQL_GET_PACKET_LEN(header); @@ -660,13 +662,17 @@ static int handle_expecting_rows(MAXROWS_SESSION_DATA *csdata) { // We have at least one complete packet. int command = (int)MYSQL_GET_COMMAND(header); + int flags = gw_mysql_get_byte2(header + MAXROWS_MYSQL_EOF_PACKET_FLAGS_OFFSET); switch (command) { - case 0xfe: // EOF, the one after the rows. + case 0xff: // ERR packet after the rows. csdata->res.offset += packetlen; ss_dassert(csdata->res.offset == buflen); - + /* + * This is the ERR packet that could terminate a Multi-Resultset. + * Reply to client is the same as in case 0x0 + */ if (csdata->state == MAXROWS_DISCARDING_RESPONSE) { rv = send_ok_upstream(csdata); @@ -675,8 +681,46 @@ static int handle_expecting_rows(MAXROWS_SESSION_DATA *csdata) { rv = send_upstream(csdata); } - csdata->state = MAXROWS_EXPECTING_NOTHING; + + break; + + case 0x0: // OK packet after the rows. + /* OK could the last packet in the Multi-Resultset transmission: + * handle DISCARD or send all the data. + * But it could also be sent instead of EOF from as in MySQL 5.7.5 + * if client sends CLIENT_DEPRECATE_EOF capability OK packet could + * have the SERVER_MORE_RESULTS_EXIST flag. + * Note: Flags in the OK packet are at the same offset as in EOF. + */ + case 0xfe: // EOF, the one after the rows. + csdata->res.offset += packetlen; + ss_dassert(csdata->res.offset == buflen); + + /* EOF could be the last packet in the transmission: + * check first whether SERVER_MORE_RESULTS_EXIST flag is set. + * If so more results set could come. The end of stream + * will be an OK packet. + */ + + if (!(flags & SERVER_MORE_RESULTS_EXIST)) + { + if (csdata->state == MAXROWS_DISCARDING_RESPONSE) + { + rv = send_ok_upstream(csdata); + } + else + { + rv = send_upstream(csdata); + } + + csdata->state = MAXROWS_EXPECTING_NOTHING; + } + else + { + C_DEBUG("EOF or OK seen with SERVER_MORE_RESULTS_EXIST flag: waiting for more data"); + } + break; case 0xfb: // NULL diff --git a/server/modules/filter/maxrows/maxrows.h b/server/modules/filter/maxrows/maxrows.h index 45d354048..7b94f9b9e 100644 --- a/server/modules/filter/maxrows/maxrows.h +++ b/server/modules/filter/maxrows/maxrows.h @@ -17,6 +17,13 @@ MXS_BEGIN_DECLS #define MAXROWS_OK_PACKET_LEN 11 +#define MAXROWS_EOF_PACKET_LEN 9 + +/* + * The EOF packet 2 bytes flags start after: + * network header (4 bytes) + eof indicator (1) + 2 bytes warnings count) + */ +#define MAXROWS_MYSQL_EOF_PACKET_FLAGS_OFFSET (MYSQL_HEADER_LEN + 1 + 2) #define MAXROWS_DEBUG_NONE 0 #define MAXROWS_DEBUG_MATCHING 1 From fa5a85858289228fe5cb025f00ff230329188adf Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Thu, 3 Nov 2016 13:38:28 +0200 Subject: [PATCH 139/215] MXS-884: Implement qc_get_fields_infos. We now collect more information about a particular field and then, if necessary, copy the data over into affected_fields if someone is interested in that. The comparison program is extended as well, but qc_get_fields_infos() is not tested, because qc_mysqlembedded does not implement this yet. --- query_classifier/qc_sqlite/qc_sqlite.c | 463 +++++++++++++++++-------- query_classifier/test/compare.cc | 118 +++++++ 2 files changed, 443 insertions(+), 138 deletions(-) diff --git a/query_classifier/qc_sqlite/qc_sqlite.c b/query_classifier/qc_sqlite/qc_sqlite.c index d73d06bea..e7d50bab1 100644 --- a/query_classifier/qc_sqlite/qc_sqlite.c +++ b/query_classifier/qc_sqlite/qc_sqlite.c @@ -61,8 +61,6 @@ typedef struct qc_sqlite_info uint32_t types; // The types of the query. qc_query_op_t operation; // The operation in question. char* affected_fields; // The affected fields. - size_t affected_fields_len; // The used length of affected_fields. - size_t affected_fields_capacity; // The capacity of affected_fields. bool is_real_query; // SELECT, UPDATE, INSERT, DELETE or a variation. bool has_clause; // Has WHERE or HAVING. char** table_names; // Array of table names used in the query. @@ -82,6 +80,9 @@ typedef struct qc_sqlite_info qc_query_op_t prepare_operation; // The operation of a prepared statement. char* preparable_stmt; // The preparable statement. size_t preparable_stmt_length; // The length of the preparable statement. + QC_FIELD_INFO *field_infos; // Pointer to array of QC_FIELD_INFOs. + size_t field_infos_len; // The used entries in field_infos. + size_t field_infos_capacity; // The capacity of the field_infos array. } QC_SQLITE_INFO; typedef enum qc_log_level @@ -123,11 +124,11 @@ typedef enum qc_token_position QC_TOKEN_RIGHT, // To the right, e.g: "b" in "a = b". } qc_token_position_t; -static void append_affected_field(QC_SQLITE_INFO* info, const char* s); static void buffer_object_free(void* data); static char** copy_string_array(char** strings, int* pn); static void enlarge_string_array(size_t n, size_t len, char*** ppzStrings, size_t* pCapacity); static bool ensure_query_is_parsed(GWBUF* query); +static void free_field_infos(QC_FIELD_INFO* infos, size_t n_infos); static void free_string_array(char** sa); static QC_SQLITE_INFO* get_query_info(GWBUF* query); static QC_SQLITE_INFO* info_alloc(void); @@ -140,17 +141,17 @@ static bool parse_query(GWBUF* query); static void parse_query_string(const char* query, size_t len); static bool query_is_parsed(GWBUF* query); static bool should_exclude(const char* zName, const ExprList* pExclude); -static void update_affected_fields(QC_SQLITE_INFO* info, - int prev_token, - const Expr* pExpr, - qc_token_position_t pos, - const ExprList* pExclude); -static void update_affected_fields_from_exprlist(QC_SQLITE_INFO* info, - const ExprList* pEList, const ExprList* pExclude); -static void update_affected_fields_from_idlist(QC_SQLITE_INFO* info, - const IdList* pIds, const ExprList* pExclude); -static void update_affected_fields_from_select(QC_SQLITE_INFO* info, - const Select* pSelect, const ExprList* pExclude); +static void update_fields_infos(QC_SQLITE_INFO* info, + int prev_token, + const Expr* pExpr, + qc_token_position_t pos, + const ExprList* pExclude); +static void update_fields_infos_from_exprlist(QC_SQLITE_INFO* info, + const ExprList* pEList, const ExprList* pExclude); +static void update_fields_infos_from_idlist(QC_SQLITE_INFO* info, + const IdList* pIds, const ExprList* pExclude); +static void update_fields_infos_from_select(QC_SQLITE_INFO* info, + const Select* pSelect, const ExprList* pExclude); static void update_database_names(QC_SQLITE_INFO* info, const char* name); static void update_names(QC_SQLITE_INFO* info, const char* zDatabase, const char* zTable); static void update_names_from_srclist(QC_SQLITE_INFO* info, const SrcList* pSrc); @@ -248,7 +249,7 @@ static bool ensure_query_is_parsed(GWBUF* query) return parsed; } -void free_field_infos(QC_FIELD_INFO* infos, size_t n_infos) +static void free_field_infos(QC_FIELD_INFO* infos, size_t n_infos) { if (infos) { @@ -311,6 +312,7 @@ static void info_finish(QC_SQLITE_INFO* info) free_string_array(info->database_names); free(info->prepare_name); free(info->preparable_stmt); + free_field_infos(info->field_infos, info->field_infos_len); } static void info_free(QC_SQLITE_INFO* info) @@ -331,8 +333,6 @@ static QC_SQLITE_INFO* info_init(QC_SQLITE_INFO* info) info->types = QUERY_TYPE_UNKNOWN; info->operation = QUERY_OP_UNDEFINED; info->affected_fields = NULL; - info->affected_fields_len = 0; - info->affected_fields_capacity = 0; info->is_real_query = false; info->has_clause = false; info->table_names = NULL; @@ -352,6 +352,9 @@ static QC_SQLITE_INFO* info_init(QC_SQLITE_INFO* info) info->prepare_operation = QUERY_OP_UNDEFINED; info->preparable_stmt = NULL; info->preparable_stmt_length = 0; + info->field_infos = NULL; + info->field_infos_len = 0; + info->field_infos_capacity = 0; return info; } @@ -643,42 +646,6 @@ static void log_invalid_data(GWBUF* query, const char* message) } } -static void append_affected_field(QC_SQLITE_INFO* info, const char* s) -{ - size_t len = strlen(s); - size_t required_len = info->affected_fields_len + len + 1; // 1 for NULL - - if (info->affected_fields_len != 0) - { - required_len += 1; // " " between fields - } - - if (required_len > info->affected_fields_capacity) - { - if (info->affected_fields_capacity == 0) - { - info->affected_fields_capacity = 32; - } - - while (required_len > info->affected_fields_capacity) - { - info->affected_fields_capacity *= 2; - } - - info->affected_fields = MXS_REALLOC(info->affected_fields, info->affected_fields_capacity); - MXS_ABORT_IF_NULL(info->affected_fields); - } - - if (info->affected_fields_len != 0) - { - strcpy(info->affected_fields + info->affected_fields_len, " "); - info->affected_fields_len += 1; - } - - strcpy(info->affected_fields + info->affected_fields_len, s); - info->affected_fields_len += len; -} - static bool should_exclude(const char* zName, const ExprList* pExclude) { int i; @@ -714,36 +681,181 @@ static bool should_exclude(const char* zName, const ExprList* pExclude) return i != pExclude->nExpr; } -static void update_affected_fields(QC_SQLITE_INFO* info, - int prev_token, - const Expr* pExpr, - qc_token_position_t pos, - const ExprList* pExclude) +static void update_field_infos(QC_SQLITE_INFO* info, + const char* database, + const char* table, + const char* column, + const ExprList* pExclude) +{ + ss_dassert(column); + + // If only a column is specified, but not a table or database and we + // have a list of expressions that should be excluded, we check if the column + // value is present in that list. This is in order to exclude the second "d" in + // a statement like "select a as d from x where d = 2". + if (column && !table && !database && pExclude && should_exclude(column, pExclude)) + { + return; + } + + QC_FIELD_INFO item = { (char*)database, (char*)table, (char*)column }; + + int i; + for (i = 0; i < info->field_infos_len; ++i) + { + QC_FIELD_INFO* field_info = info->field_infos + i; + + if (strcasecmp(item.column, field_info->column) == 0) + { + if (!item.table && !field_info->table) + { + ss_dassert(!item.database && !field_info->database); + break; + } + else if (item.table && field_info->table && (strcmp(item.table, field_info->table) == 0)) + { + if (!item.database && !field_info->database) + { + break; + } + else if (item.database && + field_info->database && + (strcmp(item.database, field_info->database) == 0)) + { + break; + } + } + } + } + + QC_FIELD_INFO* field_infos = NULL; + + if (i == info->field_infos_len) // If true, the field was not present already. + { + if (info->field_infos_len < info->field_infos_capacity) + { + field_infos = info->field_infos; + } + else + { + size_t capacity = info->field_infos_capacity ? 2 * info->field_infos_capacity : 8; + field_infos = MXS_REALLOC(info->field_infos, capacity * sizeof(QC_FIELD_INFO)); + + if (field_infos) + { + info->field_infos = field_infos; + info->field_infos_capacity = capacity; + } + } + } + + // If field_infos is NULL, then the field was found and has already been noted. + if (field_infos) + { + item.database = item.database ? MXS_STRDUP(item.database) : NULL; + item.table = item.table ? MXS_STRDUP(item.table) : NULL; + ss_dassert(item.column); + item.column = MXS_STRDUP(item.column); + + // We are happy if we at least could dup the column. + + if (item.column) + { + field_infos[info->field_infos_len++] = item; + } + } +} + +static void update_field_infos_from_expr(QC_SQLITE_INFO* info, + const struct Expr* pExpr, + const ExprList* pExclude) +{ + QC_FIELD_INFO item = {}; + + if (pExpr->op == TK_ASTERISK) + { + item.column = "*"; + } + else if (pExpr->op == TK_ID) + { + // select a from... + item.column = pExpr->u.zToken; + } + else if (pExpr->op == TK_DOT) + { + if (pExpr->pLeft->op == TK_ID && + (pExpr->pRight->op == TK_ID || pExpr->pRight->op == TK_ASTERISK)) + { + // select a.b from... + item.table = pExpr->pLeft->u.zToken; + if (pExpr->pRight->op == TK_ID) + { + item.column = pExpr->pRight->u.zToken; + } + else + { + item.column = "*"; + } + } + else if (pExpr->pLeft->op == TK_ID && + pExpr->pRight->op == TK_DOT && + pExpr->pRight->pLeft->op == TK_ID && + (pExpr->pRight->pRight->op == TK_ID || pExpr->pRight->pRight->op == TK_ASTERISK)) + { + // select a.b.c from... + item.database = pExpr->pLeft->u.zToken; + item.table = pExpr->pRight->pLeft->u.zToken; + if (pExpr->pRight->pRight->op == TK_ID) + { + item.column = pExpr->pRight->pRight->u.zToken; + } + else + { + item.column = "*"; + } + } + } + + if (item.column) + { + bool should_update = true; + + if ((pExpr->flags & EP_DblQuoted) == 0) + { + if ((strcasecmp(item.column, "true") == 0) || (strcasecmp(item.column, "false") == 0)) + { + should_update = false; + } + } + + if (should_update) + { + update_field_infos(info, item.database, item.table, item.column, pExclude); + } + } +} + + +static void update_fields_infos(QC_SQLITE_INFO* info, + int prev_token, + const Expr* pExpr, + qc_token_position_t pos, + const ExprList* pExclude) { const char* zToken = pExpr->u.zToken; switch (pExpr->op) { - case TK_ASTERISK: // "select *" - append_affected_field(info, "*"); + case TK_ASTERISK: // select * + update_field_infos_from_expr(info, pExpr, pExclude); break; - case TK_DOT: - // In case of "X.Y" qc_mysqlembedded returns "Y". - update_affected_fields(info, TK_DOT, pExpr->pRight, QC_TOKEN_RIGHT, pExclude); + case TK_DOT: // select a.b ... select a.b.c + update_field_infos_from_expr(info, pExpr, pExclude); break; - case TK_ID: - if ((pExpr->flags & EP_DblQuoted) == 0) - { - if ((strcasecmp(zToken, "true") != 0) && (strcasecmp(zToken, "false") != 0)) - { - if (!pExclude || !should_exclude(zToken, pExclude)) - { - append_affected_field(info, zToken); - } - } - } + case TK_ID: // select a + update_field_infos_from_expr(info, pExpr, pExclude); break; case TK_VARIABLE: @@ -804,12 +916,12 @@ static void update_affected_fields(QC_SQLITE_INFO* info, if (pExpr->pLeft) { - update_affected_fields(info, pExpr->op, pExpr->pLeft, QC_TOKEN_LEFT, pExclude); + update_fields_infos(info, pExpr->op, pExpr->pLeft, QC_TOKEN_LEFT, pExclude); } if (pExpr->pRight) { - update_affected_fields(info, pExpr->op, pExpr->pRight, QC_TOKEN_RIGHT, pExclude); + update_fields_infos(info, pExpr->op, pExpr->pRight, QC_TOKEN_RIGHT, pExclude); } if (pExpr->x.pList) @@ -819,7 +931,7 @@ static void update_affected_fields(QC_SQLITE_INFO* info, case TK_BETWEEN: case TK_CASE: case TK_FUNCTION: - update_affected_fields_from_exprlist(info, pExpr->x.pList, pExclude); + update_fields_infos_from_exprlist(info, pExpr->x.pList, pExclude); break; case TK_EXISTS: @@ -827,11 +939,11 @@ static void update_affected_fields(QC_SQLITE_INFO* info, case TK_SELECT: if (pExpr->flags & EP_xIsSelect) { - update_affected_fields_from_select(info, pExpr->x.pSelect, pExclude); + update_fields_infos_from_select(info, pExpr->x.pSelect, pExclude); } else { - update_affected_fields_from_exprlist(info, pExpr->x.pList, pExclude); + update_fields_infos_from_exprlist(info, pExpr->x.pList, pExclude); } break; } @@ -840,19 +952,19 @@ static void update_affected_fields(QC_SQLITE_INFO* info, } } -static void update_affected_fields_from_exprlist(QC_SQLITE_INFO* info, - const ExprList* pEList, - const ExprList* pExclude) +static void update_fields_infos_from_exprlist(QC_SQLITE_INFO* info, + const ExprList* pEList, + const ExprList* pExclude) { for (int i = 0; i < pEList->nExpr; ++i) { struct ExprList_item* pItem = &pEList->a[i]; - update_affected_fields(info, 0, pItem->pExpr, QC_TOKEN_MIDDLE, pExclude); + update_fields_infos(info, 0, pItem->pExpr, QC_TOKEN_MIDDLE, pExclude); } } -static void update_affected_fields_from_idlist(QC_SQLITE_INFO* info, +static void update_fields_infos_from_idlist(QC_SQLITE_INFO* info, const IdList* pIds, const ExprList* pExclude) { @@ -860,14 +972,11 @@ static void update_affected_fields_from_idlist(QC_SQLITE_INFO* info, { struct IdList_item* pItem = &pIds->a[i]; - if (!pExclude || !should_exclude(pItem->zName, pExclude)) - { - append_affected_field(info, pItem->zName); - } + update_field_infos(info, NULL, NULL, pItem->zName, pExclude); } } -static void update_affected_fields_from_select(QC_SQLITE_INFO* info, +static void update_fields_infos_from_select(QC_SQLITE_INFO* info, const Select* pSelect, const ExprList* pExclude) { @@ -885,7 +994,7 @@ static void update_affected_fields_from_select(QC_SQLITE_INFO* info, if (pSrc->a[i].pSelect) { - update_affected_fields_from_select(info, pSrc->a[i].pSelect, pExclude); + update_fields_infos_from_select(info, pSrc->a[i].pSelect, pExclude); } #ifdef QC_COLLECT_NAMES_FROM_USING @@ -895,7 +1004,7 @@ static void update_affected_fields_from_select(QC_SQLITE_INFO* info, // does not reveal its value, right? if (pSrc->a[i].pUsing) { - update_affected_fields_from_idlist(info, pSrc->a[i].pUsing, pSelect->pEList); + update_fields_infos_from_idlist(info, pSrc->a[i].pUsing, pSelect->pEList); } #endif } @@ -903,24 +1012,24 @@ static void update_affected_fields_from_select(QC_SQLITE_INFO* info, if (pSelect->pEList) { - update_affected_fields_from_exprlist(info, pSelect->pEList, NULL); + update_fields_infos_from_exprlist(info, pSelect->pEList, NULL); } if (pSelect->pWhere) { info->has_clause = true; - update_affected_fields(info, 0, pSelect->pWhere, QC_TOKEN_MIDDLE, pSelect->pEList); + update_fields_infos(info, 0, pSelect->pWhere, QC_TOKEN_MIDDLE, pSelect->pEList); } if (pSelect->pGroupBy) { - update_affected_fields_from_exprlist(info, pSelect->pGroupBy, pSelect->pEList); + update_fields_infos_from_exprlist(info, pSelect->pGroupBy, pSelect->pEList); } if (pSelect->pHaving) { info->has_clause = true; - update_affected_fields(info, 0, pSelect->pHaving, QC_TOKEN_MIDDLE, pSelect->pEList); + update_fields_infos(info, 0, pSelect->pHaving, QC_TOKEN_MIDDLE, pSelect->pEList); } } @@ -1165,7 +1274,7 @@ void mxs_sqlite3CreateView(Parse *pParse, /* The parsing context */ if (pSelect) { - update_affected_fields_from_select(info, pSelect, NULL); + update_fields_infos_from_select(info, pSelect, NULL); info->is_real_query = false; } @@ -1235,7 +1344,7 @@ void mxs_sqlite3DeleteFrom(Parse* pParse, SrcList* pTabList, Expr* pWhere, SrcLi if (pWhere) { - update_affected_fields(info, 0, pWhere, QC_TOKEN_MIDDLE, 0); + update_fields_infos(info, 0, pWhere, QC_TOKEN_MIDDLE, 0); } exposed_sqlite3ExprDelete(pParse->db, pWhere); @@ -1299,7 +1408,7 @@ void mxs_sqlite3EndTable(Parse *pParse, /* Parse context */ { if (pSelect) { - update_affected_fields_from_select(info, pSelect, NULL); + update_fields_infos_from_select(info, pSelect, NULL); info->is_real_query = false; } else if (pOldTable) @@ -1345,17 +1454,17 @@ void mxs_sqlite3Insert(Parse* pParse, if (pColumns) { - update_affected_fields_from_idlist(info, pColumns, NULL); + update_fields_infos_from_idlist(info, pColumns, NULL); } if (pSelect) { - update_affected_fields_from_select(info, pSelect, NULL); + update_fields_infos_from_select(info, pSelect, NULL); } if (pSet) { - update_affected_fields_from_exprlist(info, pSet, NULL); + update_fields_infos_from_exprlist(info, pSet, NULL); } exposed_sqlite3SrcListDelete(pParse->db, pTabList); @@ -1479,18 +1588,13 @@ void mxs_sqlite3Update(Parse* pParse, SrcList* pTabList, ExprList* pChanges, Exp { struct ExprList_item* pItem = &pChanges->a[i]; - if (pItem->zName) - { - append_affected_field(info, pItem->zName); - } - - update_affected_fields(info, 0, pItem->pExpr, QC_TOKEN_MIDDLE, NULL); + update_fields_infos(info, 0, pItem->pExpr, QC_TOKEN_MIDDLE, NULL); } } if (pWhere) { - update_affected_fields(info, 0, pWhere, QC_TOKEN_MIDDLE, NULL); + update_fields_infos(info, 0, pWhere, QC_TOKEN_MIDDLE, NULL); } exposed_sqlite3SrcListDelete(pParse->db, pTabList); @@ -1516,7 +1620,7 @@ void maxscaleCollectInfoFromSelect(Parse* pParse, Select* pSelect) info->types = QUERY_TYPE_READ; } - update_affected_fields_from_select(info, pSelect, NULL); + update_fields_infos_from_select(info, pSelect, NULL); } void maxscaleAlterTable(Parse *pParse, /* Parser context. */ @@ -1668,9 +1772,12 @@ void maxscaleExplain(Parse* pParse, SrcList* pName) info->status = QC_QUERY_PARSED; info->types = QUERY_TYPE_READ; update_names(info, "information_schema", "COLUMNS"); - append_affected_field(info, - "COLUMN_DEFAULT COLUMN_KEY COLUMN_NAME " - "COLUMN_TYPE EXTRA IS_NULLABLE"); + update_field_infos(info, "information_schema", "COLUMNS", "COLUMN_DEFAULT", NULL); + update_field_infos(info, "information_schema", "COLUMNS", "COLUMN_KEY", NULL); + update_field_infos(info, "information_schema", "COLUMNS", "COLUMN_NAME", NULL); + update_field_infos(info, "information_schema", "COLUMNS", "COLUMN_TYPE", NULL); + update_field_infos(info, "information_schema", "COLUMNS", "EXTRA", NULL); + update_field_infos(info, "information_schema", "COLUMNS", "IS_NULLABLE", NULL); exposed_sqlite3SrcListDelete(pParse->db, pName); } @@ -2190,7 +2297,7 @@ void maxscaleSet(Parse* pParse, int scope, mxs_set_t kind, ExprList* pList) if (pValue->op == TK_SELECT) { - update_affected_fields_from_select(info, pValue->x.pSelect, NULL); + update_fields_infos_from_select(info, pValue->x.pSelect, NULL); info->is_real_query = false; // TODO: This is what qc_mysqlembedded claims. } } @@ -2246,16 +2353,24 @@ extern void maxscaleShow(Parse* pParse, MxsShow* pShow) update_names(info, "information_schema", "COLUMNS"); if (pShow->data == MXS_SHOW_COLUMNS_FULL) { - append_affected_field(info, - "COLLATION_NAME COLUMN_COMMENT COLUMN_DEFAULT " - "COLUMN_KEY COLUMN_NAME COLUMN_TYPE EXTRA " - "IS_NULLABLE PRIVILEGES"); + update_field_infos(info, "information_schema", "COLUMNS", "COLLATION_NAME", NULL); + update_field_infos(info, "information_schema", "COLUMNS", "COLUMN_COMMENT", NULL); + update_field_infos(info, "information_schema", "COLUMNS", "COLUMN_DEFAULT", NULL); + update_field_infos(info, "information_schema", "COLUMNS", "COLUMN_KEY", NULL); + update_field_infos(info, "information_schema", "COLUMNS", "COLUMN_NAME", NULL); + update_field_infos(info, "information_schema", "COLUMNS", "COLUMN_TYPE", NULL); + update_field_infos(info, "information_schema", "COLUMNS", "EXTRA", NULL); + update_field_infos(info, "information_schema", "COLUMNS", "IS_NULLABLE", NULL); + update_field_infos(info, "information_schema", "COLUMNS", "PRIVILEGES", NULL); } else { - append_affected_field(info, - "COLUMN_DEFAULT COLUMN_KEY COLUMN_NAME " - "COLUMN_TYPE EXTRA IS_NULLABLE"); + update_field_infos(info, "information_schema", "COLUMNS", "COLUMN_DEFAULT", NULL); + update_field_infos(info, "information_schema", "COLUMNS", "COLUMN_KEY", NULL); + update_field_infos(info, "information_schema", "COLUMNS", "COLUMN_NAME", NULL); + update_field_infos(info, "information_schema", "COLUMNS", "COLUMN_TYPE", NULL); + update_field_infos(info, "information_schema", "COLUMNS", "EXTRA", NULL); + update_field_infos(info, "information_schema", "COLUMNS", "IS_NULLABLE", NULL); } } break; @@ -2278,7 +2393,7 @@ extern void maxscaleShow(Parse* pParse, MxsShow* pShow) { info->types = QUERY_TYPE_SHOW_DATABASES; update_names(info, "information_schema", "SCHEMATA"); - append_affected_field(info, "SCHEMA_NAME"); + update_field_infos(info, "information_schema", "SCHEMATA", "SCHEMA_NAME", NULL); } break; @@ -2288,10 +2403,19 @@ extern void maxscaleShow(Parse* pParse, MxsShow* pShow) { info->types = QUERY_TYPE_WRITE; update_names(info, "information_schema", "STATISTICS"); - append_affected_field(info, - "CARDINALITY COLLATION COLUMN_NAME COMMENT INDEX_COMMENT " - "INDEX_NAME INDEX_TYPE NON_UNIQUE NULLABLE PACKED SEQ_IN_INDEX " - "SUB_PART TABLE_NAME"); + update_field_infos(info, "information_schema", "STATISTICS", "CARDINALITY", NULL); + update_field_infos(info, "information_schema", "STATISTICS", "COLLATION", NULL); + update_field_infos(info, "information_schema", "STATISTICS", "COLUMN_NAME", NULL); + update_field_infos(info, "information_schema", "STATISTICS", "COMMENT", NULL); + update_field_infos(info, "information_schema", "STATISTICS", "INDEX_COMMENT", NULL); + update_field_infos(info, "information_schema", "STATISTICS", "INDEX_NAME", NULL); + update_field_infos(info, "information_schema", "STATISTICS", "INDEX_TYPE", NULL); + update_field_infos(info, "information_schema", "STATISTICS", "NON_UNIQUE", NULL); + update_field_infos(info, "information_schema", "STATISTICS", "NULLABLE", NULL); + update_field_infos(info, "information_schema", "STATISTICS", "PACKED", NULL); + update_field_infos(info, "information_schema", "STATISTICS", "SEQ_IN_INDEX", NULL); + update_field_infos(info, "information_schema", "STATISTICS", "SUB_PART", NULL); + update_field_infos(info, "information_schema", "STATISTICS", "TABLE_NAME", NULL); } break; @@ -2299,12 +2423,24 @@ extern void maxscaleShow(Parse* pParse, MxsShow* pShow) { info->types = QUERY_TYPE_WRITE; update_names(info, "information_schema", "TABLES"); - append_affected_field(info, - "AUTO_INCREMENT AVG_ROW_LENGTH CHECKSUM CHECK_TIME " - "CREATE_OPTIONS CREATE_TIME DATA_FREE DATA_LENGTH " - "ENGINE INDEX_LENGTH MAX_DATA_LENGTH ROW_FORMAT " - "TABLE_COLLATION TABLE_COMMENT TABLE_NAME " - "TABLE_ROWS UPDATE_TIME VERSION"); + update_field_infos(info, "information_schema", "TABLES", "AUTO_INCREMENT", NULL); + update_field_infos(info, "information_schema", "TABLES", "AVG_ROW_LENGTH", NULL); + update_field_infos(info, "information_schema", "TABLES", "CHECKSUM", NULL); + update_field_infos(info, "information_schema", "TABLES", "CHECK_TIME", NULL); + update_field_infos(info, "information_schema", "TABLES", "CREATE_OPTIONS", NULL); + update_field_infos(info, "information_schema", "TABLES", "CREATE_TIME", NULL); + update_field_infos(info, "information_schema", "TABLES", "DATA_FREE", NULL); + update_field_infos(info, "information_schema", "TABLES", "DATA_LENGTH", NULL); + update_field_infos(info, "information_schema", "TABLES", "ENGINE", NULL); + update_field_infos(info, "information_schema", "TABLES", "INDEX_LENGTH", NULL); + update_field_infos(info, "information_schema", "TABLES", "MAX_DATA_LENGTH", NULL); + update_field_infos(info, "information_schema", "TABLES", "ROW_FORMAT", NULL); + update_field_infos(info, "information_schema", "TABLES", "TABLE_COLLATION", NULL); + update_field_infos(info, "information_schema", "TABLES", "TABLE_COMMENT", NULL); + update_field_infos(info, "information_schema", "TABLES", "TABLE_NAME", NULL); + update_field_infos(info, "information_schema", "TABLES", "TABLE_ROWS", NULL); + update_field_infos(info, "information_schema", "TABLES", "UPDATE_TIME", NULL); + update_field_infos(info, "information_schema", "TABLES", "VERSION", NULL); } break; @@ -2318,7 +2454,8 @@ extern void maxscaleShow(Parse* pParse, MxsShow* pShow) // TODO: qc_mysqlembedded does not set the type bit. info->types = QUERY_TYPE_UNKNOWN; update_names(info, "information_schema", "SESSION_STATUS"); - append_affected_field(info, "VARIABLE_NAME VARIABLE_VALUE"); + update_field_infos(info, "information_schema", "SESSION_STATUS", "VARIABLE_NAME", NULL); + update_field_infos(info, "information_schema", "SESSION_STATUS", "VARIABLE_VALUE", NULL); break; case MXS_SHOW_STATUS_MASTER: @@ -2343,7 +2480,7 @@ extern void maxscaleShow(Parse* pParse, MxsShow* pShow) { info->types = QUERY_TYPE_SHOW_TABLES; update_names(info, "information_schema", "TABLE_NAMES"); - append_affected_field(info, "TABLE_NAME"); + update_field_infos(info, "information_schema", "TABLE_NAMES", "TABLE_NAME", NULL); } break; @@ -2358,7 +2495,8 @@ extern void maxscaleShow(Parse* pParse, MxsShow* pShow) info->types = QUERY_TYPE_SYSVAR_READ; } update_names(info, "information_schema", "SESSION_VARIABLES"); - append_affected_field(info, "VARIABLE_NAME VARIABLE_VALUE"); + update_field_infos(info, "information_schema", "SESSION_STATUS", "VARIABLE_NAME", NULL); + update_field_infos(info, "information_schema", "SESSION_STATUS", "VARIABLE_VALUE", NULL); } break; @@ -2856,7 +2994,39 @@ static char* qc_sqlite_get_affected_fields(GWBUF* query) { if (qc_info_is_valid(info->status)) { - affected_fields = info->affected_fields; + if (!info->affected_fields) + { + if (info->field_infos_len != 0) + { + // The first time qc_sqlite_get_affected_fields() is called + // we copy the column data from info->fields_infos into + // info->affected_fields. + QC_FIELD_INFO* fis = info->field_infos; + size_t fis_len = info->field_infos_len; + size_t buflen = 0; + + for (size_t i = 0; i < fis_len; ++i) + { + buflen += strlen(fis[i].column); + buflen += 1; + } + + buflen += 1; + + affected_fields = MXS_MALLOC(buflen); + MXS_ABORT_IF_NULL(affected_fields); + + affected_fields[0] = 0; + + for (size_t i = 0; i < fis_len; ++i) + { + strcat(affected_fields, fis[i].column); + strcat(affected_fields, " "); + } + + info->affected_fields = affected_fields; + } + } } else if (MXS_LOG_PRIORITY_IS_ENABLED(LOG_INFO)) { @@ -2969,16 +3139,33 @@ static qc_query_op_t qc_sqlite_get_prepare_operation(GWBUF* query) return op; } -void qc_sqlite_get_field_info(GWBUF* stmt, const QC_FIELD_INFO** infos, size_t* n_infos) +void qc_sqlite_get_field_info(GWBUF* query, const QC_FIELD_INFO** infos, size_t* n_infos) { QC_TRACE(); ss_dassert(this_unit.initialized); ss_dassert(this_thread.initialized); - MXS_ERROR("qc_get_field_info not implemented yet."); - *infos = NULL; *n_infos = 0; + + QC_SQLITE_INFO* info = get_query_info(query); + + if (info) + { + if (qc_info_is_valid(info->status)) + { + *infos = info->field_infos; + *n_infos = info->field_infos_len; + } + else if (MXS_LOG_PRIORITY_IS_ENABLED(LOG_INFO)) + { + log_invalid_data(query, "cannot report field info"); + } + } + else + { + MXS_ERROR("The query could not be parsed. Response not valid."); + } } /** diff --git a/query_classifier/test/compare.cc b/query_classifier/test/compare.cc index 99fd1f7ba..3f00439b4 100644 --- a/query_classifier/test/compare.cc +++ b/query_classifier/test/compare.cc @@ -871,6 +871,123 @@ bool compare_get_prepare_operation(QUERY_CLASSIFIER* pClassifier1, GWBUF* pCopy1 return success; } +bool operator == (const QC_FIELD_INFO& lhs, const QC_FIELD_INFO& rhs) +{ + bool rv = false; + if (lhs.column && rhs.column && (strcasecmp(lhs.column, rhs.column) == 0)) + { + if (!lhs.table && !rhs.table) + { + rv = true; + } + else if (lhs.table && rhs.table && (strcmp(lhs.table, rhs.table) == 0)) + { + if (!lhs.database && !rhs.database) + { + rv = true; + } + else if (lhs.database && rhs.database && (strcmp(lhs.database, rhs.database) == 0)) + { + rv = true; + } + } + } + + return rv; +} + +ostream& operator << (ostream& out, const QC_FIELD_INFO& x) +{ + if (x.database) + { + out << x.database; + out << "."; + ss_dassert(x.table); + } + + if (x.table) + { + out << x.table; + out << "."; + } + + ss_dassert(x.column); + out << x.column; + + return out; +} + +bool are_equal(const QC_FIELD_INFO* fields1, size_t n_fields1, + const QC_FIELD_INFO* fields2, size_t n_fields2) +{ + bool rv = (n_fields1 == n_fields2); + + if (rv) + { + size_t i = 0; + while (rv && (i < n_fields1)) + { + rv = *fields1 == *fields2; + ++i; + } + } + + return rv; +} + +ostream& print(ostream& out, const QC_FIELD_INFO* fields, size_t n_fields) +{ + size_t i = 0; + while (i < n_fields) + { + out << fields[i++]; + + if (i != n_fields) + { + out << " "; + } + } + + return out; +} + +bool compare_get_field_info(QUERY_CLASSIFIER* pClassifier1, GWBUF* pCopy1, + QUERY_CLASSIFIER* pClassifier2, GWBUF* pCopy2) +{ + bool success = false; + const char HEADING[] = "qc_get_field_info : "; + + const QC_FIELD_INFO* infos1; + const QC_FIELD_INFO* infos2; + size_t n_infos1; + size_t n_infos2; + + pClassifier1->qc_get_field_info(pCopy1, &infos1, &n_infos1); + pClassifier2->qc_get_field_info(pCopy2, &infos2, &n_infos2); + + stringstream ss; + ss << HEADING; + + if (are_equal(infos1, n_infos1, infos2, n_infos2)) + { + ss << "Ok : "; + print(ss, infos1, n_infos1); + success = true; + } + else + { + ss << "ERR: "; + print(ss, infos1, n_infos1); + ss << " != "; + print(ss, infos2, n_infos2); + } + + report(success, ss.str()); + + return success; +} + + bool compare(QUERY_CLASSIFIER* pClassifier1, QUERY_CLASSIFIER* pClassifier2, const string& s) { GWBUF* pCopy1 = create_gwbuf(s); @@ -891,6 +1008,7 @@ bool compare(QUERY_CLASSIFIER* pClassifier1, QUERY_CLASSIFIER* pClassifier2, con errors += !compare_get_database_names(pClassifier1, pCopy1, pClassifier2, pCopy2); errors += !compare_get_prepare_name(pClassifier1, pCopy1, pClassifier2, pCopy2); errors += !compare_get_prepare_operation(pClassifier1, pCopy1, pClassifier2, pCopy2); + //errors += !compare_get_field_info(pClassifier1, pCopy1, pClassifier2, pCopy2); gwbuf_free(pCopy1); gwbuf_free(pCopy2); From 3c82e7be91398ec67ce78a5efa8524aa3d9ecd6e Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Thu, 3 Nov 2016 15:46:25 +0200 Subject: [PATCH 140/215] MXS-884: qc_mysqlembedded, support qc_get_field_info --- query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc index 1041b6d44..247de682e 100644 --- a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc +++ b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc @@ -76,6 +76,9 @@ typedef struct parsing_info_st void* pi_handle; /*< parsing info object pointer */ char* pi_query_plain_str; /*< query as plain string */ void (*pi_done_fp)(void *); /*< clean-up function for parsing info */ + QC_FIELD_INFO* field_infos; + size_t field_infos_len; + size_t field_infos_capacity; #if defined(SS_DEBUG) skygw_chk_t pi_chk_tail; #endif @@ -1719,6 +1722,14 @@ static void parsing_info_done(void* ptr) free(pi->pi_query_plain_str); } + for (size_t i = 0; i < pi->field_infos_len; ++i) + { + free(pi->field_infos[i].database); + free(pi->field_infos[i].table); + free(pi->field_infos[i].column); + } + free(pi->field_infos); + free(pi); } } From c71acecce3fba320441e3edc75f829ad02103583 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Fri, 4 Nov 2016 22:19:05 +0200 Subject: [PATCH 141/215] MXS-884: qc_mysqlembedded now supports qc_get_field_info Initial implementation, one failure in the tests still to be sorted out. Plus some cleanup. --- .../qc_mysqlembedded/qc_mysqlembedded.cc | 478 +++++++++++++++++- query_classifier/test/compare.cc | 127 ++++- query_classifier/test/insert.test | 6 +- .../test/qc_sqlite_unsupported.test | 5 + 4 files changed, 582 insertions(+), 34 deletions(-) diff --git a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc index 247de682e..b3e8645a4 100644 --- a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc +++ b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc @@ -1032,6 +1032,35 @@ char* qc_get_stmtname(GWBUF* buf) } #endif +/** + * Get the parsing info structure from a GWBUF + * + * @param querybuf A GWBUF + * + * @return The parsing info object, or NULL + */ +parsing_info_t* get_pinfo(GWBUF* querybuf) +{ + parsing_info_t *pi = NULL; + + if ((querybuf != NULL) && GWBUF_IS_PARSED(querybuf)) + { + pi = (parsing_info_t *) gwbuf_get_buffer_object_data(querybuf, GWBUF_PARSING_INFO); + } + + return pi; +} + +LEX* get_lex(parsing_info_t* pi) +{ + MYSQL* mysql = (MYSQL *) pi->pi_handle; + ss_dassert(mysql); + THD* thd = (THD *) mysql->thd; + ss_dassert(thd); + + return thd->lex; +} + /** * Get the parse tree from parsed querybuf. * @param querybuf The parsed GWBUF @@ -1041,31 +1070,19 @@ char* qc_get_stmtname(GWBUF* buf) */ LEX* get_lex(GWBUF* querybuf) { + LEX* lex = NULL; + parsing_info_t* pi = get_pinfo(querybuf); - parsing_info_t* pi; - MYSQL* mysql; - THD* thd; - - if (querybuf == NULL || !GWBUF_IS_PARSED(querybuf)) + if (pi) { - return NULL; + MYSQL* mysql = (MYSQL *) pi->pi_handle; + ss_dassert(mysql); + THD* thd = (THD *) mysql->thd; + ss_dassert(thd); + lex = thd->lex; } - pi = (parsing_info_t *) gwbuf_get_buffer_object_data(querybuf, GWBUF_PARSING_INFO); - - if (pi == NULL) - { - return NULL; - } - - if ((mysql = (MYSQL *) pi->pi_handle) == NULL || - (thd = (THD *) mysql->thd) == NULL) - { - ss_dassert(mysql != NULL && thd != NULL); - return NULL; - } - - return thd->lex; + return lex; } /** @@ -2021,12 +2038,423 @@ qc_query_op_t qc_get_prepare_operation(GWBUF* stmt) return operation; } -void qc_get_field_info(GWBUF* stmt, const QC_FIELD_INFO** infos, size_t* n_infos) +static bool should_exclude(const char* name, List* excludep) { - MXS_ERROR("qc_get_field_info not implemented yet."); + bool exclude = false; + List_iterator ilist(*excludep); + Item* exclude_item; - *infos = NULL; - *n_infos = 0; + while (!exclude && (exclude_item = ilist++)) + { + if (exclude_item->name && (strcasecmp(name, exclude_item->name) == 0)) + { + exclude = true; + } + } + + return exclude; +} + +static void add_field_info(parsing_info_t* info, + const char* database, + const char* table, + const char* column, + List* excludep) +{ + ss_dassert(column); + + // If only a column is specified, but not a table or database and we + // have a list of expressions that should be excluded, we check if the column + // value is present in that list. This is in order to exclude the second "d" in + // a statement like "select a as d from x where d = 2". + if (column && !table && !database && excludep && should_exclude(column, excludep)) + { + return; + } + + QC_FIELD_INFO item = { (char*)database, (char*)table, (char*)column }; + + size_t i; + for (i = 0; i < info->field_infos_len; ++i) + { + QC_FIELD_INFO* field_info = info->field_infos + i; + + if (strcasecmp(item.column, field_info->column) == 0) + { + if (!item.table && !field_info->table) + { + ss_dassert(!item.database && !field_info->database); + break; + } + else if (item.table && field_info->table && (strcmp(item.table, field_info->table) == 0)) + { + if (!item.database && !field_info->database) + { + break; + } + else if (item.database && + field_info->database && + (strcmp(item.database, field_info->database) == 0)) + { + break; + } + } + } + } + + QC_FIELD_INFO* field_infos = NULL; + + if (i == info->field_infos_len) // If true, the field was not present already. + { + if (info->field_infos_len < info->field_infos_capacity) + { + field_infos = info->field_infos; + } + else + { + size_t capacity = info->field_infos_capacity ? 2 * info->field_infos_capacity : 8; + field_infos = (QC_FIELD_INFO*)realloc(info->field_infos, capacity * sizeof(QC_FIELD_INFO)); + + if (field_infos) + { + info->field_infos = field_infos; + info->field_infos_capacity = capacity; + } + } + } + + // If field_infos is NULL, then the field was found and has already been noted. + if (field_infos) + { + item.database = item.database ? strdup(item.database) : NULL; + item.table = item.table ? strdup(item.table) : NULL; + ss_dassert(item.column); + item.column = strdup(item.column); + + // We are happy if we at least could dup the column. + + if (item.column) + { + field_infos[info->field_infos_len++] = item; + } + } +} + +static void add_field_info(parsing_info_t* pi, Item_field* item, List* excludep) +{ + const char* database = item->db_name; + const char* table = item->table_name; + const char* column = item->field_name; + + LEX* lex = get_lex(pi); + + switch (lex->sql_command) + { + case SQLCOM_SHOW_FIELDS: + if (!database) + { + database = "information_schema"; + } + + if (!table) + { + table = "COLUMNS"; + } + break; + + case SQLCOM_SHOW_KEYS: + if (!database) + { + database = "information_schema"; + } + + if (!table) + { + table = "STATISTICS"; + } + break; + + case SQLCOM_SHOW_STATUS: + if (!database) + { + database = "information_schema"; + } + + if (!table) + { + table = "SESSION_STATUS"; + } + break; + + case SQLCOM_SHOW_TABLES: + if (!database) + { + database = "information_schema"; + } + + if (!table) + { + table = "TABLE_NAMES"; + } + break; + + case SQLCOM_SHOW_TABLE_STATUS: + if (!database) + { + database = "information_schema"; + } + + if (!table) + { + table = "TABLES"; + } + break; + + case SQLCOM_SHOW_VARIABLES: + if (!database) + { + database = "information_schema"; + } + + if (!table) + { + table = "SESSION_STATUS"; + } + break; + + default: + break; + } + + add_field_info(pi, database, table, column, excludep); +} + +static void add_field_info(parsing_info_t* pi, Item* item, List* excludep) +{ + const char* database = NULL; + const char* table = NULL; + const char* column = item->name; + + add_field_info(pi, database, table, column, excludep); +} + +static void update_field_infos(parsing_info_t* pi, + collect_source_t source, + Item* item, + List* excludep) +{ + switch (item->type()) + { + case Item::COND_ITEM: + { + Item_cond* cond_item = static_cast(item); + List_iterator ilist(*cond_item->argument_list()); + + while (Item *i = ilist++) + { + update_field_infos(pi, source, i, excludep); + } + } + break; + + case Item::FIELD_ITEM: + add_field_info(pi, static_cast(item), excludep); + break; + + case Item::REF_ITEM: + { + if (source != COLLECT_SELECT) + { + Item_ref* ref_item = static_cast(item); + + add_field_info(pi, item, excludep); + + size_t n_items = ref_item->cols(); + + for (size_t i = 0; i < n_items; ++i) + { + Item* reffed_item = ref_item->element_index(i); + + if (reffed_item != ref_item) + { + update_field_infos(pi, source, ref_item->element_index(i), excludep); + } + } + } + } + break; + + case Item::ROW_ITEM: + { + Item_row* row_item = static_cast(item); + size_t n_items = row_item->cols(); + + for (size_t i = 0; i < n_items; ++i) + { + update_field_infos(pi, source, row_item->element_index(i), excludep); + } + } + break; + + case Item::FUNC_ITEM: + case Item::SUM_FUNC_ITEM: + { + Item_func* func_item = static_cast(item); + Item** items = func_item->arguments(); + size_t n_items = func_item->argument_count(); + + for (size_t i = 0; i < n_items; ++i) + { + update_field_infos(pi, source, items[i], excludep); + } + } + break; + + case Item::SUBSELECT_ITEM: + { + Item_subselect* subselect_item = static_cast(item); + + switch (subselect_item->substype()) + { + case Item_subselect::IN_SUBS: + case Item_subselect::ALL_SUBS: + case Item_subselect::ANY_SUBS: + { + Item_in_subselect* in_subselect_item = static_cast(item); + +#if (((MYSQL_VERSION_MAJOR == 5) &&\ + ((MYSQL_VERSION_MINOR > 5) ||\ + ((MYSQL_VERSION_MINOR == 5) && (MYSQL_VERSION_PATCH >= 48))\ + )\ + ) ||\ + (MYSQL_VERSION_MAJOR >= 10)\ + ) + if (in_subselect_item->left_expr_orig) + { + update_field_infos(pi, source, + in_subselect_item->left_expr_orig, excludep); + } +#else +#pragma message "Figure out what to do with versions < 5.5.48." +#endif + // TODO: Anything else that needs to be looked into? + } + break; + + case Item_subselect::EXISTS_SUBS: + case Item_subselect::SINGLEROW_SUBS: + // TODO: Handle these explicitly as well. + break; + + case Item_subselect::UNKNOWN_SUBS: + default: + MXS_ERROR("Unknown subselect type: %d", subselect_item->substype()); + break; + } + } + break; + + default: + break; + } +} + +void qc_get_field_info(GWBUF* buf, const QC_FIELD_INFO** infos, size_t* n_infos) +{ + if (!buf) + { + return; + } + + if (!ensure_query_is_parsed(buf)) + { + return; + } + + parsing_info_t* pi = get_pinfo(buf); + + if (!pi->field_infos) + { + ss_dassert(pi); + LEX* lex = get_lex(buf); + ss_dassert(lex); + + if (!lex) + { + return; + } + + lex->current_select = lex->all_selects_list; + + while (lex->current_select) + { + List_iterator ilist(lex->current_select->item_list); + + while (Item *item = ilist++) + { + update_field_infos(pi, COLLECT_SELECT, item, NULL); + } + + if (lex->current_select->group_list.first) + { + ORDER* order = lex->current_select->group_list.first; + while (order) + { + Item* item = *order->item; + + update_field_infos(pi, COLLECT_GROUP_BY, item, &lex->current_select->item_list); + + order = order->next; + } + } + + if (lex->current_select->where) + { + update_field_infos(pi, COLLECT_WHERE, + lex->current_select->where, + &lex->current_select->item_list); + } + + if (lex->current_select->having) + { + update_field_infos(pi, COLLECT_HAVING, + lex->current_select->having, + &lex->current_select->item_list); + } + + lex->current_select = lex->current_select->next_select_in_list(); + } + + + List_iterator ilist(lex->value_list); + while (Item* item = ilist++) + { + update_field_infos(pi, COLLECT_SELECT, item, NULL); + } + + if ((lex->sql_command == SQLCOM_INSERT) || + (lex->sql_command == SQLCOM_INSERT_SELECT) || + (lex->sql_command == SQLCOM_REPLACE)) + { + List_iterator ilist(lex->field_list); + while (Item *item = ilist++) + { + update_field_infos(pi, COLLECT_SELECT, item, NULL); + } + + if (lex->insert_list) + { + List_iterator ilist(*lex->insert_list); + while (Item *item = ilist++) + { + update_field_infos(pi, COLLECT_SELECT, item, NULL); + } + } + } + } + + *infos = pi->field_infos; + *n_infos = pi->field_infos_len; } namespace diff --git a/query_classifier/test/compare.cc b/query_classifier/test/compare.cc index 3f00439b4..9fa821aea 100644 --- a/query_classifier/test/compare.cc +++ b/query_classifier/test/compare.cc @@ -917,6 +917,113 @@ ostream& operator << (ostream& out, const QC_FIELD_INFO& x) return out; } +class QcFieldInfo +{ +public: + QcFieldInfo(const QC_FIELD_INFO& info) + : m_database(info.database ? info.database : "") + , m_table(info.table ? info.table : "") + , m_column(info.column ? info.column : "") + {} + + bool eq(const QcFieldInfo& rhs) const + { + return + m_database == rhs.m_database && + m_table == rhs.m_table && + m_column == rhs.m_column; + } + + bool lt(const QcFieldInfo& rhs) const + { + bool rv = false; + + if (m_database < rhs.m_database) + { + rv = true; + } + else if (m_database > rhs.m_database) + { + rv = false; + } + else + { + if (m_table < rhs.m_table) + { + rv = true; + } + else if (m_table > rhs.m_table) + { + rv = false; + } + else + { + if (m_column < rhs.m_column) + { + rv = true; + } + } + } + + return rv; + } + + void print(ostream& out) const + { + if (!m_database.empty()) + { + out << m_database; + out << "."; + } + + if (!m_table.empty()) + { + out << m_table; + out << "."; + } + + out << m_column; + } + +private: + std::string m_database; + std::string m_table; + std::string m_column; +}; + +ostream& operator << (ostream& out, const QcFieldInfo& x) +{ + x.print(out); + return out; +} + +ostream& operator << (ostream& out, std::set& x) +{ + std::set::iterator i = x.begin(); + std::set::iterator end = x.end(); + + while (i != end) + { + out << *i++; + if (i != end) + { + out << " "; + } + } + + return out; +} + +bool operator < (const QcFieldInfo& lhs, const QcFieldInfo& rhs) +{ + return lhs.lt(rhs); +} + +bool operator == (const QcFieldInfo& lhs, const QcFieldInfo& rhs) +{ + return lhs.eq(rhs); +} + bool are_equal(const QC_FIELD_INFO* fields1, size_t n_fields1, const QC_FIELD_INFO* fields2, size_t n_fields2) { @@ -924,6 +1031,7 @@ bool are_equal(const QC_FIELD_INFO* fields1, size_t n_fields1, if (rv) { + size_t i = 0; while (rv && (i < n_fields1)) { @@ -968,18 +1076,23 @@ bool compare_get_field_info(QUERY_CLASSIFIER* pClassifier1, GWBUF* pCopy1, stringstream ss; ss << HEADING; - if (are_equal(infos1, n_infos1, infos2, n_infos2)) + int i; + + std::set f1; + f1.insert(infos1, infos1 + n_infos1); + + std::set f2; + f2.insert(infos2, infos2 + n_infos2); + + if (f1 == f2) { ss << "Ok : "; - print(ss, infos1, n_infos1); + ss << f1; success = true; } else { - ss << "ERR: "; - print(ss, infos1, n_infos1); - ss << " != "; - print(ss, infos2, n_infos2); + ss << "ERR: " << f1 << " != " << f2; } report(success, ss.str()); @@ -1008,7 +1121,7 @@ bool compare(QUERY_CLASSIFIER* pClassifier1, QUERY_CLASSIFIER* pClassifier2, con errors += !compare_get_database_names(pClassifier1, pCopy1, pClassifier2, pCopy2); errors += !compare_get_prepare_name(pClassifier1, pCopy1, pClassifier2, pCopy2); errors += !compare_get_prepare_operation(pClassifier1, pCopy1, pClassifier2, pCopy2); - //errors += !compare_get_field_info(pClassifier1, pCopy1, pClassifier2, pCopy2); + errors += !compare_get_field_info(pClassifier1, pCopy1, pClassifier2, pCopy2); gwbuf_free(pCopy1); gwbuf_free(pCopy2); diff --git a/query_classifier/test/insert.test b/query_classifier/test/insert.test index 2d95dffa1..ce3034f78 100644 --- a/query_classifier/test/insert.test +++ b/query_classifier/test/insert.test @@ -222,9 +222,11 @@ replace into t1 values (4, 4); select row_count(); # Reports that 2 rows are affected. This conforms to documentation. # (Useful for differentiating inserts from updates). -insert into t1 values (2, 2) on duplicate key update data= data + 10; +# MXSTODO: insert into t1 values (2, 2) on duplicate key update data= data + 10; +# qc_sqlite: Cannot parse "on duplicate" select row_count(); -insert into t1 values (5, 5) on duplicate key update data= data + 10; +# MXSTODO: insert into t1 values (5, 5) on duplicate key update data= data + 10; +# qc_sqlite: Cannot parse "on duplicate" select row_count(); drop table t1; diff --git a/query_classifier/test/qc_sqlite_unsupported.test b/query_classifier/test/qc_sqlite_unsupported.test index 49480924a..11a2dd999 100644 --- a/query_classifier/test/qc_sqlite_unsupported.test +++ b/query_classifier/test/qc_sqlite_unsupported.test @@ -20,3 +20,8 @@ SET @x:= (SELECT h FROM t1 WHERE (a,b,c,d,e,f,g)=(1,2,3,4,5,6,7)); # REMOVE: expr(A) ::= LP(B) expr(X) RP(E). {A.pExpr = X.pExpr; spanSet(&A,&B,&E);} # REMOVE: expr(A) ::= LP expr(X) COMMA(OP) expr(Y) RP. {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} # ADD : expr(A) ::= LP exprlist RP. { ... } + +insert into t1 values (2, 2) on duplicate key update data= data + 10; +# Problem: warning: [qc_sqlite] Statement was only partially parsed (Sqlite3 error: SQL logic error +# or missing database, near "on": syntax error): "insert into t1 values (2, 2) on duplicate +# key update data= data + 10;" \ No newline at end of file From 0aaeda0ef1e05f3c87a741a5735eace8346a83de Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Fri, 4 Nov 2016 23:36:52 +0200 Subject: [PATCH 142/215] qc_mysqlembedded: qc_get_affected_fields uses qc_get_field_info --- .../qc_mysqlembedded/qc_mysqlembedded.cc | 342 +++--------------- 1 file changed, 52 insertions(+), 290 deletions(-) diff --git a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc index b3e8645a4..33df9ef25 100644 --- a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc +++ b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc @@ -79,6 +79,7 @@ typedef struct parsing_info_st QC_FIELD_INFO* field_infos; size_t field_infos_len; size_t field_infos_capacity; + char* affected_fields; #if defined(SS_DEBUG) skygw_chk_t pi_chk_tail; #endif @@ -1304,313 +1305,58 @@ bool qc_is_drop_table_query(GWBUF* querybuf) return answer; } -inline void add_str(char** buf, int* buflen, int* bufsize, const char* str) -{ - int isize = strlen(str) + 1; - - if (*buf == NULL || isize + *buflen >= *bufsize) - { - *bufsize = (*bufsize) * 2 + isize; - char *tmp = (char*) realloc(*buf, (*bufsize) * sizeof (char)); - - if (tmp == NULL) - { - MXS_ERROR("Error: memory reallocation failed."); - free(*buf); - *buf = NULL; - *bufsize = 0; - } - - *buf = tmp; - } - - if (*buflen > 0) - { - if (*buf) - { - strcat(*buf, " "); - } - } - - if (*buf) - { - strcat(*buf, str); - } - - *buflen += isize; - -} - -typedef enum collect_source -{ - COLLECT_SELECT, - COLLECT_WHERE, - COLLECT_HAVING, - COLLECT_GROUP_BY, -} collect_source_t; - - -static void collect_name(Item* item, char** bufp, int* buflenp, int* bufsizep, List* excludep) -{ - const char* full_name = item->full_name(); - const char* name = strrchr(full_name, '.'); - - if (!name) - { - // No dot found. - name = full_name; - } - else - { - // Dot found, advance beyond it. - ++name; - } - - bool exclude = false; - - if (excludep) - { - List_iterator ilist(*excludep); - Item* exclude_item = (Item*) ilist.next(); - - for (; !exclude && (exclude_item != NULL); exclude_item = (Item*) ilist.next()) - { - if (exclude_item->name && (strcasecmp(name, exclude_item->name) == 0)) - { - exclude = true; - } - } - } - - if (!exclude) - { - add_str(bufp, buflenp, bufsizep, name); - } -} - -static void collect_affected_fields(collect_source_t source, - Item* item, char** bufp, int* buflenp, int* bufsizep, - List* excludep) -{ - switch (item->type()) - { - case Item::COND_ITEM: - { - Item_cond* cond_item = static_cast(item); - List_iterator ilist(*cond_item->argument_list()); - item = (Item*) ilist.next(); - - for (; item != NULL; item = (Item*) ilist.next()) - { - collect_affected_fields(source, item, bufp, buflenp, bufsizep, excludep); - } - } - break; - - case Item::FIELD_ITEM: - collect_name(item, bufp, buflenp, bufsizep, excludep); - break; - - case Item::REF_ITEM: - { - if (source != COLLECT_SELECT) - { - Item_ref* ref_item = static_cast(item); - - collect_name(item, bufp, buflenp, bufsizep, excludep); - - size_t n_items = ref_item->cols(); - - for (size_t i = 0; i < n_items; ++i) - { - Item* reffed_item = ref_item->element_index(i); - - if (reffed_item != ref_item) - { - collect_affected_fields(source, - ref_item->element_index(i), bufp, buflenp, bufsizep, - excludep); - } - } - } - } - break; - - case Item::ROW_ITEM: - { - Item_row* row_item = static_cast(item); - size_t n_items = row_item->cols(); - - for (size_t i = 0; i < n_items; ++i) - { - collect_affected_fields(source, row_item->element_index(i), bufp, buflenp, bufsizep, - excludep); - } - } - break; - - case Item::FUNC_ITEM: - case Item::SUM_FUNC_ITEM: - { - Item_func* func_item = static_cast(item); - Item** items = func_item->arguments(); - size_t n_items = func_item->argument_count(); - - for (size_t i = 0; i < n_items; ++i) - { - collect_affected_fields(source, items[i], bufp, buflenp, bufsizep, excludep); - } - } - break; - - case Item::SUBSELECT_ITEM: - { - Item_subselect* subselect_item = static_cast(item); - - switch (subselect_item->substype()) - { - case Item_subselect::IN_SUBS: - case Item_subselect::ALL_SUBS: - case Item_subselect::ANY_SUBS: - { - Item_in_subselect* in_subselect_item = static_cast(item); - -#if (((MYSQL_VERSION_MAJOR == 5) &&\ - ((MYSQL_VERSION_MINOR > 5) ||\ - ((MYSQL_VERSION_MINOR == 5) && (MYSQL_VERSION_PATCH >= 48))\ - )\ - ) ||\ - (MYSQL_VERSION_MAJOR >= 10)\ - ) - if (in_subselect_item->left_expr_orig) - { - collect_affected_fields(source, - in_subselect_item->left_expr_orig, bufp, buflenp, bufsizep, - excludep); - } -#else -#pragma message "Figure out what to do with versions < 5.5.48." -#endif - // TODO: Anything else that needs to be looked into? - } - break; - - case Item_subselect::EXISTS_SUBS: - case Item_subselect::SINGLEROW_SUBS: - // TODO: Handle these explicitly as well. - break; - - case Item_subselect::UNKNOWN_SUBS: - default: - MXS_ERROR("Unknown subselect type: %d", subselect_item->substype()); - break; - } - } - break; - - default: - break; - } -} - char* qc_get_affected_fields(GWBUF* buf) { - LEX* lex; - int buffsz = 0, bufflen = 0; - char* where = NULL; - Item* item; - Item::Type itype; + char* affected_fields = NULL; - if (!buf) + if (ensure_query_is_parsed(buf)) { - return NULL; - } + parsing_info_t *pi = get_pinfo(buf); - if (!ensure_query_is_parsed(buf)) - { - return NULL; - } - - if ((lex = get_lex(buf)) == NULL) - { - return NULL; - } - - lex->current_select = lex->all_selects_list; - - if ((where = (char*) malloc(sizeof (char)*1)) == NULL) - { - MXS_ERROR("Memory allocation failed."); - return NULL; - } - - *where = '\0'; - - while (lex->current_select) - { - - List_iterator ilist(lex->current_select->item_list); - item = (Item*) ilist.next(); - - for (; item != NULL; item = (Item*) ilist.next()) + if (pi->affected_fields) { - collect_affected_fields(COLLECT_SELECT, item, &where, &buffsz, &bufflen, NULL); + affected_fields = pi->affected_fields; } - - if (lex->current_select->group_list.first) + else { - ORDER* order = lex->current_select->group_list.first; - while (order) + const QC_FIELD_INFO* infos; + size_t n_infos; + + qc_get_field_info(buf, &infos, &n_infos); + + size_t buflen = 0; + + for (size_t i = 0; i < n_infos; ++i) { - Item* item = *order->item; - - collect_affected_fields(COLLECT_GROUP_BY, item, &where, &buffsz, &bufflen, - &lex->current_select->item_list); - - order = order->next; + buflen += strlen(infos[i].column); + buflen += 1; } - } - if (lex->current_select->where) - { - collect_affected_fields(COLLECT_WHERE, lex->current_select->where, &where, &buffsz, &bufflen, - &lex->current_select->item_list); - } + buflen += 1; - if (lex->current_select->having) - { - collect_affected_fields(COLLECT_HAVING, lex->current_select->having, &where, &buffsz, &bufflen, - &lex->current_select->item_list); - } + affected_fields = (char*)malloc(buflen); + affected_fields[0] = 0; - lex->current_select = lex->current_select->next_select_in_list(); - } - - if ((lex->sql_command == SQLCOM_INSERT) || - (lex->sql_command == SQLCOM_INSERT_SELECT) || - (lex->sql_command == SQLCOM_REPLACE)) - { - List_iterator ilist(lex->field_list); - item = (Item*) ilist.next(); - - for (; item != NULL; item = (Item*) ilist.next()) - { - collect_affected_fields(COLLECT_SELECT, item, &where, &buffsz, &bufflen, NULL); - } - - if (lex->insert_list) - { - List_iterator ilist(*lex->insert_list); - item = (Item*) ilist.next(); - - for (; item != NULL; item = (Item*) ilist.next()) + for (size_t i = 0; i < n_infos; ++i) { - collect_affected_fields(COLLECT_SELECT, item, &where, &buffsz, &bufflen, NULL); + strcat(affected_fields, infos[i].column); + strcat(affected_fields, " "); } + + pi->affected_fields = affected_fields; } + + ss_dassert(affected_fields); } - return where; + if (!affected_fields) + { + affected_fields = (char*)""; + } + + affected_fields = strdup(affected_fields); + + return affected_fields; } bool qc_query_has_clause(GWBUF* buf) @@ -1746,6 +1492,7 @@ static void parsing_info_done(void* ptr) free(pi->field_infos[i].column); } free(pi->field_infos); + free(pi->affected_fields); free(pi); } @@ -2046,7 +1793,14 @@ static bool should_exclude(const char* name, List* excludep) while (!exclude && (exclude_item = ilist++)) { - if (exclude_item->name && (strcasecmp(name, exclude_item->name) == 0)) + const char* exclude_name = exclude_item->full_name(); + + if (strchr(exclude_name, '.') == NULL) + { + exclude_name = exclude_item->name; + } + + if (exclude_name && (strcasecmp(name, exclude_name) == 0)) { exclude = true; } @@ -2238,6 +1992,14 @@ static void add_field_info(parsing_info_t* pi, Item* item, List* excludep) add_field_info(pi, database, table, column, excludep); } +typedef enum collect_source +{ + COLLECT_SELECT, + COLLECT_WHERE, + COLLECT_HAVING, + COLLECT_GROUP_BY, +} collect_source_t; + static void update_field_infos(parsing_info_t* pi, collect_source_t source, Item* item, From 75509a67d41c0765a651f921dede3c5ff18ef222 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Sat, 5 Nov 2016 10:09:03 +0200 Subject: [PATCH 143/215] qc: Do not collect HAVING names A HAVING clause can only refer to names that already have been mentioned or if "SELECT *" is used. Either way, the HAVING names need not be separately collected. --- .../qc_mysqlembedded/qc_mysqlembedded.cc | 12 ++++++++---- query_classifier/qc_sqlite/qc_sqlite.c | 4 ++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc index 33df9ef25..eb992d770 100644 --- a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc +++ b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc @@ -2173,16 +2173,20 @@ void qc_get_field_info(GWBUF* buf, const QC_FIELD_INFO** infos, size_t* n_infos) if (lex->current_select->where) { update_field_infos(pi, COLLECT_WHERE, - lex->current_select->where, - &lex->current_select->item_list); + lex->current_select->where, + &lex->current_select->item_list); } +#if defined(COLLECT_HAVING_AS_WELL) + // A HAVING clause can only refer to fields that already have been + // mentioned. Consequently, they need not be collected. if (lex->current_select->having) { update_field_infos(pi, COLLECT_HAVING, - lex->current_select->having, - &lex->current_select->item_list); + lex->current_select->having, + &lex->current_select->item_list); } +#endif lex->current_select = lex->current_select->next_select_in_list(); } diff --git a/query_classifier/qc_sqlite/qc_sqlite.c b/query_classifier/qc_sqlite/qc_sqlite.c index e7d50bab1..fc6767dc6 100644 --- a/query_classifier/qc_sqlite/qc_sqlite.c +++ b/query_classifier/qc_sqlite/qc_sqlite.c @@ -1029,7 +1029,11 @@ static void update_fields_infos_from_select(QC_SQLITE_INFO* info, if (pSelect->pHaving) { info->has_clause = true; +#if defined(COLLECT_HAVING_AS_WELL) + // A HAVING clause can only refer to fields that already have been + // mentioned. Consequently, they need not be collected. update_fields_infos(info, 0, pSelect->pHaving, QC_TOKEN_MIDDLE, pSelect->pEList); +#endif } } From ac4999ec7692b5305e1f467cb00d1bd2d66fc56c Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Sat, 5 Nov 2016 10:16:55 +0200 Subject: [PATCH 144/215] qc_sqlite: Only exclude exact match When checking whether a name should be excluded, there may be a match only if the name checked against is not a qualified name. --- query_classifier/qc_sqlite/qc_sqlite.c | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/query_classifier/qc_sqlite/qc_sqlite.c b/query_classifier/qc_sqlite/qc_sqlite.c index fc6767dc6..30839fcd5 100644 --- a/query_classifier/qc_sqlite/qc_sqlite.c +++ b/query_classifier/qc_sqlite/qc_sqlite.c @@ -663,18 +663,16 @@ static bool should_exclude(const char* zName, const ExprList* pExclude) Expr* pExpr = item->pExpr; - if (pExpr->op == TK_DOT) + if (pExpr->op == TK_ID) { - pExpr = pExpr->pRight; - } - - // We need to ensure that we do not report fields where there - // is only a difference in case. E.g. - // SELECT A FROM tbl WHERE a = "foo"; - // Affected fields is "A" and not "A a". - if ((pExpr->op == TK_ID) && (strcasecmp(pExpr->u.zToken, zName) == 0)) - { - break; + // We need to ensure that we do not report fields where there + // is only a difference in case. E.g. + // SELECT A FROM tbl WHERE a = "foo"; + // Affected fields is "A" and not "A a". + if (strcasecmp(pExpr->u.zToken, zName) == 0) + { + break; + } } } From add7577f088dd4dba4584a8174f38fef802404a0 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Sun, 6 Nov 2016 21:04:18 +0200 Subject: [PATCH 145/215] MXS-884: Update exclusion code --- .../qc_mysqlembedded/qc_mysqlembedded.cc | 22 ++++++++++++++----- query_classifier/qc_sqlite/qc_sqlite.c | 14 +++++++++++- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc index eb992d770..2d48cb215 100644 --- a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc +++ b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc @@ -1793,17 +1793,27 @@ static bool should_exclude(const char* name, List* excludep) while (!exclude && (exclude_item = ilist++)) { - const char* exclude_name = exclude_item->full_name(); - - if (strchr(exclude_name, '.') == NULL) - { - exclude_name = exclude_item->name; - } + const char* exclude_name = exclude_item->name; if (exclude_name && (strcasecmp(name, exclude_name) == 0)) { exclude = true; } + + if (!exclude) + { + exclude_name = strrchr(exclude_item->full_name(), '.'); + + if (exclude_name) + { + ++exclude_name; // Char after the '.' + + if (strcasecmp(name, exclude_name) == 0) + { + exclude = true; + } + } + } } return exclude; diff --git a/query_classifier/qc_sqlite/qc_sqlite.c b/query_classifier/qc_sqlite/qc_sqlite.c index 30839fcd5..eae3f00ab 100644 --- a/query_classifier/qc_sqlite/qc_sqlite.c +++ b/query_classifier/qc_sqlite/qc_sqlite.c @@ -663,6 +663,18 @@ static bool should_exclude(const char* zName, const ExprList* pExclude) Expr* pExpr = item->pExpr; + if (pExpr->op == TK_EQ) + { + // We end up here e.g with "UPDATE t set t.col = 5 ..." + // So, we pick the left branch. + pExpr = pExpr->pLeft; + } + + while (pExpr->op == TK_DOT) + { + pExpr = pExpr->pRight; + } + if (pExpr->op == TK_ID) { // We need to ensure that we do not report fields where there @@ -1596,7 +1608,7 @@ void mxs_sqlite3Update(Parse* pParse, SrcList* pTabList, ExprList* pChanges, Exp if (pWhere) { - update_fields_infos(info, 0, pWhere, QC_TOKEN_MIDDLE, NULL); + update_fields_infos(info, 0, pWhere, QC_TOKEN_MIDDLE, pChanges); } exposed_sqlite3SrcListDelete(pParse->db, pTabList); From 568153efb9dafa71883f9fd27d607bd59cadceee Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Sun, 6 Nov 2016 21:35:44 +0200 Subject: [PATCH 146/215] Fix buflen calculation in classify qc_sqlite.c is now strict as far as buffers and payloads goes. --- query_classifier/test/classify.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/query_classifier/test/classify.c b/query_classifier/test/classify.c index 9a51cc9fe..c656abae3 100644 --- a/query_classifier/test/classify.c +++ b/query_classifier/test/classify.c @@ -196,10 +196,12 @@ int test(FILE* input, FILE* expected) { tok = strpbrk(strbuff, ";"); unsigned int qlen = tok - strbuff + 1; - GWBUF* buff = gwbuf_alloc(qlen + 6); - *((unsigned char*)(GWBUF_DATA(buff))) = qlen; - *((unsigned char*)(GWBUF_DATA(buff) + 1)) = (qlen >> 8); - *((unsigned char*)(GWBUF_DATA(buff) + 2)) = (qlen >> 16); + unsigned int payload_len = qlen + 1; + unsigned int buf_len = payload_len + 4; + GWBUF* buff = gwbuf_alloc(buf_len); + *((unsigned char*)(GWBUF_DATA(buff))) = payload_len; + *((unsigned char*)(GWBUF_DATA(buff) + 1)) = (payload_len >> 8); + *((unsigned char*)(GWBUF_DATA(buff) + 2)) = (payload_len >> 16); *((unsigned char*)(GWBUF_DATA(buff) + 3)) = 0x00; *((unsigned char*)(GWBUF_DATA(buff) + 4)) = 0x03; memcpy(GWBUF_DATA(buff) + 5, strbuff, qlen); From 9f01a4612deb025cac2a215a73b24b94462ef433 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Mon, 7 Nov 2016 10:01:05 +0100 Subject: [PATCH 147/215] EOF flags are now read in case 0xfe EOF flags are now read in case 0xfe and minor improvements --- server/modules/filter/maxrows/maxrows.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/server/modules/filter/maxrows/maxrows.c b/server/modules/filter/maxrows/maxrows.c index 738eee875..3c81eea11 100644 --- a/server/modules/filter/maxrows/maxrows.c +++ b/server/modules/filter/maxrows/maxrows.c @@ -23,6 +23,7 @@ * 26/10/2016 Massimiliano Pinto Initial implementation * 04/11/2016 Massimiliano Pinto Addition of SERVER_MORE_RESULTS_EXIST flag (0x0008) * detection in handle_expecting_rows(). + * 07/11/2016 Massimiliano Pinto handle_expecting_rows renamed to handle_rows * * @endverbatim */ @@ -166,7 +167,7 @@ static void maxrows_session_data_free(MAXROWS_SESSION_DATA *data); static int handle_expecting_fields(MAXROWS_SESSION_DATA *csdata); static int handle_expecting_nothing(MAXROWS_SESSION_DATA *csdata); static int handle_expecting_response(MAXROWS_SESSION_DATA *csdata); -static int handle_expecting_rows(MAXROWS_SESSION_DATA *csdata); +static int handle_rows(MAXROWS_SESSION_DATA *csdata); static int handle_ignoring_response(MAXROWS_SESSION_DATA *csdata); static bool process_params(char **options, FILTER_PARAMETER **params, MAXROWS_CONFIG* config); @@ -368,7 +369,7 @@ static int clientReply(FILTER *instance, void *sdata, GWBUF *data) case MAXROWS_DISCARDING_RESPONSE: case MAXROWS_EXPECTING_ROWS: - rv = handle_expecting_rows(csdata); + rv = handle_rows(csdata); break; case MAXROWS_IGNORING_RESPONSE: @@ -524,7 +525,7 @@ static int handle_expecting_fields(MAXROWS_SESSION_DATA *csdata) case 0xfe: // EOF, the one after the fields. csdata->res.offset += packetlen; csdata->state = MAXROWS_EXPECTING_ROWS; - rv = handle_expecting_rows(csdata); + rv = handle_rows(csdata); break; default: // Field information. @@ -640,7 +641,7 @@ static int handle_expecting_response(MAXROWS_SESSION_DATA *csdata) * * @param csdata The maxrows session data. */ -static int handle_expecting_rows(MAXROWS_SESSION_DATA *csdata) +static int handle_rows(MAXROWS_SESSION_DATA *csdata) { ss_dassert(csdata->state == MAXROWS_EXPECTING_ROWS || csdata->state == MAXROWS_DISCARDING_RESPONSE); ss_dassert(csdata->res.data); @@ -662,7 +663,6 @@ static int handle_expecting_rows(MAXROWS_SESSION_DATA *csdata) { // We have at least one complete packet. int command = (int)MYSQL_GET_COMMAND(header); - int flags = gw_mysql_get_byte2(header + MAXROWS_MYSQL_EOF_PACKET_FLAGS_OFFSET); switch (command) { @@ -703,6 +703,8 @@ static int handle_expecting_rows(MAXROWS_SESSION_DATA *csdata) * will be an OK packet. */ + int flags = gw_mysql_get_byte2(header + MAXROWS_MYSQL_EOF_PACKET_FLAGS_OFFSET); + if (!(flags & SERVER_MORE_RESULTS_EXIST)) { if (csdata->state == MAXROWS_DISCARDING_RESPONSE) From 9fa1a0cfec110228f79ca003d8357adfad62865c Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Mon, 7 Nov 2016 11:02:49 +0200 Subject: [PATCH 148/215] Replace qc_get_affected_fields with qc_get_field_info qc_get_affected_fields that returned the fields as one string with fields interspared with spaces, will be removed. --- server/modules/filter/dbfwfilter/dbfwfilter.c | 61 +++++++++---------- 1 file changed, 28 insertions(+), 33 deletions(-) diff --git a/server/modules/filter/dbfwfilter/dbfwfilter.c b/server/modules/filter/dbfwfilter/dbfwfilter.c index 5008bfad2..e2ebd86ff 100644 --- a/server/modules/filter/dbfwfilter/dbfwfilter.c +++ b/server/modules/filter/dbfwfilter/dbfwfilter.c @@ -1701,13 +1701,12 @@ bool rule_matches(FW_INSTANCE* my_instance, RULELIST *rulelist, char* query) { - char *ptr, *where, *msg = NULL; + char *ptr, *msg = NULL; char emsg[512]; unsigned char* memptr = (unsigned char*) queue->start; bool is_sql, is_real, matches; qc_query_op_t optype = QUERY_OP_UNDEFINED; - STRLINK* strln = NULL; QUERYSPEED* queryspeed = NULL; QUERYSPEED* rule_qs = NULL; time_t time_now; @@ -1745,7 +1744,7 @@ bool rule_matches(FW_INSTANCE* my_instance, case QUERY_OP_UPDATE: case QUERY_OP_INSERT: case QUERY_OP_DELETE: - // In these cases, we have to be able to trust what qc_get_affected_fields + // In these cases, we have to be able to trust what qc_get_field_info // returns. Unless the query was parsed completely, we cannot do that. msg = create_parse_error(my_instance, "parsed completely", query, &matches); goto queryresolved; @@ -1817,32 +1816,29 @@ bool rule_matches(FW_INSTANCE* my_instance, case RT_COLUMN: if (is_sql && is_real) { - where = qc_get_affected_fields(queue); - if (where != NULL) - { - char* saveptr; - char* tok = strtok_r(where, " ", &saveptr); - while (tok) - { - strln = (STRLINK*) rulelist->rule->data; - while (strln) - { - if (strcasecmp(tok, strln->value) == 0) - { - matches = true; + const QC_FIELD_INFO* infos; + size_t n_infos; + qc_get_field_info(queue, &infos, &n_infos); - sprintf(emsg, "Permission denied to column '%s'.", strln->value); - MXS_INFO("dbfwfilter: rule '%s': query targets forbidden column: %s", - rulelist->rule->name, strln->value); - msg = MXS_STRDUP_A(emsg); - MXS_FREE(where); - goto queryresolved; - } - strln = strln->next; + for (size_t i = 0; i < n_infos; ++i) + { + const char* tok = infos[i].column; + + STRLINK* strln = (STRLINK*) rulelist->rule->data; + while (strln) + { + if (strcasecmp(tok, strln->value) == 0) + { + matches = true; + + sprintf(emsg, "Permission denied to column '%s'.", strln->value); + MXS_INFO("dbfwfilter: rule '%s': query targets forbidden column: %s", + rulelist->rule->name, strln->value); + msg = MXS_STRDUP_A(emsg); + goto queryresolved; } - tok = strtok_r(NULL, ",", &saveptr); + strln = strln->next; } - MXS_FREE(where); } } break; @@ -1850,23 +1846,22 @@ bool rule_matches(FW_INSTANCE* my_instance, case RT_WILDCARD: if (is_sql && is_real) { - char * strptr; - where = qc_get_affected_fields(queue); + const QC_FIELD_INFO* infos; + size_t n_infos; + qc_get_field_info(queue, &infos, &n_infos); - if (where != NULL) + for (size_t i = 0; i < n_infos; ++i) { - strptr = where; + const char* column = infos[i].column; - if (strchr(strptr, '*')) + if (strcmp(column, "*") == 0) { matches = true; msg = MXS_STRDUP_A("Usage of wildcard denied."); MXS_INFO("dbfwfilter: rule '%s': query contains a wildcard.", rulelist->rule->name); - MXS_FREE(where); goto queryresolved; } - MXS_FREE(where); } } break; From bf62f8950af8536b859aab268c7b73283b5821f6 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Mon, 7 Nov 2016 11:22:48 +0200 Subject: [PATCH 149/215] Remove qc_get_affected_fields Function is no longer used and it was quite unoptimal, so now removed. qc_get_prepare_name, qc_get_prepare_operation and qc_get_field_info that were missing from qc_dummy added at the same time. --- include/maxscale/query_classifier.h | 12 --- query_classifier/qc_dummy/qc_dummy.cc | 25 +++++-- .../qc_mysqlembedded/qc_mysqlembedded.cc | 57 --------------- query_classifier/qc_sqlite/qc_sqlite.c | 73 ------------------- query_classifier/test/compare.cc | 63 ---------------- server/core/query_classifier.c | 8 -- 6 files changed, 19 insertions(+), 219 deletions(-) diff --git a/include/maxscale/query_classifier.h b/include/maxscale/query_classifier.h index 00a7688c5..b7e3f4639 100644 --- a/include/maxscale/query_classifier.h +++ b/include/maxscale/query_classifier.h @@ -125,7 +125,6 @@ typedef struct query_classifier char** (*qc_get_table_names)(GWBUF* stmt, int* tblsize, bool fullnames); char* (*qc_get_canonical)(GWBUF* stmt); bool (*qc_query_has_clause)(GWBUF* stmt); - char* (*qc_get_affected_fields)(GWBUF* stmt); char** (*qc_get_database_names)(GWBUF* stmt, int* size); char* (*qc_get_prepare_name)(GWBUF* stmt); qc_query_op_t (*qc_get_prepare_operation)(GWBUF* stmt); @@ -225,17 +224,6 @@ void qc_thread_end(void); */ qc_parse_result_t qc_parse(GWBUF* stmt); -/** - * Returns the fields the statement affects, as a string of names separated - * by spaces. Note that the fields do not contain any table information. - * - * @param stmt A buffer containing a COM_QUERY packet. - * - * @return A string containing the fields or NULL if a memory allocation - * failure occurs. The string must be freed by the caller. - */ -char* qc_get_affected_fields(GWBUF* stmt); - /** * Returns information about affected fields. * diff --git a/query_classifier/qc_dummy/qc_dummy.cc b/query_classifier/qc_dummy/qc_dummy.cc index 2ee32e90c..1d326d499 100644 --- a/query_classifier/qc_dummy/qc_dummy.cc +++ b/query_classifier/qc_dummy/qc_dummy.cc @@ -45,11 +45,6 @@ bool qc_is_drop_table_query(GWBUF* querybuf) return false; } -char* qc_get_affected_fields(GWBUF* buf) -{ - return NULL; -} - bool qc_query_has_clause(GWBUF* buf) { return false; @@ -66,6 +61,22 @@ qc_query_op_t qc_get_operation(GWBUF* querybuf) return QUERY_OP_UNDEFINED; } +char* qc_sqlite_get_prepare_name(GWBUF* query) +{ + return NULL; +} + +qc_query_op_t qc_sqlite_get_prepare_operation(GWBUF* query) +{ + return QUERY_OP_UNDEFINED; +} + +void qc_sqlite_get_field_info(GWBUF* query, const QC_FIELD_INFO** infos, size_t* n_infos) +{ + *infos = NULL; + *n_infos = 0; +} + bool qc_init(const char* args) { return true; @@ -125,8 +136,10 @@ extern "C" qc_get_table_names, NULL, qc_query_has_clause, - qc_get_affected_fields, qc_get_database_names, + qc_get_prepare_name, + qc_get_prepare_operation, + qc_get_field_info, }; QUERY_CLASSIFIER* GetModuleObject() diff --git a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc index 2d48cb215..29ec2af53 100644 --- a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc +++ b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc @@ -79,7 +79,6 @@ typedef struct parsing_info_st QC_FIELD_INFO* field_infos; size_t field_infos_len; size_t field_infos_capacity; - char* affected_fields; #if defined(SS_DEBUG) skygw_chk_t pi_chk_tail; #endif @@ -1305,60 +1304,6 @@ bool qc_is_drop_table_query(GWBUF* querybuf) return answer; } -char* qc_get_affected_fields(GWBUF* buf) -{ - char* affected_fields = NULL; - - if (ensure_query_is_parsed(buf)) - { - parsing_info_t *pi = get_pinfo(buf); - - if (pi->affected_fields) - { - affected_fields = pi->affected_fields; - } - else - { - const QC_FIELD_INFO* infos; - size_t n_infos; - - qc_get_field_info(buf, &infos, &n_infos); - - size_t buflen = 0; - - for (size_t i = 0; i < n_infos; ++i) - { - buflen += strlen(infos[i].column); - buflen += 1; - } - - buflen += 1; - - affected_fields = (char*)malloc(buflen); - affected_fields[0] = 0; - - for (size_t i = 0; i < n_infos; ++i) - { - strcat(affected_fields, infos[i].column); - strcat(affected_fields, " "); - } - - pi->affected_fields = affected_fields; - } - - ss_dassert(affected_fields); - } - - if (!affected_fields) - { - affected_fields = (char*)""; - } - - affected_fields = strdup(affected_fields); - - return affected_fields; -} - bool qc_query_has_clause(GWBUF* buf) { bool clause = false; @@ -1492,7 +1437,6 @@ static void parsing_info_done(void* ptr) free(pi->field_infos[i].column); } free(pi->field_infos); - free(pi->affected_fields); free(pi); } @@ -2375,7 +2319,6 @@ static QUERY_CLASSIFIER qc = qc_get_table_names, NULL, qc_query_has_clause, - qc_get_affected_fields, qc_get_database_names, qc_get_prepare_name, qc_get_prepare_operation, diff --git a/query_classifier/qc_sqlite/qc_sqlite.c b/query_classifier/qc_sqlite/qc_sqlite.c index eae3f00ab..fc579103c 100644 --- a/query_classifier/qc_sqlite/qc_sqlite.c +++ b/query_classifier/qc_sqlite/qc_sqlite.c @@ -60,7 +60,6 @@ typedef struct qc_sqlite_info uint32_t types; // The types of the query. qc_query_op_t operation; // The operation in question. - char* affected_fields; // The affected fields. bool is_real_query; // SELECT, UPDATE, INSERT, DELETE or a variation. bool has_clause; // Has WHERE or HAVING. char** table_names; // Array of table names used in the query. @@ -305,7 +304,6 @@ static QC_SQLITE_INFO* info_alloc(void) static void info_finish(QC_SQLITE_INFO* info) { - free(info->affected_fields); free_string_array(info->table_names); free_string_array(info->table_fullnames); free(info->created_table_name); @@ -332,7 +330,6 @@ static QC_SQLITE_INFO* info_init(QC_SQLITE_INFO* info) info->types = QUERY_TYPE_UNKNOWN; info->operation = QUERY_OP_UNDEFINED; - info->affected_fields = NULL; info->is_real_query = false; info->has_clause = false; info->table_names = NULL; @@ -2587,7 +2584,6 @@ static bool qc_sqlite_is_real_query(GWBUF* query); static char** qc_sqlite_get_table_names(GWBUF* query, int* tblsize, bool fullnames); static char* qc_sqlite_get_canonical(GWBUF* query); static bool qc_sqlite_query_has_clause(GWBUF* query); -static char* qc_sqlite_get_affected_fields(GWBUF* query); static char** qc_sqlite_get_database_names(GWBUF* query, int* sizep); static bool get_key_and_value(char* arg, const char** pkey, const char** pvalue) @@ -2995,74 +2991,6 @@ static bool qc_sqlite_query_has_clause(GWBUF* query) return has_clause; } -static char* qc_sqlite_get_affected_fields(GWBUF* query) -{ - QC_TRACE(); - ss_dassert(this_unit.initialized); - ss_dassert(this_thread.initialized); - - char* affected_fields = NULL; - QC_SQLITE_INFO* info = get_query_info(query); - - if (info) - { - if (qc_info_is_valid(info->status)) - { - if (!info->affected_fields) - { - if (info->field_infos_len != 0) - { - // The first time qc_sqlite_get_affected_fields() is called - // we copy the column data from info->fields_infos into - // info->affected_fields. - QC_FIELD_INFO* fis = info->field_infos; - size_t fis_len = info->field_infos_len; - size_t buflen = 0; - - for (size_t i = 0; i < fis_len; ++i) - { - buflen += strlen(fis[i].column); - buflen += 1; - } - - buflen += 1; - - affected_fields = MXS_MALLOC(buflen); - MXS_ABORT_IF_NULL(affected_fields); - - affected_fields[0] = 0; - - for (size_t i = 0; i < fis_len; ++i) - { - strcat(affected_fields, fis[i].column); - strcat(affected_fields, " "); - } - - info->affected_fields = affected_fields; - } - } - } - else if (MXS_LOG_PRIORITY_IS_ENABLED(LOG_INFO)) - { - log_invalid_data(query, "cannot report what fields are affected"); - } - } - else - { - MXS_ERROR("The query could not be parsed. Response not valid."); - } - - if (!affected_fields) - { - affected_fields = ""; - } - - affected_fields = MXS_STRDUP(affected_fields); - MXS_ABORT_IF_NULL(affected_fields); - - return affected_fields; -} - static char** qc_sqlite_get_database_names(GWBUF* query, int* sizep) { QC_TRACE(); @@ -3203,7 +3131,6 @@ static QUERY_CLASSIFIER qc = qc_sqlite_get_table_names, NULL, qc_sqlite_query_has_clause, - qc_sqlite_get_affected_fields, qc_sqlite_get_database_names, qc_sqlite_get_prepare_name, qc_sqlite_get_prepare_operation, diff --git a/query_classifier/test/compare.cc b/query_classifier/test/compare.cc index 9fa821aea..9c267846d 100644 --- a/query_classifier/test/compare.cc +++ b/query_classifier/test/compare.cc @@ -715,68 +715,6 @@ ostream& operator << (ostream& o, const std::set& s) return o; } -bool compare_get_affected_fields(QUERY_CLASSIFIER* pClassifier1, GWBUF* pCopy1, - QUERY_CLASSIFIER* pClassifier2, GWBUF* pCopy2) -{ - bool success = false; - const char HEADING[] = "qc_get_affected_fields : "; - - char* rv1 = pClassifier1->qc_get_affected_fields(pCopy1); - char* rv2 = pClassifier2->qc_get_affected_fields(pCopy2); - - std::set fields1; - std::set fields2; - - if (rv1) - { - add_fields(fields1, rv1); - } - - if (rv2) - { - add_fields(fields2, rv2); - } - - stringstream ss; - ss << HEADING; - - if ((!rv1 && !rv2) || (rv1 && rv2 && (fields1 == fields2))) - { - ss << "Ok : " << fields1; - success = true; - } - else - { - ss << "ERR: "; - if (rv1) - { - ss << fields1; - } - else - { - ss << "NULL"; - } - - ss << " != "; - - if (rv2) - { - ss << fields2; - } - else - { - ss << "NULL"; - } - } - - report(success, ss.str()); - - free(rv1); - free(rv2); - - return success; -} - bool compare_get_database_names(QUERY_CLASSIFIER* pClassifier1, GWBUF* pCopy1, QUERY_CLASSIFIER* pClassifier2, GWBUF* pCopy2) { @@ -1117,7 +1055,6 @@ bool compare(QUERY_CLASSIFIER* pClassifier1, QUERY_CLASSIFIER* pClassifier2, con errors += !compare_get_table_names(pClassifier1, pCopy1, pClassifier2, pCopy2, false); errors += !compare_get_table_names(pClassifier1, pCopy1, pClassifier2, pCopy2, true); errors += !compare_query_has_clause(pClassifier1, pCopy1, pClassifier2, pCopy2); - errors += !compare_get_affected_fields(pClassifier1, pCopy1, pClassifier2, pCopy2); errors += !compare_get_database_names(pClassifier1, pCopy1, pClassifier2, pCopy2); errors += !compare_get_prepare_name(pClassifier1, pCopy1, pClassifier2, pCopy2); errors += !compare_get_prepare_operation(pClassifier1, pCopy1, pClassifier2, pCopy2); diff --git a/server/core/query_classifier.c b/server/core/query_classifier.c index 70d7f3cfe..3b9abef57 100644 --- a/server/core/query_classifier.c +++ b/server/core/query_classifier.c @@ -189,14 +189,6 @@ bool qc_query_has_clause(GWBUF* query) return classifier->qc_query_has_clause(query); } -char* qc_get_affected_fields(GWBUF* query) -{ - QC_TRACE(); - ss_dassert(classifier); - - return classifier->qc_get_affected_fields(query); -} - void qc_get_field_info(GWBUF* query, const QC_FIELD_INFO** infos, size_t* n_infos) { QC_TRACE(); From a097204c70115da7b7e55bdb2fe84ed2692b8a44 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 3 Nov 2016 11:24:32 +0200 Subject: [PATCH 150/215] MXS-922: Move server weight calculation into the core The core now does the server weight calculation instead of each router module doing the same thing. --- include/maxscale/service.h | 5 +- server/core/service.c | 105 +++++++++++++++++- .../routing/readconnroute/readconnroute.c | 71 +----------- .../routing/readwritesplit/readwritesplit.c | 77 +------------ 4 files changed, 106 insertions(+), 152 deletions(-) diff --git a/include/maxscale/service.h b/include/maxscale/service.h index 815e8cf62..74f9c9129 100644 --- a/include/maxscale/service.h +++ b/include/maxscale/service.h @@ -97,8 +97,9 @@ typedef struct typedef struct server_ref_t { - struct server_ref_t *next; - SERVER* server; + struct server_ref_t *next; /**< Next server reference */ + SERVER* server; /**< The actual server */ + int weight; /**< Weight of this server */ } SERVER_REF; #define SERVICE_MAX_RETRY_INTERVAL 3600 /*< The maximum interval between service start retries */ diff --git a/server/core/service.c b/server/core/service.c index c936d91bd..6613f0779 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -66,6 +66,9 @@ #include #include +/** Base value for server weights */ +#define SERVICE_BASE_SERVER_WEIGHT 1000 + /** To be used with configuration type checks */ typedef struct typelib_st { @@ -96,6 +99,7 @@ static void service_add_qualified_param(SERVICE* svc, CONFIG_PARAMETER* param); static void service_internal_restart(void *data); static void service_queue_check(void *data); +static void service_calculate_weights(SERVICE *service); /** * Allocate a new service for the gateway to support @@ -474,6 +478,9 @@ static void free_string_array(char** array) int serviceStart(SERVICE *service) { + /** Calculate the server weights */ + service_calculate_weights(service); + int listeners = 0; char **router_options = copy_string_array(service->routerOptions); @@ -729,6 +736,26 @@ int serviceHasProtocol(SERVICE *service, const char *protocol, return proto != NULL; } +/** + * Allocate a new server reference + * + * @param server Server to refer to + * @return Server reference or NULL on error + */ +static SERVER_REF* server_ref_alloc(SERVER *server) +{ + SERVER_REF *sref = MXS_MALLOC(sizeof(SERVER_REF)); + + if (sref) + { + sref->next = NULL; + sref->server = server; + sref->weight = SERVICE_BASE_SERVER_WEIGHT; + } + + return sref; +} + /** * Add a backend database server to a service * @@ -738,13 +765,10 @@ int serviceHasProtocol(SERVICE *service, const char *protocol, void serviceAddBackend(SERVICE *service, SERVER *server) { - SERVER_REF *sref = MXS_MALLOC(sizeof(SERVER_REF)); + SERVER_REF *sref = server_ref_alloc(server); if (sref) { - sref->next = NULL; - sref->server = server; - spinlock_acquire(&service->spin); if (service->dbref) { @@ -2027,3 +2051,76 @@ bool service_all_services_have_listeners() spinlock_release(&service_spin); return rval; } + +static void service_calculate_weights(SERVICE *service) +{ + char *weightby = serviceGetWeightingParameter(service); + if (weightby && service->dbref) + { + /** Service has a weighting parameter and at least one server */ + int total = 0; + + /** Calculate total weight */ + for (SERVER_REF *server = service->dbref; server; server = server->next) + { + server->weight = SERVICE_BASE_SERVER_WEIGHT; + char *param = serverGetParameter(server->server, weightby); + if (param) + { + total += atoi(param); + } + } + + if (total == 0) + { + MXS_WARNING("Weighting Parameter for service '%s' will be ignored as " + "no servers have values for the parameter '%s'.", + service->name, weightby); + } + else if (total < 0) + { + MXS_ERROR("Sum of weighting parameter '%s' for service '%s' exceeds " + "maximum value of %d. Weighting will be ignored.", + weightby, service->name, INT_MAX); + } + else + { + /** Calculate the relative weight of the servers */ + for (SERVER_REF *server = service->dbref; server; server = server->next) + { + char *param = serverGetParameter(server->server, weightby); + if (param) + { + int wght = atoi(param); + int perc = (wght * SERVICE_BASE_SERVER_WEIGHT) / total; + + if (perc == 0) + { + MXS_ERROR("Weighting parameter '%s' with a value of %d for" + " server '%s' rounds down to zero with total weight" + " of %d for service '%s'. No queries will be " + "routed to this server as long as a server with" + " positive weight is available.", + weightby, wght, server->server->unique_name, + total, service->name); + } + else if (perc < 0) + { + MXS_ERROR("Weighting parameter '%s' for server '%s' is too large, " + "maximum value is %d. No weighting will be used for this " + "server.", weightby, server->server->unique_name, + INT_MAX / SERVICE_BASE_SERVER_WEIGHT); + perc = SERVICE_BASE_SERVER_WEIGHT; + } + server->weight = perc; + } + else + { + MXS_WARNING("Server '%s' has no parameter '%s' used for weighting" + " for service '%s'.", server->server->unique_name, + weightby, service->name); + } + } + } + } +} diff --git a/server/modules/routing/readconnroute/readconnroute.c b/server/modules/routing/readconnroute/readconnroute.c index 6f78663a6..a77dad041 100644 --- a/server/modules/routing/readconnroute/readconnroute.c +++ b/server/modules/routing/readconnroute/readconnroute.c @@ -203,11 +203,8 @@ static ROUTER * createInstance(SERVICE *service, char **options) { ROUTER_INSTANCE *inst; - SERVER *server; SERVER_REF *sref; int i, n; - BACKEND *backend; - char *weightby; if ((inst = MXS_CALLOC(1, sizeof(ROUTER_INSTANCE))) == NULL) { @@ -243,77 +240,11 @@ createInstance(SERVICE *service, char **options) } inst->servers[n]->server = sref->server; inst->servers[n]->current_connection_count = 0; - inst->servers[n]->weight = 1000; + inst->servers[n]->weight = sref->weight; n++; } inst->servers[n] = NULL; - if ((weightby = serviceGetWeightingParameter(service)) != NULL) - { - int total = 0; - - for (int n = 0; inst->servers[n]; n++) - { - BACKEND *backend = inst->servers[n]; - char *param = serverGetParameter(backend->server, weightby); - if (param) - { - total += atoi(param); - } - } - if (total == 0) - { - MXS_WARNING("Weighting Parameter for service '%s' " - "will be ignored as no servers have values " - "for the parameter '%s'.", - service->name, weightby); - } - else if (total < 0) - { - MXS_ERROR("Sum of weighting parameter '%s' for service '%s' exceeds " - "maximum value of %d. Weighting will be ignored.", - weightby, service->name, INT_MAX); - } - else - { - for (int n = 0; inst->servers[n]; n++) - { - BACKEND *backend = inst->servers[n]; - char *param = serverGetParameter(backend->server, weightby); - if (param) - { - int wght = atoi(param); - int perc = (wght * 1000) / total; - - if (perc == 0) - { - perc = 1; - MXS_ERROR("Weighting parameter '%s' with a value of %d for" - " server '%s' rounds down to zero with total weight" - " of %d for service '%s'. No queries will be " - "routed to this server.", weightby, wght, - backend->server->unique_name, total, - service->name); - } - else if (perc < 0) - { - MXS_ERROR("Weighting parameter '%s' for server '%s' is too large, " - "maximum value is %d. No weighting will be used for this server.", - weightby, backend->server->unique_name, INT_MAX / 1000); - perc = 1000; - } - backend->weight = perc; - } - else - { - MXS_WARNING("Server '%s' has no parameter '%s' used for weighting" - " for service '%s'.", backend->server->unique_name, - weightby, service->name); - } - } - } - } - /* * Process the options */ diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index 9ddf0f4f5..add637776 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -232,7 +232,7 @@ static ROUTER *createInstance(SERVICE *service, char **options) router->servers[nservers]->backend_server = sref->server; router->servers[nservers]->backend_conn_count = 0; router->servers[nservers]->be_valid = false; - router->servers[nservers]->weight = 1000; + router->servers[nservers]->weight = sref->weight; #if defined(SS_DEBUG) router->servers[nservers]->be_chk_top = CHK_NUM_BACKEND; router->servers[nservers]->be_chk_tail = CHK_NUM_BACKEND; @@ -247,81 +247,6 @@ static ROUTER *createInstance(SERVICE *service, char **options) */ router->available_slaves = true; - /* - * If server weighting has been defined calculate the percentage - * of load that will be sent to each server. This is only used for - * calculating the least connections, either globally or within a - * service, or the number of current operations on a server. - */ - if ((weightby = serviceGetWeightingParameter(service)) != NULL) - { - int total = 0; - - for (int n = 0; router->servers[n]; n++) - { - BACKEND *backend = router->servers[n]; - char *param = serverGetParameter(backend->backend_server, weightby); - if (param) - { - total += atoi(param); - } - } - if (total == 0) - { - MXS_WARNING("Weighting Parameter for service '%s' " - "will be ignored as no servers have values " - "for the parameter '%s'.", - service->name, weightby); - } - else if (total < 0) - { - MXS_ERROR("Sum of weighting parameter '%s' for service '%s' exceeds " - "maximum value of %d. Weighting will be ignored.", - weightby, service->name, INT_MAX); - } - else - { - for (int n = 0; router->servers[n]; n++) - { - BACKEND *backend = router->servers[n]; - char *param = serverGetParameter(backend->backend_server, weightby); - if (param) - { - int wght = atoi(param); - int perc = (wght * 1000) / total; - - if (perc == 0) - { - MXS_ERROR("Weighting parameter '%s' with a value of %d for" - " server '%s' rounds down to zero with total weight" - " of %d for service '%s'. No queries will be " - "routed to this server as long as a server with" - " positive weight is available.", - weightby, wght, backend->backend_server->unique_name, - total, service->name); - } - else if (perc < 0) - { - MXS_ERROR("Weighting parameter '%s' for server '%s' is too large, " - "maximum value is %d. No weighting will be used for this " - "server.", - weightby, backend->backend_server->unique_name, - INT_MAX / 1000); - perc = 1000; - } - backend->weight = perc; - } - else - { - MXS_WARNING("Server '%s' has no parameter '%s' used for weighting" - " for service '%s'.", - backend->backend_server->unique_name, weightby, - service->name); - } - } - } - } - /** Enable strict multistatement handling by default */ router->rwsplit_config.rw_strict_multi_stmt = true; From a163e31b9a352114702d2b98dd0f8b50b1238945 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 3 Nov 2016 12:00:33 +0200 Subject: [PATCH 151/215] Remove BACKEND structure from readwritesplit The BACKEND structure is no longer created for the instance. This allows sessions to dynamically create new servers. --- include/maxscale/service.h | 1 + server/core/service.c | 1 + .../routing/readwritesplit/readwritesplit.c | 122 +++++------------- .../routing/readwritesplit/readwritesplit.h | 25 +--- .../routing/readwritesplit/rwsplit_mysql.c | 6 +- .../readwritesplit/rwsplit_route_stmt.c | 80 ++++++------ .../readwritesplit/rwsplit_select_backends.c | 113 ++++++++-------- .../readwritesplit/rwsplit_session_cmd.c | 12 +- 8 files changed, 135 insertions(+), 225 deletions(-) diff --git a/include/maxscale/service.h b/include/maxscale/service.h index 74f9c9129..b6d7940d5 100644 --- a/include/maxscale/service.h +++ b/include/maxscale/service.h @@ -100,6 +100,7 @@ typedef struct server_ref_t struct server_ref_t *next; /**< Next server reference */ SERVER* server; /**< The actual server */ int weight; /**< Weight of this server */ + int connections; /**< Number of connections created through this reference */ } SERVER_REF; #define SERVICE_MAX_RETRY_INTERVAL 3600 /*< The maximum interval between service start retries */ diff --git a/server/core/service.c b/server/core/service.c index 6613f0779..f8f2a77a7 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -751,6 +751,7 @@ static SERVER_REF* server_ref_alloc(SERVER *server) sref->next = NULL; sref->server = server; sref->weight = SERVICE_BASE_SERVER_WEIGHT; + sref->connections = 0; } return sref; diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index add637776..357ab9610 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -183,12 +183,7 @@ ROUTER_OBJECT *GetModuleObject() static ROUTER *createInstance(SERVICE *service, char **options) { ROUTER_INSTANCE *router; - SERVER *server; - SERVER_REF *sref; - int nservers; - int i; CONFIG_PARAMETER *param; - char *weightby; if ((router = MXS_CALLOC(1, sizeof(ROUTER_INSTANCE))) == NULL) { @@ -197,51 +192,6 @@ static ROUTER *createInstance(SERVICE *service, char **options) router->service = service; spinlock_init(&router->lock); - /** Calculate number of servers */ - sref = service->dbref; - nservers = 0; - - while (sref != NULL) - { - nservers++; - sref = sref->next; - } - router->servers = (BACKEND **)MXS_CALLOC(nservers + 1, sizeof(BACKEND *)); - - if (router->servers == NULL) - { - free_rwsplit_instance(router); - return NULL; - } - /** - * Create an array of the backend servers in the router structure to - * maintain a count of the number of connections to each - * backend server. - */ - - sref = service->dbref; - nservers = 0; - - while (sref != NULL) - { - if ((router->servers[nservers] = MXS_MALLOC(sizeof(BACKEND))) == NULL) - { - free_rwsplit_instance(router); - return NULL; - } - router->servers[nservers]->backend_server = sref->server; - router->servers[nservers]->backend_conn_count = 0; - router->servers[nservers]->be_valid = false; - router->servers[nservers]->weight = sref->weight; -#if defined(SS_DEBUG) - router->servers[nservers]->be_chk_top = CHK_NUM_BACKEND; - router->servers[nservers]->be_chk_tail = CHK_NUM_BACKEND; -#endif - nservers += 1; - sref = sref->next; - } - router->servers[nservers] = NULL; - /* * Until we know otherwise assume we have some available slaves. */ @@ -268,6 +218,13 @@ static ROUTER *createInstance(SERVICE *service, char **options) router->rwsplit_config.rw_max_sescmd_history_size = 0; } + int nservers = 0; + + for (SERVER_REF *ref = service->dbref; ref; ref = ref->next) + { + nservers++; + } + /** * Set default value for max_slave_connections as 100%. This way * LEAST_CURRENT_OPERATIONS allows us to balance evenly across all the @@ -341,8 +298,7 @@ static ROUTER *createInstance(SERVICE *service, char **options) */ static void *newSession(ROUTER *router_inst, SESSION *session) { - backend_ref_t - *backend_ref; /*< array of backend references (DCB,BACKEND,cursor) */ + backend_ref_t *backend_ref; /*< array of backend references (DCB,BACKEND,cursor) */ backend_ref_t *master_ref = NULL; /*< pointer to selected master */ ROUTER_CLIENT_SES *client_rses = NULL; ROUTER_INSTANCE *router = (ROUTER_INSTANCE *)router_inst; @@ -417,7 +373,8 @@ static void *newSession(ROUTER *router_inst, SESSION *session) * Initialize backend references with BACKEND ptr. * Initialize session command cursors for each backend reference. */ - for (i = 0; i < router_nservers; i++) + i = 0; + for (SERVER_REF *sref = router->service->dbref; sref; sref = sref->next) { #if defined(SS_DEBUG) backend_ref[i].bref_chk_top = CHK_NUM_BACKEND_REF; @@ -426,13 +383,14 @@ static void *newSession(ROUTER *router_inst, SESSION *session) backend_ref[i].bref_sescmd_cur.scmd_cur_chk_tail = CHK_NUM_SESCMD_CUR; #endif backend_ref[i].bref_state = 0; - backend_ref[i].bref_backend = router->servers[i]; + backend_ref[i].ref = sref; /** store pointers to sescmd list to both cursors */ backend_ref[i].bref_sescmd_cur.scmd_cur_rses = client_rses; backend_ref[i].bref_sescmd_cur.scmd_cur_active = false; backend_ref[i].bref_sescmd_cur.scmd_cur_ptr_property = &client_rses->rses_properties[RSES_PROP_TYPE_SESCMD]; backend_ref[i].bref_sescmd_cur.scmd_cur_cmd = NULL; + i++; } max_nslaves = rses_get_max_slavecount(client_rses, router_nservers); max_slave_rlag = rses_get_max_replication_lag(client_rses); @@ -586,7 +544,7 @@ static void closeSession(ROUTER *instance, void *router_session) */ dcb_close(dcb); /** decrease server current connection counters */ - atomic_add(&bref->bref_backend->backend_conn_count, -1); + atomic_add(&bref->ref->connections, -1); } else { @@ -729,7 +687,6 @@ static void diagnostics(ROUTER *instance, DCB *dcb) ROUTER_CLIENT_SES *router_cli_ses; ROUTER_INSTANCE *router = (ROUTER_INSTANCE *)instance; int i = 0; - BACKEND *backend; char *weightby; spinlock_acquire(&router->lock); @@ -770,13 +727,12 @@ static void diagnostics(ROUTER *instance, DCB *dcb) dcb_printf(dcb, "\t\tServer Target %% Connections " "Operations\n"); dcb_printf(dcb, "\t\t Global Router\n"); - for (i = 0; router->servers[i]; i++) + for (SERVER_REF *ref = router->service->dbref; ref; ref = ref->next) { - backend = router->servers[i]; dcb_printf(dcb, "\t\t%-20s %3.1f%% %-6d %-6d %d\n", - backend->backend_server->unique_name, (float)backend->weight / 10, - backend->backend_server->stats.n_current, backend->backend_conn_count, - backend->backend_server->stats.n_current_ops); + ref->server->unique_name, (float)ref->weight / 10, + ref->server->stats.n_current, ref->connections, + ref->server->stats.n_current_ops); } } } @@ -923,17 +879,15 @@ static void clientReply(ROUTER *instance, void *router_session, GWBUF *writebuf, { bool succp; - MXS_INFO("Backend %s:%d processed reply and starts to execute " - "active cursor.", bref->bref_backend->backend_server->name, - bref->bref_backend->backend_server->port); + MXS_INFO("Backend %s:%d processed reply and starts to execute active cursor.", + bref->ref->server->name, bref->ref->server->port); succp = execute_sescmd_in_backend(bref); if (!succp) { MXS_INFO("Backend %s:%d failed to execute session command.", - bref->bref_backend->backend_server->name, - bref->bref_backend->backend_server->port); + bref->ref->server->name, bref->ref->server->port); } } else if (bref->bref_pending_cmd != NULL) /*< non-sescmd is waiting to be routed */ @@ -1092,13 +1046,13 @@ void bref_clear_state(backend_ref_t *bref, bref_state_t state) else { /** Decrease global operation count */ - prev2 = atomic_add(&bref->bref_backend->backend_server->stats.n_current_ops, -1); + prev2 = atomic_add(&bref->ref->server->stats.n_current_ops, -1); ss_dassert(prev2 > 0); if (prev2 <= 0) { MXS_ERROR("[%s] Error: negative current operation count in backend %s:%u", - __FUNCTION__, bref->bref_backend->backend_server->name, - bref->bref_backend->backend_server->port); + __FUNCTION__, bref->ref->server->name, + bref->ref->server->port); } } } @@ -1137,19 +1091,16 @@ void bref_set_state(backend_ref_t *bref, bref_state_t state) if (prev1 < 0) { MXS_ERROR("[%s] Error: negative number of connections waiting for " - "results in backend %s:%u", - __FUNCTION__, bref->bref_backend->backend_server->name, - bref->bref_backend->backend_server->port); + "results in backend %s:%u", __FUNCTION__, + bref->ref->server->name, bref->ref->server->port); } /** Increase global operation count */ - prev2 = - atomic_add(&bref->bref_backend->backend_server->stats.n_current_ops, 1); + prev2 = atomic_add(&bref->ref->server->stats.n_current_ops, 1); ss_dassert(prev2 >= 0); if (prev2 < 0) { MXS_ERROR("[%s] Error: negative current operation count in backend %s:%u", - __FUNCTION__, bref->bref_backend->backend_server->name, - bref->bref_backend->backend_server->port); + __FUNCTION__, bref->ref->server->name, bref->ref->server->port); } } @@ -1316,7 +1267,7 @@ int router_handle_state_switch(DCB *dcb, DCB_REASON reason, void *data) bref = (backend_ref_t *)data; CHK_BACKEND_REF(bref); - srv = bref->bref_backend->backend_server; + srv = bref->ref->server; if (SERVER_IS_RUNNING(srv) && SERVER_IS_IN_CLUSTER(srv)) { @@ -1531,9 +1482,9 @@ static void handleError(ROUTER *instance, void *router_session, * handled so that session could continue. */ if (rses->rses_master_ref && rses->rses_master_ref->bref_dcb == problem_dcb && - !SERVER_IS_MASTER(rses->rses_master_ref->bref_backend->backend_server)) + !SERVER_IS_MASTER(rses->rses_master_ref->ref->server)) { - SERVER *srv = rses->rses_master_ref->bref_backend->backend_server; + SERVER *srv = rses->rses_master_ref->ref->server; backend_ref_t *bref; bref = get_bref_from_dcb(rses, problem_dcb); if (bref != NULL) @@ -1794,9 +1745,8 @@ return_succp: static int router_get_servercount(ROUTER_INSTANCE *inst) { int router_nservers = 0; - BACKEND **b = inst->servers; - /** count servers */ - while (*(b++) != NULL) + + for (SERVER_REF *ref = inst->service->dbref; ref; ref = ref->next) { router_nservers++; } @@ -2074,14 +2024,6 @@ static void free_rwsplit_instance(ROUTER_INSTANCE *router) { if (router) { - if (router->servers) - { - for (int i = 0; router->servers[i]; i++) - { - MXS_FREE(router->servers[i]); - } - } - MXS_FREE(router->servers); MXS_FREE(router); } } diff --git a/server/modules/routing/readwritesplit/readwritesplit.h b/server/modules/routing/readwritesplit/readwritesplit.h index 98020b667..5a8d9dd36 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.h +++ b/server/modules/routing/readwritesplit/readwritesplit.h @@ -199,27 +199,6 @@ typedef struct sescmd_cursor_st #endif } sescmd_cursor_t; -/** - * Internal structure used to define the set of backend servers we are routing - * connections to. This provides the storage for routing module specific data - * that is required for each of the backend servers. - * - * Owned by router_instance, referenced by each routing session. - */ -typedef struct backend_st -{ -#if defined(SS_DEBUG) - skygw_chk_t be_chk_top; -#endif - SERVER* backend_server; /*< The server itself */ - int backend_conn_count; /*< Number of connections to the server */ - bool be_valid; /*< Valid when belongs to the router's configuration */ - int weight; /*< Desired weighting on the load. Expressed in .1% increments */ -#if defined(SS_DEBUG) - skygw_chk_t be_chk_tail; -#endif -} BACKEND; - /** * Reference to BACKEND. * @@ -230,7 +209,7 @@ typedef struct backend_ref_st #if defined(SS_DEBUG) skygw_chk_t bref_chk_top; #endif - BACKEND* bref_backend; + SERVER_REF* ref; DCB* bref_dcb; bref_state_t bref_state; int bref_num_result_wait; @@ -348,8 +327,6 @@ typedef struct router_instance SERVICE* service; /*< Pointer to service */ ROUTER_CLIENT_SES* connections; /*< List of client connections */ SPINLOCK lock; /*< Lock for the instance data */ - BACKEND** servers; /*< Backend servers */ - BACKEND* master; /*< NULL or pointer */ rwsplit_config_t rwsplit_config; /*< expanded config info from SERVICE */ int rwsplit_version; /*< version number for router's config */ ROUTER_STATS stats; /*< Statistics for this router */ diff --git a/server/modules/routing/readwritesplit/rwsplit_mysql.c b/server/modules/routing/readwritesplit/rwsplit_mysql.c index dbad88573..6dcdd50d5 100644 --- a/server/modules/routing/readwritesplit/rwsplit_mysql.c +++ b/server/modules/routing/readwritesplit/rwsplit_mysql.c @@ -404,7 +404,7 @@ void print_error_packet(ROUTER_CLIENT_SES *rses, GWBUF *buf, DCB *dcb) { if (bref[i].bref_dcb == dcb) { - srv = bref[i].bref_backend->backend_server; + srv = bref[i].ref->server; } } ss_dassert(srv != NULL); @@ -453,8 +453,8 @@ void check_session_command_reply(GWBUF *writebuf, sescmd_cursor_t *scur, backend ss_dassert(len + 4 == GWBUF_LENGTH(scur->scmd_cur_cmd->my_sescmd_buf)); MXS_ERROR("Failed to execute session command in %s:%d. Error was: %s %s", - bref->bref_backend->backend_server->name, - bref->bref_backend->backend_server->port, err, replystr); + bref->ref->server->name, + bref->ref->server->port, err, replystr); MXS_FREE(err); MXS_FREE(replystr); } diff --git a/server/modules/routing/readwritesplit/rwsplit_route_stmt.c b/server/modules/routing/readwritesplit/rwsplit_route_stmt.c index cdaddc5ca..ae6e42e1f 100644 --- a/server/modules/routing/readwritesplit/rwsplit_route_stmt.c +++ b/server/modules/routing/readwritesplit/rwsplit_route_stmt.c @@ -253,10 +253,10 @@ bool route_session_write(ROUTER_CLIENT_SES *router_cli_ses, BREF_IS_IN_USE((&backend_ref[i]))) { MXS_INFO("Route query to %s \t%s:%d%s", - (SERVER_IS_MASTER(backend_ref[i].bref_backend->backend_server) + (SERVER_IS_MASTER(backend_ref[i].ref->server) ? "master" : "slave"), - backend_ref[i].bref_backend->backend_server->name, - backend_ref[i].bref_backend->backend_server->port, + backend_ref[i].ref->server->name, + backend_ref[i].ref->server->port, (i + 1 == router_cli_ses->rses_nbackends ? " <" : " ")); } @@ -368,10 +368,10 @@ bool route_session_write(ROUTER_CLIENT_SES *router_cli_ses, if (MXS_LOG_PRIORITY_IS_ENABLED(LOG_INFO)) { MXS_INFO("Route query to %s \t%s:%d%s", - (SERVER_IS_MASTER(backend_ref[i].bref_backend->backend_server) + (SERVER_IS_MASTER(backend_ref[i].ref->server) ? "master" : "slave"), - backend_ref[i].bref_backend->backend_server->name, - backend_ref[i].bref_backend->backend_server->port, + backend_ref[i].ref->server->name, + backend_ref[i].ref->server->port, (i + 1 == router_cli_ses->rses_nbackends ? " <" : " ")); } @@ -391,8 +391,8 @@ bool route_session_write(ROUTER_CLIENT_SES *router_cli_ses, { nsucc += 1; MXS_INFO("Backend %s:%d already executing sescmd.", - backend_ref[i].bref_backend->backend_server->name, - backend_ref[i].bref_backend->backend_server->port); + backend_ref[i].ref->server->name, + backend_ref[i].ref->server->port); } else { @@ -403,8 +403,8 @@ bool route_session_write(ROUTER_CLIENT_SES *router_cli_ses, else { MXS_ERROR("Failed to execute session command in %s:%d", - backend_ref[i].bref_backend->backend_server->name, - backend_ref[i].bref_backend->backend_server->port); + backend_ref[i].ref->server->name, + backend_ref[i].ref->server->port); } } } @@ -533,9 +533,9 @@ bool rwsplit_get_dcb(DCB **p_dcb, ROUTER_CLIENT_SES *rses, backend_type_t btype, for (i = 0; i < rses->rses_nbackends; i++) { - BACKEND *b = backend_ref[i].bref_backend; + SERVER_REF *b = backend_ref[i].ref; SERVER server; - server.status = backend_ref[i].bref_backend->backend_server->status; + server.status = b->server->status; /** * To become chosen: * backend must be in use, name must match, @@ -543,7 +543,7 @@ bool rwsplit_get_dcb(DCB **p_dcb, ROUTER_CLIENT_SES *rses, backend_type_t btype, * server, or master. */ if (BREF_IS_IN_USE((&backend_ref[i])) && - (strncasecmp(name, b->backend_server->unique_name, PATH_MAX) == 0) && + (strncasecmp(name, b->server->unique_name, PATH_MAX) == 0) && (SERVER_IS_SLAVE(&server) || SERVER_IS_RELAY_SERVER(&server) || SERVER_IS_MASTER(&server))) { @@ -569,10 +569,10 @@ bool rwsplit_get_dcb(DCB **p_dcb, ROUTER_CLIENT_SES *rses, backend_type_t btype, for (i = 0; i < rses->rses_nbackends; i++) { - BACKEND *b = (&backend_ref[i])->bref_backend; + SERVER_REF *b = backend_ref[i].ref; SERVER server; SERVER candidate; - server.status = backend_ref[i].bref_backend->backend_server->status; + server.status = b->server->status; /** * Unused backend or backend which is not master nor * slave can't be used @@ -596,7 +596,7 @@ bool rwsplit_get_dcb(DCB **p_dcb, ROUTER_CLIENT_SES *rses, backend_type_t btype, { /** found master */ candidate_bref = &backend_ref[i]; - candidate.status = candidate_bref->bref_backend->backend_server->status; + candidate.status = candidate_bref->ref->server->status; succp = true; } /** @@ -605,12 +605,12 @@ bool rwsplit_get_dcb(DCB **p_dcb, ROUTER_CLIENT_SES *rses, backend_type_t btype, * maximum allowed replication lag. */ else if (max_rlag == MAX_RLAG_UNDEFINED || - (b->backend_server->rlag != MAX_RLAG_NOT_AVAILABLE && - b->backend_server->rlag <= max_rlag)) + (b->server->rlag != MAX_RLAG_NOT_AVAILABLE && + b->server->rlag <= max_rlag)) { /** found slave */ candidate_bref = &backend_ref[i]; - candidate.status = candidate_bref->bref_backend->backend_server->status; + candidate.status = candidate_bref->ref->server->status; succp = true; } } @@ -620,13 +620,13 @@ bool rwsplit_get_dcb(DCB **p_dcb, ROUTER_CLIENT_SES *rses, backend_type_t btype, */ else if (SERVER_IS_MASTER(&candidate) && SERVER_IS_SLAVE(&server) && (max_rlag == MAX_RLAG_UNDEFINED || - (b->backend_server->rlag != MAX_RLAG_NOT_AVAILABLE && - b->backend_server->rlag <= max_rlag)) && + (b->server->rlag != MAX_RLAG_NOT_AVAILABLE && + b->server->rlag <= max_rlag)) && !rses->rses_config.rw_master_reads) { /** found slave */ candidate_bref = &backend_ref[i]; - candidate.status = candidate_bref->bref_backend->backend_server->status; + candidate.status = candidate_bref->ref->server->status; succp = true; } /** @@ -637,21 +637,17 @@ bool rwsplit_get_dcb(DCB **p_dcb, ROUTER_CLIENT_SES *rses, backend_type_t btype, else if (SERVER_IS_SLAVE(&server)) { if (max_rlag == MAX_RLAG_UNDEFINED || - (b->backend_server->rlag != MAX_RLAG_NOT_AVAILABLE && - b->backend_server->rlag <= max_rlag)) + (b->server->rlag != MAX_RLAG_NOT_AVAILABLE && + b->server->rlag <= max_rlag)) { - candidate_bref = - check_candidate_bref(candidate_bref, &backend_ref[i], - rses->rses_config.rw_slave_select_criteria); - candidate.status = - candidate_bref->bref_backend->backend_server->status; + candidate_bref = check_candidate_bref(candidate_bref, &backend_ref[i], + rses->rses_config.rw_slave_select_criteria); + candidate.status = candidate_bref->ref->server->status; } else { - MXS_INFO("Server %s:%d is too much behind the " - "master, %d s. and can't be chosen.", - b->backend_server->name, b->backend_server->port, - b->backend_server->rlag); + MXS_INFO("Server %s:%d is too much behind the master, %d s. and can't be chosen.", + b->server->name, b->server->port, b->server->rlag); } } } /*< for */ @@ -675,7 +671,7 @@ bool rwsplit_get_dcb(DCB **p_dcb, ROUTER_CLIENT_SES *rses, backend_type_t btype, * so copying it locally will make possible error messages * easier to understand */ SERVER server; - server.status = master_bref->bref_backend->backend_server->status; + server.status = master_bref->ref->server->status; if (BREF_IS_IN_USE(master_bref) && SERVER_IS_MASTER(&server)) { *p_dcb = master_bref->bref_dcb; @@ -687,8 +683,8 @@ bool rwsplit_get_dcb(DCB **p_dcb, ROUTER_CLIENT_SES *rses, backend_type_t btype, { MXS_ERROR("Server at %s:%d should be master but " "is %s instead and can't be chosen to master.", - master_bref->bref_backend->backend_server->name, - master_bref->bref_backend->backend_server->port, + master_bref->ref->server->name, + master_bref->ref->server->port, STRSRVSTATUS(&server)); succp = false; } @@ -1191,9 +1187,8 @@ handle_got_target(ROUTER_INSTANCE *inst, ROUTER_CLIENT_SES *rses, ss_dassert(target_dcb != NULL); MXS_INFO("Route query to %s \t%s:%d <", - (SERVER_IS_MASTER(bref->bref_backend->backend_server) ? "master" - : "slave"), bref->bref_backend->backend_server->name, - bref->bref_backend->backend_server->port); + (SERVER_IS_MASTER(bref->ref->server) ? "master" + : "slave"), bref->ref->server->name, bref->ref->server->port); /** * Store current stmt if execution of previous session command * haven't completed yet. @@ -1372,14 +1367,13 @@ static backend_ref_t *get_root_master_bref(ROUTER_CLIENT_SES *rses) if (bref == rses->rses_master_ref) { /** Store master state for better error reporting */ - master.status = bref->bref_backend->backend_server->status; + master.status = bref->ref->server->status; } - if (bref->bref_backend->backend_server->status & SERVER_MASTER) + if (SERVER_IS_MASTER(bref->ref->server)) { if (candidate_bref == NULL || - (bref->bref_backend->backend_server->depth < - candidate_bref->bref_backend->backend_server->depth)) + (bref->ref->server->depth < candidate_bref->ref->server->depth)) { candidate_bref = bref; } diff --git a/server/modules/routing/readwritesplit/rwsplit_select_backends.c b/server/modules/routing/readwritesplit/rwsplit_select_backends.c index 1198dec52..fba1d7271 100644 --- a/server/modules/routing/readwritesplit/rwsplit_select_backends.c +++ b/server/modules/routing/readwritesplit/rwsplit_select_backends.c @@ -38,7 +38,7 @@ static bool connect_server(backend_ref_t *bref, SESSION *session, bool execute_h static void log_server_connections(select_criteria_t select_criteria, backend_ref_t *backend_ref, int router_nservers); -static BACKEND *get_root_master(backend_ref_t *servers, int router_nservers); +static SERVER_REF *get_root_master(backend_ref_t *servers, int router_nservers); static int bref_cmp_global_conn(const void *bref1, const void *bref2); @@ -103,10 +103,10 @@ bool select_connect_backend_servers(backend_ref_t **p_master_ref, } /* get the root Master */ - BACKEND *master_host = get_root_master(backend_ref, router_nservers); + SERVER_REF *master_host = get_root_master(backend_ref, router_nservers); if (router->rwsplit_config.rw_master_failure_mode == RW_FAIL_INSTANTLY && - (master_host == NULL || SERVER_IS_DOWN(master_host->backend_server))) + (master_host == NULL || SERVER_IS_DOWN(master_host->server))) { MXS_ERROR("Couldn't find suitable Master from %d candidates.", router_nservers); return false; @@ -145,7 +145,7 @@ bool select_connect_backend_servers(backend_ref_t **p_master_ref, for (int i = 0; i < router_nservers && (slaves_connected < max_nslaves || !master_connected); i++) { - SERVER *serv = backend_ref[i].bref_backend->backend_server; + SERVER *serv = backend_ref[i].ref->server; if (!BREF_HAS_FAILED(&backend_ref[i]) && SERVER_IS_RUNNING(serv)) { @@ -155,7 +155,7 @@ bool select_connect_backend_servers(backend_ref_t **p_master_ref, (serv->rlag != MAX_RLAG_NOT_AVAILABLE && serv->rlag <= max_slave_rlag)) && (SERVER_IS_SLAVE(serv) || SERVER_IS_RELAY_SERVER(serv)) && - (master_host == NULL || (serv != master_host->backend_server))) + (master_host == NULL || (serv != master_host->server))) { slaves_found += 1; @@ -166,7 +166,7 @@ bool select_connect_backend_servers(backend_ref_t **p_master_ref, } } /* take the master_host for master */ - else if (master_host && (serv == master_host->backend_server)) + else if (master_host && (serv == master_host->server)) { /** p_master_ref must be assigned with this backend_ref pointer * because its original value may have been lost when backend @@ -205,9 +205,9 @@ bool select_connect_backend_servers(backend_ref_t **p_master_ref, if (BREF_IS_IN_USE((&backend_ref[i]))) { MXS_INFO("Selected %s in \t%s:%d", - STRSRVSTATUS(backend_ref[i].bref_backend->backend_server), - backend_ref[i].bref_backend->backend_server->name, - backend_ref[i].bref_backend->backend_server->port); + STRSRVSTATUS(backend_ref[i].ref->server), + backend_ref[i].ref->server->name, + backend_ref[i].ref->server->port); } } /* for */ } @@ -226,12 +226,12 @@ bool select_connect_backend_servers(backend_ref_t **p_master_ref, { if (BREF_IS_IN_USE((&backend_ref[i]))) { - ss_dassert(backend_ref[i].bref_backend->backend_conn_count > 0); + ss_dassert(backend_ref[i].ref->connections > 0); /** disconnect opened connections */ bref_clear_state(&backend_ref[i], BREF_IN_USE); /** Decrease backend's connection counter. */ - atomic_add(&backend_ref[i].bref_backend->backend_conn_count, -1); + atomic_add(&backend_ref[i].ref->connections, -1); dcb_close(backend_ref[i].bref_dcb); } } @@ -243,13 +243,12 @@ bool select_connect_backend_servers(backend_ref_t **p_master_ref, /** Compare number of connections from this router in backend servers */ static int bref_cmp_router_conn(const void *bref1, const void *bref2) { - BACKEND *b1 = ((backend_ref_t *)bref1)->bref_backend; - BACKEND *b2 = ((backend_ref_t *)bref2)->bref_backend; + SERVER_REF *b1 = ((backend_ref_t *)bref1)->ref; + SERVER_REF *b2 = ((backend_ref_t *)bref2)->ref; if (b1->weight == 0 && b2->weight == 0) { - return b1->backend_server->stats.n_current - - b2->backend_server->stats.n_current; + return b1->connections - b2->connections; } else if (b1->weight == 0) { @@ -260,20 +259,20 @@ static int bref_cmp_router_conn(const void *bref1, const void *bref2) return -1; } - return ((1000 + 1000 * b1->backend_conn_count) / b1->weight) - - ((1000 + 1000 * b2->backend_conn_count) / b2->weight); + return ((1000 + 1000 * b1->connections) / b1->weight) - + ((1000 + 1000 * b2->connections) / b2->weight); } /** Compare number of global connections in backend servers */ static int bref_cmp_global_conn(const void *bref1, const void *bref2) { - BACKEND *b1 = ((backend_ref_t *)bref1)->bref_backend; - BACKEND *b2 = ((backend_ref_t *)bref2)->bref_backend; + SERVER_REF *b1 = ((backend_ref_t *)bref1)->ref; + SERVER_REF *b2 = ((backend_ref_t *)bref2)->ref; if (b1->weight == 0 && b2->weight == 0) { - return b1->backend_server->stats.n_current - - b2->backend_server->stats.n_current; + return b1->server->stats.n_current - + b2->server->stats.n_current; } else if (b1->weight == 0) { @@ -284,32 +283,29 @@ static int bref_cmp_global_conn(const void *bref1, const void *bref2) return -1; } - return ((1000 + 1000 * b1->backend_server->stats.n_current) / b1->weight) - - ((1000 + 1000 * b2->backend_server->stats.n_current) / b2->weight); + return ((1000 + 1000 * b1->server->stats.n_current) / b1->weight) - + ((1000 + 1000 * b2->server->stats.n_current) / b2->weight); } /** Compare replication lag between backend servers */ static int bref_cmp_behind_master(const void *bref1, const void *bref2) { - BACKEND *b1 = ((backend_ref_t *)bref1)->bref_backend; - BACKEND *b2 = ((backend_ref_t *)bref2)->bref_backend; + SERVER_REF *b1 = ((backend_ref_t *)bref1)->ref; + SERVER_REF *b2 = ((backend_ref_t *)bref2)->ref; - return ((b1->backend_server->rlag < b2->backend_server->rlag) ? -1 - : ((b1->backend_server->rlag > b2->backend_server->rlag) ? 1 : 0)); + return b1->server->rlag - b2->server->rlag; } /** Compare number of current operations in backend servers */ static int bref_cmp_current_load(const void *bref1, const void *bref2) { - SERVER *s1 = ((backend_ref_t *)bref1)->bref_backend->backend_server; - SERVER *s2 = ((backend_ref_t *)bref2)->bref_backend->backend_server; - BACKEND *b1 = ((backend_ref_t *)bref1)->bref_backend; - BACKEND *b2 = ((backend_ref_t *)bref2)->bref_backend; + SERVER_REF *b1 = ((backend_ref_t *)bref1)->ref; + SERVER_REF *b2 = ((backend_ref_t *)bref2)->ref; if (b1->weight == 0 && b2->weight == 0) { - return b1->backend_server->stats.n_current - - b2->backend_server->stats.n_current; + // TODO: Fix this so that operations are used instead of connections + return b1->server->stats.n_current - b2->server->stats.n_current; } else if (b1->weight == 0) { @@ -320,8 +316,8 @@ static int bref_cmp_current_load(const void *bref1, const void *bref2) return -1; } - return ((1000 * s1->stats.n_current_ops) - b1->weight) - - ((1000 * s2->stats.n_current_ops) - b2->weight); + return ((1000 * b1->server->stats.n_current_ops) - b1->weight) - + ((1000 * b2->server->stats.n_current_ops) - b2->weight); } /** @@ -338,7 +334,7 @@ static int bref_cmp_current_load(const void *bref1, const void *bref2) */ static bool connect_server(backend_ref_t *bref, SESSION *session, bool execute_history) { - SERVER *serv = bref->bref_backend->backend_server; + SERVER *serv = bref->ref->server; bool rval = false; bref->bref_dcb = dcb_connect(serv, session, serv->protocol); @@ -354,16 +350,16 @@ static bool connect_server(backend_ref_t *bref, SESSION *session, bool execute_h &router_handle_state_switch, (void *) bref); bref->bref_state = 0; bref_set_state(bref, BREF_IN_USE); - atomic_add(&bref->bref_backend->backend_conn_count, 1); + atomic_add(&bref->ref->connections, 1); rval = true; } else { MXS_ERROR("Failed to execute session command in %s (%s:%d). See earlier " "errors for more details.", - bref->bref_backend->backend_server->unique_name, - bref->bref_backend->backend_server->name, - bref->bref_backend->backend_server->port); + bref->ref->server->unique_name, + bref->ref->server->name, + bref->ref->server->port); dcb_close(bref->bref_dcb); bref->bref_dcb = NULL; } @@ -398,33 +394,33 @@ static void log_server_connections(select_criteria_t select_criteria, for (int i = 0; i < router_nservers; i++) { - BACKEND *b = backend_ref[i].bref_backend; + SERVER_REF *b = backend_ref[i].ref; switch (select_criteria) { case LEAST_GLOBAL_CONNECTIONS: MXS_INFO("MaxScale connections : %d in \t%s:%d %s", - b->backend_server->stats.n_current, b->backend_server->name, - b->backend_server->port, STRSRVSTATUS(b->backend_server)); + b->server->stats.n_current, b->server->name, + b->server->port, STRSRVSTATUS(b->server)); break; case LEAST_ROUTER_CONNECTIONS: MXS_INFO("RWSplit connections : %d in \t%s:%d %s", - b->backend_conn_count, b->backend_server->name, - b->backend_server->port, STRSRVSTATUS(b->backend_server)); + b->connections, b->server->name, + b->server->port, STRSRVSTATUS(b->server)); break; case LEAST_CURRENT_OPERATIONS: MXS_INFO("current operations : %d in \t%s:%d %s", - b->backend_server->stats.n_current_ops, - b->backend_server->name, b->backend_server->port, - STRSRVSTATUS(b->backend_server)); + b->server->stats.n_current_ops, + b->server->name, b->server->port, + STRSRVSTATUS(b->server)); break; case LEAST_BEHIND_MASTER: MXS_INFO("replication lag : %d in \t%s:%d %s", - b->backend_server->rlag, b->backend_server->name, - b->backend_server->port, STRSRVSTATUS(b->backend_server)); + b->server->rlag, b->server->name, + b->server->port, STRSRVSTATUS(b->server)); default: break; } @@ -445,27 +441,26 @@ static void log_server_connections(select_criteria_t select_criteria, * @return The Master found * */ -static BACKEND *get_root_master(backend_ref_t *servers, int router_nservers) +static SERVER_REF *get_root_master(backend_ref_t *servers, int router_nservers) { int i = 0; - BACKEND *master_host = NULL; + SERVER_REF *master_host = NULL; for (i = 0; i < router_nservers; i++) { - BACKEND *b; - - if (servers[i].bref_backend == NULL) + if (servers[i].ref == NULL) { + /** This should not happen */ + ss_dassert(false); continue; } - b = servers[i].bref_backend; + SERVER_REF *b = servers[i].ref; - if ((b->backend_server->status & (SERVER_MASTER | SERVER_MAINT)) == - SERVER_MASTER) + if (SERVER_IS_MASTER(b->server)) { if (master_host == NULL || - (b->backend_server->depth < master_host->backend_server->depth)) + (b->server->depth < master_host->server->depth)) { master_host = b; } diff --git a/server/modules/routing/readwritesplit/rwsplit_session_cmd.c b/server/modules/routing/readwritesplit/rwsplit_session_cmd.c index b37951543..f06bf96e0 100644 --- a/server/modules/routing/readwritesplit/rwsplit_session_cmd.c +++ b/server/modules/routing/readwritesplit/rwsplit_session_cmd.c @@ -173,7 +173,7 @@ GWBUF *sescmd_cursor_process_replies(GWBUF *replybuf, { MXS_ERROR("Slave server '%s': response differs from master's response. " "Closing connection due to inconsistent session state.", - bref->bref_backend->backend_server->unique_name); + bref->ref->server->unique_name); sescmd_cursor_set_active(scur, false); bref_clear_state(bref, BREF_QUERY_ACTIVE); bref_clear_state(bref, BREF_IN_USE); @@ -205,7 +205,7 @@ GWBUF *sescmd_cursor_process_replies(GWBUF *replybuf, scmd->reply_cmd = *((unsigned char *)replybuf->start + 4); MXS_INFO("Server '%s' responded to a session command, sending the response " - "to the client.", bref->bref_backend->backend_server->unique_name); + "to the client.", bref->ref->server->unique_name); for (int i = 0; i < ses->rses_nbackends; i++) { @@ -226,8 +226,8 @@ GWBUF *sescmd_cursor_process_replies(GWBUF *replybuf, *reconnect = true; MXS_INFO("Disabling slave %s:%d, result differs from " "master's result. Master: %d Slave: %d", - ses->rses_backend_ref[i].bref_backend->backend_server->name, - ses->rses_backend_ref[i].bref_backend->backend_server->port, + ses->rses_backend_ref[i].ref->server->name, + ses->rses_backend_ref[i].ref->server->port, bref->reply_cmd, ses->rses_backend_ref[i].reply_cmd); } } @@ -237,11 +237,11 @@ GWBUF *sescmd_cursor_process_replies(GWBUF *replybuf, else { MXS_INFO("Slave '%s' responded before master to a session command. Result: %d", - bref->bref_backend->backend_server->unique_name, + bref->ref->server->unique_name, (int)bref->reply_cmd); if (bref->reply_cmd == 0xff) { - SERVER *serv = bref->bref_backend->backend_server; + SERVER *serv = bref->ref->server; MXS_ERROR("Slave '%s' (%s:%u) failed to execute session command.", serv->unique_name, serv->name, serv->port); } From ded551e19995f7d57fdd36317a3c8f9f13d5c150 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 3 Nov 2016 15:39:51 +0200 Subject: [PATCH 152/215] Add function for full memory barrier into maxscale/atomic.h This allows safer lock-free reads to be done on lists that never shrink in size. The main use-case for this is to allow servers to be added to a service without locking the service each time a new session is created. Synchronizing the memory before adding new components into a list guarantees that if a session reads from the list and sees the new list item, the memory pointed by the item is valid. --- include/maxscale/atomic.h | 37 ++++++++++++++++++++++++++++++++++++- server/core/atomic.c | 18 +----------------- 2 files changed, 37 insertions(+), 18 deletions(-) diff --git a/include/maxscale/atomic.h b/include/maxscale/atomic.h index f6d694e0c..f7614b3d3 100644 --- a/include/maxscale/atomic.h +++ b/include/maxscale/atomic.h @@ -29,7 +29,42 @@ MXS_BEGIN_DECLS -int atomic_add(int *variable, int value); +/** + * Implementation of an atomic add operation for the GCC environment, or the + * X86 processor. If we are working within GNU C then we can use the GCC + * atomic add built in function, which is portable across platforms that + * implement GCC. Otherwise, this function currently supports only X86 + * architecture (without further development). + * + * Adds a value to the contents of a location pointed to by the first parameter. + * The add operation is atomic and the return value is the value stored in the + * location prior to the operation. The number that is added may be signed, + * therefore atomic_subtract is merely an atomic add with a negative value. + * + * @param variable Pointer the the variable to add to + * @param value Value to be added + * @return The value of variable before the add occurred + */ int atomic_add(int *variable, int value); +/** + * @brief Impose a full memory barrier + * + * A full memory barrier guarantees that all store and load operations complete + * before the function is called. + * + * Currently, only the GNUC __sync_synchronize() is used. C11 introduces + * standard functions for atomic memory operations and should be taken into use. + * + * @see https://www.kernel.org/doc/Documentation/memory-barriers.txt + */ +static inline void atomic_synchronize() +{ +#ifdef __GNUC__ + __sync_synchronize(); /* Memory barrier. */ +#else +#error "No GNUC atomics available." +#endif +} + MXS_END_DECLS diff --git a/server/core/atomic.c b/server/core/atomic.c index fe5c5eb45..72b76f432 100644 --- a/server/core/atomic.c +++ b/server/core/atomic.c @@ -12,7 +12,7 @@ */ /** - * @file atomic.c - Implementation of atomic opertions for the gateway + * @file atomic.c - Implementation of atomic operations for MaxScale * * @verbatim * Revision History @@ -23,22 +23,6 @@ * @endverbatim */ -/** - * Implementation of an atomic add operation for the GCC environment, or the - * X86 processor. If we are working within GNU C then we can use the GCC - * atomic add built in function, which is portable across platforms that - * implement GCC. Otherwise, this function currently supports only X86 - * architecture (without further development). - * - * Adds a value to the contents of a location pointed to by the first parameter. - * The add operation is atomic and the return value is the value stored in the - * location prior to the operation. The number that is added may be signed, - * therefore atomic_subtract is merely an atomic add with a negative value. - * - * @param variable Pointer the the variable to add to - * @param value Value to be added - * @return The value of variable before the add occurred - */ int atomic_add(int *variable, int value) { From 59e615eb3c01414ef9508a08d5211f214b1e1c42 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 26 Jul 2016 16:50:51 +0300 Subject: [PATCH 153/215] Create initial REST API documentation The Resources document lists all currently known resources and will be converted to a list of separate resource documents once the individual resource documents are done. The Headers and Response Codes document contains a list of request and response headers and the HTTP return codes. This is a work in progress and will be expanded upon. Added initial versions of the filter, monitor, session and user resource documents. These provide information about various parts of MaxScale and allow interaction. All PATCH operations expect a JSON Patch type document in the request body. Examples modified accordingly. Add MaxScale resource document which describes the resources that give the global configuration options and show statistics. The resources that link to other resources provide values as a list of relative links. This reduces the amount of sent data when the client doesn't require all parts of the resource. Since the updating is not yet implement, it should be stated that the update API will most likely be modified at some point. The module resource is similar to `maxadmin show modules` and the log resource contains status information about logs. The log resource also has an "action" resource at `/maxscale/logs/flush` which flushes logs to disk and rotates them. Added start and stop entry points for services and monitors. Also added missing service parameters to the list of parameters that can be updated. The documentation lists listeners as a sub-resource of the service. This allows the listeners of a particular service to be queried. --- Documentation/Documentation-Contents.md | 1 + Documentation/REST-API/API.md | 453 +++++++++++++++++++ Documentation/REST-API/Resources-Filter.md | 151 +++++++ Documentation/REST-API/Resources-MaxScale.md | 216 +++++++++ Documentation/REST-API/Resources-Monitor.md | 176 +++++++ Documentation/REST-API/Resources-Server.md | 207 +++++++++ Documentation/REST-API/Resources-Service.md | 272 +++++++++++ Documentation/REST-API/Resources-Session.md | 138 ++++++ Documentation/REST-API/Resources-User.md | 81 ++++ 9 files changed, 1695 insertions(+) create mode 100644 Documentation/REST-API/API.md create mode 100644 Documentation/REST-API/Resources-Filter.md create mode 100644 Documentation/REST-API/Resources-MaxScale.md create mode 100644 Documentation/REST-API/Resources-Monitor.md create mode 100644 Documentation/REST-API/Resources-Server.md create mode 100644 Documentation/REST-API/Resources-Service.md create mode 100644 Documentation/REST-API/Resources-Session.md create mode 100644 Documentation/REST-API/Resources-User.md diff --git a/Documentation/Documentation-Contents.md b/Documentation/Documentation-Contents.md index 69070b10b..0b2fe2a43 100644 --- a/Documentation/Documentation-Contents.md +++ b/Documentation/Documentation-Contents.md @@ -31,6 +31,7 @@ - [Debug and Diagnostic Support](Reference/Debug-And-Diagnostic-Support.md) - [Routing Hints](Reference/Hint-Syntax.md) - [MaxBinlogCheck](Reference/MaxBinlogCheck.md) + - [MaxScale REST API](REST-API/API.md) ## Tutorials diff --git a/Documentation/REST-API/API.md b/Documentation/REST-API/API.md new file mode 100644 index 000000000..afa08573b --- /dev/null +++ b/Documentation/REST-API/API.md @@ -0,0 +1,453 @@ +# REST API design document + +This document describes the version 1 of the MaxScale REST API. + +## Table of Contents + +- [HTTP Headers](#http-headers) + - [Request Headers](#request-headers) + - [Response Headers](#response-headers) +- [Response Codes](#response-codes) + - [2xx Success](#2xx-success) + - [3xx Redirection](#3xx-redirection) + - [4xx Client Error](#4xx-client-error) + - [5xx Server Error](#5xx-server-error) +- [Resources](#resources) +- [Common Request Parameter](#common-request-parameters) + +## HTTP Headers + +### Request Headers + +REST makes use of the HTTP protocols in its aim to provide a natural way to +understand the workings of an API. The following request headers are understood +by this API. + +#### Accept-Charset + +Acceptable character sets. + +#### Authorization + +Credentials for authentication. + +#### Content-Type + +All PUT and POST requests must use the `Content-Type: application/json` media +type and the request body must be a valid JSON representation of a resource. All +PATCH requests must use the `Content-Type: application/json-patch` media type +and the request body must be a valid JSON Patch document which is applied to the +resource. Curently, only _add_, _remove_, _replace_ and _test_ operations are +supported. + +Read the [JSON Patch](https://tools.ietf.org/html/draft-ietf-appsawg-json-patch-08) +draft for more details on how to use it with PATCH. + +#### Date + +This header is required and should be in the RFC 1123 standard form, e.g. Mon, +18 Nov 2013 08:14:29 -0600. Please note that the date must be in English. It +will be checked by the API for being close to the current date and time. + +#### Host + +The address and port of the server. + +#### If-Match + +The request is performed only if the provided ETag value matches the one on the +server. This field should be used with PUT requests to prevent concurrent +updates to the same resource. + +The value of this header must be a value from the `ETag` header retrieved from +the same resource at an earlier point in time. + +#### If-Modified-Since + +If the content has not changed the server responds with a 304 status code. If +the content has changed the server responds with a 200 status code and the +requested resource. + +The value of this header must be a date value in the +["HTTP-date"](https://www.ietf.org/rfc/rfc2822.txt) format. + +#### If-None-Match + +If the content has not changed the server responds with a 304 status code. If +the content has changed the server responds with a 200 status code and the +requested resource. + +The value of this header must be a value from the `ETag` header retrieved from +the same resource at an earlier point in time. + +#### If-Unmodified-Since + +The request is performed only if the requested resource has not been modified +since the provided date. + +The value of this header must be a date value in the +["HTTP-date"](https://www.ietf.org/rfc/rfc2822.txt) format. + +#### X-HTTP-Method-Override + +Some clients only support GET and PUT requests. By providing the string value of +the intended method in the `X-HTTP-Method-Override` header, a client can perform +a POST, PATCH or DELETE request with the PUT method +(e.g. `X-HTTP-Method-Override: PATCH`). + +_TODO: Add API version header?_ + +### Response Headers + +#### Allow + +All resources return the Allow header with the supported HTTP methods. For +example the resource `/service` will always return the `Accept: GET, PATCH, PUT` +header. + +#### Accept-Patch + +All PATCH capable resources return the `Accept-Patch: application/json-patch` +header. + +#### Date + +Returns the RFC 1123 standard form date when the reply was sent. The date is in +English and it uses the server's local timezone. + +#### ETag + +An identifier for a specific version of a resource. The value of this header +changes whenever a resource is modified. + +When the client sends the `If-Match` or `If-None-Match` header, the provided +value should be the value of the `ETag` header of an earlier GET. + +#### Last-Modified + +The date when the resource was last modified in "HTTP-date" format. + +#### Location + +If an out of date resource location is requested, a HTTP return code of 3XX with +the `Location` header is returned. The value of the header contains the new +location of the requested resource as a relative URI. + +#### WWW-Authenticate + +The requested authentication method. For example, `WWW-Authenticate: Basic` +would require basic HTTP authentication. + +## Response Codes + +Every HTTP response starts with a line with a return code which indicates the +outcome of the request. The API uses some of the standard HTTP values: + +### 2xx Success + +- 200 OK + + - Successful HTTP requests, response has a body. + +- 201 Created + + - A new resource was created. + +- 202 Accepted + + - The request has been accepted for processing, but the processing has not + been completed. + +- 204 No Content + + - Successful HTTP requests, response has no body. + +### 3xx Redirection + +This class of status code indicates the client must take additional action to +complete the request. + +- 301 Moved Permanently + + - This and all future requests should be directed to the given URI. + +- 302 Found + + - The response to the request can be found under another URI using the same + method as in the original request. + +- 303 See Other + + - The response to the request can be found under another URI using a GET + method. + +- 304 Not Modified + + - Indicates that the resource has not been modified since the version + specified by the request headers If-Modified-Since or If-None-Match. + +- 307 Temporary Redirect + + - The request should be repeated with another URI but future requests should + use the original URI. + +- 308 Permanent Redirect + + - The request and all future requests should be repeated using another URI. + +### 4xx Client Error + +The 4xx class of status code is when the client seems to have erred. Except when +responding to a HEAD request, the body of the response contains a JSON +representation of the error in the following format. + +``` +{ + "error": "Method not supported", + "description": "The `/service` resource does not support POST." +} +``` + +The _error_ field contains a short error description and the _description_ field +contains a more detailed version of the error message. + +- 400 Bad Request + + - The server cannot or will not process the request due to client error. + +- 401 Unauthorized + + - Authentication is required. The response includes a WWW-Authenticate header. + +- 403 Forbidden + + - The request was a valid request, but the client does not have the necessary + permissions for the resource. + +- 404 Not Found + + - The requested resource could not be found. + +- 405 Method Not Allowed + + - A request method is not supported for the requested resource. + +- 406 Not Acceptable + + - The requested resource is capable of generating only content not acceptable + according to the Accept headers sent in the request. + +- 409 Conflict + + - Indicates that the request could not be processed because of conflict in the + request, such as an edit conflict be tween multiple simultaneous updates. + +- 411 Length Required + + - The request did not specify the length of its content, which is required by + the requested resource. + +- 412 Precondition Failed + + - The server does not meet one of the preconditions that the requester put on + the request. + +- 413 Payload Too Large + + - The request is larger than the server is willing or able to process. + +- 414 URI Too Long + + - The URI provided was too long for the server to process. + +- 415 Unsupported Media Type + + - The request entity has a media type which the server or resource does not + support. + +- 422 Unprocessable Entity + + - The request was well-formed but was unable to be followed due to semantic + errors. + +- 423 Locked + + - The resource that is being accessed is locked. + +- 428 Precondition Required + + - The origin server requires the request to be conditional. This error code is + returned when none of the `Modified-Since` or `Match` type headers are used. + +- 431 Request Header Fields Too Large + + - The server is unwilling to process the request because either an individual + header field, or all the header fields collectively, are too large. + +### 5xx Server Error + +The server failed to fulfill an apparently valid request. + +Response status codes beginning with the digit "5" indicate cases in which the +server is aware that it has encountered an error or is otherwise incapable of +performing the request. Except when responding to a HEAD request, the server +includes an entity containing an explanation of the error situation. + +``` +{ + "error": "Log rotation failed", + "description": "Failed to rotate log files: 13, Permission denied" +} +``` + +The _error_ field contains a short error description and the _description_ field +contains a more detailed version of the error message. + +- 500 Internal Server Error + + - A generic error message, given when an unexpected condition was encountered + and no more specific message is suitable. + +- 501 Not Implemented + + - The server either does not recognize the request method, or it lacks the + ability to fulfill the request. + +- 502 Bad Gateway + + - The server was acting as a gateway or proxy and received an invalid response + from the upstream server. + +- 503 Service Unavailable + + - The server is currently unavailable (because it is overloaded or down for + maintenance). Generally, this is a temporary state. + +- 504 Gateway Timeout + + - The server was acting as a gateway or proxy and did not receive a timely + response from the upstream server. + +- 505 HTTP Version Not Supported + + - The server does not support the HTTP protocol version used in the request. + +- 506 Variant Also Negotiates + + - Transparent content negotiation for the request results in a circular + reference. + +- 507 Insufficient Storage + + - The server is unable to store the representation needed to complete the + request. + +- 508 Loop Detected + + - The server detected an infinite loop while processing the request (sent in + lieu of 208 Already Reported). + +- 510 Not Extended + + - Further extensions to the request are required for the server to fulfil it. + +### Response Headers Reserved for Future Use + +The following response headers are not currently in use. Future versions of the +API could return them. + +- 206 Partial Content + + - The server is delivering only part of the resource (byte serving) due to a + range header sent by the client. + +- 300 Multiple Choices + + - Indicates multiple options for the resource from which the client may choose + (via agent-driven content negotiation). + +- 407 Proxy Authentication Required + + - The client must first authenticate itself with the proxy. + +- 408 Request Timeout + + - The server timed out waiting for the request. According to HTTP + specifications: "The client did not produce a request within the time that + the server was prepared to wait. The client MAY repeat the request without + modifications at any later time." + +- 410 Gone + + - Indicates that the resource requested is no longer available and will not be + available again. + +- 416 Range Not Satisfiable + + - The client has asked for a portion of the file (byte serving), but the + server cannot supply that portion. + +- 417 Expectation Failed + + - The server cannot meet the requirements of the Expect request-header field. + +- 421 Misdirected Request + + - The request was directed at a server that is not able to produce a response. + +- 424 Failed Dependency + + - The request failed due to failure of a previous request. + +- 426 Upgrade Required + + - The client should switch to a different protocol such as TLS/1.0, given in + the Upgrade header field. + +- 429 Too Many Requests + + - The user has sent too many requests in a given amount of time. Intended for + use with rate-limiting schemes. + +## Resources + +The MaxScale REST API provides the following resources. + +- [/maxscale](Resources-MaxScale.md) +- [/services](Resources-Service.md) +- [/servers](Resources-Server.md) +- [/filters](Resources-Filter.md) +- [/monitors](Resources-Monitor.md) +- [/sessions](Resources-Session.md) +- [/users](Resources-User.md) + +## Common Request Parameters + +Most of the resources that support GET also support the following +parameters. See the resource documentation for a list of supported request +parameters. + +- `fields` + + - A list of fields to return. + + This allows the returned object to be filtered so that only needed + parts are returned. The value of this parameter is a comma separated + list of fields to return. + + For example, the parameter `?fields=id,name` would return object which + would only contain the _id_ and _name_ fields. + +- `range` + + - Return a subset of the object array. + + The value of this parameter is the range of objects to return given as + a inclusive range separated by a hyphen. If the size of the array is + less than the end of the range, only the objects between the requested + start of the range and the actual end of the array are returned. This + means that + + For example, the parameter `?range=10-20` would return objects 10 + through 20 from the object array if the actual size of the original + array is greater than or equal to 20. diff --git a/Documentation/REST-API/Resources-Filter.md b/Documentation/REST-API/Resources-Filter.md new file mode 100644 index 000000000..2235209b1 --- /dev/null +++ b/Documentation/REST-API/Resources-Filter.md @@ -0,0 +1,151 @@ +# Filter Resource + +A filter resource represents an instance of a filter inside MaxScale. Multiple +services can use the same filter and a single service can use multiple filters. + +## Resource Operations + +### Get a filter + +Get a single filter. The _:name_ in the URI must be a valid filter name with all +whitespace replaced with hyphens. The filter names are case-insensitive. + +``` +GET /filters/:name +``` + +#### Response + +``` +Status: 200 OK + +{ + "name": "Query Logging Filter", + "module": "qlafilter", + "parameters": { + "filebase": { + "value": "/var/log/maxscale/qla/log.", + "configurable": false + }, + "match": { + "value": "select.*from.*t1", + "configurable": true + } + }, + "services": [ + "/services/my-service", + "/services/my-second-service" + ] +} +``` + +#### Supported Request Parameter + +- `fields` + +### Get all filters + +Get all filters. + +``` +GET /filters +``` + +#### Response + +``` +Status: 200 OK + +[ + { + "name": "Query Logging Filter", + "module": "qlafilter", + "parameters": { + "filebase": { + "value": "/var/log/maxscale/qla/log.", + "configurable": false + }, + "match": { + "value": "select.*from.*t1", + "configurable": true + } + }, + "services": [ + "/services/my-service", + "/services/my-second-service + ] + }, + { + "name": "DBFW Filter", + "module": "dbfwfilter", + "parameters": { + { + "name": "rules", + "value": "/etc/maxscale-rules", + "configurable": false + } + }, + "services": [ + "/services/my-second-service + ] + } +] +``` + +#### Supported Request Parameter + +- `fields` +- `range` + +### Update a filter + +**Note**: The update mechanisms described here are provisional and most likely + will change in the future. This description is only for design purposes and + does not yet work. + +Partially update a filter. The _:name_ in the URI must map to a filter name +and the request body must be a valid JSON Patch document which is applied to the +resource. + +``` +PATCH /filter/:name +``` + +### Modifiable Fields + +|Field |Type |Description | +|------------|-------|---------------------------------| +|parameters |object |Module specific filter parameters| + +``` +[ + { "op": "replace", "path": "/parameters/rules/value", "value": "/etc/new-rules" }, + { "op": "add", "path": "/parameters/action/value", "value": "allow" } +] +``` + +#### Response + +Response contains the modified resource. + +``` +Status: 200 OK + +{ + "name": "DBFW Filter", + "module": "dbfwfilter", + "parameters": { + "rules": { + "value": "/etc/new-rules", + "configurable": false + }, + "action": { + "value": "allow", + "configurable": true + } + } + "services": [ + "/services/my-second-service" + ] +} +``` diff --git a/Documentation/REST-API/Resources-MaxScale.md b/Documentation/REST-API/Resources-MaxScale.md new file mode 100644 index 000000000..f2959c1d4 --- /dev/null +++ b/Documentation/REST-API/Resources-MaxScale.md @@ -0,0 +1,216 @@ +# MaxScale Resource + +The MaxScale resource represents a MaxScale instance and it is the core on top +of which the modules build upon. + +## Resource Operations + +## Get global information + +Retrieve global information about a MaxScale instance. This includes various +file locations, configuration options and version information. + +``` +GET /maxscale +``` + +#### Response + +``` +Status: 200 OK + +{ + "config": "/etc/maxscale.cnf", + "cachedir": "/var/cache/maxscale/", + "datadir": "/var/lib/maxscale/" + "libdir": "/usr/lib64/maxscale/", + "piddir": "/var/run/maxscale/", + "execdir": "/usr/bin/", + "languagedir": "/var/lib/maxscale/", + "user": "maxscale", + "threads": 4, + "version": "2.1.0", + "commit": "12e7f17eb361e353f7ac413b8b4274badb41b559" + "started": "Wed, 31 Aug 2016 23:29:26 +0300" +} +``` + +#### Supported Request Parameter + +- `fields` + +## Get thread information + +Get detailed information and statistics about the threads. + +``` +GET /maxscale/threads +``` + +#### Response + +``` +Status: 200 OK + +{ + "load_average": { + "historic": 1.05, + "current": 1.00, + "1min": 0.00, + "5min": 0.00, + "15min": 0.00 + }, + "threads": [ + { + "id": 0, + "state": "processing", + "file_descriptors": 1, + "event": [ + "in", + "out" + ], + "run_time": 300 + }, + { + "id": 1, + "state": "polling", + "file_descriptors": 0, + "event": [], + "run_time": 0 + } + ] +} +``` + +#### Supported Request Parameter + +- `fields` + +## Get logging information + +Get information about the current state of logging, enabled log files and the +location where the log files are stored. + +``` +GET /maxscale/logs +``` + +#### Response + +``` +Status: 200 OK + +{ + "logdir": "/var/log/maxscale/", + "maxlog": true, + "syslog": false, + "log_levels": { + "error": true, + "warning": true, + "notice": true, + "info": false, + "debug": false + }, + "log_augmentation": { + "function": true + }, + "log_throttling": { + "limit": 8, + "window": 2000, + "suppression": 10000 + }, + "last_flushed": "Wed, 31 Aug 2016 23:29:26 +0300" +} +``` + +#### Supported Request Parameter + +- `fields` + +## Flush and rotate log files + +Flushes any pending messages to disk and reopens the log files. The body of the +message is ignored. + +``` +POST /maxscale/logs/flush +``` + +#### Response + +``` +Status: 204 No Content +``` + +## Get task schedule + +Retrieve all pending tasks that are queued for execution. + +``` +GET /maxscale/tasks +``` + +#### Response + +``` +Status: 200 OK + +[ + { + "name": "Load Average", + "type": "repeated", + "frequency": 10, + "next_due": "Fri Sep 9 14:12:37 2016" + } +} +``` + +#### Supported Request Parameter + +- `fields` + +## Get loaded modules + +Retrieve information about all loaded modules. This includes version, API and +maturity information. + +``` +GET /maxscale/modules +``` + +#### Response + +``` +Status: 200 OK + +[ + { + "name": "MySQLBackend", + "type": "Protocol", + "version": "V2.0.0", + "api_version": "1.1.0", + "maturity": "GA" + }, + { + "name": "qlafilter", + "type": "Filter", + "version": "V1.1.1", + "api_version": "1.1.0", + "maturity": "GA" + }, + { + "name": "readwritesplit", + "type": "Router", + "version": "V1.1.0", + "api_version": "1.0.0", + "maturity": "GA" + } +} +``` + +#### Supported Request Parameter + +- `fields` +- `range` + +TODO: Add epoll statistics and rest of the supported methods. diff --git a/Documentation/REST-API/Resources-Monitor.md b/Documentation/REST-API/Resources-Monitor.md new file mode 100644 index 000000000..043af6545 --- /dev/null +++ b/Documentation/REST-API/Resources-Monitor.md @@ -0,0 +1,176 @@ +# Monitor Resource + +A monitor resource represents a monitor inside MaxScale that monitors one or +more servers. + +## Resource Operations + +### Get a monitor + +Get a single monitor. The _:name_ in the URI must be a valid monitor name with +all whitespace replaced with hyphens. The monitor names are case-insensitive. + +``` +GET /monitors/:name +``` + +#### Response + +``` +Status: 200 OK + +{ + "name": "MySQL Monitor", + "module": "mysqlmon", + "state": "started", + "monitor_interval": 2500, + "connect_timeout": 5, + "read_timeout": 2, + "write_timeout": 3, + "servers": [ + "/servers/db-serv-1", + "/servers/db-serv-2", + "/servers/db-serv-3" + ] +} +``` + +#### Supported Request Parameter + +- `fields` + +### Get all monitors + +Get all monitors. + +``` +GET /monitors +``` + +#### Response + +``` +Status: 200 OK + +[ + { + "name": "MySQL Monitor", + "module": "mysqlmon", + "state": "started", + "monitor_interval": 2500, + "connect_timeout": 5, + "read_timeout": 2, + "write_timeout": 3, + "servers": [ + "/servers/db-serv-1", + "/servers/db-serv-2", + "/servers/db-serv-3" + ] + }, + { + "name": "Galera Monitor", + "module": "galeramon", + "state": "started", + "monitor_interval": 5000, + "connect_timeout": 10, + "read_timeout": 5, + "write_timeout": 5, + "servers": [ + "/servers/db-galera-1", + "/servers/db-galera-2", + "/servers/db-galera-3" + ] + } +] +``` + +#### Supported Request Parameter + +- `fields` +- `range` + +### Stop a monitor + +Stops a started monitor. + +``` +PUT /monitor/:name/stop +``` + +#### Response + +``` +Status: 204 No Content +``` + +### Start a monitor + +Starts a stopped monitor. + +``` +PUT /monitor/:name/start +``` + +#### Response + +``` +Status: 204 No Content +``` + +### Update a monitor + +**Note**: The update mechanisms described here are provisional and most likely + will change in the future. This description is only for design purposes and + does not yet work. + +Partially update a monitor. The _:name_ in the URI must map to a monitor name +and the request body must be a valid JSON Patch document which is applied to the +resource. + +``` +PATCH /monitor/:name +``` + +### Modifiable Fields + +The following values can be modified with the PATCH method. + +|Field |Type |Description | +|-----------------|------------|---------------------------------------------------| +|servers |string array|Servers monitored by this monitor | +|monitor_interval |number |Monitoring interval in milliseconds | +|connect_timeout |number |Connection timeout in seconds | +|read_timeout |number |Read timeout in seconds | +|write_timeout |number |Write timeout in seconds | + +``` +[ + { "op": "remove", "path": "/servers/0" }, + { "op": "replace", "path": "/monitor_interval", "value": 2000 }, + { "op": "replace", "path": "/connect_timeout", "value": 2 }, + { "op": "replace", "path": "/read_timeout", "value": 2 }, + { "op": "replace", "path": "/write_timeout", "value": 2 } +] +``` + +#### Response + +Response contains the modified resource. + +``` +Status: 200 OK + +{ + "name": "MySQL Monitor", + "module": "mysqlmon", + "servers": [ + "/servers/db-serv-2", + "/servers/db-serv-3" + ], + "state": "started", + "monitor_interval": 2000, + "connect_timeout": 2, + "read_timeout": 2, + "write_timeout": 2 +} +``` diff --git a/Documentation/REST-API/Resources-Server.md b/Documentation/REST-API/Resources-Server.md new file mode 100644 index 000000000..a73b0d531 --- /dev/null +++ b/Documentation/REST-API/Resources-Server.md @@ -0,0 +1,207 @@ +# Server Resource + +A server resource represents a backend database server. + +## Resource Operations + +### Get a server + +Get a single server. The _:name_ in the URI must be a valid server name with all +whitespace replaced with hyphens. The server names are case-insensitive. + +``` +GET /servers/:name +``` + +#### Response + +``` +Status: 200 OK + +{ + "name": "db-serv-1", + "address": "192.168.121.58", + "port": 3306, + "protocol": "MySQLBackend", + "status": [ + "master", + "running" + ], + "parameters": { + "report_weight": 10, + "app_weight": 2 + } +} +``` + +**Note**: The _parameters_ field contains all custom parameters for + servers, including the server weighting parameters. + +#### Supported Request Parameter + +- `fields` + +### Get all servers + +``` +GET /servers +``` + +#### Response + +``` +Status: 200 OK + +[ + { + "name": "db-serv-1", + "address": "192.168.121.58", + "port": 3306, + "protocol": "MySQLBackend", + "status": [ + "master", + "running" + ], + "parameters": { + "report_weight": 10, + "app_weight": 2 + } + }, + { + "name": "db-serv-2", + "address": "192.168.121.175", + "port": 3306, + "status": [ + "slave", + "running" + ], + "protocol": "MySQLBackend", + "parameters": { + "app_weight": 6 + } + } +] +``` + +#### Supported Request Parameter + +- `fields` +- `range` + +### Update a server + +**Note**: The update mechanisms described here are provisional and most likely + will change in the future. This description is only for design purposes and + does not yet work. + +Partially update a server. The _:name_ in the URI must map to a server name with +all whitespace replaced with hyphens and the request body must be a valid JSON +Patch document which is applied to the resource. + +``` +PATCH /servers/:name +``` + +### Modifiable Fields + +|Field |Type |Description | +|-----------|------------|-----------------------------------------------------------------------------| +|address |string |Server address | +|port |number |Server port | +|parameters |object |Server extra parameters | +|state |string array|Server state, array of `master`, `slave`, `synced`, `running` or `maintenance`. An empty array is interpreted as a server that is down.| + +``` +{ + { "op": "replace", "path": "/address", "value": "192.168.0.100" }, + { "op": "replace", "path": "/port", "value": 4006 }, + { "op": "add", "path": "/state/0", "value": "maintenance" }, + { "op": "replace", "path": "/parameters/report_weight", "value": 1 } +} +``` + +#### Response + +Response contains the modified resource. + +``` +Status: 200 OK + +{ + "name": "db-serv-1", + "protocol": "MySQLBackend", + "address": "192.168.0.100", + "port": 4006, + "state": [ + "maintenance", + "running" + ], + "parameters": { + "report_weight": 1, + "app_weight": 2 + } +} +``` + +### Get all connections to a server + +Get all connections that are connected to a server. + +``` +GET /servers/:name/connections +``` + +#### Response + +``` +Status: 200 OK + +[ + { + "state": "DCB in the polling loop", + "role": "Backend Request Handler", + "server": "/servers/db-serv-01", + "service": "/services/my-service", + "statistics": { + "reads": 2197 + "writes": 1562 + "buffered_writes": 0 + "high_water_events": 0 + "low_water_events": 0 + } + }, + { + "state": "DCB in the polling loop", + "role": "Backend Request Handler", + "server": "/servers/db-serv-01", + "service": "/services/my-second-service" + "statistics": { + "reads": 0 + "writes": 0 + "buffered_writes": 0 + "high_water_events": 0 + "low_water_events": 0 + } + } +] +``` + +#### Supported Request Parameter + +- `fields` +- `range` + +### Close all connections to a server + +Close all connections to a particular server. This will forcefully close all +backend connections. + +``` +DELETE /servers/:name/connections +``` + +#### Response + +``` +Status: 204 No Content +``` diff --git a/Documentation/REST-API/Resources-Service.md b/Documentation/REST-API/Resources-Service.md new file mode 100644 index 000000000..11c15d4e4 --- /dev/null +++ b/Documentation/REST-API/Resources-Service.md @@ -0,0 +1,272 @@ +# Service Resource + +A service resource represents a service inside MaxScale. A service is a +collection of network listeners, filters, a router and a set of backend servers. + +## Resource Operations + +### Get a service + +Get a single service. The _:name_ in the URI must be a valid service name with +all whitespace replaced with hyphens. The service names are case-insensitive. + +``` +GET /services/:name +``` + +#### Response + +``` +Status: 200 OK + +{ + "name": "My Service", + "router": "readwritesplit", + "router_options": { + "disable_sescmd_history": "true" + }, + "state": "started", + "total_connections": 10, + "current_connections": 2, + "started": "2016-08-29T12:52:31+03:00", + "filters": [ + "/filters/Query-Logging-Filter" + ], + "servers": [ + "/servers/db-serv-1", + "/servers/db-serv-2", + "/servers/db-serv-3" + ] +} +``` + +#### Supported Request Parameter + +- `fields` + +### Get all services + +Get all services. + +``` +GET /services +``` + +#### Response + +``` +Status: 200 OK + +[ + { + "name": "My Service", + "router": "readwritesplit", + "router_options": { + "disable_sescmd_history": "true" + }, + "state": "started", + "total_connections": 10, + "current_connections": 2, + "started": "2016-08-29T12:52:31+03:00", + "filters": [ + "/filters/Query-Logging-Filter" + ], + "servers": [ + "/servers/db-serv-1", + "/servers/db-serv-2", + "/servers/db-serv-3" + ] + }, + { + "name": "My Second Service", + "router": "readconnroute", + "router_options": { + "type": "master" + }, + "state": "started", + "total_connections": 10, + "current_connections": 2, + "started": "2016-08-29T12:52:31+03:00", + "servers": [ + "/servers/db-serv-1", + "/servers/db-serv-2" + ] + } +] +``` + +#### Supported Request Parameter + +- `fields` +- `range` + +### Get service listeners + +Get the listeners of a service. The _:name_ in the URI must be a valid service +name with all whitespace replaced with hyphens. The service names are +case-insensitive. + +``` +GET /services/:name/listeners +``` + +#### Response + +``` +Status: 200 OK + +[ + { + "name": "My Listener", + "protocol": "MySQLClient", + "address": "0.0.0.0", + "port": 4006 + }, + { + "name": "My SSL Listener", + "protocol": "MySQLClient", + "address": "127.0.0.1", + "port": 4006, + "ssl": "required", + "ssl_cert": "/home/markusjm/newcerts/server-cert.pem", + "ssl_key": "/home/markusjm/newcerts/server-key.pem", + "ssl_ca_cert": "/home/markusjm/newcerts/ca.pem" + } +] +``` + +#### Supported Request Parameter + +- `fields` +- `range` + +### Update a service + +**Note**: The update mechanisms described here are provisional and most likely + will change in the future. This description is only for design purposes and + does not yet work. + +Partially update a service. The _:name_ in the URI must map to a service name +and the request body must be a valid JSON Patch document which is applied to the +resource. + +``` +PATCH /services/:name +``` + +### Modifiable Fields + +|Field |Type |Description | +|--------------|------------|---------------------------------------------------| +|servers |string array|Servers used by this service, must be relative links to existing server resources| +|router_options|object |Router specific options| +|filters |string array|Service filters, configured in the same order they are declared in the array (`filters[0]` => first filter, `filters[1]` => second filter)| +|user |string |The username for the service user| +|password |string |The password for the service user| +|root_user |boolean |Allow root user to connect via this service| +|version_string|string |Custom version string given to connecting clients| +|weightby |string |Name of a server weigting parameter which is used for connection weighting| +|connection_timeout|number |Client idle timeout in seconds| +|max_connection|number |Maximum number of allowed connections| +|strip_db_esc|boolean |Strip escape characters from default database name| + +``` +[ + { "op": "replace", "path": "/servers", "value": ["/servers/db-serv-2","/servers/db-serv-3"] }, + { "op": "add", "path": "/router_options/master_failover_mode", "value": "fail_on_write" }, + { "op": "remove", "path": "/filters" } +] +``` + +#### Response + +Response contains the modified resource. + +``` +Status: 200 OK + + { + "name": "My Service", + "router": "readwritesplit", + "router_options": { + "disable_sescmd_history=false", + "master_failover_mode": "fail_on_write" + } + "state": "started", + "total_connections": 10, + "current_connections": 2, + "started": "2016-08-29T12:52:31+03:00", + "servers": [ + "/servers/db-serv-2", + "/servers/db-serv-3" + ] + } +``` + +### Stop a service + +Stops a started service. + +``` +PUT /service/:name/stop +``` + +#### Response + +``` +Status: 204 No Content +``` + +### Start a service + +Starts a stopped service. + +``` +PUT /service/:name/start +``` + +#### Response + +``` +Status: 204 No Content +``` + +### Get all sessions for a service + +Get all sessions for a particular service. + +``` +GET /services/:name/sessions +``` + +#### Response + +Relative links to all sessions for this service. + +``` +Status: 200 OK + +[ + "/sessions/1", + "/sessions/2" +] +``` + +#### Supported Request Parameter + +- `range` + +### Close all sessions for a service + +Close all sessions for a particular service. This will forcefully close all +client connections and any backend connections they have made. + +``` +DELETE /services/:name/sessions +``` + +#### Response + +``` +Status: 204 No Content +``` diff --git a/Documentation/REST-API/Resources-Session.md b/Documentation/REST-API/Resources-Session.md new file mode 100644 index 000000000..03f90d0d7 --- /dev/null +++ b/Documentation/REST-API/Resources-Session.md @@ -0,0 +1,138 @@ +# Session Resource + +A session consists of a client connection, any number of related backend +connections, a router module session and possibly filter module sessions. Each +session is created on a service and a service can have multiple sessions. + +## Resource Operations + +### Get a session + +Get a single session. _:id_ must be a valid session ID. + +``` +GET /sessions/:id +``` + +#### Response + +``` +Status: 200 OK + +{ + "id": 1, + "state": "Session ready for routing", + "user": "jdoe", + "address": "192.168.0.200", + "service": "/services/my-service", + "connected": "Wed Aug 31 03:03:12 2016", + "idle": 260 +} +``` + +#### Supported Request Parameter + +- `fields` + +### Get all sessions + +Get all sessions. + +``` +GET /sessions +``` + +#### Response + +``` +Status: 200 OK + +[ + { + "id": 1, + "state": "Session ready for routing", + "user": "jdoe", + "address": "192.168.0.200", + "service": "/services/My-Service", + "connected": "Wed Aug 31 03:03:12 2016", + "idle": 260 + }, + { + "id": 2, + "state": "Session ready for routing", + "user": "dba", + "address": "192.168.0.201", + "service": "/services/My-Service", + "connected": "Wed Aug 31 03:10:00 2016", + "idle": 1 + } +] +``` + +#### Supported Request Parameter + +- `fields` +- `range` + +### Get all connections created by a session + +Get all backend connections created by a session. _:id_ must be a valid session ID. + +``` +GET /sessions/:id/connections +``` + +#### Response + +``` +Status: 200 OK + +[ + { + "state": "DCB in the polling loop", + "role": "Backend Request Handler", + "server": "/servers/db-serv-01", + "service": "/services/my-service", + "statistics": { + "reads": 2197 + "writes": 1562 + "buffered_writes": 0 + "high_water_events": 0 + "low_water_events": 0 + } + }, + { + "state": "DCB in the polling loop", + "role": "Backend Request Handler", + "server": "/servers/db-serv-02", + "service": "/services/my-service", + "statistics": { + "reads": 0 + "writes": 0 + "buffered_writes": 0 + "high_water_events": 0 + "low_water_events": 0 + } + } +] +``` + +#### Supported Request Parameter + +- `fields` +- `range` + +### Close a session + +Close a session. This will forcefully close the client connection and any +backend connections. + +``` +DELETE /sessions/:id +``` + +#### Response + +``` +Status: 204 No Content +``` diff --git a/Documentation/REST-API/Resources-User.md b/Documentation/REST-API/Resources-User.md new file mode 100644 index 000000000..84c8fc9f8 --- /dev/null +++ b/Documentation/REST-API/Resources-User.md @@ -0,0 +1,81 @@ +# Admin User Resource + +Admin users represent administrative users that are able to query and change +MaxScale's configuration. + +## Resource Operations + +### Get all users + +Get all administrative users. + +``` +GET /users +``` + +#### Response + +``` +Status: 200 OK + +[ + { + "name": "jdoe" + }, + { + "name": "dba" + }, + { + "name": "admin" + } +] + +#### Supported Request Parameter + +- `fields` +- `range` + +### Create a user + +Create a new administrative user. + +``` +PUT /users +``` + +### Modifiable Fields + +All of the following fields need to be defined in the request body. + +|Field |Type |Description | +|---------|------|-------------------------| +|name |string|Username, consisting of alphanumeric characters| +|password |string|Password for the new user| + +``` +{ + "name": "foo", + "password": "bar" +} +``` + +#### Response + +``` +Status: 204 No Content +``` + +### Delete a user + +Delete a user. The _:name_ part of the URI must be a valid user name. The user +names are case-insensitive. + +``` +DELETE /users/:name +``` + +#### Response + +``` +Status: 204 No Content +``` From 417d6270d500ea6ccfe79e12f828fde90b4a842d Mon Sep 17 00:00:00 2001 From: ekorh475 Date: Mon, 7 Nov 2016 15:43:39 +0200 Subject: [PATCH 154/215] Fix assignment of session_id Previously the session_id incrementation was done after creating filters, giving the filters a constant zero value for session_id. Now the incrementation happens before filter creation. --- server/core/session.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/core/session.c b/server/core/session.c index 8d62f23b8..ac29c884a 100644 --- a/server/core/session.c +++ b/server/core/session.c @@ -124,6 +124,8 @@ session_alloc(SERVICE *service, DCB *client_dcb) MXS_OOM(); return NULL; } + /** Assign a session id and increase */ + session->ses_id = (size_t)atomic_add(&session_id, 1) + 1; session->ses_is_child = (bool) DCB_IS_CLONE(client_dcb); session->service = service; session->client_dcb = client_dcb; @@ -221,8 +223,6 @@ session_alloc(SERVICE *service, DCB *client_dcb) session->client_dcb->user, session->client_dcb->remote); } - /** Assign a session id and increase, insert session into list */ - session->ses_id = (size_t)atomic_add(&session_id, 1) + 1; atomic_add(&service->stats.n_sessions, 1); atomic_add(&service->stats.n_current, 1); CHK_SESSION(session); From c36e5f6ba4b610a409b546398fd2ade95650591c Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Mon, 7 Nov 2016 16:19:38 +0100 Subject: [PATCH 155/215] C_DEBUG removed and added a check for packetlen < MAXROWS_EOF_PACKET_LEN C_DEBUG removed. A check is now made for packetlen < MAXROWS_EOF_PACKET_LEN. An empty result set, OK, is send in such situation --- server/modules/filter/maxrows/maxrows.c | 62 +++++++++++++++++++------ server/modules/filter/maxrows/maxrows.h | 11 ++--- 2 files changed, 53 insertions(+), 20 deletions(-) diff --git a/server/modules/filter/maxrows/maxrows.c b/server/modules/filter/maxrows/maxrows.c index 3c81eea11..21ecda165 100644 --- a/server/modules/filter/maxrows/maxrows.c +++ b/server/modules/filter/maxrows/maxrows.c @@ -57,8 +57,6 @@ static int clientReply(FILTER *instance, void *sdata, GWBUF *queue); static void diagnostics(FILTER *instance, void *sdata, DCB *dcb); static uint64_t getCapabilities(void); -#define C_DEBUG(format, ...) MXS_LOG_MESSAGE(LOG_NOTICE, format, ##__VA_ARGS__) - /* Global symbols of the Module */ MODULE_INFO info = @@ -310,7 +308,11 @@ static int routeQuery(FILTER *instance, void *sdata, GWBUF *packet) } } - C_DEBUG("Maxrows filter is sending data."); + if (csdata->instance->config.debug & MAXROWS_DEBUG_DECISIONS) + { + MXS_NOTICE("Maxrows filter is sending data."); + } + return csdata->down.routeQuery(csdata->down.instance, csdata->down.session, packet); } @@ -343,10 +345,13 @@ static int clientReply(FILTER *instance, void *sdata, GWBUF *data) { if (gwbuf_length(csdata->res.data) > csdata->instance->config.max_resultset_size) { - C_DEBUG("Current size %uB of resultset, at least as much " - "as maximum allowed size %uKiB. Not returning data.", - gwbuf_length(csdata->res.data), - csdata->instance->config.max_resultset_size / 1024); + if (csdata->instance->config.debug & MAXROWS_DEBUG_DISCARDING) + { + MXS_NOTICE("Current size %uB of resultset, at least as much " + "as maximum allowed size %uKiB. Not returning data.", + gwbuf_length(csdata->res.data), + csdata->instance->config.max_resultset_size / 1024); + } csdata->state = MAXROWS_DISCARDING_RESPONSE; } @@ -585,19 +590,28 @@ static int handle_expecting_response(MAXROWS_SESSION_DATA *csdata) { case 0x00: // OK case 0xff: // ERR - C_DEBUG("OK or ERR"); + if (csdata->instance->config.debug & MAXROWS_DEBUG_DECISIONS) + { + MXS_NOTICE("OK or ERR"); + } rv = send_upstream(csdata); csdata->state = MAXROWS_IGNORING_RESPONSE; break; case 0xfb: // GET_MORE_CLIENT_DATA/SEND_MORE_CLIENT_DATA - C_DEBUG("GET_MORE_CLIENT_DATA"); + if (csdata->instance->config.debug & MAXROWS_DEBUG_DECISIONS) + { + MXS_NOTICE("GET_MORE_CLIENT_DATA"); + } rv = send_upstream(csdata); csdata->state = MAXROWS_IGNORING_RESPONSE; break; default: - C_DEBUG("RESULTSET"); + if (csdata->instance->config.debug & MAXROWS_DEBUG_DECISIONS) + { + MXS_NOTICE("RESULTSET"); + } if (csdata->res.n_totalfields != 0) { @@ -669,9 +683,13 @@ static int handle_rows(MAXROWS_SESSION_DATA *csdata) case 0xff: // ERR packet after the rows. csdata->res.offset += packetlen; ss_dassert(csdata->res.offset == buflen); + + if (csdata->instance->config.debug & MAXROWS_DEBUG_DECISIONS) + { + MXS_NOTICE("Error packet seen while handling result set"); + } /* * This is the ERR packet that could terminate a Multi-Resultset. - * Reply to client is the same as in case 0x0 */ if (csdata->state == MAXROWS_DISCARDING_RESPONSE) { @@ -702,11 +720,23 @@ static int handle_rows(MAXROWS_SESSION_DATA *csdata) * If so more results set could come. The end of stream * will be an OK packet. */ + if (packetlen < MAXROWS_EOF_PACKET_LEN) + { + MXS_ERROR("EOF packet has size of %lu instead of %d", packetlen, MAXROWS_EOF_PACKET_LEN); + rv = send_ok_upstream(csdata); + csdata->state = MAXROWS_EXPECTING_NOTHING; + break; + } int flags = gw_mysql_get_byte2(header + MAXROWS_MYSQL_EOF_PACKET_FLAGS_OFFSET); if (!(flags & SERVER_MORE_RESULTS_EXIST)) { + if (csdata->instance->config.debug & MAXROWS_DEBUG_DECISIONS) + { + MXS_NOTICE("OK or EOF packet seen terminating the resultset"); + } + if (csdata->state == MAXROWS_DISCARDING_RESPONSE) { rv = send_ok_upstream(csdata); @@ -720,7 +750,10 @@ static int handle_rows(MAXROWS_SESSION_DATA *csdata) } else { - C_DEBUG("EOF or OK seen with SERVER_MORE_RESULTS_EXIST flag: waiting for more data"); + if (csdata->instance->config.debug & MAXROWS_DEBUG_DECISIONS) + { + MXS_NOTICE("EOF or OK packet seen with SERVER_MORE_RESULTS_EXIST flag: waiting for more data"); + } } break; @@ -734,7 +767,10 @@ static int handle_rows(MAXROWS_SESSION_DATA *csdata) { if (csdata->res.n_rows > csdata->instance->config.max_resultset_rows) { - C_DEBUG("max_resultset_rows %lu reached, not returning the result.", csdata->res.n_rows); + if (csdata->instance->config.debug & MAXROWS_DEBUG_DISCARDING) + { + MXS_INFO("max_resultset_rows %lu reached, not returning the result.", csdata->res.n_rows); + } csdata->state = MAXROWS_DISCARDING_RESPONSE; } diff --git a/server/modules/filter/maxrows/maxrows.h b/server/modules/filter/maxrows/maxrows.h index 7b94f9b9e..aa15f84a5 100644 --- a/server/modules/filter/maxrows/maxrows.h +++ b/server/modules/filter/maxrows/maxrows.h @@ -26,15 +26,12 @@ MXS_BEGIN_DECLS #define MAXROWS_MYSQL_EOF_PACKET_FLAGS_OFFSET (MYSQL_HEADER_LEN + 1 + 2) #define MAXROWS_DEBUG_NONE 0 -#define MAXROWS_DEBUG_MATCHING 1 -#define MAXROWS_DEBUG_NON_MATCHING 2 -#define MAXROWS_DEBUG_USE 4 -#define MAXROWS_DEBUG_NON_USE 8 +#define MAXROWS_DEBUG_DISCARDING 1 +#define MAXROWS_DEBUG_DECISIONS 2 -#define MAXROWS_DEBUG_RULES (MAXROWS_DEBUG_MATCHING | MAXROWS_DEBUG_NON_MATCHING) -#define MAXROWS_DEBUG_USAGE (MAXROWS_DEBUG_USE | MAXROWS_DEBUG_NON_USE) +#define MAXROWS_DEBUG_USAGE (MAXROWS_DEBUG_DECISIONS | MAXROWS_DEBUG_DISCARDING) #define MAXROWS_DEBUG_MIN MAXROWS_DEBUG_NONE -#define MAXROWS_DEBUG_MAX (MAXROWS_DEBUG_RULES | MAXROWS_DEBUG_USAGE) +#define MAXROWS_DEBUG_MAX MAXROWS_DEBUG_USAGE // Count #define MAXROWS_DEFAULT_MAX_RESULTSET_ROWS UINT_MAX From 7358d932e6451769dcddf118c567cfccf56bd64e Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 8 Nov 2016 09:12:47 +0200 Subject: [PATCH 156/215] Fix maxrows CMakeLists.txt There was an add_subdirectory call to a nonexistent `test` directory. --- server/modules/filter/maxrows/CMakeLists.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/server/modules/filter/maxrows/CMakeLists.txt b/server/modules/filter/maxrows/CMakeLists.txt index 8ad47a642..7fc89a4f1 100644 --- a/server/modules/filter/maxrows/CMakeLists.txt +++ b/server/modules/filter/maxrows/CMakeLists.txt @@ -3,7 +3,3 @@ target_link_libraries(maxrows maxscale-common) set_target_properties(maxrows PROPERTIES VERSION "1.0.0") set_target_properties(maxrows PROPERTIES LINK_FLAGS -Wl,-z,defs) install_module(maxrows experimental) - -if(BUILD_TESTS) - add_subdirectory(test) -endif() From 689366b6b790fc0e21358efa95672a7ee26b12ba Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 8 Nov 2016 09:24:00 +0200 Subject: [PATCH 157/215] Fix possible writes to closed DCBs When persistent connections were used, it was possible that the injection of COM_CHANGE_USER statements caused a crash when a DCB in the wrong state was accessed. For MySQL protocol modules, the `data` member of the client DCB points to the shared session data, a MYSQL_session struct, but for sessions in the persistent pool, it points to NULL. The boolean, `was_persistent`, tells whether a DCB was just taken from the pool or it has been in use. The `was_persistent` status wasn't properly reset for connections that were put into the pool which caused a COM_CHANGE_USER statement to be injected for stale connections in the pool which caused a crash when the NULL `data` member was accessed. --- server/core/dcb.c | 1 + server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/server/core/dcb.c b/server/core/dcb.c index 939fe6a2b..2cc65eeb3 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -1827,6 +1827,7 @@ dcb_maybe_add_persistent(DCB *dcb) MXS_DEBUG("%lu [dcb_maybe_add_persistent] Adding DCB to persistent pool, user %s.\n", pthread_self(), dcb->user); + dcb->was_persistent = false; dcb->dcb_is_zombie = false; dcb->persistentstart = time(NULL); if (dcb->session) diff --git a/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c b/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c index 4253b1c45..fe48791b6 100644 --- a/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c +++ b/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c @@ -947,8 +947,9 @@ static int gw_MySQLWrite_backend(DCB *dcb, GWBUF *queue) CHK_DCB(dcb); spinlock_acquire(&dcb->authlock); - if (dcb->was_persistent) + if (dcb->was_persistent && dcb->state == DCB_STATE_POLLING) { + ss_dassert(dcb->persistentstart == 0); /** * This is a DCB that was just taken out of the persistent connection pool. * We need to sent a COM_CHANGE_USER query to the backend to reset the From de18f25cb882e22f61eb11a8773badebb1dc84df Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Tue, 8 Nov 2016 11:12:37 +0100 Subject: [PATCH 158/215] MXS-936: add shutdown hooks First part: add shutdown hooks for routers. Binlog is the first one with a destroyInstance() routine --- include/maxscale/router.h | 2 + server/core/service.c | 6 ++ server/modules/routing/avro/avro.c | 3 +- server/modules/routing/binlog/blr.c | 67 ++++++++++++++++++- server/modules/routing/cli/cli.c | 3 +- server/modules/routing/debugcli/debugcli.c | 3 +- server/modules/routing/maxinfo/maxinfo.c | 3 +- .../routing/readconnroute/readconnroute.c | 3 +- .../routing/readwritesplit/readwritesplit.c | 3 +- .../routing/schemarouter/schemarouter.c | 3 +- server/modules/routing/testroute/testroute.c | 3 +- server/modules/routing/webserver/webserver.c | 3 +- 12 files changed, 92 insertions(+), 10 deletions(-) diff --git a/include/maxscale/router.h b/include/maxscale/router.h index 90e8083d0..85625ae39 100644 --- a/include/maxscale/router.h +++ b/include/maxscale/router.h @@ -24,6 +24,7 @@ * 16/07/2013 Massimiliano Pinto Added router commands values * 22/10/2013 Massimiliano Pinto Added router errorReply entry point * 27/10/2015 Martin Brampton Add RCAP_TYPE_NO_RSESSION + * 08/11/2016 Massimiliano Pinto Add destroyInstance() entry point * */ @@ -82,6 +83,7 @@ typedef struct router_object error_action_t action, bool* succp); uint64_t (*getCapabilities)(void); + void (*destroyInstance)(SERVICE *service); } ROUTER_OBJECT; /** diff --git a/server/core/service.c b/server/core/service.c index f8f2a77a7..a8050ff91 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -34,6 +34,7 @@ * 03/03/15 Massimiliano Pinto Added config_enable_feedback_task() call in serviceStartAll * 19/06/15 Martin Brampton More meaningful names for temp variables * 31/05/16 Martin Brampton Implement connection throttling + * 08/11/16 Massimiliano Pinto Added: service_shutdown() calls destroyInstance() hoosk for routers * * @endverbatim */ @@ -1827,6 +1828,11 @@ void service_shutdown() while (svc != NULL) { svc->svc_do_shutdown = true; + /* Call destroyInstance hook for routers */ + if (svc->router->destroyInstance) + { + svc->router->destroyInstance(svc); + } svc = svc->next; } spinlock_release(&service_spin); diff --git a/server/modules/routing/avro/avro.c b/server/modules/routing/avro/avro.c index 54f256d3c..85c63ebfb 100644 --- a/server/modules/routing/avro/avro.c +++ b/server/modules/routing/avro/avro.c @@ -105,7 +105,8 @@ static ROUTER_OBJECT MyObject = diagnostics, clientReply, errorReply, - getCapabilities + getCapabilities, + NULL }; static SPINLOCK instlock; diff --git a/server/modules/routing/binlog/blr.c b/server/modules/routing/binlog/blr.c index eec12b98f..2cb4b0a71 100644 --- a/server/modules/routing/binlog/blr.c +++ b/server/modules/routing/binlog/blr.c @@ -50,6 +50,7 @@ * for connection error and authentication failure. * 11/07/2016 Massimiliano Pinto Added SSL backend support * 22/07/2016 Massimiliano Pinto Added semi_sync replication support + * 08/11/2016 Massimiliano Pinto Added destroyInstance() * * @endverbatim */ @@ -108,6 +109,7 @@ static int blr_check_binlog(ROUTER_INSTANCE *router); int blr_read_events_all_events(ROUTER_INSTANCE *router, int fix, int debug); void blr_master_close(ROUTER_INSTANCE *); void blr_free_ssl_data(ROUTER_INSTANCE *inst); +static void destroyInstance(SERVICE *service); /** The module object definition */ static ROUTER_OBJECT MyObject = @@ -120,7 +122,8 @@ static ROUTER_OBJECT MyObject = diagnostics, clientReply, errorReply, - getCapabilities + getCapabilities, + destroyInstance }; static void stats_func(void *); @@ -2291,3 +2294,65 @@ blr_free_ssl_data(ROUTER_INSTANCE *inst) inst->service->dbref->server->server_ssl = NULL; } } + +/** + * destroy binlog server instance + * + * @param service The service this router instance belongs to + */ +static void +destroyInstance(SERVICE *service) +{ + ROUTER_INSTANCE *inst = (ROUTER_INSTANCE *) service->router_instance; + + MXS_DEBUG("Destroying instance of router %s for service %s", + service->routerModule, service->name); + + /* Check whether master connection is active */ + if (inst->master) + { + if (inst->master->fd != -1 && inst->master->state == DCB_STATE_POLLING) + { + blr_master_close(inst); + } + } + + spinlock_acquire(&inst->lock); + + if (inst->master_state != BLRM_UNCONFIGURED) + { + inst->master_state = BLRM_SLAVE_STOPPED; + } + + if (inst->client) + { + if (inst->client->state == DCB_STATE_POLLING) + { + dcb_close(inst->client); + inst->client = NULL; + } + } + + /* Discard the queued residual data */ + while (inst->residual) + { + inst->residual = gwbuf_consume(inst->residual, GWBUF_LENGTH(inst->residual)); + } + inst->residual = NULL; + + MXS_NOTICE("%s is being stopped by MaxScale shudown. Disconnecting from master %s:%d, " + "read up to log %s, pos %lu, transaction safe pos %lu", + service->name, + service->dbref->server->name, + service->dbref->server->port, + inst->binlog_name, inst->current_pos, inst->binlog_position); + + if (inst->trx_safe && inst->pending_transaction) + { + MXS_WARNING("%s stopped by shutdown: detected mid-transaction in binlog file %s, " + "pos %lu, incomplete transaction starts at pos %lu", + service->name, inst->binlog_name, inst->current_pos, inst->binlog_position); + } + + spinlock_release(&inst->lock); +} diff --git a/server/modules/routing/cli/cli.c b/server/modules/routing/cli/cli.c index ee263009e..478a2dcdf 100644 --- a/server/modules/routing/cli/cli.c +++ b/server/modules/routing/cli/cli.c @@ -71,7 +71,8 @@ static ROUTER_OBJECT MyObject = diagnostics, NULL, NULL, - getCapabilities + getCapabilities, + NULL }; extern int execute_cmd(CLI_SESSION *cli); diff --git a/server/modules/routing/debugcli/debugcli.c b/server/modules/routing/debugcli/debugcli.c index 962fbe1b4..1f86c82fc 100644 --- a/server/modules/routing/debugcli/debugcli.c +++ b/server/modules/routing/debugcli/debugcli.c @@ -70,7 +70,8 @@ static ROUTER_OBJECT MyObject = diagnostics, NULL, NULL, - getCapabilities + getCapabilities, + NULL }; extern int execute_cmd(CLI_SESSION *cli); diff --git a/server/modules/routing/maxinfo/maxinfo.c b/server/modules/routing/maxinfo/maxinfo.c index 135de44f0..05eaa8d4d 100644 --- a/server/modules/routing/maxinfo/maxinfo.c +++ b/server/modules/routing/maxinfo/maxinfo.c @@ -96,7 +96,8 @@ static ROUTER_OBJECT MyObject = diagnostics, NULL, handleError, - getCapabilities + getCapabilities, + NULL }; static SPINLOCK instlock; diff --git a/server/modules/routing/readconnroute/readconnroute.c b/server/modules/routing/readconnroute/readconnroute.c index a77dad041..e6d2ecfc3 100644 --- a/server/modules/routing/readconnroute/readconnroute.c +++ b/server/modules/routing/readconnroute/readconnroute.c @@ -125,7 +125,8 @@ static ROUTER_OBJECT MyObject = diagnostics, clientReply, handleError, - getCapabilities + getCapabilities, + NULL }; static bool rses_begin_locked_router_action(ROUTER_CLIENT_SES* rses); diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index 357ab9610..a7f6532e8 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -101,7 +101,8 @@ static ROUTER_OBJECT MyObject = diagnostics, clientReply, handleError, - getCapabilities + getCapabilities, + NULL }; /* diff --git a/server/modules/routing/schemarouter/schemarouter.c b/server/modules/routing/schemarouter/schemarouter.c index ce6bcecec..1a7f12bda 100644 --- a/server/modules/routing/schemarouter/schemarouter.c +++ b/server/modules/routing/schemarouter/schemarouter.c @@ -113,7 +113,8 @@ static ROUTER_OBJECT MyObject = diagnostic, clientReply, handleError, - getCapabilities + getCapabilities, + NULL }; static bool rses_begin_locked_router_action(ROUTER_CLIENT_SES* rses); diff --git a/server/modules/routing/testroute/testroute.c b/server/modules/routing/testroute/testroute.c index 502e7cfab..7449822ce 100644 --- a/server/modules/routing/testroute/testroute.c +++ b/server/modules/routing/testroute/testroute.c @@ -50,7 +50,8 @@ static ROUTER_OBJECT MyObject = diagnostic, clientReply, handleError, - getCapabilities + getCapabilities, + NULL }; typedef struct diff --git a/server/modules/routing/webserver/webserver.c b/server/modules/routing/webserver/webserver.c index c9889ba77..f6546abb1 100644 --- a/server/modules/routing/webserver/webserver.c +++ b/server/modules/routing/webserver/webserver.c @@ -64,7 +64,8 @@ static ROUTER_OBJECT MyObject = diagnostic, NULL, NULL, - getCapabilities + getCapabilities, + NULL }; From 61f0603e39bfd137832a91ce5ec97434b461f325 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Tue, 8 Nov 2016 14:07:48 +0100 Subject: [PATCH 159/215] MXS-936: destroyInstance interface change MXS-936: destroyInstance interface change --- include/maxscale/router.h | 2 +- server/core/service.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/maxscale/router.h b/include/maxscale/router.h index 85625ae39..1a403d312 100644 --- a/include/maxscale/router.h +++ b/include/maxscale/router.h @@ -83,7 +83,7 @@ typedef struct router_object error_action_t action, bool* succp); uint64_t (*getCapabilities)(void); - void (*destroyInstance)(SERVICE *service); + void (*destroyInstance)(ROUTER *instance); } ROUTER_OBJECT; /** diff --git a/server/core/service.c b/server/core/service.c index a8050ff91..785d03f40 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -1831,7 +1831,7 @@ void service_shutdown() /* Call destroyInstance hook for routers */ if (svc->router->destroyInstance) { - svc->router->destroyInstance(svc); + svc->router->destroyInstance(svc->router_instance); } svc = svc->next; } From ef79686a3a29a62b005b4500a6e6a1ae51026941 Mon Sep 17 00:00:00 2001 From: ekorh475 Date: Mon, 7 Nov 2016 17:24:50 +0200 Subject: [PATCH 160/215] Add option to use a unified log file for qlafilter The qlafilter now has an option to log all messages to a single file instead of session-specific files. Session ids are printed to the beginning of the line when using this mode. Documentation updated to match. Also, added an option to flush after every write. --- Documentation/Filters/Query-Log-All-Filter.md | 6 +- server/modules/filter/qlafilter/qlafilter.c | 108 +++++++++++++++--- 2 files changed, 97 insertions(+), 17 deletions(-) diff --git a/Documentation/Filters/Query-Log-All-Filter.md b/Documentation/Filters/Query-Log-All-Filter.md index a52f7a399..db987c84a 100644 --- a/Documentation/Filters/Query-Log-All-Filter.md +++ b/Documentation/Filters/Query-Log-All-Filter.md @@ -30,8 +30,10 @@ The QLA filter accepts the following options. |ignorecase|Use case-insensitive matching | |case |Use case-sensitive matching | |extended |Use extended regular expression syntax (ERE)| - -To use multiple filter options, list them in a comma-separated list. +|session_file| Use session-specific file (default)| +|unified_file| Use one file for all sessions| +|flush_writes| Flush after every write| +To use multiple filter options, list them in a comma-separated list. If no file settings are given, default will be used. Multiple file settings can be enabled simultaneously. ``` options=case,extended diff --git a/server/modules/filter/qlafilter/qlafilter.c b/server/modules/filter/qlafilter/qlafilter.c index 420e63328..511542a25 100644 --- a/server/modules/filter/qlafilter/qlafilter.c +++ b/server/modules/filter/qlafilter/qlafilter.c @@ -62,6 +62,10 @@ static char *version_str = "V1.1.1"; /** Formatting buffer size */ #define QLA_STRING_BUFFER_SIZE 1024 +/** Log file settings flags */ +#define CONFIG_FILE_SESSION (1 << 0) // Default value, session specific files +#define CONFIG_FILE_UNIFIED (1 << 1) // One file shared by all sessions + /* * The filter entry points */ @@ -107,6 +111,10 @@ typedef struct regex_t re; /* Compiled regex text */ char *nomatch; /* Optional text to match against for exclusion */ regex_t nore; /* Compiled regex nomatch text */ + uint32_t log_mode_flags; /* Log file mode settings */ + FILE *unified_fp; /* Unified log file. The pointer needs to be shared here + * to avoid garbled printing. */ + bool flush_writes; /* Flush log file after every write */ } QLA_INSTANCE; /** @@ -125,6 +133,7 @@ typedef struct int active; char *user; char *remote; + size_t ses_id; /* The session this filter serves */ } QLA_SESSION; /** @@ -186,6 +195,9 @@ createInstance(const char *name, char **options, FILTER_PARAMETER **params) my_instance->match = NULL; my_instance->nomatch = NULL; my_instance->filebase = NULL; + my_instance->log_mode_flags = 0; + my_instance->unified_fp = NULL; + my_instance->flush_writes = false; bool error = false; if (params) @@ -239,6 +251,18 @@ createInstance(const char *name, char **options, FILTER_PARAMETER **params) { cflags |= REG_EXTENDED; } + else if (!strcasecmp(options[i], "session_file")) + { + my_instance->log_mode_flags |= CONFIG_FILE_SESSION; + } + else if (!strcasecmp(options[i], "unified_file")) + { + my_instance->log_mode_flags |= CONFIG_FILE_UNIFIED; + } + else if (!strcasecmp(options[i], "flush_writes")) + { + my_instance->flush_writes = true; + } else { MXS_ERROR("qlafilter: Unsupported option '%s'.", @@ -247,7 +271,11 @@ createInstance(const char *name, char **options, FILTER_PARAMETER **params) } } } - + if (my_instance->log_mode_flags == 0) + { + // If nothing has been set, set a default value + my_instance->log_mode_flags = CONFIG_FILE_SESSION; + } if (my_instance->filebase == NULL) { MXS_ERROR("qlafilter: No 'filebase' parameter defined."); @@ -275,6 +303,35 @@ createInstance(const char *name, char **options, FILTER_PARAMETER **params) my_instance->nomatch = NULL; error = true; } + // Try to open the unified log file + if (my_instance->log_mode_flags & CONFIG_FILE_UNIFIED && + my_instance->filebase != NULL) + { + // First calculate filename length + const char UNIFIED[] = ".unified"; + int namelen = strlen(my_instance->filebase) + sizeof(UNIFIED); + char *filename = NULL; + if ((filename = MXS_CALLOC(namelen, sizeof(char))) != NULL) + { + snprintf(filename, namelen, "%s.unified", my_instance->filebase); + // Open the file. It is only closed at program exit + my_instance->unified_fp = fopen(filename, "w"); + if (my_instance->unified_fp == NULL) + { + char errbuf[MXS_STRERROR_BUFLEN]; + MXS_ERROR("Opening output file for qla " + "filter failed due to %d, %s", + errno, + strerror_r(errno, errbuf, sizeof(errbuf))); + error = true; + } + MXS_FREE(filename); + } + else + { + error = true; + } + } if (error) { @@ -289,6 +346,10 @@ createInstance(const char *name, char **options, FILTER_PARAMETER **params) MXS_FREE(my_instance->nomatch); regfree(&my_instance->nore); } + if (my_instance->unified_fp != NULL) + { + fclose(my_instance->unified_fp); + } MXS_FREE(my_instance->filebase); MXS_FREE(my_instance->source); MXS_FREE(my_instance->userName); @@ -338,15 +399,17 @@ newSession(FILTER *instance, SESSION *session) my_session->user = userName; my_session->remote = remote; + my_session->ses_id = session->ses_id; - sprintf(my_session->filename, "%s.%d", + sprintf(my_session->filename, "%s.%lu", my_instance->filebase, - my_instance->sessions); + my_session->ses_id); // Fixed possible race condition // Multiple sessions can try to update my_instance->sessions simultaneously atomic_add(&(my_instance->sessions), 1); - if (my_session->active) + // Only open the session file if the corresponding mode setting is used + if (my_session->active && (my_instance->log_mode_flags | CONFIG_FILE_SESSION)) { my_session->fp = fopen(my_session->filename, "w"); @@ -354,7 +417,7 @@ newSession(FILTER *instance, SESSION *session) { char errbuf[MXS_STRERROR_BUFLEN]; MXS_ERROR("Opening output file for qla " - "fileter failed due to %d, %s", + "filter failed due to %d, %s", errno, strerror_r(errno, errbuf, sizeof(errbuf))); MXS_FREE(my_session->filename); @@ -363,14 +426,6 @@ newSession(FILTER *instance, SESSION *session) } } } - else - { - char errbuf[MXS_STRERROR_BUFLEN]; - MXS_ERROR("Memory allocation for qla filter failed due to " - "%d, %s.", - errno, - strerror_r(errno, errbuf, sizeof(errbuf))); - } return my_session; } @@ -458,8 +513,31 @@ routeQuery(FILTER *instance, void *session, GWBUF *queue) gettimeofday(&tv, NULL); localtime_r(&tv.tv_sec, &t); strftime(buffer, sizeof(buffer), "%F %T", &t); - fprintf(my_session->fp, "%s,%s@%s,%s\n", buffer, my_session->user, - my_session->remote, trim(squeeze_whitespace(ptr))); + + /** + * Loop over all the possible log file modes and write to + * the enabled files. + */ + char *sql_string = trim(squeeze_whitespace(ptr)); + if (my_instance->log_mode_flags & CONFIG_FILE_SESSION) + { + fprintf(my_session->fp, "%s,%s@%s,%s\n", buffer, my_session->user, + my_session->remote, sql_string); + if (my_instance->flush_writes) + { + fflush(my_session->fp); + } + } + if (my_instance->log_mode_flags & CONFIG_FILE_UNIFIED) + { + fprintf(my_instance->unified_fp, "S%zd,%s,%s@%s,%s\n", + my_session->ses_id, buffer, my_session->user, + my_session->remote, sql_string); + if (my_instance->flush_writes) + { + fflush(my_instance->unified_fp); + } + } } MXS_FREE(ptr); } From 4e36e868205fe02598229142cfbaf4bf21b46f10 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Tue, 8 Nov 2016 17:09:49 +0100 Subject: [PATCH 161/215] MXS-936: destroyInstance added for filters MXS-936: destroyInstance added for filters --- include/maxscale/filter.h | 1 + server/core/service.c | 12 ++++++++++ server/modules/filter/cache/cache.c | 3 ++- server/modules/filter/ccrfilter/ccrfilter.c | 3 ++- server/modules/filter/dbfwfilter/dbfwfilter.c | 5 ++-- server/modules/filter/gatekeeper/gatekeeper.c | 3 ++- server/modules/filter/hintfilter/hintfilter.c | 3 ++- server/modules/filter/luafilter/luafilter.c | 1 + server/modules/filter/maxrows/maxrows.c | 4 ++-- server/modules/filter/mqfilter/mqfilter.c | 1 + .../namedserverfilter/namedserverfilter.c | 3 ++- server/modules/filter/qlafilter/qlafilter.c | 1 + .../modules/filter/regexfilter/regexfilter.c | 3 ++- server/modules/filter/tee/tee.c | 1 + server/modules/filter/testfilter/testfilter.c | 24 +++++++++++++++---- server/modules/filter/topfilter/topfilter.c | 1 + server/modules/filter/tpmfilter/tpmfilter.c | 1 + 17 files changed, 56 insertions(+), 14 deletions(-) diff --git a/include/maxscale/filter.h b/include/maxscale/filter.h index 1eabbe326..27a1ef8f6 100644 --- a/include/maxscale/filter.h +++ b/include/maxscale/filter.h @@ -83,6 +83,7 @@ typedef struct filter_object int (*clientReply)(FILTER *instance, void *fsession, GWBUF *queue); void (*diagnostics)(FILTER *instance, void *fsession, DCB *dcb); uint64_t (*getCapabilities)(void); + void (*destroyInstance)(FILTER *instance); } FILTER_OBJECT; /** diff --git a/server/core/service.c b/server/core/service.c index 785d03f40..6f2de2811 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -1833,6 +1833,18 @@ void service_shutdown() { svc->router->destroyInstance(svc->router_instance); } + if (svc->n_filters) + { + FILTER_DEF **filters = svc->filters; + for (int i=0; i < svc->n_filters; i++) + { + if (filters[i]->obj->destroyInstance) + { + /* Call destroyInstance hook for filters */ + filters[i]->obj->destroyInstance(filters[i]->filter); + } + } + } svc = svc->next; } spinlock_release(&service_spin); diff --git a/server/modules/filter/cache/cache.c b/server/modules/filter/cache/cache.c index 8ddb27e8a..b7519fc3f 100644 --- a/server/modules/filter/cache/cache.c +++ b/server/modules/filter/cache/cache.c @@ -80,7 +80,8 @@ FILTER_OBJECT *GetModuleObject() routeQuery, clientReply, diagnostics, - getCapabilities + getCapabilities, + NULL, // destroyInstance }; return &object; diff --git a/server/modules/filter/ccrfilter/ccrfilter.c b/server/modules/filter/ccrfilter/ccrfilter.c index 3abda9137..157c750ed 100644 --- a/server/modules/filter/ccrfilter/ccrfilter.c +++ b/server/modules/filter/ccrfilter/ccrfilter.c @@ -75,9 +75,10 @@ static FILTER_OBJECT MyObject = setDownstream, NULL, // No Upstream requirement routeQuery, - NULL, + NULL, // No clientReply diagnostic, getCapabilities, + NULL, // No destroyInstance }; #define CCR_DEFAULT_TIME 60 diff --git a/server/modules/filter/dbfwfilter/dbfwfilter.c b/server/modules/filter/dbfwfilter/dbfwfilter.c index e2ebd86ff..b292286ff 100644 --- a/server/modules/filter/dbfwfilter/dbfwfilter.c +++ b/server/modules/filter/dbfwfilter/dbfwfilter.c @@ -114,11 +114,12 @@ static FILTER_OBJECT MyObject = closeSession, freeSession, setDownstream, - NULL, + NULL, // No setUpStream routeQuery, - NULL, + NULL, // No clientReply diagnostic, getCapabilities, + NULL, // No destroyInstance }; /** diff --git a/server/modules/filter/gatekeeper/gatekeeper.c b/server/modules/filter/gatekeeper/gatekeeper.c index 58e1d0fd2..57bb97848 100644 --- a/server/modules/filter/gatekeeper/gatekeeper.c +++ b/server/modules/filter/gatekeeper/gatekeeper.c @@ -103,9 +103,10 @@ static FILTER_OBJECT MyObject = setDownstream, NULL, // No upstream requirement routeQuery, - NULL, + NULL, // No clientReply diagnostic, getCapabilities, + NULL, // No destroyInstance }; /** diff --git a/server/modules/filter/hintfilter/hintfilter.c b/server/modules/filter/hintfilter/hintfilter.c index b6a0e7dd3..fe35db089 100644 --- a/server/modules/filter/hintfilter/hintfilter.c +++ b/server/modules/filter/hintfilter/hintfilter.c @@ -52,9 +52,10 @@ static FILTER_OBJECT MyObject = setDownstream, NULL, // No upstream requirement routeQuery, - NULL, + NULL, // No clientReply diagnostic, getCapabilities, + NULL, // No destroyInstance }; /** diff --git a/server/modules/filter/luafilter/luafilter.c b/server/modules/filter/luafilter/luafilter.c index dbfe5b2ae..f65542977 100644 --- a/server/modules/filter/luafilter/luafilter.c +++ b/server/modules/filter/luafilter/luafilter.c @@ -96,6 +96,7 @@ static FILTER_OBJECT MyObject = clientReply, diagnostic, getCapabilities, + NULL, // No destroyInstance }; /** diff --git a/server/modules/filter/maxrows/maxrows.c b/server/modules/filter/maxrows/maxrows.c index 21ecda165..5c56b0c90 100644 --- a/server/modules/filter/maxrows/maxrows.c +++ b/server/modules/filter/maxrows/maxrows.c @@ -98,7 +98,8 @@ FILTER_OBJECT *GetModuleObject() routeQuery, clientReply, diagnostics, - getCapabilities + getCapabilities, + NULL, // No destroyInstance }; return &object; @@ -917,4 +918,3 @@ static int send_ok_upstream(MAXROWS_SESSION_DATA *csdata) return rv; } - diff --git a/server/modules/filter/mqfilter/mqfilter.c b/server/modules/filter/mqfilter/mqfilter.c index 263626c1b..f9b29d4d0 100644 --- a/server/modules/filter/mqfilter/mqfilter.c +++ b/server/modules/filter/mqfilter/mqfilter.c @@ -116,6 +116,7 @@ static FILTER_OBJECT MyObject = clientReply, diagnostic, getCapabilities, + NULL, // No destroyInstance }; /** diff --git a/server/modules/filter/namedserverfilter/namedserverfilter.c b/server/modules/filter/namedserverfilter/namedserverfilter.c index 788a560d7..e31ae7632 100644 --- a/server/modules/filter/namedserverfilter/namedserverfilter.c +++ b/server/modules/filter/namedserverfilter/namedserverfilter.c @@ -69,9 +69,10 @@ static FILTER_OBJECT MyObject = setDownstream, NULL, // No Upstream requirement routeQuery, - NULL, + NULL, // No clientReply diagnostic, getCapabilities, + NULL, // No destroyInstance }; /** diff --git a/server/modules/filter/qlafilter/qlafilter.c b/server/modules/filter/qlafilter/qlafilter.c index 420e63328..b05d6149f 100644 --- a/server/modules/filter/qlafilter/qlafilter.c +++ b/server/modules/filter/qlafilter/qlafilter.c @@ -87,6 +87,7 @@ static FILTER_OBJECT MyObject = NULL, // No client reply diagnostic, getCapabilities, + NULL, // No destroyInstance }; /** diff --git a/server/modules/filter/regexfilter/regexfilter.c b/server/modules/filter/regexfilter/regexfilter.c index 21100fa55..8382ca497 100644 --- a/server/modules/filter/regexfilter/regexfilter.c +++ b/server/modules/filter/regexfilter/regexfilter.c @@ -71,9 +71,10 @@ static FILTER_OBJECT MyObject = setDownstream, NULL, // No Upstream requirement routeQuery, - NULL, + NULL, // No clientReply diagnostic, getCapabilities, + NULL, // No destroyInstance }; /** diff --git a/server/modules/filter/tee/tee.c b/server/modules/filter/tee/tee.c index c3dd9db19..d979f2e47 100644 --- a/server/modules/filter/tee/tee.c +++ b/server/modules/filter/tee/tee.c @@ -130,6 +130,7 @@ static FILTER_OBJECT MyObject = clientReply, diagnostic, getCapabilities, + NULL, // No destroyInstance }; /** diff --git a/server/modules/filter/testfilter/testfilter.c b/server/modules/filter/testfilter/testfilter.c index 5c3cb7739..fb9a451cb 100644 --- a/server/modules/filter/testfilter/testfilter.c +++ b/server/modules/filter/testfilter/testfilter.c @@ -37,7 +37,7 @@ MODULE_INFO info = "A simple query counting filter" }; -static char *version_str = "V1.0.0"; +static char *version_str = "V2.0.0"; static FILTER *createInstance(const char *name, char **options, FILTER_PARAMETER **params); static void *newSession(FILTER *instance, SESSION *session); @@ -47,6 +47,7 @@ static void setDownstream(FILTER *instance, void *fsession, DOWNSTREAM *down static int routeQuery(FILTER *instance, void *fsession, GWBUF *queue); static void diagnostic(FILTER *instance, void *fsession, DCB *dcb); static uint64_t getCapabilities(void); +static void destroyInstance(FILTER *instance); static FILTER_OBJECT MyObject = @@ -56,11 +57,12 @@ static FILTER_OBJECT MyObject = closeSession, freeSession, setDownstream, - NULL, // No upstream requirement + NULL, // No upstream requirement routeQuery, - NULL, + NULL, // No clientReply diagnostic, getCapabilities, + destroyInstance, }; /** @@ -68,7 +70,8 @@ static FILTER_OBJECT MyObject = */ typedef struct { - int sessions; + const char *name; + int sessions; } TEST_INSTANCE; /** @@ -135,6 +138,7 @@ createInstance(const char *name, char **options, FILTER_PARAMETER **params) if ((my_instance = MXS_CALLOC(1, sizeof(TEST_INSTANCE))) != NULL) { my_instance->sessions = 0; + my_instance->name = name; } return (FILTER *)my_instance; } @@ -258,3 +262,15 @@ static uint64_t getCapabilities(void) { return RCAP_TYPE_NONE; } + +/** + * destroyInstance routine. + * + * @param The filter instance. + */ +static void destroyInstance(FILTER *instance) +{ + TEST_INSTANCE *cinstance = (TEST_INSTANCE *)instance; + + MXS_INFO("Destroying filter %s", cinstance->name); +} diff --git a/server/modules/filter/topfilter/topfilter.c b/server/modules/filter/topfilter/topfilter.c index 9b5d38e4f..a423d2a19 100644 --- a/server/modules/filter/topfilter/topfilter.c +++ b/server/modules/filter/topfilter/topfilter.c @@ -82,6 +82,7 @@ static FILTER_OBJECT MyObject = clientReply, diagnostic, getCapabilities, + NULL, // No destroyInstance }; /** diff --git a/server/modules/filter/tpmfilter/tpmfilter.c b/server/modules/filter/tpmfilter/tpmfilter.c index da2c42811..0dd59002f 100644 --- a/server/modules/filter/tpmfilter/tpmfilter.c +++ b/server/modules/filter/tpmfilter/tpmfilter.c @@ -95,6 +95,7 @@ static FILTER_OBJECT MyObject = clientReply, diagnostic, getCapabilities, + NULL, // No destroyInstance }; /** From 02ec7e9b1750db8f69e1c1d8a3e09a20e9883412 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Tue, 8 Nov 2016 21:09:17 +0200 Subject: [PATCH 162/215] Wait until housekeeper has finished When calling hkshutdown(), only a shutdown flag will be set. In main() we will then actually wait for the housekeeper thread to finish before exiting. --- include/maxscale/housekeeper.h | 27 +++++++++++++++-- include/maxscale/semaphore.h | 22 ++++++++++++++ server/core/gateway.c | 13 +++++++- server/core/housekeeper.c | 54 +++++++++++++++++++++------------- 4 files changed, 92 insertions(+), 24 deletions(-) create mode 100644 include/maxscale/semaphore.h diff --git a/include/maxscale/housekeeper.h b/include/maxscale/housekeeper.h index 3a7922ceb..c25d85df0 100644 --- a/include/maxscale/housekeeper.h +++ b/include/maxscale/housekeeper.h @@ -51,11 +51,34 @@ typedef struct hktask struct hktask *next; /*< Next task in the list */ } HKTASK; -extern void hkinit(); +/** + * Initialises the housekeeper mechanism. + * + * A call to any of the other housekeeper functions can be made only if + * this function returns successfully. + * + * @return True if the housekeeper mechanism was initialized, false otherwise. + */ +extern bool hkinit(); + +/** + * Shuts down the housekeeper mechanism. + * + * Should be called @b only if @c hkinit() returned successfully. + * + * @see hkinit hkfinish + */ +extern void hkshutdown(); + +/** + * Waits for the housekeeper thread to finish. Should be called only after + * hkshutdown() has been called. + */ +extern void hkfinish(); + extern int hktask_add(const char *name, void (*task)(void *), void *data, int frequency); extern int hktask_oneshot(const char *name, void (*task)(void *), void *data, int when); extern int hktask_remove(const char *name); -extern void hkshutdown(); extern void hkshow_tasks(DCB *pdcb); MXS_END_DECLS diff --git a/include/maxscale/semaphore.h b/include/maxscale/semaphore.h new file mode 100644 index 000000000..a231bfd6f --- /dev/null +++ b/include/maxscale/semaphore.h @@ -0,0 +1,22 @@ +#pragma once +/* + * Copyright (c) 2016 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl. + * + * Change Date: 2019-07-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ + +/** + * @file semaphore.h Semaphores used by MaxScale. + */ + +// As a minimal preparation for other environments than Linux, components +// include , instead of including +// directly. +#include diff --git a/server/core/gateway.c b/server/core/gateway.c index 2099bdb7b..8b7b07d82 100644 --- a/server/core/gateway.c +++ b/server/core/gateway.c @@ -1936,7 +1936,13 @@ int main(int argc, char **argv) /* * Start the housekeeper thread */ - hkinit(); + if (!hkinit()) + { + char* logerr = "Failed to start housekeeper thread."; + print_log_n_stderr(true, true, logerr, logerr, 0); + rc = MAXSCALE_INTERNALERROR; + goto return_main; + } /*< * Start the polling threads, note this is one less than is @@ -1974,6 +1980,11 @@ int main(int argc, char **argv) */ poll_waitevents((void *)0); + /*< + * Wait for the housekeeper to finish. + */ + hkfinish(); + /*< * Wait server threads' completion. */ diff --git a/server/core/housekeeper.c b/server/core/housekeeper.c index 48096b2f3..0efc5cc14 100644 --- a/server/core/housekeeper.c +++ b/server/core/housekeeper.c @@ -10,13 +10,14 @@ * of this software will be governed by version 2 or later of the General * Public License. */ +#include #include #include #include -#include -#include +#include +#include #include -#include +#include /** * @file housekeeper.c Provide a mechanism to run periodic tasks @@ -49,22 +50,28 @@ static HKTASK *tasks = NULL; */ static SPINLOCK tasklock = SPINLOCK_INIT; -static int do_shutdown = 0; +static bool do_shutdown = 0; + long hkheartbeat = 0; /*< One heartbeat is 100 milliseconds */ static THREAD hk_thr_handle; static void hkthread(void *); -/** - * Initialise the housekeeper thread - */ -void +bool hkinit() { - if (thread_start(&hk_thr_handle, hkthread, NULL) == NULL) + bool inited = false; + + if (thread_start(&hk_thr_handle, hkthread, NULL) != NULL) { - MXS_ERROR("Failed to start housekeeper thread."); + inited = true; } + else + { + MXS_ALERT("Failed to start housekeeper thread."); + } + + return inited; } /** @@ -255,21 +262,17 @@ hkthread(void *data) void *taskdata; int i; - for (;;) + while (!do_shutdown) { for (i = 0; i < 10; i++) { - if (do_shutdown) - { - return; - } thread_millisleep(100); hkheartbeat++; } now = time(0); spinlock_acquire(&tasklock); ptr = tasks; - while (ptr) + while (!do_shutdown && ptr) { if (ptr->nextdue <= now) { @@ -297,16 +300,25 @@ hkthread(void *data) } spinlock_release(&tasklock); } + + MXS_NOTICE("Housekeeper shutting down."); } -/** - * Called to shutdown the housekeeper - * - */ void hkshutdown() { - do_shutdown = 1; + do_shutdown = true; + atomic_synchronize(); +} + +void hkfinish() +{ + ss_dassert(do_shutdown); + + MXS_NOTICE("Waiting for housekeeper to shut down."); + thread_wait(hk_thr_handle); + do_shutdown = false; + MXS_NOTICE("Housekeeper has shut down."); } /** From 802152d7d3f0495a2f9944ea90442ed2824ac852 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Wed, 9 Nov 2016 16:26:05 +0100 Subject: [PATCH 163/215] MXS-936: avro router converter routine handles shutdown MXS-936: avro router converter routine handles shutdown --- server/modules/routing/avro/avro.c | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/server/modules/routing/avro/avro.c b/server/modules/routing/avro/avro.c index 85c63ebfb..19643dd80 100644 --- a/server/modules/routing/avro/avro.c +++ b/server/modules/routing/avro/avro.c @@ -228,6 +228,13 @@ static void add_conversion_task(AVRO_INSTANCE *inst) { char tasknm[strlen(avro_task_name) + strlen(inst->service->name) + 2]; snprintf(tasknm, sizeof(tasknm), "%s-%s", inst->service->name, avro_task_name); + if (inst->service->svc_do_shutdown) + { + MXS_INFO("AVRO converter task is not added due to MaxScale shutdown"); + avro_close_binlog(inst->binlog_fd); + return; + } + MXS_INFO("Setting task for converter_func"); if (hktask_oneshot(tasknm, converter_func, inst, inst->task_delay) == 0) { MXS_ERROR("Failed to add binlog to Avro conversion task to housekeeper."); @@ -1007,13 +1014,19 @@ void converter_func(void* data) AVRO_INSTANCE* router = (AVRO_INSTANCE*) data; bool ok = true; avro_binlog_end_t binlog_end = AVRO_OK; + while (ok && binlog_end == AVRO_OK) { uint64_t start_pos = router->current_pos; + if (router->service->svc_do_shutdown) + { + MXS_INFO("AVRO converter task is exiting due to MaxScale shutdown"); + avro_close_binlog(router->binlog_fd); + break; + } + if (avro_open_binlog(router->binlogdir, router->binlog_name, &router->binlog_fd)) { - binlog_end = avro_read_all_events(router); - if (router->current_pos != start_pos) { /** We processed some data, reset the conversion task delay */ @@ -1028,6 +1041,13 @@ void converter_func(void* data) } } + if (router->service->svc_do_shutdown) + { + MXS_INFO("AVRO converter task is exiting due to MaxScale shutdown"); + avro_close_binlog(router->binlog_fd); + return; + } + /** We reached end of file, flush unwritten records to disk */ if (router->task_delay == 1) { From 62049a5bb48dcac7b966dffa3ab5bbd60f0de192 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Wed, 9 Nov 2016 17:26:28 +0100 Subject: [PATCH 164/215] MXS-936: AVRO converter_func update MXS-936: AVRO converter_func update: added new checks for pending shutdown --- server/modules/routing/avro/avro.c | 33 ++++++++++++++++-------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/server/modules/routing/avro/avro.c b/server/modules/routing/avro/avro.c index 19643dd80..663187e01 100644 --- a/server/modules/routing/avro/avro.c +++ b/server/modules/routing/avro/avro.c @@ -224,7 +224,7 @@ bool create_tables(sqlite3* handle) return true; } -static void add_conversion_task(AVRO_INSTANCE *inst) +static bool add_conversion_task(AVRO_INSTANCE *inst) { char tasknm[strlen(avro_task_name) + strlen(inst->service->name) + 2]; snprintf(tasknm, sizeof(tasknm), "%s-%s", inst->service->name, avro_task_name); @@ -232,13 +232,15 @@ static void add_conversion_task(AVRO_INSTANCE *inst) { MXS_INFO("AVRO converter task is not added due to MaxScale shutdown"); avro_close_binlog(inst->binlog_fd); - return; + return false; } MXS_INFO("Setting task for converter_func"); if (hktask_oneshot(tasknm, converter_func, inst, inst->task_delay) == 0) { MXS_ERROR("Failed to add binlog to Avro conversion task to housekeeper."); + return false; } + return true; } /** @@ -1020,13 +1022,14 @@ void converter_func(void* data) uint64_t start_pos = router->current_pos; if (router->service->svc_do_shutdown) { - MXS_INFO("AVRO converter task is exiting due to MaxScale shutdown"); - avro_close_binlog(router->binlog_fd); + MXS_INFO("AVRO converter task is not handling events due to MaxScale shutdown"); break; } if (avro_open_binlog(router->binlogdir, router->binlog_name, &router->binlog_fd)) { + binlog_end = avro_read_all_events(router); + if (router->current_pos != start_pos) { /** We processed some data, reset the conversion task delay */ @@ -1041,16 +1044,14 @@ void converter_func(void* data) } } - if (router->service->svc_do_shutdown) - { - MXS_INFO("AVRO converter task is exiting due to MaxScale shutdown"); - avro_close_binlog(router->binlog_fd); - return; - } - /** We reached end of file, flush unwritten records to disk */ if (router->task_delay == 1) { + if (router->service->svc_do_shutdown) + { + MXS_INFO("AVRO converter task is not indexing due to MaxScale shutdown"); + return; + } avro_flush_all_tables(router); avro_save_conversion_state(router); } @@ -1058,10 +1059,12 @@ void converter_func(void* data) if (binlog_end == AVRO_LAST_FILE) { router->task_delay = MXS_MIN(router->task_delay + 1, AVRO_TASK_DELAY_MAX); - add_conversion_task(router); - MXS_INFO("Stopped processing file %s at position %lu. Waiting until" - " more data is written before continuing. Next check in %d seconds.", - router->binlog_name, router->current_pos, router->task_delay); + if (add_conversion_task(router)) + { + MXS_INFO("Stopped processing file %s at position %lu. Waiting until" + " more data is written before continuing. Next check in %d seconds.", + router->binlog_name, router->current_pos, router->task_delay); + } } } From c28ffcf87e393cf4d2d001dd10989a6e4433de6f Mon Sep 17 00:00:00 2001 From: ekorh475 Date: Wed, 9 Nov 2016 12:55:35 +0200 Subject: [PATCH 165/215] Change error message when permissions on .secrets are wrong If the user running MaxScale could open the .secrets-file and the file permissions were anything other than owner:read, the secrets_readkeys() would fail with error message "Ignoring secrets file , invalid permissions." Now the message is more accurate in stating the expected permissions. --- server/core/secrets.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/core/secrets.c b/server/core/secrets.c index b197ff1e0..60cc244b5 100644 --- a/server/core/secrets.c +++ b/server/core/secrets.c @@ -175,8 +175,8 @@ secrets_readKeys(const char* path) if (secret_stats.st_mode != (S_IRUSR | S_IFREG)) { close(fd); - MXS_ERROR("Ignoring secrets file " - "%s, invalid permissions.", + MXS_ERROR("Ignoring secrets file %s, invalid permissions." + "The only permission on the file should be owner:read.", secret_file); return NULL; } From bc1c2e115275a865c0eea7b61ad5cc5c245ed2ed Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Wed, 9 Nov 2016 15:30:38 +0200 Subject: [PATCH 166/215] Ensure exit is possible If shutdown has been initiated (via maxadmin or by sending a SIGINT or SIGTERM) and a SIGTERM is received, the process is terminated. If shutdown has been initiated (via maxadmin or by sending a SIGINT or SIGTERM) and a SIGINT (Ctrl-C) is received, a warning is printed that shutdown is in progess. Then, if an additional SIGINT is received, the process is terminated. So, in practice: - If MaxScale is running as a daemon, the first SIGTERM initiates shutdown and a second one unconditionally terminates the process. - If MaxScale is running in the console, the first Ctrl-C initiates shutdown, the second prints a warning and the third terminates the process. This is to ensure that MaxScale can be forced to exit, in case some thread is hung for whatever reason, and is thus preventing the controlled shutdown. --- include/maxscale/maxscale.h | 10 ++++ server/core/gateway.c | 56 +++++++++++++++++----- server/modules/routing/debugcli/debugcmd.c | 7 ++- 3 files changed, 60 insertions(+), 13 deletions(-) diff --git a/include/maxscale/maxscale.h b/include/maxscale/maxscale.h index 93d43bd19..e6b1a0abf 100644 --- a/include/maxscale/maxscale.h +++ b/include/maxscale/maxscale.h @@ -44,4 +44,14 @@ void maxscale_reset_starttime(void); time_t maxscale_started(void); int maxscale_uptime(void); +/** + * Initiate shutdown of MaxScale. + * + * This functions informs all threads that they should stop the + * processing and exit. + * + * @return How many times maxscale_shutdown() has been called. + */ +int maxscale_shutdown(void); + MXS_END_DECLS diff --git a/server/core/gateway.c b/server/core/gateway.c index 8b7b07d82..138bc60bd 100644 --- a/server/core/gateway.c +++ b/server/core/gateway.c @@ -181,7 +181,6 @@ static int set_user(const char* user); bool pid_file_exists(); void write_child_exit_code(int fd, int code); static bool change_cwd(); -void shutdown_server(); static void log_exit_status(); static bool daemonize(); static bool sniff_configuration(const char* filepath); @@ -288,20 +287,45 @@ static void sigusr1_handler (int i) } static const char shutdown_msg[] = "\n\nShutting down MaxScale\n\n"; +static const char patience_msg[] = + "\n" + "Patience is a virtue...\n" + "Shutdown in progress, but one more Ctrl-C or SIGTERM and MaxScale goes down,\n" + "no questions asked.\n"; static void sigterm_handler(int i) { last_signal = i; - shutdown_server(); - write(STDERR_FILENO, shutdown_msg, sizeof(shutdown_msg) - 1); + int n_shutdowns = maxscale_shutdown(); + + if (n_shutdowns == 1) + { + write(STDERR_FILENO, shutdown_msg, sizeof(shutdown_msg) - 1); + } + else + { + exit(EXIT_FAILURE); + } } static void sigint_handler(int i) { last_signal = i; - shutdown_server(); - write(STDERR_FILENO, shutdown_msg, sizeof(shutdown_msg) - 1); + int n_shutdowns = maxscale_shutdown(); + + if (n_shutdowns == 1) + { + write(STDERR_FILENO, shutdown_msg, sizeof(shutdown_msg) - 1); + } + else if (n_shutdowns == 2) + { + write(STDERR_FILENO, patience_msg, sizeof(patience_msg) - 1); + } + else + { + exit(EXIT_FAILURE); + } } static void @@ -2041,14 +2065,22 @@ return_main: /*< * Shutdown MaxScale server */ -void -shutdown_server() +int maxscale_shutdown() { - service_shutdown(); - poll_shutdown(); - hkshutdown(); - memlog_flush_all(); - log_flush_shutdown(); + static int n_shutdowns = 0; + + int n = atomic_add(&n_shutdowns, 1); + + if (n == 0) + { + service_shutdown(); + poll_shutdown(); + hkshutdown(); + memlog_flush_all(); + log_flush_shutdown(); + } + + return n + 1; } static void log_flush_shutdown(void) diff --git a/server/modules/routing/debugcli/debugcmd.c b/server/modules/routing/debugcli/debugcmd.c index 950f0882a..3e828bc77 100644 --- a/server/modules/routing/debugcli/debugcmd.c +++ b/server/modules/routing/debugcli/debugcmd.c @@ -70,6 +70,7 @@ #include #include #include +#include #include #include @@ -276,7 +277,11 @@ struct subcommand listoptions[] = { {0, 0, 0} } }; -extern void shutdown_server(); +static void shutdown_server() +{ + maxscale_shutdown(); +} + static void shutdown_service(DCB *dcb, SERVICE *service); static void shutdown_monitor(DCB *dcb, MONITOR *monitor); From e9030f71bdc3a45789ded5880efd4f8b90377eb8 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Thu, 10 Nov 2016 10:16:39 +0100 Subject: [PATCH 167/215] MXS-936: AVRO shutdown update AVRO shutdown detection is only in the loop that processes new files. Flushing and status saving is done anyway --- server/modules/routing/avro/avro.c | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/server/modules/routing/avro/avro.c b/server/modules/routing/avro/avro.c index 663187e01..73a361525 100644 --- a/server/modules/routing/avro/avro.c +++ b/server/modules/routing/avro/avro.c @@ -231,7 +231,6 @@ static bool add_conversion_task(AVRO_INSTANCE *inst) if (inst->service->svc_do_shutdown) { MXS_INFO("AVRO converter task is not added due to MaxScale shutdown"); - avro_close_binlog(inst->binlog_fd); return false; } MXS_INFO("Setting task for converter_func"); @@ -1017,15 +1016,9 @@ void converter_func(void* data) bool ok = true; avro_binlog_end_t binlog_end = AVRO_OK; - while (ok && binlog_end == AVRO_OK) + while (!router->service->svc_do_shutdown && ok && binlog_end == AVRO_OK) { uint64_t start_pos = router->current_pos; - if (router->service->svc_do_shutdown) - { - MXS_INFO("AVRO converter task is not handling events due to MaxScale shutdown"); - break; - } - if (avro_open_binlog(router->binlogdir, router->binlog_name, &router->binlog_fd)) { binlog_end = avro_read_all_events(router); @@ -1047,11 +1040,6 @@ void converter_func(void* data) /** We reached end of file, flush unwritten records to disk */ if (router->task_delay == 1) { - if (router->service->svc_do_shutdown) - { - MXS_INFO("AVRO converter task is not indexing due to MaxScale shutdown"); - return; - } avro_flush_all_tables(router); avro_save_conversion_state(router); } From 6fe9fda46e9b28f98cd538d575837cdd886a5c3b Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Mon, 7 Nov 2016 15:33:30 +0200 Subject: [PATCH 168/215] MXS-935: Add support for column based rule matching In addition, test program extended to be able to test the rule matching. Tests to be extended in a subsequent commit. --- server/modules/filter/cache/rules.c | 114 ++++++++++++++- server/modules/filter/cache/test/testrules.c | 145 ++++++++++++++++--- 2 files changed, 240 insertions(+), 19 deletions(-) diff --git a/server/modules/filter/cache/rules.c b/server/modules/filter/cache/rules.c index cc2b3bdaa..7dd5bbdf0 100644 --- a/server/modules/filter/cache/rules.c +++ b/server/modules/filter/cache/rules.c @@ -782,9 +782,119 @@ static bool cache_rule_compare_n(CACHE_RULE *self, const char *value, size_t len static bool cache_rule_matches_column(CACHE_RULE *self, const char *default_db, const GWBUF *query) { ss_dassert(self->attribute == CACHE_ATTRIBUTE_COLUMN); - ss_info_dassert(!true, "Column matching not implemented yet."); - return false; + // TODO: Do this "parsing" when the rule item is created. + char buffer[strlen(self->value) + 1]; + strcpy(buffer, self->value); + + const char* rule_column = NULL; + const char* rule_table = NULL; + const char* rule_database = NULL; + char* dot1 = strchr(buffer, '.'); + char* dot2 = dot1 ? strchr(buffer, '.') : NULL; + + if (dot1 && dot2) + { + rule_database = buffer; + *dot1 = 0; + rule_table = dot1 + 1; + *dot2 = 0; + rule_column = dot2 + 1; + } + else if (dot1) + { + rule_table = buffer; + *dot1 = 0; + rule_column = dot1 + 1; + } + else + { + rule_column = buffer; + } + + const QC_FIELD_INFO *infos; + size_t n_infos; + + int n_tables; + char** tables = qc_get_table_names((GWBUF*)query, &n_tables, false); + + const char* default_table = NULL; + + if (n_tables == 1) + { + // Only if we have exactly one table can we assume anything + // about a table that has not been mentioned explicitly. + default_table = tables[0]; + } + + qc_get_field_info((GWBUF*)query, &infos, &n_infos); + + bool matches = false; + + size_t i = 0; + while (!matches && (i < n_infos)) + { + const QC_FIELD_INFO *info = (infos + i); + + if ((strcmp(info->column, rule_column) == 0) || (strcmp(info->column, "*") == 0)) + { + if (rule_table) + { + const char* check_table = info->table ? info->table : default_table; + + if (check_table && (strcmp(check_table, rule_table) == 0)) + { + if (rule_database) + { + const char *check_database = info->database ? info->database : default_db; + + if (check_database && (strcmp(check_database, rule_database) == 0)) + { + matches = true; + } + else + { + // If the rules specifies a database and either the database + // does not match or we do not know the database, the rule + // does *not* match. + matches = false; + } + } + else + { + // If the rule specifies no table, then if the table and column matches, + // the rule matches. + matches = true; + } + } + else + { + // The rules specifies a table and either the table does not match + // or we do not know the table, the rule does *not* match. + matches = false; + } + } + else + { + // If the rule specifies no table, then if the column matches, the + // rule matches. + matches = true; + } + } + + ++i; + } + + if (tables) + { + for (i = 0; i < (size_t)n_tables; ++i) + { + MXS_FREE(tables[i]); + } + MXS_FREE(tables); + } + + return matches; } /** diff --git a/server/modules/filter/cache/test/testrules.c b/server/modules/filter/cache/test/testrules.c index 3039e6605..153f342a5 100644 --- a/server/modules/filter/cache/test/testrules.c +++ b/server/modules/filter/cache/test/testrules.c @@ -14,12 +14,37 @@ #include #include "rules.h" #include +#include +#include #if !defined(SS_DEBUG) #define SS_DEBUG #endif #include -struct test_case +GWBUF* create_gwbuf(const char* s) +{ + size_t query_len = strlen(s); + size_t payload_len = query_len + 1; + size_t gwbuf_len = MYSQL_HEADER_LEN + payload_len; + + GWBUF* gwbuf = gwbuf_alloc(gwbuf_len); + ss_dassert(gwbuf); + + *((unsigned char*)((char*)GWBUF_DATA(gwbuf))) = payload_len; + *((unsigned char*)((char*)GWBUF_DATA(gwbuf) + 1)) = (payload_len >> 8); + *((unsigned char*)((char*)GWBUF_DATA(gwbuf) + 2)) = (payload_len >> 16); + *((unsigned char*)((char*)GWBUF_DATA(gwbuf) + 3)) = 0x00; + *((unsigned char*)((char*)GWBUF_DATA(gwbuf) + 4)) = 0x03; + memcpy((char*)GWBUF_DATA(gwbuf) + MYSQL_HEADER_LEN + 1, s, query_len); + + return gwbuf; +} + +// +// Test user rules. Basically tests that a user specification is translated +// into the correct pcre2 regex. +// +struct user_test_case { const char* json; struct @@ -29,31 +54,33 @@ struct test_case } expect; }; -#define TEST_CASE(op_from, from, op_to, to) \ +#define USER_TEST_CASE(op_from, from, op_to, to) \ { "{ \"use\": [ { \"attribute\": \"user\", \"op\": \"" #op_from "\", \"value\": \"" #from "\" } ] }",\ { op_to, #to } } -const struct test_case test_cases[] = +#define COLUMN_ + +const struct user_test_case user_test_cases[] = { - TEST_CASE(=, bob, CACHE_OP_LIKE, bob@.*), - TEST_CASE(=, 'bob', CACHE_OP_LIKE, bob@.*), - TEST_CASE(=, bob@%, CACHE_OP_LIKE, bob@.*), - TEST_CASE(=, 'bob'@'%.52', CACHE_OP_LIKE, bob@.*\\.52), - TEST_CASE(=, bob@127.0.0.1, CACHE_OP_EQ, bob@127.0.0.1), - TEST_CASE(=, b*b@127.0.0.1, CACHE_OP_EQ, b*b@127.0.0.1), - TEST_CASE(=, b*b@%.0.0.1, CACHE_OP_LIKE, b\\*b@.*\\.0\\.0\\.1), - TEST_CASE(=, b*b@%.0.%.1, CACHE_OP_LIKE, b\\*b@.*\\.0\\..*\\.1), + USER_TEST_CASE(=, bob, CACHE_OP_LIKE, bob@.*), + USER_TEST_CASE(=, 'bob', CACHE_OP_LIKE, bob@.*), + USER_TEST_CASE(=, bob@%, CACHE_OP_LIKE, bob@.*), + USER_TEST_CASE(=, 'bob'@'%.52', CACHE_OP_LIKE, bob@.*\\.52), + USER_TEST_CASE(=, bob@127.0.0.1, CACHE_OP_EQ, bob@127.0.0.1), + USER_TEST_CASE(=, b*b@127.0.0.1, CACHE_OP_EQ, b*b@127.0.0.1), + USER_TEST_CASE(=, b*b@%.0.0.1, CACHE_OP_LIKE, b\\*b@.*\\.0\\.0\\.1), + USER_TEST_CASE(=, b*b@%.0.%.1, CACHE_OP_LIKE, b\\*b@.*\\.0\\..*\\.1), }; -const size_t n_test_cases = sizeof(test_cases) / sizeof(test_cases[0]); +const size_t n_user_test_cases = sizeof(user_test_cases) / sizeof(user_test_cases[0]); -int test() +int test_user() { int errors = 0; - for (int i = 0; i < n_test_cases; ++i) + for (int i = 0; i < n_user_test_cases; ++i) { - const struct test_case *test_case = &test_cases[i]; + const struct user_test_case *test_case = &user_test_cases[i]; CACHE_RULES *rules = cache_rules_parse(test_case->json, 0); ss_dassert(rules); @@ -78,9 +105,86 @@ int test() rule->value); ++errors; } + + cache_rules_free(rules); } - return errors == 0 ? EXIT_SUCCESS : EXIT_FAILURE; + return errors; +} + +// +// +// +struct store_test_case +{ + const char *rule; // The rule in JSON format. + bool matches; // Whether or not the rule should match the query. + const char *default_db; // The current default db. + const char *query; // The query to be matched against the rule. +}; + +#define STORE_TEST_CASE(attribute, op, value, matches, default_db, query) \ +{ "{ \"store\": [ { \"attribute\": \"" attribute "\", \"op\": \"" op "\", \"value\": \"" value "\" } ] }",\ + matches, default_db, query } + +// In the following, +// true: The query SHOULD match the rule, +// false: The query should NOT match the rule. +const struct store_test_case store_test_cases[] = +{ + STORE_TEST_CASE("column", "=", "a", true, NULL, "SELECT a FROM tbl"), + STORE_TEST_CASE("column", "=", "b", false, NULL, "SELECT a FROM tbl") +}; + +const size_t n_store_test_cases = sizeof(store_test_cases) / sizeof(store_test_cases[0]); + +int test_store() +{ + int errors = 0; + + for (int i = 0; i < n_store_test_cases; ++i) + { + const struct store_test_case *test_case = &store_test_cases[i]; + + CACHE_RULES *rules = cache_rules_parse(test_case->rule, 0); + ss_dassert(rules); + + CACHE_RULE *rule = rules->store_rules; + ss_dassert(rule); + + GWBUF *packet = create_gwbuf(test_case->query); + + bool matches = cache_rules_should_store(rules, test_case->default_db, packet); + + if (matches != test_case->matches) + { + printf("Query : %s\n" + "Rule : %s\n" + "Expected: %s\n" + "Result : %s\n\n", + test_case->query, + test_case->rule, + test_case->matches ? "A match" : "Not a match", + matches ? "A match" : "Not a match"); + } + + gwbuf_free(packet); + + cache_rules_free(rules); + } + + return errors; +} + + +int test() +{ + int errors = 0; + + errors += test_user(); + errors += test_store(); + + return errors ? EXIT_FAILURE : EXIT_SUCCESS; } int main() @@ -89,7 +193,14 @@ int main() if (mxs_log_init(NULL, ".", MXS_LOG_TARGET_DEFAULT)) { - rc = test(); + if (qc_init("qc_sqlite", "")) + { + rc = test(); + } + else + { + MXS_ERROR("Could not initialize query classifier."); + } mxs_log_finish(); } From 77946b6adc8ddf5f751cba808a405c8bb302d151 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Thu, 10 Nov 2016 12:01:41 +0100 Subject: [PATCH 169/215] MXS-936: destroyInstance of binlog server is now updated MXS-936: destroyInstance of binlog server was not updated: not it takes the ROUTER * parameter. --- server/modules/routing/binlog/blr.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/server/modules/routing/binlog/blr.c b/server/modules/routing/binlog/blr.c index 2cb4b0a71..4561a97cb 100644 --- a/server/modules/routing/binlog/blr.c +++ b/server/modules/routing/binlog/blr.c @@ -109,7 +109,7 @@ static int blr_check_binlog(ROUTER_INSTANCE *router); int blr_read_events_all_events(ROUTER_INSTANCE *router, int fix, int debug); void blr_master_close(ROUTER_INSTANCE *); void blr_free_ssl_data(ROUTER_INSTANCE *inst); -static void destroyInstance(SERVICE *service); +static void destroyInstance(ROUTER *instance); /** The module object definition */ static ROUTER_OBJECT MyObject = @@ -2301,9 +2301,9 @@ blr_free_ssl_data(ROUTER_INSTANCE *inst) * @param service The service this router instance belongs to */ static void -destroyInstance(SERVICE *service) +destroyInstance(ROUTER *instance) { - ROUTER_INSTANCE *inst = (ROUTER_INSTANCE *) service->router_instance; + ROUTER_INSTANCE *inst = (ROUTER_INSTANCE *) instance; MXS_DEBUG("Destroying instance of router %s for service %s", service->routerModule, service->name); @@ -2340,18 +2340,18 @@ destroyInstance(SERVICE *service) } inst->residual = NULL; - MXS_NOTICE("%s is being stopped by MaxScale shudown. Disconnecting from master %s:%d, " + MXS_INFO("%s is being stopped by MaxScale shudown. Disconnecting from master %s:%d, " "read up to log %s, pos %lu, transaction safe pos %lu", - service->name, - service->dbref->server->name, - service->dbref->server->port, + inst->service->name, + inst->service->dbref->server->name, + inst->service->dbref->server->port, inst->binlog_name, inst->current_pos, inst->binlog_position); if (inst->trx_safe && inst->pending_transaction) { MXS_WARNING("%s stopped by shutdown: detected mid-transaction in binlog file %s, " "pos %lu, incomplete transaction starts at pos %lu", - service->name, inst->binlog_name, inst->current_pos, inst->binlog_position); + inst->service->name, inst->binlog_name, inst->current_pos, inst->binlog_position); } spinlock_release(&inst->lock); From 12dba14c5cfc8275dadd25a81a8a1674d84d486c Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Thu, 10 Nov 2016 13:22:26 +0100 Subject: [PATCH 170/215] MXS-936: fixed compilation error with "debug" build MXS-936: fixed compilation error with "debug" build --- server/modules/routing/binlog/blr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/modules/routing/binlog/blr.c b/server/modules/routing/binlog/blr.c index 4561a97cb..913e37a1e 100644 --- a/server/modules/routing/binlog/blr.c +++ b/server/modules/routing/binlog/blr.c @@ -2306,7 +2306,7 @@ destroyInstance(ROUTER *instance) ROUTER_INSTANCE *inst = (ROUTER_INSTANCE *) instance; MXS_DEBUG("Destroying instance of router %s for service %s", - service->routerModule, service->name); + inst->service->routerModule, inst->service->name); /* Check whether master connection is active */ if (inst->master) From 8982ee3db2738c14787a536ec906a1b9cdaa44bc Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 3 Nov 2016 15:50:43 +0200 Subject: [PATCH 171/215] Allow servers to be added and removed from services Servers can now be added and removed from services which allows routers to use them with new sessions. The routers don't fully use the new functionality in the server references which prevents new servers from being taken into use. --- include/maxscale/service.h | 3 + server/core/service.c | 58 +++++++++++++++++-- server/modules/routing/debugcli/debugcmd.c | 67 +++++++++++++++++----- 3 files changed, 107 insertions(+), 21 deletions(-) diff --git a/include/maxscale/service.h b/include/maxscale/service.h index b6d7940d5..273528f68 100644 --- a/include/maxscale/service.h +++ b/include/maxscale/service.h @@ -101,6 +101,7 @@ typedef struct server_ref_t SERVER* server; /**< The actual server */ int weight; /**< Weight of this server */ int connections; /**< Number of connections created through this reference */ + bool active; /**< Whether this reference is valid and in use*/ } SERVER_REF; #define SERVICE_MAX_RETRY_INTERVAL 3600 /*< The maximum interval between service start retries */ @@ -146,6 +147,7 @@ typedef struct service void *router_instance; /**< The router instance for this service */ char *version_string; /** version string for this service listeners */ SERVER_REF *dbref; /** server references */ + int n_dbref; /** Number of server references */ SERVICE_USER credentials; /**< The cedentials of the service user */ SPINLOCK spin; /**< The service spinlock */ SERVICE_STATS stats; /**< The service statistics */ @@ -194,6 +196,7 @@ extern int serviceAddProtocol(SERVICE *service, char *name, char *protocol, extern int serviceHasProtocol(SERVICE *service, const char *protocol, const char* address, unsigned short port); extern void serviceAddBackend(SERVICE *, SERVER *); +extern void serviceRemoveBackend(SERVICE *, const SERVER *); extern int serviceHasBackend(SERVICE *, SERVER *); extern void serviceAddRouterOption(SERVICE *, char *); extern void serviceClearRouterOptions(SERVICE *); diff --git a/server/core/service.c b/server/core/service.c index f8f2a77a7..aef3b022d 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -149,6 +149,7 @@ service_alloc(const char *servname, const char *router) service->capabilities = service->router->getCapabilities(); service->client_count = 0; + service->n_dbref = 0; service->name = (char*)servname; service->routerModule = (char*)router; service->users_from_all = false; @@ -742,7 +743,7 @@ int serviceHasProtocol(SERVICE *service, const char *protocol, * @param server Server to refer to * @return Server reference or NULL on error */ -static SERVER_REF* server_ref_alloc(SERVER *server) +static SERVER_REF* server_ref_create(SERVER *server) { SERVER_REF *sref = MXS_MALLOC(sizeof(SERVER_REF)); @@ -752,6 +753,7 @@ static SERVER_REF* server_ref_alloc(SERVER *server) sref->server = server; sref->weight = SERVICE_BASE_SERVER_WEIGHT; sref->connections = 0; + sref->active = true; } return sref; @@ -766,28 +768,72 @@ static SERVER_REF* server_ref_alloc(SERVER *server) void serviceAddBackend(SERVICE *service, SERVER *server) { - SERVER_REF *sref = server_ref_alloc(server); + SERVER_REF *new_ref = server_ref_create(server); - if (sref) + if (new_ref) { spinlock_acquire(&service->spin); + + service->n_dbref++; + if (service->dbref) { SERVER_REF *ref = service->dbref; - while (ref->next) + SERVER_REF *prev = ref; + + while (ref) { + if (ref->server == server) + { + ref->active = true; + break; + } + prev = ref; ref = ref->next; } - ref->next = sref; + + if (ref == NULL) + { + /** A new server that hasn't been used by this service */ + atomic_synchronize(); + prev->next = new_ref; + } } else { - service->dbref = sref; + atomic_synchronize(); + service->dbref = new_ref; } spinlock_release(&service->spin); } } +/** + * @brief Remove a server from a service + * + * This function sets the server reference into an inactive state. This does not + * remove the server from the list or free any of the memory. + * + * @param service Service to modify + * @param server Server to remove + */ +void serviceRemoveBackend(SERVICE *service, const SERVER *server) +{ + spinlock_acquire(&service->spin); + + service->n_dbref--; + + for (SERVER_REF *ref = service->dbref; ref; ref = ref->next) + { + if (ref->server == server) + { + ref->active = false; + break; + } + } + + spinlock_release(&service->spin); +} /** * Test if a server is part of a service * diff --git a/server/modules/routing/debugcli/debugcmd.c b/server/modules/routing/debugcli/debugcmd.c index 3e828bc77..9d072c600 100644 --- a/server/modules/routing/debugcli/debugcmd.c +++ b/server/modules/routing/debugcli/debugcmd.c @@ -715,39 +715,76 @@ struct subcommand failoptions[] = { static void telnetdAddUser(DCB *, char *user, char *password); +static void cmd_serviceAddBackend(DCB *dcb, void *a, void *b) +{ + SERVICE *service = (SERVICE*)a; + SERVER *server = (SERVER*)b; + + serviceAddBackend(service, server); + + MXS_NOTICE("Added server '%s' to service '%s'", server->unique_name, service->name); + dcb_printf(dcb, "Added server '%s' to service '%s'\n", server->unique_name, service->name); +} + /** * The subcommands of the add command */ struct subcommand addoptions[] = { - { "user", 2, telnetdAddUser, - "Add insecure account for using maxadmin over the network. E.g.:\n" - " MaxScale> add user bob somepass", - "Add insecure account for using maxadmin over the network. E.g.:\n" - " MaxScale> add user bob somepass", - {ARG_TYPE_STRING, ARG_TYPE_STRING, 0} }, + { + "user", 2, telnetdAddUser, + "Add insecure account for using maxadmin over the network. E.g.:\n" + " MaxScale> add user bob somepass", + "Add insecure account for using maxadmin over the network. E.g.:\n" + " MaxScale> add user bob somepass", + {ARG_TYPE_STRING, ARG_TYPE_STRING, 0} + }, + { + "server", 2, cmd_serviceAddBackend, + "Add a new server to a service", + "Add a new server to a service. The server must exist in the configuration file.", + {ARG_TYPE_SERVICE, ARG_TYPE_SERVER, 0} + }, { NULL, 0, NULL, NULL, NULL, - {0, 0, 0} } + {0, 0, 0}} }; static void telnetdRemoveUser(DCB *, char *user, char *password); +static void cmd_serviceRemoveBackend(DCB *dcb, void *a, void *b) +{ + SERVICE *service = (SERVICE*)a; + SERVER *server = (SERVER*)b; + + serviceRemoveBackend(service, server); + + MXS_NOTICE("Removed server '%s' from service '%s'", server->unique_name, service->name); + dcb_printf(dcb, "Removed server '%s' from service '%s'\n", server->unique_name, service->name); +} + /** * The subcommands of the remove command */ struct subcommand removeoptions[] = { { - "user", - 2, - telnetdRemoveUser, - "Remove account for using maxadmin over the network. E.g.:\n" - " MaxAdmin> remove user bob somepass", - "Remove account for using maxadmin over the network. E.g.:\n" - " MaxAdmin> remove user bob somepass", + "user", + 2, + telnetdRemoveUser, + "Remove account for using maxadmin over the network. E.g.:\n" + " MaxAdmin> remove user bob somepass", + "Remove account for using maxadmin over the network. E.g.:\n" + " MaxAdmin> remove user bob somepass", {ARG_TYPE_STRING, ARG_TYPE_STRING, 0} }, { - NULL, 0, NULL, NULL, NULL, {0, 0, 0} + "server", 2, cmd_serviceRemoveBackend, + "Remove a server from a service", + "Remove a server from a service. The server must exist in the configuration file.", + {ARG_TYPE_SERVICE, ARG_TYPE_SERVER, 0} + }, + { + NULL, 0, NULL, NULL, NULL, + {0, 0, 0} } }; From dc8c06893637d75eb4170f4b71c5251e3c3a5514 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 3 Nov 2016 15:54:31 +0200 Subject: [PATCH 172/215] Modify readwritesplit to understand changes in server references The readwritesplit now understands that the amount of servers can change and some of the items in the list of server references aren't in use. This allows dynamic changes to the number of servers used by readwritesplit. --- .../routing/readwritesplit/readwritesplit.c | 90 +++++++------------ .../routing/readwritesplit/readwritesplit.h | 1 - 2 files changed, 34 insertions(+), 57 deletions(-) diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index 357ab9610..65d9af518 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -65,6 +65,9 @@ MODULE_INFO info = * @endverbatim */ +/** Maximum number of slaves */ +#define MAX_SLAVE_COUNT 255 + static char *version_str = "V1.1.0"; /* @@ -126,7 +129,6 @@ static void handle_error_reply_client(SESSION *ses, ROUTER_CLIENT_SES *rses, static bool handle_error_new_connection(ROUTER_INSTANCE *inst, ROUTER_CLIENT_SES **rses, DCB *backend_dcb, GWBUF *errmsg); -static int router_get_servercount(ROUTER_INSTANCE *inst); static bool have_enough_servers(ROUTER_CLIENT_SES **p_rses, const int min_nsrv, int router_nsrv, ROUTER_INSTANCE *router); @@ -218,19 +220,12 @@ static ROUTER *createInstance(SERVICE *service, char **options) router->rwsplit_config.rw_max_sescmd_history_size = 0; } - int nservers = 0; - - for (SERVER_REF *ref = service->dbref; ref; ref = ref->next) - { - nservers++; - } - /** * Set default value for max_slave_connections as 100%. This way * LEAST_CURRENT_OPERATIONS allows us to balance evenly across all the * configured slaves. */ - router->rwsplit_config.rw_max_slave_conn_count = nservers; + router->rwsplit_config.rw_max_slave_conn_count = MAX_SLAVE_COUNT; if (router->rwsplit_config.rw_slave_select_criteria == UNDEFINED_CRITERIA) { @@ -306,7 +301,6 @@ static void *newSession(ROUTER *router_inst, SESSION *session) int router_nservers = 0; /*< # of servers in total */ int max_nslaves; /*< max # of slaves used in this session */ int max_slave_rlag; /*< max allowed replication lag for any slave */ - int i; const int min_nservers = 1; /*< hard-coded for now */ client_rses = (ROUTER_CLIENT_SES *)MXS_CALLOC(1, sizeof(ROUTER_CLIENT_SES)); @@ -350,7 +344,7 @@ static void *newSession(ROUTER *router_inst, SESSION *session) client_rses->have_tmp_tables = false; client_rses->forced_node = NULL; - router_nservers = router_get_servercount(router); + router_nservers = router->service->n_dbref; if (!have_enough_servers(&client_rses, min_nservers, router_nservers, router)) { @@ -373,25 +367,35 @@ static void *newSession(ROUTER *router_inst, SESSION *session) * Initialize backend references with BACKEND ptr. * Initialize session command cursors for each backend reference. */ - i = 0; - for (SERVER_REF *sref = router->service->dbref; sref; sref = sref->next) + int i = 0; + for (SERVER_REF *sref = router->service->dbref; sref && i < router_nservers; sref = sref->next) { + if (sref->active) + { #if defined(SS_DEBUG) - backend_ref[i].bref_chk_top = CHK_NUM_BACKEND_REF; - backend_ref[i].bref_chk_tail = CHK_NUM_BACKEND_REF; - backend_ref[i].bref_sescmd_cur.scmd_cur_chk_top = CHK_NUM_SESCMD_CUR; - backend_ref[i].bref_sescmd_cur.scmd_cur_chk_tail = CHK_NUM_SESCMD_CUR; + backend_ref[i].bref_chk_top = CHK_NUM_BACKEND_REF; + backend_ref[i].bref_chk_tail = CHK_NUM_BACKEND_REF; + backend_ref[i].bref_sescmd_cur.scmd_cur_chk_top = CHK_NUM_SESCMD_CUR; + backend_ref[i].bref_sescmd_cur.scmd_cur_chk_tail = CHK_NUM_SESCMD_CUR; #endif - backend_ref[i].bref_state = 0; - backend_ref[i].ref = sref; - /** store pointers to sescmd list to both cursors */ - backend_ref[i].bref_sescmd_cur.scmd_cur_rses = client_rses; - backend_ref[i].bref_sescmd_cur.scmd_cur_active = false; - backend_ref[i].bref_sescmd_cur.scmd_cur_ptr_property = - &client_rses->rses_properties[RSES_PROP_TYPE_SESCMD]; - backend_ref[i].bref_sescmd_cur.scmd_cur_cmd = NULL; - i++; + backend_ref[i].bref_state = 0; + backend_ref[i].ref = sref; + /** store pointers to sescmd list to both cursors */ + backend_ref[i].bref_sescmd_cur.scmd_cur_rses = client_rses; + backend_ref[i].bref_sescmd_cur.scmd_cur_active = false; + backend_ref[i].bref_sescmd_cur.scmd_cur_ptr_property = + &client_rses->rses_properties[RSES_PROP_TYPE_SESCMD]; + backend_ref[i].bref_sescmd_cur.scmd_cur_cmd = NULL; + i++; + } } + + if (i < router_nservers) + { + /** The service reported more servers than we took into use */ + router_nservers = i; + } + max_nslaves = rses_get_max_slavecount(client_rses, router_nservers); max_slave_rlag = rses_get_max_replication_lag(client_rses); @@ -434,7 +438,6 @@ static void *newSession(ROUTER *router_inst, SESSION *session) /** Copy backend pointers to router session. */ client_rses->rses_master_ref = master_ref; - client_rses->rses_backend_ref = backend_ref; client_rses->rses_nbackends = router_nservers; /*< # of backend servers */ if (client_rses->rses_config.rw_max_slave_conn_percent) @@ -447,11 +450,6 @@ static void *newSession(ROUTER *router_inst, SESSION *session) router->stats.n_sessions += 1; - /** - * Version is bigger than zero once initialized. - */ - atomic_add(&client_rses->rses_versno, 2); - ss_dassert(client_rses->rses_versno == 2); /** * Add this session to end of the list of active sessions in router. */ @@ -1656,7 +1654,6 @@ static bool handle_error_new_connection(ROUTER_INSTANCE *inst, { ROUTER_CLIENT_SES *myrses; SESSION *ses; - int router_nservers; int max_nslaves; int max_slave_rlag; backend_ref_t *bref; @@ -1710,8 +1707,8 @@ static bool handle_error_new_connection(ROUTER_INSTANCE *inst, */ dcb_remove_callback(backend_dcb, DCB_REASON_NOT_RESPONDING, &router_handle_state_switch, (void *)bref); - router_nservers = router_get_servercount(inst); - max_nslaves = rses_get_max_slavecount(myrses, router_nservers); + + max_nslaves = rses_get_max_slavecount(myrses, myrses->rses_nbackends); max_slave_rlag = rses_get_max_replication_lag(myrses); /** * Try to get replacement slave or at least the minimum @@ -1719,13 +1716,13 @@ static bool handle_error_new_connection(ROUTER_INSTANCE *inst, */ if (inst->rwsplit_config.rw_disable_sescmd_hist) { - succp = have_enough_servers(&myrses, 1, router_nservers, inst) ? true : false; + succp = have_enough_servers(&myrses, 1, myrses->rses_nbackends, inst) ? true : false; } else { succp = select_connect_backend_servers(&myrses->rses_master_ref, myrses->rses_backend_ref, - router_nservers, + myrses->rses_nbackends, max_nslaves, max_slave_rlag, myrses->rses_config.rw_slave_select_criteria, ses, inst); @@ -1735,25 +1732,6 @@ return_succp: return succp; } -/** - * @brief Calculate the number of backend servers - * - * @param inst Router instance - * - * @return int - count of servers - */ -static int router_get_servercount(ROUTER_INSTANCE *inst) -{ - int router_nservers = 0; - - for (SERVER_REF *ref = inst->service->dbref; ref; ref = ref->next) - { - router_nservers++; - } - - return router_nservers; -} - /** * @brief Calculate whether we have enough servers to route a query * diff --git a/server/modules/routing/readwritesplit/readwritesplit.h b/server/modules/routing/readwritesplit/readwritesplit.h index 5a8d9dd36..50bedcf40 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.h +++ b/server/modules/routing/readwritesplit/readwritesplit.h @@ -281,7 +281,6 @@ struct router_client_session skygw_chk_t rses_chk_top; #endif SPINLOCK rses_lock; /*< protects rses_deleted */ - int rses_versno; /*< even = no active update, else odd. not used 4/14 */ bool rses_closed; /*< true when closeSession is called */ rses_property_t* rses_properties[RSES_PROP_TYPE_COUNT]; /*< Properties listed by their type */ backend_ref_t* rses_master_ref; From 155161a876728da38e7854ed730425a3900843aa Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 3 Nov 2016 16:27:15 +0200 Subject: [PATCH 173/215] Remove BACKEND structure from readconnroute The BACKEND structure in readconnroute is now replaced with the use of the SERVER_REF structure of the service. This allows dynamic changes to the list of servers to be made. --- include/maxscale/service.h | 3 + .../routing/readconnroute/readconnection.h | 16 +- .../routing/readconnroute/readconnroute.c | 292 ++++++------------ 3 files changed, 99 insertions(+), 212 deletions(-) diff --git a/include/maxscale/service.h b/include/maxscale/service.h index 273528f68..b4bdfe637 100644 --- a/include/maxscale/service.h +++ b/include/maxscale/service.h @@ -104,6 +104,9 @@ typedef struct server_ref_t bool active; /**< Whether this reference is valid and in use*/ } SERVER_REF; +/** Macro to check whether a SERVER_REF is active */ +#define SERVER_REF_IS_ACTIVE(ref) (ref->active) + #define SERVICE_MAX_RETRY_INTERVAL 3600 /*< The maximum interval between service start retries */ /** Value of service timeout if timeout checks are disabled */ diff --git a/server/modules/routing/readconnroute/readconnection.h b/server/modules/routing/readconnroute/readconnection.h index 13b4fa2ad..dfcf153fd 100644 --- a/server/modules/routing/readconnroute/readconnection.h +++ b/server/modules/routing/readconnroute/readconnection.h @@ -32,18 +32,6 @@ MXS_BEGIN_DECLS -/** - * Internal structure used to define the set of backend servers we are routing - * connections to. This provides the storage for routing module specific data - * that is required for each of the backend servers. - */ -typedef struct backend -{ - SERVER *server; /*< The server itself */ - int current_connection_count; /*< Number of connections to the server */ - int weight; /*< Desired routing weight */ -} BACKEND; - /** * The client session structure used within this router. */ @@ -55,7 +43,7 @@ typedef struct router_client_session SPINLOCK rses_lock; /*< protects rses_deleted */ int rses_versno; /*< even = no active update, else odd */ bool rses_closed; /*< true when closeSession is called */ - BACKEND *backend; /*< Backend used by the client session */ + SERVER_REF *backend; /*< Backend used by the client session */ DCB *backend_dcb; /*< DCB Connection to the backend */ DCB *client_dcb; /**< Client DCB */ struct router_client_session *next; @@ -79,9 +67,7 @@ typedef struct typedef struct router_instance { SERVICE *service; /*< Pointer to the service using this router */ - ROUTER_CLIENT_SES *connections; /*< Link list of all the client connections */ SPINLOCK lock; /*< Spinlock for the instance data */ - BACKEND **servers; /*< List of backend servers */ unsigned int bitmask; /*< Bitmask to apply to server->status */ unsigned int bitvalue; /*< Required value of server->status */ ROUTER_STATS stats; /*< Statistics for this router */ diff --git a/server/modules/routing/readconnroute/readconnroute.c b/server/modules/routing/readconnroute/readconnroute.c index a77dad041..256e322c7 100644 --- a/server/modules/routing/readconnroute/readconnroute.c +++ b/server/modules/routing/readconnroute/readconnroute.c @@ -132,7 +132,7 @@ static bool rses_begin_locked_router_action(ROUTER_CLIENT_SES* rses); static void rses_end_locked_router_action(ROUTER_CLIENT_SES* rses); -static BACKEND *get_root_master(BACKEND **servers); +static SERVER_REF *get_root_master(SERVER_REF *servers); static int handle_state_switch(DCB* dcb, DCB_REASON reason, void * routersession); static SPINLOCK instlock; static ROUTER_INSTANCE *instances; @@ -178,14 +178,6 @@ static inline void free_readconn_instance(ROUTER_INSTANCE *router) { if (router) { - if (router->servers) - { - for (int i = 0; router->servers[i]; i++) - { - MXS_FREE(router->servers[i]); - } - } - MXS_FREE(router->servers); MXS_FREE(router); } } @@ -214,37 +206,6 @@ createInstance(SERVICE *service, char **options) inst->service = service; spinlock_init(&inst->lock); - /* - * We need an array of the backend servers in the instance structure so - * that we can maintain a count of the number of connections to each - * backend server. - */ - for (sref = service->dbref, n = 0; sref; sref = sref->next) - { - n++; - } - - inst->servers = (BACKEND **) MXS_CALLOC(n + 1, sizeof(BACKEND *)); - if (!inst->servers) - { - free_readconn_instance(inst); - return NULL; - } - - for (sref = service->dbref, n = 0; sref; sref = sref->next) - { - if ((inst->servers[n] = MXS_MALLOC(sizeof(BACKEND))) == NULL) - { - free_readconn_instance(inst); - return NULL; - } - inst->servers[n]->server = sref->server; - inst->servers[n]->current_connection_count = 0; - inst->servers[n]->weight = sref->weight; - n++; - } - inst->servers[n] = NULL; - /* * Process the options */ @@ -329,9 +290,9 @@ newSession(ROUTER *instance, SESSION *session) { ROUTER_INSTANCE *inst = (ROUTER_INSTANCE *) instance; ROUTER_CLIENT_SES *client_rses; - BACKEND *candidate = NULL; + SERVER_REF *candidate = NULL; int i; - BACKEND *master_host = NULL; + SERVER_REF *master_host = NULL; MXS_DEBUG("%lu [newSession] new router session with session " "%p, and inst %p.", @@ -339,7 +300,6 @@ newSession(ROUTER *instance, SESSION *session) session, inst); - client_rses = (ROUTER_CLIENT_SES *) MXS_CALLOC(1, sizeof(ROUTER_CLIENT_SES)); if (client_rses == NULL) @@ -356,7 +316,7 @@ newSession(ROUTER *instance, SESSION *session) /** * Find the Master host from available servers */ - master_host = get_root_master(inst->servers); + master_host = get_root_master(inst->service->dbref); /** * Find a backend server to connect to. This is the extent of the @@ -376,52 +336,43 @@ newSession(ROUTER *instance, SESSION *session) * become the new candidate. This has the effect of spreading the * connections over different servers during periods of very low load. */ - for (i = 0; inst->servers[i]; i++) + for (SERVER_REF *ref = inst->service->dbref; ref; ref = ref->next) { - if (inst->servers[i]) + if (!SERVER_REF_IS_ACTIVE(ref) || SERVER_IN_MAINT(ref->server) || ref->weight == 0) + { + continue; + } + else { MXS_DEBUG("%lu [newSession] Examine server in port %d with " "%d connections. Status is %s, " "inst->bitvalue is %d", pthread_self(), - inst->servers[i]->server->port, - inst->servers[i]->current_connection_count, - STRSRVSTATUS(inst->servers[i]->server), + ref->server->port, + ref->connections, + STRSRVSTATUS(ref->server), inst->bitmask); } - if (SERVER_IN_MAINT(inst->servers[i]->server)) - { - continue; - } - - if (inst->servers[i]->weight == 0) - { - continue; - } - /* Check server status bits against bitvalue from router_options */ - if (inst->servers[i] && - SERVER_IS_RUNNING(inst->servers[i]->server) && - (inst->servers[i]->server->status & inst->bitmask & inst->bitvalue)) + if (ref && SERVER_IS_RUNNING(ref->server) && + (ref->server->status & inst->bitmask & inst->bitvalue)) { if (master_host) { - if (inst->servers[i] == master_host && (inst->bitvalue & SERVER_SLAVE)) + if (ref == master_host && (inst->bitvalue & SERVER_SLAVE)) { - /* skip root Master here, as it could also be slave of an external server - * that is not in the configuration. - * Intermediate masters (Relay Servers) are also slave and will be selected - * as Slave(s) + /* Skip root master here, as it could also be slave of an external server that + * is not in the configuration. Intermediate masters (Relay Servers) are also + * slave and will be selected as Slave(s) */ continue; } - if (inst->servers[i] == master_host && (inst->bitvalue & SERVER_MASTER)) + if (ref == master_host && (inst->bitvalue & SERVER_MASTER)) { - /* If option is "master" return only the root Master as there - * could be intermediate masters (Relay Servers) - * and they must not be selected. + /* If option is "master" return only the root Master as there could be + * intermediate masters (Relay Servers) and they must not be selected. */ candidate = master_host; @@ -430,8 +381,7 @@ newSession(ROUTER *instance, SESSION *session) } else { - /* master_host is NULL, no master server. - * If requested router_option is 'master' + /* Master_host is NULL, no master server. If requested router_option is 'master' * candidate wll be NULL. */ if (inst->bitvalue & SERVER_MASTER) @@ -441,40 +391,31 @@ newSession(ROUTER *instance, SESSION *session) } } - /* If no candidate set, set first running server as - our initial candidate server */ + /* If no candidate set, set first running server as our initial candidate server */ if (candidate == NULL) { - candidate = inst->servers[i]; + candidate = ref; } - else if (((inst->servers[i]->current_connection_count + 1) - * 1000) / inst->servers[i]->weight < - ((candidate->current_connection_count + 1) * - 1000) / candidate->weight) + else if (((ref->connections + 1) * 1000) / ref->weight < + ((candidate->connections + 1) * 1000) / candidate->weight) { - /* This running server has fewer - connections, set it as a new candidate */ - candidate = inst->servers[i]; + /* This running server has fewer connections, set it as a new candidate */ + candidate = ref; } - else if (((inst->servers[i]->current_connection_count + 1) - * 1000) / inst->servers[i]->weight == - ((candidate->current_connection_count + 1) * - 1000) / candidate->weight && - inst->servers[i]->server->stats.n_connections < - candidate->server->stats.n_connections) + else if (((ref->connections + 1) * 1000) / ref->weight == + ((candidate->connections + 1) * 1000) / candidate->weight && + ref->server->stats.n_connections < candidate->server->stats.n_connections) { - /* This running server has the same number - of connections currently as the candidate - but has had fewer connections over time - than candidate, set this server to candidate*/ - candidate = inst->servers[i]; + /* This running server has the same number of connections currently as the candidate + but has had fewer connections over time than candidate, set this server to + candidate*/ + candidate = ref; } } } - /* There is no candidate server here! - * With router_option=slave a master_host could be set, so route traffic there. - * Otherwise, just clean up and return NULL + /* If we haven't found a proper candidate yet but a master server is available, we'll pick that + * with the assumption that it is "better" than a slave. */ if (!candidate) { @@ -484,9 +425,8 @@ newSession(ROUTER *instance, SESSION *session) } else { - MXS_ERROR("Failed to create new routing session. " - "Couldn't find eligible candidate server. Freeing " - "allocated resources."); + MXS_ERROR("Failed to create new routing session. Couldn't find eligible" + " candidate server. Freeing allocated resources."); MXS_FREE(client_rses); return NULL; } @@ -496,48 +436,32 @@ newSession(ROUTER *instance, SESSION *session) * We now have the server with the least connections. * Bump the connection count for this server */ - atomic_add(&candidate->current_connection_count, 1); client_rses->backend = candidate; - MXS_DEBUG("%lu [newSession] Selected server in port %d. " - "Connections : %d\n", - pthread_self(), - candidate->server->port, - candidate->current_connection_count); - /* - * Open a backend connection, putting the DCB for this - * connection in the client_rses->backend_dcb - */ - client_rses->backend_dcb = dcb_connect(candidate->server, - session, + /** Open the backend connection */ + client_rses->backend_dcb = dcb_connect(candidate->server, session, candidate->server->protocol); + if (client_rses->backend_dcb == NULL) { - atomic_add(&candidate->current_connection_count, -1); + /** The failure is reported in dcb_connect() */ MXS_FREE(client_rses); return NULL; } - dcb_add_callback( - client_rses->backend_dcb, + + atomic_add(&candidate->connections, 1); + + // TODO: Remove this as it is never called + dcb_add_callback(client_rses->backend_dcb, DCB_REASON_NOT_RESPONDING, &handle_state_switch, client_rses); inst->stats.n_sessions++; - /** - * Add this session to the list of active sessions. - */ - spinlock_acquire(&inst->lock); - client_rses->next = inst->connections; - inst->connections = client_rses; - spinlock_release(&inst->lock); - CHK_CLIENT_RSES(client_rses); - MXS_INFO("Readconnroute: New session for server %s. " - "Connections : %d", - candidate->server->unique_name, - candidate->current_connection_count); + MXS_INFO("Readconnroute: New session for server %s. Connections : %d", + candidate->server->unique_name, candidate->connections); return(void *) client_rses; } @@ -562,42 +486,11 @@ newSession(ROUTER *instance, SESSION *session) static void freeSession(ROUTER* router_instance, void* router_client_ses) { ROUTER_INSTANCE* router = (ROUTER_INSTANCE *) router_instance; - ROUTER_CLIENT_SES* router_cli_ses = - (ROUTER_CLIENT_SES *) router_client_ses; + ROUTER_CLIENT_SES* router_cli_ses = (ROUTER_CLIENT_SES *) router_client_ses; - ss_debug(int prev_val = ) atomic_add(&router_cli_ses->backend->current_connection_count, -1); + ss_debug(int prev_val = ) atomic_add(&router_cli_ses->backend->connections, -1); ss_dassert(prev_val > 0); - spinlock_acquire(&router->lock); - - if (router->connections == router_cli_ses) - { - router->connections = router_cli_ses->next; - } - else - { - ROUTER_CLIENT_SES *ptr = router->connections; - - while (ptr != NULL && ptr->next != router_cli_ses) - { - ptr = ptr->next; - } - - if (ptr != NULL) - { - ptr->next = router_cli_ses->next; - } - } - spinlock_release(&router->lock); - - MXS_DEBUG("%lu [freeSession] Unlinked router_client_session %p from " - "router %p and from server on port %d. Connections : %d. ", - pthread_self(), - router_cli_ses, - router, - router_cli_ses->backend->server->port, - prev_val - 1); - MXS_FREE(router_cli_ses); } @@ -639,6 +532,29 @@ closeSession(ROUTER *instance, void *router_session) } } +/** Log routing failure due to closed session */ +static void log_closed_session(mysql_server_cmd_t mysql_command, bool is_closed, + SERVER_REF *ref) +{ + char msg[MAX_SERVER_NAME_LEN + 200] = ""; // Extra space for message + + if (is_closed) + { + sprintf(msg, "Session is closed."); + } + else if (SERVER_IS_DOWN(ref->server)) + { + sprintf(msg, "Server '%s' is down.", ref->server->unique_name); + } + else if (!SERVER_REF_IS_ACTIVE(ref)) + { + sprintf(msg, "Server '%s' was removed from the service.", ref->server->unique_name); + } + + MXS_ERROR("Failed to route MySQL command %d to backend server. %s", + mysql_command, msg); +} + /** * We have data from the client, we must route it to the backend. * This is simply a case of sending it to the connection that was @@ -654,7 +570,7 @@ routeQuery(ROUTER *instance, void *router_session, GWBUF *queue) { ROUTER_INSTANCE *inst = (ROUTER_INSTANCE *) instance; ROUTER_CLIENT_SES *router_cli_ses = (ROUTER_CLIENT_SES *) router_session; - int rc; + int rc = 0; DCB* backend_dcb; MySQLProtocol *proto = (MySQLProtocol*)router_cli_ses->client_dcb->protocol; mysql_server_cmd_t mysql_command = proto->current_command; @@ -683,16 +599,11 @@ routeQuery(ROUTER *instance, void *router_session, GWBUF *queue) } if (rses_is_closed || backend_dcb == NULL || + !SERVER_REF_IS_ACTIVE(router_cli_ses->backend) || SERVER_IS_DOWN(router_cli_ses->backend->server)) { - MXS_ERROR("Failed to route MySQL command %d to backend " - "server.%s", - mysql_command, rses_is_closed ? " Session is closed." : ""); - rc = 0; - while ((queue = GWBUF_CONSUME_ALL(queue)) != NULL) - { - ; - } + log_closed_session(mysql_command, rses_is_closed, router_cli_ses->backend); + gwbuf_free(queue); goto return_rc; } @@ -737,23 +648,12 @@ static void diagnostics(ROUTER *router, DCB *dcb) { ROUTER_INSTANCE *router_inst = (ROUTER_INSTANCE *) router; - ROUTER_CLIENT_SES *session; - int i = 0; - BACKEND *backend; char *weightby; - spinlock_acquire(&router_inst->lock); - session = router_inst->connections; - while (session) - { - i++; - session = session->next; - } - spinlock_release(&router_inst->lock); - dcb_printf(dcb, "\tNumber of router sessions: %d\n", router_inst->stats.n_sessions); - dcb_printf(dcb, "\tCurrent no. of router sessions: %d\n", i); + dcb_printf(dcb, "\tCurrent no. of router sessions: %d\n", + router_inst->service->stats.n_current); dcb_printf(dcb, "\tNumber of queries forwarded: %d\n", router_inst->stats.n_queries); if ((weightby = serviceGetWeightingParameter(router_inst->service)) @@ -764,15 +664,13 @@ diagnostics(ROUTER *router, DCB *dcb) weightby); dcb_printf(dcb, "\t\tServer Target %% Connections\n"); - for (i = 0; router_inst->servers[i]; i++) + for (SERVER_REF *ref = router_inst->service->dbref; ref; ref = ref->next) { - backend = router_inst->servers[i]; dcb_printf(dcb, "\t\t%-20s %3.1f%% %d\n", - backend->server->unique_name, - (float) backend->weight / 10, - backend->current_connection_count); + ref->server->unique_name, + (float) ref->weight / 10, + ref->connections); } - } } @@ -933,28 +831,28 @@ static uint64_t getCapabilities(void) * */ -static BACKEND *get_root_master(BACKEND **servers) +static SERVER_REF *get_root_master(SERVER_REF *servers) { int i = 0; - BACKEND *master_host = NULL; + SERVER_REF *master_host = NULL; - for (i = 0; servers[i]; i++) + for (SERVER_REF *ref = servers; ref; ref = ref->next) { - if (servers[i] && (servers[i]->server->status & (SERVER_MASTER | SERVER_MAINT)) == SERVER_MASTER) + if (ref->active && SERVER_IS_MASTER(ref->server)) { if (master_host == NULL) { - master_host = servers[i]; + master_host = ref; } - else if (servers[i]->server->depth < master_host->server->depth || - (servers[i]->server->depth == master_host->server->depth && - servers[i]->weight > master_host->weight)) + else if (ref->server->depth < master_host->server->depth || + (ref->server->depth == master_host->server->depth && + ref->weight > master_host->weight)) { /** * This master has a lower depth than the candidate master or * the depths are equal but this master has a higher weight */ - master_host = servers[i]; + master_host = ref; } } } From 26d1986facbd605bf995bafdcd6a63440aa7392e Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 3 Nov 2016 23:19:10 +0200 Subject: [PATCH 174/215] Remove BACKEND from schemarouter and clean up code Removed the use of the BACKEND structure and replaced it with the use of the SERVER_REF structure of the service. This allows dynamic changes to be made to the list of servers. Cleaned up parts of the code and removed obsolete or useless functions. The schemarouter module could do with some refactoring since it derives from readwritesplit. --- .../routing/schemarouter/schemarouter.c | 564 +++++------------- .../routing/schemarouter/schemarouter.h | 4 +- 2 files changed, 136 insertions(+), 432 deletions(-) diff --git a/server/modules/routing/schemarouter/schemarouter.c b/server/modules/routing/schemarouter/schemarouter.c index ce6bcecec..625352369 100644 --- a/server/modules/routing/schemarouter/schemarouter.c +++ b/server/modules/routing/schemarouter/schemarouter.c @@ -84,8 +84,6 @@ static void handleError(ROUTER* instance, DCB* backend_dcb, error_action_t action, bool* succp); - -static int router_get_servercount(ROUTER_INSTANCE* router); static backend_ref_t* get_bref_from_dcb(ROUTER_CLIENT_SES* rses, DCB* dcb); static route_target_t get_shard_route_target(qc_query_type_t qtype, @@ -306,7 +304,7 @@ char* get_lenenc_str(void* data) showdb_response_t parse_showdb_response(ROUTER_CLIENT_SES* rses, backend_ref_t* bref, GWBUF** buffer) { unsigned char* ptr; - char* target = bref->bref_backend->backend_server->unique_name; + char* target = bref->bref_backend->server->unique_name; GWBUF* buf; bool duplicate_found = false; showdb_response_t rval = SHOWDB_PARTIAL_RESPONSE; @@ -393,12 +391,12 @@ showdb_response_t parse_showdb_response(ROUTER_CLIENT_SES* rses, backend_ref_t* { atomic_add(&bref->n_mapping_eof, 1); MXS_INFO("schemarouter: SHOW DATABASES fully received from %s.", - bref->bref_backend->backend_server->unique_name); + bref->bref_backend->server->unique_name); } else { MXS_INFO("schemarouter: SHOW DATABASES partially received from %s.", - bref->bref_backend->backend_server->unique_name); + bref->bref_backend->server->unique_name); } gwbuf_free(buf); @@ -453,13 +451,13 @@ int gen_databaselist(ROUTER_INSTANCE* inst, ROUTER_CLIENT_SES* session) { if (BREF_IS_IN_USE(&session->rses_backend_ref[i]) && !BREF_IS_CLOSED(&session->rses_backend_ref[i]) & - SERVER_IS_RUNNING(session->rses_backend_ref[i].bref_backend->backend_server)) + SERVER_IS_RUNNING(session->rses_backend_ref[i].bref_backend->server)) { clone = gwbuf_clone(buffer); dcb = session->rses_backend_ref[i].bref_dcb; rval |= !dcb->func.write(dcb, clone); MXS_DEBUG("schemarouter: Wrote SHOW DATABASES to %s for session %p: returned %d", - session->rses_backend_ref[i].bref_backend->backend_server->unique_name, + session->rses_backend_ref[i].bref_backend->server->unique_name, session->rses_client_dcb->session, rval); } @@ -560,7 +558,7 @@ char* get_shard_target_name(ROUTER_INSTANCE* router, for (i = 0; i < client->rses_nbackends; i++) { - char *srvnm = client->rses_backend_ref[i].bref_backend->backend_server->unique_name; + char *srvnm = client->rses_backend_ref[i].bref_backend->server->unique_name; if (strcmp(srvnm, buffer->hint->data) == 0) { rval = srvnm; @@ -596,21 +594,16 @@ char* get_shard_target_name(ROUTER_INSTANCE* router, */ bool check_shard_status(ROUTER_INSTANCE* router, char* shard) { - int i; - bool rval = false; - - for (i = 0; router->servers[i]; i++) + for (SERVER_REF *ref = router->service->dbref; ref; ref = ref->next) { - if (strcmp(router->servers[i]->backend_server->unique_name, shard) == 0) + if (strcmp(ref->server->unique_name, shard) == 0 && + SERVER_IS_RUNNING(ref->server)) { - if (SERVER_IS_RUNNING(router->servers[i]->backend_server)) - { - rval = true; - } - break; + return true; } } - return rval; + + return false; } /** @@ -694,22 +687,21 @@ ROUTER_OBJECT* GetModuleObject() static ROUTER* createInstance(SERVICE *service, char **options) { ROUTER_INSTANCE* router; - SERVER_REF* server; CONFIG_PARAMETER* conf; - int nservers; - int i; CONFIG_PARAMETER* param; if ((router = MXS_CALLOC(1, sizeof(ROUTER_INSTANCE))) == NULL) { return NULL; } + if ((router->ignored_dbs = hashtable_alloc(SCHEMAROUTER_HASHSIZE, hashkeyfun, hashcmpfun)) == NULL) { MXS_ERROR("Memory allocation failed when allocating schemarouter database ignore list."); MXS_FREE(router); return NULL; } + hashtable_memory_fns(router->ignored_dbs, hashtable_item_strdup, NULL, hashtable_item_free, NULL); if ((router->shard_maps = hashtable_alloc(SCHEMAROUTER_USERHASH_SIZE, hashkeyfun, hashcmpfun)) == NULL) @@ -739,10 +731,6 @@ static ROUTER* createInstance(SERVICE *service, char **options) router->stats.ses_shortest = (double)((unsigned long)(~0)); spinlock_init(&router->lock); - /** Calculate number of servers */ - server = service->dbref; - nservers = 0; - conf = service->svc_config_param; if ((config_get_param(conf, "auth_all_servers")) == NULL) { @@ -800,17 +788,20 @@ static ROUTER* createInstance(SERVICE *service, char **options) bool failure = false; - for (i = 0; options && options[i]; i++) + for (int i = 0; options && options[i]; i++) { - char* value; - if ((value = strchr(options[i], '=')) == NULL) + char* value = strchr(options[i], '='); + + if (value == NULL) { MXS_ERROR("Unknown router options for Schemarouter: %s", options[i]); failure = true; break; } + *value = '\0'; value++; + if (strcmp(options[i], "max_sescmd_history") == 0) { router->schemarouter_config.max_sescmd_hist = atoi(value); @@ -848,94 +839,9 @@ static ROUTER* createInstance(SERVICE *service, char **options) if (failure) { MXS_FREE(router); - return NULL; + router = NULL; } - while (server != NULL) - { - nservers++; - server = server->next; - } - router->servers = (BACKEND **)MXS_CALLOC(nservers + 1, sizeof(BACKEND *)); - - if (router->servers == NULL) - { - MXS_FREE(router); - return NULL; - } - /** - * Create an array of the backend servers in the router structure to - * maintain a count of the number of connections to each - * backend server. - */ - server = service->dbref; - nservers = 0; - - while (server != NULL) - { - if ((router->servers[nservers] = MXS_MALLOC(sizeof(BACKEND))) == NULL) - { - goto clean_up; - } - router->servers[nservers]->backend_server = server->server; - router->servers[nservers]->backend_conn_count = 0; - router->servers[nservers]->weight = 1; - router->servers[nservers]->be_valid = false; - router->servers[nservers]->stats.queries = 0; - if (server->server->monuser == NULL && service->credentials.name != NULL) - { - router->servers[nservers]->backend_server->monuser = - MXS_STRDUP_A(service->credentials.name); - } - if (server->server->monpw == NULL && service->credentials.authdata != NULL) - { - router->servers[nservers]->backend_server->monpw = - MXS_STRDUP_A(service->credentials.authdata); - } -#if defined(SS_DEBUG) - router->servers[nservers]->be_chk_top = CHK_NUM_BACKEND; - router->servers[nservers]->be_chk_tail = CHK_NUM_BACKEND; -#endif - nservers += 1; - server = server->next; - } - router->servers[nservers] = NULL; - - /** - * Process the options - */ - router->bitmask = 0; - router->bitvalue = 0; - - /** - * Read config version number from service to inform what configuration - * is used if any. - */ - router->schemarouter_version = service->svc_config_version; - - /** - * We have completed the creation of the router data, so now - * insert this router into the linked list of routers - * that have been created with this module. - */ - - spinlock_acquire(&instlock); - router->next = instances; - instances = router; - spinlock_release(&instlock); - goto retblock; - -clean_up: - /** clean up */ - for (i = 0; i < nservers; i++) - { - MXS_FREE(router->servers[i]); - } - MXS_FREE(router->servers); - MXS_FREE(router); - router = NULL; - /** Fallthrough */ -retblock: return (ROUTER *)router; } @@ -975,15 +881,12 @@ static void* newSession(ROUTER* router_inst, SESSION* session) ROUTER_INSTANCE* router = (ROUTER_INSTANCE *)router_inst; bool succp; int router_nservers = 0; /*< # of servers in total */ - int i; - char db[MYSQL_DATABASE_MAXLEN + 1]; + char db[MYSQL_DATABASE_MAXLEN + 1] = ""; MySQLProtocol* protocol = session->client_dcb->protocol; MYSQL_session* data = session->client_dcb->data; bool using_db = false; bool have_db = false; - *db = 0; - spinlock_acquire(&session->ses_lock); /* To enable connecting directly to a sharded database we first need @@ -1010,8 +913,7 @@ static void* newSession(ROUTER* router_inst, SESSION* session) if (client_rses == NULL) { - ss_dassert(false); - goto return_rses; + return NULL; } #if defined(SS_DEBUG) client_rses->rses_chk_top = CHK_NUM_ROUTER_SES; @@ -1079,47 +981,57 @@ static void* newSession(ROUTER* router_inst, SESSION* session) * responding server. */ - router_nservers = router_get_servercount(router); + router_nservers = router->service->n_dbref; /** * Create backend reference objects for this session. */ - backend_ref = (backend_ref_t *)MXS_CALLOC(1, router_nservers * sizeof(backend_ref_t)); + backend_ref = (backend_ref_t *)MXS_CALLOC(router_nservers, sizeof(backend_ref_t)); if (backend_ref == NULL) { - /** log this */ MXS_FREE(client_rses); - MXS_FREE(backend_ref); - client_rses = NULL; - goto return_rses; + return NULL; } /** * Initialize backend references with BACKEND ptr. * Initialize session command cursors for each backend reference. */ - for (i = 0; i < router_nservers; i++) + + int i = 0; + + for (SERVER_REF *ref = router->service->dbref; ref; ref = ref->next) { + if (ref->active) + { #if defined(SS_DEBUG) - backend_ref[i].bref_chk_top = CHK_NUM_BACKEND_REF; - backend_ref[i].bref_chk_tail = CHK_NUM_BACKEND_REF; - backend_ref[i].bref_sescmd_cur.scmd_cur_chk_top = CHK_NUM_SESCMD_CUR; - backend_ref[i].bref_sescmd_cur.scmd_cur_chk_tail = CHK_NUM_SESCMD_CUR; + backend_ref[i].bref_chk_top = CHK_NUM_BACKEND_REF; + backend_ref[i].bref_chk_tail = CHK_NUM_BACKEND_REF; + backend_ref[i].bref_sescmd_cur.scmd_cur_chk_top = CHK_NUM_SESCMD_CUR; + backend_ref[i].bref_sescmd_cur.scmd_cur_chk_tail = CHK_NUM_SESCMD_CUR; #endif - backend_ref[i].bref_state = 0; - backend_ref[i].n_mapping_eof = 0; - backend_ref[i].map_queue = NULL; - backend_ref[i].bref_backend = router->servers[i]; - /** store pointers to sescmd list to both cursors */ - backend_ref[i].bref_sescmd_cur.scmd_cur_rses = client_rses; - backend_ref[i].bref_sescmd_cur.scmd_cur_active = false; - backend_ref[i].bref_sescmd_cur.scmd_cur_ptr_property = - &client_rses->rses_properties[RSES_PROP_TYPE_SESCMD]; - backend_ref[i].bref_sescmd_cur.scmd_cur_cmd = NULL; + backend_ref[i].bref_state = 0; + backend_ref[i].n_mapping_eof = 0; + backend_ref[i].map_queue = NULL; + backend_ref[i].bref_backend = ref; + /** store pointers to sescmd list to both cursors */ + backend_ref[i].bref_sescmd_cur.scmd_cur_rses = client_rses; + backend_ref[i].bref_sescmd_cur.scmd_cur_active = false; + backend_ref[i].bref_sescmd_cur.scmd_cur_ptr_property = + &client_rses->rses_properties[RSES_PROP_TYPE_SESCMD]; + backend_ref[i].bref_sescmd_cur.scmd_cur_cmd = NULL; + i++; + } + } + + if (i < router_nservers) + { + router_nservers = i; } spinlock_init(&client_rses->rses_lock); client_rses->rses_backend_ref = backend_ref; + client_rses->rses_nbackends = router_nservers; /** * Find a backend servers to connect to. @@ -1129,43 +1041,23 @@ static void* newSession(ROUTER* router_inst, SESSION* session) { MXS_FREE(client_rses->rses_backend_ref); MXS_FREE(client_rses); - client_rses = NULL; - goto return_rses; + return NULL; } /** * Connect to all backend servers */ - succp = connect_backend_servers(backend_ref, - router_nservers, - session, - router); + succp = connect_backend_servers(backend_ref, router_nservers, session, router); rses_end_locked_router_action(client_rses); - /** - * Master and at least slaves must be found - */ - if (!succp) + if (!succp || !(succp = rses_begin_locked_router_action(client_rses))) { MXS_FREE(client_rses->rses_backend_ref); MXS_FREE(client_rses); - client_rses = NULL; - goto return_rses; - } - /** Copy backend pointers to router session. */ - client_rses->rses_backend_ref = backend_ref; - client_rses->rses_nbackends = router_nservers; /*< # of backend servers */ - - if (!(succp = rses_begin_locked_router_action(client_rses))) - { - MXS_FREE(client_rses->rses_backend_ref); - MXS_FREE(client_rses); - - client_rses = NULL; - goto return_rses; + return NULL; } - if (db[0] != 0x0) + if (db[0]) { /* Store the database the client is connecting to */ snprintf(client_rses->connect_db, MYSQL_DATABASE_MAXLEN + 1, "%s", db); @@ -1175,26 +1067,6 @@ static void* newSession(ROUTER* router_inst, SESSION* session) atomic_add(&router->stats.sessions, 1); - /** - * Version is bigger than zero once initialized. - */ - atomic_add(&client_rses->rses_versno, 2); - ss_dassert(client_rses->rses_versno == 2); - /** - * Add this session to end of the list of active sessions in router. - */ - spinlock_acquire(&router->lock); - client_rses->next = router->connections; - router->connections = client_rses; - spinlock_release(&router->lock); - -return_rses: -#if defined(SS_DEBUG) - if (client_rses != NULL) - { - CHK_CLIENT_RSES(client_rses); - } -#endif return (void *)client_rses; } @@ -1270,7 +1142,7 @@ static void closeSession(ROUTER* instance, void* router_session) */ dcb_close(dcb); /** decrease server current connection counters */ - atomic_add(&bref->bref_backend->backend_conn_count, -1); + atomic_add(&bref->bref_backend->connections, -1); } } @@ -1315,51 +1187,18 @@ static void closeSession(ROUTER* instance, void* router_session) static void freeSession(ROUTER* router_instance, void* router_client_session) { - ROUTER_CLIENT_SES* router_cli_ses; - ROUTER_INSTANCE* router; - int i; - backend_ref_t* bref; + ROUTER_CLIENT_SES* router_cli_ses = (ROUTER_CLIENT_SES *)router_client_session; - router_cli_ses = (ROUTER_CLIENT_SES *)router_client_session; - router = (ROUTER_INSTANCE *)router_instance; - - for (i = 0; i < router_cli_ses->rses_nbackends; i++) + for (int i = 0; i < router_cli_ses->rses_nbackends; i++) { - bref = &router_cli_ses->rses_backend_ref[i]; - while (bref->bref_pending_cmd && - (bref->bref_pending_cmd = gwbuf_consume( - bref->bref_pending_cmd, gwbuf_length(bref->bref_pending_cmd)))) - { - ; - } + gwbuf_free(router_cli_ses->rses_backend_ref[i].bref_pending_cmd); } - spinlock_acquire(&router->lock); - - if (router->connections == router_cli_ses) - { - router->connections = router_cli_ses->next; - } - else - { - ROUTER_CLIENT_SES* ptr = router->connections; - - while (ptr && ptr->next != router_cli_ses) - { - ptr = ptr->next; - } - - if (ptr) - { - ptr->next = router_cli_ses->next; - } - } - spinlock_release(&router->lock); /** * For each property type, walk through the list, finalize properties * and free the allocated memory. */ - for (i = RSES_PROP_TYPE_FIRST; i < RSES_PROP_TYPE_COUNT; i++) + for (int i = RSES_PROP_TYPE_FIRST; i < RSES_PROP_TYPE_COUNT; i++) { rses_property_t* p = router_cli_ses->rses_properties[i]; rses_property_t* q = p; @@ -1415,15 +1254,15 @@ static bool get_shard_dcb(DCB** p_dcb, for (i = 0; i < rses->rses_nbackends; i++) { - BACKEND* b = backend_ref[i].bref_backend; + SERVER_REF* b = backend_ref[i].bref_backend; /** * To become chosen: * backend must be in use, name must match, and * the backend state must be RUNNING */ if (BREF_IS_IN_USE((&backend_ref[i])) && - (strncasecmp(name, b->backend_server->unique_name, PATH_MAX) == 0) && - SERVER_IS_RUNNING(b->backend_server)) + (strncasecmp(name, b->server->unique_name, PATH_MAX) == 0) && + SERVER_IS_RUNNING(b->server)) { *p_dcb = backend_ref[i].bref_dcb; succp = true; @@ -2225,14 +2064,13 @@ static int routeQuery(ROUTER* instance, if (TARGET_IS_ANY(route_target)) { - int z; - - for (z = 0; inst->servers[z]; z++) + for (int i = 0; i < router_cli_ses->rses_nbackends; i++) { - if (SERVER_IS_RUNNING(inst->servers[z]->backend_server)) + SERVER *server = router_cli_ses->rses_backend_ref[i].bref_backend->server; + if (SERVER_IS_RUNNING(server)) { route_target = TARGET_NAMED_SERVER; - targetserver = MXS_STRDUP_A(inst->servers[z]->backend_server->unique_name); + targetserver = MXS_STRDUP_A(server->unique_name); break; } } @@ -2240,9 +2078,7 @@ static int routeQuery(ROUTER* instance, if (TARGET_IS_ANY(route_target)) { /**No valid backends alive*/ - MXS_INFO("schemarouter: No backends are running"); - MXS_ERROR("Schemarouter: Failed to route query, " - "no backends are available."); + MXS_ERROR("Schemarouter: Failed to route query, no backends are available."); rses_end_locked_router_action(router_cli_ses); ret = 0; goto retblock; @@ -2280,8 +2116,8 @@ static int routeQuery(ROUTER* instance, scur = &bref->bref_sescmd_cur; MXS_INFO("Route query to \t%s:%d <", - bref->bref_backend->backend_server->name, - bref->bref_backend->backend_server->port); + bref->bref_backend->server->name, + bref->bref_backend->server->port); /** * Store current stmt if execution of previous session command * haven't completed yet. Note that according to MySQL protocol @@ -2310,7 +2146,6 @@ static int routeQuery(ROUTER* instance, bref = get_bref_from_dcb(router_cli_ses, target_dcb); bref_set_state(bref, BREF_QUERY_ACTIVE); bref_set_state(bref, BREF_WAITING_RESULT); - atomic_add(&bref->bref_backend->stats.queries, 1); } else { @@ -2400,17 +2235,6 @@ static void diagnostic(ROUTER *instance, DCB *dcb) 100.0 * ((double)router->stats.n_sescmd / (double)router->stats.n_queries) : 0.0; - dcb_printf(dcb, "\33[1;4m%-16s%-16s%-16s\33[0m\n", "Server", "Queries", "State"); - for (i = 0; router->servers[i]; i++) - { - dcb_printf(dcb, "%-16s%-16d%-16s\n", - router->servers[i]->backend_server->unique_name, - router->servers[i]->stats.queries, - SERVER_IS_RUNNING(router->servers[i]->backend_server) ? - "\33[30;42mRUNNING\33[0m" : - "\33[30;41mDOWN\33[0m"); - } - /** Session command statistics */ dcb_printf(dcb, "\n\33[1;4mSession Commands\33[0m\n"); dcb_printf(dcb, "Total number of queries: %d\n", @@ -2514,7 +2338,7 @@ static void clientReply(ROUTER* instance, MXS_DEBUG("schemarouter: Reply from [%s] session [%p]" " mapping [%s] queries queued [%s]", - bref->bref_backend->backend_server->unique_name, + bref->bref_backend->server->unique_name, router_cli_ses->rses_client_dcb->session, router_cli_ses->init & INIT_MAPPING ? "true" : "false", router_cli_ses->queue == NULL ? "none" : @@ -2641,8 +2465,8 @@ static void clientReply(ROUTER* instance, MXS_ERROR("Failed to execute %s in %s:%d. %s %s", cmdstr, - bref->bref_backend->backend_server->name, - bref->bref_backend->backend_server->port, + bref->bref_backend->server->name, + bref->bref_backend->server->port, err, replystr); @@ -2709,8 +2533,8 @@ static void clientReply(ROUTER* instance, MXS_INFO("Backend %s:%d processed reply and starts to execute " "active cursor.", - bref->bref_backend->backend_server->name, - bref->bref_backend->backend_server->port); + bref->bref_backend->server->name, + bref->bref_backend->server->port); execute_sescmd_in_backend(bref); } @@ -2757,44 +2581,41 @@ static void clientReply(ROUTER* instance, /** Compare number of connections from this router in backend servers */ int bref_cmp_router_conn(const void* bref1, const void* bref2) { - BACKEND* b1 = ((backend_ref_t *)bref1)->bref_backend; - BACKEND* b2 = ((backend_ref_t *)bref2)->bref_backend; + SERVER_REF* b1 = ((backend_ref_t *)bref1)->bref_backend; + SERVER_REF* b2 = ((backend_ref_t *)bref2)->bref_backend; - return ((1000 * b1->backend_conn_count) / b1->weight) - - ((1000 * b2->backend_conn_count) / b2->weight); + return ((1000 * b1->connections) / b1->weight) + - ((1000 * b2->connections) / b2->weight); } /** Compare number of global connections in backend servers */ int bref_cmp_global_conn(const void* bref1, const void* bref2) { - BACKEND* b1 = ((backend_ref_t *)bref1)->bref_backend; - BACKEND* b2 = ((backend_ref_t *)bref2)->bref_backend; + SERVER_REF* b1 = ((backend_ref_t *)bref1)->bref_backend; + SERVER_REF* b2 = ((backend_ref_t *)bref2)->bref_backend; - return ((1000 * b1->backend_server->stats.n_current) / b1->weight) - - ((1000 * b2->backend_server->stats.n_current) / b2->weight); + return ((1000 * b1->server->stats.n_current) / b1->weight) + - ((1000 * b2->server->stats.n_current) / b2->weight); } /** Compare replication lag between backend servers */ int bref_cmp_behind_master(const void* bref1, const void* bref2) { - BACKEND* b1 = ((backend_ref_t *)bref1)->bref_backend; - BACKEND* b2 = ((backend_ref_t *)bref2)->bref_backend; + SERVER_REF* b1 = ((backend_ref_t *)bref1)->bref_backend; + SERVER_REF* b2 = ((backend_ref_t *)bref2)->bref_backend; - return ((b1->backend_server->rlag < b2->backend_server->rlag) ? -1 : - ((b1->backend_server->rlag > b2->backend_server->rlag) ? 1 : 0)); + return b1->server->rlag - b2->server->rlag; } /** Compare number of current operations in backend servers */ int bref_cmp_current_load(const void* bref1, const void* bref2) { - SERVER* s1 = ((backend_ref_t *)bref1)->bref_backend->backend_server; - SERVER* s2 = ((backend_ref_t *)bref2)->bref_backend->backend_server; - BACKEND* b1 = ((backend_ref_t *)bref1)->bref_backend; - BACKEND* b2 = ((backend_ref_t *)bref2)->bref_backend; + SERVER_REF* b1 = ((backend_ref_t *)bref1)->bref_backend; + SERVER_REF* b2 = ((backend_ref_t *)bref2)->bref_backend; - return ((1000 * s1->stats.n_current_ops) - b1->weight) - - ((1000 * s2->stats.n_current_ops) - b2->weight); + return ((1000 * b1->server->stats.n_current_ops) - b1->weight) + - ((1000 * b2->server->stats.n_current_ops) - b2->weight); } static void bref_clear_state(backend_ref_t* bref, bref_state_t state) @@ -2823,14 +2644,14 @@ static void bref_clear_state(backend_ref_t* bref, bref_state_t state) else { /** Decrease global operation count */ - prev2 = atomic_add(&bref->bref_backend->backend_server->stats.n_current_ops, -1); + prev2 = atomic_add(&bref->bref_backend->server->stats.n_current_ops, -1); ss_dassert(prev2 > 0); if (prev2 <= 0) { MXS_ERROR("[%s] Error: negative current operation count in backend %s:%u", __FUNCTION__, - bref->bref_backend->backend_server->name, - bref->bref_backend->backend_server->port); + bref->bref_backend->server->name, + bref->bref_backend->server->port); } } } @@ -2860,18 +2681,18 @@ static void bref_set_state(backend_ref_t* bref, bref_state_t state) MXS_ERROR("[%s] Error: negative number of connections waiting " "for results in backend %s:%u", __FUNCTION__, - bref->bref_backend->backend_server->name, - bref->bref_backend->backend_server->port); + bref->bref_backend->server->name, + bref->bref_backend->server->port); } /** Increase global operation count */ - prev2 = atomic_add(&bref->bref_backend->backend_server->stats.n_current_ops, 1); + prev2 = atomic_add(&bref->bref_backend->server->stats.n_current_ops, 1); ss_dassert(prev2 >= 0); if (prev2 < 0) { MXS_ERROR("[%s] Error: negative current operation count in backend %s:%u", __FUNCTION__, - bref->bref_backend->backend_server->name, - bref->bref_backend->backend_server->port); + bref->bref_backend->server->name, + bref->bref_backend->server->port); } } } @@ -2941,14 +2762,14 @@ static bool connect_backend_servers(backend_ref_t* backend_ref, for (i = 0; i < router_nservers; i++) { - BACKEND* b = backend_ref[i].bref_backend; + SERVER_REF* b = backend_ref[i].bref_backend; MXS_INFO("MaxScale connections : %d (%d) in \t%s:%d %s", - b->backend_conn_count, - b->backend_server->stats.n_current, - b->backend_server->name, - b->backend_server->port, - STRSRVSTATUS(b->backend_server)); + b->connections, + b->server->stats.n_current, + b->server->name, + b->server->port, + STRSRVSTATUS(b->server)); } } /*< log only */ /** @@ -2957,9 +2778,9 @@ static bool connect_backend_servers(backend_ref_t* backend_ref, */ for (i = 0; i < router_nservers; i++) { - BACKEND* b = backend_ref[i].bref_backend; + SERVER_REF* b = backend_ref[i].bref_backend; - if (SERVER_IS_RUNNING(b->backend_server)) + if (SERVER_IS_RUNNING(b->server)) { servers_found += 1; @@ -2971,9 +2792,9 @@ static bool connect_backend_servers(backend_ref_t* backend_ref, /** New server connection */ else { - backend_ref[i].bref_dcb = dcb_connect(b->backend_server, + backend_ref[i].bref_dcb = dcb_connect(b->server, session, - b->backend_server->protocol); + b->server->protocol); if (backend_ref[i].bref_dcb != NULL) { @@ -3000,7 +2821,7 @@ static bool connect_backend_servers(backend_ref_t* backend_ref, * But decreased in the calling function * of dcb_close. */ - atomic_add(&b->backend_conn_count, 1); + atomic_add(&b->connections, 1); dcb_add_callback(backend_ref[i].bref_dcb, DCB_REASON_NOT_RESPONDING, @@ -3012,8 +2833,8 @@ static bool connect_backend_servers(backend_ref_t* backend_ref, succp = false; MXS_ERROR("Unable to establish " "connection with slave %s:%d", - b->backend_server->name, - b->backend_server->port); + b->server->name, + b->server->port); /* handle connect error */ break; } @@ -3047,14 +2868,14 @@ static bool connect_backend_servers(backend_ref_t* backend_ref, { for (i = 0; i < router_nservers; i++) { - BACKEND* b = backend_ref[i].bref_backend; + SERVER_REF* b = backend_ref[i].bref_backend; if (BREF_IS_IN_USE((&backend_ref[i]))) { MXS_INFO("Connected %s in \t%s:%d", - STRSRVSTATUS(b->backend_server), - b->backend_server->name, - b->backend_server->port); + STRSRVSTATUS(b->server), + b->server->name, + b->server->port); } } /* for */ } @@ -3437,28 +3258,7 @@ static bool execute_sescmd_in_backend(backend_ref_t* backend_ref) /** Cursor is left active when function returns. */ sescmd_cursor_set_active(scur, true); } -#if defined(SS_DEBUG) - if (MXS_LOG_PRIORITY_IS_ENABLED(LOG_INFO)) - { - tracelog_routed_query(scur->scmd_cur_rses, - "execute_sescmd_in_backend", - backend_ref, - sescmd_cursor_clone_querybuf(scur)); - } - { - GWBUF* tmpbuf = sescmd_cursor_clone_querybuf(scur); - uint8_t* ptr = GWBUF_DATA(tmpbuf); - unsigned char cmd = MYSQL_GET_COMMAND(ptr); - - MXS_DEBUG("%lu [execute_sescmd_in_backend] Just before write, fd " - "%d : cmd %s.", - pthread_self(), - dcb->fd, - STRPACKETTYPE(cmd)); - gwbuf_free(tmpbuf); - } -#endif /*< SS_DEBUG */ switch (scur->scmd_cur_cmd->my_sescmd_packet_type) { case MYSQL_COM_CHANGE_USER: @@ -3567,75 +3367,6 @@ static rses_property_t* mysql_sescmd_get_property(mysql_sescmd_t* scmd) return scmd->my_sescmd_prop; } -static void tracelog_routed_query(ROUTER_CLIENT_SES* rses, - char* funcname, - backend_ref_t* bref, - GWBUF* buf) -{ - uint8_t* packet = GWBUF_DATA(buf); - unsigned char packet_type = packet[4]; - size_t len; - size_t buflen = GWBUF_LENGTH(buf); - char* querystr; - char* startpos = (char *)&packet[5]; - - CHK_BACKEND_REF(bref); - ss_debug(BACKEND *b = bref->bref_backend); - CHK_BACKEND(b); - ss_debug(DCB *dcb = bref->bref_dcb); - CHK_DCB(dcb); - - ss_debug(backend_type_t be_type = BACKEND_TYPE(b)); - - if (GWBUF_IS_TYPE_MYSQL(buf)) - { - len = packet[0]; - len += 256 * packet[1]; - len += 256 * 256 * packet[2]; - - if (packet_type == '\x03') - { - querystr = (char *)MXS_MALLOC(len); - MXS_ABORT_IF_NULL(querystr); - memcpy(querystr, startpos, len - 1); - querystr[len - 1] = '\0'; - MXS_DEBUG("%lu [%s] %d bytes long buf, \"%s\" -> %s:%d %s dcb %p", - pthread_self(), - funcname, - (int)buflen, - querystr, - b->backend_server->name, - b->backend_server->port, - STRBETYPE(be_type), - dcb); - MXS_FREE(querystr); - } - else if (packet_type == '\x22' || - packet_type == 0x22 || - packet_type == '\x26' || - packet_type == 0x26 || - true) - { - querystr = (char *)MXS_MALLOC(len); - MXS_ABORT_IF_NULL(querystr); - memcpy(querystr, startpos, len - 1); - querystr[len - 1] = '\0'; - MXS_DEBUG("%lu [%s] %d bytes long buf, \"%s\" -> %s:%d %s dcb %p", - pthread_self(), - funcname, - (int)buflen, - querystr, - b->backend_server->name, - b->backend_server->port, - STRBETYPE(be_type), - dcb); - MXS_FREE(querystr); - } - } - gwbuf_free(buf); -} - - /** * Return RCAP_TYPE_STMT_INPUT. */ @@ -3699,17 +3430,16 @@ static bool route_session_write(ROUTER_CLIENT_SES* router_cli_ses, if (MXS_LOG_PRIORITY_IS_ENABLED(LOG_INFO)) { MXS_INFO("Route query to %s\t%s:%d%s", - (SERVER_IS_MASTER(backend_ref[i].bref_backend->backend_server) ? + (SERVER_IS_MASTER(backend_ref[i].bref_backend->server) ? "master" : "slave"), - backend_ref[i].bref_backend->backend_server->name, - backend_ref[i].bref_backend->backend_server->port, + backend_ref[i].bref_backend->server->name, + backend_ref[i].bref_backend->server->port, (i + 1 == router_cli_ses->rses_nbackends ? " <" : "")); } if (BREF_IS_IN_USE((&backend_ref[i]))) { rc = dcb->func.write(dcb, gwbuf_clone(querybuf)); - atomic_add(&backend_ref[i].bref_backend->stats.queries, 1); if (rc != 1) { succp = false; @@ -3807,10 +3537,10 @@ static bool route_session_write(ROUTER_CLIENT_SES* router_cli_ses, if (MXS_LOG_PRIORITY_IS_ENABLED(LOG_INFO)) { MXS_INFO("Route query to %s\t%s:%d%s", - (SERVER_IS_MASTER(backend_ref[i].bref_backend->backend_server) ? + (SERVER_IS_MASTER(backend_ref[i].bref_backend->server) ? "master" : "slave"), - backend_ref[i].bref_backend->backend_server->name, - backend_ref[i].bref_backend->backend_server->port, + backend_ref[i].bref_backend->server->name, + backend_ref[i].bref_backend->server->port, (i + 1 == router_cli_ses->rses_nbackends ? " <" : "")); } @@ -3832,8 +3562,8 @@ static bool route_session_write(ROUTER_CLIENT_SES* router_cli_ses, succp = true; MXS_INFO("Backend %s:%d already executing sescmd.", - backend_ref[i].bref_backend->backend_server->name, - backend_ref[i].bref_backend->backend_server->port); + backend_ref[i].bref_backend->server->name, + backend_ref[i].bref_backend->server->port); } else { @@ -3843,12 +3573,8 @@ static bool route_session_write(ROUTER_CLIENT_SES* router_cli_ses, { MXS_ERROR("Failed to execute session " "command in %s:%d", - backend_ref[i].bref_backend->backend_server->name, - backend_ref[i].bref_backend->backend_server->port); - } - else - { - atomic_add(&backend_ref[i].bref_backend->stats.queries, 1); + backend_ref[i].bref_backend->server->name, + backend_ref[i].bref_backend->server->port); } } } @@ -4030,7 +3756,6 @@ static bool handle_error_new_connection(ROUTER_INSTANCE* inst, GWBUF* errmsg) { SESSION* ses; - int router_nservers, i; unsigned char cmd = *((unsigned char*)errmsg->start + 4); backend_ref_t* bref; @@ -4087,13 +3812,12 @@ static bool handle_error_new_connection(ROUTER_INSTANCE* inst, &router_handle_state_switch, (void *)bref); - router_nservers = router_get_servercount(inst); /** * Try to get replacement slave or at least the minimum * number of slave connections for router session. */ succp = connect_backend_servers(rses->rses_backend_ref, - router_nservers, + rses->rses_nbackends, ses, inst); @@ -4108,24 +3832,6 @@ return_succp: return succp; } -/** - * Count the number of servers. - * @param inst Router instance - * @return Number of servers - */ -static int router_get_servercount(ROUTER_INSTANCE* inst) -{ - int router_nservers = 0; - BACKEND** b = inst->servers; - /** count servers */ - while (*(b++) != NULL) - { - router_nservers++; - } - - return router_nservers; -} - /** * Finds out if there is a backend reference pointing at the DCB given as * parameter. @@ -4190,7 +3896,7 @@ static int router_handle_state_switch(DCB* dcb, bref = (backend_ref_t *) data; CHK_BACKEND_REF(bref); - srv = bref->bref_backend->backend_server; + srv = bref->bref_backend->server; if (SERVER_IS_RUNNING(srv)) { @@ -4200,7 +3906,7 @@ static int router_handle_state_switch(DCB* dcb, switch (reason) { case DCB_REASON_NOT_RESPONDING: - atomic_add(&bref->bref_backend->backend_conn_count, -1); + atomic_add(&bref->bref_backend->connections, -1); MXS_INFO("schemarouter: server %s not responding", srv->unique_name); dcb->func.hangup(dcb); break; @@ -4478,7 +4184,7 @@ int inspect_backend_mapping_states(ROUTER_CLIENT_SES *router_cli_ses, { router_cli_ses->rses_backend_ref[i].bref_mapped = true; MXS_DEBUG("schemarouter: Received SHOW DATABASES reply from %s for session %p", - router_cli_ses->rses_backend_ref[i].bref_backend->backend_server->unique_name, + router_cli_ses->rses_backend_ref[i].bref_backend->server->unique_name, router_cli_ses->rses_client_dcb->session); } else if (rc == SHOWDB_PARTIAL_RESPONSE) @@ -4486,7 +4192,7 @@ int inspect_backend_mapping_states(ROUTER_CLIENT_SES *router_cli_ses, bref->map_queue = writebuf; writebuf = NULL; MXS_DEBUG("schemarouter: Received partial SHOW DATABASES reply from %s for session %p", - router_cli_ses->rses_backend_ref[i].bref_backend->backend_server->unique_name, + router_cli_ses->rses_backend_ref[i].bref_backend->server->unique_name, router_cli_ses->rses_client_dcb->session); } else @@ -4539,7 +4245,7 @@ int inspect_backend_mapping_states(ROUTER_CLIENT_SES *router_cli_ses, { mapped = false; MXS_DEBUG("schemarouter: Still waiting for reply to SHOW DATABASES from %s for session %p", - bkrf[i].bref_backend->backend_server->unique_name, + bkrf[i].bref_backend->server->unique_name, router_cli_ses->rses_client_dcb->session); } } diff --git a/server/modules/routing/schemarouter/schemarouter.h b/server/modules/routing/schemarouter/schemarouter.h index 88dd01eda..c3cab3c28 100644 --- a/server/modules/routing/schemarouter/schemarouter.h +++ b/server/modules/routing/schemarouter/schemarouter.h @@ -261,7 +261,7 @@ typedef struct backend_ref_st #endif int n_mapping_eof; GWBUF* map_queue; - BACKEND* bref_backend; /*< Backend server */ + SERVER_REF* bref_backend; /*< Backend server */ DCB* bref_dcb; /*< Backend DCB */ bref_state_t bref_state; /*< State of the backend */ bool bref_mapped; /*< Whether the backend has been mapped */ @@ -357,8 +357,6 @@ typedef struct router_instance SERVICE* service; /*< Pointer to service */ ROUTER_CLIENT_SES* connections; /*< List of client connections */ SPINLOCK lock; /*< Lock for the instance data */ - BACKEND** servers; /*< Backend servers */ - BACKEND* master; /*< NULL or pointer */ schemarouter_config_t schemarouter_config; /*< expanded config info from SERVICE */ int schemarouter_version;/*< version number for router's config */ unsigned int bitmask; /*< Bitmask to apply to server->status */ From 88dca05dc7f3ccfcde4ca02648a1274ac286a9d4 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 4 Nov 2016 08:36:30 +0200 Subject: [PATCH 175/215] Remove unused code from readwritesplit There was a lot of extra debug logging behind unused defines. These were never enabled and some of the code never worked. --- .../routing/readwritesplit/readwritesplit.c | 126 ++---------------- .../routing/readwritesplit/readwritesplit.h | 20 --- .../routing/readwritesplit/rwsplit_mysql.c | 60 --------- .../readwritesplit/rwsplit_route_stmt.c | 6 - 4 files changed, 9 insertions(+), 203 deletions(-) diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index 65d9af518..d130d6321 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -307,8 +307,7 @@ static void *newSession(ROUTER *router_inst, SESSION *session) if (client_rses == NULL) { - ss_dassert(false); - goto return_rses; + return NULL; } #if defined(SS_DEBUG) client_rses->rses_chk_top = CHK_NUM_ROUTER_SES; @@ -348,7 +347,8 @@ static void *newSession(ROUTER *router_inst, SESSION *session) if (!have_enough_servers(&client_rses, min_nservers, router_nservers, router)) { - goto return_rses; + MXS_FREE(client_rses); + return NULL; } /** * Create backend reference objects for this session. @@ -359,9 +359,7 @@ static void *newSession(ROUTER *router_inst, SESSION *session) { /** log this */ MXS_FREE(client_rses); - MXS_FREE(backend_ref); - client_rses = NULL; - goto return_rses; + return NULL; } /** * Initialize backend references with BACKEND ptr. @@ -413,8 +411,7 @@ static void *newSession(ROUTER *router_inst, SESSION *session) { MXS_FREE(client_rses->rses_backend_ref); MXS_FREE(client_rses); - client_rses = NULL; - goto return_rses; + return NULL; } succp = select_connect_backend_servers(&master_ref, backend_ref, router_nservers, max_nslaves, max_slave_rlag, @@ -432,8 +429,7 @@ static void *newSession(ROUTER *router_inst, SESSION *session) { MXS_FREE(client_rses->rses_backend_ref); MXS_FREE(client_rses); - client_rses = NULL; - goto return_rses; + return NULL; } /** Copy backend pointers to router session. */ @@ -458,13 +454,6 @@ static void *newSession(ROUTER *router_inst, SESSION *session) router->connections = client_rses; spinlock_release(&router->lock); -return_rses: -#if defined(SS_DEBUG) - if (client_rses != NULL) - { - CHK_CLIENT_RSES(client_rses); - } -#endif return (void *)client_rses; } @@ -520,16 +509,8 @@ static void closeSession(ROUTER *instance, void *router_session) if (BREF_IS_IN_USE(bref)) { CHK_DCB(dcb); -#if defined(SS_DEBUG) - /** - * session must be moved to SESSION_STATE_STOPPING state before - * router session is closed. - */ - if (dcb->session != NULL) - { - ss_dassert(dcb->session->state == SESSION_STATE_STOPPING); - } -#endif + ss_dassert(dcb->session->state == SESSION_STATE_STOPPING); + /** Clean operation counter in bref and in SERVER */ if (BREF_IS_WAITING_RESULT(bref)) { @@ -764,7 +745,7 @@ static void clientReply(ROUTER *instance, void *router_session, GWBUF *writebuf, */ if (!rses_begin_locked_router_action(router_cli_ses)) { - print_error_packet(router_cli_ses, writebuf, backend_dcb); + gwbuf_free(writebuf); goto lock_failed; } /** Holding lock ensures that router session remains open */ @@ -1795,57 +1776,6 @@ static bool have_enough_servers(ROUTER_CLIENT_SES **p_rses, const int min_nsrv, return succp; } -#if defined(PREP_STMT_CACHING) -#define MAX_STMT_LEN 1024 - -static prep_stmt_t *prep_stmt_init(prep_stmt_type_t type, void *id) -{ - prep_stmt_t *pstmt; - - pstmt = (prep_stmt_t *)MXS_CALLOC(1, sizeof(prep_stmt_t)); - - if (pstmt != NULL) - { -#if defined(SS_DEBUG) - pstmt->pstmt_chk_top = CHK_NUM_PREP_STMT; - pstmt->pstmt_chk_tail = CHK_NUM_PREP_STMT; -#endif - pstmt->pstmt_state = PREP_STMT_ALLOC; - pstmt->pstmt_type = type; - - if (type == PREP_STMT_NAME) - { - pstmt->pstmt_id.name = strndup((char *)id, MAX_STMT_LEN); - } - else - { - pstmt->pstmt_id.seq = 0; - } - } - CHK_PREP_STMT(pstmt); - return pstmt; -} - -static void prep_stmt_done(prep_stmt_t *pstmt) -{ - CHK_PREP_STMT(pstmt); - - if (pstmt->pstmt_type == PREP_STMT_NAME) - { - MXS_FREE(pstmt->pstmt_id.name); - } - MXS_FREE(pstmt); -} - -static bool prep_stmt_drop(prep_stmt_t *pstmt) -{ - CHK_PREP_STMT(pstmt); - - pstmt->pstmt_state = PREP_STMT_DROPPED; - return true; -} -#endif /*< PREP_STMT_CACHING */ - /** * @brief Refresh the instance by the given parameter value. * @@ -1950,44 +1880,6 @@ static void refreshInstance(ROUTER_INSTANCE *router, } param = param->next; } - -#if defined(NOT_USED) /*< can't read monitor config parameters */ - if ((*router->servers)->backend_server->rlag == -2) - { - rlag_enabled = false; - } - else - { - rlag_enabled = true; - } - /** - * If replication lag detection is not enabled the measure can't be - * used in slave selection. - */ - if (!rlag_enabled) - { - if (rlag_limited) - { - MXS_WARNING("Configuration Failed, max_slave_replication_lag " - "is set to %d,\n\t\t but detect_replication_lag " - "is not enabled. Replication lag will not be checked.", - router->rwsplit_config.rw_max_slave_replication_lag); - } - - if (router->rwsplit_config.rw_slave_select_criteria == - LEAST_BEHIND_MASTER) - { - MXS_WARNING("Configuration Failed, router option " - "\n\t\t slave_selection_criteria=LEAST_BEHIND_MASTER " - "is specified, but detect_replication_lag " - "is not enabled.\n\t\t " - "slave_selection_criteria=%s will be used instead.", - STRCRITERIA(DEFAULT_CRITERIA)); - - router->rwsplit_config.rw_slave_select_criteria = DEFAULT_CRITERIA; - } - } -#endif /*< NOT_USED */ } /* diff --git a/server/modules/routing/readwritesplit/readwritesplit.h b/server/modules/routing/readwritesplit/readwritesplit.h index 50bedcf40..e300edf31 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.h +++ b/server/modules/routing/readwritesplit/readwritesplit.h @@ -32,26 +32,6 @@ MXS_BEGIN_DECLS -#undef PREP_STMT_CACHING - -#if defined(PREP_STMT_CACHING) - -typedef enum prep_stmt_type -{ - PREP_STMT_NAME, - PREP_STMT_ID -} prep_stmt_type_t; - -typedef enum prep_stmt_state -{ - PREP_STMT_ALLOC, - PREP_STMT_SENT, - PREP_STMT_RECV, - PREP_STMT_DROPPED -} prep_stmt_state_t; - -#endif /*< PREP_STMT_CACHING */ - typedef enum bref_state { BREF_IN_USE = 0x01, diff --git a/server/modules/routing/readwritesplit/rwsplit_mysql.c b/server/modules/routing/readwritesplit/rwsplit_mysql.c index 6dcdd50d5..6f1b31410 100644 --- a/server/modules/routing/readwritesplit/rwsplit_mysql.c +++ b/server/modules/routing/readwritesplit/rwsplit_mysql.c @@ -366,66 +366,6 @@ void live_session_reply(GWBUF **querybuf, ROUTER_CLIENT_SES *rses) } } -/* - * Uses MySQL specific mechanisms - */ -/** - * @brief Write an error message to the log for session lock failure - * - * This happens when processing a client reply and the session cannot be - * locked. - * - * @param rses Router session - * @param buf Query buffer containing reply data - * @param dcb The backend DCB that sent the reply - */ -void print_error_packet(ROUTER_CLIENT_SES *rses, GWBUF *buf, DCB *dcb) -{ -#if defined(SS_DEBUG) - if (GWBUF_IS_TYPE_MYSQL(buf)) - { - while (gwbuf_length(buf) > 0) - { - /** - * This works with MySQL protocol only ! - * Protocol specific packet print functions would be nice. - */ - uint8_t *ptr = GWBUF_DATA(buf); - size_t len = MYSQL_GET_PACKET_LEN(ptr); - - if (MYSQL_GET_COMMAND(ptr) == 0xff) - { - SERVER *srv = NULL; - backend_ref_t *bref = rses->rses_backend_ref; - int i; - char *bufstr; - - for (i = 0; i < rses->rses_nbackends; i++) - { - if (bref[i].bref_dcb == dcb) - { - srv = bref[i].ref->server; - } - } - ss_dassert(srv != NULL); - char *str = (char *)&ptr[7]; - bufstr = strndup(str, len - 3); - - MXS_ERROR("Backend server %s:%d responded with " - "error : %s", - srv->name, srv->port, bufstr); - MXS_FREE(bufstr); - } - buf = gwbuf_consume(buf, len + 4); - } - } - else - { - gwbuf_free(buf); - } -#endif /*< SS_DEBUG */ -} - /* * Uses MySQL specific mechanisms */ diff --git a/server/modules/routing/readwritesplit/rwsplit_route_stmt.c b/server/modules/routing/readwritesplit/rwsplit_route_stmt.c index ae6e42e1f..4ab38b8ed 100644 --- a/server/modules/routing/readwritesplit/rwsplit_route_stmt.c +++ b/server/modules/routing/readwritesplit/rwsplit_route_stmt.c @@ -891,9 +891,6 @@ route_target_t get_route_target(ROUTER_CLIENT_SES *rses, hint = hint->next; } /*< while (hint != NULL) */ -#if defined(SS_EXTRA_DEBUG) - MXS_INFO("Selected target \"%s\"", STRTARGET(target)); -#endif return target; } @@ -1098,9 +1095,6 @@ bool handle_slave_is_target(ROUTER_INSTANCE *inst, ROUTER_CLIENT_SES *rses, */ if (rwsplit_get_dcb(target_dcb, rses, BE_SLAVE, NULL, rlag_max)) { -#if defined(SS_EXTRA_DEBUG) - MXS_INFO("Found DCB for slave."); -#endif atomic_add(&inst->stats.n_slave, 1); return true; } From ea0dcea5d60972f3107ba2996bd2fa940dbf027d Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 4 Nov 2016 09:11:32 +0200 Subject: [PATCH 176/215] Clean up instance and session lifecycle functions The createInstace, newSession, closeSession and freeSession functions were cleaned up and reorganized to be a bit clearer for the reader. Removed unnecessary comments and replaced them with ones that explain what's happening in the code. Removed unused linked lists from both sessions and instances and replaced them with better alternatives. This should improve performance since new session don't have to acquire the instance level lock to put themselves into the session list. --- .../routing/readwritesplit/readwritesplit.c | 425 +++++++----------- .../routing/readwritesplit/readwritesplit.h | 2 - 2 files changed, 156 insertions(+), 271 deletions(-) diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index d130d6321..8653f2494 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -43,7 +43,7 @@ MODULE_INFO info = * by the entry point functions. Some of these are used by functions in other * modules of the read write split router, others are used only within this * module. - * + * * @verbatim * Revision History * @@ -107,13 +107,6 @@ static ROUTER_OBJECT MyObject = getCapabilities }; -/* - * A couple of static variables that are used throughout the router - */ - -static SPINLOCK instlock; -static ROUTER_INSTANCE *instances; - /* * Declaration of functions that are used only within this module, and are * not part of the API. @@ -129,9 +122,9 @@ static void handle_error_reply_client(SESSION *ses, ROUTER_CLIENT_SES *rses, static bool handle_error_new_connection(ROUTER_INSTANCE *inst, ROUTER_CLIENT_SES **rses, DCB *backend_dcb, GWBUF *errmsg); -static bool have_enough_servers(ROUTER_CLIENT_SES **p_rses, const int min_nsrv, +static bool have_enough_servers(ROUTER_CLIENT_SES *rses, const int min_nsrv, int router_nsrv, ROUTER_INSTANCE *router); - +static bool create_backends(ROUTER_CLIENT_SES *rses, backend_ref_t** dest, int* n_backend); /** * Implementation of the mandatory version entry point * @@ -149,8 +142,6 @@ char *version() void ModuleInit() { MXS_NOTICE("Initializing statement-based read/write split router module."); - spinlock_init(&instlock); - instances = NULL; } /** @@ -172,7 +163,7 @@ ROUTER_OBJECT *GetModuleObject() /** * @brief Create an instance of the read/write router (API). - * + * * Create an instance of read/write statement router within the MaxScale. One * instance of the router is required for each service that is defined in the * configuration as using this router. One instance of the router will handle @@ -261,15 +252,6 @@ static ROUTER *createInstance(SERVICE *service, char **options) { refreshInstance(router, param); } - /** - * We have completed the creation of the router data, so now - * insert this router into the linked list of routers - * that have been created with this module. - */ - spinlock_acquire(&instlock); - router->next = instances; - instances = router; - spinlock_release(&instlock); return (ROUTER *)router; } @@ -277,14 +259,14 @@ static ROUTER *createInstance(SERVICE *service, char **options) /** * @brief Associate a new session with this instance of the router (API). * - * The session is used to store all the data required by the router for a - * particular client connection. The instance of the router that relates to a + * The session is used to store all the data required by the router for a + * particular client connection. The instance of the router that relates to a * particular service is passed as the first parameter. The second parameter is * the session that has been created in response to the request from a client * for a connection. The passed session contains generic information; this * function creates the session structure that holds router specific data. * There is often a one to one relationship between sessions and router - * sessions, although it is possible to create configurations where a + * sessions, although it is possible to create configurations where a * connection is handled by multiple routers, one after another. * * @param instance The router instance data @@ -293,17 +275,8 @@ static ROUTER *createInstance(SERVICE *service, char **options) */ static void *newSession(ROUTER *router_inst, SESSION *session) { - backend_ref_t *backend_ref; /*< array of backend references (DCB,BACKEND,cursor) */ - backend_ref_t *master_ref = NULL; /*< pointer to selected master */ - ROUTER_CLIENT_SES *client_rses = NULL; ROUTER_INSTANCE *router = (ROUTER_INSTANCE *)router_inst; - bool succp; - int router_nservers = 0; /*< # of servers in total */ - int max_nslaves; /*< max # of slaves used in this session */ - int max_slave_rlag; /*< max allowed replication lag for any slave */ - const int min_nservers = 1; /*< hard-coded for now */ - - client_rses = (ROUTER_CLIENT_SES *)MXS_CALLOC(1, sizeof(ROUTER_CLIENT_SES)); + ROUTER_CLIENT_SES *client_rses = (ROUTER_CLIENT_SES *)MXS_CALLOC(1, sizeof(ROUTER_CLIENT_SES)); if (client_rses == NULL) { @@ -316,117 +289,50 @@ static void *newSession(ROUTER *router_inst, SESSION *session) client_rses->router = router; client_rses->client_dcb = session->client_dcb; - /** - * If service config has been changed, reload config from service to - * router instance first. - */ - spinlock_acquire(&router->lock); - - if (router->service->svc_config_version > router->rwsplit_version) - { - /** re-read all parameters to rwsplit config structure */ - refreshInstance(router, NULL); /*< scan through all parameters */ - /** increment rwsplit router's config version number */ - router->rwsplit_version = router->service->svc_config_version; - /** Read options */ - rwsplit_process_router_options(router, router->service->routerOptions); - } - /** Copy config struct from router instance */ - memcpy(&client_rses->rses_config, &router->rwsplit_config, sizeof(rwsplit_config_t)); - - spinlock_release(&router->lock); - /** - * Set defaults to session variables. - */ client_rses->rses_autocommit_enabled = true; client_rses->rses_transaction_active = false; client_rses->have_tmp_tables = false; client_rses->forced_node = NULL; + spinlock_init(&client_rses->rses_lock); + memcpy(&client_rses->rses_config, &router->rwsplit_config, sizeof(client_rses->rses_config)); - router_nservers = router->service->n_dbref; + int router_nservers = router->service->n_dbref; + const int min_nservers = 1; /*< hard-coded for now */ - if (!have_enough_servers(&client_rses, min_nservers, router_nservers, router)) + if (!have_enough_servers(client_rses, min_nservers, router_nservers, router)) { MXS_FREE(client_rses); return NULL; } + /** * Create backend reference objects for this session. */ - backend_ref = (backend_ref_t *)MXS_CALLOC(1, router_nservers * sizeof(backend_ref_t)); + backend_ref_t *backend_ref; - if (backend_ref == NULL) + if (!create_backends(client_rses, &backend_ref, &router_nservers)) { - /** log this */ MXS_FREE(client_rses); return NULL; } - /** - * Initialize backend references with BACKEND ptr. - * Initialize session command cursors for each backend reference. - */ - int i = 0; - for (SERVER_REF *sref = router->service->dbref; sref && i < router_nservers; sref = sref->next) - { - if (sref->active) - { -#if defined(SS_DEBUG) - backend_ref[i].bref_chk_top = CHK_NUM_BACKEND_REF; - backend_ref[i].bref_chk_tail = CHK_NUM_BACKEND_REF; - backend_ref[i].bref_sescmd_cur.scmd_cur_chk_top = CHK_NUM_SESCMD_CUR; - backend_ref[i].bref_sescmd_cur.scmd_cur_chk_tail = CHK_NUM_SESCMD_CUR; -#endif - backend_ref[i].bref_state = 0; - backend_ref[i].ref = sref; - /** store pointers to sescmd list to both cursors */ - backend_ref[i].bref_sescmd_cur.scmd_cur_rses = client_rses; - backend_ref[i].bref_sescmd_cur.scmd_cur_active = false; - backend_ref[i].bref_sescmd_cur.scmd_cur_ptr_property = - &client_rses->rses_properties[RSES_PROP_TYPE_SESCMD]; - backend_ref[i].bref_sescmd_cur.scmd_cur_cmd = NULL; - i++; - } - } - if (i < router_nservers) - { - /** The service reported more servers than we took into use */ - router_nservers = i; - } + int max_nslaves = rses_get_max_slavecount(client_rses, router_nservers); + int max_slave_rlag = rses_get_max_replication_lag(client_rses); - max_nslaves = rses_get_max_slavecount(client_rses, router_nservers); - max_slave_rlag = rses_get_max_replication_lag(client_rses); - - spinlock_init(&client_rses->rses_lock); client_rses->rses_backend_ref = backend_ref; + client_rses->rses_nbackends = router_nservers; /*< # of backend servers */ - /** - * Find a backend servers to connect to. - * This command requires that rsession's lock is held. - */ - - succp = rses_begin_locked_router_action(client_rses); - - if (!succp) - { - MXS_FREE(client_rses->rses_backend_ref); - MXS_FREE(client_rses); - return NULL; - } - succp = select_connect_backend_servers(&master_ref, backend_ref, router_nservers, - max_nslaves, max_slave_rlag, - client_rses->rses_config.rw_slave_select_criteria, - session, router); - - rses_end_locked_router_action(client_rses); - - /** - * Master and at least slaves must be found if the router is - * in the strict mode. If sessions without master are allowed, only - * slaves must be found. - */ - if (!succp) + backend_ref_t *master_ref = NULL; /*< pointer to selected master */ + if (!select_connect_backend_servers(&master_ref, backend_ref, router_nservers, + max_nslaves, max_slave_rlag, + client_rses->rses_config.rw_slave_select_criteria, + session, router)) { + /** + * Master and at least slaves must be found if the router is + * in the strict mode. If sessions without master are allowed, only + * slaves must be found. + */ MXS_FREE(client_rses->rses_backend_ref); MXS_FREE(client_rses); return NULL; @@ -434,7 +340,6 @@ static void *newSession(ROUTER *router_inst, SESSION *session) /** Copy backend pointers to router session. */ client_rses->rses_master_ref = master_ref; - client_rses->rses_nbackends = router_nservers; /*< # of backend servers */ if (client_rses->rses_config.rw_max_slave_conn_percent) { @@ -446,23 +351,15 @@ static void *newSession(ROUTER *router_inst, SESSION *session) router->stats.n_sessions += 1; - /** - * Add this session to end of the list of active sessions in router. - */ - spinlock_acquire(&router->lock); - client_rses->next = router->connections; - router->connections = client_rses; - spinlock_release(&router->lock); - return (void *)client_rses; } /** * @brief Close a router session (API). - * - * Close a session with the router, this is the mechanism by which a router - * may cleanup data structure etc. The instance of the router that relates to - * the relevant service is passed, along with the router session that is to + * + * Close a session with the router, this is the mechanism by which a router + * may cleanup data structure etc. The instance of the router that relates to + * the relevant service is passed, along with the router session that is to * be closed. Typically the function is used in conjunction with freeSession * which will release the resources used by a router session (see below). * @@ -471,58 +368,38 @@ static void *newSession(ROUTER *router_inst, SESSION *session) */ static void closeSession(ROUTER *instance, void *router_session) { - ROUTER_CLIENT_SES *router_cli_ses; - backend_ref_t *backend_ref; - - MXS_DEBUG("%lu [RWSplit:closeSession]", pthread_self()); - - /** - * router session can be NULL if newSession failed and it is discarding - * its connections and DCB's. - */ - if (router_session == NULL) - { - return; - } - router_cli_ses = (ROUTER_CLIENT_SES *)router_session; + ROUTER_CLIENT_SES *router_cli_ses = (ROUTER_CLIENT_SES *)router_session; CHK_CLIENT_RSES(router_cli_ses); - backend_ref = router_cli_ses->rses_backend_ref; - /** - * Lock router client session for secure read and update. - */ - if (!router_cli_ses->rses_closed && - rses_begin_locked_router_action(router_cli_ses)) + if (!router_cli_ses->rses_closed && rses_begin_locked_router_action(router_cli_ses)) { - int i; /** - * This sets router closed. Nobody is allowed to use router - * without checking this first. + * Mark router session as closed. @c rses_closed is checked at the start + * of every API function to quickly stop the processing of closed sessions. */ router_cli_ses->rses_closed = true; - for (i = 0; i < router_cli_ses->rses_nbackends; i++) + for (int i = 0; i < router_cli_ses->rses_nbackends; i++) { - backend_ref_t *bref = &backend_ref[i]; - DCB *dcb = bref->bref_dcb; - /** Close those which had been connected */ + backend_ref_t *bref = &router_cli_ses->rses_backend_ref[i]; + if (BREF_IS_IN_USE(bref)) { + /** This backend is in use and it needs to be closed */ + DCB *dcb = bref->bref_dcb; CHK_DCB(dcb); ss_dassert(dcb->session->state == SESSION_STATE_STOPPING); - /** Clean operation counter in bref and in SERVER */ if (BREF_IS_WAITING_RESULT(bref)) { + /** This backend was executing a query when the session was closed */ bref_clear_state(bref, BREF_WAITING_RESULT); } bref_clear_state(bref, BREF_IN_USE); bref_set_state(bref, BREF_CLOSED); - /** - * closes protocol and dcb - */ dcb_close(dcb); - /** decrease server current connection counters */ + + /** Decrease server reference connection count */ atomic_add(&bref->ref->connections, -1); } else @@ -539,15 +416,15 @@ static void closeSession(ROUTER *instance, void *router_session) } } } - /** Unlock */ + rses_end_locked_router_action(router_cli_ses); } } /** * @brief Free a router session (API). - * - * When a router session has been closed, freeSession can be called to free + * + * When a router session has been closed, freeSession can be called to free * allocated resources. * * @param router_instance The router instance the session belongs to @@ -556,40 +433,13 @@ static void closeSession(ROUTER *instance, void *router_session) */ static void freeSession(ROUTER *router_instance, void *router_client_session) { - ROUTER_CLIENT_SES *router_cli_ses; - ROUTER_INSTANCE *router; - int i; - - router_cli_ses = (ROUTER_CLIENT_SES *)router_client_session; - router = (ROUTER_INSTANCE *)router_instance; - - spinlock_acquire(&router->lock); - - if (router->connections == router_cli_ses) - { - router->connections = router_cli_ses->next; - } - else - { - ROUTER_CLIENT_SES *ptr = router->connections; - - while (ptr && ptr->next != router_cli_ses) - { - ptr = ptr->next; - } - - if (ptr) - { - ptr->next = router_cli_ses->next; - } - } - spinlock_release(&router->lock); + ROUTER_CLIENT_SES *router_cli_ses = (ROUTER_CLIENT_SES *)router_client_session; /** * For each property type, walk through the list, finalize properties * and free the allocated memory. */ - for (i = RSES_PROP_TYPE_FIRST; i < RSES_PROP_TYPE_COUNT; i++) + for (int i = RSES_PROP_TYPE_FIRST; i < RSES_PROP_TYPE_COUNT; i++) { rses_property_t *p = router_cli_ses->rses_properties[i]; rses_property_t *q = p; @@ -601,11 +451,7 @@ static void freeSession(ROUTER *router_instance, void *router_client_session) p = q; } } - /* - * We are no longer in the linked list, free - * all the memory and other resources associated - * to the client session. - */ + MXS_FREE(router_cli_ses->rses_backend_ref); MXS_FREE(router_cli_ses); return; @@ -663,20 +509,8 @@ static int routeQuery(ROUTER *instance, void *router_session, GWBUF *querybuf) */ static void diagnostics(ROUTER *instance, DCB *dcb) { - ROUTER_CLIENT_SES *router_cli_ses; ROUTER_INSTANCE *router = (ROUTER_INSTANCE *)instance; - int i = 0; char *weightby; - - spinlock_acquire(&router->lock); - router_cli_ses = router->connections; - while (router_cli_ses) - { - i++; - router_cli_ses = router_cli_ses->next; - } - spinlock_release(&router->lock); - double master_pct = 0.0, slave_pct = 0.0, all_pct = 0.0; if (router->stats.n_queries > 0) @@ -688,7 +522,8 @@ static void diagnostics(ROUTER *instance, DCB *dcb) dcb_printf(dcb, "\tNumber of router sessions: %d\n", router->stats.n_sessions); - dcb_printf(dcb, "\tCurrent no. of router sessions: %d\n", i); + dcb_printf(dcb, "\tCurrent no. of router sessions: %d\n", + router->service->stats.n_current); dcb_printf(dcb, "\tNumber of queries forwarded: %d\n", router->stats.n_queries); dcb_printf(dcb, "\tNumber of queries forwarded to master: %d (%.2f%%)\n", @@ -709,9 +544,9 @@ static void diagnostics(ROUTER *instance, DCB *dcb) for (SERVER_REF *ref = router->service->dbref; ref; ref = ref->next) { dcb_printf(dcb, "\t\t%-20s %3.1f%% %-6d %-6d %d\n", - ref->server->unique_name, (float)ref->weight / 10, - ref->server->stats.n_current, ref->connections, - ref->server->stats.n_current_ops); + ref->server->unique_name, (float)ref->weight / 10, + ref->server->stats.n_current, ref->connections, + ref->server->stats.n_current_ops); } } } @@ -796,7 +631,7 @@ static void clientReply(ROUTER *instance, void *router_session, GWBUF *writebuf, if (sescmd_cursor_is_active(scur)) { check_session_command_reply(writebuf, scur, bref); - + if (GWBUF_IS_TYPE_SESCMD_RESPONSE(writebuf)) { /** @@ -876,7 +711,7 @@ static void clientReply(ROUTER *instance, void *router_session, GWBUF *writebuf, CHK_GWBUF(bref->bref_pending_cmd); if ((ret = bref->bref_dcb->func.write(bref->bref_dcb, - gwbuf_clone(bref->bref_pending_cmd))) == 1) + gwbuf_clone(bref->bref_pending_cmd))) == 1) { ROUTER_INSTANCE* inst = (ROUTER_INSTANCE *)instance; atomic_add(&inst->stats.n_queries, 1); @@ -992,12 +827,12 @@ void rses_end_locked_router_action(ROUTER_CLIENT_SES *rses) /* * @brief Clear one or more bits in the backend reference state - * - * The router session holds details of the backend servers that are - * involved in the routing for this particular service. Each backend - * server has a state bit string, and this function (along with + * + * The router session holds details of the backend servers that are + * involved in the routing for this particular service. Each backend + * server has a state bit string, and this function (along with * bref_set_state) is used to manage the state. - * + * * @param bref The backend reference to be modified * @param state A bit string where the 1 bits indicate bits that should * be turned off in the bref state. @@ -1041,12 +876,12 @@ void bref_clear_state(backend_ref_t *bref, bref_state_t state) /* * @brief Set one or more bits in the backend reference state - * - * The router session holds details of the backend servers that are - * involved in the routing for this particular service. Each backend - * server has a state bit string, and this function (along with + * + * The router session holds details of the backend servers that are + * involved in the routing for this particular service. Each backend + * server has a state bit string, and this function (along with * bref_clear_state) is used to manage the state. - * + * * @param bref The backend reference to be modified * @param state A bit string where the 1 bits indicate bits that should * be turned on in the bref state. @@ -1088,9 +923,9 @@ void bref_set_state(backend_ref_t *bref, bref_state_t state) /** * @brief Free resources belonging to a property - * + * * Property is freed at the end of router client session. - * + * * @param prop The property whose resources are to be released */ void rses_property_done(rses_property_t *prop) @@ -1124,16 +959,16 @@ void rses_property_done(rses_property_t *prop) /** * @brief Get count of backend servers that are slaves. - * + * * Find out the number of read backend servers. * Depending on the configuration value type, either copy direct count * of slave connections or calculate the count from percentage value. - * + * * @param rses Router client session * @param router_nservers The number of backend servers in total */ int rses_get_max_slavecount(ROUTER_CLIENT_SES *rses, - int router_nservers) + int router_nservers) { int conf_max_nslaves; int max_nslaves; @@ -1155,7 +990,7 @@ int rses_get_max_slavecount(ROUTER_CLIENT_SES *rses, /* * @brief Get the maximum replication lag for this router - * + * * @param rses Router client session * @return Replication lag from configuration or very large number */ @@ -1180,10 +1015,10 @@ int rses_get_max_replication_lag(ROUTER_CLIENT_SES *rses) /** * @brief Find a back end reference that matches the given DCB - * + * * Finds out if there is a backend reference pointing at the DCB given as * parameter. - * + * * @param rses router client session * @param dcb DCB * @@ -1217,14 +1052,14 @@ backend_ref_t *get_bref_from_dcb(ROUTER_CLIENT_SES *rses, DCB *dcb) /** * @brief Call hang up function - * + * * Calls hang-up function for DCB if it is not both running and in * master/slave/joined/ndb role. Called by DCB's callback routine. - * + * * @param dcb DCB relating to a backend server * @param reason The reason for the state change * @param data Data is a backend reference structure belonging to this router - * + * * @return 1 for success, 0 for failure */ int router_handle_state_switch(DCB *dcb, DCB_REASON reason, void *data) @@ -1324,10 +1159,10 @@ static bool rwsplit_process_router_options(ROUTER_INSTANCE *router, if (c == UNDEFINED_CRITERIA) { MXS_ERROR("Unknown slave selection criteria \"%s\". " - "Allowed values are LEAST_GLOBAL_CONNECTIONS, " - "LEAST_ROUTER_CONNECTIONS, LEAST_BEHIND_MASTER," - "and LEAST_CURRENT_OPERATIONS.", - STRCRITERIA(router->rwsplit_config.rw_slave_select_criteria)); + "Allowed values are LEAST_GLOBAL_CONNECTIONS, " + "LEAST_ROUTER_CONNECTIONS, LEAST_BEHIND_MASTER," + "and LEAST_CURRENT_OPERATIONS.", + STRCRITERIA(router->rwsplit_config.rw_slave_select_criteria)); success = false; } else @@ -1385,9 +1220,9 @@ static bool rwsplit_process_router_options(ROUTER_INSTANCE *router, /** * @brief Router error handling routine (API) - * + * * Error Handler routine to resolve _backend_ failures. If it succeeds then - * there are enough operative backends available and connected. Otherwise it + * there are enough operative backends available and connected. Otherwise it * fails, and session is terminated. * * @param instance The router instance @@ -1450,8 +1285,8 @@ static void handleError(ROUTER *instance, void *router_session, if (!rses_begin_locked_router_action(rses)) { close_dcb = false; /* With the assumption that if the router session is closed, - * then so is the dcb. - */ + * then so is the dcb. + */ *succp = false; break; } @@ -1548,8 +1383,8 @@ static void handleError(ROUTER *instance, void *router_session, if (close_dcb) { - dcb_close(problem_dcb); -} + dcb_close(problem_dcb); + } } /** @@ -1697,7 +1532,7 @@ static bool handle_error_new_connection(ROUTER_INSTANCE *inst, */ if (inst->rwsplit_config.rw_disable_sescmd_hist) { - succp = have_enough_servers(&myrses, 1, myrses->rses_nbackends, inst) ? true : false; + succp = have_enough_servers(myrses, 1, myrses->rses_nbackends, inst) ? true : false; } else { @@ -1723,16 +1558,16 @@ return_succp: * * @return bool - whether enough, side effect is error logging */ -static bool have_enough_servers(ROUTER_CLIENT_SES **p_rses, const int min_nsrv, +static bool have_enough_servers(ROUTER_CLIENT_SES *rses, const int min_nsrv, int router_nsrv, ROUTER_INSTANCE *router) { bool succp; /** With too few servers session is not created */ if (router_nsrv < min_nsrv || - MXS_MAX((*p_rses)->rses_config.rw_max_slave_conn_count, - (router_nsrv * (*p_rses)->rses_config.rw_max_slave_conn_percent) / - 100) < min_nsrv) + MXS_MAX((rses)->rses_config.rw_max_slave_conn_count, + (router_nsrv * (rses)->rses_config.rw_max_slave_conn_percent) / + 100) < min_nsrv) { if (router_nsrv < min_nsrv) { @@ -1743,16 +1578,16 @@ static bool have_enough_servers(ROUTER_CLIENT_SES **p_rses, const int min_nsrv, } else { - int pct = (*p_rses)->rses_config.rw_max_slave_conn_percent / 100; + int pct = (rses)->rses_config.rw_max_slave_conn_percent / 100; int nservers = router_nsrv * pct; - if ((*p_rses)->rses_config.rw_max_slave_conn_count < min_nsrv) + if ((rses)->rses_config.rw_max_slave_conn_count < min_nsrv) { MXS_ERROR("Unable to start %s service. There are " "too few backend servers configured in " "MaxScale.cnf. Found %d when %d is required.", router->service->name, - (*p_rses)->rses_config.rw_max_slave_conn_count, min_nsrv); + (rses)->rses_config.rw_max_slave_conn_count, min_nsrv); } if (nservers < min_nsrv) { @@ -1762,11 +1597,9 @@ static bool have_enough_servers(ROUTER_CLIENT_SES **p_rses, const int min_nsrv, "MaxScale.cnf. Found %d%% when at least %.0f%% " "would be required.", router->service->name, - (*p_rses)->rses_config.rw_max_slave_conn_percent, dbgpct); + (rses)->rses_config.rw_max_slave_conn_percent, dbgpct); } } - MXS_FREE(*p_rses); - *p_rses = NULL; succp = false; } else @@ -1780,7 +1613,7 @@ static bool have_enough_servers(ROUTER_CLIENT_SES **p_rses, const int min_nsrv, * @brief Refresh the instance by the given parameter value. * * Used by createInstance and newSession - * + * * @param router Router instance * @param singleparam Parameter fo be reloaded * @@ -1884,11 +1717,11 @@ static void refreshInstance(ROUTER_INSTANCE *router, /* * @brief Release resources when createInstance fails to complete - * + * * Internal to createInstance - * + * * @param router Router instance - * + * */ static void free_rwsplit_instance(ROUTER_INSTANCE *router) { @@ -1898,3 +1731,57 @@ static void free_rwsplit_instance(ROUTER_INSTANCE *router) } } +/** + * @brief Create backend server references + * + * This creates a new set of backend references for the client session. Currently + * this is only used on startup but it could be used to dynamically change the + * set of used servers. + * + * @param rses Client router session + * @param dest Destination where the array of backens is stored + * @param n_backend Number of items in the array + * @return True on success, false on error + */ +static bool create_backends(ROUTER_CLIENT_SES *rses, backend_ref_t** dest, int* n_backend) +{ + backend_ref_t *backend_ref = (backend_ref_t *)MXS_CALLOC(1, *n_backend * sizeof(backend_ref_t)); + + if (backend_ref == NULL) + { + return false; + } + + int i = 0; + + for (SERVER_REF *sref = rses->router->service->dbref; sref && i < *n_backend; sref = sref->next) + { + if (sref->active) + { +#if defined(SS_DEBUG) + backend_ref[i].bref_chk_top = CHK_NUM_BACKEND_REF; + backend_ref[i].bref_chk_tail = CHK_NUM_BACKEND_REF; + backend_ref[i].bref_sescmd_cur.scmd_cur_chk_top = CHK_NUM_SESCMD_CUR; + backend_ref[i].bref_sescmd_cur.scmd_cur_chk_tail = CHK_NUM_SESCMD_CUR; +#endif + backend_ref[i].bref_state = 0; + backend_ref[i].ref = sref; + /** store pointers to sescmd list to both cursors */ + backend_ref[i].bref_sescmd_cur.scmd_cur_rses = rses; + backend_ref[i].bref_sescmd_cur.scmd_cur_active = false; + backend_ref[i].bref_sescmd_cur.scmd_cur_ptr_property = + &rses->rses_properties[RSES_PROP_TYPE_SESCMD]; + backend_ref[i].bref_sescmd_cur.scmd_cur_cmd = NULL; + i++; + } + } + + if (i < *n_backend) + { + MXS_INFO("The service reported %d servers but only took %d into use.", *n_backend, i); + *n_backend = i; + } + + *dest = backend_ref; + return true; +} diff --git a/server/modules/routing/readwritesplit/readwritesplit.h b/server/modules/routing/readwritesplit/readwritesplit.h index e300edf31..5e24a4162 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.h +++ b/server/modules/routing/readwritesplit/readwritesplit.h @@ -304,12 +304,10 @@ typedef struct typedef struct router_instance { SERVICE* service; /*< Pointer to service */ - ROUTER_CLIENT_SES* connections; /*< List of client connections */ SPINLOCK lock; /*< Lock for the instance data */ rwsplit_config_t rwsplit_config; /*< expanded config info from SERVICE */ int rwsplit_version; /*< version number for router's config */ ROUTER_STATS stats; /*< Statistics for this router */ - struct router_instance* next; /*< Next router on the list */ bool available_slaves; /*< The router has some slaves avialable */ } ROUTER_INSTANCE; From 28fc3d2b8347433ed63fe644a466b90dfdb6bc1a Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Sun, 6 Nov 2016 21:16:19 +0200 Subject: [PATCH 177/215] MXS-922: Add server serialization function The function serializes a server to a file. This is intended to be used with dynamically created servers. The output of the server serialization will eventually be stored in the configuration file directory (default is /etc/maxscale.cnf.d/) so that created servers persist even after a restart. --- include/maxscale/server.h | 14 +++++ server/core/server.c | 110 ++++++++++++++++++++++++++++++++++ server/core/test/testserver.c | 71 ++++++++++++++++++++++ 3 files changed, 195 insertions(+) diff --git a/include/maxscale/server.h b/include/maxscale/server.h index 8b85c2b13..45be5cd39 100644 --- a/include/maxscale/server.h +++ b/include/maxscale/server.h @@ -91,6 +91,7 @@ typedef struct server char *protocol; /**< Protocol module to use */ char *authenticator; /**< Authenticator module name */ void *auth_instance; /**< Authenticator instance */ + char *auth_options; /**< Authenticator options */ SSL_LISTENER *server_ssl; /**< SSL data structure for server, if any */ unsigned int status; /**< Status flag bitmap for the server */ char *monuser; /**< User name to use to monitor the db */ @@ -225,4 +226,17 @@ extern RESULTSET *serverGetList(); extern unsigned int server_map_status(char *str); extern bool server_set_version_string(SERVER* server, const char* string); +/** + * @brief Serialize a server to a file + * + * This converts @c server into an INI format file. This allows created servers + * to be persisted to disk. A new file is only created if the file pointed by + * @c filename does not exist at the time this function is called. + * + * @param server Server to serialize + * @param filename Path to a file where the server is persisted + * @return False if the serialization of the server fails, true if it was successful + */ +bool server_serialize(SERVER *server, const char *filename); + MXS_END_DECLS diff --git a/server/core/server.c b/server/core/server.c index 632aa4a94..1350bec49 100644 --- a/server/core/server.c +++ b/server/core/server.c @@ -35,6 +35,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -86,6 +89,12 @@ server_alloc(char *servname, char *protocol, unsigned short port, char *authenti return NULL; } + if (auth_options && (auth_options = MXS_STRDUP(auth_options)) == NULL) + { + MXS_FREE(authenticator); + return NULL; + } + servname = MXS_STRNDUP(servname, MAX_SERVER_NAME_LEN); protocol = MXS_STRDUP(protocol); @@ -108,6 +117,7 @@ server_alloc(char *servname, char *protocol, unsigned short port, char *authenti server->protocol = protocol; server->authenticator = authenticator; server->auth_instance = auth_instance; + server->auth_options = auth_options; server->port = port; server->status = SERVER_RUNNING; server->node_id = -1; @@ -1070,3 +1080,103 @@ bool server_set_version_string(SERVER* server, const char* string) return rval; } + +bool server_serialize(SERVER *server, const char *filename) +{ + int file = open(filename, O_EXCL | O_CREAT | O_WRONLY, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); + + if (file == -1) + { + char errbuf[MXS_STRERROR_BUFLEN]; + MXS_ERROR("Failed to open file '%s' when serializing server '%s': %d, %s", + filename, server->unique_name, errno, strerror_r(errno, errbuf, sizeof(errbuf))); + return false; + } + + // TODO: Check for return values on all of the dprintf calls + dprintf(file, "[%s]\n", server->unique_name); + dprintf(file, "type=server\n"); + dprintf(file, "protocol=%s\n", server->protocol); + dprintf(file, "address=%s\n", server->name); + dprintf(file, "port=%u\n", server->port); + dprintf(file, "authenticator=%s\n", server->authenticator); + + if (server->auth_options) + { + dprintf(file, "authenticator_options=%s\n", server->auth_options); + } + + if (server->monpw && server->monuser) + { + dprintf(file, "monitoruser=%s\n", server->monuser); + dprintf(file, "monitorpw=%s\n", server->monpw); + } + + if (server->persistpoolmax) + { + dprintf(file, "persistpoolmax=%ld\n", server->persistpoolmax); + } + + if (server->persistmaxtime) + { + dprintf(file, "persistmaxtime=%ld\n", server->persistmaxtime); + } + + if (server->server_ssl) + { + dprintf(file, "ssl=required\n"); + + if (server->server_ssl->ssl_cert) + { + dprintf(file, "ssl_cert=%s\n", server->server_ssl->ssl_cert); + } + + if (server->server_ssl->ssl_key) + { + dprintf(file, "ssl_key=%s\n", server->server_ssl->ssl_key); + } + + if (server->server_ssl->ssl_ca_cert) + { + dprintf(file, "ssl_ca_cert=%s\n", server->server_ssl->ssl_ca_cert); + } + if (server->server_ssl->ssl_cert_verify_depth) + { + dprintf(file, "ssl_cert_verify_depth=%d\n", server->server_ssl->ssl_cert_verify_depth); + } + + const char *version = NULL; + + switch (server->server_ssl->ssl_method_type) + { + case SERVICE_TLS10: + version = "TLSV10"; + break; + +#ifdef OPENSSL_1_0 + case SERVICE_TLS11: + version = "TLSV11"; + break; + + case SERVICE_TLS12: + version = "TLSV12"; + break; +#endif + case SERVICE_SSL_TLS_MAX: + version = "MAX"; + break; + + default: + break; + } + + if (version) + { + dprintf(file, "ssl_version=%s\n", version); + } + } + + close(file); + + return true; +} diff --git a/server/core/test/testserver.c b/server/core/test/testserver.c index 9c10e6edf..5a1f6f0ba 100644 --- a/server/core/test/testserver.c +++ b/server/core/test/testserver.c @@ -37,6 +37,10 @@ #include #include #include + +// This is pretty ugly but it's required to test internal functions +#include "../config.c" + /** * test1 Allocate a server and do lots of other things * @@ -103,11 +107,78 @@ test1() } +#define TEST(A, B) do { if(!(A)){ printf(B"\n"); return false; }} while(false) + +bool test_load_config(const char *input, SERVER *server) +{ + DUPLICATE_CONTEXT dcontext; + + if (duplicate_context_init(&dcontext)) + { + CONFIG_CONTEXT ccontext = {.object = ""}; + + if (config_load_single_file(input, &dcontext, &ccontext)) + { + CONFIG_CONTEXT *obj = ccontext.next; + CONFIG_PARAMETER *param = obj->parameters; + + TEST(strcmp(obj->object, server->unique_name) == 0, "Server names differ"); + TEST(strcmp(server->name, config_get_param(param, "address")->value) == 0, "Server addresses differ"); + TEST(strcmp(server->protocol, config_get_param(param, "protocol")->value) == 0, "Server protocols differ"); + TEST(strcmp(server->authenticator, config_get_param(param, "authenticator")->value) == 0, + "Server authenticators differ"); + TEST(strcmp(server->auth_options, config_get_param(param, "authenticator_options")->value) == 0, + "Server authenticator options differ"); + TEST(server->port == atoi(config_get_param(param, "port")->value), "Server ports differ"); + TEST(create_new_server(obj) == 0, "Failed to create server from loaded config"); + } + } + + return true; +} + +bool test_serialize() +{ + char name[] = "serialized-server"; + SERVER *server = server_alloc("127.0.0.1", "HTTPD", 9876, "NullAuthAllow", "fake=option"); + TEST(server, "Server allocation failed"); + server_set_unique_name(server, name); + + /** Make sure the file doesn't exist */ + unlink("./server.cnf"); + + /** Serialize server to disk */ + TEST(server_serialize(server, "./server.cnf"), "Failed to synchronize original server"); + + /** Load it again */ + TEST(test_load_config("./server.cnf", server), "Failed to load the serialized server"); + + /** We should have two identical servers */ + SERVER *created = server_find_by_unique_name(name); + TEST(created->next == server, "We should end up with two servers"); + + /** Make sure the file doesn't exist */ + unlink("./server-created.cnf"); + + /** Serialize the loaded server to disk */ + TEST(server_serialize(created, "./server-created.cnf"), "Failed to synchronize the copied server"); + + /** Check that they serialize to identical files */ + TEST(system("diff ./server.cnf ./server-created.cnf") == 0, "The files are not identical"); + + return true; +} + int main(int argc, char **argv) { int result = 0; result += test1(); + if (!test_serialize()) + { + result++; + } + exit(result); } From 65886dc38697a6d53d8f8d1feb039c81d37ee8d6 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Mon, 7 Nov 2016 11:17:01 +0200 Subject: [PATCH 178/215] Check whether server references are active Before a query is routed to a backend, the status of the server reference is checked. This allows the servers that are removed from a service to be ejected from the list of active servers for active sessions. --- .../readwritesplit/rwsplit_route_stmt.c | 35 ++++++++++++------- .../readwritesplit/rwsplit_select_backends.c | 7 ++-- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/server/modules/routing/readwritesplit/rwsplit_route_stmt.c b/server/modules/routing/readwritesplit/rwsplit_route_stmt.c index 4ab38b8ed..298358b4f 100644 --- a/server/modules/routing/readwritesplit/rwsplit_route_stmt.c +++ b/server/modules/routing/readwritesplit/rwsplit_route_stmt.c @@ -543,6 +543,7 @@ bool rwsplit_get_dcb(DCB **p_dcb, ROUTER_CLIENT_SES *rses, backend_type_t btype, * server, or master. */ if (BREF_IS_IN_USE((&backend_ref[i])) && + SERVER_REF_IS_ACTIVE(b) && (strncasecmp(name, b->server->unique_name, PATH_MAX) == 0) && (SERVER_IS_SLAVE(&server) || SERVER_IS_RELAY_SERVER(&server) || SERVER_IS_MASTER(&server))) @@ -577,7 +578,7 @@ bool rwsplit_get_dcb(DCB **p_dcb, ROUTER_CLIENT_SES *rses, backend_type_t btype, * Unused backend or backend which is not master nor * slave can't be used */ - if (!BREF_IS_IN_USE(&backend_ref[i]) || + if (!BREF_IS_IN_USE(&backend_ref[i]) || !SERVER_REF_IS_ACTIVE(b) || (!SERVER_IS_MASTER(&server) && !SERVER_IS_SLAVE(&server))) { continue; @@ -665,27 +666,37 @@ bool rwsplit_get_dcb(DCB **p_dcb, ROUTER_CLIENT_SES *rses, backend_type_t btype, */ if (btype == BE_MASTER) { - if (master_bref) + if (master_bref && SERVER_REF_IS_ACTIVE(master_bref->ref)) { /** It is possible for the server status to change at any point in time * so copying it locally will make possible error messages * easier to understand */ SERVER server; server.status = master_bref->ref->server->status; - if (BREF_IS_IN_USE(master_bref) && SERVER_IS_MASTER(&server)) + + if (BREF_IS_IN_USE(master_bref)) { - *p_dcb = master_bref->bref_dcb; - succp = true; - /** if bref is in use DCB should not be closed */ - ss_dassert(master_bref->bref_dcb->state != DCB_STATE_ZOMBIE); + if (SERVER_IS_MASTER(&server)) + { + *p_dcb = master_bref->bref_dcb; + succp = true; + /** if bref is in use DCB should not be closed */ + ss_dassert(master_bref->bref_dcb->state != DCB_STATE_ZOMBIE); + } + else + { + MXS_ERROR("Server '%s' should be master but " + "is %s instead and can't be chosen as the master.", + master_bref->ref->server->unique_name, + STRSRVSTATUS(&server)); + succp = false; + } } else { - MXS_ERROR("Server at %s:%d should be master but " - "is %s instead and can't be chosen to master.", - master_bref->ref->server->name, - master_bref->ref->server->port, - STRSRVSTATUS(&server)); + MXS_ERROR("Server '%s' is not in use and can't be " + "chosen as the master.", + master_bref->ref->server->unique_name); succp = false; } } diff --git a/server/modules/routing/readwritesplit/rwsplit_select_backends.c b/server/modules/routing/readwritesplit/rwsplit_select_backends.c index fba1d7271..35875be65 100644 --- a/server/modules/routing/readwritesplit/rwsplit_select_backends.c +++ b/server/modules/routing/readwritesplit/rwsplit_select_backends.c @@ -106,7 +106,8 @@ bool select_connect_backend_servers(backend_ref_t **p_master_ref, SERVER_REF *master_host = get_root_master(backend_ref, router_nservers); if (router->rwsplit_config.rw_master_failure_mode == RW_FAIL_INSTANTLY && - (master_host == NULL || SERVER_IS_DOWN(master_host->server))) + (master_host == NULL || !SERVER_REF_IS_ACTIVE(master_host) || + SERVER_IS_DOWN(master_host->server))) { MXS_ERROR("Couldn't find suitable Master from %d candidates.", router_nservers); return false; @@ -147,7 +148,9 @@ bool select_connect_backend_servers(backend_ref_t **p_master_ref, { SERVER *serv = backend_ref[i].ref->server; - if (!BREF_HAS_FAILED(&backend_ref[i]) && SERVER_IS_RUNNING(serv)) + if (!BREF_HAS_FAILED(&backend_ref[i]) && + SERVER_REF_IS_ACTIVE(backend_ref[i].ref) && + SERVER_IS_RUNNING(serv)) { /* check also for relay servers and don't take the master_host */ if (slaves_found < max_nslaves && From 084aacb11c9182f5c3d15613dc459813107e6477 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 8 Nov 2016 13:10:12 +0200 Subject: [PATCH 179/215] Fix debugcmd parameter processing The debugcmd parameter processing didn't actually allow the maximum number of parameters to be passed to the function. The detailed help text was never printed and most of them were only duplicates of the short description. --- client/maxadmin.c | 3 +- server/modules/include/debugcli.h | 2 +- server/modules/routing/debugcli/debugcmd.c | 1079 +++++++++++--------- 3 files changed, 583 insertions(+), 501 deletions(-) diff --git a/client/maxadmin.c b/client/maxadmin.c index 96602b95e..028c4ad01 100644 --- a/client/maxadmin.c +++ b/client/maxadmin.c @@ -108,6 +108,7 @@ static struct option long_options[] = #define MAXADMIN_DEFAULT_HOST "localhost" #define MAXADMIN_DEFAULT_PORT "6603" #define MAXADMIN_DEFAULT_USER "admin" +#define MAXADMIN_BUFFER_SIZE 2048 /** * The main for the maxadmin client @@ -125,7 +126,7 @@ main(int argc, char **argv) History *hist; HistEvent ev; #else - char buf[1024]; + char buf[MAXADMIN_BUFFER_SIZE]; #endif char *hostname = NULL; char *port = NULL; diff --git a/server/modules/include/debugcli.h b/server/modules/include/debugcli.h index a32f47374..0e384c603 100644 --- a/server/modules/include/debugcli.h +++ b/server/modules/include/debugcli.h @@ -55,7 +55,7 @@ typedef struct cli_instance * The CLI_SESSION structure. As CLI_SESSION is created for each user that logs into * the DEBUG CLI. */ -enum { CMDBUFLEN = 80 }; +#define CMDBUFLEN 2048 typedef struct cli_session { diff --git a/server/modules/routing/debugcli/debugcmd.c b/server/modules/routing/debugcli/debugcmd.c index 9d072c600..ee2732216 100644 --- a/server/modules/routing/debugcli/debugcmd.c +++ b/server/modules/routing/debugcli/debugcmd.c @@ -75,8 +75,9 @@ #include #include -#define MAXARGS 6 +#define MAXARGS 12 +#define ARG_TYPE_NONE 0 #define ARG_TYPE_ADDRESS 1 #define ARG_TYPE_STRING 2 #define ARG_TYPE_SERVICE 3 @@ -95,186 +96,261 @@ extern LIST_CONFIG SESSIONlist; * * These are the options that may be passed to a command */ -struct subcommand { +struct subcommand +{ char *arg1; - int n_args; + int argc_min; + int argc_max; void (*fn)(); char *help; char *devhelp; - int arg_types[3]; + int arg_types[MAXARGS]; }; +#define EMPTY_OPTION + static void telnetdShowUsers(DCB *); static void show_log_throttling(DCB *); /** * The subcommands of the show command */ -struct subcommand showoptions[] = { +struct subcommand showoptions[] = +{ #if defined(BUFFER_TRACE) - { "buffers", 0, dprintAllBuffers, - "Show all buffers with backtrace", - "Show all buffers with backtrace", - {0, 0, 0} }, + { + "buffers", 0, dprintAllBuffers, + "Show all buffers with backtrace", + "Show all buffers with backtrace", + {0, 0, 0} + }, #endif - { "dcblist", 0, dprintDCBList, - "Show statistics for the list of all descriptor control blocks", - "Show statistics for the list of all descriptor control blocks", - {0, 0, 0} }, - { "dcbs", 0, dprintAllDCBs, - "Show all descriptor control blocks (network connections)", - "Show all descriptor control blocks (network connections)", - {0, 0, 0} }, - { "dcb", 1, dprintDCB, - "Show a single descriptor control block e.g. show dcb 0x493340", - "Show a single descriptor control block e.g. show dcb 0x493340", - {ARG_TYPE_DCB, 0, 0} }, - { "dbusers", 1, dcb_usersPrint, - "Show statistics and user names for a service's user table.\n" - "\t\tExample : show dbusers ", - "Show statistics and user names for a service's user table.\n" - "\t\tExample : show dbusers |", - {ARG_TYPE_DBUSERS, 0, 0} }, - { "epoll", 0, dprintPollStats, - "Show the poll statistics", - "Show the poll statistics", - {0, 0, 0} }, - { "eventq", 0, dShowEventQ, - "Show the queue of events waiting to be processed", - "Show the queue of events waiting to be processed", - {0, 0, 0} }, - { "eventstats", 0, dShowEventStats, - "Show the event statistics", - "Show the event statistics", - {0, 0, 0} }, - { "feedbackreport", 0, moduleShowFeedbackReport, - "Show the report of MaxScale loaded modules, suitable for Notification Service", - "Show the report of MaxScale loaded modules, suitable for Notification Service", - {0, 0, 0} }, - { "filter", 1, dprintFilter, - "Show details of a filter, called with a filter name", - "Show details of a filter, called with the address of a filter", - {ARG_TYPE_FILTER, 0, 0} }, - { "filters", 0, dprintAllFilters, - "Show all filters", - "Show all filters", - {0, 0, 0} }, - { "log_throttling", 0, show_log_throttling, - "Show the current log throttling setting (count, window (ms), suppression (ms))", - "Show the current log throttling setting (count, window (ms), suppression (ms))", - {0, 0, 0} }, - { "modules", 0, dprintAllModules, - "Show all currently loaded modules", - "Show all currently loaded modules", - {0, 0, 0} }, - { "monitor", 1, monitorShow, - "Show the monitor details", - "Show the monitor details", - {ARG_TYPE_MONITOR, 0, 0} }, - { "monitors", 0, monitorShowAll, - "Show the monitors that are configured", - "Show the monitors that are configured", - {0, 0, 0} }, - { "persistent", 1, dprintPersistentDCBs, - "Show persistent pool for a named server, e.g. show persistent dbnode1", - "Show persistent pool for a server, e.g. show persistent 0x485390. " - "The address may also be replaced with the server name from the configuration file", - {ARG_TYPE_SERVER, 0, 0} }, - { "server", 1, dprintServer, - "Show details for a named server, e.g. show server dbnode1", - "Show details for a server, e.g. show server 0x485390. The address may also be " - "repalced with the server name from the configuration file", - {ARG_TYPE_SERVER, 0, 0} }, - { "servers", 0, dprintAllServers, - "Show all configured servers", - "Show all configured servers", - {0, 0, 0} }, - { "serversjson", 0, dprintAllServersJson, - "Show all configured servers in JSON format", - "Show all configured servers in JSON format", - {0, 0, 0} }, - { "services", 0, dprintAllServices, - "Show all configured services in MaxScale", - "Show all configured services in MaxScale", - {0, 0, 0} }, - { "service", 1, dprintService, - "Show a single service in MaxScale, may be passed a service name", - "Show a single service in MaxScale, may be passed a service name or address of a service object", - {ARG_TYPE_SERVICE, 0, 0} }, - { "session", 1, dprintSession, - "Show a single session in MaxScale, e.g. show session 0x284830", - "Show a single session in MaxScale, e.g. show session 0x284830", - {ARG_TYPE_SESSION, 0, 0} }, - { "sessionlist", 0, dprintSessionList, - "Show statistics for the list of all sessions", - "Show statistics for the list of all sessions", - {0, 0, 0} }, - { "sessions", 0, dprintAllSessions, - "Show all active sessions in MaxScale", - "Show all active sessions in MaxScale", - {0, 0, 0} }, - { "tasks", 0, hkshow_tasks, - "Show all active housekeeper tasks in MaxScale", - "Show all active housekeeper tasks in MaxScale", - {0, 0, 0} }, - { "threads", 0, dShowThreads, - "Show the status of the polling threads in MaxScale", - "Show the status of the polling threads in MaxScale", - {0, 0, 0} }, - { "users", 0, telnetdShowUsers, - "Show all maxadmin enabled Linux accounts and created maxadmin users", - "Show all maxadmin enabled Linux accounts and created maxadmin users", - {0, 0, 0} }, - { NULL, 0, NULL, NULL, NULL, - {0, 0, 0} } + { + "dcblist", 0, 0, dprintDCBList, + "Show DCB statistics", + "Show statistics for the list of all DCBs(descriptor control blocks)", + {0} + }, + { + "dcbs", 0, 0, dprintAllDCBs, + "Show all DCBs", + "Show all descriptor control blocks (network connections)", + {0} + }, + { + "dcb", 1, 1, dprintDCB, + "Show a DCB", + "Show a single descriptor control block e.g. show dcb 0x493340", + {ARG_TYPE_DCB, 0, 0} + }, + { + "dbusers", 1, 1, dcb_usersPrint, + "Show user statistics", + "Show statistics and user names for a service's user table.\n" + "\t\tExample : show dbusers |", + {ARG_TYPE_DBUSERS, 0, 0} + }, + { + "epoll", 0, 0, dprintPollStats, + "Show the poll statistics", + "Show the epoll polling system statistics", + {0, 0, 0} + }, + { + "eventq", 0, 0, dShowEventQ, + "Show event queue", + "Show the queue of events waiting to be processed", + {0, 0, 0} + }, + { + "eventstats", 0, 0, dShowEventStats, + "Show event queue statistics", + "Show event queue statistics", + {0, 0, 0} + }, + { + "feedbackreport", 0, 0, moduleShowFeedbackReport, + "Show feedback report", + "Show the report of MaxScale loaded modules, suitable for Notification Service", + {0, 0, 0} + }, + { + "filter", 1, 1, dprintFilter, + "Show filter details", + "Show details of a filter, the parameter is filter name", + {ARG_TYPE_FILTER, 0, 0} + }, + { + "filters", 0, 0, dprintAllFilters, + "Show all filters", + "Show all filters that were read from the configuration file", + {0, 0, 0} + }, + { + "log_throttling", 0, 0, show_log_throttling, + "Show log throttling setting", + "Show the current log throttling setting (count, window (ms), suppression (ms))", + {0, 0, 0} + }, + { + "modules", 0, 0, dprintAllModules, + "Show loaded modules", + "Show all currently loaded modules", + {0, 0, 0} + }, + { + "monitor", 1, 1, monitorShow, + "Show monitor details", + "Show details about a specific monitor, the parameter is monitor name", + {ARG_TYPE_MONITOR, 0, 0} + }, + { + "monitors", 0, 0, monitorShowAll, + "Show all monitors", + "Show all the monitors", + {0, 0, 0} + }, + { + "persistent", 1, 1, dprintPersistentDCBs, + "Show persistent connection pool", + "Show persistent pool for a server, e.g. show persistent dbnode1. ", + {ARG_TYPE_SERVER, 0, 0} + }, + { + "server", 1, 1, dprintServer, + "Show server details", + "Show details for a server, e.g. show server dbnode1", + {ARG_TYPE_SERVER, 0, 0} + }, + { + "servers", 0, 0, dprintAllServers, + "Show all servers", + "Show all configured servers", + {0, 0, 0} + }, + { + "serversjson", 0, 0, dprintAllServersJson, + "Show all servers in JSON", + "Show all configured servers in JSON format", + {0, 0, 0} + }, + { + "services", 0, 0, dprintAllServices, + "Show all service", + "Show all configured services in MaxScale", + {0, 0, 0} + }, + { + "service", 1, 1, dprintService, + "Show service details", + "Show a single service in MaxScale, the parameter is the service name", + {ARG_TYPE_SERVICE, 0, 0} + }, + { + "session", 1, 1, dprintSession, + "Show session details", + "Show a single session in MaxScale, e.g. show session 0x284830", + {ARG_TYPE_SESSION, 0, 0} + }, + { + "sessionlist", 0, 0, dprintSessionList, + "Show session list statistics", + "Show statistics for the list of all sessions", + {0, 0, 0} + }, + { + "sessions", 0, 0, dprintAllSessions, + "Show all sessions", + "Show all active sessions in MaxScale", + {0, 0, 0} + }, + { + "tasks", 0, 0, hkshow_tasks, + "Show housekeeper tasks", + "Show all active housekeeper tasks in MaxScale", + {0, 0, 0} + }, + { + "threads", 0, 0, dShowThreads, + "Show workter thread status", + "Show the status of the worker threads in MaxScale", + {0, 0, 0} + }, + { + "users", 0, 0, telnetdShowUsers, + "Show enabled Linux accounts", + "Show all maxadmin enabled Linux accounts and created maxadmin users", + {0, 0, 0} + }, + { EMPTY_OPTION} }; /** * The subcommands of the list command */ -struct subcommand listoptions[] = { - { "clients", 0, dListClients, - "List all the client connections to MaxScale", - "List all the client connections to MaxScale", - {0, 0, 0} }, - { "dcbs", 0, dListDCBs, - "List all the DCBs active within MaxScale", - "List all the DCBs active within MaxScale", - {0, 0, 0} }, - { "filters", 0, dListFilters, - "List all the filters defined within MaxScale", - "List all the filters defined within MaxScale", - {0, 0, 0} }, - { "listeners", 0, dListListeners, - "List all the listeners defined within MaxScale", - "List all the listeners defined within MaxScale", - {0, 0, 0} }, - { "modules", 0, dprintAllModules, - "List all currently loaded modules", - "List all currently loaded modules", - {0, 0, 0} }, - { "monitors", 0, monitorList, - "List all monitors", - "List all monitors", - {0, 0, 0} }, - { "services", 0, dListServices, - "List all the services defined within MaxScale", - "List all the services defined within MaxScale", - {0, 0, 0} }, - { "servers", 0, dListServers, - "List all the servers defined within MaxScale", - "List all the servers defined within MaxScale", - {0, 0, 0} }, - { "sessions", 0, dListSessions, - "List all the active sessions within MaxScale", - "List all the active sessions within MaxScale", - {0, 0, 0} }, - { "threads", 0, dShowThreads, - "List the status of the polling threads in MaxScale", - "List the status of the polling threads in MaxScale", - {0, 0, 0} }, - { NULL, 0, NULL, NULL, NULL, - {0, 0, 0} } +struct subcommand listoptions[] = +{ + { + "clients", 0, 0, dListClients, + "List all clients", + "List all the client connections to MaxScale", + {0, 0, 0} + }, + { + "dcbs", 0, 0, dListDCBs, + "List all DCBs", + "List all the DCBs active within MaxScale", + {0, 0, 0} + }, + { + "filters", 0, 0, dListFilters, + "List all filters", + "List all the filters defined within MaxScale", + {0, 0, 0} + }, + { + "listeners", 0, 0, dListListeners, + "List all listeners", + "List all the listeners defined within MaxScale", + {0, 0, 0} + }, + { + "modules", 0, 0, dprintAllModules, + "List all currently loaded modules", + "List all currently loaded modules", + {0, 0, 0} + }, + { + "monitors", 0, 0, monitorList, + "List all monitors", + "List all monitors", + {0, 0, 0} + }, + { + "services", 0, 0, dListServices, + "List all the services", + "List all the services defined within MaxScale", + {0, 0, 0} + }, + { + "servers", 0, 0, dListServers, + "List all servers", + "List all the servers defined within MaxScale", + {0, 0, 0} + }, + { + "sessions", 0, 0, dListSessions, + "List all sessions", + "List all the active sessions within MaxScale", + {0, 0, 0} + }, + { + "threads", 0, 0, dShowThreads, + "List polling threads", + "List the status of the polling threads in MaxScale", + {0, 0, 0} + }, + { EMPTY_OPTION} }; static void shutdown_server() @@ -288,38 +364,34 @@ static void shutdown_monitor(DCB *dcb, MONITOR *monitor); /** * The subcommands of the shutdown command */ -struct subcommand shutdownoptions[] = { +struct subcommand shutdownoptions[] = +{ { "maxscale", - 0, + 0, 0, shutdown_server, "Shutdown MaxScale", - "Shutdown MaxScale", + "Initiate a controlled shutdown of MaxScale", {0, 0, 0} }, { "monitor", - 1, + 1, 1, shutdown_monitor, - "Shutdown a monitor, e.g. shutdown monitor 0x48381e0", - "Shutdown a monitor, e.g. shutdown monitor 0x48381e0", + "Shutdown a monitor", + "E.g. shutdown monitor db-cluster-monitor", {ARG_TYPE_MONITOR, 0, 0} }, { "service", - 1, + 1, 1, shutdown_service, - "Shutdown a service, e.g. shutdown service \"Sales Database\"", - "Shutdown a service, e.g. shutdown service 0x4838320 or shutdown service \"Sales Database\"", + "Stop a service", + "E.g. shutdown service \"Sales Database\"", {ARG_TYPE_SERVICE, 0, 0} }, { - NULL, - 0, - NULL, - NULL, - NULL, - {0, 0, 0} + EMPTY_OPTION } }; @@ -332,7 +404,7 @@ static void sync_logs(DCB *dcb) else { dcb_printf(dcb, "Failed to flush logs to disk. Read the error log for " - "more details.\n"); + "more details.\n"); } } @@ -340,19 +412,14 @@ struct subcommand syncoptions[] = { { "logs", - 0, + 0, 0, sync_logs, "Flush log files to disk", "Flush log files to disk", {0, 0, 0} }, { - NULL, - 0, - NULL, - NULL, - NULL, - {0, 0, 0} + EMPTY_OPTION } }; @@ -361,17 +428,21 @@ static void restart_monitor(DCB *dcb, MONITOR *monitor); /** * The subcommands of the restart command */ -struct subcommand restartoptions[] = { - { "monitor", 1, restart_monitor, - "Restart a monitor, e.g. restart monitor 0x48181e0", - "Restart a monitor, e.g. restart monitor 0x48181e0", - {ARG_TYPE_MONITOR, 0, 0} }, - { "service", 1, restart_service, - "Restart a service, e.g. restart service \"Test Service\"", - "Restart a service, e.g. restart service 0x4838320", - {ARG_TYPE_SERVICE, 0, 0} }, - { NULL, 0, NULL, NULL, NULL, - {0, 0, 0} } +struct subcommand restartoptions[] = +{ + { + "monitor", 1, 1, restart_monitor, + "Restart a monitor", + "E.g. restart monitor db-cluster-monitor", + {ARG_TYPE_MONITOR, 0, 0} + }, + { + "service", 1, 1, restart_service, + "Restart a service", + "E.g. restart service \"Sales Database\"", + {ARG_TYPE_SERVICE, 0, 0} + }, + { EMPTY_OPTION } }; static void set_server(DCB *dcb, SERVER *server, char *bit); @@ -381,38 +452,48 @@ static void set_log_throttling(DCB *dcb, int count, int window_ms, int suppress_ /** * The subcommands of the set command */ -struct subcommand setoptions[] = { - { "server", 2, set_server, - "Set the status of a server. E.g. set server dbnode4 master", - "Set the status of a server. E.g. set server 0x4838320 master", - {ARG_TYPE_SERVER, ARG_TYPE_STRING, 0} }, - { "pollsleep", 1, set_pollsleep, - "Set the maximum poll sleep period in milliseconds", - "Set the maximum poll sleep period in milliseconds", - {ARG_TYPE_NUMERIC, 0, 0} }, - { "nbpolls", 1, set_nbpoll, - "Set the number of non-blocking polls", - "Set the number of non-blocking polls", - {ARG_TYPE_NUMERIC, 0, 0} }, - { "log_throttling", 3, set_log_throttling, - "Set the log throttling configuration", - "Set the log throttling configuration", - {ARG_TYPE_NUMERIC, ARG_TYPE_NUMERIC, ARG_TYPE_NUMERIC} }, - { NULL, 0, NULL, NULL, NULL, - {0, 0, 0} } +struct subcommand setoptions[] = +{ + { + "server", 2, 2, set_server, + "Set the status of a server", + "Set the status of a server. E.g. set server dbnode4 master", + {ARG_TYPE_SERVER, ARG_TYPE_STRING, 0} + }, + { + "pollsleep", 1, 1, set_pollsleep, + "Set poll sleep period", + "Set the maximum poll sleep period in milliseconds", + {ARG_TYPE_NUMERIC, 0, 0} + }, + { + "nbpolls", 1, 1, set_nbpoll, + "Set non-blocking polls", + "Set the number of non-blocking polls", + {ARG_TYPE_NUMERIC, 0, 0} + }, + { + "log_throttling", 3, 3, set_log_throttling, + "Set log throttling", + "Set the log throttling configuration", + {ARG_TYPE_NUMERIC, ARG_TYPE_NUMERIC, ARG_TYPE_NUMERIC} + }, + { EMPTY_OPTION } }; static void clear_server(DCB *dcb, SERVER *server, char *bit); /** * The subcommands of the clear command */ -struct subcommand clearoptions[] = { - { "server", 2, clear_server, - "Clear the status of a server. E.g. clear server dbnode2 master", - "Clear the status of a server. E.g. clear server 0x4838320 master", - {ARG_TYPE_SERVER, ARG_TYPE_STRING, 0} }, - { NULL, 0, NULL, NULL, NULL, - {0, 0, 0} } +struct subcommand clearoptions[] = +{ + { + "server", 2, 2, clear_server, + "Clear server status", + "Clear the status of a server. E.g. clear server dbnode2 master", + {ARG_TYPE_SERVER, ARG_TYPE_STRING, 0} + }, + { EMPTY_OPTION } }; static void reload_dbusers(DCB *dcb, SERVICE *service); @@ -421,17 +502,21 @@ static void reload_config(DCB *dcb); /** * The subcommands of the reload command */ -struct subcommand reloadoptions[] = { - { "config", 0, reload_config, - "Reload the configuration data for MaxScale.", - "Reload the configuration data for MaxScale.", - {0, 0, 0} }, - { "dbusers", 1, reload_dbusers, - "Reload the dbuser data for a service. E.g. reload dbusers \"splitter service\"", - "Reload the dbuser data for a service. E.g. reload dbusers 0x849420", - {ARG_TYPE_SERVICE, 0, 0} }, - { NULL, 0, NULL, NULL, NULL, - {0, 0, 0} } +struct subcommand reloadoptions[] = +{ + { + "config", 0, 0, reload_config, + "Reload the configuration", + "Reload the configuration data for MaxScale", + {0, 0, 0} + }, + { + "dbusers", 1, 1, reload_dbusers, + "Reload users table", + "Reload the users for a service. E.g. reload dbusers \"splitter service\"", + {ARG_TYPE_SERVICE, 0, 0} + }, + { EMPTY_OPTION } }; static void enable_log_action(DCB *, char *); @@ -458,106 +543,95 @@ static void disable_account(DCB *, char *user); /** * * The subcommands of the enable command * */ -struct subcommand enableoptions[] = { +struct subcommand enableoptions[] = +{ { "heartbeat", - 1, + 1, 1, enable_monitor_replication_heartbeat, - "Enable the monitor replication heartbeat, pass a monitor name as argument", - "Enable the monitor replication heartbeat, pass a monitor name as argument", + "Enable monitor replication heartbeat", + "Enable the monitor replication heartbeat, the parameter is the monitor name", {ARG_TYPE_MONITOR, 0, 0} }, { "log", - 1, + 1, 1, enable_log_action, - "[deprecated] Enable Log options for MaxScale, options 'trace' | 'error' | 'message'." - "E.g. 'enable log message'.", - "[deprecated] Enable Log options for MaxScale, options 'trace' | 'error' | 'message'." - "E.g. 'enable log message'.", + "[deprecated] Enable a logging level", + "Options 'trace' | 'error' | 'message'. E.g. 'enable log message'.", {ARG_TYPE_STRING, 0, 0} }, { "log-priority", - 1, + 1, 1, enable_log_priority, - "Enable a logging priority; options 'err' | 'warning' | 'notice' | 'info' | 'debug'. " - "E.g.: 'enable log-priority info'.", - "Enable a logging priority; options 'err' | 'warning' | 'notice' | 'info' | 'debug'. " + "Enable a logging priority", + "Enable a logging priority for MaxScale, parameters must be one of " + "'err', 'warning', 'notice', 'info' or 'debug'. " "E.g.: 'enable log-priority info'.", {ARG_TYPE_STRING, 0, 0} }, { "sessionlog", - 2, + 2, 2, enable_sess_log_action, - "[deprecated] Enable Log options for a single session. Usage: enable sessionlog [trace | error | " - "message | debug] \t E.g. enable sessionlog message 123.", - "[deprecated] Enable Log options for a single session. Usage: enable sessionlog [trace | error | " + "[deprecated] Enable a logging level for a single session", + "Usage: enable sessionlog [trace | error | " "message | debug] \t E.g. enable sessionlog message 123.", {ARG_TYPE_STRING, ARG_TYPE_STRING, 0} }, { "sessionlog-priority", - 2, + 2, 2, enable_sess_log_priority, - "Enable a logging priority for a particular session. " - "Usage: enable sessionlog-priority [err | warning | notice | info | debug] " - "message | debug] \t E.g. enable sessionlog-priority info 123.", - "Enable a logging priority for a particular session. " + "Enable a logging priority for a session", "Usage: enable sessionlog-priority [err | warning | notice | info | debug] " "message | debug] \t E.g. enable sessionlog-priority info 123.", {ARG_TYPE_STRING, ARG_TYPE_STRING, 0} }, { "root", - 1, + 1, 1, enable_service_root, - "Enable root access to a service, pass a service name to enable root access", + "Enable root user access", "Enable root access to a service, pass a service name to enable root access", {ARG_TYPE_SERVICE, 0, 0} }, { "feedback", - 0, + 0, 0, enable_feedback_action, - "Enable MaxScale modules list sending via http to notification service", + "Enable MaxScale feedback", "Enable MaxScale modules list sending via http to notification service", {0, 0, 0} }, { "syslog", - 0, + 0, 0, enable_syslog, - "Enable syslog logging", + "Enable syslog", "Enable syslog logging", {0, 0, 0} }, { "maxlog", - 0, + 0, 0, enable_maxlog, - "Enable maxlog logging", - "Enable maxlog logging", + "Enable MaxScale logging", + "Enable MaxScale logging", {0, 0, 0} }, { "account", - 1, + 1, 1, enable_account, - "Enable maxadmin usage for Linux user. E.g.:\n" - " MaxScale> enable account alice", - "Enable maxadmin usage for Linux user. E.g.:\n" + "Activate a Linux user", + "Enable maxadmin usage for a Linux user. E.g.:\n" " MaxScale> enable account alice", {ARG_TYPE_STRING, 0, 0} }, { - NULL, - 0, - NULL, - NULL, - NULL, - {0, 0, 0} + EMPTY_OPTION } }; @@ -566,106 +640,95 @@ struct subcommand enableoptions[] = { /** * * The subcommands of the disable command * */ -struct subcommand disableoptions[] = { +struct subcommand disableoptions[] = +{ { "heartbeat", - 1, + 1, 1, disable_monitor_replication_heartbeat, - "Disable the monitor replication heartbeat", + "Disable replication heartbeat", "Disable the monitor replication heartbeat", {ARG_TYPE_MONITOR, 0, 0} }, { "log", - 1, + 1, 1, disable_log_action, - "[deprecated] Disable Log for MaxScale, Options: 'debug' | 'trace' | 'error' | 'message'." - "E.g. 'disable log debug'.", - "[deprecated] Disable Log for MaxScale, Options: 'debug' | 'trace' | 'error' | 'message'." - "E.g. 'disable log debug'.", + "[deprecated] Disable log for MaxScale", + "Options: 'debug' | 'trace' | 'error' | 'message'." + "E.g. 'disable log debug'", {ARG_TYPE_STRING, 0, 0} }, { "log-priority", - 1, + 1, 1, disable_log_priority, - "Disable a logging priority; options 'err' | 'warning' | 'notice' | 'info' | 'debug'. " - "E.g.: 'disable log-priority info'.", - "Disable a logging priority; options 'err' | 'warning' | 'notice' | 'info' | 'debug'. " - "E.g.: 'disable log-priority info'.", + "Disable a logging priority", + "Options 'err' | 'warning' | 'notice' | 'info' | 'debug'. " + "E.g.: 'disable log-priority info'", {ARG_TYPE_STRING, 0, 0} }, { "sessionlog", - 2, + 2, 2, disable_sess_log_action, - "[deprecated] Disable Log options for a single session. Usage: disable sessionlog [trace | error | " - "message | debug] \t E.g. disable sessionlog message 123.", - "[deprecated] Disable Log options for a single session. Usage: disable sessionlog [trace | error | " - "message | debug] \t E.g. disable sessionlog message 123.", + "[deprecated] Disable log options", + "Disable Log options for a single session. Usage: disable sessionlog [trace | error | " + "message | debug] \t E.g. disable sessionlog message 123", {ARG_TYPE_STRING, ARG_TYPE_STRING, 0} }, { "sessionlog-priority", - 2, + 2, 2, disable_sess_log_priority, - "Disable a logging priority for a particular session. " + "Disable a logging priority for a particular session", "Usage: disable sessionlog-priority [err | warning | notice | info | debug] " - "message | debug] \t E.g. enable sessionlog-priority info 123.", - "Enable a logging priority for a particular session. " - "Usage: disable sessionlog-priority [err | warning | notice | info | debug] " - "message | debug] \t E.g. enable sessionlog-priority info 123.", + "message | debug] \t E.g. enable sessionlog-priority info 123", {ARG_TYPE_STRING, ARG_TYPE_STRING, 0} }, { "root", - 1, + 1, 1, disable_service_root, - "Disable root access to a service", + "Disable root access", "Disable root access to a service", {ARG_TYPE_SERVICE, 0, 0} }, { "feedback", - 0, + 0, 0, disable_feedback_action, - "Disable MaxScale modules list sending via http to notification service", + "Disable feedback", "Disable MaxScale modules list sending via http to notification service", {0, 0, 0} }, { "syslog", - 0, + 0, 0, disable_syslog, - "Disable syslog logging", + "Disable syslog", "Disable syslog logging", {0, 0, 0} }, { "maxlog", - 0, + 0, 0, disable_maxlog, - "Disable maxlog logging", - "Disable maxlog logging", + "Disable MaxScale logging", + "Disable MaxScale logging", {0, 0, 0} }, { "account", - 1, + 1, 1, disable_account, - "Disable maxadmin usage for Linux user. E.g.:\n" - " MaxScale> disable account alice", + "Disable Linux user", "Disable maxadmin usage for Linux user. E.g.:\n" " MaxScale> disable account alice", {ARG_TYPE_STRING, 0, 0} }, { - NULL, - 0, - NULL, - NULL, - NULL, - {0, 0, 0} + EMPTY_OPTION } }; @@ -677,7 +740,8 @@ static void fail_accept(DCB* dcb, char* arg1, char* arg2); /** * * The subcommands of the fail command * */ -struct subcommand failoptions[] = { +struct subcommand failoptions[] = +{ { "backendfd", 0, @@ -729,23 +793,22 @@ static void cmd_serviceAddBackend(DCB *dcb, void *a, void *b) /** * The subcommands of the add command */ -struct subcommand addoptions[] = { +struct subcommand addoptions[] = +{ { - "user", 2, telnetdAddUser, - "Add insecure account for using maxadmin over the network. E.g.:\n" - " MaxScale> add user bob somepass", - "Add insecure account for using maxadmin over the network. E.g.:\n" - " MaxScale> add user bob somepass", + "user", 2, 2, telnetdAddUser, + "Add account for maxadmin", + "Add insecure account for using maxadmin over the network. E.g.:\n" + " MaxScale> add user bob somepass", {ARG_TYPE_STRING, ARG_TYPE_STRING, 0} }, { - "server", 2, cmd_serviceAddBackend, - "Add a new server to a service", - "Add a new server to a service. The server must exist in the configuration file.", + "server", 2, 2, cmd_serviceAddBackend, + "Add a new server to a service", + "Takes a server name as the parameter. The server must exist in the configuration file.", {ARG_TYPE_SERVICE, ARG_TYPE_SERVER, 0} }, - { NULL, 0, NULL, NULL, NULL, - {0, 0, 0}} + { EMPTY_OPTION } }; @@ -765,26 +828,25 @@ static void cmd_serviceRemoveBackend(DCB *dcb, void *a, void *b) /** * The subcommands of the remove command */ -struct subcommand removeoptions[] = { +struct subcommand removeoptions[] = +{ { - "user", - 2, - telnetdRemoveUser, - "Remove account for using maxadmin over the network. E.g.:\n" - " MaxAdmin> remove user bob somepass", - "Remove account for using maxadmin over the network. E.g.:\n" - " MaxAdmin> remove user bob somepass", + "user", + 2, 2, + telnetdRemoveUser, + "Remove account from maxadmin", + "Remove account for using maxadmin over the network. E.g.:\n" + " MaxAdmin> remove user bob somepass", {ARG_TYPE_STRING, ARG_TYPE_STRING, 0} }, { - "server", 2, cmd_serviceRemoveBackend, - "Remove a server from a service", - "Remove a server from a service. The server must exist in the configuration file.", + "server", 2, 2, cmd_serviceRemoveBackend, + "Remove a server from a service", + "Takes a server name as the parameter. The server must exist in the configuration file.", {ARG_TYPE_SERVICE, ARG_TYPE_SERVER, 0} }, { - NULL, 0, NULL, NULL, NULL, - {0, 0, 0} + EMPTY_OPTION } }; @@ -857,25 +919,26 @@ flushlogs(DCB *pdcb) /** * The subcommands of the flush command */ -struct subcommand flushoptions[] = { +struct subcommand flushoptions[] = +{ { "log", - 1, + 1, 1, flushlog, - "Flush the content of a log file, close that log, rename it and open a new log file", + "Flush log files", "Flush the content of a log file, close that log, rename it and open a new log file", {ARG_TYPE_STRING, 0, 0} }, { "logs", - 0, + 0, 0, flushlogs, - "Flush the content of all log files, close those logs, rename them and open a new log files", + "Flush log files", "Flush the content of all log files, close those logs, rename them and open a new log files", {0, 0, 0} }, { - NULL, 0, NULL, NULL, NULL, {0, 0, 0} + EMPTY_OPTION } }; @@ -883,11 +946,15 @@ struct subcommand flushoptions[] = { /** * The debug command table */ -static struct { +static struct +{ char *cmd; struct subcommand *options; -} cmds[] = { +} cmds[] = +{ { "add", addoptions }, + { "remove", removeoptions }, + { "set", setoptions }, { "clear", clearoptions }, { "disable", disableoptions }, { "enable", enableoptions }, @@ -897,11 +964,9 @@ static struct { { "flush", flushoptions }, { "list", listoptions }, { "reload", reloadoptions }, - { "remove", removeoptions }, { "restart", restartoptions }, - { "set", setoptions }, - { "show", showoptions }, { "shutdown", shutdownoptions }, + { "show", showoptions }, { "sync", syncoptions }, { NULL, NULL } }; @@ -924,63 +989,63 @@ convert_arg(int mode, char *arg, int arg_type) switch (arg_type) { - case ARG_TYPE_ADDRESS: - return (unsigned long)strtol(arg, NULL, 0); - case ARG_TYPE_STRING: - return (unsigned long)arg; - case ARG_TYPE_SERVICE: - if (mode == CLIM_USER || (rval = (unsigned long)strtol(arg, NULL, 0)) == 0) - { - rval = (unsigned long)service_find(arg); - } - return rval; - case ARG_TYPE_SERVER: - if (mode == CLIM_USER || (rval = (unsigned long)strtol(arg, NULL, 0)) == 0) - { - rval = (unsigned long)server_find_by_unique_name(arg); - } - return rval; - case ARG_TYPE_DBUSERS: - if (mode == CLIM_USER || (rval = (unsigned long)strtol(arg, NULL, 0)) == 0) - { - service = service_find(arg); - if (service) + case ARG_TYPE_ADDRESS: + return (unsigned long)strtol(arg, NULL, 0); + case ARG_TYPE_STRING: + return (unsigned long)arg; + case ARG_TYPE_SERVICE: + if (mode == CLIM_USER || (rval = (unsigned long)strtol(arg, NULL, 0)) == 0) { - return (unsigned long)(service->ports->users); + rval = (unsigned long)service_find(arg); } - else + return rval; + case ARG_TYPE_SERVER: + if (mode == CLIM_USER || (rval = (unsigned long)strtol(arg, NULL, 0)) == 0) { - return 0; + rval = (unsigned long)server_find_by_unique_name(arg); } - } - return rval; - case ARG_TYPE_DCB: - rval = (unsigned long)strtol(arg, NULL, 0); - if (mode == CLIM_USER && dcb_isvalid((DCB *)rval) == 0) - { - rval = 0; - } - return rval; - case ARG_TYPE_SESSION: - rval = (unsigned long)strtol(arg, NULL, 0); - if (mode == CLIM_USER && session_isvalid((SESSION *)rval) == 0) - { - rval = 0; - } - return rval; - case ARG_TYPE_MONITOR: - if (mode == CLIM_USER || (rval = (unsigned long)strtol(arg, NULL, 0)) == 0) - { - rval = (unsigned long)monitor_find(arg); - } - return rval; - case ARG_TYPE_FILTER: - if (mode == CLIM_USER || (rval = (unsigned long)strtol(arg, NULL, 0)) == 0) - { - rval = (unsigned long)filter_find(arg); - } - return rval; - case ARG_TYPE_NUMERIC: + return rval; + case ARG_TYPE_DBUSERS: + if (mode == CLIM_USER || (rval = (unsigned long)strtol(arg, NULL, 0)) == 0) + { + service = service_find(arg); + if (service) + { + return (unsigned long)(service->ports->users); + } + else + { + return 0; + } + } + return rval; + case ARG_TYPE_DCB: + rval = (unsigned long)strtol(arg, NULL, 0); + if (mode == CLIM_USER && dcb_isvalid((DCB *)rval) == 0) + { + rval = 0; + } + return rval; + case ARG_TYPE_SESSION: + rval = (unsigned long)strtol(arg, NULL, 0); + if (mode == CLIM_USER && session_isvalid((SESSION *)rval) == 0) + { + rval = 0; + } + return rval; + case ARG_TYPE_MONITOR: + if (mode == CLIM_USER || (rval = (unsigned long)strtol(arg, NULL, 0)) == 0) + { + rval = (unsigned long)monitor_find(arg); + } + return rval; + case ARG_TYPE_FILTER: + if (mode == CLIM_USER || (rval = (unsigned long)strtol(arg, NULL, 0)) == 0) + { + rval = (unsigned long)filter_find(arg); + } + return rval; + case ARG_TYPE_NUMERIC: { int i; for (i = 0; arg[i]; i++) @@ -1016,7 +1081,6 @@ execute_cmd(CLI_SESSION *cli) DCB *dcb = cli->session->client_dcb; int argc, i, j, found = 0; char *args[MAXARGS + 1]; - unsigned long arg1, arg2, arg3; int in_quotes = 0, escape_next = 0; char *ptr, *lptr; bool in_space = false; @@ -1139,8 +1203,9 @@ execute_cmd(CLI_SESSION *cli) dcb_printf(dcb, "Available options to the %s command:\n", args[1]); for (j = 0; cmds[i].options[j].arg1; j++) { - dcb_printf(dcb, " %-12s %s\n", cmds[i].options[j].arg1, - cmds[i].options[j].help); + dcb_printf(dcb, "'%s' - %s\n\n\t%s\n\n", cmds[i].options[j].arg1, + cmds[i].options[j].help, cmds[i].options[j].devhelp); + } } } @@ -1166,74 +1231,89 @@ execute_cmd(CLI_SESSION *cli) if (strcasecmp(args[1], cmds[i].options[j].arg1) == 0) { found = 1; /**< command and sub-command match */ - if (argc != cmds[i].options[j].n_args) + if (argc < cmds[i].options[j].argc_min) { - dcb_printf(dcb, "Incorrect number of arguments: %s %s expects %d arguments\n", + dcb_printf(dcb, "Incorrect number of arguments: %s %s expects at least %d arguments\n", cmds[i].cmd, cmds[i].options[j].arg1, - cmds[i].options[j].n_args); + cmds[i].options[j].argc_min); } else { - switch (cmds[i].options[j].n_args) - { - case 0: - cmds[i].options[j].fn(dcb); - break; - case 1: - arg1 = convert_arg(cli->mode, args[2], cmds[i].options[j].arg_types[0]); + unsigned long arg_list[MAXARGS] = {}; - if (arg1) + for (int k = 0; k < cmds[i].options[j].argc_max && k < argc; k++) + { + arg_list[k] = convert_arg(cli->mode, args[k + 2], cmds[i].options[j].arg_types[k]); + if (arg_list[k] == 0) { - cmds[i].options[j].fn(dcb, arg1); - } - else - { - dcb_printf(dcb, "Invalid argument: %s\n", - args[2]); - } - break; - case 2: - arg1 = convert_arg(cli->mode, args[2], cmds[i].options[j].arg_types[0]); - arg2 = convert_arg(cli->mode, args[3], cmds[i].options[j].arg_types[1]); - if (arg1 && arg2) - { - cmds[i].options[j].fn(dcb, arg1, arg2); - } - else if (arg1 == 0) - { - dcb_printf(dcb, "Invalid argument: %s\n", - args[2]); - } - else - { - dcb_printf(dcb, "Invalid argument: %s\n", - args[3]); - } - break; - case 3: - arg1 = convert_arg(cli->mode, args[2], cmds[i].options[j].arg_types[0]); - arg2 = convert_arg(cli->mode, args[3], cmds[i].options[j].arg_types[1]); - arg3 = convert_arg(cli->mode, args[4], cmds[i].options[j].arg_types[2]); - if (arg1 && arg2 && arg3) - { - cmds[i].options[j].fn(dcb, arg1, arg2, arg3); - } - else if (arg1 == 0) - { - dcb_printf(dcb, "Invalid argument: %s\n", - args[2]); - } - else if (arg2 == 0) - { - dcb_printf(dcb, "Invalid argument: %s\n", - args[3]); - } - else if (arg3 == 0) - { - dcb_printf(dcb, "Invalid argument: %s\n", - args[4]); + dcb_printf(dcb, "Invalid argument: %s\n", args[k + 2]); + return 0; } } + + switch (argc) + { + case 0: + cmds[i].options[j].fn(dcb); + break; + case 1: + cmds[i].options[j].fn(dcb, arg_list[0]); + break; + case 2: + cmds[i].options[j].fn(dcb, arg_list[0], arg_list[1]); + break; + case 3: + cmds[i].options[j].fn(dcb, arg_list[0], arg_list[1], arg_list[2]); + break; + case 4: + cmds[i].options[j].fn(dcb, arg_list[0], arg_list[1], arg_list[2], + arg_list[3]); + break; + case 5: + cmds[i].options[j].fn(dcb, arg_list[0], arg_list[1], arg_list[2], + arg_list[3], arg_list[4]); + break; + case 6: + cmds[i].options[j].fn(dcb, arg_list[0], arg_list[1], arg_list[2], + arg_list[3], arg_list[4], arg_list[5]); + break; + case 7: + cmds[i].options[j].fn(dcb, arg_list[0], arg_list[1], arg_list[2], + arg_list[3], arg_list[4], arg_list[5], + arg_list[6]); + break; + case 8: + cmds[i].options[j].fn(dcb, arg_list[0], arg_list[1], arg_list[2], + arg_list[3], arg_list[4], arg_list[5], + arg_list[6], arg_list[7]); + break; + case 9: + cmds[i].options[j].fn(dcb, arg_list[0], arg_list[1], arg_list[2], + arg_list[3], arg_list[4], arg_list[5], + arg_list[6], arg_list[7], arg_list[8]); + break; + case 10: + cmds[i].options[j].fn(dcb, arg_list[0], arg_list[1], arg_list[2], + arg_list[3], arg_list[4], arg_list[5], + arg_list[6], arg_list[7], arg_list[8], + arg_list[9]); + break; + case 11: + cmds[i].options[j].fn(dcb, arg_list[0], arg_list[1], arg_list[2], + arg_list[3], arg_list[4], arg_list[5], + arg_list[6], arg_list[7], arg_list[8], + arg_list[9], arg_list[10]); + break; + case 12: + cmds[i].options[j].fn(dcb, arg_list[0], arg_list[1], arg_list[2], + arg_list[3], arg_list[4], arg_list[5], + arg_list[6], arg_list[7], arg_list[8], + arg_list[9], arg_list[10], arg_list[11]); + break; + default: + dcb_printf(dcb, "Error: Maximum argument count is %d.\n", MAXARGS); + break; + } } } } @@ -1546,11 +1626,11 @@ struct log_action_entry static bool get_log_action(const char* name, struct log_action_entry* entryp) { static const struct log_action_entry entries[] = - { - { "debug", LOG_DEBUG, "debug" }, - { "trace", LOG_INFO, "info" }, - { "message", LOG_NOTICE, "notice" }, - }; + { + { "debug", LOG_DEBUG, "debug" }, + { "trace", LOG_INFO, "info" }, + { "message", LOG_NOTICE, "notice" }, + }; const int n_entries = sizeof(entries) / sizeof(entries[0]); bool found = false; @@ -1661,13 +1741,13 @@ static int compare_log_priority_entries(const void* l, const void* r) static int string_to_priority(const char* name) { static const struct log_priority_entry LOG_PRIORITY_ENTRIES[] = - { - // NOTE: If you make changes to this array, ensure that it remains alphabetically ordered. - { "debug", LOG_DEBUG }, - { "info", LOG_INFO }, - { "notice", LOG_NOTICE }, - { "warning", LOG_WARNING }, - }; + { + // NOTE: If you make changes to this array, ensure that it remains alphabetically ordered. + { "debug", LOG_DEBUG }, + { "info", LOG_INFO }, + { "notice", LOG_NOTICE }, + { "warning", LOG_WARNING }, + }; const size_t N_LOG_PRIORITY_ENTRIES = sizeof(LOG_PRIORITY_ENTRIES) / sizeof(LOG_PRIORITY_ENTRIES[0]); @@ -2008,28 +2088,29 @@ static void fail_accept( fail_accept_errno = atoi(arg1); char errbuf[MXS_STRERROR_BUFLEN]; - switch(fail_accept_errno) { - case EAGAIN: + switch (fail_accept_errno) + { + case EAGAIN: // case EWOULDBLOCK: - case EBADF: - case EINTR: - case EINVAL: - case EMFILE: - case ENFILE: - case ENOTSOCK: - case EOPNOTSUPP: - case ENOBUFS: - case ENOMEM: - case EPROTO: - fail_next_accept = failcount; - break; + case EBADF: + case EINTR: + case EINVAL: + case EMFILE: + case ENFILE: + case ENOTSOCK: + case EOPNOTSUPP: + case ENOBUFS: + case ENOMEM: + case EPROTO: + fail_next_accept = failcount; + break; - default: - dcb_printf(dcb, - "[%d, %s] is not valid errno for accept.\n", - fail_accept_errno, - strerror_r(fail_accept_errno, errbuf, sizeof(errbuf))); - return ; + default: + dcb_printf(dcb, + "[%d, %s] is not valid errno for accept.\n", + fail_accept_errno, + strerror_r(fail_accept_errno, errbuf, sizeof(errbuf))); + return ; } } #endif /* FAKE_CODE */ From 3b465c0d412325d96322fd76a320c3ea41f1dc13 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 8 Nov 2016 16:17:18 +0200 Subject: [PATCH 180/215] Add online creation of servers New servers can be created with the new `create server` command. This will create a new server but it will not add it to any service or monitor. --- server/modules/routing/debugcli/debugcmd.c | 80 ++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/server/modules/routing/debugcli/debugcmd.c b/server/modules/routing/debugcli/debugcmd.c index ee2732216..183d64e31 100644 --- a/server/modules/routing/debugcli/debugcmd.c +++ b/server/modules/routing/debugcli/debugcmd.c @@ -942,6 +942,84 @@ struct subcommand flushoptions[] = } }; +/** This is used to prevent concurrent creation or removal of servers */ +static SPINLOCK server_mod_lock = SPINLOCK_INIT; + +static void createServer(DCB *dcb, char *name, char *address, char *port, + char *protocol, char *authenticator, char *authenticator_options) +{ + spinlock_acquire(&server_mod_lock); + + if (server_find_by_unique_name(name) == NULL) + { + if (protocol == NULL) + { + protocol = "MySQLBackend"; + } + + SERVER *server = server_alloc(address, protocol, atoi(port), authenticator, + authenticator_options); + + if (server) + { + server_set_unique_name(server, name); + // TODO: Serialize the server to disk + dcb_printf(dcb, "Created server '%s'\n", name); + } + else + { + dcb_printf(dcb, "Failed to create new server, see log file for more details\n"); + } + } + else + { + dcb_printf(dcb, "Server '%s' already exists.\n", name); + } + + spinlock_release(&server_mod_lock); +} + +struct subcommand createoptions[] = +{ + { + "server", 3, 6, createServer, + "Create a new server", + "Usage: create server NAME HOST PORT [PROTOCOL] [AUTHENTICATOR] [OPTIONS]\n" + "Create a new server from the following parameters.\n" + "NAME Server name\n" + "HOST Server host address\n" + "PORT Server port\n" + "PROTOCOL Server protocol (default MySQLBackend)\n" + "AUTHENTICATOR Authenticator module name (default MySQLAuth)\n" + "OPTIONS Options for the authenticator module\n\n" + "The first three parameters are required, the others are optional.\n", + { + ARG_TYPE_STRING, ARG_TYPE_STRING, ARG_TYPE_STRING, ARG_TYPE_STRING, + ARG_TYPE_STRING, ARG_TYPE_STRING + } + }, + { + EMPTY_OPTION + } +}; + +static void destroyServer(DCB *dcb, SERVER *server) +{ + dcb_printf(dcb, "Not yet implemented.\n"); +} + +struct subcommand destroyoptions[] = +{ + { + "server", 1, 1, destroyServer, + "Destroy a server", + "Usage: destroy server NAME", + {ARG_TYPE_STRING} + }, + { + EMPTY_OPTION + } +}; /** * The debug command table @@ -954,6 +1032,8 @@ static struct { { "add", addoptions }, { "remove", removeoptions }, + { "create", createoptions }, + { "destroy", destroyoptions }, { "set", setoptions }, { "clear", clearoptions }, { "disable", disableoptions }, From dbedf57c9ad5d65c000e9cf8d337d83b32a19136 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 8 Nov 2016 20:58:17 +0200 Subject: [PATCH 181/215] Add and remove servers for monitors The `add server` command accepts a server name and either a service name or a monitor name. It will add servers to services and monitors. Since all monitors use the MONITOR_SERVER structures directly, the monitors need to be stopped before new servers are added to them --- include/maxscale/monitor.h | 3 +- server/core/monitor.c | 78 +++++++++++++++++++- server/modules/routing/debugcli/debugcmd.c | 86 ++++++++++++++++------ 3 files changed, 141 insertions(+), 26 deletions(-) diff --git a/include/maxscale/monitor.h b/include/maxscale/monitor.h index 106b17113..5ae9df416 100644 --- a/include/maxscale/monitor.h +++ b/include/maxscale/monitor.h @@ -201,7 +201,8 @@ struct monitor extern MONITOR *monitor_alloc(char *, char *); extern void monitor_free(MONITOR *); extern MONITOR *monitor_find(char *); -extern void monitorAddServer(MONITOR *, SERVER *); +extern void monitorAddServer(MONITOR *mon, SERVER *server); +extern void monitorRemoveServer(MONITOR *mon, SERVER *server); extern void monitorAddUser(MONITOR *, char *, char *); extern void monitorAddParameters(MONITOR *monitor, CONFIG_PARAMETER *params); extern void monitorStop(MONITOR *); diff --git a/server/core/monitor.c b/server/core/monitor.c index 60bc4e238..b67e6ecf4 100644 --- a/server/core/monitor.c +++ b/server/core/monitor.c @@ -258,6 +258,13 @@ monitorAddServer(MONITOR *mon, SERVER *server) /* pending status is updated by get_replication_tree */ db->pending_status = 0; + monitor_state_t old_state = mon->state; + + if (old_state == MONITOR_STATE_RUNNING) + { + monitorStop(mon); + } + spinlock_acquire(&mon->lock); if (mon->databases == NULL) @@ -274,6 +281,23 @@ monitorAddServer(MONITOR *mon, SERVER *server) ptr->next = db; } spinlock_release(&mon->lock); + + if (old_state == MONITOR_STATE_RUNNING) + { + monitorStart(mon, mon->parameters); + } +} + +static void monitor_server_free(MONITOR_SERVERS *tofree) +{ + if (tofree) + { + if (tofree->con) + { + mysql_close(tofree->con); + } + MXS_FREE(tofree); + } } /** @@ -286,11 +310,59 @@ static void monitor_servers_free(MONITOR_SERVERS *servers) { MONITOR_SERVERS *tofree = servers; servers = servers->next; - if (tofree->con) + monitor_server_free(tofree); + } +} + +/** + * Remove a server from a monitor. + * + * @param mon The Monitor instance + * @param server The Server to remove + */ +void monitorRemoveServer(MONITOR *mon, SERVER *server) +{ + monitor_state_t old_state = mon->state; + + if (old_state == MONITOR_STATE_RUNNING) + { + monitorStop(mon); + } + + spinlock_acquire(&mon->lock); + + ss_dassert(mon->databases); + MONITOR_SERVERS *ptr = mon->databases; + + if (ptr->server == server) + { + mon->databases = mon->databases->next; + } + else + { + MONITOR_SERVERS *prev = ptr; + + while (ptr) { - mysql_close(tofree->con); + if (ptr->server == server) + { + prev->next = ptr->next; + break; + } + prev = ptr; + ptr = ptr->next; } - MXS_FREE(tofree); + } + spinlock_release(&mon->lock); + + if (ptr) + { + monitor_servers_free(ptr); + } + + if (old_state == MONITOR_STATE_RUNNING) + { + monitorStart(mon, mon->parameters); } } diff --git a/server/modules/routing/debugcli/debugcmd.c b/server/modules/routing/debugcli/debugcmd.c index 183d64e31..973b72258 100644 --- a/server/modules/routing/debugcli/debugcmd.c +++ b/server/modules/routing/debugcli/debugcmd.c @@ -779,15 +779,36 @@ struct subcommand failoptions[] = static void telnetdAddUser(DCB *, char *user, char *password); -static void cmd_serviceAddBackend(DCB *dcb, void *a, void *b) +static void cmd_AddServer(DCB *dcb, void *a, void *b) { - SERVICE *service = (SERVICE*)a; - SERVER *server = (SERVER*)b; + SERVER *server = (SERVER*)a; + char *name = (char*)b; - serviceAddBackend(service, server); + SERVICE *service = service_find(name); + MONITOR *monitor = monitor_find(name); - MXS_NOTICE("Added server '%s' to service '%s'", server->unique_name, service->name); - dcb_printf(dcb, "Added server '%s' to service '%s'\n", server->unique_name, service->name); + if (service || monitor) + { + ss_dassert(service == NULL || monitor == NULL); + + if (service) + { + serviceAddBackend(service, server); + } + else if (monitor) + { + monitorAddServer(monitor, server); + } + + const char *target = service ? "service" : "monitor"; + + MXS_NOTICE("Added server '%s' to %s '%s'", server->unique_name, target, name); + dcb_printf(dcb, "Added server '%s' to %s '%s'\n", server->unique_name, target, name); + } + else + { + dcb_printf(dcb, "No service or monitor with the name '%s'\n", name); + } } /** @@ -803,26 +824,46 @@ struct subcommand addoptions[] = {ARG_TYPE_STRING, ARG_TYPE_STRING, 0} }, { - "server", 2, 2, cmd_serviceAddBackend, + "server", 2, 2, cmd_AddServer, "Add a new server to a service", - "Takes a server name as the parameter. The server must exist in the configuration file.", - {ARG_TYPE_SERVICE, ARG_TYPE_SERVER, 0} + "Usage: add server SERVER TARGET\n" + "The TARGET must be either a service or a monitor", + {ARG_TYPE_SERVER, ARG_TYPE_STRING, 0} }, - { EMPTY_OPTION } + { EMPTY_OPTION} }; static void telnetdRemoveUser(DCB *, char *user, char *password); -static void cmd_serviceRemoveBackend(DCB *dcb, void *a, void *b) +static void cmd_RemoveServer(DCB *dcb, void *a, void *b) { - SERVICE *service = (SERVICE*)a; - SERVER *server = (SERVER*)b; + SERVER *server = (SERVER*)a; + char *name = (char*)b; + SERVICE *service = service_find(name); + MONITOR *monitor = monitor_find(name); - serviceRemoveBackend(service, server); + if (service || monitor) + { + ss_dassert(service == NULL || monitor == NULL); - MXS_NOTICE("Removed server '%s' from service '%s'", server->unique_name, service->name); - dcb_printf(dcb, "Removed server '%s' from service '%s'\n", server->unique_name, service->name); + if (service) + { + serviceRemoveBackend(service, server); + } + else if (monitor) + { + monitorRemoveServer(monitor, server); + } + + const char *target = service ? "service" : "monitor"; + MXS_NOTICE("Removed server '%s' from %s '%s'", server->unique_name, target, name); + dcb_printf(dcb, "Removed server '%s' from %s '%s'\n", server->unique_name, target, name); + } + else + { + dcb_printf(dcb, "No service or monitor with the name '%s'\n", name); + } } /** @@ -837,13 +878,14 @@ struct subcommand removeoptions[] = "Remove account from maxadmin", "Remove account for using maxadmin over the network. E.g.:\n" " MaxAdmin> remove user bob somepass", - {ARG_TYPE_STRING, ARG_TYPE_STRING, 0} + {ARG_TYPE_STRING, ARG_TYPE_STRING} }, { - "server", 2, 2, cmd_serviceRemoveBackend, - "Remove a server from a service", - "Takes a server name as the parameter. The server must exist in the configuration file.", - {ARG_TYPE_SERVICE, ARG_TYPE_SERVER, 0} + "server", 2, 2, cmd_RemoveServer, + "Remove a server from a service or a monitor", + "Usage: remove server SERVER TARGET\n" + "The TARGET must be either a service or a monitor", + {ARG_TYPE_SERVER, ARG_TYPE_STRING} }, { EMPTY_OPTION @@ -1331,7 +1373,7 @@ execute_cmd(CLI_SESSION *cli) } } - switch (argc) + switch (cmds[i].options[j].argc_max) { case 0: cmds[i].options[j].fn(dcb); From b51af51365c9be005b28718b8202134b7e58e948 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 8 Nov 2016 21:02:47 +0200 Subject: [PATCH 182/215] Allow monitors and services to start without servers MaxScale can now start without any defined monitors. This allows the core services to be configured beforehand. With the changes to dynamic modifications to servers, automatic scaling of slaves is possible. --- server/core/config.c | 61 ++++++++----------- server/core/monitor.c | 17 ++---- server/core/service.c | 3 +- .../modules/authenticator/MySQLAuth/dbusers.c | 9 +-- server/modules/monitor/mysqlmon/mysql_mon.c | 2 +- 5 files changed, 36 insertions(+), 56 deletions(-) diff --git a/server/core/config.c b/server/core/config.c index 7958cb175..31c550b9a 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -2824,12 +2824,6 @@ int configure_new_service(CONFIG_CONTEXT *context, CONFIG_CONTEXT *obj) s = strtok_r(NULL, ",", &lasts); } } - else if (servers == NULL && !is_internal_service(router)) - { - MXS_ERROR("The service '%s' is missing a definition of the servers " - "that provide the service.", obj->object); - error_count++; - } if (roptions) { @@ -2882,12 +2876,6 @@ int create_new_monitor(CONFIG_CONTEXT *context, CONFIG_CONTEXT *obj, HASHTABLE* } char *servers = config_get_value(obj->parameters, "servers"); - if (servers == NULL) - { - MXS_ERROR("Monitor '%s' is missing the 'servers' parameter that " - "lists the servers that it monitors.", obj->object); - error_count++; - } if (error_count == 0) { @@ -2934,36 +2922,39 @@ int create_new_monitor(CONFIG_CONTEXT *context, CONFIG_CONTEXT *obj, HASHTABLE* } } - /* get the servers to monitor */ - char *s, *lasts; - s = strtok_r(servers, ",", &lasts); - while (s) + if (servers) { - CONFIG_CONTEXT *obj1 = context; - int found = 0; - while (obj1) + /* get the servers to monitor */ + char *s, *lasts; + s = strtok_r(servers, ",", &lasts); + while (s) { - if (strcmp(trim(s), obj1->object) == 0 && obj->element && obj1->element) + CONFIG_CONTEXT *obj1 = context; + int found = 0; + while (obj1) { - found = 1; - if (hashtable_add(monitorhash, obj1->object, "") == 0) + if (strcmp(trim(s), obj1->object) == 0 && obj->element && obj1->element) { - MXS_WARNING("Multiple monitors are monitoring server [%s]. " - "This will cause undefined behavior.", - obj1->object); + found = 1; + if (hashtable_add(monitorhash, obj1->object, "") == 0) + { + MXS_WARNING("Multiple monitors are monitoring server [%s]. " + "This will cause undefined behavior.", + obj1->object); + } + monitorAddServer(obj->element, obj1->element); } - monitorAddServer(obj->element, obj1->element); + obj1 = obj1->next; + } + if (!found) + { + MXS_ERROR("Unable to find server '%s' that is " + "configured in the monitor '%s'.", s, obj->object); + error_count++; } - obj1 = obj1->next; - } - if (!found) - { - MXS_ERROR("Unable to find server '%s' that is " - "configured in the monitor '%s'.", s, obj->object); - error_count++; - } - s = strtok_r(NULL, ",", &lasts); + s = strtok_r(NULL, ",", &lasts); + } } char *user = config_get_value(obj->parameters, "user"); diff --git a/server/core/monitor.c b/server/core/monitor.c index b67e6ecf4..d1d2a8d88 100644 --- a/server/core/monitor.c +++ b/server/core/monitor.c @@ -58,7 +58,7 @@ const monitor_def_t monitor_event_definitions[MAX_MONITOR_EVENT] = static MONITOR *allMonitors = NULL; static SPINLOCK monLock = SPINLOCK_INIT; -static void monitor_servers_free(MONITOR_SERVERS *servers); +static void monitor_server_free_all(MONITOR_SERVERS *servers); /** * Allocate a new monitor, load the associated module for the monitor @@ -142,7 +142,7 @@ monitor_free(MONITOR *mon) } spinlock_release(&monLock); free_config_parameter(mon->parameters); - monitor_servers_free(mon->databases); + monitor_server_free_all(mon->databases); MXS_FREE(mon->name); MXS_FREE(mon); } @@ -304,7 +304,7 @@ static void monitor_server_free(MONITOR_SERVERS *tofree) * Free monitor server list * @param servers Servers to free */ -static void monitor_servers_free(MONITOR_SERVERS *servers) +static void monitor_server_free_all(MONITOR_SERVERS *servers) { while (servers) { @@ -357,7 +357,7 @@ void monitorRemoveServer(MONITOR *mon, SERVER *server) if (ptr) { - monitor_servers_free(ptr); + monitor_server_free(ptr); } if (old_state == MONITOR_STATE_RUNNING) @@ -608,13 +608,8 @@ monitorGetList() */ bool check_monitor_permissions(MONITOR* monitor, const char* query) { - if (monitor->databases == NULL) - { - MXS_ERROR("[%s] Monitor is missing the servers parameter.", monitor->name); - return false; - } - - if (config_get_global_options()->skip_permission_checks) + if (monitor->databases == NULL || // No servers to check + config_get_global_options()->skip_permission_checks) { return true; } diff --git a/server/core/service.c b/server/core/service.c index aef3b022d..c2196ac51 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -821,13 +821,12 @@ void serviceRemoveBackend(SERVICE *service, const SERVER *server) { spinlock_acquire(&service->spin); - service->n_dbref--; - for (SERVER_REF *ref = service->dbref; ref; ref = ref->next) { if (ref->server == server) { ref->active = false; + service->n_dbref--; break; } } diff --git a/server/modules/authenticator/MySQLAuth/dbusers.c b/server/modules/authenticator/MySQLAuth/dbusers.c index ec4cbad6f..d08a0d40a 100644 --- a/server/modules/authenticator/MySQLAuth/dbusers.c +++ b/server/modules/authenticator/MySQLAuth/dbusers.c @@ -2649,17 +2649,12 @@ static bool check_server_permissions(SERVICE *service, SERVER* server, bool check_service_permissions(SERVICE* service) { if (is_internal_service(service->routerModule) || - config_get_global_options()->skip_permission_checks) + config_get_global_options()->skip_permission_checks || + service->dbref == NULL) // No servers to check { return true; } - if (service->dbref == NULL) - { - MXS_ERROR("[%s] Service is missing the servers parameter.", service->name); - return false; - } - char *user, *password; if (serviceGetUser(service, &user, &password) == 0) diff --git a/server/modules/monitor/mysqlmon/mysql_mon.c b/server/modules/monitor/mysqlmon/mysql_mon.c index 86967962d..cc2b425df 100644 --- a/server/modules/monitor/mysqlmon/mysql_mon.c +++ b/server/modules/monitor/mysqlmon/mysql_mon.c @@ -215,7 +215,7 @@ bool init_server_info(MYSQL_MONITOR *handle, MONITOR_SERVERS *database) while (database) { /** Delete any existing structures and replace them with empty ones */ - hashtable_delete(handle->server_info, database->server); + hashtable_delete(handle->server_info, database->server->unique_name); if (!hashtable_add(handle->server_info, database->server->unique_name, &info)) { From 35d295939531b020a7e1e8f6b3d846bbffac98e9 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Wed, 9 Nov 2016 05:59:10 +0200 Subject: [PATCH 183/215] Enable online modification of servers The address, port, monuser and monpw parameters of an existing server can be changed at runtime. The support for enabling SSL will come in a later commit. Allowing servers to be modified could also be done by destroying and recreating them. Since the servers are never actually destroyed, it is better to allow the alteration of the existing ones. --- include/maxscale/server.h | 12 +++-- server/core/config.c | 3 +- server/core/server.c | 51 +++++++++------------ server/modules/monitor/mysqlmon/mysql_mon.c | 7 +-- server/modules/routing/debugcli/debugcmd.c | 51 +++++++++++++++++++++ 5 files changed, 83 insertions(+), 41 deletions(-) diff --git a/include/maxscale/server.h b/include/maxscale/server.h index 45be5cd39..a67199fe2 100644 --- a/include/maxscale/server.h +++ b/include/maxscale/server.h @@ -48,6 +48,8 @@ MXS_BEGIN_DECLS #define MAX_SERVER_NAME_LEN 1024 +#define MAX_SERVER_MONUSER_LEN 512 +#define MAX_SERVER_MONPW_LEN 512 #define MAX_NUM_SLAVES 128 /**< Maximum number of slaves under a single server*/ /** @@ -86,7 +88,7 @@ typedef struct server #endif SPINLOCK lock; /**< Common access lock */ char *unique_name; /**< Unique name for the server */ - char *name; /**< Server name/IP address*/ + char name[MAX_SERVER_NAME_LEN]; /**< Server name/IP address*/ unsigned short port; /**< Port to listen on */ char *protocol; /**< Protocol module to use */ char *authenticator; /**< Authenticator module name */ @@ -94,8 +96,8 @@ typedef struct server char *auth_options; /**< Authenticator options */ SSL_LISTENER *server_ssl; /**< SSL data structure for server, if any */ unsigned int status; /**< Status flag bitmap for the server */ - char *monuser; /**< User name to use to monitor the db */ - char *monpw; /**< Password to use to monitor the db */ + char monuser[MAX_SERVER_MONUSER_LEN]; /**< User name to use to monitor the db */ + char monpw[MAX_SERVER_MONPW_LEN]; /**< Password to use to monitor the db */ SERVER_STATS stats; /**< The server statistics */ struct server *next; /**< Next server */ struct server *nextdb; /**< Next server in list attached to a service */ @@ -217,7 +219,7 @@ extern void server_transfer_status(SERVER *dest_server, SERVER *source_server); extern void serverAddMonUser(SERVER *, char *, char *); extern void serverAddParameter(SERVER *, char *, char *); extern char *serverGetParameter(SERVER *, char *); -extern void server_update(SERVER *, char *, char *, char *); +extern void server_update_credentials(SERVER *, char *, char *); extern void server_set_unique_name(SERVER *, char *); extern DCB *server_get_persistent(SERVER *, char *, const char *); extern void server_update_address(SERVER *, char *); @@ -225,6 +227,8 @@ extern void server_update_port(SERVER *, unsigned short); extern RESULTSET *serverGetList(); extern unsigned int server_map_status(char *str); extern bool server_set_version_string(SERVER* server, const char* string); +extern bool server_is_ssl_parameter(const char *key); +extern void server_update_ssl(SERVER *server, const char *key, const char *value); /** * @brief Serialize a server to a file diff --git a/server/core/config.c b/server/core/config.c index 31c550b9a..fed157b28 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -1800,10 +1800,9 @@ process_config_update(CONFIG_CONTEXT *context) if (address && port && (server = server_find(address, atoi(port))) != NULL) { - char *protocol = config_get_value(obj->parameters, "protocol"); char *monuser = config_get_value(obj->parameters, "monuser"); char *monpw = config_get_value(obj->parameters, "monpw"); - server_update(server, protocol, monuser, monpw); + server_update_credentials(server, monuser, monpw); obj->element = server; } else diff --git a/server/core/server.c b/server/core/server.c index 1350bec49..749bc49d3 100644 --- a/server/core/server.c +++ b/server/core/server.c @@ -95,12 +95,11 @@ server_alloc(char *servname, char *protocol, unsigned short port, char *authenti return NULL; } - servname = MXS_STRNDUP(servname, MAX_SERVER_NAME_LEN); protocol = MXS_STRDUP(protocol); SERVER *server = (SERVER *)MXS_CALLOC(1, sizeof(SERVER)); - if (!servname || !protocol || !server || !authenticator) + if (!protocol || !server || !authenticator) { MXS_FREE(servname); MXS_FREE(protocol); @@ -113,7 +112,7 @@ server_alloc(char *servname, char *protocol, unsigned short port, char *authenti server->server_chk_top = CHK_NUM_SERVER; server->server_chk_tail = CHK_NUM_SERVER; #endif - server->name = servname; + snprintf(server->name, sizeof(server->name), "%s", servname); server->protocol = protocol; server->authenticator = authenticator; server->auth_instance = auth_instance; @@ -131,6 +130,8 @@ server_alloc(char *servname, char *protocol, unsigned short port, char *authenti server->persistmax = 0; server->persistmaxtime = 0; server->persistpoolmax = 0; + server->monuser[0] = '\0'; + server->monpw[0] = '\0'; spinlock_init(&server->persistlock); spinlock_acquire(&server_spin); @@ -785,8 +786,8 @@ server_transfer_status(SERVER *dest_server, SERVER *source_server) void serverAddMonUser(SERVER *server, char *user, char *passwd) { - server->monuser = MXS_STRDUP_A(user); - server->monpw = MXS_STRDUP_A(passwd); + snprintf(server->monuser, sizeof(server->monuser), "%s", user); + snprintf(server->monpw, sizeof(server->monpw), "%s", passwd); } /** @@ -803,28 +804,13 @@ serverAddMonUser(SERVER *server, char *user, char *passwd) * @param passwd The password to use for the monitor user */ void -server_update(SERVER *server, char *protocol, char *user, char *passwd) +server_update_credentials(SERVER *server, char *user, char *passwd) { - if (!strcmp(server->protocol, protocol)) - { - MXS_NOTICE("Update server protocol for server %s to protocol %s.", - server->name, - protocol); - MXS_FREE(server->protocol); - server->protocol = MXS_STRDUP_A(protocol); - } - if (user != NULL && passwd != NULL) { - if (strcmp(server->monuser, user) == 0 || - strcmp(server->monpw, passwd) == 0) - { - MXS_NOTICE("Update server monitor credentials for server %s", - server->name); - MXS_FREE(server->monuser); - MXS_FREE(server->monpw); - serverAddMonUser(server, user, passwd); - } + snprintf(server->monuser, sizeof(server->monuser), "%s", user); + snprintf(server->monpw, sizeof(server->monpw), "%s", passwd); + MXS_NOTICE("Updated monitor credentials for server '%s'", server->name); } } @@ -989,11 +975,7 @@ server_update_address(SERVER *server, char *address) spinlock_acquire(&server_spin); if (server && address) { - if (server->name) - { - MXS_FREE(server->name); - } - server->name = MXS_STRDUP_A(address); + strcpy(server->name, address); } spinlock_release(&server_spin); } @@ -1180,3 +1162,14 @@ bool server_serialize(SERVER *server, const char *filename) return true; } + +bool server_is_ssl_parameter(const char *key) +{ + // TODO: Implement this + return false; +} + +void server_update_ssl(SERVER *server, const char *key, const char *value) +{ + // TODO: Implement this +} diff --git a/server/modules/monitor/mysqlmon/mysql_mon.c b/server/modules/monitor/mysqlmon/mysql_mon.c index cc2b425df..5eb0e72b1 100644 --- a/server/modules/monitor/mysqlmon/mysql_mon.c +++ b/server/modules/monitor/mysqlmon/mysql_mon.c @@ -692,12 +692,7 @@ monitorDatabase(MONITOR *mon, MONITOR_SERVERS *database) unsigned long int server_version = 0; char *server_string; - if (database->server->monuser != NULL) - { - uname = database->server->monuser; - } - - if (uname == NULL) + if (!database->server->monuser[0]) { return; } diff --git a/server/modules/routing/debugcli/debugcmd.c b/server/modules/routing/debugcli/debugcmd.c index 973b72258..61a36ceb1 100644 --- a/server/modules/routing/debugcli/debugcmd.c +++ b/server/modules/routing/debugcli/debugcmd.c @@ -1063,6 +1063,56 @@ struct subcommand destroyoptions[] = } }; +static void alterServer(DCB *dcb, SERVER *server, char *key, char *value) +{ + bool unknown = false; + + if (strcmp(key, "address") == 0) + { + server_update_address(server, value); + } + else if (strcmp(key, "port") == 0) + { + server_update_port(server, atoi(value)); + } + else if (strcmp(key, "monuser") == 0) + { + server_update_credentials(server, value, server->monpw); + } + else if (strcmp(key, "monpw") == 0) + { + server_update_credentials(server, server->monuser, value); + } + else if (server_is_ssl_parameter(key)) + { + server_update_ssl(server, key, value); + } + else + { + unknown = true; + } + + if (unknown) + { + dcb_printf(dcb, "Unknown parameter '%s'", key); + } +} + +struct subcommand alteroptions[] = +{ + { + "server", 3, 3, alterServer, + "Alter server parameters", + "Usage: alter server NAME KEY VALUE\n" + "This will alter an existing parameter of a server. The accepted values\n" + "for KEY are: 'address', 'port', 'monuser', 'monpw'", + {ARG_TYPE_SERVER, ARG_TYPE_STRING, ARG_TYPE_STRING} + }, + { + EMPTY_OPTION + } +}; + /** * The debug command table */ @@ -1076,6 +1126,7 @@ static struct { "remove", removeoptions }, { "create", createoptions }, { "destroy", destroyoptions }, + { "alter", alteroptions }, { "set", setoptions }, { "clear", clearoptions }, { "disable", disableoptions }, From 4b82f83637de5290ab64f38a44f4f58bdd0982b7 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 10 Nov 2016 10:22:45 +0200 Subject: [PATCH 184/215] Enable online modification of monitors The monitor parameters can now be changed via maxadmin. These changes are not persisted to disk yet. --- include/maxscale/monitor.h | 7 +- server/core/monitor.c | 9 +-- server/modules/routing/debugcli/debugcmd.c | 86 ++++++++++++++++++++++ 3 files changed, 95 insertions(+), 7 deletions(-) diff --git a/include/maxscale/monitor.h b/include/maxscale/monitor.h index 5ae9df416..d5ca21740 100644 --- a/include/maxscale/monitor.h +++ b/include/maxscale/monitor.h @@ -137,6 +137,9 @@ typedef enum #define MONITOR_INTERVAL 10000 // in milliseconds #define MONITOR_DEFAULT_ID 1UL // unsigned long value +#define MAX_MONITOR_USER_LEN 512 +#define MAX_MONITOR_PASSWORD_LEN 512 + /* * Create declarations of the enum for monitor events and also the array of * structs containing the matching names. The data is taken from def_monitor_event.h @@ -177,8 +180,8 @@ typedef struct monitor_servers struct monitor { char *name; /**< The name of the monitor module */ - char *user; /*< Monitor username */ - char *password; /*< Monitor password */ + char user[MAX_MONITOR_USER_LEN]; /*< Monitor username */ + char password[MAX_MONITOR_PASSWORD_LEN]; /*< Monitor password */ SPINLOCK lock; CONFIG_PARAMETER* parameters; /*< configuration parameters */ MONITOR_SERVERS* databases; /*< List of databases the monitor monitors */ diff --git a/server/core/monitor.c b/server/core/monitor.c index d1d2a8d88..25aa124a0 100644 --- a/server/core/monitor.c +++ b/server/core/monitor.c @@ -93,9 +93,8 @@ monitor_alloc(char *name, char *module) mon->name = name; mon->handle = NULL; mon->databases = NULL; - mon->password = NULL; - mon->user = NULL; - mon->password = NULL; + *mon->password = '\0'; + *mon->user = '\0'; mon->read_timeout = DEFAULT_READ_TIMEOUT; mon->write_timeout = DEFAULT_WRITE_TIMEOUT; mon->connect_timeout = DEFAULT_CONNECT_TIMEOUT; @@ -377,8 +376,8 @@ void monitorRemoveServer(MONITOR *mon, SERVER *server) void monitorAddUser(MONITOR *mon, char *user, char *passwd) { - mon->user = MXS_STRDUP_A(user); - mon->password = MXS_STRDUP_A(passwd); + snprintf(mon->user, sizeof(mon->user), "%s", user); + snprintf(mon->password, sizeof(mon->password), "%s", passwd); } /** diff --git a/server/modules/routing/debugcli/debugcmd.c b/server/modules/routing/debugcli/debugcmd.c index 61a36ceb1..a59785d91 100644 --- a/server/modules/routing/debugcli/debugcmd.c +++ b/server/modules/routing/debugcli/debugcmd.c @@ -1098,6 +1098,83 @@ static void alterServer(DCB *dcb, SERVER *server, char *key, char *value) } } +/** + * @brief Convert a string value to a positive integer + * + * If the value is not a positive integer, an error is printed to @c dcb. + * + * @param dcb Client DCB + * @param value String value + * @return 0 on error, otherwise a positive integer + */ +static long get_positive_int(DCB *dcb, const char *value) +{ + char *endptr; + long ival = strtol(value, &endptr, 10); + + if (*endptr == '\0' && ival > 0) + { + return ival; + } + + dcb_printf(dcb, "Invalid value: %s", value); + return 0; +} + +static void alterMonitor(DCB *dcb, MONITOR *monitor, char *key, char *value) +{ + bool unknown = false; + if (strcmp(key, "user") == 0) + { + monitorAddUser(monitor, value, monitor->password); + } + else if (strcmp(key, "password") == 0) + { + monitorAddUser(monitor, monitor->user, value); + } + else if (strcmp(key, "monitor_interval") == 0) + { + long ival = get_positive_int(dcb, value); + if (ival) + { + monitorSetInterval(monitor, ival); + } + } + else if (strcmp(key, "backend_connect_timeout") == 0) + { + long ival = get_positive_int(dcb, value); + if (ival) + { + monitorSetNetworkTimeout(monitor, MONITOR_CONNECT_TIMEOUT, ival); + } + } + else if (strcmp(key, "backend_write_timeout") == 0) + { + long ival = get_positive_int(dcb, value); + if (ival) + { + monitorSetNetworkTimeout(monitor, MONITOR_READ_TIMEOUT, ival); + } + } + else if (strcmp(key, "backend_read_timeout") == 0) + { + long ival = get_positive_int(dcb, value); + if (ival) + { + monitorSetNetworkTimeout(monitor, MONITOR_WRITE_TIMEOUT, ival); + } + } + else + { + unknown = true; + } + + if (unknown) + { + dcb_printf(dcb, "Unknown parameter '%s'", key); + } +} + struct subcommand alteroptions[] = { { @@ -1108,6 +1185,15 @@ struct subcommand alteroptions[] = "for KEY are: 'address', 'port', 'monuser', 'monpw'", {ARG_TYPE_SERVER, ARG_TYPE_STRING, ARG_TYPE_STRING} }, + { + "monitor", 3, 3, alterMonitor, + "Alter monitor parameters", + "Usage: alter monitor NAME KEY VALUE\n" + "This will alter an existing parameter of a monitor. The accepted values\n" + "for KEY are: 'user', 'password', 'monitor_interval',\n" + "'backend_connect_timeout', 'backend_write_timeout', 'backend_read_timeout'", + {ARG_TYPE_MONITOR, ARG_TYPE_STRING, ARG_TYPE_STRING} + }, { EMPTY_OPTION } From ec894f41b8c79b3595a4dd6a13a3f8b087d73451 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 10 Nov 2016 10:35:10 +0200 Subject: [PATCH 185/215] Store persisted config directory The persisted configuration subdirectory will be used to store changes to the configuration. The gwdirs.h header now supports setting and getting the value for this directory. --- cmake/install_layout.cmake | 2 ++ include/maxscale/gwdirs.h.in | 8 +++++++- server/core/config.c | 1 + server/core/gwdirs.c | 20 ++++++++++++++++++++ 4 files changed, 30 insertions(+), 1 deletion(-) diff --git a/cmake/install_layout.cmake b/cmake/install_layout.cmake index 3eb9503f3..1abf7a7e2 100644 --- a/cmake/install_layout.cmake +++ b/cmake/install_layout.cmake @@ -20,6 +20,7 @@ set(DEFAULT_CACHE_SUBPATH "cache/maxscale" CACHE PATH "Default cache subpath") set(DEFAULT_LANG_SUBPATH "lib/maxscale" CACHE PATH "Default language file subpath") set(DEFAULT_EXEC_SUBPATH "${MAXSCALE_BINDIR}" CACHE PATH "Default executable subpath") set(DEFAULT_CONFIG_SUBPATH "etc" CACHE PATH "Default configuration subpath") +set(DEFAULT_CONFIG_PERSIST_SUBPATH "maxscale.cnf.d" CACHE PATH "Default persisted configuration subpath") set(DEFAULT_PIDDIR ${MAXSCALE_VARDIR}/${DEFAULT_PID_SUBPATH} CACHE PATH "Default PID file directory") set(DEFAULT_LOGDIR ${MAXSCALE_VARDIR}/${DEFAULT_LOG_SUBPATH} CACHE PATH "Default log directory") @@ -29,6 +30,7 @@ set(DEFAULT_CACHEDIR ${MAXSCALE_VARDIR}/${DEFAULT_CACHE_SUBPATH} CACHE PATH "Def set(DEFAULT_LANGDIR ${MAXSCALE_VARDIR}/${DEFAULT_LANG_SUBPATH} CACHE PATH "Default language file directory") set(DEFAULT_EXECDIR ${CMAKE_INSTALL_PREFIX}/${DEFAULT_EXEC_SUBPATH} CACHE PATH "Default executable directory") set(DEFAULT_CONFIGDIR /${DEFAULT_CONFIG_SUBPATH} CACHE PATH "Default configuration directory") +set(DEFAULT_CONFIG_PERSISTDIR ${DEFAULT_DATADIR}/${DEFAULT_CONFIG_PERSIST_SUBPATH} CACHE PATH "Default persisted configuration directory") # Massage TARGET_COMPONENT into a list if (TARGET_COMPONENT) diff --git a/include/maxscale/gwdirs.h.in b/include/maxscale/gwdirs.h.in index 909d324f0..695a57277 100644 --- a/include/maxscale/gwdirs.h.in +++ b/include/maxscale/gwdirs.h.in @@ -32,10 +32,12 @@ MXS_BEGIN_DECLS #define MXS_DEFAULT_LANG_SUBPATH "@DEFAULT_LANG_SUBPATH@" #define MXS_DEFAULT_EXEC_SUBPATH "@DEFAULT_EXEC_SUBPATH@" #define MXS_DEFAULT_CONFIG_SUBPATH "@DEFAULT_CONFIG_SUBPATH@" +#define MXS_DEFAULT_CONFIG_PERSIST_SUBPATH "@DEFAULT_CONFIG_PERSIST_SUBPATH@" /** Default file locations, configured by CMake */ static const char* default_cnf_fname = "maxscale.cnf"; static const char* default_configdir = "@DEFAULT_CONFIGDIR@"; + /*< This should be changed to just /run eventually, * the /var/run folder is an old standard and the newer FSH 3.0 * uses /run for PID files.*/ @@ -46,8 +48,10 @@ static const char* default_libdir = "@DEFAULT_LIBDIR@"; static const char* default_cachedir = "@DEFAULT_CACHEDIR@"; static const char* default_langdir = "@DEFAULT_LANGDIR@"; static const char* default_execdir = "@DEFAULT_EXECDIR@"; +static const char* default_config_persistdir = "@DEFAULT_CONFIG_PERSISTDIR@"; -static char* configdir = NULL; +static char* configdir = NULL; /*< Where the config file is found e.g. /etc/ */ +static char* config_persistdir = NULL;/*< Persisted configs e.g. /var/lib/maxscale.cnf.d/ */ static char* logdir = NULL; static char* libdir = NULL; static char* cachedir = NULL; @@ -62,6 +66,7 @@ void set_datadir(char* param); void set_process_datadir(char* param); void set_cachedir(char* param); void set_configdir(char* param); +void set_config_persistdir(char* param); void set_logdir(char* param); void set_langdir(char* param); void set_piddir(char* param); @@ -71,6 +76,7 @@ char* get_datadir(); char* get_process_datadir(); char* get_cachedir(); char* get_configdir(); +char* get_config_persistdir(); char* get_piddir(); char* get_logdir(); char* get_langdir(); diff --git a/server/core/config.c b/server/core/config.c index fed157b28..8b2ce58ae 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -65,6 +65,7 @@ #include #include #include +#include typedef struct duplicate_context { diff --git a/server/core/gwdirs.c b/server/core/gwdirs.c index eb8ca6394..22de96bc6 100644 --- a/server/core/gwdirs.c +++ b/server/core/gwdirs.c @@ -26,6 +26,17 @@ void set_configdir(char* str) configdir = str; } +/** + * Set the configuration parts file directory + * @param str Path to directory + */ +void set_config_persistdir(char* str) +{ + MXS_FREE(config_persistdir); + clean_up_pathname(str); + config_persistdir = str; +} + /** * Set the log file directory * @param str Path to directory @@ -160,6 +171,15 @@ char* get_configdir() return configdir ? configdir : (char*) default_configdir; } +/** + * Get the configuration file directory + * @return The path to the configuration file directory + */ +char* get_config_persistdir() +{ + return config_persistdir ? config_persistdir : (char*) default_config_persistdir; +} + /** * Get the PID file directory which contains maxscale.pid * @return Path to the PID file directory From 6847ed1c378c520ab3b6c6d92afdef96c47d60df Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 10 Nov 2016 11:24:45 +0200 Subject: [PATCH 186/215] Create and read persisted config dir The persisted configuration directory is created and/or read when MaxScale starts. This allows the servers created at runtime to be recreated when MaxScale is restarted. --- server/core/config.c | 70 +++++++++++++++++++++++++++++++------------- 1 file changed, 49 insertions(+), 21 deletions(-) diff --git a/server/core/config.c b/server/core/config.c index 8b2ce58ae..e75196b71 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -541,6 +541,46 @@ static bool config_load_dir(const char *dir, DUPLICATE_CONTEXT *dcontext, CONFIG return rv == 0; } +/** + * Check if a directory exists + * + * This function also logs warnings if the directory cannot be accessed or if + * the file is not a directory. + * @param dir Directory to check + * @return True if the file is an existing directory + */ +static bool is_directory(const char *dir) +{ + bool rval = false; + struct stat st; + if (stat(dir, &st) == -1) + { + if (errno == ENOENT) + { + MXS_NOTICE("%s does not exist, not reading.", dir); + } + else + { + char errbuf[MXS_STRERROR_BUFLEN]; + MXS_WARNING("Could not access %s, not reading: %s", + dir, strerror_r(errno, errbuf, sizeof(errbuf))); + } + } + else + { + if (S_ISDIR(st.st_mode)) + { + rval = true; + } + else + { + MXS_WARNING("%s exists, but it is not a directory. Ignoring.", dir); + } + } + + return rval; +} + /** * @brief Load the specified configuration file for MaxScale * @@ -574,30 +614,18 @@ config_load_and_process(const char* filename, bool (*process_config)(CONFIG_CONT rval = true; - struct stat st; - if (stat(dir, &st) == -1) + if (is_directory(dir)) { - if (errno == ENOENT) - { - MXS_NOTICE("%s does not exist, not reading.", dir); - } - else - { - char errbuf[MXS_STRERROR_BUFLEN]; - MXS_WARNING("Could not access %s, not reading: %s", - dir, strerror_r(errno, errbuf, sizeof(errbuf))); - } + rval = config_load_dir(dir, &dcontext, &ccontext); } - else + + /** Create the persisted configuration directory if it doesn't exist */ + const char* persist_cnf = get_config_persistdir(); + mxs_mkdir_all(persist_cnf, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); + + if (is_directory(persist_cnf)) { - if (S_ISDIR(st.st_mode)) - { - rval = config_load_dir(dir, &dcontext, &ccontext); - } - else - { - MXS_WARNING("%s exists, but it is not a directory. Ignoring.", dir); - } + rval = config_load_dir(persist_cnf, &dcontext, &ccontext); } if (rval) From aefa5c4c578b15c626e523ff07d3260f4f682208 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 10 Nov 2016 11:56:37 +0200 Subject: [PATCH 187/215] Add configurable `persistdir` The persisted configuration file directory is now configurable. --- .../Getting-Started/Configuration-Guide.md | 10 +++++ server/core/gateway.c | 40 +++++++++++++++++-- 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/Documentation/Getting-Started/Configuration-Guide.md b/Documentation/Getting-Started/Configuration-Guide.md index bf0b33fde..fc30ab168 100644 --- a/Documentation/Getting-Started/Configuration-Guide.md +++ b/Documentation/Getting-Started/Configuration-Guide.md @@ -378,6 +378,16 @@ Configure the directory where the executable files reside. All internal processe execdir=/usr/local/bin/ ``` +#### `persistdir` + +Configure the directory where persisted configurations are stored. When a new +server is created via MaxAdmin, it will be stored in this directory. Do not use +or modify the contents of this directory, use _/etc/maxscale.cnf.d/_ instead. + +``` +persistdir=/var/lib/maxscale/maxscale.cnf.d/ +``` + #### `language` Set the folder where the errmsg.sys file is located in. MariaDB MaxScale will look for the errmsg.sys file installed with MariaDB MaxScale from this folder. diff --git a/server/core/gateway.c b/server/core/gateway.c index 138bc60bd..ba02b48b5 100644 --- a/server/core/gateway.c +++ b/server/core/gateway.c @@ -128,6 +128,7 @@ static struct option long_options[] = {"configdir", required_argument, 0, 'C'}, {"datadir", required_argument, 0, 'D'}, {"execdir", required_argument, 0, 'E'}, + {"persistdir", required_argument, 0, 'F'}, {"language", required_argument, 0, 'N'}, {"piddir", required_argument, 0, 'P'}, {"basedir", required_argument, 0, 'R'}, @@ -919,8 +920,9 @@ static void usage(void) " -B, --libdir=PATH path to module directory\n" " -C, --configdir=PATH path to configuration file directory\n" " -D, --datadir=PATH path to data directory,\n" - " stored embedded mysql tables\n" + " stores internal MaxScale data\n" " -E, --execdir=PATH path to the maxscale and other executable files\n" + " -F, --persistdir=PATH path to persisted configuration directory\n" " -N, --language=PATH path to errmsg.sys file\n" " -P, --piddir=PATH path to PID file directory\n" " -R, --basedir=PATH base path for all other paths\n" @@ -944,6 +946,7 @@ static void usage(void) " execdir : %s\n" " language : %s\n" " piddir : %s\n" + " persistdir : %s\n" "\n" "If '--basedir' is provided then all other paths, including the default\n" "configuration file path, are defined relative to that. As an example,\n" @@ -954,7 +957,8 @@ static void usage(void) progname, get_configdir(), default_cnf_fname, get_configdir(), get_logdir(), get_cachedir(), get_libdir(), - get_datadir(), get_execdir(), get_langdir(), get_piddir()); + get_datadir(), get_execdir(), get_langdir(), get_piddir(), + get_config_persistdir()); } @@ -1229,6 +1233,12 @@ bool set_dirs(const char *basedir) set_piddir(path); } + if (rv && (rv = handle_path_arg(&path, basedir, MXS_DEFAULT_DATA_SUBPATH "/" + MXS_DEFAULT_CONFIG_PERSIST_SUBPATH, true, true))) + { + set_config_persistdir(path); + } + return rv; } @@ -1327,7 +1337,7 @@ int main(int argc, char **argv) } } - while ((opt = getopt_long(argc, argv, "dcf:l:vVs:S:?L:D:C:B:U:A:P:G:N:E:", + while ((opt = getopt_long(argc, argv, "dcf:l:vVs:S:?L:D:C:B:U:A:P:G:N:E:F:", long_options, &option_index)) != -1) { bool succp = true; @@ -1477,6 +1487,16 @@ int main(int argc, char **argv) succp = false; } break; + case 'F': + if (handle_path_arg(&tmp_path, optarg, NULL, true, true)) + { + set_config_persistdir(tmp_path); + } + else + { + succp = false; + } + break; case 'R': if (handle_path_arg(&tmp_path, optarg, NULL, true, false)) { @@ -2524,6 +2544,20 @@ static int cnf_preparser(void* data, const char* section, const char* name, cons } } } + else if (strcmp(name, "persistdir") == 0) + { + if (strcmp(get_config_persistdir(), default_config_persistdir) == 0) + { + if (handle_path_arg((char**)&tmp, (char*)value, NULL, true, false)) + { + set_config_persistdir(tmp); + } + else + { + return 0; + } + } + } else if (strcmp(name, "syslog") == 0) { if (!syslog_configured) From c08a6375649805b04bd01602830508354d6f86c9 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 10 Nov 2016 12:35:21 +0200 Subject: [PATCH 188/215] Fix array usage with servers The server monuser and monpw members were used with the assumption that they could be NULL. This is no longer true since they were converted to arrays. The name member of the server was freed leading to a crash when the server unit test was run. --- server/core/server.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/core/server.c b/server/core/server.c index 749bc49d3..2e5359eef 100644 --- a/server/core/server.c +++ b/server/core/server.c @@ -175,7 +175,6 @@ server_free(SERVER *tofreeserver) spinlock_release(&server_spin); /* Clean up session and free the memory */ - MXS_FREE(tofreeserver->name); MXS_FREE(tofreeserver->protocol); MXS_FREE(tofreeserver->unique_name); MXS_FREE(tofreeserver->server_string); @@ -1088,7 +1087,7 @@ bool server_serialize(SERVER *server, const char *filename) dprintf(file, "authenticator_options=%s\n", server->auth_options); } - if (server->monpw && server->monuser) + if (*server->monpw && *server->monuser) { dprintf(file, "monitoruser=%s\n", server->monuser); dprintf(file, "monitorpw=%s\n", server->monpw); From bbd3e13a54ed9f947af31559155f19aa6b022009 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 10 Nov 2016 13:00:03 +0200 Subject: [PATCH 189/215] Only use valid server credentials The server credentials are only used if both the monuser and monpw parameters are defined. This is a sort of a bugfix as a monitor connection could use a username from server but a password from the monitor. --- server/core/monitor.c | 11 +++++++++-- server/modules/monitor/mysqlmon/mysql_mon.c | 6 ------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/server/core/monitor.c b/server/core/monitor.c index 25aa124a0..9df8401ff 100644 --- a/server/core/monitor.c +++ b/server/core/monitor.c @@ -1058,8 +1058,15 @@ mon_connect_to_db(MONITOR* mon, MONITOR_SERVERS *database) if ((database->con = mysql_init(NULL))) { - char *uname = database->server->monuser ? database->server->monuser : mon->user; - char *passwd = database->server->monpw ? database->server->monpw : mon->password; + char *uname = mon->user; + char *passwd = mon->password; + + if (database->server->monuser[0] && database->server->monpw[0]) + { + uname = database->server->monuser; + passwd = database->server->monpw; + } + char *dpwd = decryptPassword(passwd); mysql_options(database->con, MYSQL_OPT_CONNECT_TIMEOUT, (void *) &mon->connect_timeout); diff --git a/server/modules/monitor/mysqlmon/mysql_mon.c b/server/modules/monitor/mysqlmon/mysql_mon.c index 5eb0e72b1..dbf19503f 100644 --- a/server/modules/monitor/mysqlmon/mysql_mon.c +++ b/server/modules/monitor/mysqlmon/mysql_mon.c @@ -688,15 +688,9 @@ monitorDatabase(MONITOR *mon, MONITOR_SERVERS *database) MYSQL_MONITOR* handle = mon->handle; MYSQL_ROW row; MYSQL_RES *result; - char *uname = mon->user; unsigned long int server_version = 0; char *server_string; - if (!database->server->monuser[0]) - { - return; - } - /* Don't probe servers in maintenance mode */ if (SERVER_IN_MAINT(database->server)) { From c9218351b8ea6e2146c3ebe46f4b0f5b954310b9 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 10 Nov 2016 13:09:27 +0200 Subject: [PATCH 190/215] MXS-922: Serialize created servers When a server is created via MaxAdmin, it will be serialized to disk. This allows created servers to be retained through a restart of MaxScale. Currently, all serialized objects are stored in one folder and there is no structure in the created files. In the future, servers could be created under a `servers` subdirectory so that it is easier to see what was added. Whether there is a need for this will be seen. --- include/maxscale/server.h | 7 ++-- server/core/server.c | 47 +++++++++++++++++++++- server/core/test/testserver.c | 22 ++++++---- server/modules/routing/debugcli/debugcmd.c | 25 +++++++++++- 4 files changed, 86 insertions(+), 15 deletions(-) diff --git a/include/maxscale/server.h b/include/maxscale/server.h index a67199fe2..b58377c5d 100644 --- a/include/maxscale/server.h +++ b/include/maxscale/server.h @@ -234,13 +234,12 @@ extern void server_update_ssl(SERVER *server, const char *key, const char *value * @brief Serialize a server to a file * * This converts @c server into an INI format file. This allows created servers - * to be persisted to disk. A new file is only created if the file pointed by - * @c filename does not exist at the time this function is called. + * to be persisted to disk. This will replace any existing files with the same + * name. * * @param server Server to serialize - * @param filename Path to a file where the server is persisted * @return False if the serialization of the server fails, true if it was successful */ -bool server_serialize(SERVER *server, const char *filename); +bool server_serialize(SERVER *server); MXS_END_DECLS diff --git a/server/core/server.c b/server/core/server.c index 2e5359eef..a2abb488d 100644 --- a/server/core/server.c +++ b/server/core/server.c @@ -47,6 +47,7 @@ #include #include #include +#include static SPINLOCK server_spin = SPINLOCK_INIT; static SERVER *allServers = NULL; @@ -1062,7 +1063,14 @@ bool server_set_version_string(SERVER* server, const char* string) return rval; } -bool server_serialize(SERVER *server, const char *filename) +/** + * Creates a server configuration at the location pointed by @c filename + * + * @param server Server to serialize into a configuration + * @param filename Filename where configuration is written + * @return True on success, false on error + */ +static bool create_server_config(SERVER *server, const char *filename) { int file = open(filename, O_EXCL | O_CREAT | O_WRONLY, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); @@ -1162,6 +1170,43 @@ bool server_serialize(SERVER *server, const char *filename) return true; } +bool server_serialize(SERVER *server) +{ + bool rval = false; + char filename[PATH_MAX]; + snprintf(filename, sizeof(filename), "%s/%s.cnf.tmp", get_config_persistdir(), + server->unique_name); + + if (unlink(filename) == -1 && errno != ENOENT) + { + char err[MXS_STRERROR_BUFLEN]; + MXS_ERROR("Failed to remove temporary server configuration at '%s': %d, %s", + filename, errno, strerror_r(errno, err, sizeof(err))); + } + else if (create_server_config(server, filename)) + { + char final_filename[PATH_MAX]; + strcpy(final_filename, filename); + + char *dot = strrchr(final_filename, '.'); + ss_dassert(dot); + *dot = '\0'; + + if (rename(filename, final_filename) == 0) + { + rval = true; + } + else + { + char err[MXS_STRERROR_BUFLEN]; + MXS_ERROR("Failed to rename temporary server configuration at '%s': %d, %s", + filename, errno, strerror_r(errno, err, sizeof(err))); + } + } + + return rval; +} + bool server_is_ssl_parameter(const char *key) { // TODO: Implement this diff --git a/server/core/test/testserver.c b/server/core/test/testserver.c index 5a1f6f0ba..bb195885e 100644 --- a/server/core/test/testserver.c +++ b/server/core/test/testserver.c @@ -140,31 +140,37 @@ bool test_load_config(const char *input, SERVER *server) bool test_serialize() { char name[] = "serialized-server"; + char config_name[] = "serialized-server.cnf"; + char old_config_name[] = "serialized-server.cnf.old"; + char *persist_dir = MXS_STRDUP_A("./"); + set_config_persistdir(persist_dir); SERVER *server = server_alloc("127.0.0.1", "HTTPD", 9876, "NullAuthAllow", "fake=option"); TEST(server, "Server allocation failed"); server_set_unique_name(server, name); - /** Make sure the file doesn't exist */ - unlink("./server.cnf"); + /** Make sure the files don't exist */ + unlink(config_name); + unlink(old_config_name); /** Serialize server to disk */ - TEST(server_serialize(server, "./server.cnf"), "Failed to synchronize original server"); + TEST(server_serialize(server), "Failed to synchronize original server"); /** Load it again */ - TEST(test_load_config("./server.cnf", server), "Failed to load the serialized server"); + TEST(test_load_config(config_name, server), "Failed to load the serialized server"); /** We should have two identical servers */ SERVER *created = server_find_by_unique_name(name); TEST(created->next == server, "We should end up with two servers"); - /** Make sure the file doesn't exist */ - unlink("./server-created.cnf"); + rename(config_name, old_config_name); /** Serialize the loaded server to disk */ - TEST(server_serialize(created, "./server-created.cnf"), "Failed to synchronize the copied server"); + TEST(server_serialize(created), "Failed to synchronize the copied server"); /** Check that they serialize to identical files */ - TEST(system("diff ./server.cnf ./server-created.cnf") == 0, "The files are not identical"); + char cmd[1024]; + sprintf(cmd, "diff ./%s ./%s", config_name, old_config_name); + TEST(system(cmd) == 0, "The files are not identical"); return true; } diff --git a/server/modules/routing/debugcli/debugcmd.c b/server/modules/routing/debugcli/debugcmd.c index a59785d91..3c0fb17dc 100644 --- a/server/modules/routing/debugcli/debugcmd.c +++ b/server/modules/routing/debugcli/debugcmd.c @@ -987,6 +987,17 @@ struct subcommand flushoptions[] = /** This is used to prevent concurrent creation or removal of servers */ static SPINLOCK server_mod_lock = SPINLOCK_INIT; +/** + * Create a new server + * + * @param dcb Client DCB + * @param name Server name + * @param address Server network address + * @param port Server port + * @param protocol Protocol, NULL for default (MySQLBackend) + * @param authenticator Authenticator module, NULL for default (MySQLBackendAuth) + * @param authenticator_options Authenticator options, NULL for no options + */ static void createServer(DCB *dcb, char *name, char *address, char *port, char *protocol, char *authenticator, char *authenticator_options) { @@ -1005,8 +1016,18 @@ static void createServer(DCB *dcb, char *name, char *address, char *port, if (server) { server_set_unique_name(server, name); - // TODO: Serialize the server to disk - dcb_printf(dcb, "Created server '%s'\n", name); + + if (server_serialize(server)) + { + dcb_printf(dcb, "Created server '%s'\n", name); + } + else + { + dcb_printf(dcb, "WARNING: The server was added to the runtime " + "configuration but persisting the new server to disk " + "failed. This server will NOT be loaded when MaxScale " + "is restarted. See log file for more details on why it failed.\n"); + } } else { From 749963061436cf85bd46112a543bc76ca121f6e1 Mon Sep 17 00:00:00 2001 From: ekorh475 Date: Thu, 10 Nov 2016 12:21:03 +0200 Subject: [PATCH 191/215] MXS-576 Check for negative values for config settings Previously, negative values were allowed for persistpoolmax and persistmaxtime. Now they cause an error. Also, monitor_interval allowed negative (or zero) values, which were then implicitly cast to unsigned, causing unintended behaviour. Now this causes a warning and the default value is used. --- server/core/config.c | 42 +++++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/server/core/config.c b/server/core/config.c index e75196b71..d772b1f7f 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -2767,22 +2767,32 @@ int create_new_server(CONFIG_CONTEXT *obj) const char *poolmax = config_get_value_string(obj->parameters, "persistpoolmax"); if (poolmax) { - server->persistpoolmax = strtol(poolmax, &endptr, 0); - if (*endptr != '\0') + long int persistpoolmax = strtol(poolmax, &endptr, 0); + if (*endptr != '\0' || persistpoolmax < 0) { MXS_ERROR("Invalid value for 'persistpoolmax' for server %s: %s", server->unique_name, poolmax); + error_count++; + } + else + { + server->persistpoolmax = persistpoolmax; } } const char *persistmax = config_get_value_string(obj->parameters, "persistmaxtime"); if (persistmax) { - server->persistmaxtime = strtol(persistmax, &endptr, 0); - if (*endptr != '\0') + long int persistmaxtime = strtol(persistmax, &endptr, 0); + if (*endptr != '\0' || persistmaxtime < 0) { MXS_ERROR("Invalid value for 'persistmaxtime' for server %s: %s", server->unique_name, persistmax); + error_count++; + } + else + { + server->persistmaxtime = persistmaxtime; } } @@ -2899,7 +2909,7 @@ int create_new_monitor(CONFIG_CONTEXT *context, CONFIG_CONTEXT *obj, HASHTABLE* else { obj->element = NULL; - MXS_ERROR("Monitor '%s' is missing the require 'module' parameter.", obj->object); + MXS_ERROR("Monitor '%s' is missing the required 'module' parameter.", obj->object); error_count++; } @@ -2909,15 +2919,29 @@ int create_new_monitor(CONFIG_CONTEXT *context, CONFIG_CONTEXT *obj, HASHTABLE* { monitorAddParameters(obj->element, obj->parameters); - char *interval = config_get_value(obj->parameters, "monitor_interval"); - if (interval) + char *interval_str = config_get_value(obj->parameters, "monitor_interval"); + if (interval_str) { - monitorSetInterval(obj->element, atoi(interval)); + char *endptr; + long interval = strtol(interval_str, &endptr, 0); + /* The interval must be >0 because it is used as a divisor. + Perhaps a greater minimum value should be added? */ + if (*endptr == '\0' && interval > 0) + { + monitorSetInterval(obj->element, (unsigned long)interval); + } + else + { + MXS_NOTICE("Invalid 'monitor_interval' parameter for monitor '%s', " + "using default value of %d milliseconds.", + obj->object, MONITOR_INTERVAL); + } } else { MXS_NOTICE("Monitor '%s' is missing the 'monitor_interval' parameter, " - "using default value of 10000 milliseconds.", obj->object); + "using default value of %d milliseconds.", + obj->object, MONITOR_INTERVAL); } char *connect_timeout = config_get_value(obj->parameters, "backend_connect_timeout"); From f0082256f57305aedad3579f3f9e36772df40bd7 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 10 Nov 2016 13:45:25 +0200 Subject: [PATCH 192/215] Only print servers that aren't destroyed If a server were to be destroyed, it should not show up among non-destroyed servers. Even though the servers aren't actually destroyed, it hides unnecessary information from the user. --- include/maxscale/server.h | 1 + server/core/server.c | 75 ++++++++++++++++++++++++++++----------- 2 files changed, 55 insertions(+), 21 deletions(-) diff --git a/include/maxscale/server.h b/include/maxscale/server.h index b58377c5d..b1ee105e3 100644 --- a/include/maxscale/server.h +++ b/include/maxscale/server.h @@ -115,6 +115,7 @@ typedef struct server long persistpoolmax; /**< Maximum size of persistent connections pool */ long persistmaxtime; /**< Maximum number of seconds connection can live */ int persistmax; /**< Maximum pool size actually achieved since startup */ + bool is_active; /**< Server is active and has not been "destroyed" */ #if defined(SS_DEBUG) skygw_chk_t server_chk_tail; #endif diff --git a/server/core/server.c b/server/core/server.c index a2abb488d..cc3e878b9 100644 --- a/server/core/server.c +++ b/server/core/server.c @@ -133,6 +133,7 @@ server_alloc(char *servname, char *protocol, unsigned short port, char *authenti server->persistpoolmax = 0; server->monuser[0] = '\0'; server->monpw[0] = '\0'; + server->is_active = true; spinlock_init(&server->persistlock); spinlock_acquire(&server_spin); @@ -283,7 +284,7 @@ server_find_by_unique_name(char *name) server = allServers; while (server) { - if (server->unique_name && strcmp(server->unique_name, name) == 0) + if (server->is_active && server->unique_name && strcmp(server->unique_name, name) == 0) { break; } @@ -309,7 +310,7 @@ server_find(char *servname, unsigned short port) server = allServers; while (server) { - if (strcmp(server->name, servname) == 0 && server->port == port) + if (server->is_active && strcmp(server->name, servname) == 0 && server->port == port) { break; } @@ -352,7 +353,10 @@ printAllServers() server = allServers; while (server) { - printServer(server); + if (server->is_active) + { + printServer(server); + } server = server->next; } spinlock_release(&server_spin); @@ -373,7 +377,10 @@ dprintAllServers(DCB *dcb) server = allServers; while (server) { - dprintServer(dcb, server); + if (server->is_active) + { + dprintServer(dcb, server); + } server = server->next; } spinlock_release(&server_spin); @@ -404,6 +411,11 @@ dprintAllServersJson(DCB *dcb) dcb_printf(dcb, "[\n"); while (server) { + if (!server->is_active) + { + server = server->next; + continue; + } dcb_printf(dcb, " {\n \"server\": \"%s\",\n", server->name); stat = server_status(server); @@ -484,6 +496,11 @@ dprintAllServersJson(DCB *dcb) void dprintServer(DCB *dcb, SERVER *server) { + if (!server->is_active) + { + return; + } + dcb_printf(dcb, "Server %p (%s)\n", server, server->unique_name); dcb_printf(dcb, "\tServer: %s\n", server->name); char* stat = server_status(server); @@ -618,25 +635,38 @@ dListServers(DCB *dcb) spinlock_acquire(&server_spin); server = allServers; + + while (server && !server->is_active) + { + server = server->next; + } + + bool have_servers = false; + if (server) { + have_servers = true; dcb_printf(dcb, "Servers.\n"); dcb_printf(dcb, "-------------------+-----------------+-------+-------------+--------------------\n"); dcb_printf(dcb, "%-18s | %-15s | Port | Connections | %-20s\n", "Server", "Address", "Status"); dcb_printf(dcb, "-------------------+-----------------+-------+-------------+--------------------\n"); } + while (server) { - stat = server_status(server); - dcb_printf(dcb, "%-18s | %-15s | %5d | %11d | %s\n", - server->unique_name, server->name, - server->port, - server->stats.n_current, stat); - MXS_FREE(stat); + if (server->is_active) + { + stat = server_status(server); + dcb_printf(dcb, "%-18s | %-15s | %5d | %11d | %s\n", + server->unique_name, server->name, + server->port, + server->stats.n_current, stat); + MXS_FREE(stat); + } server = server->next; } - if (allServers) + if (have_servers) { dcb_printf(dcb, "-------------------+-----------------+-------+-------------+--------------------\n"); } @@ -918,16 +948,19 @@ serverRowCallback(RESULTSET *set, void *data) return NULL; } (*rowno)++; - row = resultset_make_row(set); - resultset_row_set(row, 0, server->unique_name); - resultset_row_set(row, 1, server->name); - sprintf(buf, "%d", server->port); - resultset_row_set(row, 2, buf); - sprintf(buf, "%d", server->stats.n_current); - resultset_row_set(row, 3, buf); - stat = server_status(server); - resultset_row_set(row, 4, stat); - MXS_FREE(stat); + if (server->is_active) + { + row = resultset_make_row(set); + resultset_row_set(row, 0, server->unique_name); + resultset_row_set(row, 1, server->name); + sprintf(buf, "%d", server->port); + resultset_row_set(row, 2, buf); + sprintf(buf, "%d", server->stats.n_current); + resultset_row_set(row, 3, buf); + stat = server_status(server); + resultset_row_set(row, 4, stat); + MXS_FREE(stat); + } spinlock_release(&server_spin); return row; } From 261f5fdc36c36b7f3b938328830ac6d04d9a07bc Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 10 Nov 2016 13:47:41 +0200 Subject: [PATCH 193/215] Enable destruction of servers The servers can now be destroyed which removes them from the list of active servers. If the server was not created at runtime, a warning is logged. --- include/maxscale/monitor.h | 7 +++++++ include/maxscale/server.h | 9 +++++++++ include/maxscale/service.h | 7 +++++++ server/core/monitor.c | 26 ++++++++++++++++++++++++++ server/core/server.c | 35 +++++++++++++++++++++++++++++++++++ server/core/service.c | 26 ++++++++++++++++++++++++++ 6 files changed, 110 insertions(+) diff --git a/include/maxscale/monitor.h b/include/maxscale/monitor.h index d5ca21740..55e795922 100644 --- a/include/maxscale/monitor.h +++ b/include/maxscale/monitor.h @@ -233,4 +233,11 @@ connect_result_t mon_connect_to_db(MONITOR* mon, MONITOR_SERVERS *database); void mon_log_connect_error(MONITOR_SERVERS* database, connect_result_t rval); void mon_log_state_change(MONITOR_SERVERS *ptr); +/** + * Check if a monitor uses @c servers + * @param server Server that is queried + * @return True if server is used by at least one monitor + */ +bool monitor_server_in_use(const SERVER *server); + MXS_END_DECLS diff --git a/include/maxscale/server.h b/include/maxscale/server.h index b1ee105e3..7575ab127 100644 --- a/include/maxscale/server.h +++ b/include/maxscale/server.h @@ -243,4 +243,13 @@ extern void server_update_ssl(SERVER *server, const char *key, const char *value */ bool server_serialize(SERVER *server); +/** + * @brief Destroy a server + * + * This removes any created server configuration files and marks the server removed + * If the server is not in use. + * @param server Server to destroy + */ +void server_destroy(SERVER *server); + MXS_END_DECLS diff --git a/include/maxscale/service.h b/include/maxscale/service.h index b4bdfe637..9fce33362 100644 --- a/include/maxscale/service.h +++ b/include/maxscale/service.h @@ -257,4 +257,11 @@ static inline uint64_t service_get_capabilities(const SERVICE *service) return service->capabilities; } +/** + * Check if a service uses @c servers + * @param server Server that is queried + * @return True if server is used by at least one service + */ +bool service_server_in_use(const SERVER *server); + MXS_END_DECLS diff --git a/server/core/monitor.c b/server/core/monitor.c index 9df8401ff..978c45a70 100644 --- a/server/core/monitor.c +++ b/server/core/monitor.c @@ -1130,3 +1130,29 @@ void mon_log_state_change(MONITOR_SERVERS *ptr) MXS_FREE(prev); MXS_FREE(next); } + +bool monitor_server_in_use(const SERVER *server) +{ + bool rval = false; + + spinlock_acquire(&monLock); + + for (MONITOR *mon = allMonitors; mon && !rval; mon = mon->next) + { + spinlock_acquire(&mon->lock); + + for (MONITOR_SERVERS *db = mon->databases; db && !rval; db = db->next) + { + if (db->server == server) + { + rval = true; + } + } + + spinlock_release(&mon->lock); + } + + spinlock_release(&monLock); + + return rval; +} diff --git a/server/core/server.c b/server/core/server.c index cc3e878b9..baa5cf48f 100644 --- a/server/core/server.c +++ b/server/core/server.c @@ -38,6 +38,8 @@ #include #include #include +#include +#include #include #include #include @@ -1240,6 +1242,39 @@ bool server_serialize(SERVER *server) return rval; } +void server_destroy(SERVER *server) +{ + if (service_server_in_use(server) || monitor_server_in_use(server)) + { + MXS_ERROR("Cannot destroy server '%s' as it is used by at least one " + "service or monitor", server->unique_name); + } + else + { + char filename[PATH_MAX]; + snprintf(filename, sizeof(filename), "%s/%s.cnf", get_config_persistdir(), + server->unique_name); + + if (unlink(filename) == -1) + { + if (errno != ENOENT) + { + char err[MXS_STRERROR_BUFLEN]; + MXS_ERROR("Failed to remove persisted server configuration '%s': %d, %s", + filename, errno, strerror_r(errno, err, sizeof(err))); + } + else + { + MXS_WARNING("Server '%s' was not created at runtime. Remove the " + "server manually from the correct configuration file.", + server->unique_name); + } + } + + server->is_active = false; + } +} + bool server_is_ssl_parameter(const char *key) { // TODO: Implement this diff --git a/server/core/service.c b/server/core/service.c index 942bc3e42..bd4e6dccf 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -2188,3 +2188,29 @@ static void service_calculate_weights(SERVICE *service) } } } + +bool service_server_in_use(const SERVER *server) +{ + bool rval = false; + + spinlock_acquire(&service_spin); + + for (SERVICE *service = allServices; service && !rval; service = service->next) + { + spinlock_acquire(&service->spin); + + for (SERVER_REF *ref = service->dbref; ref && !rval; ref = ref->next) + { + if (ref->server == server) + { + rval = true; + } + } + + spinlock_release(&service->spin); + } + + spinlock_release(&service_spin); + + return rval; +} From e67a829daf39114a6a32b5de38e5b3a26d448a97 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 10 Nov 2016 15:34:22 +0200 Subject: [PATCH 194/215] Take server destruction into use The servers can now be destroyed with the `destroy server NAME` maxadmin command. --- include/maxscale/server.h | 3 ++- server/core/server.c | 18 +++++++++++++++-- server/modules/routing/debugcli/debugcmd.c | 23 +++++++++++++++++----- 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/include/maxscale/server.h b/include/maxscale/server.h index 7575ab127..f9e004091 100644 --- a/include/maxscale/server.h +++ b/include/maxscale/server.h @@ -249,7 +249,8 @@ bool server_serialize(SERVER *server); * This removes any created server configuration files and marks the server removed * If the server is not in use. * @param server Server to destroy + * @return True if server was destroyed */ -void server_destroy(SERVER *server); +bool server_destroy(SERVER *server); MXS_END_DECLS diff --git a/server/core/server.c b/server/core/server.c index baa5cf48f..412ad9688 100644 --- a/server/core/server.c +++ b/server/core/server.c @@ -1242,8 +1242,10 @@ bool server_serialize(SERVER *server) return rval; } -void server_destroy(SERVER *server) +bool server_destroy(SERVER *server) { + bool rval = false; + if (service_server_in_use(server) || monitor_server_in_use(server)) { MXS_ERROR("Cannot destroy server '%s' as it is used by at least one " @@ -1265,14 +1267,26 @@ void server_destroy(SERVER *server) } else { + rval = true; MXS_WARNING("Server '%s' was not created at runtime. Remove the " "server manually from the correct configuration file.", server->unique_name); } } + else + { + rval = true; + } - server->is_active = false; + if (rval) + { + MXS_NOTICE("Destroyed server '%s' at %s:%u", server->unique_name, + server->name, server->port); + server->is_active = false; + } } + + return rval; } bool server_is_ssl_parameter(const char *key) diff --git a/server/modules/routing/debugcli/debugcmd.c b/server/modules/routing/debugcli/debugcmd.c index 3c0fb17dc..e68932d1e 100644 --- a/server/modules/routing/debugcli/debugcmd.c +++ b/server/modules/routing/debugcli/debugcmd.c @@ -1023,9 +1023,9 @@ static void createServer(DCB *dcb, char *name, char *address, char *port, } else { - dcb_printf(dcb, "WARNING: The server was added to the runtime " - "configuration but persisting the new server to disk " - "failed. This server will NOT be loaded when MaxScale " + dcb_printf(dcb, "WARNING: The server was added to the runtime \n" + "configuration but persisting the new server to disk \n" + "failed. This server will NOT be loaded when MaxScale \n" "is restarted. See log file for more details on why it failed.\n"); } } @@ -1068,7 +1068,20 @@ struct subcommand createoptions[] = static void destroyServer(DCB *dcb, SERVER *server) { - dcb_printf(dcb, "Not yet implemented.\n"); + /** Do this so that we don't directly access the server. Currently, the + * destruction of a server does not free any memory and the server stays + * valid. */ + char name[strlen(server->unique_name) + 1]; + strcpy(name, server->unique_name); + + if (server_destroy(server)) + { + dcb_printf(dcb, "Destroyed server '%s'\n", name); + } + else + { + dcb_printf(dcb, "Failed to destroy server '%s', see log file for more details\n", name); + } } struct subcommand destroyoptions[] = @@ -1077,7 +1090,7 @@ struct subcommand destroyoptions[] = "server", 1, 1, destroyServer, "Destroy a server", "Usage: destroy server NAME", - {ARG_TYPE_STRING} + {ARG_TYPE_SERVER} }, { EMPTY_OPTION From 16e8aa71786028e3d9dc23c0d543cefa379f5363 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 11 Nov 2016 08:31:28 +0200 Subject: [PATCH 195/215] Fix server usage bugs in monitors and servers The MySQL Monitor did not reset the pointer to the root master reference which would lead to a crash if the master was removed. When service details were shown, it listed all servers that existed. Only servers that haven't been removed or destroyed should be shown. --- server/core/monitor.c | 10 +++------- server/core/service.c | 9 ++++++--- server/modules/monitor/mysqlmon/mysql_mon.c | 4 +++- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/server/core/monitor.c b/server/core/monitor.c index 978c45a70..94f930a8a 100644 --- a/server/core/monitor.c +++ b/server/core/monitor.c @@ -330,7 +330,6 @@ void monitorRemoveServer(MONITOR *mon, SERVER *server) spinlock_acquire(&mon->lock); - ss_dassert(mon->databases); MONITOR_SERVERS *ptr = mon->databases; if (ptr->server == server) @@ -1109,12 +1108,9 @@ void mon_log_connect_error(MONITOR_SERVERS* database, connect_result_t rval) { MXS_ERROR(rval == MONITOR_CONN_TIMEOUT ? - "Monitor timed out when connecting to " - "server %s:%d : \"%s\"" : - "Monitor was unable to connect to " - "server %s:%d : \"%s\"", - database->server->name, - database->server->port, + "Monitor timed out when connecting to server %s:%d : \"%s\"" : + "Monitor was unable to connect to server %s:%d : \"%s\"", + database->server->name, database->server->port, mysql_error(database->con)); } diff --git a/server/core/service.c b/server/core/service.c index bd4e6dccf..bd2f596e1 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -1365,8 +1365,11 @@ void dprintService(DCB *dcb, SERVICE *service) dcb_printf(dcb, "\tBackend databases:\n"); while (server) { - dcb_printf(dcb, "\t\t%s:%d Protocol: %s\n", server->server->name, server->server->port, - server->server->protocol); + if (server->active && server->server->is_active) + { + dcb_printf(dcb, "\t\t%s:%d Protocol: %s\n", server->server->name, + server->server->port, server->server->protocol); + } server = server->next; } if (service->weightby) @@ -2201,7 +2204,7 @@ bool service_server_in_use(const SERVER *server) for (SERVER_REF *ref = service->dbref; ref && !rval; ref = ref->next) { - if (ref->server == server) + if (ref->active && ref->server == server) { rval = true; } diff --git a/server/modules/monitor/mysqlmon/mysql_mon.c b/server/modules/monitor/mysqlmon/mysql_mon.c index dbf19503f..eafd1c251 100644 --- a/server/modules/monitor/mysqlmon/mysql_mon.c +++ b/server/modules/monitor/mysqlmon/mysql_mon.c @@ -270,7 +270,6 @@ startMonitor(MONITOR *monitor, const CONFIG_PARAMETER* params) handle->replicationHeartbeat = 0; handle->detectStaleMaster = true; handle->detectStaleSlave = true; - handle->master = NULL; handle->script = NULL; handle->multimaster = false; handle->mysql51_replication = false; @@ -281,6 +280,9 @@ startMonitor(MONITOR *monitor, const CONFIG_PARAMETER* params) spinlock_init(&handle->lock); } + /** This should always be reset to NULL */ + handle->master = NULL; + while (params) { if (!strcmp(params->name, "detect_stale_master")) From 08ebb88aecec4909acb02b2b9dda65ed81f76711 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 11 Nov 2016 10:13:14 +0200 Subject: [PATCH 196/215] Make server creation simpler Server creation and allocation are now the same apart from the fact that allocation only adds the server to the running configuration where as the creation of a server also persist it to disk. The server serialization should not be seen through the server.h API. This allows the serialization method to change without actually having to change the interfaces. --- include/maxscale/server.h | 75 +++++--- server/core/config.c | 6 +- server/core/server.c | 173 ++++++++++++------ server/core/test/testserver.c | 7 +- server/modules/routing/binlog/blr.c | 4 +- .../modules/routing/binlog/test/testbinlog.c | 4 +- server/modules/routing/debugcli/debugcmd.c | 24 +-- 7 files changed, 174 insertions(+), 119 deletions(-) diff --git a/include/maxscale/server.h b/include/maxscale/server.h index f9e004091..883fb909a 100644 --- a/include/maxscale/server.h +++ b/include/maxscale/server.h @@ -201,9 +201,57 @@ typedef struct server (((server)->status & (SERVER_RUNNING|SERVER_MASTER|SERVER_SLAVE|SERVER_MAINT)) == \ (SERVER_RUNNING|SERVER_MASTER|SERVER_SLAVE)) -extern SERVER *server_alloc(char *, char *, unsigned short, char*, char*); +/** + * @brief Allocate a new server + * + * This will create a new server that represents a backend server that services + * can use. This function will add the server to the running configuration but + * will not persist the changes. + * + * @param name Unique server name + * @param address The server address + * @param port The port to connect to + * @param protocol The protocol to use to connect to the server + * @param authenticator The server authenticator module + * @param auth_options Options for the authenticator module + * @return The newly created server or NULL if an error occurred + */ +extern SERVER* server_alloc(const char *name, const char *address, unsigned short port, + const char *protocol, const char *authenticator, + const char *auth_options); + +/** + * @brief Create a new server + * + * This function creates a new, persistent server by first allocating a new + * server and then storing the resulting configuration file on disk. This + * function should be used only from administrative interface modules and internal + * modules should use server_alloc() instead. + * + * @param name Server name + * @param address Network address + * @param port Network port + * @param protocol Protocol module name + * @param authenticator Authenticator module name + * @param options Options for the authenticator module + * @return True on success, false if an error occurred + */ +extern bool server_create(const char *name, const char *address, const char *port, + const char *protocol, const char *authenticator, + const char *options); + +/** + * @brief Destroy a server + * + * This removes any created server configuration files and marks the server removed + * If the server is not in use. + * @param server Server to destroy + * @return True if server was destroyed + */ +bool server_destroy(SERVER *server); + extern int server_free(SERVER *); -extern SERVER *server_find_by_unique_name(char *); +extern SERVER *server_find_by_unique_name(const char *name); extern SERVER *server_find(char *, unsigned short); extern void printServer(SERVER *); extern void printAllServers(); @@ -221,7 +269,6 @@ extern void serverAddMonUser(SERVER *, char *, char *); extern void serverAddParameter(SERVER *, char *, char *); extern char *serverGetParameter(SERVER *, char *); extern void server_update_credentials(SERVER *, char *, char *); -extern void server_set_unique_name(SERVER *, char *); extern DCB *server_get_persistent(SERVER *, char *, const char *); extern void server_update_address(SERVER *, char *); extern void server_update_port(SERVER *, unsigned short); @@ -231,26 +278,4 @@ extern bool server_set_version_string(SERVER* server, const char* string); extern bool server_is_ssl_parameter(const char *key); extern void server_update_ssl(SERVER *server, const char *key, const char *value); -/** - * @brief Serialize a server to a file - * - * This converts @c server into an INI format file. This allows created servers - * to be persisted to disk. This will replace any existing files with the same - * name. - * - * @param server Server to serialize - * @return False if the serialization of the server fails, true if it was successful - */ -bool server_serialize(SERVER *server); - -/** - * @brief Destroy a server - * - * This removes any created server configuration files and marks the server removed - * If the server is not in use. - * @param server Server to destroy - * @return True if server was destroyed - */ -bool server_destroy(SERVER *server); - MXS_END_DECLS diff --git a/server/core/config.c b/server/core/config.c index d772b1f7f..43a738397 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -2730,11 +2730,7 @@ int create_new_server(CONFIG_CONTEXT *obj) if (address && port && protocol) { - if ((obj->element = server_alloc(address, protocol, atoi(port), auth, auth_opts))) - { - server_set_unique_name(obj->element, obj->object); - } - else + if ((obj->element = server_alloc(obj->object, address, atoi(port), protocol, auth, auth_opts)) == NULL) { MXS_ERROR("Failed to create a new server, memory allocation failed."); error_count++; diff --git a/server/core/server.c b/server/core/server.c index 412ad9688..a73aa795e 100644 --- a/server/core/server.c +++ b/server/core/server.c @@ -57,57 +57,44 @@ static SERVER *allServers = NULL; static void spin_reporter(void *, char *, int); static void server_parameter_free(SERVER_PARAM *tofree); -/** - * Allocate a new server withn the gateway - * - * - * @param servname The server name - * @param protocol The protocol to use to connect to the server - * @param port The port to connect to - * - * @return The newly created server or NULL if an error occured - */ -SERVER * -server_alloc(char *servname, char *protocol, unsigned short port, char *authenticator, - char *auth_options) + +SERVER* server_alloc(const char *name, const char *address, unsigned short port, + const char *protocol, const char *authenticator, const char *auth_options) { - if (authenticator) + if (authenticator == NULL && (authenticator = get_default_authenticator(protocol)) == NULL) { - authenticator = MXS_STRDUP(authenticator); - } - else if ((authenticator = (char*)get_default_authenticator(protocol)) == NULL || - (authenticator = MXS_STRDUP(authenticator)) == NULL) - { - MXS_ERROR("No authenticator defined for server at %s:%u and no default " - "authenticator for protocol '%s'.", servname, port, protocol); + MXS_ERROR("No authenticator defined for server '%s' and no default " + "authenticator for protocol '%s'.", name, protocol); + return NULL; } void *auth_instance = NULL; if (!authenticator_init(&auth_instance, authenticator, auth_options)) { - MXS_ERROR("Failed to initialize authenticator module '%s' for server" - " at %s:%u.", authenticator, servname, port); - MXS_FREE(authenticator); + MXS_ERROR("Failed to initialize authenticator module '%s' for server '%s' ", + authenticator, name); return NULL; } - if (auth_options && (auth_options = MXS_STRDUP(auth_options)) == NULL) + char *my_auth_options = NULL; + + if (auth_options && (my_auth_options = MXS_STRDUP(auth_options)) == NULL) { - MXS_FREE(authenticator); return NULL; } - protocol = MXS_STRDUP(protocol); - SERVER *server = (SERVER *)MXS_CALLOC(1, sizeof(SERVER)); + char *my_name = MXS_STRDUP(name); + char *my_protocol = MXS_STRDUP(protocol); + char *my_authenticator = MXS_STRDUP(authenticator); - if (!protocol || !server || !authenticator) + if (!server || !my_name || !my_protocol || !my_authenticator) { - MXS_FREE(servname); - MXS_FREE(protocol); MXS_FREE(server); - MXS_FREE(authenticator); + MXS_FREE(my_name); + MXS_FREE(my_protocol); + MXS_FREE(my_authenticator); return NULL; } @@ -115,11 +102,12 @@ server_alloc(char *servname, char *protocol, unsigned short port, char *authenti server->server_chk_top = CHK_NUM_SERVER; server->server_chk_tail = CHK_NUM_SERVER; #endif - snprintf(server->name, sizeof(server->name), "%s", servname); - server->protocol = protocol; - server->authenticator = authenticator; + server->unique_name = my_name; + snprintf(server->name, sizeof(server->name), "%s", address); + server->protocol = my_protocol; + server->authenticator = my_authenticator; server->auth_instance = auth_instance; - server->auth_options = auth_options; + server->auth_options = my_auth_options; server->port = port; server->status = SERVER_RUNNING; server->node_id = -1; @@ -258,32 +246,16 @@ server_get_persistent(SERVER *server, char *user, const char *protocol) } /** - * Set a unique name for the server + * @brief Find a server with the specified name * - * @param server The server to set the name on - * @param name The unique name for the server + * @param name Name of the server + * @return The server or NULL if not found */ -void -server_set_unique_name(SERVER *server, char *name) +SERVER * server_find_by_unique_name(const char *name) { - server->unique_name = MXS_STRDUP_A(name); -} - -/** - * Find an existing server using the unique section name in - * configuration file - * - * @param servname The Server name or address - * @param port The server port - * @return The server or NULL if not found - */ -SERVER * -server_find_by_unique_name(char *name) -{ - SERVER *server; - spinlock_acquire(&server_spin); - server = allServers; + SERVER *server = allServers; + while (server) { if (server->is_active && server->unique_name && strcmp(server->unique_name, name) == 0) @@ -293,6 +265,7 @@ server_find_by_unique_name(char *name) server = server->next; } spinlock_release(&server_spin); + return server; } @@ -1205,7 +1178,17 @@ static bool create_server_config(SERVER *server, const char *filename) return true; } -bool server_serialize(SERVER *server) +/** + * @brief Serialize a server to a file + * + * This converts @c server into an INI format file. This allows created servers + * to be persisted to disk. This will replace any existing files with the same + * name. + * + * @param server Server to serialize + * @return False if the serialization of the server fails, true if it was successful + */ +static bool server_serialize(SERVER *server) { bool rval = false; char filename[PATH_MAX]; @@ -1242,6 +1225,78 @@ bool server_serialize(SERVER *server) return rval; } +/** Try to find a server with a matching name that has been destroyed */ +static SERVER* find_destroyed_server(const char *name, const char *protocol, + const char *authenticator, const char *auth_options) +{ + spinlock_acquire(&server_spin); + SERVER *server = allServers; + while (server) + { + CHK_SERVER(server); + if (strcmp(server->unique_name, name) == 0 && + strcmp(server->protocol, protocol) == 0 && + strcmp(server->authenticator, authenticator) == 0) + { + if ((auth_options == NULL && server->auth_options == NULL) || + (auth_options && server->auth_options && + strcmp(server->auth_options, auth_options) == 0)) + { + break; + } + } + server = server->next; + } + spinlock_release(&server_spin); + + return server; +} + +bool server_create(const char *name, const char *address, const char *port, + const char *protocol, const char *authenticator, + const char *authenticator_options) +{ + bool rval = false; + + if (server_find_by_unique_name(name) == NULL) + { + if (port == NULL) + { + port = "3306"; + } + if (protocol == NULL) + { + protocol = "MySQLBackend"; + } + + /** First check if this service has been created before */ + SERVER *server = find_destroyed_server(name, protocol, authenticator, + authenticator_options); + + if (server) + { + /** Found old server, replace network details with new ones and + * reactivate it */ + snprintf(server->name, sizeof(server->name), "%s", address); + server->port = atoi(port); + server->is_active = true; + rval = true; + } + else if ((server = server_alloc(name, address, atoi(port), protocol, authenticator, + authenticator_options))) + { + if (server_serialize(server)) + { + /** server_alloc will add the server to the global list of + * servers so we don't need to manually add it. */ + rval = true; + } + } + } + + return rval; +} + bool server_destroy(SERVER *server) { bool rval = false; diff --git a/server/core/test/testserver.c b/server/core/test/testserver.c index bb195885e..67991f864 100644 --- a/server/core/test/testserver.c +++ b/server/core/test/testserver.c @@ -40,6 +40,7 @@ // This is pretty ugly but it's required to test internal functions #include "../config.c" +#include "../server.c" /** * test1 Allocate a server and do lots of other things @@ -55,7 +56,7 @@ test1() /* Server tests */ ss_dfprintf(stderr, "testserver : creating server called MyServer"); set_libdir(MXS_STRDUP_A("../../modules/authenticator/")); - server = server_alloc("MyServer", "HTTPD", 9876, "NullAuthAllow", NULL); + server = server_alloc("uniquename", "127.0.0.1", 9876, "HTTPD", "NullAuthAllow", NULL); ss_info_dassert(server, "Allocating the server should not fail"); mxs_log_flush_sync(); @@ -71,7 +72,6 @@ test1() ss_dfprintf(stderr, "\t..done\nTesting Unique Name for Server."); ss_info_dassert(NULL == server_find_by_unique_name("uniquename"), "Should not find non-existent unique name."); - server_set_unique_name(server, "uniquename"); mxs_log_flush_sync(); ss_info_dassert(server == server_find_by_unique_name("uniquename"), "Should find by unique name."); ss_dfprintf(stderr, "\t..done\nTesting Status Setting for Server."); @@ -144,9 +144,8 @@ bool test_serialize() char old_config_name[] = "serialized-server.cnf.old"; char *persist_dir = MXS_STRDUP_A("./"); set_config_persistdir(persist_dir); - SERVER *server = server_alloc("127.0.0.1", "HTTPD", 9876, "NullAuthAllow", "fake=option"); + SERVER *server = server_alloc(name, "127.0.0.1", 9876, "HTTPD", "NullAuthAllow", "fake=option"); TEST(server, "Server allocation failed"); - server_set_unique_name(server, name); /** Make sure the files don't exist */ unlink(config_name); diff --git a/server/modules/routing/binlog/blr.c b/server/modules/routing/binlog/blr.c index 913e37a1e..ddc299f0f 100644 --- a/server/modules/routing/binlog/blr.c +++ b/server/modules/routing/binlog/blr.c @@ -585,7 +585,8 @@ createInstance(SERVICE *service, char **options) { SERVER *server; SSL_LISTENER *ssl_cfg; - server = server_alloc("_none_", "MySQLBackend", 3306, "MySQLBackendAuth", NULL); + server = server_alloc("binlog_router_master_host", "_none_", 3306, + "MySQLBackend", "MySQLBackendAuth", NULL); if (server == NULL) { MXS_ERROR("%s: Error for server_alloc in createInstance", @@ -617,7 +618,6 @@ createInstance(SERVICE *service, char **options) server->server_ssl = ssl_cfg; /* Set server unique name */ - server_set_unique_name(server, "binlog_router_master_host"); /* Add server to service backend list */ serviceAddBackend(inst->service, server); } diff --git a/server/modules/routing/binlog/test/testbinlog.c b/server/modules/routing/binlog/test/testbinlog.c index 2bb630e54..2eda6e461 100644 --- a/server/modules/routing/binlog/test/testbinlog.c +++ b/server/modules/routing/binlog/test/testbinlog.c @@ -103,12 +103,12 @@ int main(int argc, char **argv) { s = strtok_r(NULL, ",", &lasts); } set_libdir(MXS_STRDUP_A("../../../authenticator/")); - server = server_alloc("_none_", "MySQLBackend", 3306, "MySQLBackendAuth", NULL); + server = server_alloc("binlog_router_master_host", "_none_", 3306, + "MySQLBackend", "MySQLBackendAuth", NULL); if (server == NULL) { return 1; } - server_set_unique_name(server, "binlog_router_master_host"); serviceAddBackend(service, server); } diff --git a/server/modules/routing/debugcli/debugcmd.c b/server/modules/routing/debugcli/debugcmd.c index e68932d1e..799a4f6f9 100644 --- a/server/modules/routing/debugcli/debugcmd.c +++ b/server/modules/routing/debugcli/debugcmd.c @@ -1005,29 +1005,9 @@ static void createServer(DCB *dcb, char *name, char *address, char *port, if (server_find_by_unique_name(name) == NULL) { - if (protocol == NULL) + if (server_create(name, address, port, protocol, authenticator, authenticator_options)) { - protocol = "MySQLBackend"; - } - - SERVER *server = server_alloc(address, protocol, atoi(port), authenticator, - authenticator_options); - - if (server) - { - server_set_unique_name(server, name); - - if (server_serialize(server)) - { - dcb_printf(dcb, "Created server '%s'\n", name); - } - else - { - dcb_printf(dcb, "WARNING: The server was added to the runtime \n" - "configuration but persisting the new server to disk \n" - "failed. This server will NOT be loaded when MaxScale \n" - "is restarted. See log file for more details on why it failed.\n"); - } + dcb_printf(dcb, "Created server '%s'\n", name); } else { From ae7981f147e0fd7e53e471638846e9f9efd1e2af Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 11 Nov 2016 11:06:34 +0200 Subject: [PATCH 197/215] Add warnings for string truncation in servers Now that servers use an array for the server name, name truncation can occur. Although this is nigh impossible, a warning message is a prudent precaution. --- server/core/server.c | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/server/core/server.c b/server/core/server.c index a73aa795e..9bda4fa0e 100644 --- a/server/core/server.c +++ b/server/core/server.c @@ -98,12 +98,17 @@ SERVER* server_alloc(const char *name, const char *address, unsigned short port, return NULL; } + if (snprintf(server->name, sizeof(server->name), "%s", address) > sizeof(server->name)) + { + MXS_WARNING("Truncated server address '%s' to the maximum size of %lu characters.", + address, sizeof(server->name)); + } + #if defined(SS_DEBUG) server->server_chk_top = CHK_NUM_SERVER; server->server_chk_tail = CHK_NUM_SERVER; #endif server->unique_name = my_name; - snprintf(server->name, sizeof(server->name), "%s", address); server->protocol = my_protocol; server->authenticator = my_authenticator; server->auth_instance = auth_instance; @@ -791,8 +796,16 @@ server_transfer_status(SERVER *dest_server, SERVER *source_server) void serverAddMonUser(SERVER *server, char *user, char *passwd) { - snprintf(server->monuser, sizeof(server->monuser), "%s", user); - snprintf(server->monpw, sizeof(server->monpw), "%s", passwd); + if (snprintf(server->monuser, sizeof(server->monuser), "%s", user) > sizeof(server->monuser)) + { + MXS_WARNING("Truncated monitor user for server '%s', maximum username " + "length is %lu characters.", server->unique_name, sizeof(server->monuser)); + } + if (snprintf(server->monpw, sizeof(server->monpw), "%s", passwd) > sizeof(server->monpw)) + { + MXS_WARNING("Truncated monitor password for server '%s', maximum password " + "length is %lu characters.", server->unique_name, sizeof(server->monpw)); + } } /** @@ -813,8 +826,7 @@ server_update_credentials(SERVER *server, char *user, char *passwd) { if (user != NULL && passwd != NULL) { - snprintf(server->monuser, sizeof(server->monuser), "%s", user); - snprintf(server->monpw, sizeof(server->monpw), "%s", passwd); + serverAddMonUser(server, user, passwd); MXS_NOTICE("Updated monitor credentials for server '%s'", server->name); } } From 2fcfc102beca2e7f16ab04bbe31866e332a50b34 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 11 Nov 2016 11:05:13 +0200 Subject: [PATCH 198/215] Abstract server active checks The checks for the server active status are now hidden behind a macro. This allows for the conditions of the activity to change without having to alter the code that uses it. --- include/maxscale/server.h | 5 ++ include/maxscale/service.h | 2 +- server/core/server.c | 104 +++++++++++++++++-------------------- server/core/service.c | 2 +- 4 files changed, 55 insertions(+), 58 deletions(-) diff --git a/include/maxscale/server.h b/include/maxscale/server.h index 883fb909a..eb28e1271 100644 --- a/include/maxscale/server.h +++ b/include/maxscale/server.h @@ -140,6 +140,11 @@ typedef struct server #define SERVER_STALE_SLAVE 0x2000 /**<< Slave status is possible even without a master */ #define SERVER_RELAY_MASTER 0x4000 /**<< Server is a relay master */ +/** + * Is the server valid and active + */ +#define SERVER_IS_ACTIVE(server) (server->is_active) + /** * Is the server running - the macro returns true if the server is marked as running * regardless of it's state as a master or slave diff --git a/include/maxscale/service.h b/include/maxscale/service.h index 9fce33362..e8fd7850e 100644 --- a/include/maxscale/service.h +++ b/include/maxscale/service.h @@ -105,7 +105,7 @@ typedef struct server_ref_t } SERVER_REF; /** Macro to check whether a SERVER_REF is active */ -#define SERVER_REF_IS_ACTIVE(ref) (ref->active) +#define SERVER_REF_IS_ACTIVE(ref) (ref->active && SERVER_IS_ACTIVE(ref->server)) #define SERVICE_MAX_RETRY_INTERVAL 3600 /*< The maximum interval between service start retries */ diff --git a/server/core/server.c b/server/core/server.c index 9bda4fa0e..77accd29b 100644 --- a/server/core/server.c +++ b/server/core/server.c @@ -250,6 +250,16 @@ server_get_persistent(SERVER *server, char *user, const char *protocol) return NULL; } +static inline SERVER* next_active_server(SERVER *server) +{ + while (server && !server->is_active) + { + server = server->next; + } + + return server; +} + /** * @brief Find a server with the specified name * @@ -259,16 +269,17 @@ server_get_persistent(SERVER *server, char *user, const char *protocol) SERVER * server_find_by_unique_name(const char *name) { spinlock_acquire(&server_spin); - SERVER *server = allServers; + SERVER *server = next_active_server(allServers); while (server) { - if (server->is_active && server->unique_name && strcmp(server->unique_name, name) == 0) + if (server->unique_name && strcmp(server->unique_name, name) == 0) { break; } - server = server->next; + server = next_active_server(server->next); } + spinlock_release(&server_spin); return server; @@ -284,19 +295,20 @@ SERVER * server_find_by_unique_name(const char *name) SERVER * server_find(char *servname, unsigned short port) { - SERVER *server; - spinlock_acquire(&server_spin); - server = allServers; + SERVER *server = next_active_server(allServers); + while (server) { - if (server->is_active && strcmp(server->name, servname) == 0 && server->port == port) + if (strcmp(server->name, servname) == 0 && server->port == port) { break; } - server = server->next; + server = next_active_server(server->next); } + spinlock_release(&server_spin); + return server; } @@ -327,18 +339,15 @@ printServer(SERVER *server) void printAllServers() { - SERVER *server; - spinlock_acquire(&server_spin); - server = allServers; + SERVER *server = next_active_server(allServers); + while (server) { - if (server->is_active) - { - printServer(server); - } - server = server->next; + printServer(server); + server = next_active_server(server->next); } + spinlock_release(&server_spin); } @@ -351,18 +360,15 @@ printAllServers() void dprintAllServers(DCB *dcb) { - SERVER *server; - spinlock_acquire(&server_spin); - server = allServers; + SERVER *server = next_active_server(allServers); + while (server) { - if (server->is_active) - { - dprintServer(dcb, server); - } - server = server->next; + dprintServer(dcb, server); + server = next_active_server(server->next); } + spinlock_release(&server_spin); } @@ -375,27 +381,23 @@ dprintAllServers(DCB *dcb) void dprintAllServersJson(DCB *dcb) { - SERVER *server; char *stat; int len = 0; int el = 1; spinlock_acquire(&server_spin); - server = allServers; + SERVER *server = next_active_server(allServers); while (server) { - server = server->next; + server = next_active_server(server->next); len++; } - server = allServers; + + server = next_active_server(allServers); + dcb_printf(dcb, "[\n"); while (server) { - if (!server->is_active) - { - server = server->next; - continue; - } dcb_printf(dcb, " {\n \"server\": \"%s\",\n", server->name); stat = server_status(server); @@ -459,9 +461,10 @@ dprintAllServersJson(DCB *dcb) { dcb_printf(dcb, " }\n"); } - server = server->next; + server = next_active_server(server->next); el++; } + dcb_printf(dcb, "]\n"); spinlock_release(&server_spin); } @@ -476,7 +479,7 @@ dprintAllServersJson(DCB *dcb) void dprintServer(DCB *dcb, SERVER *server) { - if (!server->is_active) + if (!SERVER_IS_ACTIVE(server)) { return; } @@ -610,17 +613,8 @@ dprintPersistentDCBs(DCB *pdcb, SERVER *server) void dListServers(DCB *dcb) { - SERVER *server; - char *stat; - spinlock_acquire(&server_spin); - server = allServers; - - while (server && !server->is_active) - { - server = server->next; - } - + SERVER *server = next_active_server(allServers); bool have_servers = false; if (server) @@ -635,17 +629,15 @@ dListServers(DCB *dcb) while (server) { - if (server->is_active) - { - stat = server_status(server); - dcb_printf(dcb, "%-18s | %-15s | %5d | %11d | %s\n", - server->unique_name, server->name, - server->port, - server->stats.n_current, stat); - MXS_FREE(stat); - } - server = server->next; + char *stat = server_status(server); + dcb_printf(dcb, "%-18s | %-15s | %5d | %11d | %s\n", + server->unique_name, server->name, + server->port, + server->stats.n_current, stat); + MXS_FREE(stat); + server = next_active_server(server->next); } + if (have_servers) { dcb_printf(dcb, "-------------------+-----------------+-------+-------------+--------------------\n"); @@ -935,7 +927,7 @@ serverRowCallback(RESULTSET *set, void *data) return NULL; } (*rowno)++; - if (server->is_active) + if (SERVER_IS_ACTIVE(server)) { row = resultset_make_row(set); resultset_row_set(row, 0, server->unique_name); diff --git a/server/core/service.c b/server/core/service.c index bd2f596e1..7fe22d93e 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -1365,7 +1365,7 @@ void dprintService(DCB *dcb, SERVICE *service) dcb_printf(dcb, "\tBackend databases:\n"); while (server) { - if (server->active && server->server->is_active) + if (SERVER_REF_IS_ACTIVE(server)) { dcb_printf(dcb, "\t\t%s:%d Protocol: %s\n", server->server->name, server->server->port, server->server->protocol); From 0a712017a89c883d454363bf3e2259b97a9b84d2 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 11 Nov 2016 11:31:55 +0200 Subject: [PATCH 199/215] Add dynamic server configuration to release notes The release notes list the new commands and point to `maxadmin help ` for more details. --- .../MaxScale-2.1.0-Release-Notes.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Documentation/Release-Notes/MaxScale-2.1.0-Release-Notes.md b/Documentation/Release-Notes/MaxScale-2.1.0-Release-Notes.md index d5617474f..907b549bf 100644 --- a/Documentation/Release-Notes/MaxScale-2.1.0-Release-Notes.md +++ b/Documentation/Release-Notes/MaxScale-2.1.0-Release-Notes.md @@ -87,6 +87,23 @@ removed if they are no longer used by older versions of MaxScale. ## New Features +### Dynamic server configuration + +MaxScale can now change the servers of a service or a monitor at run-time. New +servers can also be created and they will persisted even after a restart. The +following new commands were added to maxadmin, see output of `maxadmin help +` for more details. + +- `create server`: Creates a new server +- `destroy server`: Destroys a created server +- `add server`: Adds a server to a service or a monitor +- `remove server`: Removes a server from a service or a monitor +- `alter server`: Alter server configuration +- `alter monitor`: Alter monitor configuration + +With these new features, you can start MaxScale without the servers and define +them later. + ### Amazon RDS Aurora monitor The new [Aurora Monitor](../Monitors/Aurora-Monitor.md) module allows monitoring From 8a8e1a6650f4bb4033fe981b6f5ad8eea4787023 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 11 Nov 2016 12:07:00 +0200 Subject: [PATCH 200/215] Add check for NULL authenticator The server creation was missing a check for NULL authenticator parameter. --- server/core/server.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/server/core/server.c b/server/core/server.c index 77accd29b..1faee1076 100644 --- a/server/core/server.c +++ b/server/core/server.c @@ -1264,6 +1264,7 @@ bool server_create(const char *name, const char *address, const char *port, if (server_find_by_unique_name(name) == NULL) { + // TODO: Get default values from the protocol module if (port == NULL) { port = "3306"; @@ -1272,6 +1273,12 @@ bool server_create(const char *name, const char *address, const char *port, { protocol = "MySQLBackend"; } + if (authenticator == NULL && (authenticator = get_default_authenticator(protocol)) == NULL) + { + MXS_ERROR("No authenticator defined for server '%s' and no default " + "authenticator for protocol '%s'.", name, protocol); + return false; + } /** First check if this service has been created before */ SERVER *server = find_destroyed_server(name, protocol, authenticator, From 03951fff119f74fa641090010e4ae245ca4e13e2 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 11 Nov 2016 12:17:22 +0200 Subject: [PATCH 201/215] Remove FAKE_CODE from use This functionality is not used in tests and parts of it didn't work. Since it is not actively maintained or even used, it is better to remove it. --- CMakeLists.txt | 4 - include/maxscale/dcb.h | 9 -- include/maxscale/debug.h | 4 - server/core/dcb.c | 96 ++----------------- server/core/gateway.c | 9 -- server/core/poll.c | 26 ----- .../MySQL/MySQLBackend/mysql_backend.c | 3 - server/modules/routing/debugcli/debugcmd.c | 95 ------------------ 8 files changed, 6 insertions(+), 240 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 218964698..69f9539ec 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -154,10 +154,6 @@ if(GCOV) set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lgcov") endif() -if(FAKE_CODE) - set(FLAGS "${FLAGS} -DFAKE_CODE" CACHE STRING "Compilation flags" FORCE) -endif() - if(PROFILE) message(STATUS "Profiling executables") set(FLAGS "${FLAGS} -pg " CACHE STRING "Compilation flags" FORCE) diff --git a/include/maxscale/dcb.h b/include/maxscale/dcb.h index b79dbeee5..98237c685 100644 --- a/include/maxscale/dcb.h +++ b/include/maxscale/dcb.h @@ -302,15 +302,6 @@ typedef enum DCB_USAGE_ALL } DCB_USAGE; -#if defined(FAKE_CODE) -extern unsigned char dcb_fake_write_errno[10240]; -extern __int32_t dcb_fake_write_ev[10240]; -extern bool fail_next_backend_fd; -extern bool fail_next_client_fd; -extern int fail_next_accept; -extern int fail_accept_errno; -#endif /* FAKE_CODE */ - /* A few useful macros */ #define DCB_SESSION(x) (x)->session #define DCB_PROTOCOL(x, type) (type *)((x)->protocol) diff --git a/include/maxscale/debug.h b/include/maxscale/debug.h index dde1f3032..8ab7db618 100644 --- a/include/maxscale/debug.h +++ b/include/maxscale/debug.h @@ -561,8 +561,4 @@ typedef enum skygw_chk_t } -#if defined(FAKE_CODE) -static bool conn_open[10240]; -#endif /* FAKE_CODE */ - MXS_END_DECLS diff --git a/server/core/dcb.c b/server/core/dcb.c index 2cc65eeb3..c07bd979b 100644 --- a/server/core/dcb.c +++ b/server/core/dcb.c @@ -93,15 +93,6 @@ #include #include -#if defined(FAKE_CODE) -unsigned char dcb_fake_write_errno[10240]; -__int32_t dcb_fake_write_ev[10240]; -bool fail_next_backend_fd; -bool fail_next_client_fd; -int fail_next_accept; -int fail_accept_errno; -#endif /* FAKE_CODE */ - /* The list of all DCBs */ static LIST_CONFIG DCBlist = {LIST_TYPE_RECYCLABLE, sizeof(DCB), SPINLOCK_INIT}; @@ -130,9 +121,6 @@ static int dcb_create_SSL(DCB* dcb, SSL_LISTENER *ssl); static int dcb_read_SSL(DCB *dcb, GWBUF **head); static GWBUF *dcb_basic_read(DCB *dcb, int bytesavailable, int maxbytes, int nreadtotal, int *nsingleread); static GWBUF *dcb_basic_read_SSL(DCB *dcb, int *nsingleread); -#if defined(FAKE_CODE) -static inline void dcb_write_fake_code(DCB *dcb); -#endif static void dcb_log_write_failure(DCB *dcb, GWBUF *queue, int eno); static inline void dcb_write_tidy_up(DCB *dcb, bool below_water); static int gw_write(DCB *dcb, GWBUF *writeq, bool *stop_writing); @@ -684,9 +672,6 @@ dcb_process_victim_queue(DCB *listofdcb) } else { -#if defined(FAKE_CODE) - conn_open[dcb->fd] = false; -#endif /* FAKE_CODE */ dcb->fd = DCBFD_CLOSED; MXS_DEBUG("%lu [dcb_process_victim_queue] Closed socket " @@ -1387,34 +1372,6 @@ dcb_write(DCB *dcb, GWBUF *queue) return 1; } -#if defined(FAKE_CODE) -/** - * Fake code for dcb_write - * (Should have fuller description) - * - * @param dcb The DCB of the client - */ -static inline void -dcb_write_fake_code(DCB *dcb) -{ - if (dcb->session != NULL) - { - if (dcb->dcb_role == DCB_ROLE_CLIENT_HANDLER && fail_next_client_fd) - { - dcb_fake_write_errno[dcb->fd] = 32; - dcb_fake_write_ev[dcb->fd] = 29; - fail_next_client_fd = false; - } - else if (dcb->dcb_role == DCB_ROLE_BACKEND_HANDLER && fail_next_backend_fd) - { - dcb_fake_write_errno[dcb->fd] = 32; - dcb_fake_write_ev[dcb->fd] = 29; - fail_next_backend_fd = false; - } - } -} -#endif /* FAKE_CODE */ - /** * Check the parameters for dcb_write * @@ -2425,28 +2382,10 @@ gw_write(DCB *dcb, GWBUF *writeq, bool *stop_writing) errno = 0; -#if defined(FAKE_CODE) - if (fd > 0 && dcb_fake_write_errno[fd] != 0) - { - ss_dassert(dcb_fake_write_ev[fd] != 0); - written = write(fd, buf, nbytes / 2); /*< leave peer to read missing bytes */ - - if (written > 0) - { - written = -1; - errno = dcb_fake_write_errno[fd]; - } - } - else if (fd > 0) - { - written = write(fd, buf, nbytes); - } -#else if (fd > 0) { written = write(fd, buf, nbytes); } -#endif /* FAKE_CODE */ saved_errno = errno; errno = 0; @@ -3093,14 +3032,9 @@ dcb_accept(DCB *listener, GWPROTOCOL *protocol_funcs) if ((c_sock = dcb_accept_one_connection(listener, (struct sockaddr *)&client_conn)) >= 0) { listener->stats.n_accepts++; -#if defined(SS_DEBUG) MXS_DEBUG("%lu [gw_MySQLAccept] Accepted fd %d.", pthread_self(), c_sock); -#endif /* SS_DEBUG */ -#if defined(FAKE_CODE) - conn_open[c_sock] = true; -#endif /* FAKE_CODE */ /* set nonblocking */ sendbuf = MXS_CLIENT_SO_SNDBUF; @@ -3233,27 +3167,12 @@ dcb_accept_one_connection(DCB *listener, struct sockaddr *client_conn) socklen_t client_len = sizeof(struct sockaddr_storage); int eno = 0; -#if defined(FAKE_CODE) - if (fail_next_accept > 0) - { - c_sock = -1; - eno = fail_accept_errno; - fail_next_accept -= 1; - } - else - { - fail_accept_errno = 0; -#endif /* FAKE_CODE */ - - /* new connection from client */ - c_sock = accept(listener->fd, - client_conn, - &client_len); - eno = errno; - errno = 0; -#if defined(FAKE_CODE) - } -#endif /* FAKE_CODE */ + /* new connection from client */ + c_sock = accept(listener->fd, + client_conn, + &client_len); + eno = errno; + errno = 0; if (c_sock == -1) { @@ -3371,9 +3290,6 @@ dcb_listen(DCB *listener, const char *config, const char *protocol_name) "attempting to register on an epoll instance."); return -1; } -#if defined(FAKE_CODE) - conn_open[listener_socket] = true; -#endif /* FAKE_CODE */ return 0; } diff --git a/server/core/gateway.c b/server/core/gateway.c index ba02b48b5..10bacd28d 100644 --- a/server/core/gateway.c +++ b/server/core/gateway.c @@ -1310,15 +1310,6 @@ int main(int argc, char **argv) progname = *argv; snprintf(datadir, PATH_MAX, "%s", default_datadir); datadir[PATH_MAX] = '\0'; -#if defined(FAKE_CODE) - memset(conn_open, 0, sizeof(bool) * 10240); - memset(dcb_fake_write_errno, 0, sizeof(unsigned char) * 10240); - memset(dcb_fake_write_ev, 0, sizeof(__int32_t) * 10240); - fail_next_backend_fd = false; - fail_next_client_fd = false; - fail_next_accept = 0; - fail_accept_errno = 0; -#endif /* FAKE_CODE */ file_write_header(stderr); /*< * Register functions which are called at exit except libmysqld-related, diff --git a/server/core/poll.c b/server/core/poll.c index ea7fa657c..90295a9bb 100644 --- a/server/core/poll.c +++ b/server/core/poll.c @@ -895,18 +895,6 @@ process_pollq(int thread_id) thread_data[thread_id].event = ev; } -#if defined(FAKE_CODE) - if (dcb_fake_write_ev[dcb->fd] != 0) - { - MXS_DEBUG("%lu [poll_waitevents] " - "Added fake events %d to ev %d.", - pthread_self(), - dcb_fake_write_ev[dcb->fd], - ev); - ev |= dcb_fake_write_ev[dcb->fd]; - dcb_fake_write_ev[dcb->fd] = 0; - } -#endif /* FAKE_CODE */ ss_debug(spinlock_acquire(&dcb->dcb_initlock)); ss_dassert(dcb->state != DCB_STATE_ALLOC); /* It isn't obvious that this is impossible */ @@ -1007,20 +995,6 @@ process_pollq(int thread_id) if (ev & EPOLLERR) { int eno = gw_getsockerrno(dcb->fd); -#if defined(FAKE_CODE) - if (eno == 0) - { - eno = dcb_fake_write_errno[dcb->fd]; - char errbuf[MXS_STRERROR_BUFLEN]; - MXS_DEBUG("%lu [poll_waitevents] " - "Added fake errno %d. " - "%s", - pthread_self(), - eno, - strerror_r(eno, errbuf, sizeof(errbuf))); - } - dcb_fake_write_errno[dcb->fd] = 0; -#endif /* FAKE_CODE */ if (eno != 0) { char errbuf[MXS_STRERROR_BUFLEN]; diff --git a/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c b/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c index fe48791b6..89de78cc5 100644 --- a/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c +++ b/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c @@ -402,9 +402,6 @@ gw_do_connect_to_backend(char *host, int port, int *fd) MXS_DEBUG("%lu [gw_do_connect_to_backend] Connected to backend server " "%s:%d, fd %d.", pthread_self(), host, port, so); -#if defined(FAKE_CODE) - conn_open[so] = true; -#endif /* FAKE_CODE */ return_rv: return rv; diff --git a/server/modules/routing/debugcli/debugcmd.c b/server/modules/routing/debugcli/debugcmd.c index 799a4f6f9..eff2e3bef 100644 --- a/server/modules/routing/debugcli/debugcmd.c +++ b/server/modules/routing/debugcli/debugcmd.c @@ -732,51 +732,6 @@ struct subcommand disableoptions[] = } }; -#if defined(FAKE_CODE) - -static void fail_backendfd(void); -static void fail_clientfd(void); -static void fail_accept(DCB* dcb, char* arg1, char* arg2); -/** - * * The subcommands of the fail command - * */ -struct subcommand failoptions[] = -{ - { - "backendfd", - 0, - fail_backendfd, - "Fail backend socket for next operation.", - "Fail backend socket for next operation.", - {ARG_TYPE_STRING, 0, 0} - }, - { - "clientfd", - 0, - fail_clientfd, - "Fail client socket for next operation.", - "Fail client socket for next operation.", - {ARG_TYPE_STRING, 0, 0} - }, - { - "accept", - 2, - fail_accept, - "Fail to accept next client connection.", - "Fail to accept next client connection.", - {ARG_TYPE_STRING, ARG_TYPE_STRING, 0} - }, - { - NULL, - 0, - NULL, - NULL, - NULL, - {0, 0, 0} - } -}; -#endif /* FAKE_CODE */ - static void telnetdAddUser(DCB *, char *user, char *password); static void cmd_AddServer(DCB *dcb, void *a, void *b) @@ -1231,9 +1186,6 @@ static struct { "clear", clearoptions }, { "disable", disableoptions }, { "enable", enableoptions }, -#if defined(FAKE_CODE) - { "fail", failoptions }, -#endif /* FAKE_CODE */ { "flush", flushoptions }, { "list", listoptions }, { "reload", reloadoptions }, @@ -2340,50 +2292,3 @@ disable_account(DCB *dcb, char *user) dcb_printf(dcb, "Failed to disable the Linux user %s: %s\n", user, err); } } - -#if defined(FAKE_CODE) -static void fail_backendfd(void) -{ - fail_next_backend_fd = true; -} - -static void fail_clientfd(void) -{ - fail_next_client_fd = true; -} - -static void fail_accept( - DCB* dcb, - char* arg1, - char* arg2) -{ - int failcount = MXS_MIN(atoi(arg2), 100); - fail_accept_errno = atoi(arg1); - char errbuf[MXS_STRERROR_BUFLEN]; - - switch (fail_accept_errno) - { - case EAGAIN: -// case EWOULDBLOCK: - case EBADF: - case EINTR: - case EINVAL: - case EMFILE: - case ENFILE: - case ENOTSOCK: - case EOPNOTSUPP: - case ENOBUFS: - case ENOMEM: - case EPROTO: - fail_next_accept = failcount; - break; - - default: - dcb_printf(dcb, - "[%d, %s] is not valid errno for accept.\n", - fail_accept_errno, - strerror_r(fail_accept_errno, errbuf, sizeof(errbuf))); - return ; - } -} -#endif /* FAKE_CODE */ From d7c476314eaff331a21c57cd5e5a64231a95d2bc Mon Sep 17 00:00:00 2001 From: ekorh475 Date: Fri, 11 Nov 2016 10:00:06 +0200 Subject: [PATCH 202/215] Have maxadmin ask for password if none was given with -p MXS-399 If maxadmin was started with -p and no password was given, it printed a help message. Now it asks for password as if no -p was given. --- client/maxadmin.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/client/maxadmin.c b/client/maxadmin.c index 028c4ad01..6deaf1ee7 100644 --- a/client/maxadmin.c +++ b/client/maxadmin.c @@ -91,12 +91,11 @@ prompt(EditLine *el __attribute__((__unused__))) } #endif - static struct option long_options[] = { {"host", required_argument, 0, 'h'}, {"user", required_argument, 0, 'u'}, - {"password", required_argument, 0, 'p'}, + {"password", optional_argument, 0, 'p'}, {"port", required_argument, 0, 'P'}, {"socket", required_argument, 0, 'S'}, {"version", no_argument, 0, 'v'}, @@ -142,7 +141,7 @@ main(int argc, char **argv) int option_index = 0; char c; - while ((c = getopt_long(argc, argv, "h:p:P:u:S:v?e", + while ((c = getopt_long(argc, argv, "h:p::P:u:S:v?e", long_options, &option_index)) >= 0) { switch (c) @@ -154,8 +153,12 @@ main(int argc, char **argv) case 'p': use_inet_socket = true; - passwd = strdup(optarg); - memset(optarg, '\0', strlen(optarg)); + // If password was not given, ask for it later + if (optarg != NULL) + { + passwd = strdup(optarg); + memset(optarg, '\0', strlen(optarg)); + } break; case 'P': From 15e9652c46004b3c5ea37a504e192540dfbbf4fb Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Tue, 8 Nov 2016 18:05:00 +0200 Subject: [PATCH 203/215] qc: Provide information about field usage Together with the field names, now qc_get_field_info also returns field usage information, that is, in what context a field is used. This allows, for instance, the cache to take action if a a particular field is selected (SELECT a FROM ...), but not if it is used in a GROUP BY clause (...GROUP BY a). This caused a significant modifications of qc_mysqlembedded that earlier did not walk the parse-tree, but instead looped over of a list of st_select_lex instances that, the name notwithstanding, also contain information about other things but SELECTs. The former approach lost all contextual information, so it was not possible to know where a particular field was used. Now the parse tree is walked, which means that the contextual information is known, and thus the field usage can be updated. --- include/maxscale/query_classifier.h | 49 ++- .../qc_mysqlembedded/qc_mysqlembedded.cc | 257 ++++++++++---- query_classifier/qc_sqlite/qc_sqlite.c | 336 ++++++++++-------- .../qc_sqlite/sqlite-src-3110100/src/parse.y | 4 +- query_classifier/test/compare.cc | 19 +- server/core/query_classifier.c | 147 +++++++- 6 files changed, 581 insertions(+), 231 deletions(-) diff --git a/include/maxscale/query_classifier.h b/include/maxscale/query_classifier.h index b7e3f4639..db1960994 100644 --- a/include/maxscale/query_classifier.h +++ b/include/maxscale/query_classifier.h @@ -86,17 +86,37 @@ typedef enum qc_parse_result QC_QUERY_PARSED = 3 /*< The query was fully parsed; completely classified. */ } qc_parse_result_t; +/** + * qc_field_usage_t defines where a particular field appears. + * + * QC_USED_IN_SELECT : The field appears on the left side of FROM in a top-level SELECT statement. + * QC_USED_IN_SUBSELECT: The field appears on the left side of FROM in a sub-select SELECT statement. + * QC_USED_IN_WHERE : The field appears in a WHERE clause. + * QC_USED_IN_SET : The field appears in the SET clause of an UPDATE statement. + * QC_USED_IN_GROUP_BY : The field appears in a GROUP BY clause. + * + * Note that multiple bits may be set at the same time. For instance, for a statement like + * "SELECT fld FROM tbl WHERE fld = 1 GROUP BY fld", the bits QC_USED_IN_SELECT, QC_USED_IN_WHERE + * and QC_USED_IN_GROUP_BY will be set. + */ +typedef enum qc_field_usage +{ + QC_USED_IN_SELECT = 0x01, /*< SELECT fld FROM... */ + QC_USED_IN_SUBSELECT = 0x02, /*< SELECT 1 FROM ... SELECT fld ... */ + QC_USED_IN_WHERE = 0x04, /*< SELECT ... FROM ... WHERE fld = ... */ + QC_USED_IN_SET = 0x08, /*< UPDATE ... SET fld = ... */ + QC_USED_IN_GROUP_BY = 0x10, /*< ... GROUP BY fld */ +} qc_field_usage_t; /** * QC_FIELD_INFO contains information about a field used in a statement. */ typedef struct qc_field_info { - char* database; /** Present if the field is of the form "a.b.c", NULL otherwise. */ - char* table; /** Present if the field is of the form "a.b", NULL otherwise. */ - char* column; /** Always present. */ - // TODO: Possibly add bits telling where the field is used; e.g. in the select - // TODO: part or the where part, or both. + char* database; /** Present if the field is of the form "a.b.c", NULL otherwise. */ + char* table; /** Present if the field is of the form "a.b", NULL otherwise. */ + char* column; /** Always present. */ + uint32_t usage; /** Bitfield denoting where the column appears. */ } QC_FIELD_INFO; /** @@ -224,6 +244,25 @@ void qc_thread_end(void); */ qc_parse_result_t qc_parse(GWBUF* stmt); +/** + * Convert a qc_field_usage_t enum to corresponding string. + * + * @param usage The value to be converted + * + * @return The corresponding string. Must @b not be freed. + */ +const char* qc_field_usage_to_string(qc_field_usage_t usage); + +/** + * Convert a mask of qc_field_usage_t enum values to corresponding string. + * + * @param usage_mask Mask of qc_field_usage_t values. + * + * @return The corresponding string, or NULL if memory allocation fails. + * @b Must be freed by the caller. + */ +char* qc_field_usage_mask_to_string(uint32_t usage_mask); + /** * Returns information about affected fields. * diff --git a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc index 29ec2af53..e13a469ea 100644 --- a/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc +++ b/query_classifier/qc_mysqlembedded/qc_mysqlembedded.cc @@ -1767,20 +1767,12 @@ static void add_field_info(parsing_info_t* info, const char* database, const char* table, const char* column, + uint32_t usage, List* excludep) { ss_dassert(column); - // If only a column is specified, but not a table or database and we - // have a list of expressions that should be excluded, we check if the column - // value is present in that list. This is in order to exclude the second "d" in - // a statement like "select a as d from x where d = 2". - if (column && !table && !database && excludep && should_exclude(column, excludep)) - { - return; - } - - QC_FIELD_INFO item = { (char*)database, (char*)table, (char*)column }; + QC_FIELD_INFO item = { (char*)database, (char*)table, (char*)column, usage }; size_t i; for (i = 0; i < info->field_infos_len; ++i) @@ -1814,22 +1806,33 @@ static void add_field_info(parsing_info_t* info, if (i == info->field_infos_len) // If true, the field was not present already. { - if (info->field_infos_len < info->field_infos_capacity) + // If only a column is specified, but not a table or database and we + // have a list of expressions that should be excluded, we check if the column + // value is present in that list. This is in order to exclude the second "d" in + // a statement like "select a as d from x where d = 2". + if (!(column && !table && !database && excludep && should_exclude(column, excludep))) { - field_infos = info->field_infos; - } - else - { - size_t capacity = info->field_infos_capacity ? 2 * info->field_infos_capacity : 8; - field_infos = (QC_FIELD_INFO*)realloc(info->field_infos, capacity * sizeof(QC_FIELD_INFO)); - - if (field_infos) + if (info->field_infos_len < info->field_infos_capacity) { - info->field_infos = field_infos; - info->field_infos_capacity = capacity; + field_infos = info->field_infos; + } + else + { + size_t capacity = info->field_infos_capacity ? 2 * info->field_infos_capacity : 8; + field_infos = (QC_FIELD_INFO*)realloc(info->field_infos, capacity * sizeof(QC_FIELD_INFO)); + + if (field_infos) + { + info->field_infos = field_infos; + info->field_infos_capacity = capacity; + } } } } + else + { + info->field_infos[i].usage |= usage; + } // If field_infos is NULL, then the field was found and has already been noted. if (field_infos) @@ -1848,7 +1851,7 @@ static void add_field_info(parsing_info_t* info, } } -static void add_field_info(parsing_info_t* pi, Item_field* item, List* excludep) +static void add_field_info(parsing_info_t* pi, Item_field* item, uint32_t usage, List* excludep) { const char* database = item->db_name; const char* table = item->table_name; @@ -1934,16 +1937,16 @@ static void add_field_info(parsing_info_t* pi, Item_field* item, List* exc break; } - add_field_info(pi, database, table, column, excludep); + add_field_info(pi, database, table, column, usage, excludep); } -static void add_field_info(parsing_info_t* pi, Item* item, List* excludep) +static void add_field_info(parsing_info_t* pi, Item* item, uint32_t usage, List* excludep) { const char* database = NULL; const char* table = NULL; const char* column = item->name; - add_field_info(pi, database, table, column, excludep); + add_field_info(pi, database, table, column, usage, excludep); } typedef enum collect_source @@ -1954,9 +1957,16 @@ typedef enum collect_source COLLECT_GROUP_BY, } collect_source_t; +static void update_field_infos(parsing_info_t* pi, + LEX* lex, + st_select_lex* select, + uint32_t usage, + List* excludep); + static void update_field_infos(parsing_info_t* pi, collect_source_t source, Item* item, + uint32_t usage, List* excludep) { switch (item->type()) @@ -1968,13 +1978,13 @@ static void update_field_infos(parsing_info_t* pi, while (Item *i = ilist++) { - update_field_infos(pi, source, i, excludep); + update_field_infos(pi, source, i, usage, excludep); } } break; case Item::FIELD_ITEM: - add_field_info(pi, static_cast(item), excludep); + add_field_info(pi, static_cast(item), usage, excludep); break; case Item::REF_ITEM: @@ -1983,7 +1993,7 @@ static void update_field_infos(parsing_info_t* pi, { Item_ref* ref_item = static_cast(item); - add_field_info(pi, item, excludep); + add_field_info(pi, item, usage, excludep); size_t n_items = ref_item->cols(); @@ -1993,7 +2003,7 @@ static void update_field_infos(parsing_info_t* pi, if (reffed_item != ref_item) { - update_field_infos(pi, source, ref_item->element_index(i), excludep); + update_field_infos(pi, source, ref_item->element_index(i), usage, excludep); } } } @@ -2007,7 +2017,7 @@ static void update_field_infos(parsing_info_t* pi, for (size_t i = 0; i < n_items; ++i) { - update_field_infos(pi, source, row_item->element_index(i), excludep); + update_field_infos(pi, source, row_item->element_index(i), usage, excludep); } } break; @@ -2021,7 +2031,7 @@ static void update_field_infos(parsing_info_t* pi, for (size_t i = 0; i < n_items; ++i) { - update_field_infos(pi, source, items[i], excludep); + update_field_infos(pi, source, items[i], usage, excludep); } } break; @@ -2048,7 +2058,21 @@ static void update_field_infos(parsing_info_t* pi, if (in_subselect_item->left_expr_orig) { update_field_infos(pi, source, - in_subselect_item->left_expr_orig, excludep); + in_subselect_item->left_expr_orig, usage, excludep); + } + st_select_lex* ssl = in_subselect_item->get_select_lex(); + if (ssl) + { + uint32_t sub_usage = usage; + + sub_usage &= ~QC_USED_IN_SELECT; + sub_usage |= QC_USED_IN_SUBSELECT; + + update_field_infos(pi, + get_lex(pi), + ssl, + sub_usage, + excludep); } #else #pragma message "Figure out what to do with versions < 5.5.48." @@ -2058,10 +2082,21 @@ static void update_field_infos(parsing_info_t* pi, break; case Item_subselect::EXISTS_SUBS: - case Item_subselect::SINGLEROW_SUBS: // TODO: Handle these explicitly as well. break; + case Item_subselect::SINGLEROW_SUBS: + { + Item_singlerow_subselect* ss_item = static_cast(item); + st_select_lex *ssl = ss_item->get_select_lex(); + + usage &= ~QC_USED_IN_SELECT; + usage |= QC_USED_IN_SUBSELECT; + + update_field_infos(pi, get_lex(pi), ssl, usage, excludep); + } + break; + case Item_subselect::UNKNOWN_SUBS: default: MXS_ERROR("Unknown subselect type: %d", subselect_item->substype()); @@ -2075,6 +2110,69 @@ static void update_field_infos(parsing_info_t* pi, } } +static void update_field_infos(parsing_info_t* pi, + LEX* lex, + st_select_lex* select, + uint32_t usage, + List* excludep) +{ + List_iterator ilist(select->item_list); + + while (Item *item = ilist++) + { + update_field_infos(pi, COLLECT_SELECT, item, usage, NULL); + } + + if (select->group_list.first) + { + ORDER* order = select->group_list.first; + while (order) + { + Item* item = *order->item; + + update_field_infos(pi, COLLECT_GROUP_BY, item, QC_USED_IN_GROUP_BY, + &select->item_list); + + order = order->next; + } + } + + if (select->where) + { + update_field_infos(pi, COLLECT_WHERE, + select->where, + QC_USED_IN_WHERE, + &select->item_list); + } + +#if defined(COLLECT_HAVING_AS_WELL) + // A HAVING clause can only refer to fields that already have been + // mentioned. Consequently, they need not be collected. + if (select->having) + { + update_field_infos(pi, COLLECT_HAVING, + select->having, + 0, + &select->item_list); + } +#endif + + TABLE_LIST* table_list = select->get_table_list(); + + if (table_list) + { + st_select_lex *sl = table_list->get_single_select(); + + if (sl) + { + // This is for "SELECT 1 FROM (SELECT ...)" + usage &= ~QC_USED_IN_SELECT; + usage |= QC_USED_IN_SUBSELECT; + update_field_infos(pi, get_lex(pi), sl, usage, excludep); + } + } +} + void qc_get_field_info(GWBUF* buf, const QC_FIELD_INFO** infos, size_t* n_infos) { if (!buf) @@ -2088,10 +2186,10 @@ void qc_get_field_info(GWBUF* buf, const QC_FIELD_INFO** infos, size_t* n_infos) } parsing_info_t* pi = get_pinfo(buf); + ss_dassert(pi); if (!pi->field_infos) { - ss_dassert(pi); LEX* lex = get_lex(buf); ss_dassert(lex); @@ -2100,56 +2198,27 @@ void qc_get_field_info(GWBUF* buf, const QC_FIELD_INFO** infos, size_t* n_infos) return; } - lex->current_select = lex->all_selects_list; + uint32_t usage = 0; - while (lex->current_select) + switch (lex->sql_command) { - List_iterator ilist(lex->current_select->item_list); + case SQLCOM_UPDATE: + case SQLCOM_UPDATE_MULTI: + usage |= QC_USED_IN_SET; + break; - while (Item *item = ilist++) - { - update_field_infos(pi, COLLECT_SELECT, item, NULL); - } - - if (lex->current_select->group_list.first) - { - ORDER* order = lex->current_select->group_list.first; - while (order) - { - Item* item = *order->item; - - update_field_infos(pi, COLLECT_GROUP_BY, item, &lex->current_select->item_list); - - order = order->next; - } - } - - if (lex->current_select->where) - { - update_field_infos(pi, COLLECT_WHERE, - lex->current_select->where, - &lex->current_select->item_list); - } - -#if defined(COLLECT_HAVING_AS_WELL) - // A HAVING clause can only refer to fields that already have been - // mentioned. Consequently, they need not be collected. - if (lex->current_select->having) - { - update_field_infos(pi, COLLECT_HAVING, - lex->current_select->having, - &lex->current_select->item_list); - } -#endif - - lex->current_select = lex->current_select->next_select_in_list(); + default: + usage |= QC_USED_IN_SELECT; } + lex->current_select = &lex->select_lex; + + update_field_infos(pi, lex, &lex->select_lex, usage, NULL); List_iterator ilist(lex->value_list); while (Item* item = ilist++) { - update_field_infos(pi, COLLECT_SELECT, item, NULL); + update_field_infos(pi, COLLECT_SELECT, item, 0, NULL); } if ((lex->sql_command == SQLCOM_INSERT) || @@ -2159,7 +2228,7 @@ void qc_get_field_info(GWBUF* buf, const QC_FIELD_INFO** infos, size_t* n_infos) List_iterator ilist(lex->field_list); while (Item *item = ilist++) { - update_field_infos(pi, COLLECT_SELECT, item, NULL); + update_field_infos(pi, COLLECT_SELECT, item, 0, NULL); } if (lex->insert_list) @@ -2167,10 +2236,44 @@ void qc_get_field_info(GWBUF* buf, const QC_FIELD_INFO** infos, size_t* n_infos) List_iterator ilist(*lex->insert_list); while (Item *item = ilist++) { - update_field_infos(pi, COLLECT_SELECT, item, NULL); + update_field_infos(pi, COLLECT_SELECT, item, 0, NULL); } } } + + if (lex->sql_command == SQLCOM_SET_OPTION) + { +#if defined(WAY_TO_DOWNCAST_SET_VAR_BASE_EXISTS) + // The list of set_var_base contains the value of variables. + // However, the actual type is a derived type of set_var_base + // and there is no information using which we could do the + // downcast... + List_iterator ilist(lex->var_list); + while (set_var_base* var = ilist++) + { + // Is set_var_base a set_var, set_var_user, set_var_password + // set_var_role + ... + } +#endif + // ...so, we will simply assume that any nested selects are + // from statements like "set @a:=(SELECT a from t1)". + + usage &= ~QC_USED_IN_SELECT; + usage |= QC_USED_IN_SUBSELECT; + + st_select_lex* select = lex->all_selects_list; + + while (select) + { + if (select->nest_level != 0) // Not the top-level select. + { + update_field_infos(pi, lex, select, usage, NULL); + } + + select = select->next_select_in_list(); + } + } } *infos = pi->field_infos; diff --git a/query_classifier/qc_sqlite/qc_sqlite.c b/query_classifier/qc_sqlite/qc_sqlite.c index fc579103c..9ff54994e 100644 --- a/query_classifier/qc_sqlite/qc_sqlite.c +++ b/query_classifier/qc_sqlite/qc_sqlite.c @@ -140,17 +140,34 @@ static bool parse_query(GWBUF* query); static void parse_query_string(const char* query, size_t len); static bool query_is_parsed(GWBUF* query); static bool should_exclude(const char* zName, const ExprList* pExclude); -static void update_fields_infos(QC_SQLITE_INFO* info, - int prev_token, - const Expr* pExpr, - qc_token_position_t pos, - const ExprList* pExclude); -static void update_fields_infos_from_exprlist(QC_SQLITE_INFO* info, - const ExprList* pEList, const ExprList* pExclude); -static void update_fields_infos_from_idlist(QC_SQLITE_INFO* info, - const IdList* pIds, const ExprList* pExclude); -static void update_fields_infos_from_select(QC_SQLITE_INFO* info, - const Select* pSelect, const ExprList* pExclude); +static void update_field_info(QC_SQLITE_INFO* info, + const char* database, + const char* table, + const char* column, + uint32_t usage, + const ExprList* pExclude); +static void update_field_infos_from_expr(QC_SQLITE_INFO* info, + const struct Expr* pExpr, + uint32_t usage, + const ExprList* pExclude); +static void update_field_infos(QC_SQLITE_INFO* info, + int prev_token, + const Expr* pExpr, + uint32_t usage, + qc_token_position_t pos, + const ExprList* pExclude); +static void update_field_infos_from_exprlist(QC_SQLITE_INFO* info, + const ExprList* pEList, + uint32_t usage, + const ExprList* pExclude); +static void update_field_infos_from_idlist(QC_SQLITE_INFO* info, + const IdList* pIds, + uint32_t usage, + const ExprList* pExclude); +static void update_field_infos_from_select(QC_SQLITE_INFO* info, + const Select* pSelect, + uint32_t usage, + const ExprList* pExclude); static void update_database_names(QC_SQLITE_INFO* info, const char* name); static void update_names(QC_SQLITE_INFO* info, const char* zDatabase, const char* zTable); static void update_names_from_srclist(QC_SQLITE_INFO* info, const SrcList* pSrc); @@ -185,7 +202,7 @@ extern void exposed_sqlite3StartTable(Parse *pParse, /* Parser context */ int isView, /* True if this is a VIEW */ int isVirtual, /* True if this is a VIRTUAL table */ int noErr); /* Do nothing if table already exists */ -extern void maxscaleCollectInfoFromSelect(Parse*, Select*); +extern void maxscaleCollectInfoFromSelect(Parse*, Select*, int); /** * Used for freeing a QC_SQLITE_INFO object added to a GWBUF. @@ -688,24 +705,16 @@ static bool should_exclude(const char* zName, const ExprList* pExclude) return i != pExclude->nExpr; } -static void update_field_infos(QC_SQLITE_INFO* info, - const char* database, - const char* table, - const char* column, - const ExprList* pExclude) +static void update_field_info(QC_SQLITE_INFO* info, + const char* database, + const char* table, + const char* column, + uint32_t usage, + const ExprList* pExclude) { ss_dassert(column); - // If only a column is specified, but not a table or database and we - // have a list of expressions that should be excluded, we check if the column - // value is present in that list. This is in order to exclude the second "d" in - // a statement like "select a as d from x where d = 2". - if (column && !table && !database && pExclude && should_exclude(column, pExclude)) - { - return; - } - - QC_FIELD_INFO item = { (char*)database, (char*)table, (char*)column }; + QC_FIELD_INFO item = { (char*)database, (char*)table, (char*)column, usage }; int i; for (i = 0; i < info->field_infos_len; ++i) @@ -739,22 +748,33 @@ static void update_field_infos(QC_SQLITE_INFO* info, if (i == info->field_infos_len) // If true, the field was not present already. { - if (info->field_infos_len < info->field_infos_capacity) + // If only a column is specified, but not a table or database and we + // have a list of expressions that should be excluded, we check if the column + // value is present in that list. This is in order to exclude the second "d" in + // a statement like "select a as d from x where d = 2". + if (!(column && !table && !database && pExclude && should_exclude(column, pExclude))) { - field_infos = info->field_infos; - } - else - { - size_t capacity = info->field_infos_capacity ? 2 * info->field_infos_capacity : 8; - field_infos = MXS_REALLOC(info->field_infos, capacity * sizeof(QC_FIELD_INFO)); - - if (field_infos) + if (info->field_infos_len < info->field_infos_capacity) { - info->field_infos = field_infos; - info->field_infos_capacity = capacity; + field_infos = info->field_infos; + } + else + { + size_t capacity = info->field_infos_capacity ? 2 * info->field_infos_capacity : 8; + field_infos = MXS_REALLOC(info->field_infos, capacity * sizeof(QC_FIELD_INFO)); + + if (field_infos) + { + info->field_infos = field_infos; + info->field_infos_capacity = capacity; + } } } } + else + { + info->field_infos[i].usage |= usage; + } // If field_infos is NULL, then the field was found and has already been noted. if (field_infos) @@ -775,6 +795,7 @@ static void update_field_infos(QC_SQLITE_INFO* info, static void update_field_infos_from_expr(QC_SQLITE_INFO* info, const struct Expr* pExpr, + uint32_t usage, const ExprList* pExclude) { QC_FIELD_INFO item = {}; @@ -837,32 +858,33 @@ static void update_field_infos_from_expr(QC_SQLITE_INFO* info, if (should_update) { - update_field_infos(info, item.database, item.table, item.column, pExclude); + update_field_info(info, item.database, item.table, item.column, usage, pExclude); } } } -static void update_fields_infos(QC_SQLITE_INFO* info, - int prev_token, - const Expr* pExpr, - qc_token_position_t pos, - const ExprList* pExclude) +static void update_field_infos(QC_SQLITE_INFO* info, + int prev_token, + const Expr* pExpr, + uint32_t usage, + qc_token_position_t pos, + const ExprList* pExclude) { const char* zToken = pExpr->u.zToken; switch (pExpr->op) { case TK_ASTERISK: // select * - update_field_infos_from_expr(info, pExpr, pExclude); + update_field_infos_from_expr(info, pExpr, usage, pExclude); break; case TK_DOT: // select a.b ... select a.b.c - update_field_infos_from_expr(info, pExpr, pExclude); + update_field_infos_from_expr(info, pExpr, usage, pExclude); break; case TK_ID: // select a - update_field_infos_from_expr(info, pExpr, pExclude); + update_field_infos_from_expr(info, pExpr, usage, pExclude); break; case TK_VARIABLE: @@ -923,12 +945,17 @@ static void update_fields_infos(QC_SQLITE_INFO* info, if (pExpr->pLeft) { - update_fields_infos(info, pExpr->op, pExpr->pLeft, QC_TOKEN_LEFT, pExclude); + update_field_infos(info, pExpr->op, pExpr->pLeft, usage, QC_TOKEN_LEFT, pExclude); } if (pExpr->pRight) { - update_fields_infos(info, pExpr->op, pExpr->pRight, QC_TOKEN_RIGHT, pExclude); + if (usage & QC_USED_IN_SET) + { + usage &= ~QC_USED_IN_SET; + } + + update_field_infos(info, pExpr->op, pExpr->pRight, usage, QC_TOKEN_RIGHT, pExclude); } if (pExpr->x.pList) @@ -938,7 +965,7 @@ static void update_fields_infos(QC_SQLITE_INFO* info, case TK_BETWEEN: case TK_CASE: case TK_FUNCTION: - update_fields_infos_from_exprlist(info, pExpr->x.pList, pExclude); + update_field_infos_from_exprlist(info, pExpr->x.pList, usage, pExclude); break; case TK_EXISTS: @@ -946,11 +973,15 @@ static void update_fields_infos(QC_SQLITE_INFO* info, case TK_SELECT: if (pExpr->flags & EP_xIsSelect) { - update_fields_infos_from_select(info, pExpr->x.pSelect, pExclude); + uint32_t sub_usage = usage; + + sub_usage &= ~QC_USED_IN_SELECT; + sub_usage |= QC_USED_IN_SUBSELECT; + update_field_infos_from_select(info, pExpr->x.pSelect, sub_usage, pExclude); } else { - update_fields_infos_from_exprlist(info, pExpr->x.pList, pExclude); + update_field_infos_from_exprlist(info, pExpr->x.pList, usage, pExclude); } break; } @@ -959,33 +990,36 @@ static void update_fields_infos(QC_SQLITE_INFO* info, } } -static void update_fields_infos_from_exprlist(QC_SQLITE_INFO* info, - const ExprList* pEList, - const ExprList* pExclude) +static void update_field_infos_from_exprlist(QC_SQLITE_INFO* info, + const ExprList* pEList, + uint32_t usage, + const ExprList* pExclude) { for (int i = 0; i < pEList->nExpr; ++i) { struct ExprList_item* pItem = &pEList->a[i]; - update_fields_infos(info, 0, pItem->pExpr, QC_TOKEN_MIDDLE, pExclude); + update_field_infos(info, 0, pItem->pExpr, usage, QC_TOKEN_MIDDLE, pExclude); } } -static void update_fields_infos_from_idlist(QC_SQLITE_INFO* info, - const IdList* pIds, - const ExprList* pExclude) +static void update_field_infos_from_idlist(QC_SQLITE_INFO* info, + const IdList* pIds, + uint32_t usage, + const ExprList* pExclude) { for (int i = 0; i < pIds->nId; ++i) { struct IdList_item* pItem = &pIds->a[i]; - update_field_infos(info, NULL, NULL, pItem->zName, pExclude); + update_field_info(info, NULL, NULL, pItem->zName, usage, pExclude); } } -static void update_fields_infos_from_select(QC_SQLITE_INFO* info, - const Select* pSelect, - const ExprList* pExclude) +static void update_field_infos_from_select(QC_SQLITE_INFO* info, + const Select* pSelect, + uint32_t usage, + const ExprList* pExclude) { if (pSelect->pSrc) { @@ -1001,7 +1035,12 @@ static void update_fields_infos_from_select(QC_SQLITE_INFO* info, if (pSrc->a[i].pSelect) { - update_fields_infos_from_select(info, pSrc->a[i].pSelect, pExclude); + uint32_t sub_usage = usage; + + sub_usage &= ~QC_USED_IN_SELECT; + sub_usage |= QC_USED_IN_SUBSELECT; + + update_field_infos_from_select(info, pSrc->a[i].pSelect, sub_usage, pExclude); } #ifdef QC_COLLECT_NAMES_FROM_USING @@ -1011,7 +1050,7 @@ static void update_fields_infos_from_select(QC_SQLITE_INFO* info, // does not reveal its value, right? if (pSrc->a[i].pUsing) { - update_fields_infos_from_idlist(info, pSrc->a[i].pUsing, pSelect->pEList); + update_field_infos_from_idlist(info, pSrc->a[i].pUsing, 0, pSelect->pEList); } #endif } @@ -1019,18 +1058,18 @@ static void update_fields_infos_from_select(QC_SQLITE_INFO* info, if (pSelect->pEList) { - update_fields_infos_from_exprlist(info, pSelect->pEList, NULL); + update_field_infos_from_exprlist(info, pSelect->pEList, usage, NULL); } - if (pSelect->pWhere) + if (pSelect->pWhere) { info->has_clause = true; - update_fields_infos(info, 0, pSelect->pWhere, QC_TOKEN_MIDDLE, pSelect->pEList); + update_field_infos(info, 0, pSelect->pWhere, QC_USED_IN_WHERE, QC_TOKEN_MIDDLE, pSelect->pEList); } if (pSelect->pGroupBy) { - update_fields_infos_from_exprlist(info, pSelect->pGroupBy, pSelect->pEList); + update_field_infos_from_exprlist(info, pSelect->pGroupBy, QC_USED_IN_GROUP_BY, pSelect->pEList); } if (pSelect->pHaving) @@ -1039,7 +1078,7 @@ static void update_fields_infos_from_select(QC_SQLITE_INFO* info, #if defined(COLLECT_HAVING_AS_WELL) // A HAVING clause can only refer to fields that already have been // mentioned. Consequently, they need not be collected. - update_fields_infos(info, 0, pSelect->pHaving, QC_TOKEN_MIDDLE, pSelect->pEList); + update_field_infos(info, 0, pSelect->pHaving, 0, QC_TOKEN_MIDDLE, pSelect->pEList); #endif } } @@ -1285,7 +1324,7 @@ void mxs_sqlite3CreateView(Parse *pParse, /* The parsing context */ if (pSelect) { - update_fields_infos_from_select(info, pSelect, NULL); + update_field_infos_from_select(info, pSelect, QC_USED_IN_SELECT, NULL); info->is_real_query = false; } @@ -1355,7 +1394,7 @@ void mxs_sqlite3DeleteFrom(Parse* pParse, SrcList* pTabList, Expr* pWhere, SrcLi if (pWhere) { - update_fields_infos(info, 0, pWhere, QC_TOKEN_MIDDLE, 0); + update_field_infos(info, 0, pWhere, QC_USED_IN_WHERE, QC_TOKEN_MIDDLE, 0); } exposed_sqlite3ExprDelete(pParse->db, pWhere); @@ -1419,7 +1458,7 @@ void mxs_sqlite3EndTable(Parse *pParse, /* Parse context */ { if (pSelect) { - update_fields_infos_from_select(info, pSelect, NULL); + update_field_infos_from_select(info, pSelect, QC_USED_IN_SELECT, NULL); info->is_real_query = false; } else if (pOldTable) @@ -1465,17 +1504,28 @@ void mxs_sqlite3Insert(Parse* pParse, if (pColumns) { - update_fields_infos_from_idlist(info, pColumns, NULL); + update_field_infos_from_idlist(info, pColumns, 0, NULL); } if (pSelect) { - update_fields_infos_from_select(info, pSelect, NULL); + uint32_t usage; + + if (pSelect->selFlags & SF_Values) // Synthesized from VALUES clause + { + usage = 0; + } + else + { + usage = QC_USED_IN_SELECT; + } + + update_field_infos_from_select(info, pSelect, usage, NULL); } if (pSet) { - update_fields_infos_from_exprlist(info, pSet, NULL); + update_field_infos_from_exprlist(info, pSet, 0, NULL); } exposed_sqlite3SrcListDelete(pParse->db, pTabList); @@ -1511,7 +1561,7 @@ int mxs_sqlite3Select(Parse* pParse, Select* p, SelectDest* pDest) info->status = QC_QUERY_PARSED; info->operation = QUERY_OP_SELECT; - maxscaleCollectInfoFromSelect(pParse, p); + maxscaleCollectInfoFromSelect(pParse, p, 0); // NOTE: By convention, the select is deleted in parse.y. } else @@ -1599,13 +1649,13 @@ void mxs_sqlite3Update(Parse* pParse, SrcList* pTabList, ExprList* pChanges, Exp { struct ExprList_item* pItem = &pChanges->a[i]; - update_fields_infos(info, 0, pItem->pExpr, QC_TOKEN_MIDDLE, NULL); + update_field_infos(info, 0, pItem->pExpr, QC_USED_IN_SET, QC_TOKEN_MIDDLE, NULL); } } if (pWhere) { - update_fields_infos(info, 0, pWhere, QC_TOKEN_MIDDLE, pChanges); + update_field_infos(info, 0, pWhere, QC_USED_IN_WHERE, QC_TOKEN_MIDDLE, pChanges); } exposed_sqlite3SrcListDelete(pParse->db, pTabList); @@ -1613,7 +1663,7 @@ void mxs_sqlite3Update(Parse* pParse, SrcList* pTabList, ExprList* pChanges, Exp exposed_sqlite3ExprDelete(pParse->db, pWhere); } -void maxscaleCollectInfoFromSelect(Parse* pParse, Select* pSelect) +void maxscaleCollectInfoFromSelect(Parse* pParse, Select* pSelect, int sub_select) { QC_SQLITE_INFO* info = this_thread.info; ss_dassert(info); @@ -1631,7 +1681,9 @@ void maxscaleCollectInfoFromSelect(Parse* pParse, Select* pSelect) info->types = QUERY_TYPE_READ; } - update_fields_infos_from_select(info, pSelect, NULL); + uint32_t usage = sub_select ? QC_USED_IN_SUBSELECT : QC_USED_IN_SELECT; + + update_field_infos_from_select(info, pSelect, usage, NULL); } void maxscaleAlterTable(Parse *pParse, /* Parser context. */ @@ -1783,12 +1835,13 @@ void maxscaleExplain(Parse* pParse, SrcList* pName) info->status = QC_QUERY_PARSED; info->types = QUERY_TYPE_READ; update_names(info, "information_schema", "COLUMNS"); - update_field_infos(info, "information_schema", "COLUMNS", "COLUMN_DEFAULT", NULL); - update_field_infos(info, "information_schema", "COLUMNS", "COLUMN_KEY", NULL); - update_field_infos(info, "information_schema", "COLUMNS", "COLUMN_NAME", NULL); - update_field_infos(info, "information_schema", "COLUMNS", "COLUMN_TYPE", NULL); - update_field_infos(info, "information_schema", "COLUMNS", "EXTRA", NULL); - update_field_infos(info, "information_schema", "COLUMNS", "IS_NULLABLE", NULL); + uint32_t u = QC_USED_IN_SELECT; + update_field_info(info, "information_schema", "COLUMNS", "COLUMN_DEFAULT", u, NULL); + update_field_info(info, "information_schema", "COLUMNS", "COLUMN_KEY", u, NULL); + update_field_info(info, "information_schema", "COLUMNS", "COLUMN_NAME", u, NULL); + update_field_info(info, "information_schema", "COLUMNS", "COLUMN_TYPE", u, NULL); + update_field_info(info, "information_schema", "COLUMNS", "EXTRA", u, NULL); + update_field_info(info, "information_schema", "COLUMNS", "IS_NULLABLE", u, NULL); exposed_sqlite3SrcListDelete(pParse->db, pName); } @@ -2308,7 +2361,8 @@ void maxscaleSet(Parse* pParse, int scope, mxs_set_t kind, ExprList* pList) if (pValue->op == TK_SELECT) { - update_fields_infos_from_select(info, pValue->x.pSelect, NULL); + update_field_infos_from_select(info, pValue->x.pSelect, + QC_USED_IN_SUBSELECT, NULL); info->is_real_query = false; // TODO: This is what qc_mysqlembedded claims. } } @@ -2356,6 +2410,8 @@ extern void maxscaleShow(Parse* pParse, MxsShow* pShow) zName = name; } + uint32_t u = QC_USED_IN_SELECT; + switch (pShow->what) { case MXS_SHOW_COLUMNS: @@ -2364,24 +2420,24 @@ extern void maxscaleShow(Parse* pParse, MxsShow* pShow) update_names(info, "information_schema", "COLUMNS"); if (pShow->data == MXS_SHOW_COLUMNS_FULL) { - update_field_infos(info, "information_schema", "COLUMNS", "COLLATION_NAME", NULL); - update_field_infos(info, "information_schema", "COLUMNS", "COLUMN_COMMENT", NULL); - update_field_infos(info, "information_schema", "COLUMNS", "COLUMN_DEFAULT", NULL); - update_field_infos(info, "information_schema", "COLUMNS", "COLUMN_KEY", NULL); - update_field_infos(info, "information_schema", "COLUMNS", "COLUMN_NAME", NULL); - update_field_infos(info, "information_schema", "COLUMNS", "COLUMN_TYPE", NULL); - update_field_infos(info, "information_schema", "COLUMNS", "EXTRA", NULL); - update_field_infos(info, "information_schema", "COLUMNS", "IS_NULLABLE", NULL); - update_field_infos(info, "information_schema", "COLUMNS", "PRIVILEGES", NULL); + update_field_info(info, "information_schema", "COLUMNS", "COLLATION_NAME", u, NULL); + update_field_info(info, "information_schema", "COLUMNS", "COLUMN_COMMENT", u, NULL); + update_field_info(info, "information_schema", "COLUMNS", "COLUMN_DEFAULT", u, NULL); + update_field_info(info, "information_schema", "COLUMNS", "COLUMN_KEY", u, NULL); + update_field_info(info, "information_schema", "COLUMNS", "COLUMN_NAME", u, NULL); + update_field_info(info, "information_schema", "COLUMNS", "COLUMN_TYPE", u, NULL); + update_field_info(info, "information_schema", "COLUMNS", "EXTRA", u, NULL); + update_field_info(info, "information_schema", "COLUMNS", "IS_NULLABLE", u, NULL); + update_field_info(info, "information_schema", "COLUMNS", "PRIVILEGES", u, NULL); } else { - update_field_infos(info, "information_schema", "COLUMNS", "COLUMN_DEFAULT", NULL); - update_field_infos(info, "information_schema", "COLUMNS", "COLUMN_KEY", NULL); - update_field_infos(info, "information_schema", "COLUMNS", "COLUMN_NAME", NULL); - update_field_infos(info, "information_schema", "COLUMNS", "COLUMN_TYPE", NULL); - update_field_infos(info, "information_schema", "COLUMNS", "EXTRA", NULL); - update_field_infos(info, "information_schema", "COLUMNS", "IS_NULLABLE", NULL); + update_field_info(info, "information_schema", "COLUMNS", "COLUMN_DEFAULT", u, NULL); + update_field_info(info, "information_schema", "COLUMNS", "COLUMN_KEY", u, NULL); + update_field_info(info, "information_schema", "COLUMNS", "COLUMN_NAME", u, NULL); + update_field_info(info, "information_schema", "COLUMNS", "COLUMN_TYPE", u, NULL); + update_field_info(info, "information_schema", "COLUMNS", "EXTRA", u, NULL); + update_field_info(info, "information_schema", "COLUMNS", "IS_NULLABLE", u, NULL); } } break; @@ -2404,7 +2460,7 @@ extern void maxscaleShow(Parse* pParse, MxsShow* pShow) { info->types = QUERY_TYPE_SHOW_DATABASES; update_names(info, "information_schema", "SCHEMATA"); - update_field_infos(info, "information_schema", "SCHEMATA", "SCHEMA_NAME", NULL); + update_field_info(info, "information_schema", "SCHEMATA", "SCHEMA_NAME", u, NULL); } break; @@ -2414,19 +2470,19 @@ extern void maxscaleShow(Parse* pParse, MxsShow* pShow) { info->types = QUERY_TYPE_WRITE; update_names(info, "information_schema", "STATISTICS"); - update_field_infos(info, "information_schema", "STATISTICS", "CARDINALITY", NULL); - update_field_infos(info, "information_schema", "STATISTICS", "COLLATION", NULL); - update_field_infos(info, "information_schema", "STATISTICS", "COLUMN_NAME", NULL); - update_field_infos(info, "information_schema", "STATISTICS", "COMMENT", NULL); - update_field_infos(info, "information_schema", "STATISTICS", "INDEX_COMMENT", NULL); - update_field_infos(info, "information_schema", "STATISTICS", "INDEX_NAME", NULL); - update_field_infos(info, "information_schema", "STATISTICS", "INDEX_TYPE", NULL); - update_field_infos(info, "information_schema", "STATISTICS", "NON_UNIQUE", NULL); - update_field_infos(info, "information_schema", "STATISTICS", "NULLABLE", NULL); - update_field_infos(info, "information_schema", "STATISTICS", "PACKED", NULL); - update_field_infos(info, "information_schema", "STATISTICS", "SEQ_IN_INDEX", NULL); - update_field_infos(info, "information_schema", "STATISTICS", "SUB_PART", NULL); - update_field_infos(info, "information_schema", "STATISTICS", "TABLE_NAME", NULL); + update_field_info(info, "information_schema", "STATISTICS", "CARDINALITY", u, NULL); + update_field_info(info, "information_schema", "STATISTICS", "COLLATION", u, NULL); + update_field_info(info, "information_schema", "STATISTICS", "COLUMN_NAME", u, NULL); + update_field_info(info, "information_schema", "STATISTICS", "COMMENT", u, NULL); + update_field_info(info, "information_schema", "STATISTICS", "INDEX_COMMENT", u, NULL); + update_field_info(info, "information_schema", "STATISTICS", "INDEX_NAME", u, NULL); + update_field_info(info, "information_schema", "STATISTICS", "INDEX_TYPE", u, NULL); + update_field_info(info, "information_schema", "STATISTICS", "NON_UNIQUE", u, NULL); + update_field_info(info, "information_schema", "STATISTICS", "NULLABLE", u, NULL); + update_field_info(info, "information_schema", "STATISTICS", "PACKED", u, NULL); + update_field_info(info, "information_schema", "STATISTICS", "SEQ_IN_INDEX", u, NULL); + update_field_info(info, "information_schema", "STATISTICS", "SUB_PART", u, NULL); + update_field_info(info, "information_schema", "STATISTICS", "TABLE_NAME", u, NULL); } break; @@ -2434,24 +2490,24 @@ extern void maxscaleShow(Parse* pParse, MxsShow* pShow) { info->types = QUERY_TYPE_WRITE; update_names(info, "information_schema", "TABLES"); - update_field_infos(info, "information_schema", "TABLES", "AUTO_INCREMENT", NULL); - update_field_infos(info, "information_schema", "TABLES", "AVG_ROW_LENGTH", NULL); - update_field_infos(info, "information_schema", "TABLES", "CHECKSUM", NULL); - update_field_infos(info, "information_schema", "TABLES", "CHECK_TIME", NULL); - update_field_infos(info, "information_schema", "TABLES", "CREATE_OPTIONS", NULL); - update_field_infos(info, "information_schema", "TABLES", "CREATE_TIME", NULL); - update_field_infos(info, "information_schema", "TABLES", "DATA_FREE", NULL); - update_field_infos(info, "information_schema", "TABLES", "DATA_LENGTH", NULL); - update_field_infos(info, "information_schema", "TABLES", "ENGINE", NULL); - update_field_infos(info, "information_schema", "TABLES", "INDEX_LENGTH", NULL); - update_field_infos(info, "information_schema", "TABLES", "MAX_DATA_LENGTH", NULL); - update_field_infos(info, "information_schema", "TABLES", "ROW_FORMAT", NULL); - update_field_infos(info, "information_schema", "TABLES", "TABLE_COLLATION", NULL); - update_field_infos(info, "information_schema", "TABLES", "TABLE_COMMENT", NULL); - update_field_infos(info, "information_schema", "TABLES", "TABLE_NAME", NULL); - update_field_infos(info, "information_schema", "TABLES", "TABLE_ROWS", NULL); - update_field_infos(info, "information_schema", "TABLES", "UPDATE_TIME", NULL); - update_field_infos(info, "information_schema", "TABLES", "VERSION", NULL); + update_field_info(info, "information_schema", "TABLES", "AUTO_INCREMENT", u, NULL); + update_field_info(info, "information_schema", "TABLES", "AVG_ROW_LENGTH", u, NULL); + update_field_info(info, "information_schema", "TABLES", "CHECKSUM", u, NULL); + update_field_info(info, "information_schema", "TABLES", "CHECK_TIME", u, NULL); + update_field_info(info, "information_schema", "TABLES", "CREATE_OPTIONS", u, NULL); + update_field_info(info, "information_schema", "TABLES", "CREATE_TIME", u, NULL); + update_field_info(info, "information_schema", "TABLES", "DATA_FREE", u, NULL); + update_field_info(info, "information_schema", "TABLES", "DATA_LENGTH", u, NULL); + update_field_info(info, "information_schema", "TABLES", "ENGINE", u, NULL); + update_field_info(info, "information_schema", "TABLES", "INDEX_LENGTH", u, NULL); + update_field_info(info, "information_schema", "TABLES", "MAX_DATA_LENGTH", u, NULL); + update_field_info(info, "information_schema", "TABLES", "ROW_FORMAT", u, NULL); + update_field_info(info, "information_schema", "TABLES", "TABLE_COLLATION", u, NULL); + update_field_info(info, "information_schema", "TABLES", "TABLE_COMMENT", u, NULL); + update_field_info(info, "information_schema", "TABLES", "TABLE_NAME", u, NULL); + update_field_info(info, "information_schema", "TABLES", "TABLE_ROWS", u, NULL); + update_field_info(info, "information_schema", "TABLES", "UPDATE_TIME", u, NULL); + update_field_info(info, "information_schema", "TABLES", "VERSION", u, NULL); } break; @@ -2465,8 +2521,8 @@ extern void maxscaleShow(Parse* pParse, MxsShow* pShow) // TODO: qc_mysqlembedded does not set the type bit. info->types = QUERY_TYPE_UNKNOWN; update_names(info, "information_schema", "SESSION_STATUS"); - update_field_infos(info, "information_schema", "SESSION_STATUS", "VARIABLE_NAME", NULL); - update_field_infos(info, "information_schema", "SESSION_STATUS", "VARIABLE_VALUE", NULL); + update_field_info(info, "information_schema", "SESSION_STATUS", "VARIABLE_NAME", u, NULL); + update_field_info(info, "information_schema", "SESSION_STATUS", "VARIABLE_VALUE", u, NULL); break; case MXS_SHOW_STATUS_MASTER: @@ -2491,7 +2547,7 @@ extern void maxscaleShow(Parse* pParse, MxsShow* pShow) { info->types = QUERY_TYPE_SHOW_TABLES; update_names(info, "information_schema", "TABLE_NAMES"); - update_field_infos(info, "information_schema", "TABLE_NAMES", "TABLE_NAME", NULL); + update_field_info(info, "information_schema", "TABLE_NAMES", "TABLE_NAME", u, NULL); } break; @@ -2506,8 +2562,8 @@ extern void maxscaleShow(Parse* pParse, MxsShow* pShow) info->types = QUERY_TYPE_SYSVAR_READ; } update_names(info, "information_schema", "SESSION_VARIABLES"); - update_field_infos(info, "information_schema", "SESSION_STATUS", "VARIABLE_NAME", NULL); - update_field_infos(info, "information_schema", "SESSION_STATUS", "VARIABLE_VALUE", NULL); + update_field_info(info, "information_schema", "SESSION_STATUS", "VARIABLE_NAME", u, NULL); + update_field_info(info, "information_schema", "SESSION_STATUS", "VARIABLE_VALUE", u, NULL); } break; diff --git a/query_classifier/qc_sqlite/sqlite-src-3110100/src/parse.y b/query_classifier/qc_sqlite/sqlite-src-3110100/src/parse.y index b015bc5c7..ab6b50ea7 100644 --- a/query_classifier/qc_sqlite/sqlite-src-3110100/src/parse.y +++ b/query_classifier/qc_sqlite/sqlite-src-3110100/src/parse.y @@ -94,7 +94,7 @@ extern int mxs_sqlite3Select(Parse*, Select*, SelectDest*); extern void mxs_sqlite3StartTable(Parse*,Token*,Token*,int,int,int,int); extern void mxs_sqlite3Update(Parse*, SrcList*, ExprList*, Expr*, int); -extern void maxscaleCollectInfoFromSelect(Parse*, Select*); +extern void maxscaleCollectInfoFromSelect(Parse*, Select*, int); extern void maxscaleAlterTable(Parse*, mxs_alter_t command, SrcList*, Token*); extern void maxscaleCall(Parse*, SrcList* pName); @@ -1444,7 +1444,7 @@ table_factor(A) ::= nm(X) DOT nm(Y) as_opt id(Z). { } table_factor(A) ::= LP oneselect(S) RP as_opt id. { - maxscaleCollectInfoFromSelect(pParse, S); + maxscaleCollectInfoFromSelect(pParse, S, 1); sqlite3SelectDelete(pParse->db, S); A = 0; } diff --git a/query_classifier/test/compare.cc b/query_classifier/test/compare.cc index 9c267846d..102b9e42b 100644 --- a/query_classifier/test/compare.cc +++ b/query_classifier/test/compare.cc @@ -862,6 +862,7 @@ public: : m_database(info.database ? info.database : "") , m_table(info.table ? info.table : "") , m_column(info.column ? info.column : "") + , m_usage(info.usage) {} bool eq(const QcFieldInfo& rhs) const @@ -869,7 +870,8 @@ public: return m_database == rhs.m_database && m_table == rhs.m_table && - m_column == rhs.m_column; + m_column == rhs.m_column && + m_usage == rhs.m_usage; } bool lt(const QcFieldInfo& rhs) const @@ -900,6 +902,14 @@ public: { rv = true; } + else if (m_column > rhs.m_column) + { + rv = false; + } + else + { + rv = (m_usage < rhs.m_usage); + } } } @@ -921,12 +931,19 @@ public: } out << m_column; + + out << "("; + char* s = qc_field_usage_mask_to_string(m_usage); + out << s; + free(s); + out << ")"; } private: std::string m_database; std::string m_table; std::string m_column; + uint32_t m_usage; }; ostream& operator << (ostream& out, const QcFieldInfo& x) diff --git a/server/core/query_classifier.c b/server/core/query_classifier.c index 3b9abef57..0a12f374a 100644 --- a/server/core/query_classifier.c +++ b/server/core/query_classifier.c @@ -27,6 +27,12 @@ #define QC_TRACE() #endif +struct type_name_info +{ + const char* name; + size_t name_len; +}; + static const char default_qc_name[] = "qc_sqlite"; static QUERY_CLASSIFIER* classifier; @@ -213,6 +219,141 @@ char* qc_get_prepare_name(GWBUF* query) return classifier->qc_get_prepare_name(query); } +struct type_name_info field_usage_to_type_name_info(qc_field_usage_t usage) +{ + struct type_name_info info; + + switch (usage) + { + case QC_USED_IN_SELECT: + { + static const char name[] = "QC_USED_IN_SELECT"; + info.name = name; + info.name_len = sizeof(name) - 1; + } + break; + + case QC_USED_IN_SUBSELECT: + { + static const char name[] = "QC_USED_IN_SUBSELECT"; + info.name = name; + info.name_len = sizeof(name) - 1; + } + break; + + case QC_USED_IN_WHERE: + { + static const char name[] = "QC_USED_IN_WHERE"; + info.name = name; + info.name_len = sizeof(name) - 1; + } + break; + + case QC_USED_IN_SET: + { + static const char name[] = "QC_USED_IN_SET"; + info.name = name; + info.name_len = sizeof(name) - 1; + } + break; + + case QC_USED_IN_GROUP_BY: + { + static const char name[] = "QC_USED_IN_GROUP_BY"; + info.name = name; + info.name_len = sizeof(name) - 1; + } + break; + + default: + { + static const char name[] = "UNKNOWN_FIELD_USAGE"; + info.name = name; + info.name_len = sizeof(name) - 1; + } + break; + } + + return info; +} + + + +const char* qc_field_usage_to_string(qc_field_usage_t usage) +{ + return field_usage_to_type_name_info(usage).name; +} + +static const qc_field_usage_t FIELD_USAGE_VALUES[] = +{ + QC_USED_IN_SELECT, + QC_USED_IN_SUBSELECT, + QC_USED_IN_WHERE, + QC_USED_IN_SET, + QC_USED_IN_GROUP_BY, +}; + +static const int N_FIELD_USAGE_VALUES = + sizeof(FIELD_USAGE_VALUES) / sizeof(FIELD_USAGE_VALUES[0]); +static const int FIELD_USAGE_MAX_LEN = 20; // strlen("QC_USED_IN_SUBSELECT"); + +char* qc_field_usage_mask_to_string(uint32_t mask) +{ + size_t len = 0; + + // First calculate how much space will be needed. + for (int i = 0; i < N_FIELD_USAGE_VALUES; ++i) + { + if (mask & FIELD_USAGE_VALUES[i]) + { + if (len != 0) + { + ++len; // strlen("|"); + } + + len += FIELD_USAGE_MAX_LEN; + } + } + + ++len; + + // Then make one allocation and build the string. + char* s = (char*) MXS_MALLOC(len); + + if (s) + { + if (len > 1) + { + char* p = s; + + for (int i = 0; i < N_FIELD_USAGE_VALUES; ++i) + { + qc_field_usage_t value = FIELD_USAGE_VALUES[i]; + + if (mask & value) + { + if (p != s) + { + strcpy(p, "|"); + ++p; + } + + struct type_name_info info = field_usage_to_type_name_info(value); + + strcpy(p, info.name); + p += info.name_len; + } + } + } + else + { + *s = 0; + } + } + + return s; +} + const char* qc_op_to_string(qc_query_op_t op) { switch (op) @@ -261,12 +402,6 @@ const char* qc_op_to_string(qc_query_op_t op) } } -struct type_name_info -{ - const char* name; - size_t name_len; -}; - struct type_name_info type_to_type_name_info(qc_query_type_t type) { struct type_name_info info; From fbb46f72eb09f2f63baf4814740a214ff58ed847 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Fri, 11 Nov 2016 11:27:29 +0100 Subject: [PATCH 204/215] MXS-930: Maxrows filter documentation MXS-930: Maxrows filter documentation --- Documentation/Filters/Maxrows.md | 66 ++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 Documentation/Filters/Maxrows.md diff --git a/Documentation/Filters/Maxrows.md b/Documentation/Filters/Maxrows.md new file mode 100644 index 000000000..376e9fec9 --- /dev/null +++ b/Documentation/Filters/Maxrows.md @@ -0,0 +1,66 @@ +# Maxrows + +## Overview +The maxrows filter is capable of restricting the amount of rows that a SELECT, a prepared statement or stored procedure +could return to the client application. +If a resultset from a backend server has more rows than the configured limit or the resultset size exceeds the configured size, +an empty result will be sent to the client. + +## Configuration + +The maxrows filter is easy to configure and to add to any existing service. + +``` +[MaxRows] +type=filter +module=maxrows + +[MaxRows Routing Service] +type=service +... +filters=maxrows +``` + +### Filter Parameters + +The maxrows filter has no mandatory parameters. +Optional parameters are: + +#### `max_resultset_rows` + +Specifies the maximum number of rows a resultset can have in order to be +returned to the usere. If a resultset is larger than this an empty result will be sent instead. + +``` +max_resultset_rows=1000 +``` +Zero or a negative value is interpreted as no limitation. + +The default value is `-1`. + +#### `max_resultset_size` + +Specifies the maximum size a resultset can have, measured in kibibytes, +in order to be sent to the client. A resultset larger than this, will +not be sent: an empty resultset will be sent instead. + +``` +max_resultset_size=128 +``` +The default value is 64. + +#### `debug` + +An integer value, using which the level of debug logging made by the maxrows +filter can be controlled. The value is actually a bitfield with different bits +denoting different logging. + + * ` 0` (`0b00000`) No logging is made. + * ` 1` (`0b00001`) A decision to handle data form server is logged. + * ` 2` (`0b00010`) Reached max_resultset_rows or max_resultset_size limit is logged. + +Default is `0`. To log everything, give `debug` a value of `3`. + +``` +debug=2 +``` From 11fc7682dd1564e0d578287b8400c44853129317 Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Fri, 11 Nov 2016 16:43:02 +0100 Subject: [PATCH 205/215] Maxrows filter documentation update Maxrows filter documentation update --- Documentation/Filters/Maxrows.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/Documentation/Filters/Maxrows.md b/Documentation/Filters/Maxrows.md index 376e9fec9..47edb688f 100644 --- a/Documentation/Filters/Maxrows.md +++ b/Documentation/Filters/Maxrows.md @@ -1,10 +1,12 @@ # Maxrows ## Overview -The maxrows filter is capable of restricting the amount of rows that a SELECT, a prepared statement or stored procedure -could return to the client application. -If a resultset from a backend server has more rows than the configured limit or the resultset size exceeds the configured size, -an empty result will be sent to the client. +The maxrows filter is capable of restricting the amount of rows that a SELECT, + a prepared statement or stored procedure could return to the client application. + +If a resultset from a backend server has more rows than the configured limit +or the resultset size exceeds the configured size, + an empty result will be sent to the client. ## Configuration @@ -28,8 +30,10 @@ Optional parameters are: #### `max_resultset_rows` -Specifies the maximum number of rows a resultset can have in order to be -returned to the usere. If a resultset is larger than this an empty result will be sent instead. +Specifies the maximum number of rows a resultset can have in order to be returned + to the user. + +If a resultset is larger than this an empty result will be sent instead. ``` max_resultset_rows=1000 @@ -57,7 +61,7 @@ denoting different logging. * ` 0` (`0b00000`) No logging is made. * ` 1` (`0b00001`) A decision to handle data form server is logged. - * ` 2` (`0b00010`) Reached max_resultset_rows or max_resultset_size limit is logged. + * ` 2` (`0b00010`) Reached max_resultset_rows or max_resultset_size is logged. Default is `0`. To log everything, give `debug` a value of `3`. From 11bee30f614ffdc818065b17fe9a5a96677fd2bd Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Sat, 12 Nov 2016 13:19:14 +0200 Subject: [PATCH 206/215] Fix backend SSL The backend SSL connections weren't authenticated due to an inverted check. This caused all SSL connections to fail. --- server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c | 5 +++++ server/modules/protocol/MySQL/mysql_common.c | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c b/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c index 89de78cc5..df89c4b01 100644 --- a/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c +++ b/server/modules/protocol/MySQL/MySQLBackend/mysql_backend.c @@ -617,6 +617,11 @@ gw_read_backend_event(DCB *dcb) gw_reply_on_error(dcb, proto->protocol_auth_state); } } + else if (proto->protocol_auth_state == MXS_AUTH_STATE_CONNECTED && + dcb->ssl_state == SSL_ESTABLISHED) + { + proto->protocol_auth_state = gw_send_backend_auth(dcb); + } } return rc; diff --git a/server/modules/protocol/MySQL/mysql_common.c b/server/modules/protocol/MySQL/mysql_common.c index 5104720be..9873b30ae 100644 --- a/server/modules/protocol/MySQL/mysql_common.c +++ b/server/modules/protocol/MySQL/mysql_common.c @@ -1339,7 +1339,7 @@ mxs_auth_state_t gw_send_backend_auth(DCB *dcb) (dcb->session->state != SESSION_STATE_READY && dcb->session->state != SESSION_STATE_ROUTER_READY) || (dcb->server->server_ssl && - dcb->ssl_state != SSL_HANDSHAKE_FAILED)) + dcb->ssl_state == SSL_HANDSHAKE_FAILED)) { return MXS_AUTH_STATE_FAILED; } From b893ca7ba8f7a7f25ab0b473a904f6e4f76f21ca Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Sat, 12 Nov 2016 13:22:18 +0200 Subject: [PATCH 207/215] Move configuration context processing into subfunctions The functions allow simple operations on configuration context objects. This makes it easier to understand what the code does and allows reuse of the configuration processing code. --- include/maxscale/config.h | 81 ++++++++++++++++- include/maxscale/server.h | 2 - server/core/config.c | 177 +++++++++++++++++++++++--------------- server/core/server.c | 11 --- 4 files changed, 186 insertions(+), 85 deletions(-) diff --git a/include/maxscale/config.h b/include/maxscale/config.h index 8e616aae4..6f02ec613 100644 --- a/include/maxscale/config.h +++ b/include/maxscale/config.h @@ -32,6 +32,7 @@ #include #include #include +#include MXS_BEGIN_DECLS @@ -129,13 +130,91 @@ typedef struct } GATEWAY_CONF; +/** + * @brief Creates an empty configuration context + * + * @param section Context name + * @return New context or NULL on memory allocation failure + */ +CONFIG_CONTEXT* config_context_create(const char *section); + +/** + * @brief Free a configuration context + * + * @param context The context to free + */ +void config_context_free(CONFIG_CONTEXT *context); + +/** + * @brief Get a configuration parameter + * + * @param params List of parameters + * @param name Name of parameter to get + * @return The parameter or NULL if the parameter was not found + */ +CONFIG_PARAMETER* config_get_param(CONFIG_PARAMETER* params, const char* name); + +/** + * @brief Add a parameter to a configuration context + * + * @param obj Context where the parameter should be added + * @param key Key to add + * @param value Value for the key + * @return True on success, false on memory allocation error + */ +bool config_add_param(CONFIG_CONTEXT* obj, const char* key, const char* value); + +/** + * @brief Append to an existing parameter + * + * @param obj Configuration context + * @param key Parameter name + * @param value Value to append to the parameter + * @return True on success, false on memory allocation error + */ +bool config_append_param(CONFIG_CONTEXT* obj, const char* key, const char* value); + +/** + * @brief Check if all SSL parameters are defined + * + * Helper function to check whether all of the required SSL parameters are defined + * in the configuration context. The checked parameters are 'ssl', 'ssl_key', + * 'ssl_cert' and 'ssl_ca_cert'. The 'ssl' parameter must also have a value of + * 'required'. + * + * @param obj Configuration context + * @return True if all required parameters are present + */ +bool config_have_required_ssl_params(CONFIG_CONTEXT *obj); + +/** + * @brief Helper function for checking SSL parameters + * + * @param key Parameter name + * @return True if the parameter is an SSL parameter + */ +bool config_is_ssl_parameter(const char *key); + +/** + * @brief Construct an SSL structure + * + * The SSL structure is used by both listeners and servers. + * + * TODO: Rename to something like @c config_construct_ssl + * + * @param obj Configuration context + * @param require_cert Whether certificates are required + * @param error_count Pointer to an int which is incremented for each error + * @return New SSL_LISTENER structure or NULL on error + */ +SSL_LISTENER *make_ssl_structure(CONFIG_CONTEXT *obj, bool require_cert, int *error_count); + char* config_clean_string_list(const char* str); CONFIG_PARAMETER* config_clone_param(const CONFIG_PARAMETER* param); void config_enable_feedback_task(void); void config_disable_feedback_task(void); unsigned long config_get_gateway_id(void); GATEWAY_CONF* config_get_global_options(); -CONFIG_PARAMETER* config_get_param(CONFIG_PARAMETER* params, const char* name); config_param_type_t config_get_paramtype(const CONFIG_PARAMETER* param); bool config_get_valint(int* val, const CONFIG_PARAMETER* param, diff --git a/include/maxscale/server.h b/include/maxscale/server.h index eb28e1271..d47493237 100644 --- a/include/maxscale/server.h +++ b/include/maxscale/server.h @@ -280,7 +280,5 @@ extern void server_update_port(SERVER *, unsigned short); extern RESULTSET *serverGetList(); extern unsigned int server_map_status(char *str); extern bool server_set_version_string(SERVER* server, const char* string); -extern bool server_is_ssl_parameter(const char *key); -extern void server_update_ssl(SERVER *server, const char *key, const char *value); MXS_END_DECLS diff --git a/server/core/config.c b/server/core/config.c index 43a738397..6e8abb7a1 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -80,7 +80,6 @@ static void duplicate_context_finish(DUPLICATE_CONTEXT* context); extern int setipaddress(struct in_addr *, char *); static bool process_config_context(CONFIG_CONTEXT *); static bool process_config_update(CONFIG_CONTEXT *); -static void free_config_context(CONFIG_CONTEXT *); static char *config_get_value(CONFIG_PARAMETER *, const char *); static char *config_get_password(CONFIG_PARAMETER *); static const char *config_get_value_string(CONFIG_PARAMETER *, const char *); @@ -90,13 +89,11 @@ static void global_defaults(); static void feedback_defaults(); static bool check_config_objects(CONFIG_CONTEXT *context); static int maxscale_getline(char** dest, int* size, FILE* file); -static SSL_LISTENER *make_ssl_structure(CONFIG_CONTEXT *obj, bool require_cert, int *error_count); int config_truth_value(char *str); int config_get_ifaddr(unsigned char *output); static int config_get_release_string(char* release); FEEDBACK_CONF *config_get_feedback_data(); -void config_add_param(CONFIG_CONTEXT*, char*, char*); bool config_has_duplicate_sections(const char* config, DUPLICATE_CONTEXT* context); int create_new_service(CONFIG_CONTEXT *obj); int create_new_server(CONFIG_CONTEXT *obj); @@ -326,6 +323,20 @@ char* config_clean_string_list(const char* str) return dest; } +CONFIG_CONTEXT* config_context_create(const char *section) +{ + CONFIG_CONTEXT* ctx = (CONFIG_CONTEXT *)MXS_MALLOC(sizeof(CONFIG_CONTEXT)); + if (ctx) + { + ctx->object = MXS_STRDUP_A(section); + ctx->parameters = NULL; + ctx->next = NULL; + ctx->element = NULL; + } + + return ctx; +} + /** * Config item handler for the ini file reader * @@ -340,7 +351,6 @@ ini_handler(void *userdata, const char *section, const char *name, const char *v { CONFIG_CONTEXT *cntxt = (CONFIG_CONTEXT *)userdata; CONFIG_CONTEXT *ptr = cntxt; - CONFIG_PARAMETER *param, *p1; if (strcmp(section, "gateway") == 0 || strcasecmp(section, "MaxScale") == 0) { @@ -368,54 +378,24 @@ ini_handler(void *userdata, const char *section, const char *name, const char *v if (!ptr) { - if ((ptr = (CONFIG_CONTEXT *)MXS_MALLOC(sizeof(CONFIG_CONTEXT))) == NULL) + if ((ptr = config_context_create(section)) == NULL) { return 0; } - ptr->object = MXS_STRDUP_A(section); - ptr->parameters = NULL; ptr->next = cntxt->next; - ptr->element = NULL; cntxt->next = ptr; } - /* Check to see if the parameter already exists for the section */ - p1 = ptr->parameters; - while (p1) - { - if (!strcmp(p1->name, name)) - { - char *tmp; - int paramlen = strlen(p1->value) + strlen(value) + 2; - if ((tmp = MXS_REALLOC(p1->value, sizeof(char) * (paramlen))) == NULL) - { - return 0; - } - strcat(tmp, ","); - strcat(tmp, value); - if ((p1->value = config_clean_string_list(tmp)) == NULL) - { - p1->value = tmp; - MXS_ERROR("[%s] Cleaning configuration parameter failed.", __FUNCTION__); - return 0; - } - MXS_FREE(tmp); - return 1; - } - p1 = p1->next; - } - - if ((param = (CONFIG_PARAMETER *)MXS_MALLOC(sizeof(CONFIG_PARAMETER))) == NULL) + if (config_get_param(ptr->parameters, name) && + !config_append_param(ptr, name, value)) + { + return 0; + } + else if (!config_add_param(ptr, name, value)) { return 0; } - - param->name = MXS_STRDUP_A(name); - param->value = MXS_STRDUP_A(value); - param->next = ptr->parameters; - param->qfd_param_type = UNDEFINED_TYPE; - ptr->parameters = param; return 1; } @@ -637,7 +617,7 @@ config_load_and_process(const char* filename, bool (*process_config)(CONFIG_CONT } } - free_config_context(ccontext.next); + config_context_free(ccontext.next); duplicate_context_finish(&dcontext); } @@ -876,10 +856,7 @@ config_get_value_string(CONFIG_PARAMETER *params, const char *name) return ""; } - -CONFIG_PARAMETER* config_get_param( - CONFIG_PARAMETER* params, - const char* name) +CONFIG_PARAMETER* config_get_param(CONFIG_PARAMETER* params, const char* name) { while (params) { @@ -1042,22 +1019,15 @@ void free_config_parameter(CONFIG_PARAMETER* p1) } } -/** - * Free a config tree - * - * @param context The configuration data - */ -static void -free_config_context(CONFIG_CONTEXT *context) +void config_context_free(CONFIG_CONTEXT *context) { CONFIG_CONTEXT *obj; - CONFIG_PARAMETER *p1, *p2; while (context) { - MXS_FREE(context->object); - free_config_parameter(context->parameters); obj = context->next; + free_config_parameter(context->parameters); + MXS_FREE(context->object); MXS_FREE(context); context = obj; } @@ -1362,8 +1332,7 @@ free_ssl_structure(SSL_LISTENER *ssl) * @param *error_count An error count which may be incremented * @return SSL_LISTENER structure or NULL */ -static SSL_LISTENER * -make_ssl_structure (CONFIG_CONTEXT *obj, bool require_cert, int *error_count) +SSL_LISTENER* make_ssl_structure (CONFIG_CONTEXT *obj, bool require_cert, int *error_count) { char *ssl, *ssl_version, *ssl_cert, *ssl_key, *ssl_ca_cert, *ssl_cert_verify_depth; int local_errors = 0; @@ -2288,26 +2257,57 @@ unsigned long config_get_gateway_id() return gateway.id; } -void config_add_param(CONFIG_CONTEXT* obj, char* key, char* value) +bool config_add_param(CONFIG_CONTEXT* obj, const char* key, const char* value) { - key = MXS_STRDUP(key); - value = MXS_STRDUP(value); + ss_dassert(config_get_param(obj->parameters, key) == NULL); + bool rval = false; + char *my_key = MXS_STRDUP(key); + char *my_value = MXS_STRDUP(value); + CONFIG_PARAMETER* param = (CONFIG_PARAMETER *)MXS_MALLOC(sizeof(*param)); - CONFIG_PARAMETER* param = (CONFIG_PARAMETER *)MXS_MALLOC(sizeof(CONFIG_PARAMETER)); - - if (!key || !value || !param) + if (my_key && my_value && param) { - MXS_FREE(key); - MXS_FREE(value); + param->name = my_key; + param->value = my_value; + param->qfd_param_type = UNDEFINED_TYPE; + param->next = obj->parameters; + obj->parameters = param; + rval = true; + } + else + { + MXS_FREE(my_key); + MXS_FREE(my_value); MXS_FREE(param); - return; } - param->name = key; - param->value = value; - param->next = obj->parameters; - obj->parameters = param; + return rval; } + +bool config_append_param(CONFIG_CONTEXT* obj, const char* key, const char* value) +{ + CONFIG_PARAMETER *param = config_get_param(obj->parameters, key); + ss_dassert(param); + int paramlen = strlen(param->value) + strlen(value) + 2; + char tmp[paramlen]; + bool rval = false; + + strcpy(tmp, param->value); + strcat(tmp, ","); + strcat(tmp, value); + + char *new_value = config_clean_string_list(tmp); + + if (new_value) + { + MXS_FREE(param->value); + param->value = new_value; + rval = true; + } + + return rval; +} + /** * Return the pointer to the global options for MaxScale. * @return Pointer to the GATEWAY_CONF structure. This is a static structure and @@ -3158,3 +3158,38 @@ int create_new_filter(CONFIG_CONTEXT *obj) return error_count; } + +bool config_have_required_ssl_params(CONFIG_CONTEXT *obj) +{ + CONFIG_PARAMETER *param = obj->parameters; + + return config_get_param(param, "ssl") && + config_get_param(param, "ssl_key") && + config_get_param(param, "ssl_cert") && + config_get_param(param, "ssl_ca_cert") && + strcmp(config_get_value_string(param, "ssl"), "required") == 0; +} + +bool config_is_ssl_parameter(const char *key) +{ + const char *ssl_params[] = + { + "ssl_cert", + "ssl_ca_cert", + "ssl", + "ssl_key", + "ssl_version", + "ssl_cert_verify_depth", + NULL + }; + + for (int i = 0; ssl_params[i]; i++) + { + if (strcmp(key, ssl_params[i]) == 0) + { + return true; + } + } + + return false; +} diff --git a/server/core/server.c b/server/core/server.c index 1faee1076..6c096709f 100644 --- a/server/core/server.c +++ b/server/core/server.c @@ -1354,14 +1354,3 @@ bool server_destroy(SERVER *server) return rval; } - -bool server_is_ssl_parameter(const char *key) -{ - // TODO: Implement this - return false; -} - -void server_update_ssl(SERVER *server, const char *key, const char *value) -{ - // TODO: Implement this -} From 8b692b0754fae603a2533ec0f1b6a597aa5334e8 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Sat, 12 Nov 2016 13:23:50 +0200 Subject: [PATCH 208/215] Add SSL support for created servers Servers created at runtime can now be configured to use SSL. The configuration is only possible if the server is not in use. The `alter server` command in maxadmin now takes a list of `key=value` strings. This allows the user to define multiple alter operations with one command. --- include/maxscale/server.h | 12 +++ server/core/server.c | 18 ++-- server/modules/routing/debugcli/debugcmd.c | 115 +++++++++++++++++++-- 3 files changed, 122 insertions(+), 23 deletions(-) diff --git a/include/maxscale/server.h b/include/maxscale/server.h index d47493237..51fe48bea 100644 --- a/include/maxscale/server.h +++ b/include/maxscale/server.h @@ -245,6 +245,18 @@ extern bool server_create(const char *name, const char *address, const char *por const char *protocol, const char *authenticator, const char *options); +/** + * @brief Serialize a server to a file + * + * This converts @c server into an INI format file. This allows created servers + * to be persisted to disk. This will replace any existing files with the same + * name. + * + * @param server Server to serialize + * @return False if the serialization of the server fails, true if it was successful + */ +bool server_serialize(const SERVER *server); + /** * @brief Destroy a server * diff --git a/server/core/server.c b/server/core/server.c index 6c096709f..a51aa2521 100644 --- a/server/core/server.c +++ b/server/core/server.c @@ -987,6 +987,8 @@ server_update_address(SERVER *server, char *address) spinlock_acquire(&server_spin); if (server && address) { + MXS_NOTICE("Updated the address of server '%s' from '%s' to '%s'.", + server->unique_name, server->name, address); strcpy(server->name, address); } spinlock_release(&server_spin); @@ -1005,6 +1007,8 @@ server_update_port(SERVER *server, unsigned short port) spinlock_acquire(&server_spin); if (server && port > 0) { + MXS_NOTICE("Updated the port of server '%s' from %d to %d.", + server->unique_name, server->port, port); server->port = port; } spinlock_release(&server_spin); @@ -1082,7 +1086,7 @@ bool server_set_version_string(SERVER* server, const char* string) * @param filename Filename where configuration is written * @return True on success, false on error */ -static bool create_server_config(SERVER *server, const char *filename) +static bool create_server_config(const SERVER *server, const char *filename) { int file = open(filename, O_EXCL | O_CREAT | O_WRONLY, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); @@ -1182,17 +1186,7 @@ static bool create_server_config(SERVER *server, const char *filename) return true; } -/** - * @brief Serialize a server to a file - * - * This converts @c server into an INI format file. This allows created servers - * to be persisted to disk. This will replace any existing files with the same - * name. - * - * @param server Server to serialize - * @return False if the serialization of the server fails, true if it was successful - */ -static bool server_serialize(SERVER *server) +bool server_serialize(const SERVER *server) { bool rval = false; char filename[PATH_MAX]; diff --git a/server/modules/routing/debugcli/debugcmd.c b/server/modules/routing/debugcli/debugcmd.c index eff2e3bef..47dde9896 100644 --- a/server/modules/routing/debugcli/debugcmd.c +++ b/server/modules/routing/debugcli/debugcmd.c @@ -1032,9 +1032,9 @@ struct subcommand destroyoptions[] = } }; -static void alterServer(DCB *dcb, SERVER *server, char *key, char *value) +static bool handle_alter_server(SERVER *server, char *key, char *value) { - bool unknown = false; + bool valid = true; if (strcmp(key, "address") == 0) { @@ -1052,18 +1052,108 @@ static void alterServer(DCB *dcb, SERVER *server, char *key, char *value) { server_update_credentials(server, server->monuser, value); } - else if (server_is_ssl_parameter(key)) + else { - server_update_ssl(server, key, value); + valid = false; + } + + return valid; +} + +void handle_server_ssl(DCB *dcb, SERVER *server, CONFIG_CONTEXT *obj) +{ + if (config_have_required_ssl_params(obj)) + { + int err = 0; + SSL_LISTENER *ssl = make_ssl_structure(obj, true, &err); + + if (err == 0 && ssl && listener_init_SSL(ssl) == 0) + { + /** Sync to prevent reads on partially initialized server_ssl */ + atomic_synchronize(); + + server->server_ssl = ssl; + if (server_serialize(server)) + { + dcb_printf(dcb, "SSL enabled for server '%s'\n", server->unique_name); + } + else + { + dcb_printf(dcb, "SSL enabled for server '%s' but persisting " + "it to disk failed, see log for more details.\n", + server->unique_name); + } + } + else + { + dcb_printf(dcb, "Enabling SSL for server '%s' failed, see log " + "for more details.\n", server->unique_name); + } } else { - unknown = true; + dcb_printf(dcb, "Error: SSL configuration requires the following parameters:\n" + "ssl=required ssl_key=PATH ssl_cert=PATH ssl_ca_cert=PATH\n"); + } +} + +/** + * @brief Process multiple alter operations at once + * + * This is a somewhat ugly way to handle multiple key-value changes in one operation + * with one function. This could be handled with a variadic function but the + * required complexity would probably negate any benefits. + */ +static void alterServer(DCB *dcb, SERVER *server, char *v1, char *v2, char *v3, + char *v4, char *v5, char *v6, char *v7, char *v8, char *v9, + char *v10, char *v11) +{ + char *values[11] = {v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11}; + const int items = sizeof(values) / sizeof(values[0]); + CONFIG_CONTEXT *obj = NULL; + + for (int i = 0; i < items; i++) + { + if (values[i]) + { + char *key = values[i]; + char *value = strchr(key, '='); + + if (value) + { + *value++ = '\0'; + + if (config_is_ssl_parameter(key)) + { + /** + * All the required SSL parameters must be defined at once to + * enable SSL for created servers. This removes the problem + * of partial configuration and allows a somewhat atomic + * operation. + */ + if ((obj == NULL && (obj = config_context_create(server->unique_name)) == NULL) || + (!config_add_param(obj, key, value))) + { + dcb_printf(dcb, "Internal error, see log for more details\n"); + } + } + else if (!handle_alter_server(server, key, value)) + { + dcb_printf(dcb, "Error: Unknown key-value parameter: %s=%s\n", key, value); + } + } + else + { + dcb_printf(dcb, "Error: not a key-value parameter: %s\n", values[i]); + } + } } - if (unknown) + if (obj) { - dcb_printf(dcb, "Unknown parameter '%s'", key); + /** We have SSL parameters, try to process them */ + handle_server_ssl(dcb, server, obj); + config_context_free(obj); } } @@ -1147,12 +1237,15 @@ static void alterMonitor(DCB *dcb, MONITOR *monitor, char *key, char *value) struct subcommand alteroptions[] = { { - "server", 3, 3, alterServer, + "server", 2, 12, alterServer, "Alter server parameters", - "Usage: alter server NAME KEY VALUE\n" + "Usage: alter server NAME KEY=VALUE ...\n" "This will alter an existing parameter of a server. The accepted values\n" - "for KEY are: 'address', 'port', 'monuser', 'monpw'", - {ARG_TYPE_SERVER, ARG_TYPE_STRING, ARG_TYPE_STRING} + "for KEY are: 'address', 'port', 'monuser', 'monpw'\n" + "A maximum of 11 parameters can be changed at one time\n", + { ARG_TYPE_SERVER, ARG_TYPE_STRING, ARG_TYPE_STRING, ARG_TYPE_STRING, + ARG_TYPE_STRING, ARG_TYPE_STRING, ARG_TYPE_STRING, ARG_TYPE_STRING, + ARG_TYPE_STRING, ARG_TYPE_STRING, ARG_TYPE_STRING, ARG_TYPE_STRING} }, { "monitor", 3, 3, alterMonitor, From a651eff633d3923e8b2e49c9f076c4ab1dcd051c Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Sat, 12 Nov 2016 13:56:31 +0200 Subject: [PATCH 209/215] Serialize repurposed servers When a destroyed server is taken into use, it needs to be serialized. This will allow the server to be created on restart. --- server/core/server.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/server/core/server.c b/server/core/server.c index a51aa2521..729f8916a 100644 --- a/server/core/server.c +++ b/server/core/server.c @@ -1287,15 +1287,19 @@ bool server_create(const char *name, const char *address, const char *port, server->is_active = true; rval = true; } - else if ((server = server_alloc(name, address, atoi(port), protocol, authenticator, - authenticator_options))) + else { - if (server_serialize(server)) - { - /** server_alloc will add the server to the global list of - * servers so we don't need to manually add it. */ - rval = true; - } + /** + * server_alloc will add the server to the global list of + * servers so we don't need to manually add it. + */ + server = server_alloc(name, address, atoi(port), protocol, + authenticator, authenticator_options); + } + + if (server && server_serialize(server)) + { + rval = true; } } From a17aa28eedae9393ce3ba9a1b4fbe3f2fe026c51 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Sat, 12 Nov 2016 21:55:42 +0200 Subject: [PATCH 210/215] Persist changes to the list of monitored servers When a server is added to a monitor, an supplementary configuration file is generated to persist this information. This will allow dynamic modifications to server lists which will survive restarts and unexpected downtime. The monitor will only add new servers to its list of monitored servers. This prevents duplicate entries in the list and makes it safe to persist all used servers to the supplementary configuration file instead of only the ones that are not listed in the main configuration. --- include/maxscale/monitor.h | 16 ++ server/core/config.c | 66 ++++++++- server/core/monitor.c | 162 +++++++++++++++++---- server/modules/routing/debugcli/debugcmd.c | 2 + 4 files changed, 214 insertions(+), 32 deletions(-) diff --git a/include/maxscale/monitor.h b/include/maxscale/monitor.h index 55e795922..1db55f8c3 100644 --- a/include/maxscale/monitor.h +++ b/include/maxscale/monitor.h @@ -233,6 +233,22 @@ connect_result_t mon_connect_to_db(MONITOR* mon, MONITOR_SERVERS *database); void mon_log_connect_error(MONITOR_SERVERS* database, connect_result_t rval); void mon_log_state_change(MONITOR_SERVERS *ptr); +/** + * @brief Serialize a monitor to a file + * + * This partially converts @c monitor into an INI format file. Only the servers + * of the monitor are serialized. This allows the monitor to keep monitoring + * the servers that were added at runtime even after a restart. + * + * NOTE: This does not persist the complete monitor configuration and requires + * that an existing monitor configuration is in the main configuration file. + * Changes to monitor parameters are not persisted. + * + * @param monitor Monitor to serialize + * @return False if the serialization of the monitor fails, true if it was successful + */ +bool monitor_serialize_servers(const MONITOR *monitor); + /** * Check if a monitor uses @c servers * @param server Server that is queried diff --git a/server/core/config.c b/server/core/config.c index 6e8abb7a1..26c1c4784 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -561,6 +561,47 @@ static bool is_directory(const char *dir) return rval; } +/** + * @brief Check if a directory contains .cnf files + * + * @param path Path to a directory + * @return True if the directory contained one or more .cnf files + */ +static bool contains_cnf_files(const char *path) +{ + bool rval = false; + glob_t matches; + const char suffix[] = "/*.cnf"; + char pattern[strlen(path) + sizeof(suffix)]; + + strcpy(pattern, path); + strcat(pattern, suffix); + int rc = glob(pattern, GLOB_NOSORT, NULL, &matches); + + switch (rc) + { + case 0: + rval = true; + break; + + case GLOB_NOSPACE: + MXS_OOM(); + break; + + case GLOB_ABORTED: + MXS_ERROR("Failed to read directory '%s'", path); + break; + + default: + ss_dassert(rc == GLOB_NOMATCH); + break; + } + + globfree(&matches); + + return rval; +} + /** * @brief Load the specified configuration file for MaxScale * @@ -605,7 +646,23 @@ config_load_and_process(const char* filename, bool (*process_config)(CONFIG_CONT if (is_directory(persist_cnf)) { - rval = config_load_dir(persist_cnf, &dcontext, &ccontext); + DUPLICATE_CONTEXT p_dcontext; + /** + * We need to initialize a second duplicate context for the + * generated configuration files as the monitors and services will + * have duplicate sections. The duplicate sections are used to + * store changes to the list of servers the services and monitors + * use, and thus should not be treated as errors. + */ + if (duplicate_context_init(&p_dcontext)) + { + rval = config_load_dir(persist_cnf, &p_dcontext, &ccontext); + duplicate_context_finish(&p_dcontext); + } + else + { + rval = false; + } } if (rval) @@ -613,6 +670,13 @@ config_load_and_process(const char* filename, bool (*process_config)(CONFIG_CONT if (!check_config_objects(ccontext.next) || !process_config(ccontext.next)) { rval = false; + if (contains_cnf_files(persist_cnf)) + { + MXS_WARNING("One or more generated configurations were found at '%s'. " + "If the error relates to any of the files located there, " + "remove the offending configurations from this directory.", + persist_cnf); + } } } } diff --git a/server/core/monitor.c b/server/core/monitor.c index 94f930a8a..39c2c14f1 100644 --- a/server/core/monitor.c +++ b/server/core/monitor.c @@ -40,6 +40,7 @@ #include #include #include +#include /* * Create declarations of the enum for monitor events and also the array of @@ -244,46 +245,62 @@ monitorStopAll() void monitorAddServer(MONITOR *mon, SERVER *server) { - MONITOR_SERVERS *db = (MONITOR_SERVERS *)MXS_MALLOC(sizeof(MONITOR_SERVERS)); - MXS_ABORT_IF_NULL(db); - - db->server = server; - db->con = NULL; - db->next = NULL; - db->mon_err_count = 0; - db->log_version_err = true; - /** Server status is uninitialized */ - db->mon_prev_status = -1; - /* pending status is updated by get_replication_tree */ - db->pending_status = 0; - - monitor_state_t old_state = mon->state; - - if (old_state == MONITOR_STATE_RUNNING) - { - monitorStop(mon); - } - + bool new_server = true; spinlock_acquire(&mon->lock); - if (mon->databases == NULL) + for (MONITOR_SERVERS *db = mon->databases; db; db = db->next) { - mon->databases = db; - } - else - { - MONITOR_SERVERS *ptr = mon->databases; - while (ptr->next != NULL) + if (db->server == server) { - ptr = ptr->next; + new_server = false; } - ptr->next = db; } + spinlock_release(&mon->lock); - if (old_state == MONITOR_STATE_RUNNING) + if (new_server) { - monitorStart(mon, mon->parameters); + MONITOR_SERVERS *db = (MONITOR_SERVERS *)MXS_MALLOC(sizeof(MONITOR_SERVERS)); + MXS_ABORT_IF_NULL(db); + + db->server = server; + db->con = NULL; + db->next = NULL; + db->mon_err_count = 0; + db->log_version_err = true; + /** Server status is uninitialized */ + db->mon_prev_status = -1; + /* pending status is updated by get_replication_tree */ + db->pending_status = 0; + + monitor_state_t old_state = mon->state; + + if (old_state == MONITOR_STATE_RUNNING) + { + monitorStop(mon); + } + + spinlock_acquire(&mon->lock); + + if (mon->databases == NULL) + { + mon->databases = db; + } + else + { + MONITOR_SERVERS *ptr = mon->databases; + while (ptr->next != NULL) + { + ptr = ptr->next; + } + ptr->next = db; + } + spinlock_release(&mon->lock); + + if (old_state == MONITOR_STATE_RUNNING) + { + monitorStart(mon, mon->parameters); + } } } @@ -1152,3 +1169,86 @@ bool monitor_server_in_use(const SERVER *server) return rval; } + +/** + * Creates a monitor configuration at the location pointed by @c filename + * + * @param monitor Monitor to serialize into a configuration + * @param filename Filename where configuration is written + * @return True on success, false on error + */ +static bool create_monitor_config(const MONITOR *monitor, const char *filename) +{ + int file = open(filename, O_EXCL | O_CREAT | O_WRONLY, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); + + if (file == -1) + { + char errbuf[MXS_STRERROR_BUFLEN]; + MXS_ERROR("Failed to open file '%s' when serializing monitor '%s': %d, %s", + filename, monitor->name, errno, strerror_r(errno, errbuf, sizeof(errbuf))); + return false; + } + + /** + * Only additional parameters are added to the configuration. This prevents + * duplication or addition of parameters that don't support it. + * + * TODO: Check for return values on all of the dprintf calls + */ + dprintf(file, "[%s]\n", monitor->name); + + if (monitor->databases) + { + dprintf(file, "servers="); + for (MONITOR_SERVERS *db = monitor->databases; db; db = db->next) + { + if (db != monitor->databases) + { + dprintf(file, ","); + } + dprintf(file, "%s", db->server->unique_name); + } + dprintf(file, "\n"); + } + + close(file); + + return true; +} + +bool monitor_serialize_servers(const MONITOR *monitor) +{ + bool rval = false; + char filename[PATH_MAX]; + snprintf(filename, sizeof(filename), "%s/%s.cnf.tmp", get_config_persistdir(), + monitor->name); + + if (unlink(filename) == -1 && errno != ENOENT) + { + char err[MXS_STRERROR_BUFLEN]; + MXS_ERROR("Failed to remove temporary monitor configuration at '%s': %d, %s", + filename, errno, strerror_r(errno, err, sizeof(err))); + } + else if (create_monitor_config(monitor, filename)) + { + char final_filename[PATH_MAX]; + strcpy(final_filename, filename); + + char *dot = strrchr(final_filename, '.'); + ss_dassert(dot); + *dot = '\0'; + + if (rename(filename, final_filename) == 0) + { + rval = true; + } + else + { + char err[MXS_STRERROR_BUFLEN]; + MXS_ERROR("Failed to rename temporary monitor configuration at '%s': %d, %s", + filename, errno, strerror_r(errno, err, sizeof(err))); + } + } + + return rval; +} diff --git a/server/modules/routing/debugcli/debugcmd.c b/server/modules/routing/debugcli/debugcmd.c index 47dde9896..19e33470e 100644 --- a/server/modules/routing/debugcli/debugcmd.c +++ b/server/modules/routing/debugcli/debugcmd.c @@ -753,6 +753,7 @@ static void cmd_AddServer(DCB *dcb, void *a, void *b) else if (monitor) { monitorAddServer(monitor, server); + monitor_serialize_servers(monitor); } const char *target = service ? "service" : "monitor"; @@ -809,6 +810,7 @@ static void cmd_RemoveServer(DCB *dcb, void *a, void *b) else if (monitor) { monitorRemoveServer(monitor, server); + monitor_serialize_servers(monitor); } const char *target = service ? "service" : "monitor"; From 878d01e276aed790d6ac6b90263e6e7bddd2d93e Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Sat, 12 Nov 2016 22:49:27 +0200 Subject: [PATCH 211/215] Persist server changes to services When a service is added or removed from a service, a supplementary configuration file is created. This allows MaxScale to survive restars and unexpected downtime even if runtime changes to the servers of a service have been made. With these changes, it is possible to start MaxScale without any servers, create servers, add the created servers to services and monitors and restart Maxscale without losing the runtime configuration changes. --- include/maxscale/service.h | 18 ++- server/core/service.c | 148 ++++++++++++++++----- server/modules/routing/debugcli/debugcmd.c | 2 + 3 files changed, 136 insertions(+), 32 deletions(-) diff --git a/include/maxscale/service.h b/include/maxscale/service.h index e8fd7850e..95ac954c6 100644 --- a/include/maxscale/service.h +++ b/include/maxscale/service.h @@ -200,7 +200,7 @@ extern int serviceHasProtocol(SERVICE *service, const char *protocol, const char* address, unsigned short port); extern void serviceAddBackend(SERVICE *, SERVER *); extern void serviceRemoveBackend(SERVICE *, const SERVER *); -extern int serviceHasBackend(SERVICE *, SERVER *); +extern bool serviceHasBackend(SERVICE *, SERVER *); extern void serviceAddRouterOption(SERVICE *, char *); extern void serviceClearRouterOptions(SERVICE *); extern int serviceStart(SERVICE *); @@ -264,4 +264,20 @@ static inline uint64_t service_get_capabilities(const SERVICE *service) */ bool service_server_in_use(const SERVER *server); +/** + * @brief Serialize a service to a file + * + * This partially converts @c service into an INI format file. Only the servers + * of the service are serialized. This allows the service to keep using the servers + * added at runtime even after a restart. + * + * NOTE: This does not persist the complete service configuration and requires + * that an existing service configuration is in the main configuration file. + * Changes to service parameters are not persisted. + * + * @param service Service to serialize + * @return False if the serialization of the service fails, true if it was successful + */ +bool service_serialize_servers(const SERVICE *service); + MXS_END_DECLS diff --git a/server/core/service.c b/server/core/service.c index 7fe22d93e..f205338ed 100644 --- a/server/core/service.c +++ b/server/core/service.c @@ -43,6 +43,10 @@ #include #include #include +#include +#include +#include +#include #include #include #include @@ -56,12 +60,9 @@ #include #include #include -#include -#include #include #include #include -#include #include #include #include @@ -769,43 +770,46 @@ static SERVER_REF* server_ref_create(SERVER *server) void serviceAddBackend(SERVICE *service, SERVER *server) { - SERVER_REF *new_ref = server_ref_create(server); - - if (new_ref) + if (!serviceHasBackend(service, server)) { - spinlock_acquire(&service->spin); + SERVER_REF *new_ref = server_ref_create(server); - service->n_dbref++; - - if (service->dbref) + if (new_ref) { - SERVER_REF *ref = service->dbref; - SERVER_REF *prev = ref; + spinlock_acquire(&service->spin); - while (ref) + service->n_dbref++; + + if (service->dbref) { - if (ref->server == server) + SERVER_REF *ref = service->dbref; + SERVER_REF *prev = ref; + + while (ref) { - ref->active = true; - break; + if (ref->server == server) + { + ref->active = true; + break; + } + prev = ref; + ref = ref->next; } - prev = ref; - ref = ref->next; - } - if (ref == NULL) - { - /** A new server that hasn't been used by this service */ - atomic_synchronize(); - prev->next = new_ref; + if (ref == NULL) + { + /** A new server that hasn't been used by this service */ + atomic_synchronize(); + prev->next = new_ref; + } } + else + { + atomic_synchronize(); + service->dbref = new_ref; + } + spinlock_release(&service->spin); } - else - { - atomic_synchronize(); - service->dbref = new_ref; - } - spinlock_release(&service->spin); } } @@ -841,7 +845,7 @@ void serviceRemoveBackend(SERVICE *service, const SERVER *server) * @param server The server to add * @return Non-zero if the server is already part of the service */ -int +bool serviceHasBackend(SERVICE *service, SERVER *server) { SERVER_REF *ptr; @@ -2217,3 +2221,85 @@ bool service_server_in_use(const SERVER *server) return rval; } + +/** + * Creates a service configuration at the location pointed by @c filename + * + * @param service Service to serialize into a configuration + * @param filename Filename where configuration is written + * @return True on success, false on error + */ +static bool create_service_config(const SERVICE *service, const char *filename) +{ + int file = open(filename, O_EXCL | O_CREAT | O_WRONLY, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); + + if (file == -1) + { + char errbuf[MXS_STRERROR_BUFLEN]; + MXS_ERROR("Failed to open file '%s' when serializing service '%s': %d, %s", + filename, service->name, errno, strerror_r(errno, errbuf, sizeof(errbuf))); + return false; + } + + /** + * Only additional parameters are added to the configuration. This prevents + * duplication or addition of parameters that don't support it. + * + * TODO: Check for return values on all of the dprintf calls + */ + dprintf(file, "[%s]\n", service->name); + if (service->dbref) + { + dprintf(file, "servers="); + for (SERVER_REF *db = service->dbref; db; db = db->next) + { + if (db != service->dbref) + { + dprintf(file, ","); + } + dprintf(file, "%s", db->server->unique_name); + } + dprintf(file, "\n"); + } + + close(file); + + return true; +} + +bool service_serialize_servers(const SERVICE *service) +{ + bool rval = false; + char filename[PATH_MAX]; + snprintf(filename, sizeof(filename), "%s/%s.cnf.tmp", get_config_persistdir(), + service->name); + + if (unlink(filename) == -1 && errno != ENOENT) + { + char err[MXS_STRERROR_BUFLEN]; + MXS_ERROR("Failed to remove temporary service configuration at '%s': %d, %s", + filename, errno, strerror_r(errno, err, sizeof(err))); + } + else if (create_service_config(service, filename)) + { + char final_filename[PATH_MAX]; + strcpy(final_filename, filename); + + char *dot = strrchr(final_filename, '.'); + ss_dassert(dot); + *dot = '\0'; + + if (rename(filename, final_filename) == 0) + { + rval = true; + } + else + { + char err[MXS_STRERROR_BUFLEN]; + MXS_ERROR("Failed to rename temporary service configuration at '%s': %d, %s", + filename, errno, strerror_r(errno, err, sizeof(err))); + } + } + + return rval; +} diff --git a/server/modules/routing/debugcli/debugcmd.c b/server/modules/routing/debugcli/debugcmd.c index 19e33470e..ad568d625 100644 --- a/server/modules/routing/debugcli/debugcmd.c +++ b/server/modules/routing/debugcli/debugcmd.c @@ -749,6 +749,7 @@ static void cmd_AddServer(DCB *dcb, void *a, void *b) if (service) { serviceAddBackend(service, server); + service_serialize_servers(service); } else if (monitor) { @@ -806,6 +807,7 @@ static void cmd_RemoveServer(DCB *dcb, void *a, void *b) if (service) { serviceRemoveBackend(service, server); + service_serialize_servers(service); } else if (monitor) { From a5bb02bd14ac4d03db8a3640030b654d40907324 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Sat, 12 Nov 2016 23:49:26 +0200 Subject: [PATCH 212/215] Expect at least two arguments for `create server` When a server is created in server_create, it sets the port to the default of 3306 if no explicit port is defined. The code that called this function still expected a minimum of three arguments: name, address and port. --- server/modules/routing/debugcli/debugcmd.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/modules/routing/debugcli/debugcmd.c b/server/modules/routing/debugcli/debugcmd.c index ad568d625..bf92d5d43 100644 --- a/server/modules/routing/debugcli/debugcmd.c +++ b/server/modules/routing/debugcli/debugcmd.c @@ -984,9 +984,9 @@ static void createServer(DCB *dcb, char *name, char *address, char *port, struct subcommand createoptions[] = { { - "server", 3, 6, createServer, + "server", 2, 6, createServer, "Create a new server", - "Usage: create server NAME HOST PORT [PROTOCOL] [AUTHENTICATOR] [OPTIONS]\n" + "Usage: create server NAME HOST [PORT] [PROTOCOL] [AUTHENTICATOR] [OPTIONS]\n" "Create a new server from the following parameters.\n" "NAME Server name\n" "HOST Server host address\n" From f18cf407a796b48439d198f2cb01a5f08bc3b207 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Sun, 13 Nov 2016 00:18:13 +0200 Subject: [PATCH 213/215] Improve maxadmin error messages The error messages now report more detailed information about the expected number of arguments to commands. --- server/modules/routing/debugcli/debugcmd.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/server/modules/routing/debugcli/debugcmd.c b/server/modules/routing/debugcli/debugcmd.c index bf92d5d43..e076c7ee7 100644 --- a/server/modules/routing/debugcli/debugcmd.c +++ b/server/modules/routing/debugcli/debugcmd.c @@ -1553,12 +1553,29 @@ execute_cmd(CLI_SESSION *cli) if (strcasecmp(args[1], cmds[i].options[j].arg1) == 0) { found = 1; /**< command and sub-command match */ - if (argc < cmds[i].options[j].argc_min) + + if (cmds[i].options[j].argc_min == cmds[i].options[j].argc_max && + argc != cmds[i].options[j].argc_min) { + /** Wrong number of arguments */ + dcb_printf(dcb, "Incorrect number of arguments: %s %s expects %d arguments\n", + cmds[i].cmd, cmds[i].options[j].arg1, + cmds[i].options[j].argc_min); + } + else if (argc < cmds[i].options[j].argc_min) + { + /** Not enough arguments */ dcb_printf(dcb, "Incorrect number of arguments: %s %s expects at least %d arguments\n", cmds[i].cmd, cmds[i].options[j].arg1, cmds[i].options[j].argc_min); } + else if (argc > cmds[i].options[j].argc_max) + { + /** Too many arguments */ + dcb_printf(dcb, "Incorrect number of arguments: %s %s expects at most %d arguments\n", + cmds[i].cmd, cmds[i].options[j].arg1, + cmds[i].options[j].argc_max); + } else { unsigned long arg_list[MAXARGS] = {}; From 2202ec7a334fa1f6cf00ac28c80f42faa0e310c9 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Sun, 13 Nov 2016 10:51:09 +0200 Subject: [PATCH 214/215] Allow multiple monitor alterations at one time The monitor alteration now also uses a list of key-value pairs. This allows multiple changes to be made in one command. --- server/modules/routing/debugcli/debugcmd.c | 79 +++++++++++++++------- 1 file changed, 56 insertions(+), 23 deletions(-) diff --git a/server/modules/routing/debugcli/debugcmd.c b/server/modules/routing/debugcli/debugcmd.c index e076c7ee7..0b19e3bb3 100644 --- a/server/modules/routing/debugcli/debugcmd.c +++ b/server/modules/routing/debugcli/debugcmd.c @@ -1143,7 +1143,7 @@ static void alterServer(DCB *dcb, SERVER *server, char *v1, char *v2, char *v3, } else if (!handle_alter_server(server, key, value)) { - dcb_printf(dcb, "Error: Unknown key-value parameter: %s=%s\n", key, value); + dcb_printf(dcb, "Error: Bad key-value parameter: %s=%s\n", key, value); } } else @@ -1166,11 +1166,10 @@ static void alterServer(DCB *dcb, SERVER *server, char *v1, char *v2, char *v3, * * If the value is not a positive integer, an error is printed to @c dcb. * - * @param dcb Client DCB * @param value String value * @return 0 on error, otherwise a positive integer */ -static long get_positive_int(DCB *dcb, const char *value) +static long get_positive_int(const char *value) { char *endptr; long ival = strtol(value, &endptr, 10); @@ -1180,62 +1179,93 @@ static long get_positive_int(DCB *dcb, const char *value) return ival; } - dcb_printf(dcb, "Invalid value: %s", value); return 0; } -static void alterMonitor(DCB *dcb, MONITOR *monitor, char *key, char *value) +static bool handle_alter_monitor(MONITOR *monitor, char *key, char *value) { - bool unknown = false; + bool valid = false; + if (strcmp(key, "user") == 0) { + valid = true; monitorAddUser(monitor, value, monitor->password); } else if (strcmp(key, "password") == 0) { + valid = true; monitorAddUser(monitor, monitor->user, value); } else if (strcmp(key, "monitor_interval") == 0) { - long ival = get_positive_int(dcb, value); + long ival = get_positive_int(value); if (ival) { + valid = true; monitorSetInterval(monitor, ival); } } else if (strcmp(key, "backend_connect_timeout") == 0) { - long ival = get_positive_int(dcb, value); + long ival = get_positive_int(value); if (ival) { + valid = true; monitorSetNetworkTimeout(monitor, MONITOR_CONNECT_TIMEOUT, ival); } } else if (strcmp(key, "backend_write_timeout") == 0) { - long ival = get_positive_int(dcb, value); + long ival = get_positive_int(value); if (ival) { - monitorSetNetworkTimeout(monitor, MONITOR_READ_TIMEOUT, ival); + valid = true; + monitorSetNetworkTimeout(monitor, MONITOR_WRITE_TIMEOUT, ival); } } else if (strcmp(key, "backend_read_timeout") == 0) { - long ival = get_positive_int(dcb, value); + long ival = get_positive_int(value); if (ival) { - monitorSetNetworkTimeout(monitor, MONITOR_WRITE_TIMEOUT, ival); + valid = true; + monitorSetNetworkTimeout(monitor, MONITOR_READ_TIMEOUT, ival); } } - else + + return valid; +} + +static void alterMonitor(DCB *dcb, MONITOR *monitor, char *v1, char *v2, char *v3, + char *v4, char *v5, char *v6, char *v7, char *v8, char *v9, + char *v10, char *v11) +{ + char *values[11] = {v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11}; + const int items = sizeof(values) / sizeof(values[0]); + + for (int i = 0; i < items; i++) { - unknown = true; + if (values[i]) + { + char *key = values[i]; + char *value = strchr(key, '='); + + if (value) + { + *value++ = '\0'; + + if (!handle_alter_monitor(monitor, key, value)) + { + dcb_printf(dcb, "Error: Bad key-value parameter: %s=%s\n", key, value); + } + } + else + { + dcb_printf(dcb, "Error: not a key-value parameter: %s\n", values[i]); + } + } } - if (unknown) - { - dcb_printf(dcb, "Unknown parameter '%s'", key); - } } struct subcommand alteroptions[] = @@ -1246,19 +1276,22 @@ struct subcommand alteroptions[] = "Usage: alter server NAME KEY=VALUE ...\n" "This will alter an existing parameter of a server. The accepted values\n" "for KEY are: 'address', 'port', 'monuser', 'monpw'\n" - "A maximum of 11 parameters can be changed at one time\n", + "A maximum of 11 parameters can be changed at one time", { ARG_TYPE_SERVER, ARG_TYPE_STRING, ARG_TYPE_STRING, ARG_TYPE_STRING, ARG_TYPE_STRING, ARG_TYPE_STRING, ARG_TYPE_STRING, ARG_TYPE_STRING, ARG_TYPE_STRING, ARG_TYPE_STRING, ARG_TYPE_STRING, ARG_TYPE_STRING} }, { - "monitor", 3, 3, alterMonitor, + "monitor", 2, 12, alterMonitor, "Alter monitor parameters", - "Usage: alter monitor NAME KEY VALUE\n" + "Usage: alter monitor NAME KEY=VALUE ...\n" "This will alter an existing parameter of a monitor. The accepted values\n" "for KEY are: 'user', 'password', 'monitor_interval',\n" - "'backend_connect_timeout', 'backend_write_timeout', 'backend_read_timeout'", - {ARG_TYPE_MONITOR, ARG_TYPE_STRING, ARG_TYPE_STRING} + "'backend_connect_timeout', 'backend_write_timeout', 'backend_read_timeout'\n" + "A maximum of 11 parameters can be changed at one time", + {ARG_TYPE_MONITOR, ARG_TYPE_STRING, ARG_TYPE_STRING, ARG_TYPE_STRING, + ARG_TYPE_STRING, ARG_TYPE_STRING, ARG_TYPE_STRING, ARG_TYPE_STRING, + ARG_TYPE_STRING, ARG_TYPE_STRING, ARG_TYPE_STRING, ARG_TYPE_STRING} }, { EMPTY_OPTION From 548182afe39b7b9b584af977880a84e970106c9e Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Tue, 15 Nov 2016 10:50:06 +0200 Subject: [PATCH 215/215] Fix debug assertion in config.c The debug assertion was triggered due to a wrongly structured conditional statement. --- server/core/config.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/server/core/config.c b/server/core/config.c index 26c1c4784..ea812487f 100644 --- a/server/core/config.c +++ b/server/core/config.c @@ -387,10 +387,12 @@ ini_handler(void *userdata, const char *section, const char *name, const char *v cntxt->next = ptr; } - if (config_get_param(ptr->parameters, name) && - !config_append_param(ptr, name, value)) + if (config_get_param(ptr->parameters, name)) { - return 0; + if (!config_append_param(ptr, name, value)) + { + return 0; + } } else if (!config_add_param(ptr, name, value)) {