From a8118b98b44e2ff345f279f84bd5233748129574 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Wed, 21 Sep 2016 13:53:15 +0300 Subject: [PATCH 01/16] Further refine the tarball instructions --- .../Install-MariaDB-MaxScale-Using-a-Tarball.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Documentation/Getting-Started/Install-MariaDB-MaxScale-Using-a-Tarball.md b/Documentation/Getting-Started/Install-MariaDB-MaxScale-Using-a-Tarball.md index 25ec59a86..041f3f5b0 100644 --- a/Documentation/Getting-Started/Install-MariaDB-MaxScale-Using-a-Tarball.md +++ b/Documentation/Getting-Started/Install-MariaDB-MaxScale-Using-a-Tarball.md @@ -18,7 +18,7 @@ The required steps are as follows: $ cd maxscale $ sudo chown -R maxscale var -Creating the symbolic link is necessary, since MariaDB MaxScale has been built with with the assumption that its base-directory, that is, the directory under which all its sub-directories are found, is `/usr/local/maxscale`. +Creating the symbolic link is necessary, since MariaDB MaxScale has been built with with the assumption that the plugin directory is `/usr/local/maxscale/lib/maxscale`. The symbolic link also makes it easy to switch between different versions of MariaDB MaxScale that have been installed side by side in `/usr/local`; just make the symbolic link point to another installation. @@ -46,6 +46,12 @@ The `-d` flag causes maxscale _not_ to turn itself into a daemon, which is advis If you want to place the configuration file somewhere else but in `/etc` you can invoke MariaDB MaxScale with the `--config` flag, for instance, `--config=/usr/local/maxscale/etc/maxscale.cnf`. +Note also that if you want to keep _everything_ under `/usr/local/maxscale` you can invoke MariaDB MaxScale using the flag `--basedir`. + + $ sudo bin/maxscale --user=maxscale --basedir=/usr/local/maxscale -d + +That will cause MariaDB MaxScale to look for its configuration file in `/usr/local/maxscale/etc` and to store all runtime files under `/usr/local/maxscale/var`. + ## Installing in any Directory Enter a directory where you have the right to create a subdirectory. Then do as follows. From 97858c164d8de0a66b83ebcc65ce14d74841a703 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 22 Sep 2016 12:38:58 +0300 Subject: [PATCH 02/16] Update library dependencies Libcurl was not listed in the list of required packages. The tarball installation instruction did not list any of the required libraries. --- .../Getting-Started/Building-MaxScale-from-Source-Code.md | 1 + .../Install-MariaDB-MaxScale-Using-a-Tarball.md | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/Documentation/Getting-Started/Building-MaxScale-from-Source-Code.md b/Documentation/Getting-Started/Building-MaxScale-from-Source-Code.md index afd1da429..7b04dadca 100644 --- a/Documentation/Getting-Started/Building-MaxScale-from-Source-Code.md +++ b/Documentation/Getting-Started/Building-MaxScale-from-Source-Code.md @@ -6,6 +6,7 @@ requirements are as follows: * CMake version 2.8 or later (Packaging requires version 2.8.12 or later) * GCC version 4.4.7 or later * libaio +* libcurl * OpenSSL * Bison 2.7 or later * Flex 2.5.35 or later diff --git a/Documentation/Getting-Started/Install-MariaDB-MaxScale-Using-a-Tarball.md b/Documentation/Getting-Started/Install-MariaDB-MaxScale-Using-a-Tarball.md index 041f3f5b0..d8eeb4b3f 100644 --- a/Documentation/Getting-Started/Install-MariaDB-MaxScale-Using-a-Tarball.md +++ b/Documentation/Getting-Started/Install-MariaDB-MaxScale-Using-a-Tarball.md @@ -2,6 +2,12 @@ MariaDB MaxScale is also made available as a tarball, which is named like `maxscale-x.y.z.OS.tar.gz` where `x.y.z` is the same as the corresponding version and `OS` identifies the operating system, e.g. `maxscale-2.0.1.centos.7.tar.gz`. +In order to use the tarball, the following libraries are required: + +- libcurl +- libaio +- OpenSSL + The tarball has been built with the assumption that it will be installed in `/usr/local`. However, it is possible to install it in any directory, but in that case MariaDB MaxScale must be invoked with a flag. ## Installing as root in `/usr/local` From 049f823d3781260a565024aefd87dd68fa185bec Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Thu, 22 Sep 2016 13:50:57 +0300 Subject: [PATCH 03/16] Update tag information --- .../Release-Notes/MaxScale-2.0.1-Release-Notes.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Documentation/Release-Notes/MaxScale-2.0.1-Release-Notes.md b/Documentation/Release-Notes/MaxScale-2.0.1-Release-Notes.md index 509cffad5..06ae13832 100644 --- a/Documentation/Release-Notes/MaxScale-2.0.1-Release-Notes.md +++ b/Documentation/Release-Notes/MaxScale-2.0.1-Release-Notes.md @@ -139,9 +139,8 @@ Packages can be downloaded [here](https://mariadb.com/resources/downloads). ## Source Code -The source code of MaxScale is tagged at GitHub with a tag, which is identical -with the version of MaxScale. For instance, the tag of version X.Y.Z of MaxScale -is maxscale-X.Y.Z. Further, *master* always refers to the latest released -non-beta version. +The source code of MaxScale is tagged at GitHub with a tag, which is derived +from the version of MaxScale. For instance, the tag of version `X.Y.Z` of MaxScale +is `maxscale-X.Y.Z`. The source code is available [here](https://github.com/mariadb-corporation/MaxScale). From 542666ed16530e138a95f4270b97fb0b24420d11 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Fri, 23 Sep 2016 11:26:48 +0300 Subject: [PATCH 04/16] qc_sqlite: Remove leak Previously a query like e.g. delete t11.*, t12.* from t11,t12 where t11.a = t12.a; would cause a leak. The code for freeing that memory was present but commented out because it used to cause a crash. Now no crash appears, so it would seem that the crash was caused by something else that no longer is present. --- query_classifier/qc_sqlite/qc_sqlite.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/query_classifier/qc_sqlite/qc_sqlite.c b/query_classifier/qc_sqlite/qc_sqlite.c index 3ee36d6c1..b4251bdfb 100644 --- a/query_classifier/qc_sqlite/qc_sqlite.c +++ b/query_classifier/qc_sqlite/qc_sqlite.c @@ -1185,10 +1185,8 @@ void mxs_sqlite3DeleteFrom(Parse* pParse, SrcList* pTabList, Expr* pWhere, SrcLi update_affected_fields(info, 0, pWhere, QC_TOKEN_MIDDLE, 0); } - //TODO: Figure out why the following statements, causes a crash - //TODO: long down the road. - //TODO: exposed_sqlite3SrcListDelete(pParse->db, pTabList); - //TODO: exposed_sqlite3ExprDelete(pParse->db, pWhere); + exposed_sqlite3ExprDelete(pParse->db, pWhere); + exposed_sqlite3SrcListDelete(pParse->db, pTabList); exposed_sqlite3SrcListDelete(pParse->db, pUsing); } From e618370cdbd56a05dfc1da7c89b90022018d170c Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 22 Sep 2016 23:50:07 +0300 Subject: [PATCH 05/16] MXS-875: Fix regexfilter matching The return values of pcre2_match are now properly handled. A positive match is a return value which is greater than or equal to zero. This fix should give a small performance boost to as memory is no longer needlessly allocated. --- server/modules/filter/regexfilter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/modules/filter/regexfilter.c b/server/modules/filter/regexfilter.c index 65802d83b..4d12ee560 100644 --- a/server/modules/filter/regexfilter.c +++ b/server/modules/filter/regexfilter.c @@ -482,7 +482,7 @@ regex_replace(const char *sql, pcre2_code *re, pcre2_match_data *match_data, con size_t result_size; /** This should never fail with rc == 0 because we used pcre2_match_data_create_from_pattern() */ - if (pcre2_match(re, (PCRE2_SPTR) sql, PCRE2_ZERO_TERMINATED, 0, 0, match_data, NULL)) + if (pcre2_match(re, (PCRE2_SPTR) sql, PCRE2_ZERO_TERMINATED, 0, 0, match_data, NULL) > 0) { result_size = strlen(sql) + strlen(replace); result = malloc(result_size); From 649efb91b5c4a5c8ae83ff3eac4b3752f55f801b Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Fri, 23 Sep 2016 16:29:26 +0300 Subject: [PATCH 06/16] Fix broken upgrades on CentOS 7 The upgrade process removed the /var/lib/maxscale directory as the newer version didn't use it. This can be fixed by installing an empty directory into /var/lib/maxscale. --- CMakeLists.txt | 1 - cmake/package_rpm.cmake | 3 +++ query_classifier/qc_mysqlembedded/CMakeLists.txt | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b37d03670..c61ea96c6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -264,7 +264,6 @@ if(WITH_MAXSCALE_CNF) install(FILES server/maxscale_template.cnf DESTINATION ${MAXSCALE_CONFDIR} RENAME maxscale.cnf.template) endif() install(FILES server/maxscale_binlogserver_template.cnf DESTINATION ${MAXSCALE_SHAREDIR}) -install(PROGRAMS ${ERRMSG} DESTINATION ${MAXSCALE_VARDIR}/lib/maxscale) install(FILES ${CMAKE_SOURCE_DIR}/COPYRIGHT DESTINATION ${MAXSCALE_SHAREDIR}) install(FILES ${CMAKE_SOURCE_DIR}/README DESTINATION ${MAXSCALE_SHAREDIR}) install(FILES ${CMAKE_SOURCE_DIR}/LICENSE.TXT DESTINATION ${MAXSCALE_SHAREDIR}) diff --git a/cmake/package_rpm.cmake b/cmake/package_rpm.cmake index fabf2a650..4f6c84036 100644 --- a/cmake/package_rpm.cmake +++ b/cmake/package_rpm.cmake @@ -23,3 +23,6 @@ set(IGNORED_DIRS "%ignore ${CMAKE_INSTALL_PREFIX}/share/man/man1") set(CPACK_RPM_USER_FILELIST "${IGNORED_DIRS}") + +# Installing this prevents RPM from deleting the /var/lib/maxscale folder +install(DIRECTORY DESTINATION ${MAXSCALE_VARDIR}/lib/maxscale) diff --git a/query_classifier/qc_mysqlembedded/CMakeLists.txt b/query_classifier/qc_mysqlembedded/CMakeLists.txt index b8ee02e6f..73a091a1f 100644 --- a/query_classifier/qc_mysqlembedded/CMakeLists.txt +++ b/query_classifier/qc_mysqlembedded/CMakeLists.txt @@ -15,4 +15,5 @@ if (BUILD_QC_MYSQLEMBEDDED) set_target_properties(qc_mysqlembedded PROPERTIES LINK_FLAGS -Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/qc_mysqlembedded.map) #set_target_properties(qc_mysqlembedded PROPERTIES LINK_FLAGS -Wl,-z,defs) install(TARGETS qc_mysqlembedded COMPONENT lib DESTINATION ${MAXSCALE_LIBDIR}) + install(PROGRAMS ${ERRMSG} DESTINATION ${MAXSCALE_VARDIR}/lib/maxscale) endif() From dd65062531769ee4f00a211d089bf17ba3953b37 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Sat, 24 Sep 2016 10:27:29 +0300 Subject: [PATCH 07/16] Fix broken link lists and clean up tutorials Some of the tutorials didn't follow proper Markdown formatting rules. --- .../Tutorials/Administration-Tutorial.md | 14 +- ...era-Cluster-Connection-Routing-Tutorial.md | 217 ++++++++++-------- 2 files changed, 132 insertions(+), 99 deletions(-) diff --git a/Documentation/Tutorials/Administration-Tutorial.md b/Documentation/Tutorials/Administration-Tutorial.md index bb9c22e9e..e9f4e5929 100644 --- a/Documentation/Tutorials/Administration-Tutorial.md +++ b/Documentation/Tutorials/Administration-Tutorial.md @@ -6,13 +6,13 @@ Last updated 24th June 2015 The purpose of this tutorial is to introduce the MariaDB MaxScale Administrator to a few of the common administration tasks that need to be performed with MariaDB MaxScale. It is not intended as a reference to all the tasks that may be performed, more this is aimed as an introduction for administrators who are new to MariaDB MaxScale. -[Starting MariaDB MaxScale](#starting) -[Stopping MariaDB MaxScale](#stopping) -[Checking The Status Of The MariaDB MaxScale Services](#checking) -[Persistent Connections](#persistent) -[What Clients Are Connected To MariaDB MaxScale](#clients) -[Rotating the Log File](#rotating) -[Taking A Database Server Out Of Use](#outofuse) +- [Starting MariaDB MaxScale](#starting) +- [Stopping MariaDB MaxScale](#stopping) +- [Checking The Status Of The MariaDB MaxScale Services](#checking) +- [Persistent Connections](#persistent) +- [What Clients Are Connected To MariaDB MaxScale](#clients) +- [Rotating the Log File](#rotating) +- [Taking A Database Server Out Of Use](#outofuse) ### Starting MariaDB MaxScale diff --git a/Documentation/Tutorials/Galera-Cluster-Connection-Routing-Tutorial.md b/Documentation/Tutorials/Galera-Cluster-Connection-Routing-Tutorial.md index b37e61008..06a15d2e7 100644 --- a/Documentation/Tutorials/Galera-Cluster-Connection-Routing-Tutorial.md +++ b/Documentation/Tutorials/Galera-Cluster-Connection-Routing-Tutorial.md @@ -16,152 +16,185 @@ MariaDB MaxScale configuration is held in an ini file that is located in the fil A global, maxscale, section is included within every MariaDB MaxScale configuration file; this is used to set the values of various MariaDB MaxScale wide parameters, perhaps the most important of these is the number of threads that MariaDB MaxScale will use to execute the code that forwards requests and handles responses for clients. - [maxscale] - threads=4 +``` +[maxscale] +threads=4 +``` Since we are using Galera Cluster and connection routing we want a single to which the client application can connect; MariaDB MaxScale will then route connections to this port onwards to the various nodes within the Galera Cluster. To achieve this within MariaDB MaxScale we need to define a service in the ini file. Create a section for each in your MariaDB MaxScale configuration file and set the type to service, the section name is the names of the service and should be meaningful to the administrator. Names may contain whitespace. - [Galera Service] - type=service +``` +[Galera Service] +type=service +``` The router for this section the readconnroute module, also the service should be provided with the list of servers that will be part of the cluster. The server names given here are actually the names of server sections in the configuration file and not the physical hostnames or addresses of the servers. - [Galera Service] - type=service - router=readconnroute - servers=dbserv1, dbserv2, dbserv3 +``` +[Galera Service] +type=service +router=readconnroute +servers=dbserv1, dbserv2, dbserv3 +``` In order to instruct the router to which servers it should route we must add router options to the service. The router options are compared to the status that the monitor collects from the servers and used to restrict the eligible set of servers to which that service may route. In our case we use the option that restricts us to servers that are fully functional members of the Galera cluster which are able to support SQL operations on the cluster. To achieve this we use the router option synced. - [Galera Service] - type=service - router=readconnroute - router_options=synced - servers=dbserv1, dbserv2, dbserv3 +``` +[Galera Service] +type=service +router=readconnroute +router_options=synced +servers=dbserv1, dbserv2, dbserv3 +``` The final step in the service section is to add the username and password that will be used to populate the user data from the database cluster. There are two options for representing the password, either plain text or encrypted passwords may be used. In order to use encrypted passwords a set of keys must be generated that will be used by the encryption and decryption process. To generate the keys use the maxkeys command and pass the name of the secrets file in which the keys are stored. - % maxkeys /var/lib/maxscale/.secrets - % +``` +% maxkeys /var/lib/maxscale/.secrets +% +``` Once the keys have been created the maxpasswd command can be used to generate the encrypted password. - % maxpasswd plainpassword - 96F99AA1315BDC3604B006F427DD9484 - % +``` +% maxpasswd plainpassword +96F99AA1315BDC3604B006F427DD9484 +% +``` The username and password, either encrypted or plain text, are stored in the service section using the user and passwd parameters. - [Galera Service] - type=service - router=readconnroute - router_options=synced - servers=dbserv1, dbserv2, dbserv3 - user=maxscale - passwd=96F99AA1315BDC3604B006F427DD9484 +``` +[Galera Service] +type=service +router=readconnroute +router_options=synced +servers=dbserv1, dbserv2, dbserv3 +user=maxscale +passwd=96F99AA1315BDC3604B006F427DD9484 +``` This completes the definitions required by the service, however listening ports must be associated with a service in order to allow network connections. This is done by creating a series of listener sections. These sections again are named for the convenience of the administrator and should be of type listener with an entry labeled service which contains the name of the service to associate the listener with. Each service may have multiple listeners. - [Galera Listener] - type=listener - service=Galera Service +``` +[Galera Listener] +type=listener +service=Galera Service +``` A listener must also define the protocol module it will use for the incoming network protocol, currently this should be the MySQLClient protocol for all database listeners. The listener may then supply a network port to listen on and/or a socket within the file system. - [Galera Listener] - type=listener - service=Galera Service - protocol=MySQLClient - port=4306 - socket=/tmp/DB.Cluster +``` +[Galera Listener] +type=listener +service=Galera Service +protocol=MySQLClient +port=4306 +socket=/tmp/DB.Cluster +``` An address parameter may be given if the listener is required to bind to a particular network address when using hosts with multiple network addresses. The default behavior is to listen on all network interfaces. The next stage is the configuration is to define the server information. This defines how to connect to each of the servers within the cluster, again a section is created for each server, with the type set to server, the network address and port to connect to and the protocol to use to connect to the server. Currently the protocol for all database connections in MySQLBackend. - [dbserv1] - type=server - address=192.168.2.1 - port=3306 - protocol=MySQLBackend - [dbserv2] - type=server - address=192.168.2.2 - port=3306 - protocol=MySQLBackend - [dbserv3] - type=server - address=192.168.2.3 - port=3306 - protocol=MySQLBackend +``` +[dbserv1] +type=server +address=192.168.2.1 +port=3306 +protocol=MySQLBackend + +[dbserv2] +type=server +address=192.168.2.2 +port=3306 +protocol=MySQLBackend + +[dbserv3] +type=server +address=192.168.2.3 +port=3306 +protocol=MySQLBackend +``` In order for MariaDB MaxScale to monitor the servers using the correct monitoring mechanisms a section should be provided that defines the monitor to use and the servers to monitor. Once again a section is created with a symbolic name for the monitor, with the type set to monitor. Parameters are added for the module to use, the list of servers to monitor and the username and password to use when connecting to the the servers with the monitor. - [Galera Monitor] - type=monitor - module=galeramon - servers=dbserv1, dbserv2, dbserv3 - user=maxscale - passwd=96F99AA1315BDC3604B006F427DD9484 +``` +[Galera Monitor] +type=monitor +module=galeramon +servers=dbserv1, dbserv2, dbserv3 +user=maxscale +passwd=96F99AA1315BDC3604B006F427DD9484 +``` As with the password definition in the server either plain text or encrypted passwords may be used. The final stage in the configuration is to add the option service which is used by the maxadmin command to connect to MariaDB MaxScale for monitoring and administration purposes. This creates a service section and a listener section. - [CLI] - type=service - router=cli - [CLI Listener] - type=listener - service=CLI - protocol=maxscaled - socket=default +``` +[CLI] +type=service +router=cli +[CLI Listener] +type=listener +service=CLI +protocol=maxscaled +socket=default +``` ## Starting MariaDB MaxScale Upon completion of the configuration process MariaDB MaxScale is ready to be started for the first time. This may either be done manually by running the maxscale command or via the service interface. - % maxscale +``` +% maxscale +``` or - % service maxscale start +``` +% service maxscale start +``` Check the error log in /var/log/maxscale to see if any errors are detected in the configuration file and to confirm MariaDB MaxScale has been started. Also the maxadmin command may be used to confirm that MariaDB MaxScale is running and the services, listeners etc have been correctly configured. - % maxadmin list services +``` +% maxadmin list services - Services. - --------------------------+----------------------+--------+--------------- - Service Name | Router Module | #Users | Total Sessions - --------------------------+----------------------+--------+--------------- - Galera Service | readconnroute | 1 | 1 - CLI | cli | 2 | 2 - --------------------------+----------------------+--------+--------------- - % maxadmin list servers - Servers. - -------------------+-----------------+-------+-------------+------------------- - Server | Address | Port | Connections | Status - -------------------+-----------------+-------+-------------+-------------------- - dbserv1 | 192.168.2.1 | 3306 | 0 | Running, Synced, Master - dbserv2 | 192.168.2.2 | 3306 | 0 | Running, Synced, Slave - dbserv3 | 192.168.2.3 | 3306 | 0 | Running, Synced, Slave - -------------------+-----------------+-------+-------------+-------------------- +Services. +--------------------------+----------------------+--------+--------------- +Service Name | Router Module | #Users | Total Sessions +--------------------------+----------------------+--------+--------------- +Galera Service | readconnroute | 1 | 1 +CLI | cli | 2 | 2 +--------------------------+----------------------+--------+--------------- +% maxadmin list servers +Servers. +-------------------+-----------------+-------+-------------+------------------- +Server | Address | Port | Connections | Status +-------------------+-----------------+-------+-------------+-------------------- +dbserv1 | 192.168.2.1 | 3306 | 0 | Running, Synced, Master +dbserv2 | 192.168.2.2 | 3306 | 0 | Running, Synced, Slave +dbserv3 | 192.168.2.3 | 3306 | 0 | Running, Synced, Slave +-------------------+-----------------+-------+-------------+-------------------- +``` A Galera Cluster is a multi-master clustering technology, however the monitor is able to impose false notions of master and slave roles within a Galera Cluster in order to facilitate the use of Galera as if it were a standard MySQL Replication setup. This is merely an internal MariaDB MaxScale convenience and has no impact on the behavior of the cluster. You can control which Galera node is the master server by using the _priority_ mechanism of the Galera Monitor module. For more details, read the [Galera Monitor](../Monitors/Galera-Monitor.md) documentation. - % maxadmin list listeners +``` +% maxadmin list listeners - Listeners. - ---------------------+--------------------+-----------------+-------+-------- - Service Name | Protocol Module | Address | Port | State - ---------------------+--------------------+-----------------+-------+-------- - Galera Service | MySQLClient | * | 4306 | Running - CLI | maxscaled | localhost | 6603 | Running - ---------------------+--------------------+-----------------+-------+-------- - % +Listeners. +---------------------+--------------------+-----------------+-------+-------- +Service Name | Protocol Module | Address | Port | State +---------------------+--------------------+-----------------+-------+-------- +Galera Service | MySQLClient | * | 4306 | Running +CLI | maxscaled | localhost | 6603 | Running +---------------------+--------------------+-----------------+-------+-------- +% +``` MariaDB MaxScale is now ready to start accepting client connections and routing them to the master or slaves within your cluster. Other configuration options are available that can alter the criteria used for routing, such as using weights to obtain unequal balancing operations. These options may be found in the MariaDB MaxScale Configuration Guide. More detail on the use of maxadmin can be found in the document ["MaxAdmin - The MariaDB MaxScale Administration & Monitoring Client Application"](../Reference/MaxAdmin.md). - From 997fe6b90bba462eea81315afbfca376289954c2 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Sat, 24 Sep 2016 10:14:59 +0300 Subject: [PATCH 08/16] Add a fail-safe for active operation counters If a backend is not in use but it is waiting for a result, the state should anyways be cleared to keep the operation counters correct. --- server/modules/routing/readwritesplit/readwritesplit.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index ac5a8781d..0d026b800 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -958,6 +958,15 @@ static void closeSession(ROUTER *instance, void *router_session) else { ss_dassert(!BREF_IS_WAITING_RESULT(bref)); + + /** This should never be true unless a backend reference is taken + * out of use before clearing the BREF_WAITING_RESULT state */ + if (BREF_IS_WAITING_RESULT(bref)) + { + MXS_WARNING("A closed backend was expecting a result, this should not be possible. " + "Decrementing active operation counter for this backend."); + bref_clear_state(bref, BREF_WAITING_RESULT); + } } } /** Unlock */ From 8fc6f52eea42c89ebfc5b25dffef9c8a0d25d2ea Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Mon, 26 Sep 2016 18:12:36 +0300 Subject: [PATCH 09/16] Update ChangeLog --- Documentation/Changelog.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Documentation/Changelog.md b/Documentation/Changelog.md index 026e7dfb6..264a2038a 100644 --- a/Documentation/Changelog.md +++ b/Documentation/Changelog.md @@ -5,14 +5,13 @@ * SSL can be used in the communication between MariaDB MaxScale and the backend servers. * The number of allowed connections can explicitly be throttled. * MariaDB MaxScale can continue serving read request even if the master has gone down. -* The security of MaxAdmin has been improved; you can only connect from the - same host MariaDB MaxScale is running on, and the Linux identity is used for - authorization. +* The security of MaxAdmin has been improved; Unix domain sockets can be used in the + communication with MariaDB MaxScale and the Linux identity can be used for authorization. * MariaDB MaxScale can in real time make binlog events available as raw AVRO or as JSON objects (beta level functionality). For more details, please refer to: -* [MariaDB MaxScale 2.0.0 Release Notes](Release-Notes/MaxScale-2.0.0-Release-Notes.md) +* [MariaDB MaxScale 2.0.1 Release Notes](Release-Notes/MaxScale-2.0.1-Release-Notes.md) ## MariaDB MaxScale 1.4 * Authentication now allows table level resolution of grants. MaxScale service From 8f233d32cfd55f041fa64ee8a99e255fd537c863 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Wed, 28 Sep 2016 09:04:42 +0300 Subject: [PATCH 10/16] Always update state of bref behind lock --- .../routing/readwritesplit/readwritesplit.c | 55 +++++++++++++++---- 1 file changed, 45 insertions(+), 10 deletions(-) diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index 0d026b800..3b3cf17d1 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -4276,6 +4276,8 @@ static void handleError(ROUTER *instance, void *router_session, } session = problem_dcb->session; + bool close_dcb = true; + if (session == NULL || rses == NULL) { *succp = false; @@ -4295,6 +4297,9 @@ static void handleError(ROUTER *instance, void *router_session, { if (!rses_begin_locked_router_action(rses)) { + close_dcb = false; /* With the assumption that if the router session is closed, + * then so is the dcb. + */ *succp = false; break; } @@ -4363,6 +4368,9 @@ static void handleError(ROUTER *instance, void *router_session, */ *succp = handle_error_new_connection(inst, &rses, problem_dcb, errmsgbuf); } + + dcb_close(problem_dcb); + close_dcb = false; /* Free the lock if rses still exists */ if (rses) { @@ -4374,16 +4382,22 @@ static void handleError(ROUTER *instance, void *router_session, case ERRACT_REPLY_CLIENT: { handle_error_reply_client(session, rses, problem_dcb, errmsgbuf); + close_dcb = false; *succp = false; /*< no new backend servers were made available */ break; } default: + ss_dassert(!true); *succp = false; break; } } - dcb_close(problem_dcb); + + if (close_dcb) + { + dcb_close(problem_dcb); + } } static void handle_error_reply_client(SESSION *ses, ROUTER_CLIENT_SES *rses, @@ -4398,18 +4412,39 @@ static void handle_error_reply_client(SESSION *ses, ROUTER_CLIENT_SES *rses, client_dcb = ses->client_dcb; spinlock_release(&ses->ses_lock); - /** - * If bref exists, mark it closed - */ - if ((bref = get_bref_from_dcb(rses, backend_dcb)) != NULL) + if (rses_begin_locked_router_action(rses)) { - CHK_BACKEND_REF(bref); - bref_clear_state(bref, BREF_IN_USE); - bref_set_state(bref, BREF_CLOSED); - if (BREF_IS_WAITING_RESULT(bref)) + /** + * If bref exists, mark it closed + */ + if ((bref = get_bref_from_dcb(rses, backend_dcb)) != NULL) { - bref_clear_state(bref, BREF_WAITING_RESULT); + CHK_BACKEND_REF(bref); + + if (BREF_IS_IN_USE(bref)) + { + bref_clear_state(bref, BREF_IN_USE); + bref_set_state(bref, BREF_CLOSED); + if (BREF_IS_WAITING_RESULT(bref)) + { + bref_clear_state(bref, BREF_WAITING_RESULT); + } + + dcb_close(backend_dcb); + } } + else + { + // All dcbs should be associated with a backend reference. + ss_dassert(!true); + } + + rses_end_locked_router_action(rses); + } + else + { + // The session has already been closed, hence the dcb has been + // closed as well. } if (sesstate == SESSION_STATE_ROUTER_READY) From 4df543157280b80a2ec4e2b4c4e647055ae946a6 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Wed, 28 Sep 2016 12:20:14 +0300 Subject: [PATCH 11/16] qc_sqlite: Reduce logging when query cannot be parsed If a query could not be parsed and details about the query are asked, we log an info level message instead of an error. --- query_classifier/qc_sqlite/qc_sqlite.c | 65 +++++++++++++++++++------- 1 file changed, 47 insertions(+), 18 deletions(-) diff --git a/query_classifier/qc_sqlite/qc_sqlite.c b/query_classifier/qc_sqlite/qc_sqlite.c index b4251bdfb..9c7b086ce 100644 --- a/query_classifier/qc_sqlite/qc_sqlite.c +++ b/query_classifier/qc_sqlite/qc_sqlite.c @@ -21,6 +21,7 @@ #include #include #include +#include #include "builtin_functions.h" //#define QC_TRACE_ENABLED @@ -188,6 +189,7 @@ static void info_finish(QC_SQLITE_INFO* info); static void info_free(QC_SQLITE_INFO* info); static QC_SQLITE_INFO* info_init(QC_SQLITE_INFO* info); static bool is_submitted_query(const QC_SQLITE_INFO* info, const Parse* pParse); +static void log_invalid_data(GWBUF* query, const char* message); static bool parse_query(GWBUF* query); static void parse_query_string(const char* query, size_t len); static bool query_is_parsed(GWBUF* query); @@ -596,6 +598,33 @@ static bool is_submitted_query(const QC_SQLITE_INFO* info, const Parse* pParse) return rv; } +/** + * Logs information about invalid data. + * + * @param query The query that could not be parsed. + * @param message What is being asked for. + */ +static void log_invalid_data(GWBUF* query, const char* message) +{ + // At this point the query should be contiguous, but better safe than sorry. + + if (GWBUF_LENGTH(query) >= MYSQL_HEADER_LEN + 1) + { + char *sql; + int length; + + if (modutil_extract_SQL(query, &sql, &length)) + { + if (length > GWBUF_LENGTH(query) - MYSQL_HEADER_LEN - 1) + { + length = GWBUF_LENGTH(query) - MYSQL_HEADER_LEN - 1; + } + + MXS_INFO("qc_sqlite: Parsing the query failed, %s: %*s", message, length, sql); + } + } +} + static void append_affected_field(QC_SQLITE_INFO* info, const char* s) { size_t len = strlen(s); @@ -2549,9 +2578,9 @@ static uint32_t qc_sqlite_get_type(GWBUF* query) { types = info->types; } - else + else if (MXS_LOG_PRIORITY_IS_ENABLED(LOG_INFO)) { - MXS_ERROR("qc_sqlite: The query operation was not resolved. Response not valid."); + log_invalid_data(query, "cannot report query type"); } } else @@ -2577,9 +2606,9 @@ static qc_query_op_t qc_sqlite_get_operation(GWBUF* query) { op = info->operation; } - else + else if (MXS_LOG_PRIORITY_IS_ENABLED(LOG_INFO)) { - MXS_ERROR("qc_sqlite: The query operation was not resolved. Response not valid."); + log_invalid_data(query, "cannot report query operation"); } } else @@ -2608,9 +2637,9 @@ static char* qc_sqlite_get_created_table_name(GWBUF* query) created_table_name = mxs_strdup(info->created_table_name); } } - else + else if (MXS_LOG_PRIORITY_IS_ENABLED(LOG_INFO)) { - MXS_ERROR("qc_sqlite: The query operation was not resolved. Response not valid."); + log_invalid_data(query, "cannot report created tables"); } } else @@ -2636,9 +2665,9 @@ static bool qc_sqlite_is_drop_table_query(GWBUF* query) { is_drop_table = info->is_drop_table; } - else + else if (MXS_LOG_PRIORITY_IS_ENABLED(LOG_INFO)) { - MXS_ERROR("qc_sqlite: The query operation was not resolved. Response not valid."); + log_invalid_data(query, "cannot report whether query is drop table"); } } else @@ -2664,9 +2693,9 @@ static bool qc_sqlite_is_real_query(GWBUF* query) { is_real_query = info->is_real_query; } - else + else if (MXS_LOG_PRIORITY_IS_ENABLED(LOG_INFO)) { - MXS_ERROR("qc_sqlite: The query operation was not resolved. Response not valid."); + log_invalid_data(query, "cannot report whether query is a real query"); } } else @@ -2708,9 +2737,9 @@ static char** qc_sqlite_get_table_names(GWBUF* query, int* tblsize, bool fullnam *tblsize = 0; } } - else + else if (MXS_LOG_PRIORITY_IS_ENABLED(LOG_INFO)) { - MXS_ERROR("qc_sqlite: The query operation was not resolved. Response not valid."); + log_invalid_data(query, "cannot report what tables are accessed"); } } else @@ -2747,9 +2776,9 @@ static bool qc_sqlite_query_has_clause(GWBUF* query) { has_clause = info->has_clause; } - else + else if (MXS_LOG_PRIORITY_IS_ENABLED(LOG_INFO)) { - MXS_ERROR("qc_sqlite: The query operation was not resolved. Response not valid."); + log_invalid_data(query, "cannot report whether the query has a where clause"); } } else @@ -2775,9 +2804,9 @@ static char* qc_sqlite_get_affected_fields(GWBUF* query) { affected_fields = info->affected_fields; } - else + else if (MXS_LOG_PRIORITY_IS_ENABLED(LOG_INFO)) { - MXS_ERROR("qc_sqlite: The query operation was not resolved. Response not valid."); + log_invalid_data(query, "cannot report what fields are affected"); } } else @@ -2811,9 +2840,9 @@ static char** qc_sqlite_get_database_names(GWBUF* query, int* sizep) database_names = copy_string_array(info->database_names, sizep); } } - else + else if (MXS_LOG_PRIORITY_IS_ENABLED(LOG_INFO)) { - MXS_ERROR("qc_sqlite: The query operation was not resolved. Response not valid."); + log_invalid_data(query, "cannot report what databases are accessed"); } } else From 3d5cfee348e8556b222516bf57a3031788e7bac7 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Thu, 29 Sep 2016 09:34:35 +0300 Subject: [PATCH 12/16] housekeeper: Copy data to prevent access of freed data --- server/core/housekeeper.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/server/core/housekeeper.c b/server/core/housekeeper.c index 1dff3ad7b..c9294eab7 100644 --- a/server/core/housekeeper.c +++ b/server/core/housekeeper.c @@ -275,11 +275,16 @@ hkthread(void *data) ptr->nextdue = now + ptr->frequency; taskfn = ptr->task; taskdata = ptr->data; + // We need to copy type and name, in case hktask_remove is called from + // the callback. Otherwise we will access freed data. + HKTASK_TYPE type = ptr->type; + char name[strlen(ptr->name) + 1]; + strcpy(name, ptr->name); spinlock_release(&tasklock); (*taskfn)(taskdata); - if (ptr->type == HK_ONESHOT) + if (type == HK_ONESHOT) { - hktask_remove(ptr->name); + hktask_remove(name); } spinlock_acquire(&tasklock); ptr = tasks; From dcf55d409934d19e29532609e9647f0a3cd5bb23 Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 29 Sep 2016 14:57:44 +0300 Subject: [PATCH 13/16] Fix possible out of bounds read in CDCPlainAuth When the authentication string was decoded from hexadecimal to binary, it was possible that an out of bounds read was done if the length of the data was not an even number. --- server/modules/authenticator/cdc_plain_auth.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/modules/authenticator/cdc_plain_auth.c b/server/modules/authenticator/cdc_plain_auth.c index 0c605f675..e50dcb482 100644 --- a/server/modules/authenticator/cdc_plain_auth.c +++ b/server/modules/authenticator/cdc_plain_auth.c @@ -285,6 +285,12 @@ cdc_auth_set_client_data(CDC_session *client_data, uint8_t *client_auth_packet, int client_auth_packet_size) { + if (client_auth_packet_size % 2 != 0) + { + /** gw_hex2bin expects an even number of bytes */ + client_auth_packet_size--; + } + int rval = CDC_STATE_AUTH_ERR; int decoded_size = client_auth_packet_size / 2; char decoded_buffer[decoded_size + 1]; // Extra for terminating null From 4658a289656cbd1b881776d990d17e77df334eae Mon Sep 17 00:00:00 2001 From: Markus Makela Date: Thu, 29 Sep 2016 15:28:57 +0300 Subject: [PATCH 14/16] Fix out of bounds read in avro_client_process_command When the last transaction was queried, it caused an out of bounds read when strstr was used on the raw data of a GWBUF. --- server/modules/routing/avro/avro_client.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/modules/routing/avro/avro_client.c b/server/modules/routing/avro/avro_client.c index 0d4a870d4..6ab00117a 100644 --- a/server/modules/routing/avro/avro_client.c +++ b/server/modules/routing/avro/avro_client.c @@ -448,7 +448,10 @@ avro_client_process_command(AVRO_INSTANCE *router, AVRO_CLIENT *client, GWBUF *q const char req_last_gtid[] = "QUERY-LAST-TRANSACTION"; const char req_gtid[] = "QUERY-TRANSACTION"; const size_t req_data_len = sizeof(req_data) - 1; - uint8_t *data = GWBUF_DATA(queue); + size_t buflen = gwbuf_length(queue); + uint8_t data[buflen + 1]; + gwbuf_copy_data(queue, 0, buflen, data); + data[buflen] = '\0'; char *command_ptr = strstr((char *)data, req_data); if (command_ptr != NULL) From ca76cb15760d7edd0cc12cd7b31a7381c9c597a9 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Thu, 29 Sep 2016 22:50:18 +0300 Subject: [PATCH 15/16] qc_sqlite: Recognize backslash as escape character Without this change, e.g. insert into t1 values('\''); causes a buffer overflow and crash. --- query_classifier/qc_sqlite/sqlite-src-3110100/src/util.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/query_classifier/qc_sqlite/sqlite-src-3110100/src/util.c b/query_classifier/qc_sqlite/sqlite-src-3110100/src/util.c index 37b585b2e..94b10b481 100644 --- a/query_classifier/qc_sqlite/sqlite-src-3110100/src/util.c +++ b/query_classifier/qc_sqlite/sqlite-src-3110100/src/util.c @@ -219,6 +219,15 @@ int sqlite3Dequote(char *z){ } for(i=1, j=0;; i++){ assert( z[i] ); +#ifdef MAXSCALE + if ( z[i]=='\\' ){ + z[j++] = '\\'; + if ( z[i+1]==quote || z[i+1]=='\\' ){ + z[j++] = quote; + i++; + } + } else +#endif if( z[i]==quote ){ if( z[i+1]==quote ){ z[j++] = quote; From fa2a66719554d13a00db5c81c5c9ffd5b3a2ce14 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Thu, 29 Sep 2016 23:48:21 +0300 Subject: [PATCH 16/16] qc_sqlite: Handle a name like ```a`. --- query_classifier/qc_sqlite/qc_sqlite.c | 1 + query_classifier/qc_sqlite/sqlite-src-3110100/src/util.c | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/query_classifier/qc_sqlite/qc_sqlite.c b/query_classifier/qc_sqlite/qc_sqlite.c index 9c7b086ce..6253b2a33 100644 --- a/query_classifier/qc_sqlite/qc_sqlite.c +++ b/query_classifier/qc_sqlite/qc_sqlite.c @@ -919,6 +919,7 @@ static void update_database_names(QC_SQLITE_INFO* info, const char* zDatabase) static void update_names(QC_SQLITE_INFO* info, const char* zDatabase, const char* zTable) { char* zCopy = mxs_strdup(zTable); + // TODO: Is this call really needed. Check also sqlite3Dequote. exposed_sqlite3Dequote(zCopy); enlarge_string_array(1, info->table_names_len, &info->table_names, &info->table_names_capacity); diff --git a/query_classifier/qc_sqlite/sqlite-src-3110100/src/util.c b/query_classifier/qc_sqlite/sqlite-src-3110100/src/util.c index 94b10b481..2030e0e0c 100644 --- a/query_classifier/qc_sqlite/sqlite-src-3110100/src/util.c +++ b/query_classifier/qc_sqlite/sqlite-src-3110100/src/util.c @@ -220,7 +220,13 @@ int sqlite3Dequote(char *z){ for(i=1, j=0;; i++){ assert( z[i] ); #ifdef MAXSCALE - if ( z[i]=='\\' ){ + if ( z[i]==0 ){ + // TODO: This is needed only because exposed_sqlite3Dequote() is called + // TODO: in qc_sqlite.c:update_names(). That call probably is not needed + // TODO: and should be removed, in which case this check could also be + // TODO: removed. + break; + }else if ( z[i]=='\\' ){ z[j++] = '\\'; if ( z[i+1]==quote || z[i+1]=='\\' ){ z[j++] = quote;