From 7e38d8f6b79a16dbfe621609ce94a9161264444d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Mon, 24 Jul 2017 12:59:11 +0300 Subject: [PATCH 01/26] Enable info log for mariadb_tests_hartmut_galera The test fails for unknown reasons from time to time and enabling the info log level should help resolve it. --- maxscale-system-test/CMakeLists.txt | 2 +- .../cnf/maxscale.cnf.template.galera_hartmut | 89 +++++++++++++++++++ 2 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 maxscale-system-test/cnf/maxscale.cnf.template.galera_hartmut diff --git a/maxscale-system-test/CMakeLists.txt b/maxscale-system-test/CMakeLists.txt index 22b142605..3da543dc6 100644 --- a/maxscale-system-test/CMakeLists.txt +++ b/maxscale-system-test/CMakeLists.txt @@ -309,7 +309,7 @@ add_test_executable(lots_of_rows.cpp lots_of_rows galera LABELS readwritesplit H add_test_script(mariadb_tests_hartmut mariadb_tests_hartmut.sh replication LABELS readwritesplit REPL_BACKEND) # A set of MariaDB server tests executed against Maxscale RWSplit (Galera backend) -add_test_script(mariadb_tests_hartmut_galera mariadb_tests_hartmut.sh galera LABELS readwritesplit GALERA_BACKEND) +add_test_script(mariadb_tests_hartmut_galera mariadb_tests_hartmut.sh galera_hartmut LABELS readwritesplit GALERA_BACKEND) # Creates a number of connections > max_connections setting add_test_executable(max_connections.cpp max_connections replication LABELS MySQLAuth MySQLProtocol UNSTABLE HEAVY REPL_BACKEND) diff --git a/maxscale-system-test/cnf/maxscale.cnf.template.galera_hartmut b/maxscale-system-test/cnf/maxscale.cnf.template.galera_hartmut new file mode 100644 index 000000000..fbfbb4d6c --- /dev/null +++ b/maxscale-system-test/cnf/maxscale.cnf.template.galera_hartmut @@ -0,0 +1,89 @@ +[maxscale] +threads=###threads### +log_info=1 + +[Galera Monitor] +type=monitor +module=galeramon +servers=server1,server2,server3,server4 +user=maxskysql +passwd=skysql +monitor_interval=1000 +root_node_as_master=false + +[RW Split Router] +type=service +router=readwritesplit +servers=server1,server2,server3,server4 +user=maxskysql +passwd=skysql +router_options=slave_selection_criteria=LEAST_GLOBAL_CONNECTIONS +max_slave_connections=1 + +[Read Connection Router Slave] +type=service +router=readconnroute +router_options=slave +servers=server1,server2,server3,server4 +user=maxskysql +passwd=skysql + +[Read Connection Router Master] +type=service +router=readconnroute +router_options=master +servers=server1,server2,server3,server4 +user=maxskysql +passwd=skysql + +[RW Split Listener] +type=listener +service=RW Split Router +protocol=MySQLClient +port=4006 + +[Read Connection Listener Slave] +type=listener +service=Read Connection Router Slave +protocol=MySQLClient +port=4009 + +[Read Connection Listener Master] +type=listener +service=Read Connection Router Master +protocol=MySQLClient +port=4008 + +[CLI] +type=service +router=cli + +[CLI Listener] +type=listener +service=CLI +protocol=maxscaled +socket=default + +[server1] +type=server +address=###galera_server_IP_1### +port=###galera_server_port_1### +protocol=MySQLBackend + +[server2] +type=server +address=###galera_server_IP_2### +port=###galera_server_port_2### +protocol=MySQLBackend + +[server3] +type=server +address=###galera_server_IP_3### +port=###galera_server_port_3### +protocol=MySQLBackend + +[server4] +type=server +address=###galera_server_IP_4### +port=###galera_server_port_4### +protocol=MySQLBackend From 24d20d2a1bcc7b7303537605c047cc86741367b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Mon, 24 Jul 2017 13:55:17 +0300 Subject: [PATCH 02/26] Increse timeouts for mxs1071_maxrows As the test does a lot of large inserts, the timeouts can be triggered at times when the test is not actually stuck but just waiting for the insert to complete. Increasing the timeout from 10 to 30 seconds should give it enough time to finish processing. --- maxscale-system-test/mxs1071_maxrows.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/maxscale-system-test/mxs1071_maxrows.cpp b/maxscale-system-test/mxs1071_maxrows.cpp index c52a295b1..3f6253c5d 100644 --- a/maxscale-system-test/mxs1071_maxrows.cpp +++ b/maxscale-system-test/mxs1071_maxrows.cpp @@ -207,7 +207,7 @@ int compare_expected(TestConnections * Test, const char * sql, my_ulonglong exp_ my_ulonglong *rows = new my_ulonglong[30]; my_ulonglong i; - Test->set_timeout(10); + Test->set_timeout(30); execute_query_num_of_rows(Test->conn_rwsplit, sql, rows, &i); Test->tprintf("Result sets number is %llu\n", i); @@ -245,7 +245,7 @@ int compare_stmt_expected(TestConnections * Test, MYSQL_STMT * stmt, my_ulonglon my_ulonglong *rows = new my_ulonglong[30]; my_ulonglong i; - Test->set_timeout(10); + Test->set_timeout(30); execute_stmt_num_of_rows(stmt, rows, &i); Test->tprintf("Result sets number is %llu\n", i); @@ -290,7 +290,7 @@ int main(int argc, char *argv[]) MYSQL_STMT * stmt; TestConnections * Test = new TestConnections(argc, argv); - Test->set_timeout(10); + Test->set_timeout(30); Test->connect_rwsplit(); create_t1(Test->conn_rwsplit); From 6ac3484e317076715f1d8b3b588227a9a86828ea Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Tue, 25 Jul 2017 10:31:55 +0300 Subject: [PATCH 03/26] Update 2.1.5 version number --- VERSION21.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION21.cmake b/VERSION21.cmake index ba1a23389..56646f85c 100644 --- a/VERSION21.cmake +++ b/VERSION21.cmake @@ -5,7 +5,7 @@ set(MAXSCALE_VERSION_MAJOR "2" CACHE STRING "Major version") set(MAXSCALE_VERSION_MINOR "1" CACHE STRING "Minor version") -set(MAXSCALE_VERSION_PATCH "4" CACHE STRING "Patch version") +set(MAXSCALE_VERSION_PATCH "5" CACHE STRING "Patch version") # This should only be incremented if a package is rebuilt set(MAXSCALE_BUILD_NUMBER 1 CACHE STRING "Release number") From 17a09412cc2d919382bd06f77050d494e9e11dab Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Tue, 25 Jul 2017 10:34:46 +0300 Subject: [PATCH 04/26] Update ChangeLog and Upgrading --- Documentation/Changelog.md | 1 + Documentation/Upgrading/Upgrading-To-MaxScale-2.1.md | 1 + 2 files changed, 2 insertions(+) diff --git a/Documentation/Changelog.md b/Documentation/Changelog.md index 34a68e631..84f55e6bc 100644 --- a/Documentation/Changelog.md +++ b/Documentation/Changelog.md @@ -21,6 +21,7 @@ * MaxScale now supports IPv6 For more details, please refer to: +* [MariaDB MaxScale 2.1.5 Release Notes](Release-Notes/MaxScale-2.1.5-Release-Notes.md) * [MariaDB MaxScale 2.1.4 Release Notes](Release-Notes/MaxScale-2.1.4-Release-Notes.md) * [MariaDB MaxScale 2.1.3 Release Notes](Release-Notes/MaxScale-2.1.3-Release-Notes.md) * [MariaDB MaxScale 2.1.2 Release Notes](Release-Notes/MaxScale-2.1.2-Release-Notes.md) diff --git a/Documentation/Upgrading/Upgrading-To-MaxScale-2.1.md b/Documentation/Upgrading/Upgrading-To-MaxScale-2.1.md index aee13f8dc..a07949329 100644 --- a/Documentation/Upgrading/Upgrading-To-MaxScale-2.1.md +++ b/Documentation/Upgrading/Upgrading-To-MaxScale-2.1.md @@ -7,6 +7,7 @@ For more information about MariaDB MaxScale 2.1, please refer to the [ChangeLog](../Changelog.md). For a complete list of changes in MaxScale 2.1, refer to the +[MaxScale 2.1.5 Release Notes](../Release-Notes/MaxScale-2.1.5-Release-Notes.md). [MaxScale 2.1.4 Release Notes](../Release-Notes/MaxScale-2.1.4-Release-Notes.md). [MaxScale 2.1.3 Release Notes](../Release-Notes/MaxScale-2.1.3-Release-Notes.md). [MaxScale 2.1.2 Release Notes](../Release-Notes/MaxScale-2.1.2-Release-Notes.md). From 2148d78d3f8477ec0fd6013823a2a8f205740805 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Tue, 25 Jul 2017 09:23:42 +0300 Subject: [PATCH 05/26] MXS-1324: Fix build scripts for Debian-based systems The avrorouter dependencies weren't correctly built for Debian-based systems. This is due to the different way Debian and RHEL store libraries. --- BUILD/install_build_deps.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/BUILD/install_build_deps.sh b/BUILD/install_build_deps.sh index 9b374eae4..00d822655 100755 --- a/BUILD/install_build_deps.sh +++ b/BUILD/install_build_deps.sh @@ -9,7 +9,7 @@ command -v apt-get if [ $? == 0 ] then # DEB-based distro - + install_libdir=/usr/lib sudo apt-get update sudo apt-get install -y --force-yes dpkg-dev git gcc g++ ncurses-dev bison \ @@ -18,6 +18,7 @@ then sqlite3 liblua5.1 liblua5.1-dev wget else ## RPM-based distro + install_libdir=/usr/lib64 command -v yum if [ $? != 0 ] @@ -123,7 +124,7 @@ fi mkdir -p jansson/build pushd jansson/build -cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_C_FLAGS=-fPIC -DJANSSON_INSTALL_LIB_DIR=/usr/lib64 +cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_C_FLAGS=-fPIC -DJANSSON_INSTALL_LIB_DIR=$install_libdir make sudo make install popd From ed44c45be1d7fcf3efc6d0f747bcd2cab7316358 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Tue, 25 Jul 2017 09:46:40 +0300 Subject: [PATCH 06/26] MXS-1323: Fix crash on attempted retry of read When a backend is waiting for a response but no statement is stored for the session, the buffer where the stored statement is copied is not modified. This means that it needs to be initialized to a NULL value. Added a test that checks that the behavior works as expected even with persistent connections. A second test reproduces the crash by executing parallel SET commands while slaves are blocked. There is still a behavioral problem in readwritesplit. If a session command is being executed and it fails on a slave, an error is sent to the client. In this case it would not be necessary to close the session if the master is still alive. --- maxscale-system-test/CMakeLists.txt | 4 + .../cnf/maxscale.cnf.template.mxs1323 | 51 ++++++++++++ maxscale-system-test/mariadb_nodes.cpp | 8 ++ maxscale-system-test/mariadb_nodes.h | 2 + maxscale-system-test/mxs1323_retry_read.cpp | 50 ++++++++++++ maxscale-system-test/mxs1323_stress.cpp | 81 +++++++++++++++++++ .../routing/readwritesplit/readwritesplit.c | 14 +++- 7 files changed, 206 insertions(+), 4 deletions(-) create mode 100644 maxscale-system-test/cnf/maxscale.cnf.template.mxs1323 create mode 100644 maxscale-system-test/mxs1323_retry_read.cpp create mode 100644 maxscale-system-test/mxs1323_stress.cpp diff --git a/maxscale-system-test/CMakeLists.txt b/maxscale-system-test/CMakeLists.txt index 3da543dc6..9908a6135 100644 --- a/maxscale-system-test/CMakeLists.txt +++ b/maxscale-system-test/CMakeLists.txt @@ -302,6 +302,10 @@ add_test_executable(mxs1110_16mb.cpp mxs1110_16mb longblob_filters LABELS readwr # Schemarouter implicit database detection add_test_executable(mxs1310_implicit_db.cpp mxs1310_implicit_db mxs1310_implicit_db LABELS schemarouter REPL_BACKEND) +# Retry reads with persistent connections +add_test_executable(mxs1323_retry_read.cpp mxs1323_retry_read mxs1323 LABELS readwritesplit LIGHT REPL_BACKEND) +add_test_executable(mxs1323_stress.cpp mxs1323_stress mxs1323 LABELS readwritesplit REPL_BACKEND) + # INSERT extremelly big number of rows add_test_executable(lots_of_rows.cpp lots_of_rows galera LABELS readwritesplit HEAVY GALERA_BACKEND) diff --git a/maxscale-system-test/cnf/maxscale.cnf.template.mxs1323 b/maxscale-system-test/cnf/maxscale.cnf.template.mxs1323 new file mode 100644 index 000000000..f5fff9976 --- /dev/null +++ b/maxscale-system-test/cnf/maxscale.cnf.template.mxs1323 @@ -0,0 +1,51 @@ +[maxscale] +threads=###threads### +log_info=1 + +[MySQL Monitor] +type=monitor +module=mysqlmon +###repl51### +servers= server1,server2 +user=maxskysql +passwd= skysql +monitor_interval=500 + +[RW Split Router] +type=service +router= readwritesplit +servers=server1,server2 +user=maxskysql +passwd=skysql + +[RW Split Listener] +type=listener +service=RW Split Router +protocol=MySQLClient +port=4006 + +[CLI] +type=service +router=cli + +[CLI Listener] +type=listener +service=CLI +protocol=maxscaled +socket=default + +[server1] +type=server +address=###node_server_IP_1### +port=###node_server_port_1### +protocol=MySQLBackend +persistpoolmax=10 +persistmaxtime=300 + +[server2] +type=server +address=###node_server_IP_2### +port=###node_server_port_2### +protocol=MySQLBackend +persistpoolmax=10 +persistmaxtime=300 diff --git a/maxscale-system-test/mariadb_nodes.cpp b/maxscale-system-test/mariadb_nodes.cpp index 85860f4db..cd21436af 100644 --- a/maxscale-system-test/mariadb_nodes.cpp +++ b/maxscale-system-test/mariadb_nodes.cpp @@ -14,6 +14,7 @@ #include "sql_const.h" #include #include +#include #include Mariadb_nodes::Mariadb_nodes(const char *pref, const char *test_cwd, bool verbose): @@ -918,6 +919,13 @@ int Mariadb_nodes::get_server_id(int index) return id; } +std::string Mariadb_nodes::get_server_id_str(int index) +{ + std::stringstream ss; + ss << get_server_id(index); + return ss.str(); +} + void Mariadb_nodes::generate_ssh_cmd(char *cmd, int node, const char *ssh, bool sudo) { if (strcmp(IP[node], "127.0.0.1") == 0) diff --git a/maxscale-system-test/mariadb_nodes.h b/maxscale-system-test/mariadb_nodes.h index bc01f34dc..f7052c0c2 100644 --- a/maxscale-system-test/mariadb_nodes.h +++ b/maxscale-system-test/mariadb_nodes.h @@ -16,6 +16,7 @@ #include "mariadb_func.h" #include +#include /** * @brief A class to handle backend nodes @@ -329,6 +330,7 @@ public: * @return Node id of the server or -1 on error */ int get_server_id(int index); + std::string get_server_id_str(int index); /** * @brief Generate command line to execute command on the node via ssh diff --git a/maxscale-system-test/mxs1323_retry_read.cpp b/maxscale-system-test/mxs1323_retry_read.cpp new file mode 100644 index 000000000..7d17f337f --- /dev/null +++ b/maxscale-system-test/mxs1323_retry_read.cpp @@ -0,0 +1,50 @@ +/** + * Test for MXS-1323. + * - Check that retried reads work with persistent connections + */ + +#include "testconnections.h" + +void* async_block(void* data) +{ + TestConnections *test = (TestConnections*)data; + sleep(5); + test->tprintf("Blocking slave"); + test->repl->block_node(1); + return NULL; +} + +std::string do_query(TestConnections& test) +{ + MYSQL* conn = test.open_rwsplit_connection(); + + const char* query = "SELECT SLEEP(10), @@server_id"; + char output[512] = ""; + + find_field(conn, query, "@@server_id", output); + mysql_close(conn); + + return std::string(output); +} + +int main(int argc, char *argv[]) +{ + TestConnections test(argc, argv); + + char server_id[2][1024]; + test.repl->connect(); + std::string master = test.repl->get_server_id_str(0); + std::string slave = test.repl->get_server_id_str(1); + test.repl->close_connections(); + + test.set_timeout(60); + test.add_result(do_query(test) != slave, "The slave should respond to the first query"); + + pthread_t thr; + pthread_create(&thr, NULL, async_block, &test); + test.add_result(do_query(test) != master, "The master should respond to the second query"); + pthread_join(thr, NULL); + test.repl->unblock_node(1); + + return test.global_result; +} diff --git a/maxscale-system-test/mxs1323_stress.cpp b/maxscale-system-test/mxs1323_stress.cpp new file mode 100644 index 000000000..f20275144 --- /dev/null +++ b/maxscale-system-test/mxs1323_stress.cpp @@ -0,0 +1,81 @@ +/** + * Test for MXS-1323. + * - Check that retried reads work with persistent connections + */ + +#include "testconnections.h" +#include + +static bool running = true; + +void* async_query(void* data) +{ + TestConnections *test = (TestConnections*)data; + + while (running && test->global_result == 0) + { + MYSQL* conn = test->open_rwsplit_connection(); + + for (int i = 0; i < 50; i++) + { + const char* query = "SET @a = (SELECT SLEEP(1))"; + test->try_query(conn, query); + } + + mysql_close(conn); + } + + return NULL; +} + +#define NUM_THR 5 + +int main(int argc, char *argv[]) +{ + TestConnections test(argc, argv); + pthread_t query_thr[NUM_THR]; + std::stringstream ss; + + ss << "CREATE OR REPLACE TABLE test.t1 (id INT)"; + test.connect_maxscale(); + test.try_query(test.conn_rwsplit, ss.str().c_str()); + + ss.str(""); + ss << "INSERT INTO test.t1 VALUES (0)"; + for (int i = 1; i <= 10000; i++) + { + ss << ",(" << i << ")"; + } + test.try_query(test.conn_rwsplit, ss.str().c_str()); + + test.close_maxscale_connections(); + + if (test.global_result) + { + return test.global_result; + } + + for (int i = 0; i < NUM_THR; i++) + { + pthread_create(&query_thr[i], NULL, async_query, &test); + } + + for (int i = 0; i < 3 && test.global_result == 0; i++) + { + test.tprintf("Round %d", i + 1); + test.repl->block_node(1); + sleep(5); + test.repl->unblock_node(1); + sleep(5); + } + + running = false; + + for (int i = 0; i < NUM_THR; i++) + { + test.set_timeout(10); + pthread_join(query_thr[i], NULL); + } + + return test.global_result; +} diff --git a/server/modules/routing/readwritesplit/readwritesplit.c b/server/modules/routing/readwritesplit/readwritesplit.c index 89a8a4d50..ce15f6cad 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.c +++ b/server/modules/routing/readwritesplit/readwritesplit.c @@ -1541,8 +1541,8 @@ static bool handle_error_new_connection(ROUTER_INSTANCE *inst, */ if (BREF_IS_WAITING_RESULT(bref)) { - GWBUF *stored; - const SERVER *target; + GWBUF *stored = NULL; + const SERVER *target = NULL; if (!session_take_stmt(backend_dcb->session, &stored, &target) || target != bref->ref->server || @@ -1554,8 +1554,14 @@ static bool handle_error_new_connection(ROUTER_INSTANCE *inst, */ gwbuf_free(stored); - DCB *client_dcb = ses->client_dcb; - client_dcb->func.write(client_dcb, gwbuf_clone(errmsg)); + if (!sescmd_cursor_is_active(&bref->bref_sescmd_cur)) + { + /** The client expects a response from this exact backend. + * We need to route an error to the client to let it know + * that the query failed. */ + DCB *client_dcb = ses->client_dcb; + client_dcb->func.write(client_dcb, gwbuf_clone(errmsg)); + } } } From f16e93c1129f2ae7c0a5dae539a4a11a2e7de21d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Tue, 25 Jul 2017 10:32:50 +0300 Subject: [PATCH 07/26] MXS-1285: Check that upstart config exists before copying it The existence of the file should be checked before it is copied. Otherwise the installation will produce errors. --- etc/postinst.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/postinst.in b/etc/postinst.in index a46ec3bd9..592fcf0ce 100755 --- a/etc/postinst.in +++ b/etc/postinst.in @@ -53,7 +53,7 @@ then systemctl daemon-reload fi else - if [ -d "/etc/init/" ] + if [ -d "/etc/init/" ] && [ -f "@CMAKE_INSTALL_PREFIX@/@MAXSCALE_SHAREDIR@/upstart/maxscale.conf" ] then cp @CMAKE_INSTALL_PREFIX@/@MAXSCALE_SHAREDIR@/upstart/maxscale.conf /etc/init/ fi From ba3912ddb44a996463b6244a55b7f9e53b422da5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Tue, 25 Jul 2017 10:34:35 +0300 Subject: [PATCH 08/26] MXS-1326: Fix postrm script The script could only work on RPM based systems as DEB based uninstall scripts are called with different parameters, namely "remove" instead of "0". --- etc/postrm.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/postrm.in b/etc/postrm.in index 7474dda40..4a10dcaf3 100755 --- a/etc/postrm.in +++ b/etc/postrm.in @@ -3,7 +3,7 @@ # The first argument is the number of packages left after # this one has been removed. If it is 0 then the package is being # removed from the system. -if [ "$1" -eq 0 ] +if [ "$1" = "0" ] || [ "$1" = "remove" ] then if [ -f /etc/init.d/maxscale ] then From fc81a8097fd9778dba462920e428a4929edd3e69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Tue, 25 Jul 2017 21:08:50 +0300 Subject: [PATCH 09/26] Add missing `length` and `real_type` fields to cdc_schema.go The Go version of the CDC schema generator was missing the new `length` and `real_type` fields. --- .../modules/protocol/examples/cdc_schema.go | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/server/modules/protocol/examples/cdc_schema.go b/server/modules/protocol/examples/cdc_schema.go index bcd9c43e1..277cd9f08 100644 --- a/server/modules/protocol/examples/cdc_schema.go +++ b/server/modules/protocol/examples/cdc_schema.go @@ -15,11 +15,11 @@ import ( "database/sql" "encoding/json" "flag" + "fmt" "log" "os" "regexp" "strconv" - "fmt" ) import _ "github.com/go-sql-driver/mysql" @@ -45,8 +45,10 @@ The "user" and "password" flags are required. // Avro field type Field struct { - Name string `json:"name"` - Type string `json:"type"` + Name string `json:"name"` + Type string `json:"type"` + RealType string `json:"real_type"` + Length int `json:"length"` } // Avro schema @@ -68,13 +70,20 @@ func LogObject(obj interface{}) { } var field_re *regexp.Regexp +var length_re *regexp.Regexp // Convert the SQL type to the appropriate Avro type func (f *Field) ToAvroType() { + orig := f.Type f.Type = field_re.ReplaceAllString(f.Type, "") + f.Length = -1 + f.RealType = f.Type switch f.Type { case "date", "datetime", "time", "timestamp", "year", "tinytext", "text", - "mediumtext", "longtext", "char", "varchar", "enum", "set": + "mediumtext", "longtext", "char", "varchar": + f.Type = "string" + f.Length, _ = strconv.Atoi(length_re.ReplaceAllString(orig, "$1")) + case "enum", "set": f.Type = "string" case "tinyblob", "blob", "mediumblob", "longblob", "binary", "varbinary": f.Type = "bytes" @@ -127,11 +136,18 @@ func StoreSchema(db *sql.DB, schema, table string) { func main() { var err error field_re, err = regexp.Compile("[(].*") + if err != nil { log.Fatal("Error: ", err) } - flag.Usage = PrintUsage; + length_re, err = regexp.Compile(".*[(](.*)[)].*") + + if err != nil { + log.Fatal("Error: ", err) + } + + flag.Usage = PrintUsage flag.Parse() if len(*user) == 0 || len(*passwd) == 0 { From e26ec4f8b93a78546897959c1caadc2a457ab141 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Wed, 26 Jul 2017 22:25:36 +0300 Subject: [PATCH 10/26] MXS-1329: Only parse text protocol statements The query classifier should only be used to parse text protocol statements. The insertstream filter exploited the fact that any statements that the filter did not expect would be classified as an unknown commands. This led to repetitive error messages with binary protocol statements. --- server/modules/filter/insertstream/insertstream.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/modules/filter/insertstream/insertstream.c b/server/modules/filter/insertstream/insertstream.c index 1b2c7b9b0..1c5eb9dca 100644 --- a/server/modules/filter/insertstream/insertstream.c +++ b/server/modules/filter/insertstream/insertstream.c @@ -574,7 +574,8 @@ static bool extract_insert_target(GWBUF *buffer, char* target, int len) { bool rval = false; - if (qc_get_operation(buffer) == QUERY_OP_INSERT && + if (MYSQL_GET_COMMAND(GWBUF_DATA(buffer)) == MYSQL_COM_QUERY && + qc_get_operation(buffer) == QUERY_OP_INSERT && only_implicit_values(buffer)) { int n_tables = 0; From 563fdb4db5e613ddbe616ceccaaa9e986d1fffbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 27 Jul 2017 08:28:16 +0300 Subject: [PATCH 11/26] Backport testing fixes to 2.1 Some of the testing fixes weren't included into 2.1 and due to that, a few of the tests appear to fail. --- maxscale-system-test/mariadb_tests_hartmut.sh | 2 +- maxscale-system-test/testconnections.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/maxscale-system-test/mariadb_tests_hartmut.sh b/maxscale-system-test/mariadb_tests_hartmut.sh index ef8b6e64d..a456d6917 100755 --- a/maxscale-system-test/mariadb_tests_hartmut.sh +++ b/maxscale-system-test/mariadb_tests_hartmut.sh @@ -4,7 +4,7 @@ # # TODO: Don't test correctness of routing with mysqltest # -./non_native_setup $1 + master_id=`echo "SELECT @@server_id" | mysql -u$node_user -p$node_password -h $node_000_network $ssl_options -P $node_000_port | tail -n1` echo "--disable_query_log" > Hartmut_tests/maxscale-mysqltest/testconf.inc echo "SET @TMASTER_ID=$master_id;" >> Hartmut_tests/maxscale-mysqltest/testconf.inc diff --git a/maxscale-system-test/testconnections.cpp b/maxscale-system-test/testconnections.cpp index 9d6a2e446..25d52b81f 100644 --- a/maxscale-system-test/testconnections.cpp +++ b/maxscale-system-test/testconnections.cpp @@ -1672,7 +1672,7 @@ void *timeout_thread( void *ptr ) Test->timeout--; } Test->tprintf("\n **** Timeout! *** \n"); - delete Test; + Test->~TestConnections(); exit(250); } From e48fb6a0710506dfbf6ed2af3464db4708ca8555 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Tue, 11 Jul 2017 22:27:42 +0300 Subject: [PATCH 12/26] Assume that the `test` database exists The tests should not drop the test database and then recreate it. This adds an unnecessary burden on replication which will cause false positives when replication is not fast enough. This is commit `2410c23e6664bef7e449dfd5a767121f309559fe` from develop cherry-picked to 2.1. --- maxscale-system-test/bug529.cpp | 3 --- maxscale-system-test/bug649.cpp | 3 +-- maxscale-system-test/short_sessions.cpp | 2 -- 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/maxscale-system-test/bug529.cpp b/maxscale-system-test/bug529.cpp index 5f5de6d36..0607cd740 100644 --- a/maxscale-system-test/bug529.cpp +++ b/maxscale-system-test/bug529.cpp @@ -150,10 +150,7 @@ int main(int argc, char *argv[]) Test->repl->connect(); conn = Test->open_rwsplit_connection(); - execute_query(conn, (char *) "DROP DATABASE IF EXISTS test"); - execute_query(conn, (char *) "CREATE DATABASE test"); execute_query(conn, (char *) "USE test;"); - create_t1(conn); mysql_close(conn); Test->tprintf("Table t1 is created\n"); diff --git a/maxscale-system-test/bug649.cpp b/maxscale-system-test/bug649.cpp index a414a6ea7..fbb95d594 100644 --- a/maxscale-system-test/bug649.cpp +++ b/maxscale-system-test/bug649.cpp @@ -126,8 +126,7 @@ int main(int argc, char *argv[]) /** Clean up */ Test->repl->connect(); - execute_query(Test->repl->nodes[0], "DROP DATABASE test"); - execute_query(Test->repl->nodes[0], "CREATE DATABASE test"); + execute_query(Test->repl->nodes[0], "DROP TABLE IF EXISTS t1;"); int rval = Test->global_result; delete Test; diff --git a/maxscale-system-test/short_sessions.cpp b/maxscale-system-test/short_sessions.cpp index 6eb25789e..3b7df1ee5 100644 --- a/maxscale-system-test/short_sessions.cpp +++ b/maxscale-system-test/short_sessions.cpp @@ -24,8 +24,6 @@ int main(int argc, char *argv[]) test.repl->connect(); MYSQL *conn = test.open_rwsplit_connection(); - execute_query(conn, "DROP DATABASE IF EXISTS test;"); - execute_query(conn, "CREATE DATABASE test;"); execute_query(conn, "USE test;"); create_t1(conn); mysql_close(conn); From 448d56a273cb9f2637de95bb57e27206ab8f42bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 27 Jul 2017 08:45:43 +0300 Subject: [PATCH 13/26] Disable replication_manager and fix mxs1323_stress false positive The replication_manager is only designed for systems that have yum installed which means it will always fail on non-RHEL/CentOS systems. The query threads in mxs1323_stress were not checking whether the test had ended while they were executing the queries. This caused test timeouts as the queries can take a relatively long time. --- maxscale-system-test/CMakeLists.txt | 2 +- maxscale-system-test/mxs1323_stress.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/maxscale-system-test/CMakeLists.txt b/maxscale-system-test/CMakeLists.txt index 9908a6135..b2798cb78 100644 --- a/maxscale-system-test/CMakeLists.txt +++ b/maxscale-system-test/CMakeLists.txt @@ -546,7 +546,7 @@ add_test_executable(rwsplit_multi_stmt.cpp rwsplit_multi_stmt rwsplit_multi_stmt add_test_executable(rwsplit_read_only_trx.cpp rwsplit_read_only_trx rwsplit_read_only_trx LABELS readwritesplit REPL_BACKEND) # Test replication-manager with MaxScale -add_test_executable(replication_manager.cpp replication_manager replication_manager LABELS maxscale REPL_BACKEND) +#add_test_executable(replication_manager.cpp replication_manager replication_manager LABELS maxscale REPL_BACKEND) #add_test_executable_notest(replication_manager_2nodes.cpp replication_manager_2nodes replication_manager_2nodes LABELS maxscale REPL_BACKEND) #add_test_executable_notest(replication_manager_3nodes.cpp replication_manager_3nodes replication_manager_3nodes LABELS maxscale REPL_BACKEND) diff --git a/maxscale-system-test/mxs1323_stress.cpp b/maxscale-system-test/mxs1323_stress.cpp index f20275144..efae35e66 100644 --- a/maxscale-system-test/mxs1323_stress.cpp +++ b/maxscale-system-test/mxs1323_stress.cpp @@ -16,7 +16,7 @@ void* async_query(void* data) { MYSQL* conn = test->open_rwsplit_connection(); - for (int i = 0; i < 50; i++) + for (int i = 0; i < 50 && running && test->global_result == 0; i++) { const char* query = "SET @a = (SELECT SLEEP(1))"; test->try_query(conn, query); From d54616f0756ed7ae723533f4b506b76b7cc51b13 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Tue, 25 Jul 2017 10:50:32 +0300 Subject: [PATCH 14/26] Update 2.1.5 release notes --- .../Release-Notes/MaxScale-2.1.5-Release-Notes.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Documentation/Release-Notes/MaxScale-2.1.5-Release-Notes.md b/Documentation/Release-Notes/MaxScale-2.1.5-Release-Notes.md index 9025c9581..2cac79a2f 100644 --- a/Documentation/Release-Notes/MaxScale-2.1.5-Release-Notes.md +++ b/Documentation/Release-Notes/MaxScale-2.1.5-Release-Notes.md @@ -43,7 +43,14 @@ will still go to the central database. [Here is a list of bugs fixed since the release of MaxScale 2.1.4.] (https://jira.mariadb.org/issues/?jql=project%20%3D%20MXS%20AND%20issuetype%20%3D%20Bug%20AND%20status%20%3D%20Closed%20AND%20fixVersion%20%3D%202.1.5) +* [MXS-1319](https://jira.mariadb.org/browse/MXS-1319) Maxscale selecting extra whitespace while loading users +* [MXS-1318](https://jira.mariadb.org/browse/MXS-1318) Use SSL_CTX_use_certificate_chain_file in Maxscale to use CA signed certificates +* [MXS-1316](https://jira.mariadb.org/browse/MXS-1316) error using Kafka with binlog router +* [MXS-1313](https://jira.mariadb.org/browse/MXS-1313) Character set is not updated if servers are down +* [MXS-1312](https://jira.mariadb.org/browse/MXS-1312) Rule with only on_queries do not work +* [MXS-1311](https://jira.mariadb.org/browse/MXS-1311) Function type rule that blocks function results in syntax error * [MXS-1310](https://jira.mariadb.org/browse/MXS-1310) schemarouter ignores local copy of duplicate schemas on JOIN +* [MXS-1309](https://jira.mariadb.org/browse/MXS-1309) ALTER TABLE detection is broken ## Known Issues and Limitations From 4985bf9b6ebacca99c202bc6825b7eeeb7e87e72 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Thu, 27 Jul 2017 09:46:34 +0300 Subject: [PATCH 15/26] MXS-1328 Mark regexp functions as builtin functions Regexp functions were not listed among the builtin read-only functions and consequently any SELECT using one of those was routed to master. --- query_classifier/qc_sqlite/builtin_functions.c | 10 ++++++++++ query_classifier/test/maxscale.test | 2 ++ 2 files changed, 12 insertions(+) diff --git a/query_classifier/qc_sqlite/builtin_functions.c b/query_classifier/qc_sqlite/builtin_functions.c index 37fde1477..4fe8b950f 100644 --- a/query_classifier/qc_sqlite/builtin_functions.c +++ b/query_classifier/qc_sqlite/builtin_functions.c @@ -360,6 +360,16 @@ static const char* BUILTIN_FUNCTIONS[] = "to_base64", "weight_string", + /* + * Regex functions + * https://mariadb.com/kb/en/mariadb/regular-expressions-functions/ + */ + "regexp", + "regexp_instr", + "regexp_replace", + "regexp_substr", + "rlike", + /* * http://dev.mysql.com/doc/refman/5.7/en/row-subqueries.html */ diff --git a/query_classifier/test/maxscale.test b/query_classifier/test/maxscale.test index 95a3c130e..52a10478c 100644 --- a/query_classifier/test/maxscale.test +++ b/query_classifier/test/maxscale.test @@ -77,3 +77,5 @@ ROLLBACK TO SAVEPOINT id; ROLLBACK WORK TO id; ROLLBACK TO id; RELEASE SAVEPOINT id; + +SELECT her FROM (SELECT @@server_id as her) as t WHERE her REGEXP '.*'; From 86994457e1657ab000ed4434dd89c4bdf9c32e0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 27 Jul 2017 15:26:52 +0300 Subject: [PATCH 16/26] Update 2.1.5 release notes Added all fixed bugs and short descriptions of the major changes. --- .../MaxScale-2.1.5-Release-Notes.md | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/Documentation/Release-Notes/MaxScale-2.1.5-Release-Notes.md b/Documentation/Release-Notes/MaxScale-2.1.5-Release-Notes.md index 2cac79a2f..aeaa7bb5a 100644 --- a/Documentation/Release-Notes/MaxScale-2.1.5-Release-Notes.md +++ b/Documentation/Release-Notes/MaxScale-2.1.5-Release-Notes.md @@ -18,12 +18,26 @@ report at [Jira](https://jira.mariadb.org). ## Changed Features +### SSL CA Certificates + +Before MaxScale 2.1.5, MaxScale would only use the first certificate file found +in the CA certificate file. In MaxScale 2.1.5, the first certificate is loaded +and the rest of the certificates on the file are stored in the chain store. + +This change should not cause any changes in MaxScale's behavior. + ### Schemarouter Starting with MaxScale 2.1.5, the _schemarouter_ will prioritize the current database over an explicit database if tables in the the current database are used in a query. +### Dbfwfilter + +The function type rule will now accept backtick quoted values. This allows +keywords such as `insert` and `function` to be used as values for a function +rule. + ## New Features ### Schemarouter @@ -40,9 +54,15 @@ will still go to the central database. ## Bug fixes -[Here is a list of bugs fixed since the release of MaxScale 2.1.4.] +[Here is a list of bugs fixed in MaxScale 2.1.5.] (https://jira.mariadb.org/issues/?jql=project%20%3D%20MXS%20AND%20issuetype%20%3D%20Bug%20AND%20status%20%3D%20Closed%20AND%20fixVersion%20%3D%202.1.5) +* [MXS-1330](https://jira.mariadb.org/browse/MXS-1330) insertstream attempts to parse all buffers +* [MXS-1329](https://jira.mariadb.org/browse/MXS-1329) Using filters with SSL and keep alive can cause errors +* [MXS-1328](https://jira.mariadb.org/browse/MXS-1328) Strange behavior with routes between master / slaves +* [MXS-1326](https://jira.mariadb.org/browse/MXS-1326) Upgrade error on Ubuntu Xenial +* [MXS-1324](https://jira.mariadb.org/browse/MXS-1324) MaxScale 2.1.4 compiled without the avrorouter? +* [MXS-1323](https://jira.mariadb.org/browse/MXS-1323) Maxscale2.1.3 coredump * [MXS-1319](https://jira.mariadb.org/browse/MXS-1319) Maxscale selecting extra whitespace while loading users * [MXS-1318](https://jira.mariadb.org/browse/MXS-1318) Use SSL_CTX_use_certificate_chain_file in Maxscale to use CA signed certificates * [MXS-1316](https://jira.mariadb.org/browse/MXS-1316) error using Kafka with binlog router @@ -51,6 +71,7 @@ will still go to the central database. * [MXS-1311](https://jira.mariadb.org/browse/MXS-1311) Function type rule that blocks function results in syntax error * [MXS-1310](https://jira.mariadb.org/browse/MXS-1310) schemarouter ignores local copy of duplicate schemas on JOIN * [MXS-1309](https://jira.mariadb.org/browse/MXS-1309) ALTER TABLE detection is broken +* [MXS-1285](https://jira.mariadb.org/browse/MXS-1285) cannot stat `/usr/share/maxscale/upstart/maxscale.conf': No such file or directory ## Known Issues and Limitations From 08c51aa59891d848989cbda82a12052b7d36ea85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 27 Jul 2017 15:31:34 +0300 Subject: [PATCH 17/26] Remove broken links The links in old upgrading guides and the changelog pointed to documents that do not exist. --- Documentation/Changelog.md | 24 +++++++++---------- .../Upgrading/Upgrading-To-MaxScale-1.3.md | 2 +- .../Upgrading/Upgrading-To-MaxScale-1.4.md | 8 +++---- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Documentation/Changelog.md b/Documentation/Changelog.md index 84f55e6bc..a23fbb436 100644 --- a/Documentation/Changelog.md +++ b/Documentation/Changelog.md @@ -39,13 +39,13 @@ For more details, please refer to: as JSON objects (beta level functionality). For more details, please refer to: -* [MariaDB MaxScale 2.0.6 Release Notes](Release-Notes/MaxScale-2.0.6-Release-Notes.md) -* [MariaDB MaxScale 2.0.5 Release Notes](Release-Notes/MaxScale-2.0.5-Release-Notes.md) -* [MariaDB MaxScale 2.0.4 Release Notes](Release-Notes/MaxScale-2.0.4-Release-Notes.md) -* [MariaDB MaxScale 2.0.3 Release Notes](Release-Notes/MaxScale-2.0.3-Release-Notes.md) -* [MariaDB MaxScale 2.0.2 Release Notes](Release-Notes/MaxScale-2.0.2-Release-Notes.md) -* [MariaDB MaxScale 2.0.1 Release Notes](Release-Notes/MaxScale-2.0.1-Release-Notes.md) -* [MariaDB MaxScale 2.0.0 Release Notes](Release-Notes/MaxScale-2.0.0-Release-Notes.md) +* MariaDB MaxScale 2.0.6 Release Notes +* MariaDB MaxScale 2.0.5 Release Notes +* MariaDB MaxScale 2.0.4 Release Notes +* MariaDB MaxScale 2.0.3 Release Notes +* MariaDB MaxScale 2.0.2 Release Notes +* MariaDB MaxScale 2.0.1 Release Notes +* MariaDB MaxScale 2.0.0 Release Notes ## MariaDB MaxScale 1.4 * Authentication now allows table level resolution of grants. MaxScale service @@ -55,17 +55,17 @@ For more details, please refer to: * Client side SSL works. For more details, please refer to -* [MariaDB MaxScale 1.4.3 Release Notes](Release-Notes/MaxScale-1.4.3-Release-Notes.md) -* [MariaDB MaxScale 1.4.2 Release Notes](Release-Notes/MaxScale-1.4.2-Release-Notes.md) -* [MariaDB MaxScale 1.4.1 Release Notes](Release-Notes/MaxScale-1.4.1-Release-Notes.md) -* [MariaDB MaxScale 1.4.0 Release Notes](Release-Notes/MaxScale-1.4.0-Release-Notes.md). +* MariaDB MaxScale 1.4.3 Release Notes +* MariaDB MaxScale 1.4.2 Release Notes +* MariaDB MaxScale 1.4.1 Release Notes +* MariaDB MaxScale 1.4.0 Release Notes ## MariaDB MaxScale 1.3 * Added support for persistent backend connections * The *binlog server* is now an integral component of MariaDB MaxScale. * The logging has been changed; instead of different log files there is one log file and different message priorities. -For more details, please refer to [MariaDB MaxScale 1.3 Release Notes](Release-Notes/MaxScale-1.3.0-Release-Notes.md) +For more details, please refer to MariaDB MaxScale 1.3 Release Notes ## MariaDB MaxScale 1.2 * Logfiles have been renamed. The log names are now named error.log, messages.log, trace.log and debug.log. diff --git a/Documentation/Upgrading/Upgrading-To-MaxScale-1.3.md b/Documentation/Upgrading/Upgrading-To-MaxScale-1.3.md index 4c814b4a5..78ca57ccf 100644 --- a/Documentation/Upgrading/Upgrading-To-MaxScale-1.3.md +++ b/Documentation/Upgrading/Upgrading-To-MaxScale-1.3.md @@ -3,7 +3,7 @@ This document describes upgrading MaxScale from version 1.2.1 to 1.3. The major changes can be found in the [changelog](../Changelog.md) and more -detailed information in the [release notes](../Release-Notes/MaxScale-1.3.0-Release-Notes.md). +detailed information in the MaxScale 1.3.0 release notes. ## Installation diff --git a/Documentation/Upgrading/Upgrading-To-MaxScale-1.4.md b/Documentation/Upgrading/Upgrading-To-MaxScale-1.4.md index 23f15ccaa..ba18eb8e4 100644 --- a/Documentation/Upgrading/Upgrading-To-MaxScale-1.4.md +++ b/Documentation/Upgrading/Upgrading-To-MaxScale-1.4.md @@ -3,10 +3,10 @@ This document describes upgrading MaxScale from version 1.3 to 1.4. For more detailed information about MaxScale 1.4, please refer to: -* [MaxScale 1.4.3 release notes](../Release-Notes/MaxScale-1.4.3-Release-Notes.md) -* [MaxScale 1.4.2 release notes](../Release-Notes/MaxScale-1.4.2-Release-Notes.md) -* [MaxScale 1.4.1 release notes](../Release-Notes/MaxScale-1.4.1-Release-Notes.md) -* [MaxScale 1.4.0 release notes](../Release-Notes/MaxScale-1.4.0-Release-Notes.md) +* MaxScale 1.4.3 release notes +* MaxScale 1.4.2 release notes +* MaxScale 1.4.1 release notes +* MaxScale 1.4.0 release notes ## Installation From 0256be660f230902299d1a159aa0e2390bc17797 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 27 Jul 2017 16:34:08 +0300 Subject: [PATCH 18/26] Fix missing zlib in build scripts The zlib library wasn't explicitly installed which caused it to fail on Debian 9. --- BUILD/install_build_deps.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/BUILD/install_build_deps.sh b/BUILD/install_build_deps.sh index 00d822655..183e150da 100755 --- a/BUILD/install_build_deps.sh +++ b/BUILD/install_build_deps.sh @@ -15,7 +15,8 @@ then sudo apt-get install -y --force-yes dpkg-dev git gcc g++ ncurses-dev bison \ build-essential libssl-dev libaio-dev perl make libtool libcurl4-openssl-dev \ libpcre3-dev flex tcl libeditline-dev uuid-dev liblzma-dev libsqlite3-dev \ - sqlite3 liblua5.1 liblua5.1-dev wget + sqlite3 liblua5.1 liblua5.1-dev wget zlib1g-dev + else ## RPM-based distro install_libdir=/usr/lib64 From 6d9500d898166136bec29ab639ac1448f93b348c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Fri, 28 Jul 2017 09:46:34 +0300 Subject: [PATCH 19/26] Add links to release notes in Changelog.md The Markdown links do not work as the release note files do not exist but the resulting KB links do work. --- Documentation/Changelog.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Documentation/Changelog.md b/Documentation/Changelog.md index a23fbb436..84f55e6bc 100644 --- a/Documentation/Changelog.md +++ b/Documentation/Changelog.md @@ -39,13 +39,13 @@ For more details, please refer to: as JSON objects (beta level functionality). For more details, please refer to: -* MariaDB MaxScale 2.0.6 Release Notes -* MariaDB MaxScale 2.0.5 Release Notes -* MariaDB MaxScale 2.0.4 Release Notes -* MariaDB MaxScale 2.0.3 Release Notes -* MariaDB MaxScale 2.0.2 Release Notes -* MariaDB MaxScale 2.0.1 Release Notes -* MariaDB MaxScale 2.0.0 Release Notes +* [MariaDB MaxScale 2.0.6 Release Notes](Release-Notes/MaxScale-2.0.6-Release-Notes.md) +* [MariaDB MaxScale 2.0.5 Release Notes](Release-Notes/MaxScale-2.0.5-Release-Notes.md) +* [MariaDB MaxScale 2.0.4 Release Notes](Release-Notes/MaxScale-2.0.4-Release-Notes.md) +* [MariaDB MaxScale 2.0.3 Release Notes](Release-Notes/MaxScale-2.0.3-Release-Notes.md) +* [MariaDB MaxScale 2.0.2 Release Notes](Release-Notes/MaxScale-2.0.2-Release-Notes.md) +* [MariaDB MaxScale 2.0.1 Release Notes](Release-Notes/MaxScale-2.0.1-Release-Notes.md) +* [MariaDB MaxScale 2.0.0 Release Notes](Release-Notes/MaxScale-2.0.0-Release-Notes.md) ## MariaDB MaxScale 1.4 * Authentication now allows table level resolution of grants. MaxScale service @@ -55,17 +55,17 @@ For more details, please refer to: * Client side SSL works. For more details, please refer to -* MariaDB MaxScale 1.4.3 Release Notes -* MariaDB MaxScale 1.4.2 Release Notes -* MariaDB MaxScale 1.4.1 Release Notes -* MariaDB MaxScale 1.4.0 Release Notes +* [MariaDB MaxScale 1.4.3 Release Notes](Release-Notes/MaxScale-1.4.3-Release-Notes.md) +* [MariaDB MaxScale 1.4.2 Release Notes](Release-Notes/MaxScale-1.4.2-Release-Notes.md) +* [MariaDB MaxScale 1.4.1 Release Notes](Release-Notes/MaxScale-1.4.1-Release-Notes.md) +* [MariaDB MaxScale 1.4.0 Release Notes](Release-Notes/MaxScale-1.4.0-Release-Notes.md). ## MariaDB MaxScale 1.3 * Added support for persistent backend connections * The *binlog server* is now an integral component of MariaDB MaxScale. * The logging has been changed; instead of different log files there is one log file and different message priorities. -For more details, please refer to MariaDB MaxScale 1.3 Release Notes +For more details, please refer to [MariaDB MaxScale 1.3 Release Notes](Release-Notes/MaxScale-1.3.0-Release-Notes.md) ## MariaDB MaxScale 1.2 * Logfiles have been renamed. The log names are now named error.log, messages.log, trace.log and debug.log. From fcabda15a85c4a0d3f5b2390d77bc164edf5a3c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Fri, 28 Jul 2017 11:52:49 +0300 Subject: [PATCH 20/26] MXS-1335: Disable root_node_as_master by default Enabling the option hinders the use of maintenance mode with the root master node in most use-cases. This behavior occurs due to the fact that the maintenance mode causes a server to be treted as if it was down. The Galera monitor waits for the cluster to reorganize before assigning a new master node. This is correct (but very unexpected) behavior for single instance use-cases. --- Documentation/Monitors/Galera-Monitor.md | 5 +++-- .../Release-Notes/MaxScale-2.1.5-Release-Notes.md | 7 +++++++ server/modules/monitor/galeramon/galeramon.c | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Documentation/Monitors/Galera-Monitor.md b/Documentation/Monitors/Galera-Monitor.md index 4014d610d..0104d3d48 100644 --- a/Documentation/Monitors/Galera-Monitor.md +++ b/Documentation/Monitors/Galera-Monitor.md @@ -65,8 +65,9 @@ use_priority=true ### `root_node_as_master` This option controls whether the write master Galera node requires a -_wsrep_local_index_ value of 0. This option is enabled by default and was -introduced in MaxScale 2.1.0. +_wsrep_local_index_ value of 0. This option was introduced in MaxScale 2.1.0 and +it is disabled by default in versions 2.1.5 and newer. In versions 2.1.4 and +older, the option was enabled by default. A Galera cluster will always have a node which has a _wsrep_local_index_ value of 0. Based on this information, multiple MaxScale instances can always pick the diff --git a/Documentation/Release-Notes/MaxScale-2.1.5-Release-Notes.md b/Documentation/Release-Notes/MaxScale-2.1.5-Release-Notes.md index aeaa7bb5a..ab0fb6191 100644 --- a/Documentation/Release-Notes/MaxScale-2.1.5-Release-Notes.md +++ b/Documentation/Release-Notes/MaxScale-2.1.5-Release-Notes.md @@ -26,6 +26,12 @@ and the rest of the certificates on the file are stored in the chain store. This change should not cause any changes in MaxScale's behavior. +### `root_node_as_master` + +The galeramon parameter `root_node_as_master` is now disabled by default. The +option should be enabled when it is of great importance to know that all +MaxScale instances treat a shared Galera cluster in the same way. + ### Schemarouter Starting with MaxScale 2.1.5, the _schemarouter_ will prioritize the current @@ -57,6 +63,7 @@ will still go to the central database. [Here is a list of bugs fixed in MaxScale 2.1.5.] (https://jira.mariadb.org/issues/?jql=project%20%3D%20MXS%20AND%20issuetype%20%3D%20Bug%20AND%20status%20%3D%20Closed%20AND%20fixVersion%20%3D%202.1.5) +* [MXS-1335](https://jira.mariadb.org/browse/MXS-1335) root_node_as_master should not be enabled by default * [MXS-1330](https://jira.mariadb.org/browse/MXS-1330) insertstream attempts to parse all buffers * [MXS-1329](https://jira.mariadb.org/browse/MXS-1329) Using filters with SSL and keep alive can cause errors * [MXS-1328](https://jira.mariadb.org/browse/MXS-1328) Strange behavior with routes between master / slaves diff --git a/server/modules/monitor/galeramon/galeramon.c b/server/modules/monitor/galeramon/galeramon.c index b249b3949..f748e2fd8 100644 --- a/server/modules/monitor/galeramon/galeramon.c +++ b/server/modules/monitor/galeramon/galeramon.c @@ -93,7 +93,7 @@ MXS_MODULE* MXS_CREATE_MODULE() {"disable_master_failback", MXS_MODULE_PARAM_BOOL, "false"}, {"available_when_donor", MXS_MODULE_PARAM_BOOL, "false"}, {"disable_master_role_setting", MXS_MODULE_PARAM_BOOL, "false"}, - {"root_node_as_master", MXS_MODULE_PARAM_BOOL, "true"}, + {"root_node_as_master", MXS_MODULE_PARAM_BOOL, "false"}, {"use_priority", MXS_MODULE_PARAM_BOOL, "false"}, { "script", From d0d19f3dedcf2a8a93b8255f55d1ea8105aba769 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Sat, 29 Jul 2017 05:26:06 +0300 Subject: [PATCH 21/26] Tune timeouts for bug662 The timeouts were a bit too demanding and increasing them should lower the rate of false positives. --- maxscale-system-test/bug662.cpp | 38 +++++++-------------------------- 1 file changed, 8 insertions(+), 30 deletions(-) diff --git a/maxscale-system-test/bug662.cpp b/maxscale-system-test/bug662.cpp index af7e514f3..72e3b5c65 100644 --- a/maxscale-system-test/bug662.cpp +++ b/maxscale-system-test/bug662.cpp @@ -1,39 +1,18 @@ /** * @file bug662.cpp regression case for bug 662 ("MaxScale hangs in startup if backend server is not responsive"), covers also bug680 ("RWSplit can't load DB user if backend is not available at MaxScale start") * - * - block all Mariadb servers Firewall - * - restart MaxScale - * - check it took no more then 20 seconds - * - unblock Mariadb servers - * - sleep one minute - * - check if Maxscale is alive + * - Block all Mariadb servers + * - Restart MaxScale + * - Unblock Mariadb servers + * - Sleep and check if Maxscale is alive */ -/* -Vilho Raatikka 2014-12-29 08:38:28 UTC -During startup, load_mysql_users tries to read the contents of the mysql.user table. If the chosen backend is not responsive, connection hangs for a long time. -Comment 1 Vilho Raatikka 2014-12-29 11:41:32 UTC -The issue causes long stalls for the executing thread whenever getUsers function is called and one or more backends are not responsive. -Comment 2 Vilho Raatikka 2014-12-29 11:50:10 UTC -dbusers.c: Added function for setting read, write and connection timeout values. Set default timeouts for getUsers. Defaults are listed in service.c - gateway.c:shutdown_server is called whenever MaxScale is to be shut down. Added call for service_shutdown to shutdown_server. - service.c:service_alloc: replaced malloc with calloc and removed unnecessary zero/NULL initialization statements as a consequence. - serviceStart: Exit serviceStartPort loop if shutdown flag is set for the service. - serviceStartAll: Exit serviceStart loop if shutdown flag is set for the service. - service.c: Added service_shutdown which sets shutdown flag for each service found in allServices list. - service.h: Added prototype for service_shutdown -*/ - - - -#include #include "testconnections.h" #include "maxadmin_operations.h" int main(int argc, char *argv[]) { TestConnections * Test = new TestConnections(argc, argv); - Test->set_timeout(20); int i; Test->tprintf("Connecting to Maxscale %s\n", Test->maxscale_IP); @@ -44,20 +23,21 @@ int main(int argc, char *argv[]) for (i = 0; i < Test->repl->N; i++) { + Test->set_timeout(30); Test->tprintf("Setup firewall to block mysql on node %d\n", i); Test->repl->block_node(i); fflush(stdout); } - Test->set_timeout(100); + Test->set_timeout(120); Test->restart_maxscale(); - Test->set_timeout(20); Test->tprintf("Checking if MaxScale is alive by connecting to MaxAdmin\n"); Test->add_result(Test->execute_maxadmin_command((char* ) "show servers"), "Maxadmin execution failed.\n"); for (i = 0; i < Test->repl->N; i++) { + Test->set_timeout(30); Test->tprintf("Setup firewall back to allow mysql on node %d\n", i); Test->repl->unblock_node(i); fflush(stdout); @@ -67,13 +47,11 @@ int main(int argc, char *argv[]) Test->tprintf("Sleeping 30 seconds\n"); sleep(30); - Test->set_timeout(20); - + Test->set_timeout(30); Test->tprintf("Checking Maxscale is alive\n"); Test->check_maxscale_alive(); int rval = Test->global_result; delete Test; return rval; - //} } From af99ab3b72a6eb8fcb7a84f8fbb4c0773453f8e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 6 Jul 2017 13:31:38 +0300 Subject: [PATCH 22/26] Print the JSON in avro_alter The test now prints the output in avro_alter. The replicate_from_master function also stops maxscale before removing the files. --- maxscale-system-test/avro_alter.cpp | 1 + maxscale-system-test/testconnections.cpp | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/maxscale-system-test/avro_alter.cpp b/maxscale-system-test/avro_alter.cpp index ab6b51ebc..3fdf0146b 100644 --- a/maxscale-system-test/avro_alter.cpp +++ b/maxscale-system-test/avro_alter.cpp @@ -55,6 +55,7 @@ int main(int argc, char *argv[]) { json_error_t err; json_t* json = json_loads(line.c_str(), 0, &err); + test.tprintf("%s", line.c_str()); test.add_result(json == NULL, "Failed to parse JSON: %s", line.c_str()); json_decref(json); nrows++; diff --git a/maxscale-system-test/testconnections.cpp b/maxscale-system-test/testconnections.cpp index 25d52b81f..d7b1f0209 100644 --- a/maxscale-system-test/testconnections.cpp +++ b/maxscale-system-test/testconnections.cpp @@ -968,8 +968,9 @@ bool TestConnections::replicate_from_master() mysql_close(conn); /** Clean up MaxScale directories */ + ssh_maxscale(true, "service maxscale stop"); prepare_binlog(); - ssh_maxscale(true, "service maxscale restart"); + ssh_maxscale(true, "service maxscale start"); char log_file[256] = ""; char log_pos[256] = "4"; From 174dad4b5bdb682433f60cf830737fe15ff6b935 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Mon, 3 Jul 2017 16:53:25 +0300 Subject: [PATCH 23/26] Fix master server version checks in tests The master server version was not processed adequately and false positives of old master versions were possible. --- maxscale-system-test/testconnections.cpp | 31 ++++++++++++------------ 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/maxscale-system-test/testconnections.cpp b/maxscale-system-test/testconnections.cpp index d7b1f0209..59307c7a6 100644 --- a/maxscale-system-test/testconnections.cpp +++ b/maxscale-system-test/testconnections.cpp @@ -796,36 +796,35 @@ int TestConnections::copy_all_logs_periodic() int TestConnections::prepare_binlog() { - char version_str[1024]; - find_field(repl->nodes[0], "SELECT @@VERSION", "@@version", version_str); - tprintf("Master server version %s\n", version_str); + char version_str[1024] = ""; - if ((strstr(version_str, "10.0") != NULL) || - (strstr(version_str, "10.1") != NULL) || - (strstr(version_str, "10.2") != NULL)) - { - tprintf("10.0!\n"); - } - else + repl->connect(); + find_field(repl->nodes[0], "SELECT @@version", "@@version", version_str); + tprintf("Master server version '%s'", version_str); + + if (*version_str && + strstr(version_str, "10.0") == NULL && + strstr(version_str, "10.1") == NULL && + strstr(version_str, "10.2") == NULL) { add_result(ssh_maxscale(true, "sed -i \"s/,mariadb10-compatibility=1//\" %s", maxscale_cnf), "Error editing maxscale.cnf"); } - tprintf("Removing all binlog data from Maxscale node\n"); + tprintf("Removing all binlog data from Maxscale node"); add_result(ssh_maxscale(true, "rm -rf %s", maxscale_binlog_dir), - "Removing binlog data failed\n"); + "Removing binlog data failed"); - tprintf("Creating binlog dir\n"); + tprintf("Creating binlog dir"); add_result(ssh_maxscale(true, "mkdir -p %s", maxscale_binlog_dir), - "Creating binlog data dir failed\n"); - tprintf("Set 'maxscale' as a owner of binlog dir\n"); + "Creating binlog data dir failed"); + tprintf("Set 'maxscale' as a owner of binlog dir"); add_result(ssh_maxscale(false, "%s mkdir -p %s; %s chown maxscale:maxscale -R %s", maxscale_access_sudo, maxscale_binlog_dir, maxscale_access_sudo, maxscale_binlog_dir), - "directory ownership change failed\n"); + "directory ownership change failed"); return 0; } From f95fff66320fc90d848e7983574e4fce51a4a0d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 6 Jul 2017 13:54:59 +0300 Subject: [PATCH 24/26] Stop replication before configuring binlogrouter The replication needs to be stopped before the binlogrouter is started. If the replication is stopped after this, it is possible that two servers with the same value of server_id attempt to register as slaves which causes the later of them to fail. --- maxscale-system-test/testconnections.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/maxscale-system-test/testconnections.cpp b/maxscale-system-test/testconnections.cpp index 59307c7a6..9fc9042b3 100644 --- a/maxscale-system-test/testconnections.cpp +++ b/maxscale-system-test/testconnections.cpp @@ -966,6 +966,8 @@ bool TestConnections::replicate_from_master() } mysql_close(conn); + repl->execute_query_all_nodes("STOP SLAVE"); + /** Clean up MaxScale directories */ ssh_maxscale(true, "service maxscale stop"); prepare_binlog(); @@ -974,7 +976,6 @@ bool TestConnections::replicate_from_master() char log_file[256] = ""; char log_pos[256] = "4"; - repl->execute_query_all_nodes("STOP SLAVE"); repl->connect(); execute_query(repl->nodes[0], "RESET MASTER"); From 931ca0f497ffd2c5534fcd64846a635da8d4f25d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Sun, 30 Jul 2017 19:26:29 +0300 Subject: [PATCH 25/26] Stop timeout in avro_alter before fixing replication The resetting of the replication can take some time. By disabling the timeout during the reset, false positives are less likely. --- maxscale-system-test/avro_alter.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/maxscale-system-test/avro_alter.cpp b/maxscale-system-test/avro_alter.cpp index 3fdf0146b..2ee641875 100644 --- a/maxscale-system-test/avro_alter.cpp +++ b/maxscale-system-test/avro_alter.cpp @@ -65,6 +65,7 @@ int main(int argc, char *argv[]) free(rows); } + test.stop_timeout(); execute_query(test.repl->nodes[0], "DROP TABLE test.t1;RESET MASTER"); test.repl->fix_replication(); From 95c254a8b0f87f27e7370f290dc9e04e5a63c644 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Mon, 31 Jul 2017 15:45:59 +0300 Subject: [PATCH 26/26] Update release date --- Documentation/Release-Notes/MaxScale-2.1.5-Release-Notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/Release-Notes/MaxScale-2.1.5-Release-Notes.md b/Documentation/Release-Notes/MaxScale-2.1.5-Release-Notes.md index ab0fb6191..5c58eca86 100644 --- a/Documentation/Release-Notes/MaxScale-2.1.5-Release-Notes.md +++ b/Documentation/Release-Notes/MaxScale-2.1.5-Release-Notes.md @@ -1,4 +1,4 @@ -# MariaDB MaxScale 2.1.5 Release Notes +# MariaDB MaxScale 2.1.5 Release Notes -- 2017-07-31 Release 2.1.5 is a GA release.