diff --git a/BUILD/build_deb_local.sh b/BUILD/build_deb_local.sh index f53e2d8ba..a063fc339 100755 --- a/BUILD/build_deb_local.sh +++ b/BUILD/build_deb_local.sh @@ -17,7 +17,7 @@ make if [[ "$cmake_flags" =~ "BUILD_TESTS" ]] then # All tests must pass otherwise the build is considered a failure - make test || exit 1 + ctest --output-on-failure || exit 1 fi export LD_LIBRARY_PATH=$(for i in `find $PWD/ -name '*.so*'`; do echo $(dirname $i); done|sort|uniq|xargs|sed -e 's/[[:space:]]/:/g') diff --git a/BUILD/build_rpm_local.sh b/BUILD/build_rpm_local.sh index 629a3b1ba..78e2d018f 100755 --- a/BUILD/build_rpm_local.sh +++ b/BUILD/build_rpm_local.sh @@ -15,7 +15,7 @@ make if [[ "$cmake_flags" =~ "BUILD_TESTS" ]] then # All tests must pass otherwise the build is considered a failure - make test || exit 1 + ctest --output-on-failure || exit 1 fi if [ $remove_strip == "yes" ] ; then diff --git a/Documentation/Release-Notes/MaxScale-2.1.13-Release-Notes.md b/Documentation/Release-Notes/MaxScale-2.1.13-Release-Notes.md index 6888bb079..a6e6acd1b 100644 --- a/Documentation/Release-Notes/MaxScale-2.1.13-Release-Notes.md +++ b/Documentation/Release-Notes/MaxScale-2.1.13-Release-Notes.md @@ -34,6 +34,7 @@ For any problems you encounter, please consider submitting a bug report at * [MXS-1581](https://jira.mariadb.org/browse/MXS-1581) CREATE TABLE AS not supported * [MXS-1580](https://jira.mariadb.org/browse/MXS-1580) Invalid handling of BIT values * [MXS-1527](https://jira.mariadb.org/browse/MXS-1527) SELECT with session var is not supported +* [MXS-1516](https://jira.mariadb.org/browse/MXS-1516) existing connection don't change routing even if master switched ## Packaging diff --git a/Documentation/Routers/Avrorouter.md b/Documentation/Routers/Avrorouter.md index dbdeacacd..aa51b884d 100644 --- a/Documentation/Routers/Avrorouter.md +++ b/Documentation/Routers/Avrorouter.md @@ -197,6 +197,18 @@ Start or stop the binary log to Avro conversion. The first parameter is the name of the service to stop and the second parameter tells whether to start the conversion process or to stop it. +### `avrorouter::purge SERVICE` + +This command will delete all files created by the avrorouter. This includes all +.avsc schema files and .avro data files as well as the internal state tracking +files. Use this to completely reset the conversion process. + +**Note:** Once the command has completed, MaxScale must be restarted to restart +the conversion process. Issuing a `convert start` command **will not work**. + +**WARNING:** You will lose any and all converted data when this command is + executed. + # Files Created by the Avrorouter The avrorouter creates two files in the location pointed by _avrodir_: @@ -205,6 +217,14 @@ the locations of the GTIDs in the .avro files. The _avro-conversion.ini_ contain the last converted position and GTID in the binlogs. If you need to reset the conversion process, delete these two files and restart MaxScale. +# Resetting the Conversion Process + +To reset the binlog conversion process, issue the `purge` module command by +executing it via MaxAdmin and stop MaxScale. If manually created schema files +were used, they need to be recreated once MaxScale is stopped. After stopping +MaxScale and optionally creating the schema files, the conversion process can be +started by starting MaxScale. + # Example Client The avrorouter comes with an example client program, _cdc.py_, written in Python 3. diff --git a/maxscale-system-test/CMakeLists.txt b/maxscale-system-test/CMakeLists.txt index 85908b8ae..22283e486 100644 --- a/maxscale-system-test/CMakeLists.txt +++ b/maxscale-system-test/CMakeLists.txt @@ -556,6 +556,14 @@ add_test_executable(mxs1476.cpp mxs1476 mxs1476 LABELS GALERA_BACKEND) # https://jira.mariadb.org/browse/MXS-1509 add_test_executable(mxs1509.cpp mxs1509 mxs1509 LABELS REPL_BACKEND) + # MXS-1516: existing connection don't change routing, even if master switched + # https://jira.mariadb.org/browse/MXS-1516 +add_test_executable(mxs1516.cpp mxs1516 replication LABELS REPL_BACKEND) + + # MXS-1542: Check that UTF16 strings work + # https://jira.mariadb.org/browse/MXS-1542 +add_test_executable(mxs1542.cpp mxs1542 avro LABELS REPL_BACKEND) + # MXS-1585: Crash in MaxScale 2.1.12 # https://jira.mariadb.org/browse/MXS-1585 add_test_executable(mxs1585.cpp mxs1585 mxs1585 LABELS REPL_BACKEND) diff --git a/maxscale-system-test/mxs1516.cpp b/maxscale-system-test/mxs1516.cpp new file mode 100644 index 000000000..c1081b4e2 --- /dev/null +++ b/maxscale-system-test/mxs1516.cpp @@ -0,0 +1,26 @@ +/** + * MXS-1516: existing connection don't change routing, even if master switched + * + * https://jira.mariadb.org/browse/MXS-1516 + */ + +#include "testconnections.h" + +int main(int argc, char** argv) +{ + TestConnections test(argc, argv); + + test.maxscales->connect(); + test.try_query(test.maxscales->conn_master[0], "SELECT 1"); + + // Change master mid-session + test.repl->connect(); + test.repl->change_master(1, 0); + + test.add_result(execute_query_silent(test.maxscales->conn_master[0], "SELECT 1") == 0, "Query should fail"); + + // Change the master back to the original one + test.repl->change_master(0, 1); + + return test.global_result; +} diff --git a/maxscale-system-test/mxs1542.cpp b/maxscale-system-test/mxs1542.cpp new file mode 100644 index 000000000..b5e3cde56 --- /dev/null +++ b/maxscale-system-test/mxs1542.cpp @@ -0,0 +1,40 @@ +/** + * MXS-1542: https://jira.mariadb.org/browse/MXS-1542 + * + * Check that UTF16 strings work. + */ + +#include "testconnections.h" + +int main(int argc, char** argv) +{ + TestConnections::skip_maxscale_start(true); + TestConnections::check_nodes(false); + TestConnections test(argc, argv); + + test.replicate_from_master(); + + test.repl->connect(); + execute_query(test.repl->nodes[0], + "CREATE OR REPLACE TABLE t1 (data varchar(30) NOT NULL) DEFAULT CHARSET=utf16"); + execute_query(test.repl->nodes[0], + "INSERT INTO t1 VALUES ('Hello World'), ('Բարեւ աշխարհ'), ('こんにちは世界'), ('你好，世界'), ('Привет мир')"); + + // Wait for the data to be processed + const char* logmsg = "Waiting until more data is written"; + test.maxscales->ssh_node_f(0, true, + "for ((i=0;i<15;i++)); do grep '%s' /var/log/maxscale/maxscale.log && break || sleep 1; done", logmsg); + + // Check if the Avro file contains the inserted value + int rc = test.maxscales->ssh_node_f(0, true, + "maxavrocheck -d /var/lib/maxscale/avro/test.t1.000001.avro|grep 'Hello World'"); + test.add_result(rc == 0, "Data is converted when a failure to convert is expected"); + + printf("\n" + "o-------------------------------------------------------------------o\n" + "|The test is expected to fail, change it when the MXS-1542 is fixed.|\n" + "o-------------------------------------------------------------------o\n" + "\n"); + + return test.global_result; +} diff --git a/server/core/skygw_utils.cc b/server/core/skygw_utils.cc index e05755246..262db6889 100644 --- a/server/core/skygw_utils.cc +++ b/server/core/skygw_utils.cc @@ -29,7 +29,6 @@ #include #include "internal/skygw_utils.h" #include -#include #include #if !defined(PATH_MAX) @@ -460,7 +459,7 @@ void acquire_lock(int* l) misscount += 1; if (misscount > 10) { - ts1.tv_nsec = (random_jkiss() % misscount) * 1000000; + ts1.tv_nsec = misscount * 1000000; nanosleep(&ts1, NULL); } } diff --git a/server/modules/routing/avrorouter/avro.c b/server/modules/routing/avrorouter/avro.c index ba5f3f127..a637ef6eb 100644 --- a/server/modules/routing/avrorouter/avro.c +++ b/server/modules/routing/avrorouter/avro.c @@ -26,9 +26,10 @@ #include #include #include - -#include - +#include +#include +#include +#include #include #include #include @@ -43,6 +44,7 @@ #include #include #include +#include #ifndef BINLOG_NAMEFMT #define BINLOG_NAMEFMT "%s.%06d" @@ -111,6 +113,7 @@ bool avro_handle_convert(const MODULECMD_ARG *args, json_t** output) return rval; } + static const MXS_ENUM_VALUE codec_values[] = { {"null", MXS_AVRO_CODEC_NULL}, @@ -120,6 +123,70 @@ static const MXS_ENUM_VALUE codec_values[] = {NULL} }; +static bool do_unlink(const char* format, ...) +{ + va_list args; + va_start(args, format); + + char filename[PATH_MAX + 1]; + vsnprintf(filename, sizeof(filename), format, args); + + va_end(args); + + int rc = unlink(filename); + return rc == 0 || rc == ENOENT; +} + +static bool do_unlink_with_pattern(const char* format, ...) +{ + bool rval = true; + va_list args; + va_start(args, format); + + char filename[PATH_MAX + 1]; + vsnprintf(filename, sizeof(filename), format, args); + + va_end(args); + + glob_t g; + int rc = glob(filename, 0, NULL, &g); + + if (rc == 0) + { + for (size_t i = 0; i < g.gl_pathc; i++) + { + if (!do_unlink("%s", g.gl_pathv[i])) + { + rval = false; + } + } + } + else if (rc != GLOB_NOMATCH) + { + modulecmd_set_error("Failed to search '%s': %d, %s", + filename, errno, mxs_strerror(errno)); + rval = false; + } + + globfree(&g); + + return rval; +} + +static bool avro_handle_purge(const MODULECMD_ARG *args, json_t** output) +{ + AVRO_INSTANCE* inst = (AVRO_INSTANCE*)args->argv[0].value.service->router_instance; + + // First stop the conversion service + conversion_task_ctl(inst, false); + + // Then delete the files + return do_unlink("%s/%s", inst->avrodir, AVRO_PROGRESS_FILE) && // State file + do_unlink("/%s/%s", inst->avrodir, avro_index_name) && // Index database + do_unlink_with_pattern("/%s/*.avro", inst->avrodir) && // .avro files + do_unlink_with_pattern("/%s/*.avsc", inst->avrodir); // .avsc files +} + /** * The module entry point routine. It is this routine that * must populate the structure that is referred to as the @@ -133,15 +200,27 @@ MXS_MODULE* MXS_CREATE_MODULE() spinlock_init(&instlock); instances = NULL; - static modulecmd_arg_type_t args[] = + static modulecmd_arg_type_t args_convert[] = { { MODULECMD_ARG_SERVICE | MODULECMD_ARG_NAME_MATCHES_DOMAIN, "The avrorouter service" }, { MODULECMD_ARG_STRING, "Action, whether to 'start' or 'stop' the conversion process" } }; modulecmd_register_command(MXS_MODULE_NAME, "convert", MODULECMD_TYPE_ACTIVE, - avro_handle_convert, 2, args, + avro_handle_convert, 2, args_convert, "Start or stop the binlog to avro conversion process"); + static modulecmd_arg_type_t args_purge[] = + { + { + MODULECMD_ARG_SERVICE | MODULECMD_ARG_NAME_MATCHES_DOMAIN, + "The avrorouter service to purge (NOTE: THIS REMOVES ALL CONVERTED FILES)" + } + }; + modulecmd_register_command(MXS_MODULE_NAME, "purge", MODULECMD_TYPE_ACTIVE, + avro_handle_purge, 1, args_purge, + "Purge created Avro files and reset conversion state. " + "NOTE: MaxScale must be restarted after this call."); + static MXS_ROUTER_OBJECT MyObject = { createInstance, diff --git a/server/modules/routing/avrorouter/avro_schema.c b/server/modules/routing/avrorouter/avro_schema.c index 95c6f5534..f92a254db 100644 --- a/server/modules/routing/avrorouter/avro_schema.c +++ b/server/modules/routing/avrorouter/avro_schema.c @@ -123,10 +123,13 @@ char* json_new_schema_from_table(TABLE_MAP *map) json_array_append_new(array, json_pack_ex(&err, 0, "{s:s, s:o}", "name", avro_event_type, "type", event_types)); - for (uint64_t i = 0; i < map->columns; i++) + for (uint64_t i = 0; i < map->columns && i < create->columns; i++) { ss_info_dassert(create->column_names[i] && *create->column_names[i], "Column name should not be empty or NULL"); + ss_info_dassert(create->column_types[i] && *create->column_types[i], + "Column type should not be empty or NULL"); + json_array_append_new(array, json_pack_ex(&err, 0, "{s:s, s:s, s:s, s:i}", "name", create->column_names[i], "type", column_type_to_avro_type(map->column_types[i]), diff --git a/server/modules/routing/readconnroute/readconnroute.c b/server/modules/routing/readconnroute/readconnroute.c index dc5690af5..daad7121f 100644 --- a/server/modules/routing/readconnroute/readconnroute.c +++ b/server/modules/routing/readconnroute/readconnroute.c @@ -507,7 +507,7 @@ closeSession(MXS_ROUTER *instance, MXS_ROUTER_SESSION *router_session) /** Log routing failure due to closed session */ static void log_closed_session(mxs_mysql_cmd_t mysql_command, bool is_closed, - SERVER_REF *ref) + SERVER_REF *ref, bool valid) { char msg[MAX_SERVER_ADDRESS_LEN + 200] = ""; // Extra space for message @@ -523,11 +523,44 @@ static void log_closed_session(mxs_mysql_cmd_t mysql_command, bool is_closed, { sprintf(msg, "Server '%s' is in maintenance.", ref->server->unique_name); } + else if (!valid) + { + sprintf(msg, "Server '%s' no longer qualifies as a target server.", + ref->server->unique_name); + } MXS_ERROR("Failed to route MySQL command %d to backend server. %s", mysql_command, msg); } +/** + * Check if the server we're connected to is still valid + * + * @param inst Router instance + * @param router_cli_ses Router session + * + * @return True if the backend connection is still valid + */ +static inline bool connection_is_valid(ROUTER_INSTANCE* inst, ROUTER_CLIENT_SES* router_cli_ses) +{ + bool rval = false; + + if (SERVER_IS_RUNNING(router_cli_ses->backend->server) && + (router_cli_ses->backend->server->status & inst->bitmask & inst->bitvalue)) + { + if (inst->bitvalue & SERVER_MASTER) + { + rval = router_cli_ses->backend == get_root_master(inst->service->dbref); + } + else + { + rval = true; + } + } + + return rval; +} + /** * We have data from the client, we must route it to the backend. * This is simply a case of sending it to the connection that was @@ -571,10 +604,12 @@ routeQuery(MXS_ROUTER *instance, MXS_ROUTER_SESSION *router_session, GWBUF *queu rses_end_locked_router_action(router_cli_ses); } + bool valid; + if (rses_is_closed || backend_dcb == NULL || - !SERVER_IS_RUNNING(router_cli_ses->backend->server)) + (valid = !connection_is_valid(inst, router_cli_ses))) { - log_closed_session(mysql_command, rses_is_closed, router_cli_ses->backend); + log_closed_session(mysql_command, rses_is_closed, router_cli_ses->backend, valid); gwbuf_free(queue); goto return_rc;