From b99390f3e0b6a49982d6752d06a8e42a5b06c98c Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Mon, 3 Sep 2018 16:35:43 +0300 Subject: [PATCH 1/4] install Maxscale on all VMs in run_test_snapshot --- .../mdbci/run_test_snapshot.sh | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/maxscale-system-test/mdbci/run_test_snapshot.sh b/maxscale-system-test/mdbci/run_test_snapshot.sh index a3e6c643d..577163cf0 100755 --- a/maxscale-system-test/mdbci/run_test_snapshot.sh +++ b/maxscale-system-test/mdbci/run_test_snapshot.sh @@ -58,13 +58,23 @@ fi . ${script_dir}/set_env.sh "$name" -${mdbci_dir}/mdbci sudo --command 'yum remove maxscale -y' $name/maxscale -${mdbci_dir}/mdbci sudo --command 'yum clean all' $name/maxscale +if [ ${maxscale_N} -gt 1 ] ; then + maxscales_vm=`env | grep maxscale | grep network | sed 's/_network.*//' | grep "_"` +else + maxscales_vm="maxscale" +fi -${mdbci_dir}/mdbci setup_repo --product maxscale_ci --product-version ${target} $name/maxscale -${mdbci_dir}/mdbci install_product --product maxscale_ci $name/maxscale +for maxscale_vm_name in ${maxscales_vm} +do + ${mdbci_dir}/mdbci sudo --command 'yum remove maxscale -y' $name/${maxscale_vm_name} + ${mdbci_dir}/mdbci sudo --command 'yum clean all' $name/${maxscale_vm_name} + + ${mdbci_dir}/mdbci setup_repo --product maxscale_ci --product-version ${target} $name/${maxscale_vm_name} + ${mdbci_dir}/mdbci install_product --product maxscale_ci $name/${maxscale_vm_name} + + checkExitStatus $? "Error installing Maxscale" $snapshot_lock_file +done -checkExitStatus $? "Error installing Maxscale" $snapshot_lock_file cd ${script_dir}/.. From d7dd636c9a710881cca3368c5761fa4a7ced51eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Fri, 31 Aug 2018 13:48:55 +0300 Subject: [PATCH 2/4] Add table filtering to avrorouter The filtering is implemented with PCRE2 regular expressions and as such is not the most user-friendly interface. --- Documentation/Routers/Avrorouter.md | 15 ++++++++ server/modules/routing/avrorouter/avro.c | 38 ++++++++++++++++++- server/modules/routing/avrorouter/avro_rbr.c | 16 +++++++- .../modules/routing/avrorouter/avrorouter.h | 8 ++++ 4 files changed, 73 insertions(+), 4 deletions(-) diff --git a/Documentation/Routers/Avrorouter.md b/Documentation/Routers/Avrorouter.md index 311cf255e..5b8d2448c 100644 --- a/Documentation/Routers/Avrorouter.md +++ b/Documentation/Routers/Avrorouter.md @@ -66,6 +66,21 @@ _deflate_. These are the mandatory compression algorithms required by the Avro specification. For more information about the compression types, refer to the [Avro specification](https://avro.apache.org/docs/current/spec.html#Required+Codecs). +### `match` + +Only process events for tables that match this PCRE2 regular expression. See +[Regular Expressions](../Getting-Started/Configuration-Guide.md#regular-expressions) +for more information about regular expressions. + +This parameter was added in MaxScale 2.2.14. + +### `exclude` + +Ignore events for tables that match this PCRE2 regular expression. This can be +combined with the `match` parameter to implement table event filtering. + +This parameter was added in MaxScale 2.2.14. + **Note:** Since the 2.1 version of MaxScale, all of the router options can also be defined as parameters. diff --git a/server/modules/routing/avrorouter/avro.c b/server/modules/routing/avrorouter/avro.c index a2c4ed5aa..74bc6cfbb 100644 --- a/server/modules/routing/avrorouter/avro.c +++ b/server/modules/routing/avrorouter/avro.c @@ -44,6 +44,7 @@ #include #include #include +#include #include #ifndef BINLOG_NAMEFMT @@ -272,6 +273,8 @@ MXS_MODULE* MXS_CREATE_MODULE() {"start_index", MXS_MODULE_PARAM_COUNT, "1"}, {"block_size", MXS_MODULE_PARAM_SIZE, "0"}, {"codec", MXS_MODULE_PARAM_ENUM, "null", MXS_MODULE_OPT_ENUM_UNIQUE, codec_values}, + {"match", MXS_MODULE_PARAM_REGEX}, + {"exclude", MXS_MODULE_PARAM_REGEX}, {MXS_END_MODULE_PARAMS} } }; @@ -481,6 +484,18 @@ createInstance(SERVICE *service, char **options) AVRO_INSTANCE *inst; int i; + MXS_CONFIG_PARAMETER *params = service->svc_config_param; + pcre2_code* match = config_get_compiled_regex(params, "match", 0, NULL); + pcre2_code* exclude = config_get_compiled_regex(params, "exclude", 0, NULL); + pcre2_match_data* md_match = NULL; + pcre2_match_data* md_exclude = NULL; + + if ((match && (md_match = pcre2_match_data_create_from_pattern(match, NULL)) == NULL) || + (exclude && (md_exclude = pcre2_match_data_create_from_pattern(exclude, NULL)) == NULL)) + { + return NULL; + } + if ((inst = MXS_CALLOC(1, sizeof(AVRO_INSTANCE))) == NULL) { return NULL; @@ -502,8 +517,6 @@ createInstance(SERVICE *service, char **options) inst->trx_count = 0; inst->binlogdir = NULL; - MXS_CONFIG_PARAMETER *params = service->svc_config_param; - inst->avrodir = MXS_STRDUP_A(config_get_string(params, "avrodir")); inst->fileroot = MXS_STRDUP_A(config_get_string(params, "filestem")); inst->row_target = config_get_integer(params, "group_rows"); @@ -511,6 +524,10 @@ createInstance(SERVICE *service, char **options) inst->codec = config_get_enum(params, "codec", codec_values); int first_file = config_get_integer(params, "start_index"); inst->block_size = config_get_size(params, "block_size"); + inst->match = match; + inst->exclude = exclude; + inst->md_match = md_match; + inst->md_exclude = md_exclude; MXS_CONFIG_PARAMETER *param = config_get_param(params, "source"); inst->gtid.domain = 0; @@ -1284,3 +1301,20 @@ static bool ensure_dir_ok(const char* path, int mode) return rval; } + +bool table_matches(AVRO_INSTANCE* inst, const char* ident) +{ + bool rval = false; + + if (!inst->match || pcre2_match(inst->match, (PCRE2_SPTR)ident, PCRE2_ZERO_TERMINATED, + 0, 0, inst->md_match, NULL) > 0) + { + if (!inst->exclude || pcre2_match(inst->exclude, (PCRE2_SPTR)ident, PCRE2_ZERO_TERMINATED, + 0, 0, inst->md_exclude, NULL) == PCRE2_ERROR_NOMATCH) + { + rval = true; + } + } + + return rval; +} diff --git a/server/modules/routing/avrorouter/avro_rbr.c b/server/modules/routing/avrorouter/avro_rbr.c index 38d57ed07..83d8110db 100644 --- a/server/modules/routing/avrorouter/avro_rbr.c +++ b/server/modules/routing/avrorouter/avro_rbr.c @@ -100,6 +100,12 @@ bool handle_table_map_event(AVRO_INSTANCE *router, REP_HEADER *hdr, uint8_t *ptr int ev_len = router->event_type_hdr_lens[hdr->event_type]; read_table_info(ptr, ev_len, &id, table_ident, sizeof(table_ident)); + + if (!table_matches(router, table_ident)) + { + return true; + } + TABLE_CREATE* create = hashtable_fetch(router->created_tables, table_ident); if (create) @@ -292,6 +298,12 @@ bool handle_row_event(AVRO_INSTANCE *router, REP_HEADER *hdr, uint8_t *ptr) { char table_ident[MYSQL_TABLE_MAXLEN + MYSQL_DATABASE_MAXLEN + 2]; snprintf(table_ident, sizeof(table_ident), "%s.%s", map->database, map->table); + + if (!table_matches(router, table_ident)) + { + return true; + } + AVRO_TABLE* table = hashtable_fetch(router->open_tables, table_ident); TABLE_CREATE* create = map->table_create; ss_dassert(hashtable_fetch(router->created_tables, table_ident) == create); @@ -369,8 +381,8 @@ bool handle_row_event(AVRO_INSTANCE *router, REP_HEADER *hdr, uint8_t *ptr) } else { - MXS_ERROR("Row event for unknown table mapped to ID %lu. Data will not " - "be processed.", table_id); + MXS_INFO("Row event for unknown table mapped to ID %lu. Data will not " + "be processed.", table_id); } return rval; diff --git a/server/modules/routing/avrorouter/avrorouter.h b/server/modules/routing/avrorouter/avrorouter.h index 67029e1c5..3bcf6202a 100644 --- a/server/modules/routing/avrorouter/avrorouter.h +++ b/server/modules/routing/avrorouter/avrorouter.h @@ -303,6 +303,12 @@ typedef struct avro_instance * a flush of all tables */ uint64_t block_size; /**< Avro datablock size */ enum mxs_avro_codec_type codec; /**< Avro codec type, defaults to `null` */ + + /** Match and exclude patterns for tables */ + pcre2_code* match; + pcre2_code* exclude; + pcre2_match_data* md_match; + pcre2_match_data* md_exclude; struct avro_instance *next; } AVRO_INSTANCE; @@ -355,6 +361,8 @@ extern void avro_flush_all_tables(AVRO_INSTANCE *router, enum avrorouter_file_op #define AVRO_CS_BUSY 0x0001 #define AVRO_WAIT_DATA 0x0002 +bool table_matches(AVRO_INSTANCE* inst, const char* ident); + MXS_END_DECLS #endif From 15eba3ac7509245fbedbcf9c24bb378af087b281 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Sat, 1 Sep 2018 00:00:46 +0300 Subject: [PATCH 3/4] Add 2.2.14 release notes --- .../MaxScale-2.2.14-Release-Notes.md | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 Documentation/Release-Notes/MaxScale-2.2.14-Release-Notes.md diff --git a/Documentation/Release-Notes/MaxScale-2.2.14-Release-Notes.md b/Documentation/Release-Notes/MaxScale-2.2.14-Release-Notes.md new file mode 100644 index 000000000..55a0de2a5 --- /dev/null +++ b/Documentation/Release-Notes/MaxScale-2.2.14-Release-Notes.md @@ -0,0 +1,51 @@ +# MariaDB MaxScale 2.2.14 Release Notes + +Release 2.2.14 is a GA release. + +This document describes the changes in release 2.2.14, when compared to +release 2.2.13. + +For any problems you encounter, please consider submitting a bug +report on [our Jira](https://jira.mariadb.org/projects/MXS). + +## New Features + +### Avrorouter Table Filtering + +The tables that the avrorouter processes can now be filtered with two new +parameters: [`match`](../Routers/Avrorouter.md#match) and +[`exclude`](../Routers/Avrorouter.md#exclude). + +## Bug fixes + +* [MXS-2034](https://jira.mariadb.org/browse/MXS-2034) query_retry_timeout was not set +* [MXS-2027](https://jira.mariadb.org/browse/MXS-2027) LOAD DATA LOCAL INFILE is not ignored by protocol modules +* [MXS-2024](https://jira.mariadb.org/browse/MXS-2024) Crash in reauthenticate_client +* [MXS-2019](https://jira.mariadb.org/browse/MXS-2019) All atexit handlers aren't called +* [MXS-2015](https://jira.mariadb.org/browse/MXS-2015) CDC Connector ignores errors after registration is successful +* [MXS-2007](https://jira.mariadb.org/browse/MXS-2007) Fatal: MaxScale 2.2.13 received fatal signal 11 (Aurora Monitor) +* [MXS-1999](https://jira.mariadb.org/browse/MXS-1999) Invalid null relationship handling in REST API +* [MXS-1996](https://jira.mariadb.org/browse/MXS-1996) Avrorouter writes misleading error messages to the log +* [MXS-1880](https://jira.mariadb.org/browse/MXS-1880) MaxScale 2.2.5-1 Crashes after a non clean Start +* [MXS-1736](https://jira.mariadb.org/browse/MXS-1736) Clarify the usage of maxpasswd in documentation +* [MXS-1735](https://jira.mariadb.org/browse/MXS-1735) Clarify SSL documentation + +## Known Issues and Limitations + +There are some limitations and known issues within this version of MaxScale. +For more information, please refer to the [Limitations](../About/Limitations.md) document. + +## Packaging + +RPM and Debian packages are provided for supported the Linux distributions. + +Packages can be downloaded [here](https://mariadb.com/downloads/mariadb-tx/maxscale). + +## Source Code + +The source code of MaxScale is tagged at GitHub with a tag, which is identical +with the version of MaxScale. For instance, the tag of version X.Y.Z of MaxScale +is `maxscale-X.Y.Z`. Further, the default branch is always the latest GA version +of MaxScale. + +The source code is available [here](https://github.com/mariadb-corporation/MaxScale). From 2e90e77baac5283abe031addd51835285b127c84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Sat, 1 Sep 2018 11:21:24 +0300 Subject: [PATCH 4/4] MXS-2027: Set load to inactive on error The data loading should be set to inactive if an error is encountered, not to active. --- server/modules/routing/readwritesplit/readwritesplit.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/modules/routing/readwritesplit/readwritesplit.cc b/server/modules/routing/readwritesplit/readwritesplit.cc index 4d518f9a0..2abb6a608 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.cc +++ b/server/modules/routing/readwritesplit/readwritesplit.cc @@ -1204,7 +1204,7 @@ static void clientReply(MXS_ROUTER *instance, { // Server responded with an error to the LOAD DATA LOCAL INFILE rses->load_data_state = LOAD_DATA_INACTIVE; - session_set_load_active(backend_dcb->session, true); + session_set_load_active(backend_dcb->session, false); } if (backend->get_reply_state() == REPLY_STATE_DONE)