From 77f44ba92bcb025309ab53a95afd1acc8a5c3e00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Tue, 20 Jun 2017 13:36:53 +0300 Subject: [PATCH] Add missing match/exclude functionality to tee The tee filter was missing the match/exclude functionality that was in the previous implementation. The filter was upgraded to use PCRE2 regular expressions instead of POSIX regular expressions. Documented that the match/exclude patterns should use PCRE2 syntax. --- Documentation/Filters/Tee-Filter.md | 76 +++++++++---------- .../MaxScale-2.2.0-Release-Notes.md | 4 + .../Upgrading/Upgrading-To-MaxScale-2.2.md | 4 +- server/modules/filter/tee/tee.cc | 72 +++++++----------- server/modules/filter/tee/tee.hh | 27 +++++-- server/modules/filter/tee/teesession.cc | 71 ++++++++++++++++- server/modules/filter/tee/teesession.hh | 12 ++- 7 files changed, 164 insertions(+), 102 deletions(-) diff --git a/Documentation/Filters/Tee-Filter.md b/Documentation/Filters/Tee-Filter.md index 72e7f4424..99fb0eec8 100644 --- a/Documentation/Filters/Tee-Filter.md +++ b/Documentation/Filters/Tee-Filter.md @@ -31,9 +31,41 @@ passwd=mypasswd filters=DataMartFilter ``` -## Filter Options +## Filter Parameters -The tee filter accepts the following options. +The tee filter requires a mandatory parameter to define the service to replicate +statements to and accepts a number of optional parameters. + +### `match` + +An optional parameter used to limit the queries that will be replicated by the +tee filter. The parameter value is a PCRE2 regular expression that is used to +match against the SQL text. Only SQL statements that match the text passed as +the value of this parameter will be sent to the service defined in the filter +section. + +``` +match=/insert.*into.*order*/ +``` + +### `exclude` + +An optional parameter used to limit the queries that will be replicated by the +tee filter. The parameter value is a PCRE2 regular expression that is used to +match against the SQL text. Any SQL statements that match the text passed as the +value of this parameter will be excluded from the replication stream. + +``` +exclude=/select.*from.*t1/ +``` + +If both `match` and `exclude` parameters are defined, `exclude` takes +precedence. + +### `options` + +The options parameter controls the regular expression options. The following +options are accepted. |Option |Description | |----------|--------------------------------------------| @@ -47,43 +79,7 @@ To use multiple filter options, list them in a comma-separated list. options=case,extended ``` -## Filter Parameters - -The tee filter requires a mandatory parameter to define the service to replicate -statements to and accepts a number of optional parameters. - -### Match - -An optional parameter used to limit the queries that will be replicated by the -tee filter. The parameter value is a regular expression that is used to match -against the SQL text. Only SQL statements that matches the text passed as the -value of this parameter will be sent to the service defined in the filter -section. - -``` -match=insert.*into.*order* -``` - -All regular expressions are evaluated with the option to ignore the case of the -text, therefore a match option of select will match both insert, INSERT and any -form of the word with upper or lowercase characters. - -### Exclude - -An optional parameter used to limit the queries that will be replicated by the -tee filter. The parameter value is a regular expression that is used to match -against the SQL text. SQL statements that match the text passed as the value of -this parameter will be excluded from the replication stream. - -``` -exclude=select -``` - -All regular expressions are evaluated with the option to ignore the case of the -text, therefore an exclude option of select will exclude statements that contain -both select, SELECT or any form of the word with upper or lowercase characters. - -### Source +### `source` The optional source parameter defines an address that is used to match against the address from which the client connection to MariaDB MaxScale originates. @@ -93,7 +89,7 @@ Only sessions that originate from this address will be replicated. source=127.0.0.1 ``` -### User +### `user` The optional user parameter defines a user name that is used to match against the user from which the client connection to MariaDB MaxScale originates. Only diff --git a/Documentation/Release-Notes/MaxScale-2.2.0-Release-Notes.md b/Documentation/Release-Notes/MaxScale-2.2.0-Release-Notes.md index c36ad58e7..9e2ef5826 100644 --- a/Documentation/Release-Notes/MaxScale-2.2.0-Release-Notes.md +++ b/Documentation/Release-Notes/MaxScale-2.2.0-Release-Notes.md @@ -50,6 +50,10 @@ In addition to the aforementioned requirements, a failure to create a branched session no longer causes the actual client session to be closed. In most cases, this is desired behavior. +The `match` and `exclude` parameters were changed to use PCRE2 syntax for the +regular expressions. The regular expression should be enclosed by slashes +e.g. `match=/select.*from.*test/`. + ## Dropped Features ### MaxAdmin diff --git a/Documentation/Upgrading/Upgrading-To-MaxScale-2.2.md b/Documentation/Upgrading/Upgrading-To-MaxScale-2.2.md index ced87f3e5..1eee3012f 100644 --- a/Documentation/Upgrading/Upgrading-To-MaxScale-2.2.md +++ b/Documentation/Upgrading/Upgrading-To-MaxScale-2.2.md @@ -15,6 +15,6 @@ file. ### Regular Expression Parameters Modules may now use a built-in regular expression string parameter type instead -of a normal string when accepting patterns. The only module using the new regex -parameter type is currently *QLAFilter*. When inputting pattern, enclose the +of a normal string when accepting patterns. The modules that use the new regex +parameter type are *qlafilter* and *tee*. When inputting pattern, enclose the string in slashes, e.g. `match=/^select/` defines the pattern `^select`. diff --git a/server/modules/filter/tee/tee.cc b/server/modules/filter/tee/tee.cc index d92fb2c70..5aa508c25 100644 --- a/server/modules/filter/tee/tee.cc +++ b/server/modules/filter/tee/tee.cc @@ -29,31 +29,23 @@ static const MXS_ENUM_VALUE option_values[] = { - {"ignorecase", REG_ICASE}, - {"case", 0}, - {"extended", REG_EXTENDED}, + {"ignorecase", PCRE2_CASELESS}, + {"case", 0}, + {"extended", PCRE2_EXTENDED}, {NULL} }; -Tee::Tee(SERVICE* service, const char* user, const char* remote, - const char* match, const char* nomatch, int cflags): +Tee::Tee(SERVICE* service, std::string user, std::string remote, + pcre2_code* match, std::string match_string, + pcre2_code* exclude, std::string exclude_string): m_service(service), m_user(user), m_source(remote), - m_match(match), - m_nomatch(nomatch) + m_match_code(match), + m_exclude_code(exclude), + m_match(match_string), + m_exclude(exclude_string) { - if (*match) - { - ss_debug(int rc = )regcomp(&m_re, match, cflags); - ss_dassert(rc == 0); - } - - if (*nomatch) - { - ss_debug(int rc = )regcomp(&m_nore, nomatch, cflags); - ss_dassert(rc == 0); - } } /** @@ -68,34 +60,22 @@ Tee::Tee(SERVICE* service, const char* user, const char* remote, */ Tee* Tee::create(const char *name, char **options, MXS_CONFIG_PARAMETER *params) { - Tee *my_instance = NULL; - SERVICE* service = config_get_service(params, "service"); const char* source = config_get_string(params, "source"); const char* user = config_get_string(params, "user"); - const char* match = config_get_string(params, "match"); - const char* nomatch = config_get_string(params, "exclude"); + uint32_t cflags = config_get_enum(params, "options", option_values); + pcre2_code* match = config_get_compiled_regex(params, "match", cflags, NULL); + pcre2_code* exclude = config_get_compiled_regex(params, "exclude", cflags, NULL); + const char* match_str = config_get_string(params, "match"); + const char* exclude_str = config_get_string(params, "exclude"); - int cflags = config_get_enum(params, "options", option_values); - regex_t re; - regex_t nore; + Tee* my_instance = new (std::nothrow) Tee(service, source, user, match, + match_str, exclude, exclude_str); - if (*match && regcomp(&re, match, cflags) != 0) + if (my_instance == NULL) { - MXS_ERROR("Invalid regular expression '%s' for the match parameter.", match); - } - else if (*nomatch && regcomp(&nore, nomatch, cflags) != 0) - { - MXS_ERROR("Invalid regular expression '%s' for the nomatch parameter.", nomatch); - - if (*match) - { - regfree(&re); - } - } - else - { - my_instance = new (std::nothrow) Tee(service, source, user, match, nomatch, cflags); + pcre2_code_free(match); + pcre2_code_free(exclude); } return my_instance; @@ -136,10 +116,10 @@ void Tee::diagnostics(DCB *dcb) dcb_printf(dcb, "\t\tInclude queries that match %s\n", m_match.c_str()); } - if (m_nomatch.c_str()) + if (m_exclude.c_str()) { dcb_printf(dcb, "\t\tExclude queries that match %s\n", - m_nomatch.c_str()); + m_exclude.c_str()); } } @@ -174,9 +154,9 @@ json_t* Tee::diagnostics_json() const json_object_set_new(rval, "match", json_string(m_match.c_str())); } - if (m_nomatch.length()) + if (m_exclude.length()) { - json_object_set_new(rval, "exclude", json_string(m_nomatch.c_str())); + json_object_set_new(rval, "exclude", json_string(m_exclude.c_str())); } return rval; @@ -210,8 +190,8 @@ MXS_MODULE* MXS_CREATE_MODULE() NULL, /* Thread finish. */ { {"service", MXS_MODULE_PARAM_SERVICE, NULL, MXS_MODULE_OPT_REQUIRED}, - {"match", MXS_MODULE_PARAM_STRING}, - {"exclude", MXS_MODULE_PARAM_STRING}, + {"match", MXS_MODULE_PARAM_REGEX}, + {"exclude", MXS_MODULE_PARAM_REGEX}, {"source", MXS_MODULE_PARAM_STRING}, {"user", MXS_MODULE_PARAM_STRING}, { diff --git a/server/modules/filter/tee/tee.hh b/server/modules/filter/tee/tee.hh index df096583c..a17e65309 100644 --- a/server/modules/filter/tee/tee.hh +++ b/server/modules/filter/tee/tee.hh @@ -57,15 +57,26 @@ public: return m_service; } + pcre2_code* get_match() const + { + return m_match_code; + } + + pcre2_code* get_exclude() const + { + return m_exclude_code; + } + private: - Tee(SERVICE* service, const char* user, const char* remote, - const char* match, const char* nomatch, int cflags); + Tee(SERVICE* service, std::string user, std::string remote, + pcre2_code* match, std::string match_string, + pcre2_code* exclude, std::string exclude_string); SERVICE* m_service; - std::string m_user; /* The user name to filter on */ - std::string m_source; /* The source of the client connection */ - std::string m_match; /* Optional text to match against */ - std::string m_nomatch; /* Optional text to match against for exclusion */ - regex_t m_re; /* Compiled regex text */ - regex_t m_nore; /* Compiled regex nomatch text */ + std::string m_user; /* The user name to filter on */ + std::string m_source; /* The source of the client connection */ + pcre2_code* m_match_code; /* Compiled match pattern */ + pcre2_code* m_exclude_code; /* Compiled exclude pattern*/ + std::string m_match; /* Pattern for matching queries */ + std::string m_exclude; /* Pattern for excluding queries */ }; diff --git a/server/modules/filter/tee/teesession.cc b/server/modules/filter/tee/teesession.cc index 59a8bef78..0274ddfc5 100644 --- a/server/modules/filter/tee/teesession.cc +++ b/server/modules/filter/tee/teesession.cc @@ -17,6 +17,8 @@ #include #include +#include + /** * Detect loops in the filter chain. */ @@ -57,9 +59,15 @@ bool recursive_tee_usage(std::set& services, SERVICE* service) return false; } -TeeSession::TeeSession(MXS_SESSION* session, LocalClient* client): +TeeSession::TeeSession(MXS_SESSION* session, LocalClient* client, + pcre2_code* match, pcre2_match_data* md_match, + pcre2_code* exclude, pcre2_match_data* md_exclude): mxs::FilterSession(session), - m_client(client) + m_client(client), + m_match(match), + m_md_match(md_match), + m_exclude(exclude), + m_md_exclude(md_exclude) { } @@ -75,17 +83,43 @@ TeeSession* TeeSession::create(Tee* my_instance, MXS_SESSION* session) } LocalClient* client = NULL; + pcre2_code* match = NULL; + pcre2_code* exclude = NULL; + pcre2_match_data* md_match = NULL; + pcre2_match_data* md_exclude = NULL; if (my_instance->user_matches(session_get_user(session)) && my_instance->remote_matches(session_get_remote(session))) { + match = my_instance->get_match(); + exclude = my_instance->get_exclude(); + + if ((match && (md_match = pcre2_match_data_create_from_pattern(match, NULL)) == NULL) || + (exclude && (md_exclude = pcre2_match_data_create_from_pattern(exclude, NULL)) == NULL)) + { + return NULL; + } + if ((client = LocalClient::create(session, my_instance->get_service())) == NULL) { return NULL; } } - return new (std::nothrow) TeeSession(session, client); + TeeSession* tee = new (std::nothrow) TeeSession(session, client, match, md_match, exclude, md_exclude); + + if (!tee) + { + pcre2_match_data_free(md_match); + pcre2_match_data_free(md_exclude); + + if (client) + { + delete client; + } + } + + return tee; } TeeSession::~TeeSession() @@ -99,7 +133,7 @@ void TeeSession::close() int TeeSession::routeQuery(GWBUF* queue) { - if (m_client) + if (m_client && query_matches(queue)) { m_client->queue_query(queue); } @@ -115,3 +149,32 @@ json_t* TeeSession::diagnostics_json() const { return NULL; } + +bool TeeSession::query_matches(GWBUF* buffer) +{ + bool rval = true; + + if (m_match || m_exclude) + { + char* sql; + int len; + + if (modutil_extract_SQL(buffer, &sql, &len)) + { + if (m_match && pcre2_match_8(m_match, (PCRE2_SPTR)sql, len, 0, 0, + m_md_match, NULL) < 0) + { + MXS_INFO("Query does not match the 'match' pattern: %.*s", len, sql); + rval = false; + } + else if (m_exclude && pcre2_match_8(m_exclude, (PCRE2_SPTR)sql, len, + 0, 0, m_md_exclude, NULL) >= 0) + { + MXS_INFO("Query matches the 'exclude' pattern: %.*s", len, sql); + rval = false; + } + } + } + + return rval; +} diff --git a/server/modules/filter/tee/teesession.hh b/server/modules/filter/tee/teesession.hh index 9d731b0b7..5c9580d52 100644 --- a/server/modules/filter/tee/teesession.hh +++ b/server/modules/filter/tee/teesession.hh @@ -38,6 +38,14 @@ public: json_t* diagnostics_json() const; private: - TeeSession(MXS_SESSION* session, LocalClient* client); - LocalClient* m_client; /**< The client connection to the local service */ + TeeSession(MXS_SESSION* session, LocalClient* client, + pcre2_code* match, pcre2_match_data* md_match, + pcre2_code* exclude, pcre2_match_data* md_exclude); + bool query_matches(GWBUF* buffer); + + LocalClient* m_client; /**< The client connection to the local service */ + pcre2_code* m_match; + pcre2_match_data* m_md_match; + pcre2_code* m_exclude; + pcre2_match_data* m_md_exclude; };