From 9de06a52b00943396d48ded62553c51463ef9e43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Wed, 10 Jul 2019 10:01:43 +0300 Subject: [PATCH 01/40] Allow infinite refreshes of users The hard limit of 10 seconds is too strict when taking into account the fact that infinite refreshes was possible before the bug was fixed. This also makes testing a lot easier where rapid reloads are necessary. --- Documentation/Getting-Started/Configuration-Guide.md | 6 +++++- include/maxscale/service.h | 1 - .../cnf/maxscale.cnf.template.mxs922_base | 1 + server/core/config.cc | 11 +---------- 4 files changed, 7 insertions(+), 12 deletions(-) diff --git a/Documentation/Getting-Started/Configuration-Guide.md b/Documentation/Getting-Started/Configuration-Guide.md index 8fb6fac7e..bb95617d8 100644 --- a/Documentation/Getting-Started/Configuration-Guide.md +++ b/Documentation/Getting-Started/Configuration-Guide.md @@ -867,13 +867,17 @@ MaxScale will at startup load the users from the backend server, but if the authentication of a user fails, MaxScale assumes it is because a new user has been created and will thus refresh the users. By default, MaxScale will do that at most once per 30 seconds and with this configuration option -that can be changed. The minimum allowed value is 10 seconds. A negative +that can be changed. A value of 0 allows infinite refreshes and a negative value disables the refreshing entirelly. Note that using `maxadmin` it is possible to explicitly cause the users of a service to be reloaded. + ``` users_refresh_time=120 ``` +In MaxScale 2.3.9 and older versions, the minimum allowed value was 10 seconds +but, due to a bug, the default value was 0 which allowed infinite refreshes. + ### `retain_last_statements` How many statements MaxScale should store for each session. This is for diff --git a/include/maxscale/service.h b/include/maxscale/service.h index e3f030a2b..dea4624df 100644 --- a/include/maxscale/service.h +++ b/include/maxscale/service.h @@ -83,7 +83,6 @@ typedef struct server_ref_t /* Refresh rate limits for load users from database */ #define USERS_REFRESH_TIME_DEFAULT 30 /* Allowed time interval (in seconds) after last update*/ -#define USERS_REFRESH_TIME_MIN 10 /* Minimum allowed time interval (in seconds)*/ /** Default timeout values used by the connections which fetch user authentication data */ #define DEFAULT_AUTH_CONNECT_TIMEOUT 3 diff --git a/maxscale-system-test/cnf/maxscale.cnf.template.mxs922_base b/maxscale-system-test/cnf/maxscale.cnf.template.mxs922_base index 81b609976..cbb9d16ce 100644 --- a/maxscale-system-test/cnf/maxscale.cnf.template.mxs922_base +++ b/maxscale-system-test/cnf/maxscale.cnf.template.mxs922_base @@ -1,5 +1,6 @@ [maxscale] threads=###threads### +users_refresh_time=0 [rwsplit-service] type=service diff --git a/server/core/config.cc b/server/core/config.cc index 3ca07e3f6..e3d910317 100644 --- a/server/core/config.cc +++ b/server/core/config.cc @@ -2337,7 +2337,7 @@ static int handle_global_item(const char* name, const char* value) return 0; } - decltype(gateway.qc_cache_properties.max_size)max_size = int_value; + decltype(gateway.qc_cache_properties.max_size) max_size = int_value; if (max_size >= 0) { @@ -2515,15 +2515,6 @@ static int handle_global_item(const char* name, const char* value) // but I just don't beleave the uptime will be that long. users_refresh_time = INT32_MAX; } - else if (users_refresh_time < USERS_REFRESH_TIME_MIN) - { - MXS_WARNING("%s is less than the allowed minimum value of %d for the " - "configuration option '%s', using the minimum value.", - value, - USERS_REFRESH_TIME_MIN, - CN_USERS_REFRESH_TIME); - users_refresh_time = USERS_REFRESH_TIME_MIN; - } if (users_refresh_time > INT32_MAX) { From 3e686e0ba5e1aa4b621f5e34399d07ca95950cab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Wed, 10 Jul 2019 10:48:26 +0300 Subject: [PATCH 02/40] Extend stacktrace printing There was a case that wasn't handled which would happen with executables. Also removed the path truncation to show where the library was loaded from. --- maxutils/maxbase/src/stacktrace.cc | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/maxutils/maxbase/src/stacktrace.cc b/maxutils/maxbase/src/stacktrace.cc index 8f52709a3..11d352153 100644 --- a/maxutils/maxbase/src/stacktrace.cc +++ b/maxutils/maxbase/src/stacktrace.cc @@ -71,7 +71,7 @@ static void extract_file_and_line(char* symbols, char* cmd, size_t size) const char* symname_start = filename_end + 1; - if (*symname_start != '+') + if (*symname_start != '+' && symname_start != symname_end) { // We have a string form symbol name and an offset, we need to // extract the symbol address @@ -111,6 +111,17 @@ static void extract_file_and_line(char* symbols, char* cmd, size_t size) } else { + if (symname_start == symname_end) + { + // Symbol is of the format `./executable [0xdeadbeef]` + if (!(symname_start = strchr(symname_start, '[')) + || !(symname_end = strchr(symname_start, ']'))) + { + snprintf(cmd, size, "Unexpected symbol format"); + return; + } + } + // Raw offset into library symname_start++; snprintf(symname, sizeof(symname), "%.*s", (int)(symname_end - symname_start), symname_start); @@ -126,13 +137,6 @@ static void extract_file_and_line(char* symbols, char* cmd, size_t size) memmove(cmd, str, strlen(cmd) - (str - cmd) + 1); } - // Strip the directory name from the symbols (could this be useful?) - if (char* str = strrchr(symbols, '/')) - { - ++str; - memmove(symbols, str, strlen(symbols) - (str - symbols) + 1); - } - // Remove the address where the symbol is in memory (i.e. the [0xdeadbeef] that follows the // (main+0xa1) part), we're only interested where it is in the library. if (char* str = strchr(symbols, '[')) From 0394c9c525f19ffd1c95193adbb4680ecc2dc25d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Wed, 10 Jul 2019 12:38:10 +0300 Subject: [PATCH 03/40] MXS-2449: Fix maxinfo monitor status output The status use an AND operation when it should do an equality comparison. --- server/core/monitor.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/core/monitor.cc b/server/core/monitor.cc index a96816507..5c85a5138 100644 --- a/server/core/monitor.cc +++ b/server/core/monitor.cc @@ -743,7 +743,7 @@ std::unique_ptr monitor_get_list() for (MXS_MONITOR* ptr = allMonitors; ptr; ptr = ptr->next) { - const char* state = ptr->state & MONITOR_STATE_RUNNING ? "Running" : "Stopped"; + const char* state = ptr->state == MONITOR_STATE_RUNNING ? "Running" : "Stopped"; set->add_row({ptr->name, state}); } From 8c84a2b2a44d77d8a4d65c271739d9322f38b26c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 11 Jul 2019 09:39:06 +0300 Subject: [PATCH 04/40] Fix error detection in service creation If a server that did not exist was added to the service, the error would not cause a startup failure. --- server/core/config.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/core/config.cc b/server/core/config.cc index e3d910317..1f6298bcb 100644 --- a/server/core/config.cc +++ b/server/core/config.cc @@ -3734,12 +3734,11 @@ int create_new_service(CONFIG_CONTEXT* obj) config_add_defaults(obj, config_service_params); config_add_defaults(obj, module->parameters); + int error_count = 0; Service* service = service_alloc(obj->object, router, obj->parameters); if (service) { - int error_count = 0; - for (auto& a : mxs::strtok(config_get_string(obj->parameters, CN_SERVERS), ",")) { fix_object_name(a); @@ -3771,9 +3770,10 @@ int create_new_service(CONFIG_CONTEXT* obj) else { MXS_ERROR("Service '%s' creation failed.", obj->object); + error_count++; } - return service ? 0 : 1; + return error_count; } /** From ee83a6ca0d93a7985b46fb8e408768657813a458 Mon Sep 17 00:00:00 2001 From: "wuzang.hdp" Date: Fri, 12 Jul 2019 15:09:10 +0800 Subject: [PATCH 05/40] fix memory leak on handling COM_CHANGE_USER --- server/modules/protocol/MySQL/mariadbbackend/mysql_backend.c | 1 + 1 file changed, 1 insertion(+) diff --git a/server/modules/protocol/MySQL/mariadbbackend/mysql_backend.c b/server/modules/protocol/MySQL/mariadbbackend/mysql_backend.c index 534fe9212..55e5a2fd8 100644 --- a/server/modules/protocol/MySQL/mariadbbackend/mysql_backend.c +++ b/server/modules/protocol/MySQL/mariadbbackend/mysql_backend.c @@ -858,6 +858,7 @@ gw_read_and_write(DCB *dcb) if (auth_change_requested(read_buffer) && handle_auth_change_response(read_buffer, proto, dcb)) { + gwbuf_free(read_buffer); return 0; } else From 797069df52c2a33cf5ea8cb505aa4f9577cbe905 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Tue, 16 Jul 2019 09:34:02 +0300 Subject: [PATCH 06/40] Fix npm audit warnings Updated lodash to a newer version. --- maxctrl/package-lock.json | 243 ++++++-------------- maxctrl/package.json | 2 +- server/core/test/rest-api/package-lock.json | 6 +- 3 files changed, 71 insertions(+), 180 deletions(-) diff --git a/maxctrl/package-lock.json b/maxctrl/package-lock.json index 402f714e6..0f7f1649c 100644 --- a/maxctrl/package-lock.json +++ b/maxctrl/package-lock.json @@ -404,6 +404,18 @@ "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==", "dev": true }, + "handlebars": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.1.2.tgz", + "integrity": "sha512-nvfrjqvt9xQ8Z/w0ijewdD/vvWDTOweBUm96NTr66Wfvo1mJenBLwcYmPs3TIBP5ruzYGD7Hx/DaM9RmhroGPw==", + "dev": true, + "requires": { + "neo-async": "^2.6.0", + "optimist": "^0.6.1", + "source-map": "^0.6.1", + "uglify-js": "^3.1.4" + } + }, "har-schema": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", @@ -568,9 +580,9 @@ } }, "lodash": { - "version": "4.17.11", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", - "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" + "version": "4.17.14", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.14.tgz", + "integrity": "sha512-mmKYbW3GLuJeX+iGP+Y7Gp1AiGHGbXHCOh/jZmrawMmsE7MS4znI3RL2FsjbqOyMayHInjOeykW7PEajUk1/xw==" }, "lodash-getpath": { "version": "0.2.4", @@ -661,6 +673,12 @@ "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", "dev": true }, + "neo-async": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.1.tgz", + "integrity": "sha512-iyam8fBuCUpWeKPGpaNMetEocMt364qkCsfL9JuhjXX6dRnguRVOfk2GZaDpPjcOKiiXCPINZC1GczQ7iTq3Zw==", + "dev": true + }, "normalize-package-data": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.4.0.tgz", @@ -720,23 +738,6 @@ "yargs-parser": "^8.0.0" }, "dependencies": { - "align-text": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/align-text/-/align-text-0.1.4.tgz", - "integrity": "sha1-DNkKVhCT810KmSVsIrcGlDP60Rc=", - "dev": true, - "requires": { - "kind-of": "^3.0.2", - "longest": "^1.0.1", - "repeat-string": "^1.5.2" - } - }, - "amdefine": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/amdefine/-/amdefine-1.0.1.tgz", - "integrity": "sha1-SlKCrBZHKek2Gbz9OtFR+BfOkfU=", - "dev": true - }, "ansi-regex": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", @@ -800,12 +801,6 @@ "integrity": "sha1-WWZ/QfrdTyDMvCu5a41Pf3jsA2c=", "dev": true }, - "async": { - "version": "1.5.2", - "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", - "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", - "dev": true - }, "atob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.1.tgz", @@ -1060,24 +1055,6 @@ "write-file-atomic": "^1.1.4" } }, - "camelcase": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-1.2.1.tgz", - "integrity": "sha1-m7UwTS4LVmmLLHWLCKPqqdqlijk=", - "dev": true, - "optional": true - }, - "center-align": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/center-align/-/center-align-0.1.3.tgz", - "integrity": "sha1-qg0yYptu6XIgBBHL1EYckHvCt60=", - "dev": true, - "optional": true, - "requires": { - "align-text": "^0.1.3", - "lazy-cache": "^1.0.3" - } - }, "chalk": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", @@ -1120,27 +1097,6 @@ } } }, - "cliui": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-2.1.0.tgz", - "integrity": "sha1-S0dXYP+AJkx2LDoXGQMukcf+oNE=", - "dev": true, - "optional": true, - "requires": { - "center-align": "^0.1.1", - "right-align": "^0.1.1", - "wordwrap": "0.0.2" - }, - "dependencies": { - "wordwrap": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.2.tgz", - "integrity": "sha1-t5Zpu0LstAn4PVg8rVLKF+qhZD8=", - "dev": true, - "optional": true - } - } - }, "code-point-at": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", @@ -1595,29 +1551,6 @@ "integrity": "sha1-Dovf5NHduIVNZOBOp8AOKgJuVlg=", "dev": true }, - "handlebars": { - "version": "4.0.11", - "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.0.11.tgz", - "integrity": "sha1-Ywo13+ApS8KB7a5v/F0yn8eYLcw=", - "dev": true, - "requires": { - "async": "^1.4.0", - "optimist": "^0.6.1", - "source-map": "^0.4.4", - "uglify-js": "^2.6" - }, - "dependencies": { - "source-map": { - "version": "0.4.4", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.4.4.tgz", - "integrity": "sha1-66T12pwNyZneaAMti092FzZSA2s=", - "dev": true, - "requires": { - "amdefine": ">=0.0.4" - } - } - } - }, "has-ansi": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", @@ -2001,13 +1934,6 @@ "is-buffer": "^1.1.5" } }, - "lazy-cache": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-1.0.4.tgz", - "integrity": "sha1-odePw6UEdMuAhF07O24dpJpEbo4=", - "dev": true, - "optional": true - }, "lcid": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/lcid/-/lcid-1.0.0.tgz", @@ -2050,14 +1976,7 @@ }, "lodash": { "version": "4.17.10", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.10.tgz", - "integrity": "sha512-UejweD1pDoXu+AD825lWwp4ZGtSwgnpZxb3JDViD7StjQz+Nb/6l093lx4OQ0foGWNRoc19mWy7BzL+UAK2iVg==", - "dev": true - }, - "longest": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/longest/-/longest-1.0.1.tgz", - "integrity": "sha1-MKCy2jj3N3DoKUoNIuZiXtd9AJc=", + "resolved": "", "dev": true }, "loose-envify": { @@ -2187,8 +2106,7 @@ }, "mixin-deep": { "version": "1.3.1", - "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.1.tgz", - "integrity": "sha512-8ZItLHeEgaqEvd5lYBXfm4EZSFCX29Jb9K+lAHhDKzReKBQKj3R+7NOF6tjqYi9t4oI8VUfaWITJQm86wnXGNQ==", + "resolved": "", "dev": true, "requires": { "for-in": "^1.0.2", @@ -2359,16 +2277,6 @@ "wrappy": "1" } }, - "optimist": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz", - "integrity": "sha1-2j6nRob6IaGaERwybpDrFaAZZoY=", - "dev": true, - "requires": { - "minimist": "~0.0.1", - "wordwrap": "~0.0.2" - } - }, "os-homedir": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", @@ -2623,16 +2531,6 @@ "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==", "dev": true }, - "right-align": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/right-align/-/right-align-0.1.3.tgz", - "integrity": "sha1-YTObci/mo1FWiSENJOFMlhSGE+8=", - "dev": true, - "optional": true, - "requires": { - "align-text": "^0.1.1" - } - }, "rimraf": { "version": "2.6.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.2.tgz", @@ -2665,8 +2563,7 @@ }, "set-value": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.0.tgz", - "integrity": "sha512-hw0yxk9GT/Hr5yJEYnHNKYXkIA8mVJgd9ditYZCe16ZczcaELYYcfvaXesNACk2O8O0nTiPQcQhGUQj8JLzeeg==", + "resolved": "", "dev": true, "requires": { "extend-shallow": "^2.0.1", @@ -3322,44 +3219,9 @@ "integrity": "sha1-yy4SAwZ+DI3h9hQJS5/kVwTqYAM=", "dev": true }, - "uglify-js": { - "version": "2.8.29", - "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-2.8.29.tgz", - "integrity": "sha1-KcVzMUgFe7Th913zW3qcty5qWd0=", - "dev": true, - "optional": true, - "requires": { - "source-map": "~0.5.1", - "uglify-to-browserify": "~1.0.0", - "yargs": "~3.10.0" - }, - "dependencies": { - "yargs": { - "version": "3.10.0", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-3.10.0.tgz", - "integrity": "sha1-9+572FfdfB0tOMDnTvvWgdFDH9E=", - "dev": true, - "optional": true, - "requires": { - "camelcase": "^1.0.2", - "cliui": "^2.1.0", - "decamelize": "^1.0.0", - "window-size": "0.1.0" - } - } - } - }, - "uglify-to-browserify": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/uglify-to-browserify/-/uglify-to-browserify-1.0.2.tgz", - "integrity": "sha1-bgkk1r2mta/jSeOabWMoUKD4grc=", - "dev": true, - "optional": true - }, "union-value": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.0.tgz", - "integrity": "sha1-XHHDTLW61dzr4+oM0IIHulqhrqQ=", + "resolved": "", "dev": true, "requires": { "arr-union": "^3.1.0", @@ -3485,19 +3347,6 @@ "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=", "dev": true }, - "window-size": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/window-size/-/window-size-0.1.0.tgz", - "integrity": "sha1-VDjNLqk7IC76Ohn+iIeu58lPnJ0=", - "dev": true, - "optional": true - }, - "wordwrap": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz", - "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=", - "dev": true - }, "wrap-ansi": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", @@ -3655,6 +3504,16 @@ "wrappy": "1" } }, + "optimist": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz", + "integrity": "sha1-2j6nRob6IaGaERwybpDrFaAZZoY=", + "dev": true, + "requires": { + "minimist": "~0.0.1", + "wordwrap": "~0.0.2" + } + }, "os-locale": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-2.1.0.tgz", @@ -3881,6 +3740,12 @@ "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=" }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + }, "spdx-correct": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.0.0.tgz", @@ -4008,6 +3873,26 @@ "integrity": "sha1-diIXzAbbJY7EiQihKY6LlRIejqI=", "dev": true }, + "uglify-js": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.6.0.tgz", + "integrity": "sha512-W+jrUHJr3DXKhrsS7NUVxn3zqMOFn0hL/Ei6v0anCIMoKC93TjcflTagwIHLW7SfMFfiQuktQyFVCFHGUE0+yg==", + "dev": true, + "optional": true, + "requires": { + "commander": "~2.20.0", + "source-map": "~0.6.1" + }, + "dependencies": { + "commander": { + "version": "2.20.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.0.tgz", + "integrity": "sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ==", + "dev": true, + "optional": true + } + } + }, "uuid": { "version": "3.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", @@ -4045,6 +3930,12 @@ "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=" }, + "wordwrap": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz", + "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=", + "dev": true + }, "wrap-ansi": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", diff --git a/maxctrl/package.json b/maxctrl/package.json index 434ba2fd9..7db8d5b77 100644 --- a/maxctrl/package.json +++ b/maxctrl/package.json @@ -18,7 +18,7 @@ "license": "SEE LICENSE IN ../LICENSE.TXT", "dependencies": { "cli-table": "^0.3.1", - "lodash": "^4.17.11", + "lodash": "^4.17.14", "lodash-getpath": "^0.2.4", "readline-sync": "^1.4.9", "request": "^2.88.0", diff --git a/server/core/test/rest-api/package-lock.json b/server/core/test/rest-api/package-lock.json index d11210320..662d0c412 100644 --- a/server/core/test/rest-api/package-lock.json +++ b/server/core/test/rest-api/package-lock.json @@ -370,9 +370,9 @@ } }, "lodash": { - "version": "4.17.11", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", - "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" + "version": "4.17.14", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.14.tgz", + "integrity": "sha512-mmKYbW3GLuJeX+iGP+Y7Gp1AiGHGbXHCOh/jZmrawMmsE7MS4znI3RL2FsjbqOyMayHInjOeykW7PEajUk1/xw==" }, "mime-db": { "version": "1.36.0", From bc2f7f4040f3a44617100d23e64948d9db140dfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Tue, 16 Jul 2019 10:03:27 +0300 Subject: [PATCH 07/40] MXS-2597: Log libmicrohttpd errors on startup By enabling the debug messages only at startup, we'll get log messages for any daemon startup failures and we exlude the verbose parsing errors that malformed requests cause. --- server/core/admin.cc | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/server/core/admin.cc b/server/core/admin.cc index c99771115..386a4e666 100644 --- a/server/core/admin.cc +++ b/server/core/admin.cc @@ -372,6 +372,19 @@ static bool load_ssl_certificates() return rval; } +static bool log_daemon_errors = true; + +void admin_log_error(void* arg, const char* fmt, va_list ap) +{ + if (log_daemon_errors) + { + char buf[1024]; + vsnprintf(buf, sizeof(buf), fmt, ap); + trim(buf); + MXS_ERROR("HTTP daemon error: %s\n", buf); + } +} + bool mxs_admin_init() { struct sockaddr_storage addr; @@ -380,7 +393,7 @@ bool mxs_admin_init() config_get_global_options()->admin_port, &addr)) { - int options = MHD_USE_EPOLL_INTERNALLY_LINUX_ONLY; + int options = MHD_USE_EPOLL_INTERNALLY_LINUX_ONLY | MHD_USE_DEBUG; if (addr.ss_family == AF_INET6) { @@ -395,6 +408,7 @@ bool mxs_admin_init() // The port argument is ignored and the port in the struct sockaddr is used instead http_daemon = MHD_start_daemon(options, 0, NULL, NULL, handle_client, NULL, + MHD_OPTION_EXTERNAL_LOGGER, admin_log_error, NULL, MHD_OPTION_NOTIFY_COMPLETED, close_client, NULL, MHD_OPTION_SOCK_ADDR, &addr, !using_ssl ? MHD_OPTION_END : @@ -404,6 +418,9 @@ bool mxs_admin_init() MHD_OPTION_END); } + // Silence all other errors to prevent malformed requests from flooding the log + log_daemon_errors = false; + return http_daemon != NULL; } From 3649efde0b7d167ceaaf2c028eaa3348e1fd4e72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Tue, 16 Jul 2019 21:45:13 +0300 Subject: [PATCH 08/40] MXS-2605: Remove false debug assertion The assertion doesn't count executed session commands and thus is not reliable. --- server/modules/routing/readwritesplit/readwritesplit.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/modules/routing/readwritesplit/readwritesplit.cc b/server/modules/routing/readwritesplit/readwritesplit.cc index d0e1dd887..7f13893fc 100644 --- a/server/modules/routing/readwritesplit/readwritesplit.cc +++ b/server/modules/routing/readwritesplit/readwritesplit.cc @@ -415,8 +415,6 @@ json_t* RWSplit::diagnostics_json() const for (const auto& a : all_server_stats()) { - mxb_assert(a.second.total == a.second.read + a.second.write); - ServerStats::CurrentStats stats = a.second.current_stats(); json_t* obj = json_object(); From f139991a2c19fa0ac58654257c00aba4eb69d487 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Wed, 10 Jul 2019 08:55:16 +0300 Subject: [PATCH 09/40] MXS-2559: Log source of loaded users MySQLAuth now logs the server where the users were loaded from. As only the initial loading of users causes a log message, it is still possible for the source server to change without any indication of it. --- server/modules/authenticator/MySQLAuth/dbusers.cc | 9 +++++---- server/modules/authenticator/MySQLAuth/mysql_auth.cc | 7 +++++-- server/modules/authenticator/MySQLAuth/mysql_auth.h | 3 ++- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/server/modules/authenticator/MySQLAuth/dbusers.cc b/server/modules/authenticator/MySQLAuth/dbusers.cc index 69f17ac35..a80ade0b8 100644 --- a/server/modules/authenticator/MySQLAuth/dbusers.cc +++ b/server/modules/authenticator/MySQLAuth/dbusers.cc @@ -136,7 +136,7 @@ const char* mariadb_users_query // We only care about users that have a default role assigned "WHERE t.default_role = u.user %s;"; -static int get_users(SERV_LISTENER* listener, bool skip_local); +static int get_users(SERV_LISTENER* listener, bool skip_local, SERVER** srv); static MYSQL* gw_mysql_init(void); static int gw_mysql_set_timeouts(MYSQL* handle); static char* mysql_format_user_entry(void* data); @@ -192,9 +192,9 @@ static char* get_users_query(const char* server_version, int version, bool inclu return rval; } -int replace_mysql_users(SERV_LISTENER* listener, bool skip_local) +int replace_mysql_users(SERV_LISTENER* listener, bool skip_local, SERVER** srv) { - int i = get_users(listener, skip_local); + int i = get_users(listener, skip_local, srv); return i; } @@ -1092,7 +1092,7 @@ int get_users_from_server(MYSQL* con, SERVER_REF* server_ref, SERVICE* service, * @param users The users table into which to load the users * @return -1 on any error or the number of users inserted */ -static int get_users(SERV_LISTENER* listener, bool skip_local) +static int get_users(SERV_LISTENER* listener, bool skip_local, SERVER** srv) { const char* service_user = NULL; const char* service_passwd = NULL; @@ -1148,6 +1148,7 @@ static int get_users(SERV_LISTENER* listener, bool skip_local) if (users > total_users) { + *srv = server->server; total_users = users; } diff --git a/server/modules/authenticator/MySQLAuth/mysql_auth.cc b/server/modules/authenticator/MySQLAuth/mysql_auth.cc index e9a53ca45..5107a77aa 100644 --- a/server/modules/authenticator/MySQLAuth/mysql_auth.cc +++ b/server/modules/authenticator/MySQLAuth/mysql_auth.cc @@ -787,7 +787,8 @@ static int mysql_auth_load_users(SERV_LISTENER* port) first_load = true; } - int loaded = replace_mysql_users(port, first_load); + SERVER* srv = nullptr; + int loaded = replace_mysql_users(port, first_load, &srv); bool injected = false; if (loaded <= 0) @@ -834,7 +835,9 @@ static int mysql_auth_load_users(SERV_LISTENER* port) } else if (loaded > 0 && first_load) { - MXS_NOTICE("[%s] Loaded %d MySQL users for listener %s.", service->name, loaded, port->name); + mxb_assert(srv); + MXS_NOTICE("[%s] Loaded %d MySQL users for listener %s from server %s.", + service->name, loaded, port->name, srv->name); } return rc; diff --git a/server/modules/authenticator/MySQLAuth/mysql_auth.h b/server/modules/authenticator/MySQLAuth/mysql_auth.h index 2f4c6d986..f278ce1a0 100644 --- a/server/modules/authenticator/MySQLAuth/mysql_auth.h +++ b/server/modules/authenticator/MySQLAuth/mysql_auth.h @@ -198,10 +198,11 @@ bool dbusers_save(sqlite3* src, const char* filename); * * @param service The current service * @param skip_local Skip loading of users on local MaxScale services + * @param srv Server where the users were loaded from (output) * * @return -1 on any error or the number of users inserted (0 means no users at all) */ -int replace_mysql_users(SERV_LISTENER* listener, bool skip_local); +int replace_mysql_users(SERV_LISTENER* listener, bool skip_local, SERVER** srv); /** * @brief Verify the user has access to the database From f8ee11cf552de31d1a33d5cfed0829402b6fb68c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Wed, 17 Jul 2019 14:42:32 +0300 Subject: [PATCH 10/40] MXS-2606: Sort servers before loading users By sorting the servers in descending order based on their role we make sure that the users are loaded from a master if one is available. --- .../authenticator/MySQLAuth/dbusers.cc | 78 +++++++++++-------- 1 file changed, 44 insertions(+), 34 deletions(-) diff --git a/server/modules/authenticator/MySQLAuth/dbusers.cc b/server/modules/authenticator/MySQLAuth/dbusers.cc index a80ade0b8..1cc5f53f5 100644 --- a/server/modules/authenticator/MySQLAuth/dbusers.cc +++ b/server/modules/authenticator/MySQLAuth/dbusers.cc @@ -21,6 +21,9 @@ #include #include +#include +#include + #include #include #include @@ -1025,17 +1028,17 @@ bool query_and_process_users(const char* query, MYSQL* con, sqlite3* handle, SER return rval; } -int get_users_from_server(MYSQL* con, SERVER_REF* server_ref, SERVICE* service, SERV_LISTENER* listener) +int get_users_from_server(MYSQL* con, SERVER* server, SERVICE* service, SERV_LISTENER* listener) { - if (server_ref->server->version_string[0] == 0) + if (server->version_string[0] == 0) { - mxs_mysql_update_server_version(con, server_ref->server); + mxs_mysql_update_server_version(con, server); } - char* query = get_users_query(server_ref->server->version_string, - server_ref->server->version, + char* query = get_users_query(server->version_string, + server->version, service->enable_root, - roles_are_available(con, service, server_ref->server)); + roles_are_available(con, service, server)); MYSQL_AUTH* instance = (MYSQL_AUTH*)listener->auth_instance; sqlite3* handle = get_handle(instance); @@ -1043,20 +1046,20 @@ int get_users_from_server(MYSQL* con, SERVER_REF* server_ref, SERVICE* service, bool rv = query_and_process_users(query, con, handle, service, &users); - if (!rv && have_mdev13453_problem(con, server_ref->server)) + if (!rv && have_mdev13453_problem(con, server)) { /** * Try to work around MDEV-13453 by using a query without CTEs. Masquerading as * a 10.1.10 server makes sure CTEs aren't used. */ MXS_FREE(query); - query = get_users_query(server_ref->server->version_string, 100110, service->enable_root, true); + query = get_users_query(server->version_string, 100110, service->enable_root, true); rv = query_and_process_users(query, con, handle, service, &users); } if (!rv) { - MXS_ERROR("Failed to load users from server '%s': %s", server_ref->server->name, mysql_error(con)); + MXS_ERROR("Failed to load users from server '%s': %s", server->name, mysql_error(con)); } MXS_FREE(query); @@ -1084,6 +1087,28 @@ int get_users_from_server(MYSQL* con, SERVER_REF* server_ref, SERVICE* service, return users; } +// Sorts candidates servers so that masters are before slaves which are before only running servers +static std::vector get_candidates(SERVICE* service, bool skip_local) +{ + std::vector candidates; + + for (auto server = service->dbref; server; server = server->next) + { + if (SERVER_REF_IS_ACTIVE(server) && server_is_running(server->server) + && (!skip_local || !server_is_mxs_service(server->server))) + { + candidates.push_back(server->server); + } + } + + std::sort(candidates.begin(), candidates.end(), [](SERVER* a, SERVER* b) { + return (server_is_master(a) && !server_is_master(b)) + || (server_is_slave(a) && !server_is_slave(b) && !server_is_master(b)); + }); + + return candidates; +} + /** * Load the user/passwd form mysql.user table into the service users' hashtable * environment. @@ -1112,33 +1137,18 @@ static int get_users(SERV_LISTENER* listener, bool skip_local, SERVER** srv) sqlite3* handle = get_handle(instance); delete_mysql_users(handle); - SERVER_REF* server = service->dbref; int total_users = -1; - bool no_active_servers = true; + auto candidates = get_candidates(service, skip_local); - for (server = service->dbref; !maxscale_is_shutting_down() && server; server = server->next) + for (auto server : candidates) { - if (!SERVER_REF_IS_ACTIVE(server) || !server_is_active(server->server) - || (skip_local && server_is_mxs_service(server->server)) - || !server_is_running(server->server)) + if (MYSQL* con = gw_mysql_init()) { - continue; - } - - no_active_servers = false; - - MYSQL* con = gw_mysql_init(); - if (con) - { - if (mxs_mysql_real_connect(con, server->server, service_user, dpwd) == NULL) + if (mxs_mysql_real_connect(con, server, service_user, dpwd) == NULL) { - MXS_ERROR("Failure loading users data from backend " - "[%s:%i] for service [%s]. MySQL error %i, %s", - server->server->address, - server->server->port, - service->name, - mysql_errno(con), - mysql_error(con)); + MXS_ERROR("Failure loading users data from backend [%s:%i] for service [%s]. " + "MySQL error %i, %s", server->address, server->port, service->name, + mysql_errno(con), mysql_error(con)); mysql_close(con); } else @@ -1148,7 +1158,7 @@ static int get_users(SERV_LISTENER* listener, bool skip_local, SERVER** srv) if (users > total_users) { - *srv = server->server; + *srv = server; total_users = users; } @@ -1164,12 +1174,12 @@ static int get_users(SERV_LISTENER* listener, bool skip_local, SERVER** srv) MXS_FREE(dpwd); - if (no_active_servers) + if (candidates.empty()) { // This service has no servers or all servers are local MaxScale services total_users = 0; } - else if (server == NULL && total_users == -1) + else if (*srv == nullptr && total_users == -1) { MXS_ERROR("Unable to get user data from backend database for service [%s]." " Failed to connect to any of the backend databases.", From 84f4688ebbbe4bdf710b6373b1d35bc0f2a0f054 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Wed, 17 Jul 2019 14:45:02 +0300 Subject: [PATCH 11/40] Fix readwritesplit response count assertion The assertion in routeQuery that expects there to be at least one ongoing query would be triggered if a query was received after a master had failed but before the session would close. To make sure the internal logic stays consistent, the error handler should only decrement the expected response count if the session can continue. --- server/modules/routing/readwritesplit/rwsplitsession.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/server/modules/routing/readwritesplit/rwsplitsession.cc b/server/modules/routing/readwritesplit/rwsplitsession.cc index e92d89db1..fc1faaf13 100644 --- a/server/modules/routing/readwritesplit/rwsplitsession.cc +++ b/server/modules/routing/readwritesplit/rwsplitsession.cc @@ -985,7 +985,6 @@ void RWSplitSession::handleError(GWBUF* errmsgbuf, { // We were expecting a response but we aren't going to get one mxb_assert(m_expected_responses > 0); - m_expected_responses--; errmsg += " Lost connection to master server while waiting for a result."; if (can_retry_query()) @@ -1000,6 +999,14 @@ void RWSplitSession::handleError(GWBUF* errmsgbuf, can_continue = true; send_readonly_error(m_client); } + + // Decrement the expected response count only if we know we can continue the sesssion. + // This keeps the internal logic sound even if another query is routed before the session + // is closed. + if (can_continue) + { + m_expected_responses--; + } } if (session_trx_is_active(session) && m_otrx_state == OTRX_INACTIVE) From 1ddcbc9ae1bfbd2083a93341527f709fbb4ab725 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 18 Jul 2019 17:58:22 +0300 Subject: [PATCH 12/40] Log error message on slave session command failure If the slave's response differs from the master and the slave sent an error packet, log the contents of the error. This should make it obvious as to what caused the failure. --- server/modules/routing/readwritesplit/rwsplit_session_cmd.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/modules/routing/readwritesplit/rwsplit_session_cmd.cc b/server/modules/routing/readwritesplit/rwsplit_session_cmd.cc index 762cd10ef..0f13a18ab 100644 --- a/server/modules/routing/readwritesplit/rwsplit_session_cmd.cc +++ b/server/modules/routing/readwritesplit/rwsplit_session_cmd.cc @@ -149,8 +149,8 @@ void RWSplitSession::process_sescmd_response(SRWBackend& backend, GWBUF** ppPack { if (cmd == MYSQL_REPLY_ERR && m_sescmd_responses[id] != MYSQL_REPLY_ERR) { - MXS_INFO("Session command failed on slave '%s': %s", - backend->name(), extract_error(*ppPacket).c_str()); + MXS_WARNING("Session command failed on slave '%s': %s", + backend->name(), extract_error(*ppPacket).c_str()); } discard_if_response_differs(backend, m_sescmd_responses[id], cmd, sescmd); From e3b1eebb1559f76bd0b3dc51f741333ab3d4153d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Fri, 19 Jul 2019 08:22:25 +0300 Subject: [PATCH 13/40] Update unit test configs Removed some of the useless parameters, tuned some values and reordered it to declare objects before they are referred to. --- test/maxscale_test.cnf | 78 ++++++++++++-------------------- test/maxscale_test_secondary.cnf | 78 ++++++++++++-------------------- 2 files changed, 60 insertions(+), 96 deletions(-) diff --git a/test/maxscale_test.cnf b/test/maxscale_test.cnf index d2a9ff810..3006e6c45 100644 --- a/test/maxscale_test.cnf +++ b/test/maxscale_test.cnf @@ -1,5 +1,5 @@ [maxscale] -threads=4 +threads=auto libdir=@CMAKE_INSTALL_PREFIX@/@MAXSCALE_LIBDIR@ logdir=@CMAKE_INSTALL_PREFIX@/log/maxscale/ datadir=@CMAKE_INSTALL_PREFIX@/lib/maxscale @@ -8,13 +8,37 @@ language=@CMAKE_INSTALL_PREFIX@/lib/maxscale/ piddir=@CMAKE_INSTALL_PREFIX@/run/maxscale/ admin_auth=false +[server1] +type=server +address=127.0.0.1 +port=3000 +protocol=MariaDBBackend + +[server2] +type=server +address=127.0.0.1 +port=3001 +protocol=MariaDBBackend + +[server3] +type=server +address=127.0.0.1 +port=3002 +protocol=MariaDBBackend + +[server4] +type=server +address=127.0.0.1 +port=3003 +protocol=MariaDBBackend + [MariaDB-Monitor] type=monitor module=mariadbmon servers=server1,server2,server3,server4 user=maxuser password=maxpwd -monitor_interval=10000 +monitor_interval=5000 [RW-Split-Router] type=service @@ -22,7 +46,6 @@ router=readwritesplit servers=server1,server2,server3,server4 user=maxuser password=maxpwd -max_slave_connections=100% [SchemaRouter-Router] type=service @@ -30,7 +53,6 @@ router=schemarouter servers=server1,server2,server3,server4 user=maxuser password=maxpwd -auth_all_servers=1 [RW-Split-Hint-Router] type=service @@ -38,7 +60,6 @@ router=readwritesplit servers=server1,server2,server3,server4 user=maxuser password=maxpwd -max_slave_connections=100% filters=Hint [Read-Connection-Router] @@ -54,21 +75,6 @@ filters=QLA type=filter module=hintfilter -[recurse3] -type=filter -module=tee -service=RW-Split-Router - -[recurse2] -type=filter -module=tee -service=Read-Connection-Router - -[recurse1] -type=filter -module=tee -service=RW-Split-Hint-Router - [QLA] type=filter module=qlafilter @@ -77,10 +83,6 @@ append=false flush=true filebase=/tmp/qla.log -[CLI] -type=service -router=cli - [Read-Connection-Listener] type=listener service=Read-Connection-Router @@ -105,32 +107,12 @@ service=RW-Split-Hint-Router protocol=MariaDBClient port=4009 +[CLI] +type=service +router=cli + [CLI-Listener] type=listener service=CLI protocol=maxscaled socket=default - -[server1] -type=server -address=127.0.0.1 -port=3000 -protocol=MariaDBBackend - -[server2] -type=server -address=127.0.0.1 -port=3001 -protocol=MariaDBBackend - -[server3] -type=server -address=127.0.0.1 -port=3002 -protocol=MariaDBBackend - -[server4] -type=server -address=127.0.0.1 -port=3003 -protocol=MariaDBBackend diff --git a/test/maxscale_test_secondary.cnf b/test/maxscale_test_secondary.cnf index d0eb78095..b16a12179 100644 --- a/test/maxscale_test_secondary.cnf +++ b/test/maxscale_test_secondary.cnf @@ -1,5 +1,5 @@ [maxscale] -threads=4 +threads=auto libdir=@CMAKE_INSTALL_PREFIX@/@MAXSCALE_LIBDIR@ logdir=@CMAKE_INSTALL_PREFIX@/secondary/log/maxscale/ datadir=@CMAKE_INSTALL_PREFIX@/secondary/lib/maxscale @@ -10,13 +10,37 @@ persistdir=@CMAKE_INSTALL_PREFIX@/secondary/lib/maxscale/maxscale.cnf.d/ admin_auth=false admin_port=8990 +[server1] +type=server +address=127.0.0.1 +port=3000 +protocol=MariaDBBackend + +[server2] +type=server +address=127.0.0.1 +port=3001 +protocol=MariaDBBackend + +[server3] +type=server +address=127.0.0.1 +port=3002 +protocol=MariaDBBackend + +[server4] +type=server +address=127.0.0.1 +port=3003 +protocol=MariaDBBackend + [MariaDB-Monitor] type=monitor module=mariadbmon servers=server1,server2,server3,server4 user=maxuser password=maxpwd -monitor_interval=10000 +monitor_interval=5000 [RW-Split-Router] type=service @@ -24,7 +48,6 @@ router=readwritesplit servers=server1,server2,server3,server4 user=maxuser password=maxpwd -max_slave_connections=100% [SchemaRouter-Router] type=service @@ -32,7 +55,6 @@ router=schemarouter servers=server1,server2,server3,server4 user=maxuser password=maxpwd -auth_all_servers=1 [RW-Split-Hint-Router] type=service @@ -40,7 +62,6 @@ router=readwritesplit servers=server1,server2,server3,server4 user=maxuser password=maxpwd -max_slave_connections=100% filters=Hint [Read-Connection-Router] @@ -56,21 +77,6 @@ filters=QLA type=filter module=hintfilter -[recurse3] -type=filter -module=tee -service=RW-Split-Router - -[recurse2] -type=filter -module=tee -service=Read-Connection-Router - -[recurse1] -type=filter -module=tee -service=RW-Split-Hint-Router - [QLA] type=filter module=qlafilter @@ -79,10 +85,6 @@ append=false flush=true filebase=/tmp/qla2.log -[CLI] -type=service -router=cli - [Read-Connection-Listener] type=listener service=Read-Connection-Router @@ -107,32 +109,12 @@ service=RW-Split-Hint-Router protocol=MariaDBClient port=5009 +[CLI] +type=service +router=cli + [CLI-Listener] type=listener service=CLI protocol=maxscaled socket=/tmp/maxadmin2.sock - -[server1] -type=server -address=127.0.0.1 -port=3000 -protocol=MariaDBBackend - -[server2] -type=server -address=127.0.0.1 -port=3001 -protocol=MariaDBBackend - -[server3] -type=server -address=127.0.0.1 -port=3002 -protocol=MariaDBBackend - -[server4] -type=server -address=127.0.0.1 -port=3003 -protocol=MariaDBBackend From 146b9402456c2f13e7b87dc06b8b32e0d25a8c9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Fri, 19 Jul 2019 09:39:15 +0300 Subject: [PATCH 14/40] Dump statements on fatal signal This should make it easier to figure out what is going on at the time of the crash if statement collection is enabled. --- server/core/gateway.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/server/core/gateway.cc b/server/core/gateway.cc index 7700b7a85..a77b13138 100644 --- a/server/core/gateway.cc +++ b/server/core/gateway.cc @@ -427,6 +427,14 @@ static void sigfatal_handler(int i) cnf->sysname, cnf->release_string); + if (DCB* dcb = dcb_get_current()) + { + if (dcb->session) + { + session_dump_statements(dcb->session); + } + } + auto cb = [](const char* symbol, const char* cmd) { MXS_ALERT(" %s: %s", symbol, cmd); }; From 8e23a6cdb96b61fa9231c38538fcc750ac8a93e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 18 Jul 2019 23:40:11 +0300 Subject: [PATCH 15/40] MXS-2607: Remove trailing spaces in MaxCtrl --tvs mode As the TSV format uses tabs as delimiters, the trailing spaces would acutally be interpreted as data. --- maxctrl/lib/common.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/maxctrl/lib/common.js b/maxctrl/lib/common.js index b3555c8f1..035a50ace 100644 --- a/maxctrl/lib/common.js +++ b/maxctrl/lib/common.js @@ -142,6 +142,9 @@ module.exports = function() { if (this.argv.tsv) { // Based on the regex found in: https://github.com/jonschlinkert/strip-color str = str.replace( /\x1B\[[(?);]{0,2}(;?\d)*./g, '') + + // Trim trailing whitespace that cli-table generates + str = str.split(os.EOL).map(s => s.split('\t').map(s => s.trimEnd()).join('\t')).join(os.EOL) } return str } From fbeb5d9c84c35f09088464cc2d049cbc3a089f88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Tue, 23 Jul 2019 09:20:33 +0300 Subject: [PATCH 16/40] Always ignore server-internal databases The mysql, information_schema and performance_schema databases should always be ignored if found on multiple servers. --- server/modules/routing/schemarouter/schemaroutersession.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/server/modules/routing/schemarouter/schemaroutersession.cc b/server/modules/routing/schemarouter/schemaroutersession.cc index 4b7bfa1d8..ff9cd4fdb 100644 --- a/server/modules/routing/schemarouter/schemaroutersession.cc +++ b/server/modules/routing/schemarouter/schemaroutersession.cc @@ -1192,11 +1192,13 @@ char* get_lenenc_str(void* data) return rval; } +static const std::set always_ignore = {"mysql", "information_schema", "performance_schema"}; + bool SchemaRouterSession::ignore_duplicate_database(const char* data) { bool rval = false; - if (m_config->ignored_dbs.find(data) != m_config->ignored_dbs.end()) + if (m_config->ignored_dbs.count(data) || always_ignore.count(data)) { rval = true; } @@ -1379,8 +1381,7 @@ void SchemaRouterSession::query_databases() "LEFT JOIN information_schema.tables AS t ON s.schema_name = t.table_schema " "WHERE t.table_name IS NULL " "UNION " - "SELECT CONCAT (table_schema, '.', table_name) FROM information_schema.tables " - "WHERE table_schema NOT IN ('information_schema', 'performance_schema', 'mysql');"); + "SELECT CONCAT (table_schema, '.', table_name) FROM information_schema.tables"); gwbuf_set_type(buffer, GWBUF_TYPE_COLLECT_RESULT); for (SSRBackendList::iterator it = m_backends.begin(); it != m_backends.end(); it++) From 12bd26398f6f84c74116d7d6b6bce44748a0819d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Tue, 23 Jul 2019 09:46:05 +0300 Subject: [PATCH 17/40] MXS-2486: Add missing schemarouter capabilities The capabilities that the schemarouter declared were missing the RCAP_TYPE_PACKET_OUTPUT which caused partial packets to be returned to it. --- server/modules/protocol/MySQL/rwbackend.cc | 1 + .../schemarouter/schemarouterinstance.cc | 31 ++++++++++--------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/server/modules/protocol/MySQL/rwbackend.cc b/server/modules/protocol/MySQL/rwbackend.cc index 0840c6cb6..2dc98d16e 100644 --- a/server/modules/protocol/MySQL/rwbackend.cc +++ b/server/modules/protocol/MySQL/rwbackend.cc @@ -294,6 +294,7 @@ void RWBackend::process_packets(GWBUF* result) auto it = buffer.begin(); MXB_AT_DEBUG(size_t total_len = buffer.length()); MXB_AT_DEBUG(size_t used_len = 0); + mxb_assert(dcb()->session->service->capabilities & (RCAP_TYPE_PACKET_OUTPUT | RCAP_TYPE_STMT_OUTPUT)); while (it != buffer.end()) { diff --git a/server/modules/routing/schemarouter/schemarouterinstance.cc b/server/modules/routing/schemarouter/schemarouterinstance.cc index 49d4a90a2..066955a62 100644 --- a/server/modules/routing/schemarouter/schemarouterinstance.cc +++ b/server/modules/routing/schemarouter/schemarouterinstance.cc @@ -265,9 +265,12 @@ json_t* SchemaRouter::diagnostics_json() const return rval; } +static const uint64_t CAPABILITIES = RCAP_TYPE_CONTIGUOUS_INPUT | RCAP_TYPE_PACKET_OUTPUT + | RCAP_TYPE_RUNTIME_CONFIG; + uint64_t SchemaRouter::getCapabilities() { - return RCAP_TYPE_CONTIGUOUS_INPUT | RCAP_TYPE_RUNTIME_CONFIG; + return schemarouter::CAPABILITIES; } } @@ -288,21 +291,21 @@ extern "C" MXS_MODULE* MXS_CREATE_MODULE() MXS_ROUTER_VERSION, "A database sharding router for simple sharding", "V1.0.0", - RCAP_TYPE_CONTIGUOUS_INPUT | RCAP_TYPE_RUNTIME_CONFIG, + schemarouter::CAPABILITIES, &schemarouter::SchemaRouter::s_object, - NULL, /* Process init. */ - NULL, /* Process finish. */ - NULL, /* Thread init. */ - NULL, /* Thread finish. */ + NULL, + NULL, + NULL, + NULL, { - {"ignore_databases", MXS_MODULE_PARAM_STRING }, - {"ignore_databases_regex", MXS_MODULE_PARAM_STRING }, - {"max_sescmd_history", MXS_MODULE_PARAM_COUNT, "0"}, - {"disable_sescmd_history", MXS_MODULE_PARAM_BOOL, "false"}, - {"refresh_databases", MXS_MODULE_PARAM_BOOL, "true"}, - {"refresh_interval", MXS_MODULE_PARAM_COUNT, DEFAULT_REFRESH_INTERVAL}, - {"debug", MXS_MODULE_PARAM_BOOL, "false"}, - {"preferred_server", MXS_MODULE_PARAM_SERVER }, + {"ignore_databases", MXS_MODULE_PARAM_STRING }, + {"ignore_databases_regex", MXS_MODULE_PARAM_STRING }, + {"max_sescmd_history", MXS_MODULE_PARAM_COUNT, "0"}, + {"disable_sescmd_history", MXS_MODULE_PARAM_BOOL, "false"}, + {"refresh_databases", MXS_MODULE_PARAM_BOOL, "true"}, + {"refresh_interval", MXS_MODULE_PARAM_COUNT, DEFAULT_REFRESH_INTERVAL}, + {"debug", MXS_MODULE_PARAM_BOOL, "false"}, + {"preferred_server", MXS_MODULE_PARAM_SERVER }, {MXS_END_MODULE_PARAMS} } }; From f31f09fc66fd0967a836dad087c0a0c02f007e64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Tue, 23 Jul 2019 14:12:58 +0300 Subject: [PATCH 18/40] Don't print errors with master_failure_mode=error_on_write The errors are expeted and should not be logged. --- server/modules/routing/readwritesplit/rwsplit_route_stmt.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/modules/routing/readwritesplit/rwsplit_route_stmt.cc b/server/modules/routing/readwritesplit/rwsplit_route_stmt.cc index 53797d280..1ec1d8076 100644 --- a/server/modules/routing/readwritesplit/rwsplit_route_stmt.cc +++ b/server/modules/routing/readwritesplit/rwsplit_route_stmt.cc @@ -350,7 +350,7 @@ bool RWSplitSession::route_single_stmt(GWBUF* querybuf) succp = true; MXS_INFO("Delaying routing: %s", extract_sql(querybuf).c_str()); } - else + else if (m_config.master_failure_mode != RW_ERROR_ON_WRITE) { MXS_ERROR("Could not find valid server for target type %s, closing " "connection.", route_target_to_string(route_target)); From 9171dbaad674c683c21c91525bed8a46f8178602 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Fri, 26 Jul 2019 08:59:21 +0300 Subject: [PATCH 19/40] Replace trimEnd with trim trimEnd was introduced in Node.js 10.0 which is not in the 6.0 version we use. --- maxctrl/lib/common.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maxctrl/lib/common.js b/maxctrl/lib/common.js index 035a50ace..796fa1834 100644 --- a/maxctrl/lib/common.js +++ b/maxctrl/lib/common.js @@ -144,7 +144,7 @@ module.exports = function() { str = str.replace( /\x1B\[[(?);]{0,2}(;?\d)*./g, '') // Trim trailing whitespace that cli-table generates - str = str.split(os.EOL).map(s => s.split('\t').map(s => s.trimEnd()).join('\t')).join(os.EOL) + str = str.split(os.EOL).map(s => s.split('\t').map(s => s.trim()).join('\t')).join(os.EOL) } return str } From 9a7153dcec86b28720864461906137aaf539dca8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Fri, 26 Jul 2019 09:09:40 +0300 Subject: [PATCH 20/40] Fix mysqlmon_multimaster It failed to compile with a new-ish compiler as the output argument was guaranteed to be null. --- maxscale-system-test/mysqlmon_multimaster.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maxscale-system-test/mysqlmon_multimaster.cpp b/maxscale-system-test/mysqlmon_multimaster.cpp index 6022f5a55..7423c16e3 100644 --- a/maxscale-system-test/mysqlmon_multimaster.cpp +++ b/maxscale-system-test/mysqlmon_multimaster.cpp @@ -56,7 +56,7 @@ json_t* get_json_data(TestConnections& test, const char* query) char* output = test.maxscales->ssh_node_output(0, query, true, &exit_code); if (output == NULL) { - test.add_result(1, "Query '%s' execution error, no output.\ni", output); + test.add_result(1, "Query '%s' execution error, no output.", query); } else { From b07ffdb2fad57704ba0e3dcb7085a719b0495bb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Fri, 26 Jul 2019 09:34:08 +0300 Subject: [PATCH 21/40] Fix hang on transaction replay The expected response counter was not decremented if a transaction replay was started. This caused the connections to hang which in turn caused the failure of the mxs1507_trx_stress test case. --- .../routing/readwritesplit/rwsplitsession.cc | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/server/modules/routing/readwritesplit/rwsplitsession.cc b/server/modules/routing/readwritesplit/rwsplitsession.cc index fc1faaf13..fb4ee458b 100644 --- a/server/modules/routing/readwritesplit/rwsplitsession.cc +++ b/server/modules/routing/readwritesplit/rwsplitsession.cc @@ -963,7 +963,9 @@ void RWSplitSession::handleError(GWBUF* errmsgbuf, MXS_INFO("Master '%s' failed: %s", backend->name(), extract_error(errmsgbuf).c_str()); /** The connection to the master has failed */ - if (!backend->is_waiting_result()) + bool expected_response = backend->is_waiting_result(); + + if (!expected_response) { /** The failure of a master is not considered a critical * failure as partial functionality still remains. If @@ -999,14 +1001,6 @@ void RWSplitSession::handleError(GWBUF* errmsgbuf, can_continue = true; send_readonly_error(m_client); } - - // Decrement the expected response count only if we know we can continue the sesssion. - // This keeps the internal logic sound even if another query is routed before the session - // is closed. - if (can_continue) - { - m_expected_responses--; - } } if (session_trx_is_active(session) && m_otrx_state == OTRX_INACTIVE) @@ -1034,6 +1028,14 @@ void RWSplitSession::handleError(GWBUF* errmsgbuf, } } + // Decrement the expected response count only if we know we can continue the sesssion. + // This keeps the internal logic sound even if another query is routed before the session + // is closed. + if (can_continue && expected_response) + { + m_expected_responses--; + } + backend->close(); backend->set_close_reason("Master connection failed: " + extract_error(errmsgbuf)); } From 918a2964d5acc7d85bda728060bf88b8cf39d92f Mon Sep 17 00:00:00 2001 From: Marko Date: Wed, 24 Jul 2019 18:29:58 +0300 Subject: [PATCH 22/40] MXS-2592 Add configuration for session specific in-memory log When enabled each session will write log messages in the in-memory log. If session ends in error this log is written to the actual log in disk. --- include/maxscale/session.h | 11 +++ maxctrl/lib/show.js | 3 +- maxutils/maxbase/include/maxbase/log.h | 17 ++++- maxutils/maxbase/include/maxbase/log.hh | 9 +-- maxutils/maxbase/include/maxbase/maxbase.hh | 5 +- maxutils/maxbase/src/log.cc | 31 +++++++- maxutils/maxbase/src/maxbase.cc | 5 +- server/core/config.cc | 17 +++++ server/core/gateway.cc | 1 + server/core/internal/session.hh | 6 +- server/core/log.cc | 11 ++- server/core/session.cc | 70 ++++++++++++++++++- .../MySQL/mariadbclient/mysql_client.cc | 5 ++ .../routing/readwritesplit/rwsplitsession.cc | 1 + 14 files changed, 176 insertions(+), 16 deletions(-) diff --git a/include/maxscale/session.h b/include/maxscale/session.h index 154a730f3..076f17b28 100644 --- a/include/maxscale/session.h +++ b/include/maxscale/session.h @@ -679,6 +679,17 @@ session_dump_statements_t session_get_dump_statements(); */ const char* session_get_dump_statements_str(); + +void session_set_session_trace(uint32_t value); + +uint32_t session_get_session_trace(); + +const char* session_get_session_log(MXS_SESSION* pSession); + +void session_append_log(MXS_SESSION* pSession, const char* log); + +void session_dump_log(MXS_SESSION* pSession); + /** * @brief Route the query again after a delay * diff --git a/maxctrl/lib/show.js b/maxctrl/lib/show.js index a9583e139..067af86b2 100644 --- a/maxctrl/lib/show.js +++ b/maxctrl/lib/show.js @@ -59,7 +59,8 @@ const session_fields = [ {'Idle': 'attributes.idle'}, {'Connections': 'attributes.connections[].server'}, {'Connection IDs': 'attributes.connections[].protocol_diagnostics.connection_id'}, - {'Queries': 'attributes.queries[].statement'} + {'Queries': 'attributes.queries[].statement'}, + {'Log': 'attributes.log'} ] const filter_fields = [ diff --git a/maxutils/maxbase/include/maxbase/log.h b/maxutils/maxbase/include/maxbase/log.h index f5c6d3dbe..0c2a1f6ff 100644 --- a/maxutils/maxbase/include/maxbase/log.h +++ b/maxutils/maxbase/include/maxbase/log.h @@ -85,6 +85,8 @@ typedef struct MXB_LOG_THROTTLING */ typedef size_t (* mxb_log_context_provider_t)(char* buffer, size_t len); +typedef void (* mxb_in_memory_log_t)(const char* buffer, size_t len); + /** * @brief Initialize the log * @@ -105,7 +107,8 @@ bool mxb_log_init(const char* ident, const char* logdir, const char* filename, mxb_log_target_t target, - mxb_log_context_provider_t context_provider); + mxb_log_context_provider_t context_provider, + mxb_in_memory_log_t in_memory_log); /** * @brief Finalize the log @@ -150,6 +153,8 @@ const char* mxb_log_get_filename(); */ bool mxb_log_set_priority_enabled(int priority, bool enabled); +bool mxb_log_get_session_trace(); + /** * Query whether a particular syslog priority is enabled. * @@ -233,6 +238,14 @@ void mxb_log_get_throttling(MXB_LOG_THROTTLING* throttling); */ void mxs_log_redirect_stdout(bool redirect); +/** + * Set session specific in-memory log + * + * @param enabled True or false to enable or disable session in-memory logging + */ +void mxb_log_set_session_trace(bool enabled); + + /** * Log a message of a particular priority. * @@ -278,7 +291,7 @@ int mxb_log_oom(const char* message); * MXB_ERROR, MXB_WARNING, etc. macros instead. */ #define MXB_LOG_MESSAGE(priority, format, ...) \ - (mxb_log_is_priority_enabled(priority) \ + (mxb_log_is_priority_enabled(priority) || mxb_log_get_session_trace() \ ? mxb_log_message(priority, MXB_MODULE_NAME, __FILE__, __LINE__, __func__, format, ##__VA_ARGS__) \ : 0) diff --git a/maxutils/maxbase/include/maxbase/log.hh b/maxutils/maxbase/include/maxbase/log.hh index 99dd6f907..29ac6ecc5 100644 --- a/maxutils/maxbase/include/maxbase/log.hh +++ b/maxutils/maxbase/include/maxbase/log.hh @@ -30,7 +30,7 @@ */ inline bool mxb_log_init(mxb_log_target_t target = MXB_LOG_TARGET_FS) { - return mxb_log_init(nullptr, ".", nullptr, target, nullptr); + return mxb_log_init(nullptr, ".", nullptr, target, nullptr, nullptr); } namespace maxbase @@ -52,16 +52,17 @@ public: const char* logdir, const char* filename, mxb_log_target_t target, - mxb_log_context_provider_t context_provider) + mxb_log_context_provider_t context_provider, + mxb_in_memory_log_t in_memory_log) { - if (!mxb_log_init(ident, logdir, filename, target, context_provider)) + if (!mxb_log_init(ident, logdir, filename, target, context_provider, in_memory_log)) { throw std::runtime_error("Failed to initialize the log."); } } Log(mxb_log_target_t target = MXB_LOG_TARGET_FS) - : Log(nullptr, ".", nullptr, target, nullptr) + : Log(nullptr, ".", nullptr, target, nullptr, nullptr) { } diff --git a/maxutils/maxbase/include/maxbase/maxbase.hh b/maxutils/maxbase/include/maxbase/maxbase.hh index 1e0a0007b..4f254e84e 100644 --- a/maxutils/maxbase/include/maxbase/maxbase.hh +++ b/maxutils/maxbase/include/maxbase/maxbase.hh @@ -78,7 +78,8 @@ public: const char* zLogdir, const char* zFilename, mxb_log_target_t target, - mxb_log_context_provider_t context_provider); + mxb_log_context_provider_t context_provider, + mxb_in_memory_log_t in_memory_log); /** * @brief Initializes MaxBase and the MaxBase log. @@ -88,7 +89,7 @@ public: * @throws std::runtime_error if the initialization failed. */ MaxBase(mxb_log_target_t target) - : MaxBase(nullptr, ".", nullptr, target, nullptr) + : MaxBase(nullptr, ".", nullptr, target, nullptr, nullptr) { } diff --git a/maxutils/maxbase/src/log.cc b/maxutils/maxbase/src/log.cc index c80b59f07..659d2d611 100644 --- a/maxutils/maxbase/src/log.cc +++ b/maxutils/maxbase/src/log.cc @@ -386,10 +386,12 @@ struct this_unit bool do_syslog; // Can change during the lifetime of log_manager. bool do_maxlog; // Can change during the lifetime of log_manager. bool redirect_stdout; + bool session_trace; MXB_LOG_THROTTLING throttling; // Can change during the lifetime of log_manager. std::unique_ptr sLogger; std::unique_ptr sMessage_registry; size_t (* context_provider)(char* buffer, size_t len); + void (* in_memory_log)(const char* buffer, size_t len); } this_unit = { DEFAULT_LOG_AUGMENTATION, // augmentation @@ -397,6 +399,7 @@ struct this_unit true, // do_syslog true, // do_maxlog false, // redirect_stdout + false, // session_trace DEFAULT_LOG_THROTTLING, // throttling }; @@ -449,7 +452,8 @@ bool mxb_log_init(const char* ident, const char* logdir, const char* filename, mxb_log_target_t target, - mxb_log_context_provider_t context_provider) + mxb_log_context_provider_t context_provider, + mxb_in_memory_log_t in_memory_log) { assert(!this_unit.sLogger && !this_unit.sMessage_registry); @@ -511,6 +515,7 @@ bool mxb_log_init(const char* ident, if (this_unit.sLogger && this_unit.sMessage_registry) { this_unit.context_provider = context_provider; + this_unit.in_memory_log = in_memory_log; openlog(ident, LOG_PID | LOG_ODELAY, LOG_USER); } @@ -614,6 +619,16 @@ void mxs_log_redirect_stdout(bool redirect) this_unit.redirect_stdout = redirect; } +void mxb_log_set_session_trace(bool enabled) +{ + this_unit.session_trace = enabled; +} + +bool mxb_log_get_session_trace() +{ + return this_unit.session_trace; +} + bool mxb_log_rotate() { bool rval = this_unit.sLogger->rotate(); @@ -874,7 +889,19 @@ int mxb_log_message(int priority, // Add a final newline into the message msg.push_back('\n'); - err = this_unit.sLogger->write(msg.c_str(), msg.length()) ? 0 : -1; + if (this_unit.session_trace) + { + this_unit.in_memory_log(msg.c_str(), msg.length()); + } + + if (mxb_log_is_priority_enabled(priority)) + { + err = this_unit.sLogger->write(msg.c_str(), msg.length()) ? 0 : -1; + } + else + { + err = 0; + } } } } diff --git a/maxutils/maxbase/src/maxbase.cc b/maxutils/maxbase/src/maxbase.cc index b8bd4cea2..007641761 100644 --- a/maxutils/maxbase/src/maxbase.cc +++ b/maxutils/maxbase/src/maxbase.cc @@ -92,14 +92,15 @@ MaxBase::MaxBase(const char* zIdent, const char* zLogdir, const char* zFilename, mxb_log_target_t target, - mxb_log_context_provider_t context_provider) + mxb_log_context_provider_t context_provider, + mxb_in_memory_log_t in_memory_log) : m_log_inited(false) { const char* zMessage = nullptr; if (maxbase::init()) { - m_log_inited = mxb_log_init(zIdent, zLogdir, zFilename, target, context_provider); + m_log_inited = mxb_log_init(zIdent, zLogdir, zFilename, target, context_provider, in_memory_log); if (!m_log_inited) { diff --git a/server/core/config.cc b/server/core/config.cc index 1f6298bcb..b55169f31 100644 --- a/server/core/config.cc +++ b/server/core/config.cc @@ -160,6 +160,7 @@ const char CN_SERVER[] = "server"; const char CN_SERVICES[] = "services"; const char CN_SERVICE[] = "service"; const char CN_SESSIONS[] = "sessions"; +const char CN_SESSION_TRACE[] = "session_trace"; const char CN_SESSION_TRACK_TRX_STATE[] = "session_track_trx_state"; const char CN_SKIP_PERMISSION_CHECKS[] = "skip_permission_checks"; const char CN_SOCKET[] = "socket"; @@ -317,6 +318,7 @@ const MXS_MODULE_PARAM config_service_params[] = {CN_RETRY_ON_FAILURE, MXS_MODULE_PARAM_BOOL, "true"}, {CN_SESSION_TRACK_TRX_STATE, MXS_MODULE_PARAM_BOOL, "false"}, {CN_RETAIN_LAST_STATEMENTS, MXS_MODULE_PARAM_INT, "-1"}, + {CN_SESSION_TRACE, MXS_MODULE_PARAM_INT, "0"}, {NULL} }; @@ -2602,6 +2604,21 @@ static int handle_global_item(const char* name, const char* value) return 0; } } + else if (strcmp(name, CN_SESSION_TRACE) == 0) + { + char* endptr; + int intval = strtol(value, &endptr, 0); + if (*endptr == '\0' && intval >= 0) + { + session_set_session_trace(intval); + mxb_log_set_session_trace(true); + } + else + { + MXS_ERROR("Invalid value for '%s': %s", CN_SESSION_TRACE, value); + return 0; + } + } else if (strcmp(name, CN_LOAD_PERSISTED_CONFIGS) == 0) { int b = config_truth_value(value); diff --git a/server/core/gateway.cc b/server/core/gateway.cc index a77b13138..d7d484744 100644 --- a/server/core/gateway.cc +++ b/server/core/gateway.cc @@ -432,6 +432,7 @@ static void sigfatal_handler(int i) if (dcb->session) { session_dump_statements(dcb->session); + session_dump_log(dcb->session); } } diff --git a/server/core/internal/session.hh b/server/core/internal/session.hh index 1a5fb9fff..ccc5ed499 100644 --- a/server/core/internal/session.hh +++ b/server/core/internal/session.hh @@ -97,7 +97,7 @@ public: }; typedef std::deque QueryInfos; - + using Log = std::deque; using FilterList = std::vector; Session(SERVICE* service); @@ -121,8 +121,11 @@ public: void book_server_response(SERVER* pServer, bool final_response); void book_last_as_complete(); void reset_server_bookkeeping(); + void append_session_log(std::string); + void dump_session_log(); json_t* queries_as_json() const; + json_t* log_as_json() const; void link_backend_dcb(DCB* dcb) { @@ -148,6 +151,7 @@ private: int m_current_query = -1; /*< The index of the current query */ DCBSet m_dcb_set; /*< Set of associated backend DCBs */ uint32_t m_retain_last_statements; /*< How many statements be retained */ + Log m_log; /*< Session specific in-memory log */ }; } diff --git a/server/core/log.cc b/server/core/log.cc index e0c4bb666..94768cf7f 100644 --- a/server/core/log.cc +++ b/server/core/log.cc @@ -47,13 +47,22 @@ size_t mxs_get_context(char* buffer, size_t len) return len; } + +void mxs_log_in_memory(const char* msg, size_t len) +{ + MXS_SESSION* session = session_get_current(); + if (session) + { + session_append_log(session, msg); + } +} } bool mxs_log_init(const char* ident, const char* logdir, mxs_log_target_t target) { mxb::Logger::set_ident("MariaDB MaxScale"); - return mxb_log_init(ident, logdir, LOGFILE_NAME, target, mxs_get_context); + return mxb_log_init(ident, logdir, LOGFILE_NAME, target, mxs_get_context, mxs_log_in_memory); } namespace diff --git a/server/core/session.cc b/server/core/session.cc index 7def7937a..99230f623 100644 --- a/server/core/session.cc +++ b/server/core/session.cc @@ -62,11 +62,13 @@ struct uint64_t next_session_id; uint32_t retain_last_statements; session_dump_statements_t dump_statements; + uint32_t session_trace; } this_unit = { 1, 0, - SESSION_DUMP_STATEMENTS_NEVER + SESSION_DUMP_STATEMENTS_NEVER, + 0 }; static struct session dummy_session() @@ -909,6 +911,9 @@ json_t* session_json_data(const Session* session, const char* host) json_t* queries = session->queries_as_json(); json_object_set_new(attr, "queries", queries); + json_t* log = session->log_as_json(); + json_object_set_new(attr, "log", log); + json_object_set_new(data, CN_ATTRIBUTES, attr); json_object_set_new(data, CN_LINKS, mxs_json_self_link(host, CN_SESSIONS, ss.str().c_str())); @@ -1111,6 +1116,32 @@ void session_dump_statements(MXS_SESSION* session) pSession->dump_statements(); } +void session_set_session_trace(uint32_t value) +{ + this_unit.session_trace = value; +} + +uint32_t session_get_session_trace() +{ + return this_unit.session_trace; +} + +void session_append_log(MXS_SESSION* pSession, const char* log) +{ + // Ignore dummy and listener sessions + if (pSession->state != SESSION_STATE_DUMMY + && pSession->state != SESSION_STATE_LISTENER + && pSession->state != SESSION_STATE_LISTENER_STOPPED) + { + static_cast(pSession)->append_session_log(std::string(log)); + } +} + +void session_dump_log(MXS_SESSION* pSession) +{ + static_cast(pSession)->dump_session_log(); +} + class DelayedRoutingTask { DelayedRoutingTask(const DelayedRoutingTask&) = delete; @@ -1365,6 +1396,18 @@ json_t* Session::queries_as_json() const return pQueries; } +json_t* Session::log_as_json() const +{ + json_t* pLog = json_array(); + + for (const auto& i : m_log) + { + json_array_append_new(pLog, json_string(i.c_str())); + } + + return pLog; +} + bool Session::setup_filters(Service* service) { for (const auto& a : service->get_filters()) @@ -1724,3 +1767,28 @@ void Session::QueryInfo::reset_server_bookkeeping() m_completed.tv_nsec = 0; m_complete = false; } + +void Session::append_session_log(std::string log) +{ + m_log.push_front(log); + + if (m_log.size() >= this_unit.session_trace) + { + m_log.pop_back(); + } +} + +void Session::dump_session_log() +{ + if (!(m_log.empty())) + { + std::string log; + + for (const auto& s : m_log) + { + log += s; + } + + MXS_NOTICE("Session log for session (%" PRIu64"): \n%s ", ses_id, log.c_str()); + } +} \ No newline at end of file diff --git a/server/modules/protocol/MySQL/mariadbclient/mysql_client.cc b/server/modules/protocol/MySQL/mariadbclient/mysql_client.cc index c6fe58f24..7b8ea6b30 100644 --- a/server/modules/protocol/MySQL/mariadbclient/mysql_client.cc +++ b/server/modules/protocol/MySQL/mariadbclient/mysql_client.cc @@ -1585,6 +1585,11 @@ static int gw_client_hangup_event(DCB* dcb) session_dump_statements(session); } + if (session_get_session_trace()) + { + session_dump_log(session); + } + // The client did not send a COM_QUIT packet std::string errmsg {"Connection killed by MaxScale"}; std::string extra {session_get_close_reason(dcb->session)}; diff --git a/server/modules/routing/readwritesplit/rwsplitsession.cc b/server/modules/routing/readwritesplit/rwsplitsession.cc index fb4ee458b..a5ddb2585 100644 --- a/server/modules/routing/readwritesplit/rwsplitsession.cc +++ b/server/modules/routing/readwritesplit/rwsplitsession.cc @@ -395,6 +395,7 @@ static void log_unexpected_response(SRWBackend& backend, GWBUF* buffer, GWBUF* c backend->current_command(), sql.c_str()); session_dump_statements(backend->dcb()->session); + session_dump_log(backend->dcb()->session); mxb_assert(false); } } From dda02b45ee8e40ab37e0a7068668be5a0da9899f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Fri, 26 Jul 2019 20:37:53 +0300 Subject: [PATCH 23/40] MXS-2520: Do master replacement if it's possible This allows a new master to be chosen regardless of the target server type. --- .../readwritesplit/rwsplit_route_stmt.cc | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/server/modules/routing/readwritesplit/rwsplit_route_stmt.cc b/server/modules/routing/readwritesplit/rwsplit_route_stmt.cc index 1ec1d8076..4ac85bed1 100644 --- a/server/modules/routing/readwritesplit/rwsplit_route_stmt.cc +++ b/server/modules/routing/readwritesplit/rwsplit_route_stmt.cc @@ -210,6 +210,17 @@ bool RWSplitSession::route_single_stmt(GWBUF* querybuf) { update_trx_statistics(); + auto next_master = get_target_backend(BE_MASTER, NULL, MXS_RLAG_UNDEFINED); + + if (should_replace_master(next_master)) + { + MXS_INFO("Replacing old master '%s' with new master '%s'", + m_current_master ? + m_current_master->name() : "", + next_master->name()); + replace_master(next_master); + } + if (m_qc.is_trx_starting() // A transaction is starting && !session_trx_is_read_only(m_client->session) // Not explicitly read-only && should_try_trx_on_slave(route_target)) // Qualifies for speculative routing @@ -1052,15 +1063,6 @@ bool RWSplitSession::handle_master_is_target(SRWBackend* dest) SRWBackend target = get_target_backend(BE_MASTER, NULL, MXS_RLAG_UNDEFINED); bool succp = true; - if (should_replace_master(target)) - { - MXS_INFO("Replacing old master '%s' with new master '%s'", - m_current_master ? - m_current_master->name() : "", - target->name()); - replace_master(target); - } - if (target && target == m_current_master) { mxb::atomic::add(&m_router->stats().n_master, 1, mxb::atomic::RELAXED); From 3ddcc4928862c3725874200194ffb116f022b050 Mon Sep 17 00:00:00 2001 From: Marko Date: Mon, 29 Jul 2019 12:07:02 +0300 Subject: [PATCH 24/40] MXS-2592 Use the correct log level variable --- maxutils/maxbase/src/log.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maxutils/maxbase/src/log.cc b/maxutils/maxbase/src/log.cc index 659d2d611..ff204de48 100644 --- a/maxutils/maxbase/src/log.cc +++ b/maxutils/maxbase/src/log.cc @@ -894,7 +894,7 @@ int mxb_log_message(int priority, this_unit.in_memory_log(msg.c_str(), msg.length()); } - if (mxb_log_is_priority_enabled(priority)) + if (mxb_log_is_priority_enabled(level)) { err = this_unit.sLogger->write(msg.c_str(), msg.length()) ? 0 : -1; } From c592328ea2cc393a18ec04de0bb7c45935f542ea Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Tue, 30 Jul 2019 12:29:01 +0300 Subject: [PATCH 25/40] MXS-2613 Access correct argument The cache show-command accessed the wrong argument. That is why [maxadmin|maxctrl] call command cache show TheCache did not work. --- server/modules/filter/cache/cachefilter.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/modules/filter/cache/cachefilter.cc b/server/modules/filter/cache/cachefilter.cc index ed6b31e4d..0a20f046e 100644 --- a/server/modules/filter/cache/cachefilter.cc +++ b/server/modules/filter/cache/cachefilter.cc @@ -94,9 +94,9 @@ void cache_config_reset(CACHE_CONFIG& config) bool cache_command_show(const MODULECMD_ARG* pArgs, json_t** output) { mxb_assert(pArgs->argc == 1); - mxb_assert(MODULECMD_GET_TYPE(&pArgs->argv[1].type) == MODULECMD_ARG_FILTER); + mxb_assert(MODULECMD_GET_TYPE(&pArgs->argv[0].type) == MODULECMD_ARG_FILTER); - const MXS_FILTER_DEF* pFilterDef = pArgs->argv[1].value.filter; + const MXS_FILTER_DEF* pFilterDef = pArgs->argv[0].value.filter; mxb_assert(pFilterDef); CacheFilter* pFilter = reinterpret_cast(filter_def_get_instance(pFilterDef)); From 18169a17d9759a85a99d9dc6be28fab1cf3c39d5 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Thu, 1 Aug 2019 08:23:39 +0300 Subject: [PATCH 26/40] Update 2.3 changelog and add 2.3.10 release notes --- Documentation/Changelog.md | 1 + .../MaxScale-2.3.10-Release-Notes.md | 49 +++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 Documentation/Release-Notes/MaxScale-2.3.10-Release-Notes.md diff --git a/Documentation/Changelog.md b/Documentation/Changelog.md index e5160d7ae..bf171504f 100644 --- a/Documentation/Changelog.md +++ b/Documentation/Changelog.md @@ -31,6 +31,7 @@ For more details, please refer to: +* [MariaDB MaxScale 2.3.10 Release Notes](Release-Notes/MaxScale-2.3.10-Release-Notes.md) * [MariaDB MaxScale 2.3.9 Release Notes](Release-Notes/MaxScale-2.3.9-Release-Notes.md) * [MariaDB MaxScale 2.3.8 Release Notes](Release-Notes/MaxScale-2.3.8-Release-Notes.md) * [MariaDB MaxScale 2.3.7 Release Notes](Release-Notes/MaxScale-2.3.7-Release-Notes.md) diff --git a/Documentation/Release-Notes/MaxScale-2.3.10-Release-Notes.md b/Documentation/Release-Notes/MaxScale-2.3.10-Release-Notes.md new file mode 100644 index 000000000..b675b27dd --- /dev/null +++ b/Documentation/Release-Notes/MaxScale-2.3.10-Release-Notes.md @@ -0,0 +1,49 @@ +# MariaDB MaxScale 2.3.10 Release Notes + +Release 2.3.10 is a GA release. + +This document describes the changes in release 2.3.10, when compared to the +previous release in the same series. + +For any problems you encounter, please consider submitting a bug +report on [our Jira](https://jira.mariadb.org/projects/MXS). + +## Bug fixes + +* [MXS-2613](https://jira.mariadb.org/browse/MXS-2613) Fix cachefilter diagnostics +* [MXS-2607](https://jira.mariadb.org/browse/MXS-2607) Unexpected trailing spaces with --tsv option in MaxCtrl +* [MXS-2606](https://jira.mariadb.org/browse/MXS-2606) Users are loaded from the first available server +* [MXS-2605](https://jira.mariadb.org/browse/MXS-2605) debug assert at readwritesplit.cc:418 failed: a.second.total == a.second.read + a.second.write +* [MXS-2598](https://jira.mariadb.org/browse/MXS-2598) memory leak on handling COM_CHANGE_USER +* [MXS-2597](https://jira.mariadb.org/browse/MXS-2597) MaxScale doesn't handle errors from microhttpd +* [MXS-2594](https://jira.mariadb.org/browse/MXS-2594) Enabling use_priority for not set priority on server level triggers an election +* [MXS-2587](https://jira.mariadb.org/browse/MXS-2587) mxs1507_trx_replay: debug assert in routeQuery +* [MXS-2586](https://jira.mariadb.org/browse/MXS-2586) user_refresh_time default value is wrong +* [MXS-2559](https://jira.mariadb.org/browse/MXS-2559) Log doesn't tell from which server users are loaded from +* [MXS-2520](https://jira.mariadb.org/browse/MXS-2520) Readwritesplit won't connect to master for reads +* [MXS-2502](https://jira.mariadb.org/browse/MXS-2502) Specifying 'information_schema' as default schema upon connection gives 'access denied' +* [MXS-2490](https://jira.mariadb.org/browse/MXS-2490) Unknown prepared statement handler (0) given to mysqld_stmt_execute +* [MXS-2486](https://jira.mariadb.org/browse/MXS-2486) MaxScale 2.3.6 received fatal signal 11 +* [MXS-2449](https://jira.mariadb.org/browse/MXS-2449) Maxadmin shows wrong monitor status +* [MXS-2261](https://jira.mariadb.org/browse/MXS-2261) maxkeys overwrites existing key without warning +* [MXS-1901](https://jira.mariadb.org/browse/MXS-1901) Multi continues COM_STMT_SEND_LONG_DATA route to different backends + +## Known Issues and Limitations + +There are some limitations and known issues within this version of MaxScale. +For more information, please refer to the [Limitations](../About/Limitations.md) document. + +## Packaging + +RPM and Debian packages are provided for supported the Linux distributions. + +Packages can be downloaded [here](https://mariadb.com/downloads/#mariadb_platform-mariadb_maxscale). + +## Source Code + +The source code of MaxScale is tagged at GitHub with a tag, which is identical +with the version of MaxScale. For instance, the tag of version X.Y.Z of MaxScale +is `maxscale-X.Y.Z`. Further, the default branch is always the latest GA version +of MaxScale. + +The source code is available [here](https://github.com/mariadb-corporation/MaxScale). From 8347a48b644dd00cbae7e9bb1c9c5805d8990052 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Thu, 1 Aug 2019 08:32:30 +0300 Subject: [PATCH 27/40] Update 2.3 maintance release --- VERSION23.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION23.cmake b/VERSION23.cmake index afc3c44fc..2f928171c 100644 --- a/VERSION23.cmake +++ b/VERSION23.cmake @@ -5,7 +5,7 @@ set(MAXSCALE_VERSION_MAJOR "2" CACHE STRING "Major version") set(MAXSCALE_VERSION_MINOR "3" CACHE STRING "Minor version") -set(MAXSCALE_VERSION_PATCH "10" CACHE STRING "Patch version") +set(MAXSCALE_VERSION_PATCH "11" CACHE STRING "Patch version") # This should only be incremented if a package is rebuilt set(MAXSCALE_BUILD_NUMBER 1 CACHE STRING "Release number") From e1a730ed869d3f29b34a5c6ef3468ab667b95595 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Thu, 1 Aug 2019 16:06:26 +0300 Subject: [PATCH 28/40] Update release date for 2.3.10 --- Documentation/Release-Notes/MaxScale-2.3.10-Release-Notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/Release-Notes/MaxScale-2.3.10-Release-Notes.md b/Documentation/Release-Notes/MaxScale-2.3.10-Release-Notes.md index b675b27dd..c8a0fed77 100644 --- a/Documentation/Release-Notes/MaxScale-2.3.10-Release-Notes.md +++ b/Documentation/Release-Notes/MaxScale-2.3.10-Release-Notes.md @@ -1,4 +1,4 @@ -# MariaDB MaxScale 2.3.10 Release Notes +# MariaDB MaxScale 2.3.10 Release Notes -- 2019-08-01 Release 2.3.10 is a GA release. From 527f9d31361bd95281b1f4915e82355eed767619 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Sun, 28 Jul 2019 09:35:20 +0300 Subject: [PATCH 29/40] Rewrite open_close_connections The test now uses standard library threads and lambda functions to make the code simpler. Also made the test more robust by ignoring any errors that are caused by the exhaustion of available client side TCP ports. --- .../open_close_connections.cpp | 134 +++++++----------- 1 file changed, 50 insertions(+), 84 deletions(-) diff --git a/maxscale-system-test/open_close_connections.cpp b/maxscale-system-test/open_close_connections.cpp index 167a8bd9b..d565bb740 100644 --- a/maxscale-system-test/open_close_connections.cpp +++ b/maxscale-system-test/open_close_connections.cpp @@ -6,113 +6,79 @@ #include "testconnections.h" -typedef struct -{ - int exit_flag; - int thread_id; - long i; - int rwsplit_only; - TestConnections* Test; -} openclose_thread_data; +#include +#include -void* query_thread1(void* ptr); -int threads_num = 20; +std::atomic run {true}; + +void query_thread(TestConnections& test, int thread_id) +{ + uint64_t i = 0; + + auto validate = [&](MYSQL* conn){ + unsigned int port = 0; + const char* host = ""; + mariadb_get_infov(conn, MARIADB_CONNECTION_PORT, &port); + mariadb_get_infov(conn, MARIADB_CONNECTION_HOST, &host); + + test.expect(mysql_errno(conn) == 0 || errno == EADDRNOTAVAIL, + "Error opening conn to %s:%u, thread num is %d, iteration %ld, error is: %s\n", + host, port, thread_id, i, mysql_error(conn)); + + if (conn && mysql_errno(conn) == 0) + { + test.try_query(conn, "USE test"); + mysql_close(conn); + } + }; + + // Keep running the test until we exhaust all available ports + while (run && test.global_result == 0 && errno != EADDRNOTAVAIL) + { + validate(test.maxscales->open_rwsplit_connection(0)); + validate(test.maxscales->open_readconn_master_connection(0)); + validate(test.maxscales->open_readconn_slave_connection(0)); + i++; + } +} int main(int argc, char* argv[]) { - TestConnections* Test = new TestConnections(argc, argv); - int run_time = Test->smoke ? 10 : 300; - - openclose_thread_data data[threads_num]; - for (int i = 0; i < threads_num; i++) - { - data[i].i = 0; - data[i].exit_flag = 0; - data[i].Test = Test; - data[i].thread_id = i; - } + TestConnections test(argc, argv); // Tuning these kernel parameters removes any system limitations on how many // connections can be created within a short period - Test->maxscales->ssh_node_f(0, + test.maxscales->ssh_node_f(0, true, "sysctl net.ipv4.tcp_tw_reuse=1 net.ipv4.tcp_tw_recycle=1 " "net.core.somaxconn=10000 net.ipv4.tcp_max_syn_backlog=10000"); - Test->repl->execute_query_all_nodes((char*) "set global max_connections = 50000;"); - Test->repl->sync_slaves(); + test.repl->execute_query_all_nodes((char*) "set global max_connections = 50000;"); + test.repl->sync_slaves(); - pthread_t thread1[threads_num]; + std::vector threads; + constexpr int threads_num = 20; - /* Create independent threads each of them will execute function */ for (int i = 0; i < threads_num; i++) { - pthread_create(&thread1[i], NULL, query_thread1, &data[i]); + threads.emplace_back(query_thread, std::ref(test), i); } - Test->tprintf("Threads are running %d seconds \n", run_time); + constexpr int run_time = 10; + test.tprintf("Threads are running for %d seconds", run_time); - for (int i = 0; i < run_time && Test->global_result == 0; i++) + for (int i = 0; i < run_time && test.global_result == 0; i++) { sleep(1); } - for (int i = 0; i < threads_num; i++) + run = false; + + for (auto& a : threads) { - data[i].exit_flag = 1; - pthread_join(thread1[i], NULL); + a.join(); } - Test->check_maxscale_alive(0); - int rval = Test->global_result; - delete Test; - return rval; -} - -void* query_thread1(void* ptr) -{ - openclose_thread_data* data = (openclose_thread_data*) ptr; - - while (data->exit_flag == 0 && data->Test->global_result == 0) - { - MYSQL* conn1 = data->Test->maxscales->open_rwsplit_connection(0); - data->Test->add_result(mysql_errno(conn1), - "Error opening RWsplit conn, thread num is %d, iteration %li, error is: %s\n", - data->thread_id, data->i, mysql_error(conn1)); - MYSQL* conn2 = data->Test->maxscales->open_readconn_master_connection(0); - data->Test->add_result(mysql_errno( - conn2), - "Error opening ReadConn master conn, thread num is %d, iteration %li, error is: %s\n", - data->thread_id, - data->i, - mysql_error(conn2)); - MYSQL* conn3 = data->Test->maxscales->open_readconn_slave_connection(0); - data->Test->add_result(mysql_errno( - conn3), - "Error opening ReadConn master conn, thread num is %d, iteration %li, error is: %s\n", - data->thread_id, - data->i, - mysql_error(conn3)); - // USE test here is a hack to prevent Maxscale from failure; should be removed when fixed - if (conn1 != NULL) - { - data->Test->try_query(conn1, (char*) "USE test"); - mysql_close(conn1); - } - - if (conn2 != NULL) - { - data->Test->try_query(conn2, (char*) "USE test"); - mysql_close(conn2); - } - if (conn3 != NULL) - { - data->Test->try_query(conn3, (char*) "USE test"); - mysql_close(conn3); - } - - data->i++; - } - - return NULL; + test.check_maxscale_alive(0); + return test.global_result; } From 15461c1f8e502392bcfaae80ee93e58c7de333ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Fri, 2 Aug 2019 09:41:19 +0300 Subject: [PATCH 30/40] MXS-2621: Add test case The test reproduces the problem with lower_case_table_names. --- maxscale-system-test/CMakeLists.txt | 11 ++++++++ ...ale.cnf.template.mxs2621_lower_case_tables | 26 +++++++++++++++++++ .../mxs2621_lower_case_tables.cpp | 15 +++++++++++ 3 files changed, 52 insertions(+) create mode 100644 maxscale-system-test/cnf/maxscale.cnf.template.mxs2621_lower_case_tables create mode 100644 maxscale-system-test/mxs2621_lower_case_tables.cpp diff --git a/maxscale-system-test/CMakeLists.txt b/maxscale-system-test/CMakeLists.txt index e07a028ef..0dabd0bbc 100644 --- a/maxscale-system-test/CMakeLists.txt +++ b/maxscale-system-test/CMakeLists.txt @@ -389,6 +389,10 @@ add_test_executable(csmon_test.cpp csmon_test csmon_test LABELS csmon CS_BACKEND # END: ColumnStore tests # ############################################ +############################################ +# BEGIN: Normal tests # +############################################ + # Test monitor state change events when manually clearing server bits add_test_executable(false_monitor_state_change.cpp false_monitor_state_change replication LABELS mysqlmon REPL_BACKEND) @@ -972,6 +976,13 @@ add_test_executable(mxs2521_double_exec.cpp mxs2521_double_exec mxs2521_double_e # MXS-2490: Direct execution doesn't work with MaxScale add_test_executable(mxs2490_ps_execute_direct.cpp mxs2490_ps_execute_direct replication LABELS REPL_BACKEND readwritesplit) +# MXS-2621: Incorrect SQL if lower_case_table_names is used. +add_test_executable(mxs2621_lower_case_tables.cpp mxs2621_lower_case_tables mxs2621_lower_case_tables LABELS REPL_BACKEND) + +############################################ +# END: Normal tests # +############################################ + ############################################ # BEGIN: binlogrouter and avrorouter tests # ############################################ diff --git a/maxscale-system-test/cnf/maxscale.cnf.template.mxs2621_lower_case_tables b/maxscale-system-test/cnf/maxscale.cnf.template.mxs2621_lower_case_tables new file mode 100644 index 000000000..4c27e3f7d --- /dev/null +++ b/maxscale-system-test/cnf/maxscale.cnf.template.mxs2621_lower_case_tables @@ -0,0 +1,26 @@ +[maxscale] +threads=###threads### + +###server### + +[MySQL-Monitor] +type=monitor +module=mysqlmon +servers=###server_line### +user=maxskysql +password=skysql +monitor_interval=2000 + +[RW-Split-Router] +type=service +router=readwritesplit +servers=###server_line### +user=maxskysql +password=skysql + +[RW-Split-Listener] +type=listener +service=RW-Split-Router +protocol=MySQLClient +port=4006 +authenticator_options=lower_case_table_names=true diff --git a/maxscale-system-test/mxs2621_lower_case_tables.cpp b/maxscale-system-test/mxs2621_lower_case_tables.cpp new file mode 100644 index 000000000..62351237a --- /dev/null +++ b/maxscale-system-test/mxs2621_lower_case_tables.cpp @@ -0,0 +1,15 @@ +/** + * MXS-2621: Incorrect SQL if lower_case_table_names is used. + * https://jira.mariadb.org/browse/MXS-2621 + */ + +#include "testconnections.h" + +int main(int argc, char* argv[]) +{ + TestConnections test(argc, argv); + test.maxscales->connect(); + test.try_query(test.maxscales->conn_rwsplit[0], "SELECT 123"); + test.maxscales->disconnect(); + return test.global_result; +} From 110bc32b256eba2d6a28f853172ee81e5e60816c Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Fri, 2 Aug 2019 09:42:49 +0300 Subject: [PATCH 31/40] MXS-2621 Fix broken authorization SQL --- server/modules/authenticator/MySQLAuth/mysql_auth.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/modules/authenticator/MySQLAuth/mysql_auth.h b/server/modules/authenticator/MySQLAuth/mysql_auth.h index f278ce1a0..a4ad9bc7d 100644 --- a/server/modules/authenticator/MySQLAuth/mysql_auth.h +++ b/server/modules/authenticator/MySQLAuth/mysql_auth.h @@ -71,7 +71,7 @@ static const char mysqlauth_validate_user_query[] = static const char mysqlauth_validate_user_query_lower[] = "SELECT password FROM " MYSQLAUTH_USERS_TABLE_NAME " WHERE user = '%s' AND ( '%s' = host OR '%s' LIKE host)" - " AND (anydb = '1' OR LOWER('%s') IN ('', 'information_schema') OR LOWER('%s') LIKE LOWER(db)" + " AND (anydb = '1' OR LOWER('%s') IN ('', 'information_schema') OR LOWER('%s') LIKE LOWER(db))" " LIMIT 1"; /** Query that only checks if there's a matching user */ From 36355922281a6820de63b76fb76c9203861e3988 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Fri, 2 Aug 2019 09:50:01 +0300 Subject: [PATCH 32/40] Add 2.3.11 release notes and update 2.3 change log --- Documentation/Changelog.md | 1 + .../MaxScale-2.3.11-Release-Notes.md | 33 +++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 Documentation/Release-Notes/MaxScale-2.3.11-Release-Notes.md diff --git a/Documentation/Changelog.md b/Documentation/Changelog.md index bf171504f..cbd1a6af2 100644 --- a/Documentation/Changelog.md +++ b/Documentation/Changelog.md @@ -31,6 +31,7 @@ For more details, please refer to: +* [MariaDB MaxScale 2.3.11 Release Notes](Release-Notes/MaxScale-2.3.11-Release-Notes.md) * [MariaDB MaxScale 2.3.10 Release Notes](Release-Notes/MaxScale-2.3.10-Release-Notes.md) * [MariaDB MaxScale 2.3.9 Release Notes](Release-Notes/MaxScale-2.3.9-Release-Notes.md) * [MariaDB MaxScale 2.3.8 Release Notes](Release-Notes/MaxScale-2.3.8-Release-Notes.md) diff --git a/Documentation/Release-Notes/MaxScale-2.3.11-Release-Notes.md b/Documentation/Release-Notes/MaxScale-2.3.11-Release-Notes.md new file mode 100644 index 000000000..e05f9a282 --- /dev/null +++ b/Documentation/Release-Notes/MaxScale-2.3.11-Release-Notes.md @@ -0,0 +1,33 @@ +# MariaDB MaxScale 2.3.11 Release Notes + +Release 2.3.11 is a GA release. + +This document describes the changes in release 2.3.11, when compared to the +previous release in the same series. + +For any problems you encounter, please consider submitting a bug +report on [our Jira](https://jira.mariadb.org/projects/MXS). + +## Bug fixes + +* [MXS-2621](https://jira.mariadb.org/browse/MXS-2621) Incorrect SQL if lower_case_table_names is used. + +## Known Issues and Limitations + +There are some limitations and known issues within this version of MaxScale. +For more information, please refer to the [Limitations](../About/Limitations.md) document. + +## Packaging + +RPM and Debian packages are provided for supported the Linux distributions. + +Packages can be downloaded [here](https://mariadb.com/downloads/#mariadb_platform-mariadb_maxscale). + +## Source Code + +The source code of MaxScale is tagged at GitHub with a tag, which is identical +with the version of MaxScale. For instance, the tag of version X.Y.Z of MaxScale +is `maxscale-X.Y.Z`. Further, the default branch is always the latest GA version +of MaxScale. + +The source code is available [here](https://github.com/mariadb-corporation/MaxScale). From 83db2a4dc1126f85b573d856fdbfab640bb35003 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Fri, 2 Aug 2019 09:54:17 +0300 Subject: [PATCH 33/40] Update 2.3 maintenance release number --- VERSION23.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION23.cmake b/VERSION23.cmake index 2f928171c..f6d1dc661 100644 --- a/VERSION23.cmake +++ b/VERSION23.cmake @@ -5,7 +5,7 @@ set(MAXSCALE_VERSION_MAJOR "2" CACHE STRING "Major version") set(MAXSCALE_VERSION_MINOR "3" CACHE STRING "Minor version") -set(MAXSCALE_VERSION_PATCH "11" CACHE STRING "Patch version") +set(MAXSCALE_VERSION_PATCH "12" CACHE STRING "Patch version") # This should only be incremented if a package is rebuilt set(MAXSCALE_BUILD_NUMBER 1 CACHE STRING "Release number") From f8150b776cbd702473bdec23a20c478c30bed9ab Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Fri, 2 Aug 2019 15:46:03 +0300 Subject: [PATCH 34/40] Update 2.3.11 release date --- Documentation/Release-Notes/MaxScale-2.3.11-Release-Notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/Release-Notes/MaxScale-2.3.11-Release-Notes.md b/Documentation/Release-Notes/MaxScale-2.3.11-Release-Notes.md index e05f9a282..52c2f9204 100644 --- a/Documentation/Release-Notes/MaxScale-2.3.11-Release-Notes.md +++ b/Documentation/Release-Notes/MaxScale-2.3.11-Release-Notes.md @@ -1,4 +1,4 @@ -# MariaDB MaxScale 2.3.11 Release Notes +# MariaDB MaxScale 2.3.11 Release Notes -- 2019-08-02 Release 2.3.11 is a GA release. From fd72332ea471b73091f8cf567997e075ccd7ac4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 1 Aug 2019 16:52:41 +0300 Subject: [PATCH 35/40] Improve master failure error message The message will now always contain the server name. --- .../routing/readwritesplit/rwsplitsession.cc | 20 +++++-------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/server/modules/routing/readwritesplit/rwsplitsession.cc b/server/modules/routing/readwritesplit/rwsplitsession.cc index a5ddb2585..c3693465a 100644 --- a/server/modules/routing/readwritesplit/rwsplitsession.cc +++ b/server/modules/routing/readwritesplit/rwsplitsession.cc @@ -1012,21 +1012,11 @@ void RWSplitSession::handleError(GWBUF* errmsgbuf, if (!can_continue) { - if (!backend->is_master() && !backend->server()->master_err_is_logged) - { - MXS_ERROR("Server %s (%s) lost the master status while waiting" - " for a result. Client sessions will be closed.", - backend->name(), - backend->uri()); - backend->server()->master_err_is_logged = true; - } - else - { - int64_t idle = mxs_clock() - backend->dcb()->last_read; - MXS_ERROR("Lost connection to the master server, closing session.%s " - "Connection has been idle for %.1f seconds. Error caused by: %s", - errmsg.c_str(), (float)idle / 10.f, extract_error(errmsgbuf).c_str()); - } + int64_t idle = mxs_clock() - backend->dcb()->last_read; + MXS_ERROR("Lost connection to the master server '%s', closing session.%s " + "Connection has been idle for %.1f seconds. Error caused by: %s", + backend->name(), errmsg.c_str(), (float)idle / 10.f, + extract_error(errmsgbuf).c_str()); } // Decrement the expected response count only if we know we can continue the sesssion. From 2fa336d142661f7b902ed36ae87a0ad98f97febb Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Wed, 7 Aug 2019 14:58:19 +0300 Subject: [PATCH 36/40] create symlink for sles15 --- BUILD/mdbci/copy_repos.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/BUILD/mdbci/copy_repos.sh b/BUILD/mdbci/copy_repos.sh index 71c3483ec..d07cc356e 100755 --- a/BUILD/mdbci/copy_repos.sh +++ b/BUILD/mdbci/copy_repos.sh @@ -28,10 +28,14 @@ if [ "$box_type" == "RPM" ] ; then cd $path_prefix/$platform ln -s $platform_version "$platform_version"server ln -s $platform_version "$platform_version"Server + cd .. if [ "$platform" == "centos" ] ; then - cd .. ln -s centos rhel fi + if [ "$platform" == "opensuse" ] ; then + ln -s opensuse/$paltform_version sles/$paltform_version + fi + eval "cat < Date: Wed, 7 Aug 2019 15:41:18 +0300 Subject: [PATCH 37/40] add sles directory creation --- BUILD/mdbci/copy_repos.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/BUILD/mdbci/copy_repos.sh b/BUILD/mdbci/copy_repos.sh index d07cc356e..4cd37de3c 100755 --- a/BUILD/mdbci/copy_repos.sh +++ b/BUILD/mdbci/copy_repos.sh @@ -33,6 +33,7 @@ if [ "$box_type" == "RPM" ] ; then ln -s centos rhel fi if [ "$platform" == "opensuse" ] ; then + mkdir -p sles ln -s opensuse/$paltform_version sles/$paltform_version fi From bb43e6193e068c36edf5abecede19d7db7b6eece Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Wed, 7 Aug 2019 16:00:12 +0300 Subject: [PATCH 38/40] fix typo in copy_repos.sh --- BUILD/mdbci/copy_repos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BUILD/mdbci/copy_repos.sh b/BUILD/mdbci/copy_repos.sh index 4cd37de3c..9a3bd91f3 100755 --- a/BUILD/mdbci/copy_repos.sh +++ b/BUILD/mdbci/copy_repos.sh @@ -34,7 +34,7 @@ if [ "$box_type" == "RPM" ] ; then fi if [ "$platform" == "opensuse" ] ; then mkdir -p sles - ln -s opensuse/$paltform_version sles/$paltform_version + ln -s opensuse/$platform_version sles/$platform_version fi From 169fb256edb014af0068be4861d0de6565c6a21e Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Wed, 7 Aug 2019 16:02:40 +0300 Subject: [PATCH 39/40] Create symlink for sles15 Currently build is executed for OpenSuse 15 and SLES15 repo is not available. To avoid missing repo symlink sles -> opensuse is created --- BUILD/mdbci/copy_repos.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/BUILD/mdbci/copy_repos.sh b/BUILD/mdbci/copy_repos.sh index 71c3483ec..9a3bd91f3 100755 --- a/BUILD/mdbci/copy_repos.sh +++ b/BUILD/mdbci/copy_repos.sh @@ -28,10 +28,15 @@ if [ "$box_type" == "RPM" ] ; then cd $path_prefix/$platform ln -s $platform_version "$platform_version"server ln -s $platform_version "$platform_version"Server + cd .. if [ "$platform" == "centos" ] ; then - cd .. ln -s centos rhel fi + if [ "$platform" == "opensuse" ] ; then + mkdir -p sles + ln -s opensuse/$platform_version sles/$platform_version + fi + eval "cat < Date: Wed, 7 Aug 2019 19:45:52 +0300 Subject: [PATCH 40/40] fix sles link creation --- BUILD/mdbci/copy_repos.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/BUILD/mdbci/copy_repos.sh b/BUILD/mdbci/copy_repos.sh index 9a3bd91f3..517ef82dd 100755 --- a/BUILD/mdbci/copy_repos.sh +++ b/BUILD/mdbci/copy_repos.sh @@ -34,7 +34,9 @@ if [ "$box_type" == "RPM" ] ; then fi if [ "$platform" == "opensuse" ] ; then mkdir -p sles - ln -s opensuse/$platform_version sles/$platform_version + cd sles + ln -s ../opensuse/$platform_version $platform_version + cd .. fi