From c3dff9c60c43ce560dc508f6c3e26ff4e9e55bcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Fri, 29 May 2020 08:31:35 +0300 Subject: [PATCH 1/6] MXS-3014: Add missing global parameters to REST API A few global parameters weren't included in the list of parameters. A few non-parameter values that were in the parameter object were moved out into the attributes object. Sorted the parameter alphabetically. --- server/core/config.cc | 85 +++++++++++++++++++++---------------------- 1 file changed, 42 insertions(+), 43 deletions(-) diff --git a/server/core/config.cc b/server/core/config.cc index 3ce0388a4..fbd784459 100644 --- a/server/core/config.cc +++ b/server/core/config.cc @@ -2344,7 +2344,7 @@ static int handle_global_item(const char* name, const char* value) return 0; } } - else if (strcmp(name, "sql_mode") == 0) + else if (strcmp(name, CN_SQL_MODE) == 0) { if (strcasecmp(value, "default") == 0) { @@ -4524,55 +4524,53 @@ int config_parse_server_list(const char* servers, char*** output_array) json_t* config_maxscale_to_json(const char* host) { - json_t* param = json_object(); - json_object_set_new(param, "libdir", json_string(get_libdir())); - json_object_set_new(param, "datadir", json_string(get_datadir())); - json_object_set_new(param, "process_datadir", json_string(get_process_datadir())); - json_object_set_new(param, "cachedir", json_string(get_cachedir())); - json_object_set_new(param, "configdir", json_string(get_configdir())); - json_object_set_new(param, "config_persistdir", json_string(get_config_persistdir())); - json_object_set_new(param, "module_configdir", json_string(get_module_configdir())); - json_object_set_new(param, "piddir", json_string(get_piddir())); - json_object_set_new(param, "logdir", json_string(get_logdir())); - json_object_set_new(param, "langdir", json_string(get_langdir())); - json_object_set_new(param, "execdir", json_string(get_execdir())); - json_object_set_new(param, "connector_plugindir", json_string(get_connector_plugindir())); - json_object_set_new(param, CN_THREADS, json_integer(config_threadcount())); - json_object_set_new(param, CN_THREAD_STACK_SIZE, json_integer(config_thread_stack_size())); - json_object_set_new(param, CN_WRITEQ_HIGH_WATER, json_integer(config_writeq_high_water())); - json_object_set_new(param, CN_WRITEQ_LOW_WATER, json_integer(config_writeq_low_water())); - MXS_CONFIG* cnf = config_get_global_options(); + json_t* param = json_object(); + json_object_set_new(param, CN_ADMIN_AUTH, json_boolean(cnf->admin_auth)); + json_object_set_new(param, CN_ADMIN_ENABLED, json_boolean(cnf->admin_enabled)); + json_object_set_new(param, CN_ADMIN_HOST, json_string(cnf->admin_host)); + json_object_set_new(param, CN_ADMIN_LOG_AUTH_FAILURES, json_boolean(cnf->admin_log_auth_failures)); + json_object_set_new(param, CN_ADMIN_PORT, json_integer(cnf->admin_port)); + json_object_set_new(param, CN_ADMIN_SSL_CA_CERT, json_string(cnf->admin_ssl_ca_cert)); + json_object_set_new(param, CN_ADMIN_SSL_CERT, json_string(cnf->admin_ssl_cert)); + json_object_set_new(param, CN_ADMIN_SSL_KEY, json_string(cnf->admin_ssl_key)); json_object_set_new(param, CN_AUTH_CONNECT_TIMEOUT, json_integer(cnf->auth_conn_timeout)); json_object_set_new(param, CN_AUTH_READ_TIMEOUT, json_integer(cnf->auth_read_timeout)); json_object_set_new(param, CN_AUTH_WRITE_TIMEOUT, json_integer(cnf->auth_write_timeout)); - json_object_set_new(param, CN_SKIP_PERMISSION_CHECKS, json_boolean(cnf->skip_permission_checks)); - json_object_set_new(param, CN_ADMIN_AUTH, json_boolean(cnf->admin_auth)); - json_object_set_new(param, CN_ADMIN_ENABLED, json_boolean(cnf->admin_enabled)); - json_object_set_new(param, CN_ADMIN_LOG_AUTH_FAILURES, json_boolean(cnf->admin_log_auth_failures)); - json_object_set_new(param, CN_ADMIN_HOST, json_string(cnf->admin_host)); - json_object_set_new(param, CN_ADMIN_PORT, json_integer(cnf->admin_port)); - json_object_set_new(param, CN_ADMIN_SSL_KEY, json_string(cnf->admin_ssl_key)); - json_object_set_new(param, CN_ADMIN_SSL_CERT, json_string(cnf->admin_ssl_cert)); - json_object_set_new(param, CN_ADMIN_SSL_CA_CERT, json_string(cnf->admin_ssl_ca_cert)); - json_object_set_new(param, CN_PASSIVE, json_boolean(cnf->passive)); - - json_object_set_new(param, CN_QUERY_CLASSIFIER, json_string(cnf->qc_name)); - - if (cnf->qc_args) - { - json_object_set_new(param, CN_QUERY_CLASSIFIER_ARGS, json_string(cnf->qc_args)); - } - - json_object_set_new(param, - CN_QUERY_CLASSIFIER_CACHE_SIZE, - json_integer(cnf->qc_cache_properties.max_size)); - - json_object_set_new(param, CN_RETAIN_LAST_STATEMENTS, json_integer(session_get_retain_last_statements())); + json_object_set_new(param, CN_CACHEDIR, json_string(get_cachedir())); + json_object_set_new(param, CN_CONNECTOR_PLUGINDIR, json_string(get_connector_plugindir())); + json_object_set_new(param, CN_DATADIR, json_string(get_datadir())); json_object_set_new(param, CN_DUMP_LAST_STATEMENTS, json_string(session_get_dump_statements_str())); - json_object_set_new(param, CN_SESSION_TRACE, json_integer(session_get_session_trace())); + json_object_set_new(param, CN_EXECDIR, json_string(get_execdir())); + json_object_set_new(param, CN_LANGUAGE, json_string(get_langdir())); + json_object_set_new(param, CN_LIBDIR, json_string(get_libdir())); json_object_set_new(param, CN_LOAD_PERSISTED_CONFIGS, json_boolean(cnf->load_persisted_configs)); + json_object_set_new(param, CN_LOCAL_ADDRESS, + cnf->local_address ? json_string(cnf->local_address) : json_null()); + json_object_set_new(param, CN_LOGDIR, json_string(get_logdir())); + json_object_set_new(param, CN_MODULE_CONFIGDIR, json_string(get_module_configdir())); + json_object_set_new(param, CN_PASSIVE, json_boolean(cnf->passive)); + json_object_set_new(param, CN_PERSISTDIR, json_string(get_config_persistdir())); + json_object_set_new(param, CN_PIDDIR, json_string(get_piddir())); + json_object_set_new(param, CN_QUERY_CLASSIFIER, json_string(cnf->qc_name)); + json_object_set_new(param, CN_QUERY_CLASSIFIER_ARGS, + cnf->qc_args ? json_string(cnf->qc_args) : json_null()); + json_object_set_new(param, CN_QUERY_CLASSIFIER_CACHE_SIZE, + json_integer(cnf->qc_cache_properties.max_size)); + json_object_set_new(param, CN_QUERY_RETRIES, json_integer(cnf->query_retries)); + json_object_set_new(param, CN_QUERY_RETRY_TIMEOUT, json_integer(cnf->query_retry_timeout)); + json_object_set_new(param, CN_RETAIN_LAST_STATEMENTS, json_integer(session_get_retain_last_statements())); + json_object_set_new(param, CN_SESSION_TRACE, json_integer(session_get_session_trace())); + json_object_set_new(param, CN_SKIP_PERMISSION_CHECKS, json_boolean(cnf->skip_permission_checks)); + json_object_set_new(param, CN_SQL_MODE, + json_string(cnf->qc_sql_mode == QC_SQL_MODE_DEFAULT ? "default" : "oracle")); + json_object_set_new(param, CN_SUBSTITUTE_VARIABLES, json_boolean(cnf->substitute_variables)); + json_object_set_new(param, CN_THREADS, json_integer(config_threadcount())); + json_object_set_new(param, CN_THREAD_STACK_SIZE, json_integer(config_thread_stack_size())); + json_object_set_new(param, CN_USERS_REFRESH_TIME, json_integer(cnf->users_refresh_time)); + json_object_set_new(param, CN_WRITEQ_HIGH_WATER, json_integer(config_writeq_high_water())); + json_object_set_new(param, CN_WRITEQ_LOW_WATER, json_integer(config_writeq_low_water())); json_t* attr = json_object(); time_t started = maxscale_started(); @@ -4583,6 +4581,7 @@ json_t* config_maxscale_to_json(const char* host) json_object_set_new(attr, "started_at", json_string(http_to_date(started).c_str())); json_object_set_new(attr, "activated_at", json_string(http_to_date(activated).c_str())); json_object_set_new(attr, "uptime", json_integer(maxscale_uptime())); + json_object_set_new(attr, "process_datadir", json_string(get_process_datadir())); json_t* obj = json_object(); json_object_set_new(obj, CN_ATTRIBUTES, attr); From 0f1f9426c3bcbadb19a370de41269544f6ee13b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Mon, 1 Jun 2020 09:16:01 +0300 Subject: [PATCH 2/6] MXS-2983: Document list option formats The options that take lists of objects as arguments did not properly document the expected format. --- maxctrl/lib/create.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/maxctrl/lib/create.js b/maxctrl/lib/create.js index 2a54114a2..05d33a938 100644 --- a/maxctrl/lib/create.js +++ b/maxctrl/lib/create.js @@ -80,11 +80,11 @@ exports.builder = function(yargs) { // Create server .group(['services', 'monitors'], 'Create server options:') .option('services', { - describe: 'Link the created server to these services', + describe: 'Link the created server to these services. All non-option arguments after --services are interpreted as service names e.g. `--services my-service-1 my-service-2`.', type: 'array' }) .option('monitors', { - describe: 'Link the created server to these monitors', + describe: 'Link the created server to this monitor', type: 'array' }) .command('server ', 'Create a new server', function(yargs) { @@ -135,7 +135,7 @@ exports.builder = function(yargs) { // Create monitor .group(['servers', 'monitor-user', 'monitor-password'], 'Create monitor options:') .option('servers', { - describe: 'Link the created monitor to these servers', + describe: 'Link the created monitor to these servers. All non-option arguments after --servers are interpreted as server names e.g. `--servers srv1 srv2 srv3`.', type: 'array' }) .option('monitor-user', { @@ -194,7 +194,7 @@ exports.builder = function(yargs) { // Create service .group(['servers', 'filters'], 'Create service options:') .option('servers', { - describe: 'Link the created service to these servers', + describe: 'Link the created service to these servers. All non-option arguments after --servers are interpreted as server names e.g. `--servers srv1 srv2 srv3`.', type: 'array' }) .option('filters', { From ce437bc77936158193734516816cfa95dcd43de7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Mon, 1 Jun 2020 09:20:46 +0300 Subject: [PATCH 3/6] MXS-3016: Rename --monitors to --monitor Renamed the option and changed the type to a string argument. The alias for --monitors still allows old code to use the same format. --- maxctrl/lib/create.js | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/maxctrl/lib/create.js b/maxctrl/lib/create.js index 05d33a938..64bbceaac 100644 --- a/maxctrl/lib/create.js +++ b/maxctrl/lib/create.js @@ -83,9 +83,10 @@ exports.builder = function(yargs) { describe: 'Link the created server to these services. All non-option arguments after --services are interpreted as service names e.g. `--services my-service-1 my-service-2`.', type: 'array' }) - .option('monitors', { + .option('monitor', { + alias: 'monitors', describe: 'Link the created server to this monitor', - type: 'array' + type: 'string' }) .command('server ', 'Create a new server', function(yargs) { return yargs.epilog('The created server will not be used by any services or monitors ' + @@ -121,10 +122,8 @@ exports.builder = function(yargs) { } } - if (argv.monitors) { - for (i = 0; i < argv.monitors.length; i++) { - _.set(server, 'data.relationships.monitors.data[' + i + ']', {id: argv.monitors[i], type: 'monitors'}) - } + if (argv.monitor) { + _.set(server, 'data.relationships.monitors.data[0]', {id: argv.monitor, type: 'monitors'}) } maxctrl(argv, function(host) { From 2e1b0c5ddf37674ba48f2c1e09f63c6a1aab9bfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Mon, 1 Jun 2020 09:26:33 +0300 Subject: [PATCH 4/6] MXS-2984: Fix `list listeners` Backported the changed from 2.5 to 2.3 and added a test case. --- maxctrl/lib/common.js | 2 +- maxctrl/lib/list.js | 2 +- maxctrl/test/diagnostics.js | 10 ++++++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/maxctrl/lib/common.js b/maxctrl/lib/common.js index 89627f9fc..167646846 100644 --- a/maxctrl/lib/common.js +++ b/maxctrl/lib/common.js @@ -197,7 +197,7 @@ module.exports = function() { row = [] fields.forEach(function(p) { - var v = _.getPath(i, p[p.name], '') + var v = _.getPath(i, p.path, '') if (Array.isArray(v) && typeof(v[0]) != 'object') { v = v.join(', ') diff --git a/maxctrl/lib/list.js b/maxctrl/lib/list.js index 776d6f473..b17689ecf 100644 --- a/maxctrl/lib/list.js +++ b/maxctrl/lib/list.js @@ -87,7 +87,7 @@ const list_listeners_fields = [ }, { name: 'Host', - path: 'attributes.parameters.host', + path: 'attributes.parameters.address', description: 'The address or socket where the listener listens' }, { diff --git a/maxctrl/test/diagnostics.js b/maxctrl/test/diagnostics.js index 1ca3ae672..8722356bc 100644 --- a/maxctrl/test/diagnostics.js +++ b/maxctrl/test/diagnostics.js @@ -42,6 +42,16 @@ describe("Diagnostic Commands", function() { }); }) + it('MXS-2984: Malformed `list listeners` output', function() { + return doCommand('list listeners RW-Split-Router --tsv') + .then(res => { + var d = res.split('\t') + d[0].should.equal('RW-Split-Listener') + d[1].should.equal('4006') + d[2].should.equal('::') + }) + }); + after(stopMaxScale) }); From 11960a1e931b496a29186540691e2fe88b087b34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Mon, 1 Jun 2020 16:22:14 +0300 Subject: [PATCH 5/6] Document transaction_replay_max_size inner workings The documentation didn't explain how to estimate how much memory a transaction takes and whether it was a global limit or a per session limit. --- Documentation/Routers/ReadWriteSplit.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Documentation/Routers/ReadWriteSplit.md b/Documentation/Routers/ReadWriteSplit.md index c0469d02e..fa6c76b3f 100644 --- a/Documentation/Routers/ReadWriteSplit.md +++ b/Documentation/Routers/ReadWriteSplit.md @@ -446,8 +446,16 @@ be replayed successfully but it will still be attempted. ### `transaction_replay_max_size` The limit on transaction size for transaction replay in bytes. Any transaction -that exceeds this limit will not be replayed. The default size limit is 1 -MiB. Read [the configuration guide](../Getting-Started/Configuration-Guide.md#sizes) +that exceeds this limit will not be replayed. The default value is 1 MiB. This +limit applies at a session level which means that the total peak memory +consumption can be `transaction_replay_max_size` times the number of client +connections. + +The amount of memory needed to store a particular transaction will be slightly +larger than the length in bytes of the SQL used in the transaction. If the limit +is ever exceeded, a message will be logged at the info level. + +Read [the configuration guide](../Getting-Started/Configuration-Guide.md#sizes) for more details on size type parameters in MaxScale. ### `optimistic_trx` From cb8b4546cb0cacfadefc1dc779bb5bee1ef1eb89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Mon, 1 Jun 2020 19:28:48 +0300 Subject: [PATCH 6/6] Don't use client SHA1 for fake responses When a fake handshake response is generated for a connection that hasn't received the server's handshake, the client's SHA1 would be used with a static scramble. This, in theory, would weaken the authentication to some extend so to completely prevent this, a null password is used. This removes any possibility of the password being exposed. --- server/modules/protocol/MySQL/mariadbbackend/mysql_backend.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/modules/protocol/MySQL/mariadbbackend/mysql_backend.cc b/server/modules/protocol/MySQL/mariadbbackend/mysql_backend.cc index e23890ff5..3362a022a 100644 --- a/server/modules/protocol/MySQL/mariadbbackend/mysql_backend.cc +++ b/server/modules/protocol/MySQL/mariadbbackend/mysql_backend.cc @@ -1416,6 +1416,10 @@ static int gw_backend_close(DCB* dcb) { MYSQL_session client; gw_get_shared_session_auth_info(dcb, &client); + + // Don't use the actual client SHA1. This prevents the password from being used with the constant + // null scramble we use in these cases. + memset(client.client_sha1, 0, sizeof(client.client_sha1)); memset(proto->scramble, 0, sizeof(proto->scramble)); dcb_write(dcb, gw_generate_auth_response(&client, proto, false, false, 0)); }