diff --git a/Documentation/REST-API/Resources-MaxScale.md b/Documentation/REST-API/Resources-MaxScale.md index 1c1b1fe66..c6e067632 100644 --- a/Documentation/REST-API/Resources-MaxScale.md +++ b/Documentation/REST-API/Resources-MaxScale.md @@ -52,12 +52,17 @@ file locations, configuration options and version information. "admin_ssl_key": "", "admin_ssl_cert": "", "admin_ssl_ca_cert": "", - "query_classifier": "" + "query_classifier": "", + "query_classifier_cache_size": 416215859, + "retain_last_statements": 2, + "dump_last_statements": "never", + "load_persisted_configs": false }, - "version": "2.2.0", - "commit": "aa1a413cd961d467083d1974c2a027f612201845", - "started_at": "Wed, 06 Sep 2017 06:51:54 GMT", - "uptime": 1227 + "version": "2.3.6", + "commit": "47158faf12c156775c39388652a77f8a8c542d28", + "started_at": "Thu, 04 Apr 2019 21:04:06 GMT", + "activated_at": "Thu, 04 Apr 2019 21:04:06 GMT", + "uptime": 337 }, "id": "maxscale", "type": "maxscale" diff --git a/Documentation/Release-Notes/MaxScale-2.3.6-Release-Notes.md b/Documentation/Release-Notes/MaxScale-2.3.6-Release-Notes.md new file mode 100644 index 000000000..0ac6a919e --- /dev/null +++ b/Documentation/Release-Notes/MaxScale-2.3.6-Release-Notes.md @@ -0,0 +1,48 @@ +# MariaDB MaxScale 2.3.6 Release Notes + +Release 2.3.6 is a GA release. + +This document describes the changes in release 2.3.6, when compared to the +previous release in the same series. + +For any problems you encounter, please consider submitting a bug +report on [our Jira](https://jira.mariadb.org/projects/MXS). + +## New Features + +* [MXS-2417](https://jira.mariadb.org/browse/MXS-2417) MaxScale main config should take precedence over runtime config on restart + +### REST API & MaxCtrl: Hard maintenance mode + +The new `--force` option for the `set server` command in MaxCtrl allows all +connections to the server in question to be closed when it is set into +maintenance mode. This causes idle connections to be closed immediately. + +For more information, read the +[REST-API](../REST-API/Resources-Server.md#set-server-state) documentation for +the `set` endpoint. + +## Bug fixes + +* [MXS-2419](https://jira.mariadb.org/browse/MXS-2419) Hangs on query during multiple transaction replays +* [MXS-2418](https://jira.mariadb.org/browse/MXS-2418) Crash on transaction replay if log_info is on and session starts with no master + +## Known Issues and Limitations + +There are some limitations and known issues within this version of MaxScale. +For more information, please refer to the [Limitations](../About/Limitations.md) document. + +## Packaging + +RPM and Debian packages are provided for supported the Linux distributions. + +Packages can be downloaded [here](https://mariadb.com/downloads/mariadb-tx/maxscale). + +## Source Code + +The source code of MaxScale is tagged at GitHub with a tag, which is identical +with the version of MaxScale. For instance, the tag of version X.Y.Z of MaxScale +is `maxscale-X.Y.Z`. Further, the default branch is always the latest GA version +of MaxScale. + +The source code is available [here](https://github.com/mariadb-corporation/MaxScale). diff --git a/include/maxscale/dcb.hh b/include/maxscale/dcb.hh index 948020a61..e63658598 100644 --- a/include/maxscale/dcb.hh +++ b/include/maxscale/dcb.hh @@ -243,6 +243,7 @@ int dcb_write(DCB*, GWBUF*); DCB* dcb_alloc(DCB::Role, MXS_SESSION*); DCB* dcb_connect(struct SERVER*, MXS_SESSION*, const char*); int dcb_read(DCB*, GWBUF**, int); +int dcb_bytes_readable(DCB* dcb); int dcb_drain_writeq(DCB*); void dcb_close(DCB*); diff --git a/include/maxscale/protocol/rwbackend.hh b/include/maxscale/protocol/rwbackend.hh index ee63ef8e7..fd57f9538 100644 --- a/include/maxscale/protocol/rwbackend.hh +++ b/include/maxscale/protocol/rwbackend.hh @@ -40,6 +40,7 @@ enum reply_state_t REPLY_STATE_START, /**< Query sent to backend */ REPLY_STATE_DONE, /**< Complete reply received */ REPLY_STATE_RSET_COLDEF, /**< Resultset response, waiting for column definitions */ + REPLY_STATE_RSET_COLDEF_EOF,/**< Resultset response, waiting for EOF for column definitions */ REPLY_STATE_RSET_ROWS /**< Resultset response, waiting for rows */ }; @@ -70,6 +71,30 @@ public: return m_reply_state; } + const char* reply_state_str() const + { + switch (m_reply_state) + { + case REPLY_STATE_START: + return "START"; + + case REPLY_STATE_DONE: + return "DONE"; + + case REPLY_STATE_RSET_COLDEF: + return "COLDEF"; + + case REPLY_STATE_RSET_COLDEF_EOF: + return "COLDEF_EOF"; + + case REPLY_STATE_RSET_ROWS: + return "ROWS"; + + default: + return "UNKNOWN"; + } + } + void add_ps_handle(uint32_t id, uint32_t handle); uint32_t get_ps_handle(uint32_t id) const; @@ -132,6 +157,9 @@ public: return m_reply_state == REPLY_STATE_DONE; } + void process_packets(GWBUF* buffer); + void process_reply_start(mxs::Buffer::iterator it); + // Controlled by the session ResponseStat& response_stat(); @@ -152,6 +180,7 @@ private: uint32_t m_expected_rows; /**< Number of rows a COM_STMT_FETCH is retrieving */ bool m_local_infile_requested; /**< Whether a LOCAL INFILE was requested */ ResponseStat m_response_stat; + uint64_t m_num_coldefs = 0; inline bool is_opening_cursor() const { diff --git a/include/maxscale/session.hh b/include/maxscale/session.hh index cc085ce4d..230368d18 100644 --- a/include/maxscale/session.hh +++ b/include/maxscale/session.hh @@ -555,6 +555,11 @@ char* session_set_variable_value(MXS_SESSION* session, */ void session_set_retain_last_statements(uint32_t n); +/** + * Get retain_last_statements + */ +uint32_t session_get_retain_last_statements(); + /** * @brief Retain provided statement, if configured to do so. * @@ -604,6 +609,11 @@ void session_set_dump_statements(session_dump_statements_t value); */ session_dump_statements_t session_get_dump_statements(); +/** + * String version of session_get_dump_statements + */ +const char* session_get_dump_statements_str(); + /** * @brief Route the query again after a delay * diff --git a/maxctrl/lib/classify.js b/maxctrl/lib/classify.js index 0f50f6fe6..06d1928de 100644 --- a/maxctrl/lib/classify.js +++ b/maxctrl/lib/classify.js @@ -28,11 +28,13 @@ exports.handler = function (argv) { return doRequest(host, 'maxscale/query_classifier/classify?sql=' + argv.statement, (res) => { - var a = res.data.attributes.parameters.functions.map((f) => { - return f.name + ': (' + f.arguments.join(', ') + ')' - }); + if (res.data.attributes.parameters.functions) { + var a = res.data.attributes.parameters.functions.map((f) => { + return f.name + ': (' + f.arguments.join(', ') + ')' + }); - res.data.attributes.parameters.functions = a; + res.data.attributes.parameters.functions = a; + } return formatResource(classify_fields, res.data) }) @@ -44,14 +46,6 @@ exports.builder = function(yargs) { .epilog('Classify the statement using MaxScale and display the result. ' + 'The possible values for "Parse result", "Type mask" and "Operation" ' + 'can be looked up in ' + - 'https://github.com/mariadb-corporation/MaxScale/blob/' + - '2.3/include/maxscale/query_classifier.h') + 'https://github.com/mariadb-corporation/MaxScale/blob/2.3/include/maxscale/query_classifier.h') .help() - .command('*', 'the default command', {}, function(argv) { - console.log("*"); - maxctrl(argv, function(host) { - console.log(argv.statement); - return error('Unknown command. See output of `help stop` for a list of commands.') - }) - }) } diff --git a/maxctrl/test/classify.js b/maxctrl/test/classify.js new file mode 100644 index 000000000..d50892182 --- /dev/null +++ b/maxctrl/test/classify.js @@ -0,0 +1,27 @@ +require('../test_utils.js')() + +describe("Classify Commands", function() { + before(startMaxScale) + + it('classifies query', function() { + return doCommand('--tsv classify SELECT\t1') + .should.eventually.match(/QC_QUERY_PARSED/) + }) + + it('classifies query with function', function() { + return doCommand('--tsv classify SELECT\tspecial_function("hello",5)') + .should.eventually.match(/special_function/) + }) + + it('classifies invalid query', function() { + return doCommand('--tsv classify This-should-fail') + .should.eventually.match(/QC_QUERY_INVALID/) + }) + + it('rejects no query', function() { + return doCommand('classify') + .should.be.rejected + }) + + after(stopMaxScale) +}); diff --git a/maxctrl/test/states.js b/maxctrl/test/states.js index f104df737..502276bd7 100644 --- a/maxctrl/test/states.js +++ b/maxctrl/test/states.js @@ -22,6 +22,20 @@ describe("Set/Clear Commands", function() { }) }) + it('force maintenance mode', function() { + return verifyCommand('set server server1 maintenance --force', 'servers/server1') + .then(function(res) { + res.data.attributes.state.should.match(/Maintenance/) + }) + }) + + it('clear maintenance mode', function() { + return verifyCommand('clear server server1 maintenance', 'servers/server1') + .then(function(res) { + res.data.attributes.state.should.not.match(/Maintenance/) + }) + }) + it('reject set incorrect state', function() { return doCommand('set server server2 something') .should.be.rejected diff --git a/maxscale-system-test/CMakeLists.txt b/maxscale-system-test/CMakeLists.txt index 1bcf6e892..fbf9ad175 100644 --- a/maxscale-system-test/CMakeLists.txt +++ b/maxscale-system-test/CMakeLists.txt @@ -940,6 +940,9 @@ add_test_executable(mxs2326_hint_clone.cpp mxs2326_hint_clone mxs2326_hint_clone # MXS-2313: `rank` functional test add_test_executable(mxs2313_rank.cpp mxs2313_rank mxs2313_rank LABELS readwritesplit REPL_BACKEND) +# MXS-2417: Ignore persisted configs with load_persisted_configs=false +add_test_executable(mxs2417_ignore_persisted_cnf.cpp mxs2417_ignore_persisted_cnf mxs2417_ignore_persisted_cnf LABELS REPL_BACKEND) + ############################################ # BEGIN: binlogrouter and avrorouter tests # ############################################ diff --git a/maxscale-system-test/cnf/maxscale.cnf.template.mxs2417_ignore_persisted_cnf b/maxscale-system-test/cnf/maxscale.cnf.template.mxs2417_ignore_persisted_cnf new file mode 100644 index 000000000..2e90be3d1 --- /dev/null +++ b/maxscale-system-test/cnf/maxscale.cnf.template.mxs2417_ignore_persisted_cnf @@ -0,0 +1,3 @@ +[maxscale] +threads=###threads### +load_persisted_configs=false diff --git a/maxscale-system-test/labels_table.cpp b/maxscale-system-test/labels_table.cpp index 79910be7f..c4a6e3a6e 100644 --- a/maxscale-system-test/labels_table.cpp +++ b/maxscale-system-test/labels_table.cpp @@ -2,6 +2,7 @@ #include #include #include "labels_table.h" +#include "testconnections.h" std::string get_mdbci_lables(const char *labels_string) { @@ -9,12 +10,20 @@ std::string get_mdbci_lables(const char *labels_string) for (size_t i = 0; i < sizeof(labels_table) / sizeof(labels_table_t); i++) { - printf("%lu\t %s\n", i, labels_table[i].test_label); + if (TestConnections::verbose) + { + printf("%lu\t %s\n", i, labels_table[i].test_label); + } + if (strstr(labels_string, labels_table[i].test_label)) { mdbci_labels += "," + std::string(labels_table[i].mdbci_label); } } - printf("mdbci labels %s\n", mdbci_labels.c_str()); + + if (TestConnections::verbose) + { + printf("mdbci labels %s\n", mdbci_labels.c_str()); + } return mdbci_labels; } diff --git a/maxscale-system-test/mariadb_nodes.cpp b/maxscale-system-test/mariadb_nodes.cpp index 0834ccf04..426ec134e 100644 --- a/maxscale-system-test/mariadb_nodes.cpp +++ b/maxscale-system-test/mariadb_nodes.cpp @@ -18,6 +18,8 @@ #include #include #include +#include +#include #include "envv.h" using std::cout; @@ -1102,19 +1104,22 @@ std::string Mariadb_nodes::get_lowest_version() int Mariadb_nodes::truncate_mariadb_logs() { - int local_result = 0; + std::vector> results; + for (int node = 0; node < N; node++) { if (strcmp(IP[node], "127.0.0.1") != 0) { - local_result += ssh_node_f(node, true, - "truncate -s 0 /var/lib/mysql/*.err;" - "truncate -s 0 /var/log/syslog;" - "truncate -s 0 /var/log/messages;" - "rm -f /etc/my.cnf.d/binlog_enc*;"); + auto f = std::async(std::launch::async, &Nodes::ssh_node_f, this, node, true, + "truncate -s 0 /var/lib/mysql/*.err;" + "truncate -s 0 /var/log/syslog;" + "truncate -s 0 /var/log/messages;" + "rm -f /etc/my.cnf.d/binlog_enc*;"); + results.push_back(std::move(f)); } } - return local_result; + + return std::count_if(results.begin(), results.end(), std::mem_fn(&std::future::get)); } int Mariadb_nodes::configure_ssl(bool require) diff --git a/maxscale-system-test/mxs2417_ignore_persisted_cnf.cpp b/maxscale-system-test/mxs2417_ignore_persisted_cnf.cpp new file mode 100644 index 000000000..75049f0d5 --- /dev/null +++ b/maxscale-system-test/mxs2417_ignore_persisted_cnf.cpp @@ -0,0 +1,24 @@ +/** + * MXS-2417: Ignore persisted configs with load_persisted_configs=false + * https://jira.mariadb.org/browse/MXS-2417 + */ + +#include "testconnections.h" + +int main(int argc, char* argv[]) +{ + TestConnections test(argc, argv); + + test.tprintf("Creating a server and verifying it exists"); + test.check_maxctrl("create server server1234 127.0.0.1 3306"); + test.check_maxctrl("show server server1234"); + + test.tprintf("Restarting MaxScale"); + test.maxscales->restart_maxscale(); + + test.tprintf("Creating the server again and verifying it is successful"); + test.check_maxctrl("create server server1234 127.0.0.1 3306"); + test.check_maxctrl("show server server1234"); + + return test.global_result; +} diff --git a/maxscale-system-test/nodes.cpp b/maxscale-system-test/nodes.cpp index 7431182bd..412fcafee 100644 --- a/maxscale-system-test/nodes.cpp +++ b/maxscale-system-test/nodes.cpp @@ -2,6 +2,9 @@ #include #include #include +#include +#include +#include #include "envv.h" @@ -9,36 +12,29 @@ Nodes::Nodes() { } -int Nodes::check_node_ssh(int node) +bool Nodes::check_node_ssh(int node) { - int res = 0; + bool res = true; - if (ssh_node(node, (char*) "ls > /dev/null", false) != 0) + if (ssh_node(node, "ls > /dev/null", false) != 0) { - printf("Node %d is not available\n", node); - fflush(stdout); - res = 1; - } - else - { - fflush(stdout); + std::cout << "Node " << node << " is not available" << std::endl; + res = false; } + return res; } -int Nodes::check_nodes() +bool Nodes::check_nodes() { - std::cout << "Checking nodes..." << std::endl; + std::vector> f; for (int i = 0; i < N; i++) { - if (check_node_ssh(i) != 0) - { - return 1; - } + f.push_back(std::async(std::launch::async, &Nodes::check_node_ssh, this, i)); } - return 0; + return std::all_of(f.begin(), f.end(), std::mem_fn(&std::future::get)); } void Nodes::generate_ssh_cmd(char* cmd, int node, const char* ssh, bool sudo) @@ -168,6 +164,11 @@ int Nodes::ssh_node(int node, const char* ssh, bool sudo) verbose ? "" : " > /dev/null"); } + if (verbose) + { + std::cout << ssh << std::endl; + } + int rc = 1; FILE* in = popen(cmd, "w"); diff --git a/maxscale-system-test/nodes.h b/maxscale-system-test/nodes.h index 615fee934..ba2b26192 100644 --- a/maxscale-system-test/nodes.h +++ b/maxscale-system-test/nodes.h @@ -168,9 +168,9 @@ public: /** * @brief Check node via ssh and restart it if it is not resposible * @param node Node index - * @return 0 if node is ok, 1 if start failed + * @return True if node is ok, false if start failed */ - int check_nodes(); + bool check_nodes(); /** * @brief read_basic_env Read IP, sshkey, etc - common parameters for all kinds of nodes @@ -206,5 +206,5 @@ public: int stop_vm(int node); private: - int check_node_ssh(int node); + bool check_node_ssh(int node); }; diff --git a/maxscale-system-test/testconnections.cpp b/maxscale-system-test/testconnections.cpp index 5ff036484..3f4c41ac0 100644 --- a/maxscale-system-test/testconnections.cpp +++ b/maxscale-system-test/testconnections.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include "mariadb_func.h" @@ -119,6 +120,8 @@ void TestConnections::restart_galera(bool value) maxscale::restart_galera = value; } +bool TestConnections::verbose = false; + TestConnections::TestConnections(int argc, char* argv[]) : enable_timeouts(true) , global_result(0) @@ -126,7 +129,6 @@ TestConnections::TestConnections(int argc, char* argv[]) , local_maxscale(false) , no_backend_log_copy(false) , no_maxscale_log_copy(false) - , verbose(false) , smoke(true) , binlog_cmd_option(0) , ssl(false) @@ -303,7 +305,7 @@ TestConnections::TestConnections(int argc, char* argv[]) mdbci_call_needed = true; tprintf("Machines with label '%s' are not running, MDBCI UP call is needed", label.c_str()); } - else + else if (verbose) { tprintf("Machines with label '%s' are running, MDBCI UP call is not needed", label.c_str()); } @@ -321,13 +323,19 @@ TestConnections::TestConnections(int argc, char* argv[]) if (mdbci_labels.find(std::string("REPL_BACKEND")) == std::string::npos) { no_repl = true; - tprintf("No need to use Master/Slave"); + if (verbose) + { + tprintf("No need to use Master/Slave"); + } } if (mdbci_labels.find(std::string("GALERA_BACKEND")) == std::string::npos) { no_galera = true; - tprintf("No need to use Galera"); + if (verbose) + { + tprintf("No need to use Galera"); + } } get_logs_command = (char *) malloc(strlen(test_dir) + 14); @@ -345,19 +353,16 @@ TestConnections::TestConnections(int argc, char* argv[]) exit(0); } + std::future repl_future; + std::future galera_future; + if (!no_repl) { repl = new Mariadb_nodes("node", test_dir, verbose, network_config); repl->use_ipv6 = use_ipv6; repl->take_snapshot_command = take_snapshot_command; repl->revert_snapshot_command = revert_snapshot_command; - if (repl->check_nodes()) - { - if (call_mdbci("--recreate")) - { - exit(MDBCI_FAUILT); - } - } + repl_future = std::async(std::launch::async, &Mariadb_nodes::check_nodes, repl); } else { @@ -371,13 +376,7 @@ TestConnections::TestConnections(int argc, char* argv[]) galera->use_ipv6 = false; galera->take_snapshot_command = take_snapshot_command; galera->revert_snapshot_command = revert_snapshot_command; - if (galera->check_nodes()) - { - if (call_mdbci("--recreate")) - { - exit(MDBCI_FAUILT); - } - } + galera_future = std::async(std::launch::async, &Galera_nodes::check_nodes, galera); } else { @@ -385,23 +384,26 @@ TestConnections::TestConnections(int argc, char* argv[]) } maxscales = new Maxscales("maxscale", test_dir, verbose, use_valgrind, network_config); - if (maxscales->check_nodes() || - ((maxscales->N < 2) && (mdbci_labels.find(std::string("SECOND_MAXSCALE")) != std::string::npos)) - ) + + bool maxscale_ok = maxscales->check_nodes(); + bool repl_ok = no_repl || repl_future.get(); + bool galera_ok = no_galera || galera_future.get(); + bool node_error = !maxscale_ok || !repl_ok || !galera_ok; + + if (node_error || too_many_maxscales()) { + tprintf("Recreating VMs: %s", node_error ? "node check failed" : "too many maxscales"); + if (call_mdbci("--recreate")) { exit(MDBCI_FAUILT); } } - if (reinstall_maxscale) + if (reinstall_maxscale && reinstall_maxscales()) { - if (reinstall_maxscales()) - { - tprintf("Failed to install Maxscale: target is %s", target); - exit(MDBCI_FAUILT); - } + tprintf("Failed to install Maxscale: target is %s", target); + exit(MDBCI_FAUILT); } std::string src = std::string(test_dir) + "/mdbci/add_core_cnf.sh"; @@ -422,39 +424,33 @@ TestConnections::TestConnections(int argc, char* argv[]) } } - if (repl) + if (repl && maxscale::required_repl_version.length()) { - if (maxscale::required_repl_version.length()) - { - int ver_repl_required = get_int_version(maxscale::required_repl_version); - std::string ver_repl = repl->get_lowest_version(); - int int_ver_repl = get_int_version(ver_repl); + int ver_repl_required = get_int_version(maxscale::required_repl_version); + std::string ver_repl = repl->get_lowest_version(); + int int_ver_repl = get_int_version(ver_repl); - if (int_ver_repl < ver_repl_required) - { - tprintf("Test requires a higher version of backend servers, skipping test."); - tprintf("Required version: %s", maxscale::required_repl_version.c_str()); - tprintf("Master-slave version: %s", ver_repl.c_str()); - exit(0); - } + if (int_ver_repl < ver_repl_required) + { + tprintf("Test requires a higher version of backend servers, skipping test."); + tprintf("Required version: %s", maxscale::required_repl_version.c_str()); + tprintf("Master-slave version: %s", ver_repl.c_str()); + exit(0); } } - if (galera) + if (galera && maxscale::required_galera_version.length()) { - if (maxscale::required_galera_version.length()) - { - int ver_galera_required = get_int_version(maxscale::required_galera_version); - std::string ver_galera = galera->get_lowest_version(); - int int_ver_galera = get_int_version(ver_galera); + int ver_galera_required = get_int_version(maxscale::required_galera_version); + std::string ver_galera = galera->get_lowest_version(); + int int_ver_galera = get_int_version(ver_galera); - if (int_ver_galera < ver_galera_required) - { - tprintf("Test requires a higher version of backend servers, skipping test."); - tprintf("Required version: %s", maxscale::required_galera_version.c_str()); - tprintf("Galera version: %s", ver_galera.c_str()); - exit(0); - } + if (int_ver_galera < ver_galera_required) + { + tprintf("Test requires a higher version of backend servers, skipping test."); + tprintf("Required version: %s", maxscale::required_galera_version.c_str()); + tprintf("Galera version: %s", ver_galera.c_str()); + exit(0); } } @@ -464,22 +460,15 @@ TestConnections::TestConnections(int argc, char* argv[]) galera->start_replication(); } - if (maxscale::check_nodes) { - if (repl) + if (repl && !repl->fix_replication()) { - if (!repl->fix_replication() ) - { - exit(BROKEN_VM_FAUILT); - } + exit(BROKEN_VM_FAUILT); } - if (galera) + if (galera && !galera->fix_replication()) { - if (!galera->fix_replication()) - { - exit(BROKEN_VM_FAUILT); - } + exit(BROKEN_VM_FAUILT); } } diff --git a/maxscale-system-test/testconnections.h b/maxscale-system-test/testconnections.h index 369f7b34e..aebcb86aa 100644 --- a/maxscale-system-test/testconnections.h +++ b/maxscale-system-test/testconnections.h @@ -186,7 +186,7 @@ public: /** * @brief verbose if true more printing activated */ - bool verbose; + static bool verbose; /** * @brief smoke if true all tests are executed in quick mode @@ -700,9 +700,15 @@ public: private: void report_result(const char* format, va_list argp); - void copy_one_mariadb_log(Mariadb_nodes *nrepl, int i, std::string filename); + void copy_one_mariadb_log(Mariadb_nodes* nrepl, int i, std::string filename); - std::vector> m_on_destroy; + bool too_many_maxscales() const + { + return maxscales->N < 2 + && mdbci_labels.find("SECOND_MAXSCALE") != std::string::npos; + } + + std::vector> m_on_destroy; }; /** diff --git a/query_classifier/qc_sqlite/qc_sqlite.cc b/query_classifier/qc_sqlite/qc_sqlite.cc index 549d9def4..3b871058d 100644 --- a/query_classifier/qc_sqlite/qc_sqlite.cc +++ b/query_classifier/qc_sqlite/qc_sqlite.cc @@ -1586,18 +1586,16 @@ public: } } - void mxs_sqlite3BeginTrigger(Parse* pParse, /* The parse context of the CREATE TRIGGER statement - * */ - Token* pName1, /* The name of the trigger */ - Token* pName2, /* The name of the trigger */ - int tr_tm, /* One of TK_BEFORE, TK_AFTER, TK_INSTEAD */ - int op, /* One of TK_INSERT, TK_UPDATE, TK_DELETE */ - IdList* pColumns, /* column list if this is an UPDATE OF trigger */ - SrcList* pTableName, /* The name of the table/view the trigger applies to - * */ - Expr* pWhen, /* WHEN clause */ - int isTemp, /* True if the TEMPORARY keyword is present */ - int noErr) /* Suppress errors if the trigger already exists */ + void mxs_sqlite3BeginTrigger(Parse* pParse, /* The parse context of the CREATE TRIGGER statement */ + Token* pName1, /* The name of the trigger */ + Token* pName2, /* The name of the trigger */ + int tr_tm, /* One of TK_BEFORE, TK_AFTER, TK_INSTEAD */ + int op, /* One of TK_INSERT, TK_UPDATE, TK_DELETE */ + IdList* pColumns, /* column list if this is an UPDATE OF trigger */ + SrcList* pTableName, /* The name of the table/view the trigger applies to */ + Expr* pWhen, /* WHEN clause */ + int isTemp, /* True if the TEMPORARY keyword is present */ + int noErr) /* Suppress errors if the trigger already exists */ { mxb_assert(this_thread.initialized); @@ -2643,6 +2641,11 @@ public: m_type_mask = (QUERY_TYPE_WRITE | QUERY_TYPE_COMMIT); break; + case TK_XA: + m_status = QC_QUERY_TOKENIZED; + m_type_mask = QUERY_TYPE_WRITE; + break; + default: ; } diff --git a/query_classifier/qc_sqlite/sqlite-src-3110100/src/parse.y b/query_classifier/qc_sqlite/sqlite-src-3110100/src/parse.y index 67088e537..917b68f68 100644 --- a/query_classifier/qc_sqlite/sqlite-src-3110100/src/parse.y +++ b/query_classifier/qc_sqlite/sqlite-src-3110100/src/parse.y @@ -639,6 +639,7 @@ columnid(A) ::= nm(X). { VALUE VIEW /*VIRTUAL*/ /*WITH*/ WORK + XA %endif . %wildcard ANY. diff --git a/query_classifier/qc_sqlite/sqlite-src-3110100/src/tokenize.c b/query_classifier/qc_sqlite/sqlite-src-3110100/src/tokenize.c index 84e4a4538..a698d29ba 100644 --- a/query_classifier/qc_sqlite/sqlite-src-3110100/src/tokenize.c +++ b/query_classifier/qc_sqlite/sqlite-src-3110100/src/tokenize.c @@ -626,6 +626,22 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){ /* If it is not a BLOB literal, then it must be an ID, since no ** SQL keywords start with the letter 'x'. Fall through */ } +#endif +#ifdef MAXSCALE + // It may be the "XA" keyword. + // If the next character is 'a' or 'A', followed by whitespace or a + // comment, then we are indeed dealing with the "XA" keyword. + if (( z[1]=='a' || z[1]=='A' ) && + (sqlite3Isspace(z[2]) || // Whitespace + (z[2]=='/' && z[3]=='*') || // Beginning of /* comment + (z[2]=='#') || // # eol comment + (z[2]=='-' && z[3]=='-' && sqlite3Isspace(z[4])))) { // -- eol comment + extern int maxscaleKeyword(int); + + *tokenType = TK_XA; + maxscaleKeyword(*tokenType); + return 2; + } #endif case CC_ID: { i = 1; diff --git a/query_classifier/qc_sqlite/sqlite-src-3110100/tool/mkkeywordhash.c b/query_classifier/qc_sqlite/sqlite-src-3110100/tool/mkkeywordhash.c index 675c543c4..a8746a7df 100644 --- a/query_classifier/qc_sqlite/sqlite-src-3110100/tool/mkkeywordhash.c +++ b/query_classifier/qc_sqlite/sqlite-src-3110100/tool/mkkeywordhash.c @@ -500,6 +500,7 @@ static Keyword aKeywordTable[] = { #ifdef MAXSCALE { "WORK", "TK_WORK", ALWAYS }, { "WRITE", "TK_WRITE", ALWAYS }, + { "XA", "TK_XA", ALWAYS }, #endif { "ZEROFILL", "TK_ZEROFILL", ALWAYS }, }; diff --git a/query_classifier/test/maxscale.test b/query_classifier/test/maxscale.test index 9c8e9825e..2a5758211 100644 --- a/query_classifier/test/maxscale.test +++ b/query_classifier/test/maxscale.test @@ -121,3 +121,11 @@ SELECT X(coordinates), Y(coordinates), ST_X(coordinates), ST_Y(coordinates) FROM # MXS-2248 SELECT curdate() + interval '60' day; + +# MXS-2431 +XA BEGIN 'xid'; +XA END 'xid'; +XA PREPARE 'xid'; +XA COMMIT 'xid'; +XA ROLLBACK 'xid' +XA RECOVER 'xid'; diff --git a/server/core/config.cc b/server/core/config.cc index add372d70..edd67cb23 100644 --- a/server/core/config.cc +++ b/server/core/config.cc @@ -4659,6 +4659,10 @@ json_t* config_maxscale_to_json(const char* host) CN_QUERY_CLASSIFIER_CACHE_SIZE, json_integer(cnf->qc_cache_properties.max_size)); + json_object_set_new(param, CN_RETAIN_LAST_STATEMENTS, json_integer(session_get_retain_last_statements())); + json_object_set_new(param, CN_DUMP_LAST_STATEMENTS, json_string(session_get_dump_statements_str())); + json_object_set_new(param, CN_LOAD_PERSISTED_CONFIGS, json_boolean(cnf->load_persisted_configs)); + json_t* attr = json_object(); time_t started = maxscale_started(); time_t activated = started + MXS_CLOCK_TO_SEC(cnf->promoted_at); diff --git a/server/core/dcb.cc b/server/core/dcb.cc index 53d249206..8583c0c44 100644 --- a/server/core/dcb.cc +++ b/server/core/dcb.cc @@ -99,7 +99,6 @@ static inline DCB* dcb_find_in_list(DCB* dcb); static void dcb_stop_polling_and_shutdown(DCB* dcb); static bool dcb_maybe_add_persistent(DCB*); static inline bool dcb_write_parameter_check(DCB* dcb, GWBUF* queue); -static int dcb_bytes_readable(DCB* dcb); static int dcb_read_no_bytes_available(DCB* dcb, int nreadtotal); static int dcb_create_SSL(DCB* dcb, SSL_LISTENER* ssl); static int dcb_read_SSL(DCB* dcb, GWBUF** head); @@ -586,9 +585,10 @@ int dcb_read(DCB* dcb, * Find the number of bytes available for the DCB's socket * * @param dcb The DCB to read from + * * @return -1 on error, otherwise the total number of bytes available */ -static int dcb_bytes_readable(DCB* dcb) +int dcb_bytes_readable(DCB* dcb) { int bytesavailable; diff --git a/server/core/modutil.cc b/server/core/modutil.cc index a470dbca2..e39321faa 100644 --- a/server/core/modutil.cc +++ b/server/core/modutil.cc @@ -1458,9 +1458,16 @@ std::string get_canonical(GWBUF* querybuf) break; } } - else if (is_space(*it) && (i == 0 || is_space(rval[i - 1]))) + else if (is_space(*it)) { - // Repeating space, skip it + if (i == 0 || is_space(rval[i - 1])) + { + // Leading or repeating whitespace, skip it + } + else + { + rval[i++] = ' '; + } } else if (*it == '/' && is_next(it, buf.end(), "/*")) { @@ -1564,6 +1571,12 @@ std::string get_canonical(GWBUF* querybuf) mxb_assert(it != buf.end()); } + // Remove trailing whitespace + while (i > 0 && is_space(rval[i - 1])) + { + --i; + } + // Shrink the buffer so that the internal bookkeeping of std::string remains up to date rval.resize(i); diff --git a/server/core/session.cc b/server/core/session.cc index 4332da5bf..1bf883f95 100644 --- a/server/core/session.cc +++ b/server/core/session.cc @@ -938,6 +938,11 @@ void session_set_retain_last_statements(uint32_t n) this_unit.retain_last_statements = n; } +uint32_t session_get_retain_last_statements() +{ + return this_unit.retain_last_statements; +} + void session_set_dump_statements(session_dump_statements_t value) { this_unit.dump_statements = value; @@ -948,6 +953,25 @@ session_dump_statements_t session_get_dump_statements() return this_unit.dump_statements; } +const char* session_get_dump_statements_str() +{ + switch (this_unit.dump_statements) + { + case SESSION_DUMP_STATEMENTS_NEVER: + return "never"; + + case SESSION_DUMP_STATEMENTS_ON_CLOSE: + return "on_close"; + + case SESSION_DUMP_STATEMENTS_ON_ERROR: + return "on_error"; + + default: + mxb_assert(!true); + return "unknown"; + } +} + void session_retain_statement(MXS_SESSION* pSession, GWBUF* pBuffer) { static_cast(pSession)->retain_statement(pBuffer); @@ -1380,7 +1404,7 @@ void Session::retain_statement(GWBUF* pBuffer) { mxb_assert(m_last_queries.size() <= m_retain_last_statements); - std::shared_ptr sBuffer(gwbuf_clone(pBuffer)); + std::shared_ptr sBuffer(gwbuf_clone(pBuffer), std::default_delete()); m_last_queries.push_front(QueryInfo(sBuffer)); diff --git a/server/core/test/rest-api/test/server.js b/server/core/test/rest-api/test/server.js index 36bc95ec7..ca6cd7da2 100644 --- a/server/core/test/rest-api/test/server.js +++ b/server/core/test/rest-api/test/server.js @@ -156,6 +156,18 @@ describe("Server State", function() { }) }); + it("force server into maintenance", function() { + return request.put(base_url + "/servers/" + server.data.id + "/set?state=maintenance&force=yes") + .then(function(resp) { + return request.get(base_url + "/servers/" + server.data.id) + }) + .then(function(resp) { + var srv = JSON.parse(resp) + srv.data.attributes.state.should.match(/Maintenance/) + srv.data.attributes.statistics.connections.should.be.equal(0) + }) + }); + it("clear maintenance", function() { return request.put(base_url + "/servers/" + server.data.id + "/clear?state=maintenance") .then(function(resp) { diff --git a/server/modules/protocol/MySQL/mariadbclient/mysql_client.cc b/server/modules/protocol/MySQL/mariadbclient/mysql_client.cc index dd8cc5c3a..8bbafda32 100644 --- a/server/modules/protocol/MySQL/mariadbclient/mysql_client.cc +++ b/server/modules/protocol/MySQL/mariadbclient/mysql_client.cc @@ -485,7 +485,10 @@ int gw_read_client_event(DCB* dcb) { max_bytes = 36; } - return_code = dcb_read(dcb, &read_buffer, max_bytes); + + const uint32_t max_single_read = GW_MYSQL_MAX_PACKET_LEN + MYSQL_HEADER_LEN; + return_code = dcb_read(dcb, &read_buffer, max_bytes > 0 ? max_bytes : max_single_read); + if (return_code < 0) { dcb_close(dcb); @@ -495,6 +498,13 @@ int gw_read_client_event(DCB* dcb) return return_code; } + if (nbytes_read == max_single_read && dcb_bytes_readable(dcb) > 0) + { + // We read a maximally long packet, route it first. This is done in case there's a lot more data + // waiting and we have to start throttling the reads. + poll_fake_read_event(dcb); + } + return_code = 0; switch (protocol->protocol_auth_state) diff --git a/server/modules/protocol/MySQL/mysql_common.cc b/server/modules/protocol/MySQL/mysql_common.cc index e87a50109..9ca1740ff 100644 --- a/server/modules/protocol/MySQL/mysql_common.cc +++ b/server/modules/protocol/MySQL/mysql_common.cc @@ -1078,6 +1078,14 @@ int gw_decode_mysql_server_handshake(MySQLProtocol* conn, uint8_t* payload) // get ThreadID: 4 bytes uint32_t tid = gw_mysql_get_byte4(payload); + + // LocalClient also uses this code and it doesn't populate the server pointer + // TODO: fix it + if (conn->owner_dcb && conn->owner_dcb->server) + { + MXS_INFO("Connected to '%s' with thread id %u", conn->owner_dcb->server->name(), tid); + } + /* TODO: Correct value of thread id could be queried later from backend if * there is any worry it might be larger than 32bit allows. */ conn->thread_id = tid; diff --git a/server/modules/protocol/MySQL/rwbackend.cc b/server/modules/protocol/MySQL/rwbackend.cc index 995ba55e3..6d4bd8390 100644 --- a/server/modules/protocol/MySQL/rwbackend.cc +++ b/server/modules/protocol/MySQL/rwbackend.cc @@ -128,9 +128,23 @@ void RWBackend::close(close_type type) bool RWBackend::consume_fetched_rows(GWBUF* buffer) { - m_expected_rows -= modutil_count_packets(buffer); - mxb_assert(m_expected_rows >= 0); - return m_expected_rows == 0; + bool rval = false; + bool more = false; + int n_eof = modutil_count_signal_packets(buffer, 0, &more, &m_modutil_state); + + // If the server responded with an error, n_eof > 0 + if (n_eof > 0) + { + rval = true; + } + else + { + m_expected_rows -= modutil_count_packets(buffer); + mxb_assert(m_expected_rows >= 0); + rval = m_expected_rows == 0; + } + + return rval; } static inline bool have_next_packet(GWBUF* buffer) @@ -139,6 +153,197 @@ static inline bool have_next_packet(GWBUF* buffer) return gwbuf_length(buffer) > len; } +template +uint64_t get_encoded_int(Iter it) +{ + uint64_t len = *it++; + + switch (len) + { + case 0xfc: + len = *it++; + len |= ((uint64_t)*it++) << 8; + break; + + case 0xfd: + len = *it++; + len |= ((uint64_t)*it++) << 8; + len |= ((uint64_t)*it++) << 16; + break; + + case 0xfe: + len = *it++; + len |= ((uint64_t)*it++) << 8; + len |= ((uint64_t)*it++) << 16; + len |= ((uint64_t)*it++) << 24; + len |= ((uint64_t)*it++) << 32; + len |= ((uint64_t)*it++) << 40; + len |= ((uint64_t)*it++) << 48; + len |= ((uint64_t)*it++) << 56; + break; + + default: + break; + } + + return len; +} + +template +Iter skip_encoded_int(Iter it) +{ + switch (*it) + { + case 0xfc: + return std::next(it, 3); + + case 0xfd: + return std::next(it, 4); + + case 0xfe: + return std::next(it, 9); + + default: + return std::next(it); + } +} + +template +uint64_t is_last_ok(Iter it) +{ + ++it; // Skip the command byte + it = skip_encoded_int(it); // Affected rows + it = skip_encoded_int(it); // Last insert ID + uint16_t status = *it++; + status |= (*it++) << 8; + return (status & SERVER_MORE_RESULTS_EXIST) == 0; +} + +template +uint64_t is_last_eof(Iter it) +{ + std::advance(it, 3); // Skip the command byte and warning count + uint16_t status = *it++; + status |= (*it++) << 8; + return (status & SERVER_MORE_RESULTS_EXIST) == 0; +} + +void RWBackend::process_reply_start(mxs::Buffer::iterator it) +{ + uint8_t cmd = *it; + m_local_infile_requested = false; + + switch (cmd) + { + case MYSQL_REPLY_OK: + if (is_last_ok(it)) + { + // No more results + set_reply_state(REPLY_STATE_DONE); + } + break; + + case MYSQL_REPLY_LOCAL_INFILE: + m_local_infile_requested = true; + set_reply_state(REPLY_STATE_DONE); + break; + + case MYSQL_REPLY_ERR: + // Nothing ever follows an error packet + set_reply_state(REPLY_STATE_DONE); + break; + + case MYSQL_REPLY_EOF: + // EOF packets are never expected as the first response + mxb_assert(!true); + break; + + default: + if (current_command() == MXS_COM_FIELD_LIST) + { + // COM_FIELD_LIST sends a strange kind of a result set + set_reply_state(REPLY_STATE_RSET_ROWS); + } + else + { + // Start of a result set + m_num_coldefs = get_encoded_int(it); + set_reply_state(REPLY_STATE_RSET_COLDEF); + } + + break; + } +} + +void RWBackend::process_packets(GWBUF* result) +{ + mxs::Buffer buffer(result); + auto it = buffer.begin(); + + while (it != buffer.end()) + { + // Extract packet length and command byte + uint32_t len = *it++; + len |= (*it++) << 8; + len |= (*it++) << 16; + ++it; // Skip the sequence + mxb_assert(it != buffer.end()); + auto end = std::next(it, len); + uint8_t cmd = *it; + + switch (m_reply_state) + { + case REPLY_STATE_START: + process_reply_start(it); + break; + + case REPLY_STATE_DONE: + // This should never happen + mxb_assert(!true); + MXS_ERROR("Unexpected result state. cmd: 0x%02hhx, len: %u", cmd, len); + break; + + case REPLY_STATE_RSET_COLDEF: + mxb_assert(m_num_coldefs > 0); + --m_num_coldefs; + + if (m_num_coldefs == 0) + { + set_reply_state(REPLY_STATE_RSET_COLDEF_EOF); + // Skip this state when DEPRECATE_EOF capability is supported + } + break; + + case REPLY_STATE_RSET_COLDEF_EOF: + mxb_assert(cmd == MYSQL_REPLY_EOF && len == MYSQL_EOF_PACKET_LEN - MYSQL_HEADER_LEN); + set_reply_state(REPLY_STATE_RSET_ROWS); + + if (is_opening_cursor()) + { + set_cursor_opened(); + MXS_INFO("Cursor successfully opened"); + set_reply_state(REPLY_STATE_DONE); + } + break; + + case REPLY_STATE_RSET_ROWS: + if (cmd == MYSQL_REPLY_EOF && len == MYSQL_EOF_PACKET_LEN - MYSQL_HEADER_LEN) + { + set_reply_state(is_last_eof(it) ? REPLY_STATE_DONE : REPLY_STATE_START); + } + else if (cmd == MYSQL_REPLY_ERR) + { + set_reply_state(REPLY_STATE_DONE); + } + break; + } + + it = end; + } + + buffer.release(); +} + /** * @brief Process a possibly partial response from the backend * @@ -148,109 +353,22 @@ void RWBackend::process_reply(GWBUF* buffer) { if (current_command() == MXS_COM_STMT_FETCH) { - bool more = false; - int n_eof = modutil_count_signal_packets(buffer, 0, &more, &m_modutil_state); - // If the server responded with an error, n_eof > 0 - if (n_eof > 0 || consume_fetched_rows(buffer)) + if (consume_fetched_rows(buffer)) { set_reply_state(REPLY_STATE_DONE); } } - else if (current_command() == MXS_COM_STATISTICS) + else if (current_command() == MXS_COM_STATISTICS || GWBUF_IS_COLLECTED_RESULT(buffer)) { - // COM_STATISTICS returns a single string and thus requires special handling + // COM_STATISTICS returns a single string and thus requires special handling. + // Collected result are all in one buffer and need no processing. set_reply_state(REPLY_STATE_DONE); } - else if (get_reply_state() == REPLY_STATE_START - && (!mxs_mysql_is_result_set(buffer) || GWBUF_IS_COLLECTED_RESULT(buffer))) - { - m_local_infile_requested = false; - - if (GWBUF_IS_COLLECTED_RESULT(buffer) - || current_command() == MXS_COM_STMT_PREPARE - || !mxs_mysql_is_ok_packet(buffer) - || !mxs_mysql_more_results_after_ok(buffer)) - { - /** Not a result set, we have the complete response */ - set_reply_state(REPLY_STATE_DONE); - - if (mxs_mysql_is_local_infile(buffer)) - { - m_local_infile_requested = true; - } - } - else - { - // This is an OK packet and more results will follow - mxb_assert(mxs_mysql_is_ok_packet(buffer) - && mxs_mysql_more_results_after_ok(buffer)); - - if (have_next_packet(buffer)) - { - // TODO: Don't clone the buffer - GWBUF* tmp = gwbuf_clone(buffer); - tmp = gwbuf_consume(tmp, mxs_mysql_get_packet_len(tmp)); - - // Consume repeating OK packets - while (mxs_mysql_more_results_after_ok(buffer) && have_next_packet(tmp)) - { - tmp = gwbuf_consume(tmp, mxs_mysql_get_packet_len(tmp)); - mxb_assert(tmp); - } - - process_reply(tmp); - gwbuf_free(tmp); - return; - } - } - } else { - bool more = false; - int n_old_eof = get_reply_state() == REPLY_STATE_RSET_ROWS ? 1 : 0; - int n_eof = modutil_count_signal_packets(buffer, n_old_eof, &more, &m_modutil_state); - - if (n_eof > 2) - { - /** - * We have multiple results in the buffer, we only care about - * the state of the last one. Skip the complete result sets and act - * like we're processing a single result set. - */ - n_eof = n_eof % 2 ? 1 : 2; - } - - if (n_eof == 0) - { - /** Waiting for the EOF packet after the column definitions */ - set_reply_state(REPLY_STATE_RSET_COLDEF); - } - else if (n_eof == 1 && current_command() != MXS_COM_FIELD_LIST) - { - /** Waiting for the EOF packet after the rows */ - set_reply_state(REPLY_STATE_RSET_ROWS); - - if (is_opening_cursor()) - { - set_cursor_opened(); - MXS_INFO("Cursor successfully opened"); - set_reply_state(REPLY_STATE_DONE); - } - } - else - { - /** We either have a complete result set or a response to - * a COM_FIELD_LIST command */ - mxb_assert(n_eof == 2 || (n_eof == 1 && current_command() == MXS_COM_FIELD_LIST)); - set_reply_state(REPLY_STATE_DONE); - - if (more) - { - /** The server will send more resultsets */ - set_reply_state(REPLY_STATE_START); - } - } + // Normal result, process it one packet at a time + process_packets(buffer); } if (get_reply_state() == REPLY_STATE_DONE) diff --git a/server/modules/routing/readwritesplit/rwsplitsession.cc b/server/modules/routing/readwritesplit/rwsplitsession.cc index c02f8a2b0..5f765b110 100644 --- a/server/modules/routing/readwritesplit/rwsplitsession.cc +++ b/server/modules/routing/readwritesplit/rwsplitsession.cc @@ -190,23 +190,9 @@ bool RWSplitSession::route_stored_query() while (!m_query_queue.empty()) { MXS_INFO(">>> Routing stored queries"); - auto query = std::move(m_query_queue.front()); m_query_queue.pop_front(); - mxb_assert(!query.empty()); - mxb_assert_message(modutil_count_packets(query.get()) == 1, "Buffer must contain only one packet"); - - if (query.empty()) - { - MXS_ALERT("Queued query unexpectedly empty, dumping query queue contents"); - for (auto& a : m_query_queue) - { - gwbuf_hexdump(a, LOG_ALERT); - } - return true; - } - /** Store the query queue locally for the duration of the routeQuery call. * This prevents recursive calls into this function. */ decltype(m_query_queue) temp_storage; @@ -232,7 +218,12 @@ bool RWSplitSession::route_stored_query() } else { - /** Routing was stopped, we need to wait for a response before retrying */ + /** + * Routing was stopped, we need to wait for a response before retrying. + * temp_storage holds the tail end of the queue and m_query_queue contains the query we attempted + * to route. + */ + mxb_assert(m_query_queue.size() == 1); temp_storage.push_front(std::move(m_query_queue.front())); m_query_queue = std::move(temp_storage); break; @@ -810,6 +801,11 @@ bool RWSplitSession::start_trx_replay() m_trx.close(); m_trx = m_orig_trx; m_current_query.copy_from(m_orig_stmt); + + // Erase all replayed queries from the query queue to prevent checksum mismatches + m_query_queue.erase(std::remove_if(m_query_queue.begin(), m_query_queue.end(), [](mxs::Buffer b) { + return GWBUF_IS_REPLAYED(b.get()); + }), m_query_queue.end()); } if (m_trx.have_stmts() || m_current_query.get())