From aa2de5205480ee9609777a33fca87c8ed7842d3c Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Fri, 9 Dec 2016 15:11:58 +0200 Subject: [PATCH] Factor out .test-reading capability The capability for reading MySQL/MariaDB .test-files has now been factored out from the compare.cc test program. That way, the functionality can be used from other test programs as well. --- query_classifier/test/CMakeLists.txt | 2 +- query_classifier/test/compare.cc | 320 ++--------------------- query_classifier/test/testreader.cc | 374 +++++++++++++++++++++++++++ query_classifier/test/testreader.hh | 82 ++++++ 4 files changed, 480 insertions(+), 298 deletions(-) create mode 100644 query_classifier/test/testreader.cc create mode 100644 query_classifier/test/testreader.hh diff --git a/query_classifier/test/CMakeLists.txt b/query_classifier/test/CMakeLists.txt index 68de663a7..1311ad572 100644 --- a/query_classifier/test/CMakeLists.txt +++ b/query_classifier/test/CMakeLists.txt @@ -19,7 +19,7 @@ if (BUILD_QC_MYSQLEMBEDDED) add_executable(classify classify.c) target_link_libraries(classify maxscale-common) - add_executable(compare compare.cc) + add_executable(compare compare.cc testreader.cc) target_link_libraries(compare maxscale-common) add_executable(crash_qc_sqlite crash_qc_sqlite.c) diff --git a/query_classifier/test/compare.cc b/query_classifier/test/compare.cc index 1a5d6fd20..8c51124c7 100644 --- a/query_classifier/test/compare.cc +++ b/query_classifier/test/compare.cc @@ -27,6 +27,7 @@ #include #include #include +#include "testreader.hh" using std::cerr; using std::cin; using std::cout; @@ -1108,312 +1109,39 @@ static void trim(std::string &s) rtrim(s); } -enum skip_action_t -{ - SKIP_NOTHING, // Skip nothing. - SKIP_BLOCK, // Skip until the end of next { ... } - SKIP_DELIMITER, // Skip the new delimiter. - SKIP_LINE, // Skip current line. - SKIP_NEXT_STATEMENT, // Skip statement starting on line following this line. - SKIP_STATEMENT, // Skip statment starting on this line. - SKIP_TERMINATE, // Cannot handle this, terminate. -}; - -typedef std::map KeywordActionMapping; - -static KeywordActionMapping mtl_keywords; - -void init_keywords() -{ - struct Keyword - { - const char* z_keyword; - skip_action_t action; - }; - - static const Keyword KEYWORDS[] = - { - { "append_file", SKIP_LINE }, - { "cat_file", SKIP_LINE }, - { "change_user", SKIP_LINE }, - { "character_set", SKIP_LINE }, - { "chmod", SKIP_LINE }, - { "connect", SKIP_LINE }, - { "connection", SKIP_LINE }, - { "copy_file", SKIP_LINE }, - { "dec", SKIP_LINE }, - { "delimiter", SKIP_DELIMITER }, - { "die", SKIP_LINE }, - { "diff_files", SKIP_LINE }, - { "dirty_close", SKIP_LINE }, - { "disable_abort_on_error", SKIP_LINE }, - { "disable_connect_log", SKIP_LINE }, - { "disable_info", SKIP_LINE }, - { "disable_metadata", SKIP_LINE }, - { "disable_parsing", SKIP_LINE }, - { "disable_ps_protocol", SKIP_LINE }, - { "disable_query_log", SKIP_LINE }, - { "disable_reconnect", SKIP_LINE }, - { "disable_result_log", SKIP_LINE }, - { "disable_rpl_parse", SKIP_LINE }, - { "disable_session_track_info", SKIP_LINE }, - { "disable_warnings", SKIP_LINE }, - { "disconnect", SKIP_LINE }, - { "echo", SKIP_LINE }, - { "enable_abort_on_error", SKIP_LINE }, - { "enable_connect_log", SKIP_LINE }, - { "enable_info", SKIP_LINE }, - { "enable_metadata", SKIP_LINE }, - { "enable_parsing", SKIP_LINE }, - { "enable_ps_protocol", SKIP_LINE }, - { "enable_query_log", SKIP_LINE }, - { "enable_reconnect", SKIP_LINE }, - { "enable_result_log", SKIP_LINE }, - { "enable_rpl_parse", SKIP_LINE }, - { "enable_session_track_info", SKIP_LINE }, - { "enable_warnings", SKIP_LINE }, - { "end_timer", SKIP_LINE }, - { "error", SKIP_NEXT_STATEMENT }, - { "eval", SKIP_STATEMENT }, - { "exec", SKIP_LINE }, - { "exit", SKIP_LINE }, - { "file_exists", SKIP_LINE }, - { "horizontal_results", SKIP_LINE }, - { "if", SKIP_BLOCK }, - { "inc", SKIP_LINE }, - { "let", SKIP_LINE }, - { "let", SKIP_LINE }, - { "list_files", SKIP_LINE }, - { "list_files_append_file", SKIP_LINE }, - { "list_files_write_file", SKIP_LINE }, - { "lowercase_result", SKIP_LINE }, - { "mkdir", SKIP_LINE }, - { "move_file", SKIP_LINE }, - { "output", SKIP_LINE }, - { "perl", SKIP_TERMINATE }, - { "ping", SKIP_LINE }, - { "print", SKIP_LINE }, - { "query", SKIP_LINE }, - { "query_get_value", SKIP_LINE }, - { "query_horizontal", SKIP_LINE }, - { "query_vertical", SKIP_LINE }, - { "real_sleep", SKIP_LINE }, - { "reap", SKIP_LINE }, - { "remove_file", SKIP_LINE }, - { "remove_files_wildcard", SKIP_LINE }, - { "replace_column", SKIP_LINE }, - { "replace_regex", SKIP_LINE }, - { "replace_result", SKIP_LINE }, - { "require", SKIP_LINE }, - { "reset_connection", SKIP_LINE }, - { "result", SKIP_LINE }, - { "result_format", SKIP_LINE }, - { "rmdir", SKIP_LINE }, - { "same_master_pos", SKIP_LINE }, - { "send", SKIP_LINE }, - { "send_eval", SKIP_LINE }, - { "send_quit", SKIP_LINE }, - { "send_shutdown", SKIP_LINE }, - { "skip", SKIP_LINE }, - { "sleep", SKIP_LINE }, - { "sorted_result", SKIP_LINE }, - { "source", SKIP_LINE }, - { "start_timer", SKIP_LINE }, - { "sync_slave_with_master", SKIP_LINE }, - { "sync_with_master", SKIP_LINE }, - { "system", SKIP_LINE }, - { "vertical_results", SKIP_LINE }, - { "while", SKIP_BLOCK }, - { "write_file", SKIP_LINE }, - }; - - const size_t N_KEYWORDS = sizeof(KEYWORDS)/sizeof(KEYWORDS[0]); - - for (size_t i = 0; i < N_KEYWORDS; ++i) - { - mtl_keywords[KEYWORDS[i].z_keyword] = KEYWORDS[i].action; - } -} - -skip_action_t get_action(const string& keyword) -{ - skip_action_t action = SKIP_NOTHING; - - string key(keyword); - - std::transform(key.begin(), key.end(), key.begin(), ::tolower); - - KeywordActionMapping::iterator i = mtl_keywords.find(key); - - if (i != mtl_keywords.end()) - { - action = i->second; - } - - return action; -} - -void skip_block(istream& in) -{ - int c; - - // Find first '{' - while (in && ((c = in.get()) != '{')) - { - if (c == '\n') - { - ++global.line; - } - } - - int n = 1; - - while ((n > 0) && in) - { - c = in.get(); - - switch (c) - { - case '{': - ++n; - break; - - case '}': - --n; - break; - - case '\n': - ++global.line; - break; - - default: - ; - } - } -} - - int run(QUERY_CLASSIFIER* pClassifier1, QUERY_CLASSIFIER* pClassifier2, istream& in) { bool stop = false; // Whether we should exit. - bool skip = false; // Whether next statement should be skipped. - char delimiter = ';'; - string query; - while (!stop && std::getline(in, query)) + maxscale::TestReader reader(in); + + while (!stop && (reader.get_statement(global.query) == maxscale::TestReader::RESULT_STMT)) { - trim(query); + global.line = reader.line(); + global.query_printed = false; + global.result_printed = false; - global.line++; + ++global.n_statements; - if (!query.empty() && (query.at(0) != '#')) + if (global.verbosity >= VERBOSITY_EXTENDED) { - if (!skip) + // In case the execution crashes, we want the query printed. + report_query(); + } + + bool success = compare(pClassifier1, pClassifier2, global.query); + + if (!success) + { + ++global.n_errors; + + if (global.stop_at_error) { - if (query.substr(0, 2) == "--") - { - query = query.substr(2); - trim(query); - } - - string::iterator i = std::find_if(query.begin(), query.end(), - std::ptr_fun(std::isspace)); - string keyword = query.substr(0, i - query.begin()); - - skip_action_t action = get_action(keyword); - - switch (action) - { - case SKIP_NOTHING: - break; - - case SKIP_BLOCK: - skip_block(in); - continue; - - case SKIP_DELIMITER: - query = query.substr(i - query.begin()); - trim(query); - if (query.length() > 0) - { - delimiter = query.at(0); - } - continue; - - case SKIP_LINE: - continue; - - case SKIP_NEXT_STATEMENT: - skip = true; - continue; - - case SKIP_STATEMENT: - skip = true; - break; - - case SKIP_TERMINATE: - cout << "error: Cannot handle line " << global.line - << ", terminating: " << query << endl; - stop = true; - break; - } - } - - global.query += query; - - char c = query.at(query.length() - 1); - - if (c == delimiter) - { - if (c != ';') - { - // If the delimiter was something else but ';' we need to - // remove that before giving the query to the classifiers. - global.query.erase(global.query.length() - 1); - } - - if (!skip) - { - global.query_printed = false; - global.result_printed = false; - - ++global.n_statements; - - if (global.verbosity >= VERBOSITY_EXTENDED) - { - // In case the execution crashes, we want the query printed. - report_query(); - } - - bool success = compare(pClassifier1, pClassifier2, global.query); - - if (!success) - { - ++global.n_errors; - - if (global.stop_at_error) - { - stop = true; - } - } - } - else - { - skip = false; - } - - global.query.clear(); - } - else - { - global.query += " "; + stop = true; } } - else if (query.substr(0, 7) == "--error") - { - // Next statement is supposed to fail, no need to check. - skip = true; - } + + global.query.clear(); } return global.n_errors == 0 ? EXIT_SUCCESS : EXIT_FAILURE; @@ -1502,8 +1230,6 @@ int main(int argc, char* argv[]) if ((rc == EXIT_SUCCESS) && (v >= VERBOSITY_MIN && v <= VERBOSITY_MAX)) { - init_keywords(); - rc = EXIT_FAILURE; global.verbosity = static_cast(v); diff --git a/query_classifier/test/testreader.cc b/query_classifier/test/testreader.cc new file mode 100644 index 000000000..0a4d14926 --- /dev/null +++ b/query_classifier/test/testreader.cc @@ -0,0 +1,374 @@ +/* + * Copyright (c) 2016 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl. + * + * Change Date: 2019-07-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ + +#include "testreader.hh" +#include +#include + +using std::istream; +using std::string; +using std::map; + +namespace +{ + +enum skip_action_t +{ + SKIP_NOTHING, // Skip nothing. + SKIP_BLOCK, // Skip until the end of next { ... } + SKIP_DELIMITER, // Skip the new delimiter. + SKIP_LINE, // Skip current line. + SKIP_NEXT_STATEMENT, // Skip statement starting on line following this line. + SKIP_STATEMENT, // Skip statment starting on this line. + SKIP_TERMINATE, // Cannot handle this, terminate. +}; + +typedef std::map KeywordActionMapping; + +static KeywordActionMapping mtl_keywords; + +void init_keywords() +{ + struct Keyword + { + const char* z_keyword; + skip_action_t action; + }; + + static const Keyword KEYWORDS[] = + { + { "append_file", SKIP_LINE }, + { "cat_file", SKIP_LINE }, + { "change_user", SKIP_LINE }, + { "character_set", SKIP_LINE }, + { "chmod", SKIP_LINE }, + { "connect", SKIP_LINE }, + { "connection", SKIP_LINE }, + { "copy_file", SKIP_LINE }, + { "dec", SKIP_LINE }, + { "delimiter", SKIP_DELIMITER }, + { "die", SKIP_LINE }, + { "diff_files", SKIP_LINE }, + { "dirty_close", SKIP_LINE }, + { "disable_abort_on_error", SKIP_LINE }, + { "disable_connect_log", SKIP_LINE }, + { "disable_info", SKIP_LINE }, + { "disable_metadata", SKIP_LINE }, + { "disable_parsing", SKIP_LINE }, + { "disable_ps_protocol", SKIP_LINE }, + { "disable_query_log", SKIP_LINE }, + { "disable_reconnect", SKIP_LINE }, + { "disable_result_log", SKIP_LINE }, + { "disable_rpl_parse", SKIP_LINE }, + { "disable_session_track_info", SKIP_LINE }, + { "disable_warnings", SKIP_LINE }, + { "disconnect", SKIP_LINE }, + { "echo", SKIP_LINE }, + { "enable_abort_on_error", SKIP_LINE }, + { "enable_connect_log", SKIP_LINE }, + { "enable_info", SKIP_LINE }, + { "enable_metadata", SKIP_LINE }, + { "enable_parsing", SKIP_LINE }, + { "enable_ps_protocol", SKIP_LINE }, + { "enable_query_log", SKIP_LINE }, + { "enable_reconnect", SKIP_LINE }, + { "enable_result_log", SKIP_LINE }, + { "enable_rpl_parse", SKIP_LINE }, + { "enable_session_track_info", SKIP_LINE }, + { "enable_warnings", SKIP_LINE }, + { "end_timer", SKIP_LINE }, + { "error", SKIP_NEXT_STATEMENT }, + { "eval", SKIP_STATEMENT }, + { "exec", SKIP_LINE }, + { "exit", SKIP_LINE }, + { "file_exists", SKIP_LINE }, + { "horizontal_results", SKIP_LINE }, + { "if", SKIP_BLOCK }, + { "inc", SKIP_LINE }, + { "let", SKIP_LINE }, + { "let", SKIP_LINE }, + { "list_files", SKIP_LINE }, + { "list_files_append_file", SKIP_LINE }, + { "list_files_write_file", SKIP_LINE }, + { "lowercase_result", SKIP_LINE }, + { "mkdir", SKIP_LINE }, + { "move_file", SKIP_LINE }, + { "output", SKIP_LINE }, + { "perl", SKIP_TERMINATE }, + { "ping", SKIP_LINE }, + { "print", SKIP_LINE }, + { "query", SKIP_LINE }, + { "query_get_value", SKIP_LINE }, + { "query_horizontal", SKIP_LINE }, + { "query_vertical", SKIP_LINE }, + { "real_sleep", SKIP_LINE }, + { "reap", SKIP_LINE }, + { "remove_file", SKIP_LINE }, + { "remove_files_wildcard", SKIP_LINE }, + { "replace_column", SKIP_LINE }, + { "replace_regex", SKIP_LINE }, + { "replace_result", SKIP_LINE }, + { "require", SKIP_LINE }, + { "reset_connection", SKIP_LINE }, + { "result", SKIP_LINE }, + { "result_format", SKIP_LINE }, + { "rmdir", SKIP_LINE }, + { "same_master_pos", SKIP_LINE }, + { "send", SKIP_LINE }, + { "send_eval", SKIP_LINE }, + { "send_quit", SKIP_LINE }, + { "send_shutdown", SKIP_LINE }, + { "skip", SKIP_LINE }, + { "sleep", SKIP_LINE }, + { "sorted_result", SKIP_LINE }, + { "source", SKIP_LINE }, + { "start_timer", SKIP_LINE }, + { "sync_slave_with_master", SKIP_LINE }, + { "sync_with_master", SKIP_LINE }, + { "system", SKIP_LINE }, + { "vertical_results", SKIP_LINE }, + { "while", SKIP_BLOCK }, + { "write_file", SKIP_LINE }, + }; + + const size_t N_KEYWORDS = sizeof(KEYWORDS)/sizeof(KEYWORDS[0]); + + for (size_t i = 0; i < N_KEYWORDS; ++i) + { + mtl_keywords[KEYWORDS[i].z_keyword] = KEYWORDS[i].action; + } +} + +skip_action_t get_action(const string& keyword) +{ + skip_action_t action = SKIP_NOTHING; + + string key(keyword); + + std::transform(key.begin(), key.end(), key.begin(), ::tolower); + + KeywordActionMapping::iterator i = mtl_keywords.find(key); + + if (i != mtl_keywords.end()) + { + action = i->second; + } + + return action; +} + +inline void ltrim(std::string &s) +{ + s.erase(s.begin(), std::find_if(s.begin(), s.end(), std::not1(std::ptr_fun(std::isspace)))); +} + +inline void rtrim(std::string &s) +{ + s.erase(std::find_if(s.rbegin(), s.rend(), + std::not1(std::ptr_fun(std::isspace))).base(), s.end()); +} + +void trim(std::string &s) +{ + ltrim(s); + rtrim(s); +} + +} + +namespace maxscale +{ + +TestReader::TestReader(istream& in, + size_t line) + : m_in(in) + , m_line(line) + , m_delimiter(';') +{ + init(); +} + +TestReader::result_t TestReader::get_statement(std::string& stmt) +{ + bool error = false; // Whether an error has occurred. + bool found = false; // Whether we have found a statement. + bool skip = false; // Whether next statement should be skipped. + + stmt.clear(); + + string line; + + while (!error && !found && std::getline(m_in, line)) + { + trim(line); + + m_line++; + + if (!line.empty() && (line.at(0) != '#')) + { + if (!skip) + { + if (line.substr(0, 2) == "--") + { + line = line.substr(2); + trim(line); + } + + string::iterator i = std::find_if(line.begin(), line.end(), + std::ptr_fun(std::isspace)); + string keyword = line.substr(0, i - line.begin()); + + skip_action_t action = get_action(keyword); + + switch (action) + { + case SKIP_NOTHING: + break; + + case SKIP_BLOCK: + skip_block(); + continue; + + case SKIP_DELIMITER: + line = line.substr(i - line.begin()); + trim(line); + if (line.length() > 0) + { + m_delimiter = line.at(0); + } + continue; + + case SKIP_LINE: + continue; + + case SKIP_NEXT_STATEMENT: + skip = true; + continue; + + case SKIP_STATEMENT: + skip = true; + break; + + case SKIP_TERMINATE: + MXS_ERROR("Cannot handle line %u: %s", (unsigned)m_line, line.c_str()); + error = true; + break; + } + } + + stmt += line; + + char c = line.at(line.length() - 1); + + if (c == m_delimiter) + { + if (c != ';') + { + // If the delimiter was something else but ';' we need to + // remove that before giving the line to the classifiers. + stmt.erase(stmt.length() - 1); + } + + if (!skip) + { + found = true; + } + else + { + skip = false; + stmt.clear(); + } + } + else if (!skip) + { + stmt += " "; + } + } + else if (line.substr(0, 7) == "--error") + { + // Next statement is supposed to fail, no need to check. + skip = true; + } + } + + result_t result; + + if (error) + { + result = RESULT_ERROR; + } + else if (found) + { + result = RESULT_STMT; + } + else + { + result = RESULT_EOF; + } + + return result; +} + +// static +void TestReader::init() +{ + static bool inited = false; + + if (!inited) + { + inited = true; + + init_keywords(); + } +} + +void TestReader::skip_block() +{ + int c; + + // Find first '{' + while (m_in && ((c = m_in.get()) != '{')) + { + if (c == '\n') + { + ++m_line; + } + } + + int n = 1; + + while ((n > 0) && m_in) + { + c = m_in.get(); + + switch (c) + { + case '{': + ++n; + break; + + case '}': + --n; + break; + + case '\n': + ++m_line; + break; + + default: + ; + } + } +} + +} diff --git a/query_classifier/test/testreader.hh b/query_classifier/test/testreader.hh new file mode 100644 index 000000000..3ee36c5c6 --- /dev/null +++ b/query_classifier/test/testreader.hh @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2016 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl. + * + * Change Date: 2019-07-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ + +#include +#include + +namespace maxscale +{ + +/** + * @class TestReader + * + * The TestReader class is capable of reading a MySQL/MariaDB test file, + * such like the ones in [MySQL|MariaDB]/server/mysql-test/t, and return + * the SQL statements one by one. It does this by skipping test commands + * and by coalescing lines to obtain a full statement. + */ +class TestReader +{ +public: + enum result_t + { + RESULT_ERROR, /*< The input is probably not a test file. */ + RESULT_EOF, /*< End of file was reached. */ + RESULT_STMT, /*< A statement was returned. */ + }; + + /** + * Initialize internal shared tables. This will automatically be called + * by the TestReader constructor, but if multiple threads are used it + * is adviseable to call this function explicitly from the main thread. + */ + static void init(); + + + /** + * Creates a TestReader instance. + * + * @param in An input stream. + * @param line Optionally specify the initial line number. + */ + TestReader(std::istream& in, + size_t line = 0); + + /** + * @return The current line number. + */ + size_t line() const { return m_line; } + + /** + * Get next full SQL statement. + * + * @param stmt String where statement will be stored. + * + * @return RESULT_STMT if a statement was returned in @c stmt. + */ + result_t get_statement(std::string& stmt); + +private: + void skip_block(); + +private: + TestReader(const TestReader&); + TestReader& operator = (const TestReader&); + +private: + std::istream& m_in; /*< The stream we are using. */ + size_t m_line; /*< The current line. */ + char m_delimiter; /*< The current delimiter. */ +}; + +};