From 83c19c0b1a14fd515c06029047b9e63881401ed5 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Fri, 16 Dec 2016 15:47:54 +0200 Subject: [PATCH] Cache: Factor out commonality of test[raw|lru]storage.cc --- .../modules/filter/cache/test/CMakeLists.txt | 11 +- .../filter/cache/test/testerlrustorage.cc | 2 + .../filter/cache/test/testerstorage.cc | 7 + .../filter/cache/test/testlrustorage.cc | 105 +++--------- .../filter/cache/test/testrawstorage.cc | 105 +++--------- .../modules/filter/cache/test/teststorage.cc | 149 ++++++++++++++++++ .../modules/filter/cache/test/teststorage.hh | 65 ++++++++ 7 files changed, 274 insertions(+), 170 deletions(-) create mode 100644 server/modules/filter/cache/test/teststorage.cc create mode 100644 server/modules/filter/cache/test/teststorage.hh diff --git a/server/modules/filter/cache/test/CMakeLists.txt b/server/modules/filter/cache/test/CMakeLists.txt index f5a013624..2a849b870 100644 --- a/server/modules/filter/cache/test/CMakeLists.txt +++ b/server/modules/filter/cache/test/CMakeLists.txt @@ -5,6 +5,7 @@ add_library(cachetester testerstorage.cc testerlrustorage.cc testerrawstorage.cc + teststorage.cc ../../../../../query_classifier/test/testreader.cc ) @@ -28,8 +29,10 @@ add_test(TestCache_rules testrules) add_test(TestCache_inmemory_keygeneration testkeygeneration storage_inmemory ${CMAKE_CURRENT_SOURCE_DIR}/input.test) add_test(TestCache_rocksdb_keygeneration testkeygeneration storage_rocksdb ${CMAKE_CURRENT_SOURCE_DIR}/input.test) -add_test(TestCache_storage_inmemory testrawstorage 10 storage_inmemory ${CMAKE_CURRENT_SOURCE_DIR}/input.test) -add_test(TestCache_storage_rocksdb testrawstorage 10 storage_rocksdb ${CMAKE_CURRENT_SOURCE_DIR}/input.test) +#usage: testrawstorage storage-module [threads [time [items [min-size [max-size]]]]]\n" +add_test(TestCache_storage_inmemory testrawstorage storage_inmemory 0 10 1000 1024 1024000) +add_test(TestCache_storage_rocksdb testrawstorage storage_rocksdb 0 10 1000 1024 1024000) -add_test(TestCache_lru_inmemory testlrustorage 10 storage_inmemory ${CMAKE_CURRENT_SOURCE_DIR}/input.test) -add_test(TestCache_lru_rocksdb testlrustorage 10 storage_rocksdb ${CMAKE_CURRENT_SOURCE_DIR}/input.test) +#usage: testlrustorage storage-module [threads [time [items [min-size [max-size]]]]]\n" +add_test(TestCache_lru_inmemory testlrustorage storage_inmemory 0 10 1000 1024 1024000) +add_test(TestCache_lru_rocksdb testlrustorage storage_inmemory 0 10 1000 1024 1024000) diff --git a/server/modules/filter/cache/test/testerlrustorage.cc b/server/modules/filter/cache/test/testerlrustorage.cc index ff8f3ca08..414ed6c25 100644 --- a/server/modules/filter/cache/test/testerlrustorage.cc +++ b/server/modules/filter/cache/test/testerlrustorage.cc @@ -30,6 +30,8 @@ int TesterLRUStorage::execute(size_t n_threads, size_t n_seconds, const CacheIte size_t max_count = cache_items.size() / 4; + out() << "LRU max-count: " << max_count << "\n" << endl; + pStorage = m_factory.createStorage(CACHE_THREAD_MODEL_MT, "unspecified", 0, // No TTL diff --git a/server/modules/filter/cache/test/testerstorage.cc b/server/modules/filter/cache/test/testerstorage.cc index 39afd3688..3b0877753 100644 --- a/server/modules/filter/cache/test/testerstorage.cc +++ b/server/modules/filter/cache/test/testerstorage.cc @@ -191,6 +191,13 @@ int TesterStorage::run(size_t n_threads, { rv = EXIT_FAILURE; } + + ++i; + } + + if (rv == EXIT_SUCCESS) + { + rv = execute(n_threads, n_seconds, cache_items); } clear_cache_items(cache_items); diff --git a/server/modules/filter/cache/test/testlrustorage.cc b/server/modules/filter/cache/test/testlrustorage.cc index 600cf6697..050d52b16 100644 --- a/server/modules/filter/cache/test/testlrustorage.cc +++ b/server/modules/filter/cache/test/testlrustorage.cc @@ -13,12 +13,7 @@ #include #include -#include -#include -#include -#include -#include -#include "storagefactory.hh" +#include "teststorage.hh" #include "testerlrustorage.hh" using namespace std; @@ -26,88 +21,32 @@ using namespace std; namespace { -void print_usage(const char* zProgram) +class TestLRUStorage : public TestStorage { - cout << "usage: " << zProgram << " time storage-module text-file\n" - << "\n" - << "where:\n" - << " time is the number of seconds we should run,\n" - << " storage-module is the name of a storage module,\n" - << " test-file is the name of a text file." << endl; -} +public: + TestLRUStorage(std::ostream* pOut) + : TestStorage(pOut) + {} + +private: + int execute(StorageFactory& factory, + size_t threads, + size_t seconds, + size_t items, + size_t min_size, + size_t max_size) + { + TesterLRUStorage tester(&out(), &factory); + + return tester.run(threads, seconds, items, min_size, max_size); + } +}; } int main(int argc, char* argv[]) { - int rv = EXIT_FAILURE; + TestLRUStorage test(&cout); - if ((argc == 3) || (argc == 4)) - { - if (mxs_log_init(NULL, ".", MXS_LOG_TARGET_DEFAULT)) - { - size_t n_seconds = atoi(argv[1]); - - if (qc_init(NULL, NULL)) - { - const char* zModule = argv[2]; - - const char FORMAT[] = "../storage/%s"; - char libdir[sizeof(FORMAT) + strlen(zModule)]; - sprintf(libdir, FORMAT, zModule); - - set_libdir(MXS_STRDUP_A(libdir)); - - StorageFactory* pFactory = StorageFactory::Open(zModule); - - if (pFactory) - { - TesterLRUStorage tester(&cout, pFactory); - - size_t n_threads = get_processor_count() + 1; - size_t n_max_items = n_threads * n_seconds * 10; - - if (argc == 3) - { - rv = tester.run(n_threads, n_seconds, n_max_items, cin); - } - else - { - fstream in(argv[3]); - - if (in) - { - rv = tester.run(n_threads, n_seconds, n_max_items, in); - } - else - { - cerr << "error: Could not open " << argv[3] << "." << endl; - } - } - - delete pFactory; - } - else - { - cerr << "error: Could not initialize factory " << zModule << "." << endl; - } - } - else - { - cerr << "error: Could not initialize query classifier." << endl; - } - - mxs_log_finish(); - } - else - { - cerr << "error: Could not initialize log." << endl; - } - } - else - { - print_usage(argv[0]); - } - - return rv; + return test.run(argc, argv); } diff --git a/server/modules/filter/cache/test/testrawstorage.cc b/server/modules/filter/cache/test/testrawstorage.cc index 675a8674f..4fdabb9b4 100644 --- a/server/modules/filter/cache/test/testrawstorage.cc +++ b/server/modules/filter/cache/test/testrawstorage.cc @@ -13,12 +13,7 @@ #include #include -#include -#include -#include -#include -#include -#include "storagefactory.hh" +#include "teststorage.hh" #include "testerrawstorage.hh" using namespace std; @@ -26,88 +21,32 @@ using namespace std; namespace { -void print_usage(const char* zProgram) +class TestRawStorage : public TestStorage { - cout << "usage: " << zProgram << " time storage-module text-file\n" - << "\n" - << "where:\n" - << " time is the number of seconds we should run,\n" - << " storage-module is the name of a storage module,\n" - << " test-file is the name of a text file." << endl; -} +public: + TestRawStorage(ostream* pOut) + : TestStorage(pOut) + {} + +private: + int execute(StorageFactory& factory, + size_t threads, + size_t seconds, + size_t items, + size_t min_size, + size_t max_size) + { + TesterRawStorage tester(&out(), &factory); + + return tester.run(threads, seconds, items, min_size, max_size); + } +}; } int main(int argc, char* argv[]) { - int rv = EXIT_FAILURE; + TestRawStorage test(&cout); - if ((argc == 3) || (argc == 4)) - { - if (mxs_log_init(NULL, ".", MXS_LOG_TARGET_DEFAULT)) - { - size_t n_seconds = atoi(argv[1]); - - if (qc_init(NULL, NULL)) - { - const char* zModule = argv[2]; - - const char FORMAT[] = "../storage/%s"; - char libdir[sizeof(FORMAT) + strlen(zModule)]; - sprintf(libdir, FORMAT, zModule); - - set_libdir(MXS_STRDUP_A(libdir)); - - StorageFactory* pFactory = StorageFactory::Open(zModule); - - if (pFactory) - { - TesterRawStorage tester(&cout, pFactory); - - size_t n_threads = get_processor_count() + 1; - size_t n_max_items = n_threads * n_seconds * 10; - - if (argc == 3) - { - rv = tester.run(n_threads, n_seconds, n_max_items, cin); - } - else - { - fstream in(argv[3]); - - if (in) - { - rv = tester.run(n_threads, n_seconds, n_max_items, in); - } - else - { - cerr << "error: Could not open " << argv[3] << "." << endl; - } - } - - delete pFactory; - } - else - { - cerr << "error: Could not initialize factory " << zModule << "." << endl; - } - } - else - { - cerr << "error: Could not initialize query classifier." << endl; - } - - mxs_log_finish(); - } - else - { - cerr << "error: Could not initialize log." << endl; - } - } - else - { - print_usage(argv[0]); - } - - return rv; + return test.run(argc, argv); } diff --git a/server/modules/filter/cache/test/teststorage.cc b/server/modules/filter/cache/test/teststorage.cc new file mode 100644 index 000000000..c7dc68d1c --- /dev/null +++ b/server/modules/filter/cache/test/teststorage.cc @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2016 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl. + * + * Change Date: 2019-07-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "storagefactory.hh" +#include "teststorage.hh" + +using namespace std; + +TestStorage::TestStorage(ostream* pOut, + size_t threads, + size_t seconds, + size_t items, + size_t min_size, + size_t max_size) + : m_out(*pOut) + , m_threads(threads) + , m_seconds(seconds) + , m_items(items) + , m_min_size(min_size) + , m_max_size(max_size) +{ +} + +TestStorage::~TestStorage() +{ +} + +int TestStorage::run(int argc, char** argv) +{ + int rv = EXIT_FAILURE; + + if ((argc >= 2) || (argc <= 7)) + { + if (mxs_log_init(NULL, ".", MXS_LOG_TARGET_DEFAULT)) + { + if (qc_init(NULL, NULL)) + { + const char* zModule = NULL; + size_t threads = m_threads; + size_t seconds = m_seconds; + size_t items = m_items; + size_t min_size = m_min_size; + size_t max_size = m_max_size; + + switch (argc) + { + default: + ss_dassert(!true); + case 7: + max_size = atoi(argv[6]); + case 6: + min_size = atoi(argv[5]); + case 5: + items = atoi(argv[4]); + case 4: + seconds = atoi(argv[3]); + case 3: + threads = atoi(argv[2]); + case 2: + zModule = argv[1]; + } + + if (threads == 0) + { + threads = get_processor_count() + 1; + } + + if (items == 0) + { + items = threads * seconds * 10; + } + + const char FORMAT[] = "../storage/%s"; + char libdir[sizeof(FORMAT) + strlen(zModule)]; + sprintf(libdir, FORMAT, zModule); + + set_libdir(MXS_STRDUP_A(libdir)); + + StorageFactory* pFactory = StorageFactory::Open(zModule); + + if (pFactory) + { + out() << "Module : " << zModule << "\n" + << "Threads : " << threads << "\n" + << "Seconds : " << seconds << "\n" + << "Items : " << items << "\n" + << "Min-Size: " << min_size << "\n" + << "Max-Size: " << max_size << "\n" + << endl; + + rv = execute(*pFactory, threads, seconds, items, min_size, max_size); + + delete pFactory; + } + else + { + cerr << "error: Could not initialize factory " << zModule << "." << endl; + } + } + else + { + cerr << "error: Could not initialize query classifier." << endl; + } + + mxs_log_finish(); + } + else + { + cerr << "error: Could not initialize log." << endl; + } + } + else + { + print_usage(argv[0]); + } + + return rv; +} + +void TestStorage::print_usage(const char* zProgram) +{ + cout << "usage: " << zProgram << " storage-module [threads [time [items [min-size [max-size]]]]]\n" + << "\n" + << "where:\n" + << " storage-module is the name of a storage module,\n" + << " threads is the number of threads to use (if 0, #cores + 1 is used,\n" + << " time is the number of seconds we should run,\n" + << " items is the number of items to use when populating the cache,\n" + << " if 0, threads * seconds * 10 is used\n" + << " min-size is the minimum size of a cache value, and\n" + << " max-size is the maximum size of a cache value." << endl; +} diff --git a/server/modules/filter/cache/test/teststorage.hh b/server/modules/filter/cache/test/teststorage.hh new file mode 100644 index 000000000..db895ad6c --- /dev/null +++ b/server/modules/filter/cache/test/teststorage.hh @@ -0,0 +1,65 @@ +#pragma once +/* + * Copyright (c) 2016 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl. + * + * Change Date: 2019-07-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ + +#include + +class StorageFactory; + +class TestStorage +{ +public: + ~TestStorage(); + + enum + { + DEFAULT_THREADS = 4, + DEFAULT_SECONDS = 10, + DEFAULT_ITEMS = 400, + DEFAULT_MIN_SIZE = 1024, + DEFAULT_MAX_SIZE = 1024 * 1024 + }; + + int run(int argc, char** argv); + +protected: + TestStorage(std::ostream* pOut, + size_t threads = DEFAULT_THREADS, + size_t seconds = DEFAULT_SECONDS, + size_t items = DEFAULT_ITEMS, + size_t min_size = DEFAULT_MIN_SIZE, + size_t max_size = DEFAULT_MAX_SIZE); + + virtual int execute(StorageFactory& factory, + size_t threads, + size_t seconds, + size_t items, + size_t min_size, + size_t max_size) = 0; + + virtual void print_usage(const char* zProgram); + + std::ostream& out() const { return m_out; } + +private: + TestStorage(const TestStorage&); + TestStorage& operator = (const TestStorage&); + +private: + std::ostream& m_out; + size_t m_threads; + size_t m_seconds; + size_t m_items; + size_t m_min_size; + size_t m_max_size; +};