Cache: Factor out commonality of test[raw|lru]storage.cc

server/modules/filter/cache/test/CMakeLists.txt

@@ -5,6 +5,7 @@ add_library(cachetester
   testerstorage.cc
   testerlrustorage.cc
   testerrawstorage.cc
+  teststorage.cc
   ../../../../../query_classifier/test/testreader.cc
   )
 
@@ -28,8 +29,10 @@ add_test(TestCache_rules testrules)
 add_test(TestCache_inmemory_keygeneration testkeygeneration storage_inmemory ${CMAKE_CURRENT_SOURCE_DIR}/input.test)
 add_test(TestCache_rocksdb_keygeneration testkeygeneration storage_rocksdb ${CMAKE_CURRENT_SOURCE_DIR}/input.test)
 
-add_test(TestCache_storage_inmemory testrawstorage 10 storage_inmemory ${CMAKE_CURRENT_SOURCE_DIR}/input.test)
+#usage: testrawstorage storage-module [threads [time [items [min-size [max-size]]]]]\n"
-add_test(TestCache_storage_rocksdb  testrawstorage 10 storage_rocksdb ${CMAKE_CURRENT_SOURCE_DIR}/input.test)
+add_test(TestCache_storage_inmemory testrawstorage storage_inmemory 0 10 1000 1024 1024000)
+add_test(TestCache_storage_rocksdb  testrawstorage storage_rocksdb  0 10 1000 1024 1024000)
 
-add_test(TestCache_lru_inmemory testlrustorage 10 storage_inmemory ${CMAKE_CURRENT_SOURCE_DIR}/input.test)
+#usage: testlrustorage storage-module [threads [time [items [min-size [max-size]]]]]\n"
-add_test(TestCache_lru_rocksdb  testlrustorage 10 storage_rocksdb ${CMAKE_CURRENT_SOURCE_DIR}/input.test)
+add_test(TestCache_lru_inmemory testlrustorage storage_inmemory 0 10 1000 1024 1024000)
+add_test(TestCache_lru_rocksdb  testlrustorage storage_inmemory 0 10 1000 1024 1024000)

server/modules/filter/cache/test/testerlrustorage.cc

@@ -30,6 +30,8 @@ int TesterLRUStorage::execute(size_t n_threads, size_t n_seconds, const CacheIte
 
     size_t max_count = cache_items.size() / 4;
 
+    out() << "LRU max-count: " << max_count << "\n" << endl;
+
     pStorage = m_factory.createStorage(CACHE_THREAD_MODEL_MT,
                                        "unspecified",
                                        0, // No TTL

server/modules/filter/cache/test/testerstorage.cc

@@ -191,6 +191,13 @@ int TesterStorage::run(size_t n_threads,
         {
             rv = EXIT_FAILURE;
         }
+
+        ++i;
+    }
+
+    if (rv == EXIT_SUCCESS)
+    {
+        rv = execute(n_threads, n_seconds, cache_items);
     }
 
     clear_cache_items(cache_items);

server/modules/filter/cache/test/testlrustorage.cc

@@ -13,12 +13,7 @@
 
 #include <maxscale/cppdefs.hh>
 #include <iostream>
-#include <fstream>
+#include "teststorage.hh"
-#include <maxscale/alloc.h>
-#include <maxscale/gwdirs.h>
-#include <maxscale/log_manager.h>
-#include <maxscale/query_classifier.h>
-#include "storagefactory.hh"
 #include "testerlrustorage.hh"
 
 using namespace std;
@@ -26,88 +21,32 @@ using namespace std;
 namespace
 {
 
-void print_usage(const char* zProgram)
+class TestLRUStorage : public TestStorage
 {
-    cout << "usage: " << zProgram << " time storage-module text-file\n"
+public:
-         << "\n"
+    TestLRUStorage(std::ostream* pOut)
-         << "where:\n"
+        : TestStorage(pOut)
-         << "  time            is the number of seconds we should run,\n"
+    {}
-         << "  storage-module  is the name of a storage module,\n"
+
-         << "  test-file       is the name of a text file." << endl;
+private:
+    int execute(StorageFactory& factory,
+                size_t threads,
+                size_t seconds,
+                size_t items,
+                size_t min_size,
+                size_t max_size)
+    {
+        TesterLRUStorage tester(&out(), &factory);
+
+        return tester.run(threads, seconds, items, min_size, max_size);
     }
+};
 
 }
 
 int main(int argc, char* argv[])
 {
-    int rv = EXIT_FAILURE;
+    TestLRUStorage test(&cout);
 
-    if ((argc == 3) || (argc == 4))
+    return test.run(argc, argv);
-    {
-        if (mxs_log_init(NULL, ".", MXS_LOG_TARGET_DEFAULT))
-        {
-            size_t n_seconds = atoi(argv[1]);
-
-            if (qc_init(NULL, NULL))
-            {
-                const char* zModule = argv[2];
-
-                const char FORMAT[] = "../storage/%s";
-                char libdir[sizeof(FORMAT) + strlen(zModule)];
-                sprintf(libdir, FORMAT, zModule);
-
-                set_libdir(MXS_STRDUP_A(libdir));
-
-                StorageFactory* pFactory = StorageFactory::Open(zModule);
-
-                if (pFactory)
-                {
-                    TesterLRUStorage tester(&cout, pFactory);
-
-                    size_t n_threads = get_processor_count() + 1;
-                    size_t n_max_items = n_threads * n_seconds * 10;
-
-                    if (argc == 3)
-                    {
-                        rv = tester.run(n_threads, n_seconds, n_max_items, cin);
-                    }
-                    else
-                    {
-                        fstream in(argv[3]);
-
-                        if (in)
-                        {
-                            rv = tester.run(n_threads, n_seconds, n_max_items, in);
-                        }
-                        else
-                        {
-                            cerr << "error: Could not open " << argv[3] << "." << endl;
-                        }
-                    }
-
-                    delete pFactory;
-                }
-                else
-                {
-                    cerr << "error: Could not initialize factory " << zModule << "." << endl;
-                }
-            }
-            else
-            {
-                cerr << "error: Could not initialize query classifier." << endl;
-            }
-
-            mxs_log_finish();
-        }
-        else
-        {
-            cerr << "error: Could not initialize log." << endl;
-        }
-    }
-    else
-    {
-        print_usage(argv[0]);
-    }
-
-    return rv;
 }

server/modules/filter/cache/test/testrawstorage.cc

@@ -13,12 +13,7 @@
 
 #include <maxscale/cppdefs.hh>
 #include <iostream>
-#include <fstream>
+#include "teststorage.hh"
-#include <maxscale/alloc.h>
-#include <maxscale/gwdirs.h>
-#include <maxscale/log_manager.h>
-#include <maxscale/query_classifier.h>
-#include "storagefactory.hh"
 #include "testerrawstorage.hh"
 
 using namespace std;
@@ -26,88 +21,32 @@ using namespace std;
 namespace
 {
 
-void print_usage(const char* zProgram)
+class TestRawStorage : public TestStorage
 {
-    cout << "usage: " << zProgram << " time storage-module text-file\n"
+public:
-         << "\n"
+    TestRawStorage(ostream* pOut)
-         << "where:\n"
+        : TestStorage(pOut)
-         << "  time            is the number of seconds we should run,\n"
+    {}
-         << "  storage-module  is the name of a storage module,\n"
+
-         << "  test-file       is the name of a text file." << endl;
+private:
+    int execute(StorageFactory& factory,
+                size_t threads,
+                size_t seconds,
+                size_t items,
+                size_t min_size,
+                size_t max_size)
+    {
+        TesterRawStorage tester(&out(), &factory);
+
+        return tester.run(threads, seconds, items, min_size, max_size);
     }
+};
 
 }
 
 int main(int argc, char* argv[])
 {
-    int rv = EXIT_FAILURE;
+    TestRawStorage test(&cout);
 
-    if ((argc == 3) || (argc == 4))
+    return test.run(argc, argv);
-    {
-        if (mxs_log_init(NULL, ".", MXS_LOG_TARGET_DEFAULT))
-        {
-            size_t n_seconds = atoi(argv[1]);
-
-            if (qc_init(NULL, NULL))
-            {
-                const char* zModule = argv[2];
-
-                const char FORMAT[] = "../storage/%s";
-                char libdir[sizeof(FORMAT) + strlen(zModule)];
-                sprintf(libdir, FORMAT, zModule);
-
-                set_libdir(MXS_STRDUP_A(libdir));
-
-                StorageFactory* pFactory = StorageFactory::Open(zModule);
-
-                if (pFactory)
-                {
-                    TesterRawStorage tester(&cout, pFactory);
-
-                    size_t n_threads = get_processor_count() + 1;
-                    size_t n_max_items = n_threads * n_seconds * 10;
-
-                    if (argc == 3)
-                    {
-                        rv = tester.run(n_threads, n_seconds, n_max_items, cin);
-                    }
-                    else
-                    {
-                        fstream in(argv[3]);
-
-                        if (in)
-                        {
-                            rv = tester.run(n_threads, n_seconds, n_max_items, in);
-                        }
-                        else
-                        {
-                            cerr << "error: Could not open " << argv[3] << "." << endl;
-                        }
-                    }
-
-                    delete pFactory;
-                }
-                else
-                {
-                    cerr << "error: Could not initialize factory " << zModule << "." << endl;
-                }
-            }
-            else
-            {
-                cerr << "error: Could not initialize query classifier." << endl;
-            }
-
-            mxs_log_finish();
-        }
-        else
-        {
-            cerr << "error: Could not initialize log." << endl;
-        }
-    }
-    else
-    {
-        print_usage(argv[0]);
-    }
-
-    return rv;
 }

server/modules/filter/cache/test/teststorage.cc

@@ -0,0 +1,149 @@
+/*
+ * Copyright (c) 2016 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl.
+ *
+ * Change Date: 2019-07-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#include <maxscale/cppdefs.hh>
+#include <stdlib.h>
+#include <iostream>
+#include <maxscale/alloc.h>
+#include <maxscale/gwdirs.h>
+#include <maxscale/log_manager.h>
+#include <maxscale/query_classifier.h>
+#include "storagefactory.hh"
+#include "teststorage.hh"
+
+using namespace std;
+
+TestStorage::TestStorage(ostream* pOut,
+                         size_t threads,
+                         size_t seconds,
+                         size_t items,
+                         size_t min_size,
+                         size_t max_size)
+    : m_out(*pOut)
+    , m_threads(threads)
+    , m_seconds(seconds)
+    , m_items(items)
+    , m_min_size(min_size)
+    , m_max_size(max_size)
+{
+}
+
+TestStorage::~TestStorage()
+{
+}
+
+int TestStorage::run(int argc, char** argv)
+{
+    int rv = EXIT_FAILURE;
+
+    if ((argc >= 2) || (argc <= 7))
+    {
+        if (mxs_log_init(NULL, ".", MXS_LOG_TARGET_DEFAULT))
+        {
+            if (qc_init(NULL, NULL))
+            {
+                const char* zModule = NULL;
+                size_t threads = m_threads;
+                size_t seconds = m_seconds;
+                size_t items = m_items;
+                size_t min_size = m_min_size;
+                size_t max_size = m_max_size;
+
+                switch (argc)
+                {
+                default:
+                    ss_dassert(!true);
+                case 7:
+                    max_size = atoi(argv[6]);
+                case 6:
+                    min_size = atoi(argv[5]);
+                case 5:
+                    items = atoi(argv[4]);
+                case 4:
+                    seconds = atoi(argv[3]);
+                case 3:
+                    threads = atoi(argv[2]);
+                case 2:
+                    zModule = argv[1];
+                }
+
+                if (threads == 0)
+                {
+                    threads = get_processor_count() + 1;
+                }
+
+                if (items == 0)
+                {
+                    items = threads * seconds * 10;
+                }
+
+                const char FORMAT[] = "../storage/%s";
+                char libdir[sizeof(FORMAT) + strlen(zModule)];
+                sprintf(libdir, FORMAT, zModule);
+
+                set_libdir(MXS_STRDUP_A(libdir));
+
+                StorageFactory* pFactory = StorageFactory::Open(zModule);
+
+                if (pFactory)
+                {
+                    out() << "Module  : " << zModule << "\n"
+                          << "Threads : " << threads << "\n"
+                          << "Seconds : " << seconds << "\n"
+                          << "Items   : " << items << "\n"
+                          << "Min-Size: " << min_size << "\n"
+                          << "Max-Size: " << max_size << "\n"
+                          << endl;
+
+                    rv = execute(*pFactory, threads, seconds, items, min_size, max_size);
+
+                    delete pFactory;
+                }
+                else
+                {
+                    cerr << "error: Could not initialize factory " << zModule << "." << endl;
+                }
+            }
+            else
+            {
+                cerr << "error: Could not initialize query classifier." << endl;
+            }
+
+            mxs_log_finish();
+        }
+        else
+        {
+            cerr << "error: Could not initialize log." << endl;
+        }
+    }
+    else
+    {
+        print_usage(argv[0]);
+    }
+
+    return rv;
+}
+
+void TestStorage::print_usage(const char* zProgram)
+{
+    cout << "usage: " << zProgram << " storage-module [threads [time [items [min-size [max-size]]]]]\n"
+         << "\n"
+         << "where:\n"
+         << "  storage-module  is the name of a storage module,\n"
+         << "  threads         is the number of threads to use (if 0, #cores + 1 is used,\n"
+         << "  time            is the number of seconds we should run,\n"
+         << "  items           is the number of items to use when populating the cache,\n"
+         << "                  if 0, threads * seconds * 10 is used\n"
+         << "  min-size        is the minimum size of a cache value, and\n"
+         << "  max-size        is the maximum size of a cache value." << endl;
+}

server/modules/filter/cache/test/teststorage.hh

@@ -0,0 +1,65 @@
+#pragma once
+/*
+ * Copyright (c) 2016 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl.
+ *
+ * Change Date: 2019-07-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#include <maxscale/cppdefs.hh>
+
+class StorageFactory;
+
+class TestStorage
+{
+public:
+    ~TestStorage();
+
+    enum
+    {
+        DEFAULT_THREADS  = 4,
+        DEFAULT_SECONDS  = 10,
+        DEFAULT_ITEMS    = 400,
+        DEFAULT_MIN_SIZE = 1024,
+        DEFAULT_MAX_SIZE = 1024 * 1024
+    };
+
+    int run(int argc, char** argv);
+
+protected:
+    TestStorage(std::ostream* pOut,
+                size_t threads  = DEFAULT_THREADS,
+                size_t seconds  = DEFAULT_SECONDS,
+                size_t items    = DEFAULT_ITEMS,
+                size_t min_size = DEFAULT_MIN_SIZE,
+                size_t max_size = DEFAULT_MAX_SIZE);
+
+    virtual int execute(StorageFactory& factory,
+                        size_t threads,
+                        size_t seconds,
+                        size_t items,
+                        size_t min_size,
+                        size_t max_size) = 0;
+
+    virtual void print_usage(const char* zProgram);
+
+    std::ostream& out() const { return m_out; }
+
+private:
+    TestStorage(const TestStorage&);
+    TestStorage& operator = (const TestStorage&);
+
+private:
+    std::ostream& m_out;
+    size_t m_threads;
+    size_t m_seconds;
+    size_t m_items;
+    size_t m_min_size;
+    size_t m_max_size;
+};