From fab7accef3f24e88ace3a1b4058de5e061f0d37c Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Thu, 8 Dec 2016 14:30:46 +0200 Subject: [PATCH] Cache: Collect RocksDB statistics --- Documentation/Filters/Cache.md | 15 ++++++ server/modules/filter/cache/cachefilter.cc | 3 +- .../storage/storage_rocksdb/rocksdbstorage.cc | 47 +++++++++++++++---- .../storage/storage_rocksdb/rocksdbstorage.h | 3 +- 4 files changed, 57 insertions(+), 11 deletions(-) diff --git a/Documentation/Filters/Cache.md b/Documentation/Filters/Cache.md index 04e68ee8c..046bc51c0 100644 --- a/Documentation/Filters/Cache.md +++ b/Documentation/Filters/Cache.md @@ -457,3 +457,18 @@ storage_options=cache_directory=/mnt/maxscale-cache With the above setting a directory `/mnt/macscale-cache/storage_rocksdb` will created, under which the actual instance specific cache directories are created. + +#### `collect_statistics` + +Specifies whether RocksDB should collect statistics that later can be queried +using `maxadmin`. It should be noted, though, that collecting RocksDB statistics +is not without a cost. From the [RocksDB Documentation](https://github.com/facebook/rocksdb/wiki/Statistics) + +_The overhead of statistics is usually small but non-negligible. We usually +observe an overhead of 5%-10%._ + +The value is a boolean and the default is `false`. + +``` +storage_options=collect_statistics=true +``` \ No newline at end of file diff --git a/server/modules/filter/cache/cachefilter.cc b/server/modules/filter/cache/cachefilter.cc index 4fabcbdac..bf2953053 100644 --- a/server/modules/filter/cache/cachefilter.cc +++ b/server/modules/filter/cache/cachefilter.cc @@ -463,8 +463,9 @@ static bool process_params(char **pzOptions, FILTER_PARAMETER **ppParams, CACHE_ int argc = 1; char *arg = config.storage_options; - while ((arg = strchr(config.storage_options, ','))) + while ((arg = strchr(arg, ','))) { + arg = arg + 1; ++argc; } diff --git a/server/modules/filter/cache/storage/storage_rocksdb/rocksdbstorage.cc b/server/modules/filter/cache/storage/storage_rocksdb/rocksdbstorage.cc index a82eaa28b..8f4b14af8 100644 --- a/server/modules/filter/cache/storage/storage_rocksdb/rocksdbstorage.cc +++ b/server/modules/filter/cache/storage/storage_rocksdb/rocksdbstorage.cc @@ -20,13 +20,10 @@ #include #include #include +#include #include #include -extern "C" -{ -// TODO: Add extern "C" to modutil.h #include -} #include #include "rocksdbinternals.h" @@ -231,6 +228,7 @@ RocksDBStorage* RocksDBStorage::Create(const char* zName, uint32_t ttl, int argc ss_dassert(zName); string storageDirectory = get_cachedir(); + bool collectStatistics = false; for (int i = 0; i < argc; ++i) { @@ -261,6 +259,13 @@ RocksDBStorage* RocksDBStorage::Create(const char* zName, uint32_t ttl, int argc zKey, get_cachedir()); } } + else if (strcmp(zKey, "collect_statistics") == 0) + { + if (zValue) + { + collectStatistics = config_truth_value(zValue); + } + } else { MXS_WARNING("Unknown argument '%s'.", zKey); @@ -269,11 +274,14 @@ RocksDBStorage* RocksDBStorage::Create(const char* zName, uint32_t ttl, int argc storageDirectory += "/storage_rocksdb"; - return Create(storageDirectory, zName, ttl); + return Create(storageDirectory, zName, ttl, collectStatistics); } // static -RocksDBStorage* RocksDBStorage::Create(const string& storageDirectory, const char* zName, uint32_t ttl) +RocksDBStorage* RocksDBStorage::Create(const string& storageDirectory, + const char* zName, + uint32_t ttl, + bool collectStatistics) { RocksDBStorage* pStorage = nullptr; @@ -302,6 +310,11 @@ RocksDBStorage* RocksDBStorage::Create(const string& storageDirectory, const cha options.create_if_missing = true; options.error_if_exists = true; + if (collectStatistics) + { + options.statistics = rocksdb::CreateDBStatistics(); + } + rocksdb::DBWithTTL* pDb; rocksdb::Status status; rocksdb::Slice key(STORAGE_ROCKSDB_VERSION_KEY); @@ -348,11 +361,27 @@ RocksDBStorage* RocksDBStorage::Create(const string& storageDirectory, const cha cache_result_t RocksDBStorage::getInfo(uint32_t what, json_t** ppInfo) const { - *ppInfo = json_object(); + json_t* pInfo = json_object(); - // TODO: Fill with RocksDB statistics. + if (pInfo) + { + auto sStatistics = m_sDb->GetOptions().statistics; - return *ppInfo ? CACHE_RESULT_OK : CACHE_RESULT_OUT_OF_RESOURCES; + for_each(rocksdb::TickersNameMap.begin(), rocksdb::TickersNameMap.end(), + [pInfo, sStatistics](const std::pair& tickerName) { + json_t* pValue = json_integer(sStatistics->getTickerCount(tickerName.first)); + + if (pValue) + { + json_object_set(pInfo, tickerName.second.c_str(), pValue); + json_decref(pValue); + } + }); + + *ppInfo = pInfo; + } + + return pInfo ? CACHE_RESULT_OK : CACHE_RESULT_OUT_OF_RESOURCES; } cache_result_t RocksDBStorage::getKey(const char* zDefaultDB, const GWBUF* pQuery, CACHE_KEY* pKey) diff --git a/server/modules/filter/cache/storage/storage_rocksdb/rocksdbstorage.h b/server/modules/filter/cache/storage/storage_rocksdb/rocksdbstorage.h index d325ee062..2391a6246 100644 --- a/server/modules/filter/cache/storage/storage_rocksdb/rocksdbstorage.h +++ b/server/modules/filter/cache/storage/storage_rocksdb/rocksdbstorage.h @@ -46,7 +46,8 @@ private: static RocksDBStorage* Create(const std::string& storageDirectory, const char* zName, - uint32_t ttl); + uint32_t ttl, + bool collectStatistics); static const rocksdb::WriteOptions& writeOptions() {