hcq/MaxScale
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Cache: Provide storage config as an object

Browse Source 
Now all storage configuration values are provided in a single object.
That way it'll be easier to provide more if necessary and also makes
it straightforward to fetch the configuration, which makes it possible,
for instance, to adapt tests according to the way the storage has been
configured.
This commit is contained in:
Johan Wikman
2016-12-20 12:07:06 +02:00
parent 34b3f5c06e
commit ee422fd3ec
28 changed files with 279 additions and 207 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
server/modules/filter/cache/cache_storage_api.h vendored
View File

@ -95,6 +95,38 @@ static inline bool cache_storage_has_cap(uint32_t capabilities, uint32_t mask)
return (capabilities & mask) == mask;
}
typedef struct cache_storage_config_t
{
/**
* Specifies whether the storage will be used in a single thread or multi
* thread context. In the latter case the storage must perform thread
* synchronization as appropriate, in the former case it need not.
*/
cache_thread_model_t thread_model;
/**
* Time to live; number of seconds the value is valid. A value of 0 means
* that there is no time-to-live, but that the value is considered fresh
* as long as it is available.
*/
uint32_t ttl;
/**
* The maximum number of items the storage may store, before it should
* evict some items. A value of 0 means that there is no limit. The caller
* should specify 0, unless CACHE_STORAGE_CAP_MAX_COUNT is returned at
* initialization.
*/
uint32_t max_count;
/**
* The maximum size of the storage may may occupy, before it should evict
* some items. A value if 0 means that there is no limit. The caller should
* specify 0, unless CACHE_STORAGE_CAP_MAX_SIZE is returned at initialization.
*/
uint64_t max_size;
} CACHE_STORAGE_CONFIG;
typedef struct cache_storage_api
{
/**
@ -112,22 +144,8 @@ typedef struct cache_storage_api
* create the actual storage, initialize it and prepare to put and get
* cache items.
*
* @param model Whether the storage will be used in a single thread or
* multi thread context. In the latter case the storage must
* perform thread synchronization as appropriate, in the former
* case it need not.
* @param name The name of the cache instance.
* @param ttl Time to live; number of seconds the value is valid.
* A value of 0 means that there is no time-to-live, but that
* the value is considered fresh as long as it is available.
* @param max_count The maximum number of items the storage may store, before
* it should evict some items. A value of 0 means that there is
* no limit. The caller should specify 0, unless
* CACHE_STORAGE_CAP_MAX_COUNT is returned at initialization.
* @param max_count The maximum size of the storage may may occupy, before it
* should evict some items. A value if 0 means that there is
* no limit. The caller should specify 0, unless
* CACHE_STORAGE_CAP_MAX_SIZE is returned at initialization.
* @param config The storage configuration.
* @param argc The number of elements in the argv array.
* @param argv Array of arguments, as passed in the `storage_options`
* parameter in the cache section in the MaxScale configuration
@ -136,11 +154,8 @@ typedef struct cache_storage_api
* @return A new cache instance, or NULL if the instance could not be
* created.
*/
CACHE_STORAGE* (*createInstance)(cache_thread_model_t model,
const char *name,
uint32_t ttl,
uint32_t max_count,
uint64_t max_size,
CACHE_STORAGE* (*createInstance)(const char *name,
const CACHE_STORAGE_CONFIG* config,
int argc, char* argv[]);
/**
@ -162,6 +177,15 @@ typedef struct cache_storage_api
*/
void (*freeInstance)(CACHE_STORAGE* instance);
/**
* Returns the configuration the storage was created with.
*
* @param storage Pointer to a CACHE_STORAGE
* @param config Pointer to variable that will be updated with the config.
*/
void (*getConfig)(CACHE_STORAGE* storage,
CACHE_STORAGE_CONFIG* config);
/**
* Returns information about the storage.
*

31
server/modules/filter/cache/cache_storage_api.hh vendored
View File

@ -67,3 +67,34 @@ public:
memset(data, 0, sizeof(data));
}
};
class CacheStorageConfig : public CACHE_STORAGE_CONFIG
{
public:
CacheStorageConfig(cache_thread_model_t thread_model,
uint32_t ttl = 0,
uint32_t max_count = 0,
uint64_t max_size = 0)
{
this->thread_model = thread_model;
this->ttl = ttl;
this->max_count = max_count;
this->max_size = max_count;
}
CacheStorageConfig()
{
thread_model = CACHE_THREAD_MODEL_MT;
ttl = 0;
max_count = 0;
max_size = 0;
}
CacheStorageConfig(const CACHE_STORAGE_CONFIG& config)
{
thread_model = config.thread_model;
ttl = config.ttl;
max_count = config.max_count;
max_size = config.max_size;
}
};

11
server/modules/filter/cache/cachemt.cc vendored
View File

@ -84,16 +84,15 @@ CacheMT* CacheMT::Create(const std::string& name,
{
CacheMT* pCache = NULL;
uint32_t ttl = pConfig->ttl;
uint32_t maxCount = pConfig->max_count;
uint32_t maxSize = pConfig->max_size;
CacheStorageConfig storage_config(CACHE_THREAD_MODEL_MT,
pConfig->ttl,
pConfig->max_count,
pConfig->max_size);
int argc = pConfig->storage_argc;
char** argv = pConfig->storage_argv;
Storage* pStorage = sFactory->createStorage(CACHE_THREAD_MODEL_MT, name.c_str(),
ttl, maxCount, maxSize,
argc, argv);
Storage* pStorage = sFactory->createStorage(name.c_str(), storage_config, argc, argv);
if (pStorage)
{

11
server/modules/filter/cache/cachest.cc vendored
View File

@ -88,16 +88,15 @@ CacheST* CacheST::Create(const std::string& name,
{
CacheST* pCache = NULL;
uint32_t ttl = pConfig->ttl;
uint32_t maxCount = pConfig->max_count;
uint32_t maxSize = pConfig->max_size;
CacheStorageConfig storage_config(CACHE_THREAD_MODEL_ST,
pConfig->ttl,
pConfig->max_count,
pConfig->max_size);
int argc = pConfig->storage_argc;
char** argv = pConfig->storage_argv;
Storage* pStorage = sFactory->createStorage(CACHE_THREAD_MODEL_ST, name.c_str(),
ttl, maxCount, maxSize,
argc, argv);
Storage* pStorage = sFactory->createStorage(name.c_str(), storage_config, argc, argv);
if (pStorage)
{

14
server/modules/filter/cache/lrustorage.cc vendored
View File

@ -14,10 +14,11 @@
#define MXS_MODULE_NAME "cache"
#include "lrustorage.hh"
LRUStorage::LRUStorage(Storage* pstorage, size_t max_count, size_t max_size)
: pstorage_(pstorage)
, max_count_(max_count != 0 ? max_count : UINT64_MAX)
, max_size_(max_size != 0 ? max_size : UINT64_MAX)
LRUStorage::LRUStorage(const CACHE_STORAGE_CONFIG& config, Storage* pstorage)
: config_(config)
, pstorage_(pstorage)
, max_count_(config.max_count != 0 ? config.max_count : UINT64_MAX)
, max_size_(config.max_size != 0 ? config.max_size : UINT64_MAX)
, phead_(NULL)
, ptail_(NULL)
{
@ -35,6 +36,11 @@ LRUStorage::~LRUStorage()
delete pstorage_;
}
void LRUStorage::get_config(CACHE_STORAGE_CONFIG* pConfig)
{
*pConfig = config_;
}
cache_result_t LRUStorage::get_key(const char* zdefault_db,
const GWBUF* pquery,
CACHE_KEY* pkey) const

12
server/modules/filter/cache/lrustorage.hh vendored
View File

@ -23,6 +23,11 @@ class LRUStorage : public Storage
public:
~LRUStorage();
/**
* @see Storage::get_config
*/
void get_config(CACHE_STORAGE_CONFIG* pConfig);
/**
* @see Storage::get_key
*/
@ -31,7 +36,7 @@ public:
CACHE_KEY* pKey) const;
protected:
LRUStorage(Storage* pstorage, uint64_t max_count, uint64_t max_size);
LRUStorage(const CACHE_STORAGE_CONFIG& config, Storage* pstorage);
/**
* @see Storage::get_info
@ -234,9 +239,10 @@ private:
uint64_t evictions; /*< How many times an item has been evicted from the cache. */
};
const CACHE_STORAGE_CONFIG config_; /*< The configuration. */
Storage* pstorage_; /*< The actual storage. */
uint64_t max_count_; /*< The maximum number of items in the LRU list, */
uint64_t max_size_; /*< The maximum size of all cached items. */
const uint64_t max_count_; /*< The maximum number of items in the LRU list, */
const uint64_t max_size_; /*< The maximum size of all cached items. */
mutable Stats stats_; /*< Cache statistics. */
mutable NodesByKey nodes_by_key_; /*< Mapping from cache keys to corresponding Node. */
mutable Node* phead_; /*< The node at the LRU list. */

8
server/modules/filter/cache/lrustoragemt.cc vendored
View File

@ -16,8 +16,8 @@
using maxscale::SpinLockGuard;
LRUStorageMT::LRUStorageMT(Storage* pstorage, uint64_t max_count, uint64_t max_size)
: LRUStorage(pstorage, max_count, max_size)
LRUStorageMT::LRUStorageMT(const CACHE_STORAGE_CONFIG& config, Storage* pstorage)
: LRUStorage(config, pstorage)
{
spinlock_init(&lock_);
@ -28,11 +28,11 @@ LRUStorageMT::~LRUStorageMT()
{
}
LRUStorageMT* LRUStorageMT::create(Storage* pstorage, uint64_t max_count, uint64_t max_size)
LRUStorageMT* LRUStorageMT::create(const CACHE_STORAGE_CONFIG& config, Storage* pstorage)
{
LRUStorageMT* plru_storage = NULL;
MXS_EXCEPTION_GUARD(plru_storage = new LRUStorageMT(pstorage, max_count, max_size));
MXS_EXCEPTION_GUARD(plru_storage = new LRUStorageMT(config, pstorage));
return plru_storage;
}

4
server/modules/filter/cache/lrustoragemt.hh vendored
View File

@ -21,7 +21,7 @@ class LRUStorageMT : public LRUStorage
public:
~LRUStorageMT();
static LRUStorageMT* create(Storage* pstorage, uint64_t max_count, uint64_t max_size);
static LRUStorageMT* create(const CACHE_STORAGE_CONFIG& config, Storage* pstorage);
cache_result_t get_info(uint32_t what,
json_t** ppInfo) const;
@ -46,7 +46,7 @@ public:
cache_result_t get_items(uint64_t* pItems) const;
private:
LRUStorageMT(Storage* pstorage, uint64_t max_count, uint64_t max_size);
LRUStorageMT(const CACHE_STORAGE_CONFIG& config, Storage* pstorage);
LRUStorageMT(const LRUStorageMT&);
LRUStorageMT& operator = (const LRUStorageMT&);

8
server/modules/filter/cache/lrustoragest.cc vendored
View File

@ -14,8 +14,8 @@
#define MXS_MODULE_NAME "cache"
#include "lrustoragest.hh"
LRUStorageST::LRUStorageST(Storage* pstorage, uint64_t max_count, uint64_t max_size)
: LRUStorage(pstorage, max_count, max_size)
LRUStorageST::LRUStorageST(const CACHE_STORAGE_CONFIG& config, Storage* pstorage)
: LRUStorage(config, pstorage)
{
MXS_NOTICE("Created single threaded LRU storage.");
}
@ -24,11 +24,11 @@ LRUStorageST::~LRUStorageST()
{
}
LRUStorageST* LRUStorageST::create(Storage* pstorage, uint64_t max_count, uint64_t max_size)
LRUStorageST* LRUStorageST::create(const CACHE_STORAGE_CONFIG& config, Storage* pstorage)
{
LRUStorageST* plru_storage = NULL;
MXS_EXCEPTION_GUARD(plru_storage = new LRUStorageST(pstorage, max_count, max_size));
MXS_EXCEPTION_GUARD(plru_storage = new LRUStorageST(config, pstorage));
return plru_storage;
}

4
server/modules/filter/cache/lrustoragest.hh vendored
View File

@ -20,7 +20,7 @@ class LRUStorageST : public LRUStorage
public:
~LRUStorageST();
static LRUStorageST* create(Storage* pstorage, uint64_t max_count, uint64_t max_size);
static LRUStorageST* create(const CACHE_STORAGE_CONFIG& config, Storage* pstorage);
cache_result_t get_info(uint32_t what,
json_t** ppInfo) const;
@ -45,7 +45,7 @@ public:
cache_result_t get_items(uint64_t* pItems) const;
private:
LRUStorageST(Storage* pstorage, uint64_t max_count, uint64_t max_size);
LRUStorageST(const CACHE_STORAGE_CONFIG& config, Storage* pstorage);
LRUStorageST(const LRUStorageST&);
LRUStorageST& operator = (const LRUStorageST&);

7
server/modules/filter/cache/storage.hh vendored
View File

@ -25,6 +25,13 @@ public:
virtual ~Storage();
/**
* Returns the configuration the storage was created with.
*
* @param pConfig Pointer to object that will be updated.
*/
virtual void get_config(CACHE_STORAGE_CONFIG* pConfig) = 0;
/**
* Returns information about the storage.
*

12
server/modules/filter/cache/storage/storage_inmemory/inmemorystorage.cc vendored
View File

@ -35,10 +35,9 @@ const size_t INMEMORY_KEY_LENGTH = 2 * SHA512_DIGEST_LENGTH;
}
InMemoryStorage::InMemoryStorage(const string& name,
uint32_t ttl)
InMemoryStorage::InMemoryStorage(const string& name, const CACHE_STORAGE_CONFIG& config)
: name_(name)
, ttl_(ttl)
, config_(config)
{
}
@ -108,6 +107,11 @@ cache_result_t InMemoryStorage::get_key(const char* zdefault_db, const GWBUF* pq
return CACHE_RESULT_OK;
}
void InMemoryStorage::get_config(CACHE_STORAGE_CONFIG* pConfig)
{
*pConfig = config_;
}
cache_result_t InMemoryStorage::get_head(CACHE_KEY* pKey, GWBUF** ppHead) const
{
return CACHE_RESULT_OUT_OF_RESOURCES;
@ -154,7 +158,7 @@ cache_result_t InMemoryStorage::do_get_value(const CACHE_KEY& key, uint32_t flag
uint32_t now = time(NULL);
bool is_stale = ttl_ == 0 ? false : (now - entry.time > ttl_);
bool is_stale = config_.ttl == 0 ? false : (now - entry.time > config_.ttl);
if (!is_stale || ((flags & CACHE_FLAGS_INCLUDE_STALE) != 0))
{

6
server/modules/filter/cache/storage/storage_inmemory/inmemorystorage.hh vendored
View File

@ -26,6 +26,7 @@ public:
static cache_result_t get_key(const char* zdefault_db, const GWBUF* pquery, CACHE_KEY* pkey);
void get_config(CACHE_STORAGE_CONFIG* pConfig);
virtual cache_result_t get_info(uint32_t what, json_t** ppInfo) const = 0;
virtual cache_result_t get_value(const CACHE_KEY& key, uint32_t flags, GWBUF** ppresult) = 0;
virtual cache_result_t put_value(const CACHE_KEY& key, const GWBUF* pvalue) = 0;
@ -37,7 +38,8 @@ public:
cache_result_t get_items(uint64_t* pItems) const;
protected:
InMemoryStorage(const std::string& name, uint32_t ttl);
InMemoryStorage(const std::string& name,
const CACHE_STORAGE_CONFIG& config);
cache_result_t do_get_info(uint32_t what, json_t** ppInfo) const;
cache_result_t do_get_value(const CACHE_KEY& key, uint32_t flags, GWBUF** ppresult);
@ -85,7 +87,7 @@ private:
typedef std::tr1::unordered_map<CACHE_KEY, Entry> Entries;
std::string name_;
uint32_t ttl_;
const CACHE_STORAGE_CONFIG config_;
Entries entries_;
Stats stats_;
};

9
server/modules/filter/cache/storage/storage_inmemory/inmemorystoragemt.cc vendored
View File

@ -17,8 +17,9 @@
using maxscale::SpinLockGuard;
using std::auto_ptr;
InMemoryStorageMT::InMemoryStorageMT(const std::string& name, uint32_t ttl)
: InMemoryStorage(name, ttl)
InMemoryStorageMT::InMemoryStorageMT(const std::string& name,
const CACHE_STORAGE_CONFIG& config)
: InMemoryStorage(name, config)
{
spinlock_init(&lock_);
}
@ -29,10 +30,10 @@ InMemoryStorageMT::~InMemoryStorageMT()
// static
auto_ptr<InMemoryStorageMT> InMemoryStorageMT::create(const std::string& name,
uint32_t ttl,
const CACHE_STORAGE_CONFIG& config,
int argc, char* argv[])
{
return auto_ptr<InMemoryStorageMT>(new InMemoryStorageMT(name, ttl));
return auto_ptr<InMemoryStorageMT>(new InMemoryStorageMT(name, config));
}
cache_result_t InMemoryStorageMT::get_info(uint32_t what, json_t** ppInfo) const

6
server/modules/filter/cache/storage/storage_inmemory/inmemorystoragemt.hh vendored
View File

@ -23,7 +23,9 @@ public:
typedef std::auto_ptr<InMemoryStorageMT> SInMemoryStorageMT;
static SInMemoryStorageMT create(const std::string& name, uint32_t ttl, int argc, char* argv[]);
static SInMemoryStorageMT create(const std::string& name,
const CACHE_STORAGE_CONFIG& config,
int argc, char* argv[]);
cache_result_t get_info(uint32_t what, json_t** ppInfo) const;
cache_result_t get_value(const CACHE_KEY& key, uint32_t flags, GWBUF** ppresult);
@ -31,7 +33,7 @@ public:
cache_result_t del_value(const CACHE_KEY& key);
private:
InMemoryStorageMT(const std::string& name, uint32_t ttl);
InMemoryStorageMT(const std::string& name, const CACHE_STORAGE_CONFIG& config);
private:
InMemoryStorageMT(const InMemoryStorageMT&);

9
server/modules/filter/cache/storage/storage_inmemory/inmemorystoragest.cc vendored
View File

@ -16,8 +16,9 @@
using std::auto_ptr;
InMemoryStorageST::InMemoryStorageST(const std::string& name, uint32_t ttl)
: InMemoryStorage(name, ttl)
InMemoryStorageST::InMemoryStorageST(const std::string& name,
const CACHE_STORAGE_CONFIG& config)
: InMemoryStorage(name, config)
{
}
@ -27,10 +28,10 @@ InMemoryStorageST::~InMemoryStorageST()
// static
auto_ptr<InMemoryStorageST> InMemoryStorageST::create(const std::string& name,
uint32_t ttl,
const CACHE_STORAGE_CONFIG& config,
int argc, char* argv[])
{
return auto_ptr<InMemoryStorageST>(new InMemoryStorageST(name, ttl));
return auto_ptr<InMemoryStorageST>(new InMemoryStorageST(name, config));
}
cache_result_t InMemoryStorageST::get_info(uint32_t what, json_t** ppinfo) const

6
server/modules/filter/cache/storage/storage_inmemory/inmemorystoragest.hh vendored
View File

@ -22,7 +22,9 @@ public:
typedef std::auto_ptr<InMemoryStorageST> SInMemoryStorageST;
static SInMemoryStorageST create(const std::string& name, uint32_t ttl, int argc, char* argv[]);
static SInMemoryStorageST create(const std::string& name,
const CACHE_STORAGE_CONFIG& config,
int argc, char* argv[]);
cache_result_t get_info(uint32_t what, json_t** ppinfo) const;
cache_result_t get_value(const CACHE_KEY& key, uint32_t flags, GWBUF** ppresult);
@ -30,7 +32,7 @@ public:
cache_result_t del_value(const CACHE_KEY& key);
private:
InMemoryStorageST(const std::string& name, uint32_t ttl);
InMemoryStorageST(const std::string& name, const CACHE_STORAGE_CONFIG& config);
private:
InMemoryStorageST(const InMemoryStorageST&);

34
server/modules/filter/cache/storage/storage_inmemory/storage_inmemory.cc vendored
View File

@ -31,40 +31,38 @@ bool initialize(uint32_t* pcapabilities)
return true;
}
CACHE_STORAGE* createInstance(cache_thread_model_t model,
const char* zname,
uint32_t ttl,
uint32_t max_count,
uint64_t max_size,
CACHE_STORAGE* createInstance(const char* zname,
const CACHE_STORAGE_CONFIG* pConfig,
int argc, char* argv[])
{
ss_dassert(zname);
if (max_count != 0)
if (pConfig->max_count != 0)
{
MXS_WARNING("A maximum item count of %u specified, although 'storage_inMemory' "
"does not enforce such a limit.", (unsigned int)max_count);
"does not enforce such a limit.", (unsigned int)pConfig->max_count);
}
if (max_size != 0)
if (pConfig->max_size != 0)
{
MXS_WARNING("A maximum size of %lu specified, although 'storage_inMemory' "
"does not enforce such a limit.", (unsigned long)max_size);
"does not enforce such a limit.", (unsigned long)pConfig->max_size);
}
auto_ptr<InMemoryStorage> sStorage;
switch (model)
switch (pConfig->thread_model)
{
case CACHE_THREAD_MODEL_ST:
MXS_EXCEPTION_GUARD(sStorage = InMemoryStorageST::create(zname, ttl, argc, argv));
MXS_EXCEPTION_GUARD(sStorage = InMemoryStorageST::create(zname, *pConfig, argc, argv));
break;
default:
ss_dassert(!true);
MXS_ERROR("Unknown thread model %d, creating multi-thread aware storage.", (int)model);
MXS_ERROR("Unknown thread model %d, creating multi-thread aware storage.",
(int)pConfig->thread_model);
case CACHE_THREAD_MODEL_MT:
MXS_EXCEPTION_GUARD(sStorage = InMemoryStorageMT::create(zname, ttl, argc, argv));
MXS_EXCEPTION_GUARD(sStorage = InMemoryStorageMT::create(zname, *pConfig, argc, argv));
break;
}
@ -96,6 +94,15 @@ void freeInstance(CACHE_STORAGE* pinstance)
MXS_EXCEPTION_GUARD(delete reinterpret_cast<InMemoryStorage*>(pinstance));
}
void getConfig(CACHE_STORAGE* pStorage,
CACHE_STORAGE_CONFIG* pConfig)
{
ss_dassert(pStorage);
ss_dassert(pConfig);
MXS_EXCEPTION_GUARD(reinterpret_cast<InMemoryStorage*>(pStorage)->get_config(pConfig));
}
cache_result_t getInfo(CACHE_STORAGE* pStorage,
uint32_t what,
json_t** ppInfo)
@ -219,6 +226,7 @@ CACHE_STORAGE_API* CacheGetStorageAPI()
createInstance,
getKey,
freeInstance,
getConfig,
getInfo,
getValue,
putValue,

33
server/modules/filter/cache/storage/storage_rocksdb/rocksdbstorage.cc vendored
View File

@ -193,14 +193,14 @@ bool deletePath(const string& path)
rocksdb::WriteOptions RocksDBStorage::s_writeOptions;
//private
RocksDBStorage::RocksDBStorage(unique_ptr<rocksdb::DBWithTTL>& sDb,
const string& name,
RocksDBStorage::RocksDBStorage(const string& name,
const CACHE_STORAGE_CONFIG& config,
const string& path,
uint32_t ttl)
: m_sDb(std::move(sDb))
, m_name(name)
unique_ptr<rocksdb::DBWithTTL>& sDb)
: m_name(name)
, m_config(config)
, m_path(path)
, m_ttl(ttl)
, m_sDb(std::move(sDb))
{
}
@ -224,7 +224,7 @@ bool RocksDBStorage::Initialize()
//static
unique_ptr<RocksDBStorage> RocksDBStorage::Create(const char* zName,
uint32_t ttl,
const CACHE_STORAGE_CONFIG& config,
int argc, char* argv[])
{
ss_dassert(zName);
@ -276,13 +276,13 @@ unique_ptr<RocksDBStorage> RocksDBStorage::Create(const char* zName,
storageDirectory += "/storage_rocksdb";
return Create(storageDirectory, zName, ttl, collectStatistics);
return Create(zName, config, storageDirectory, collectStatistics);
}
// static
unique_ptr<RocksDBStorage> RocksDBStorage::Create(const string& storageDirectory,
const char* zName,
uint32_t ttl,
unique_ptr<RocksDBStorage> RocksDBStorage::Create(const char* zName,
const CACHE_STORAGE_CONFIG& config,
const string& storageDirectory,
bool collectStatistics)
{
unique_ptr<RocksDBStorage> sStorage;
@ -321,7 +321,7 @@ unique_ptr<RocksDBStorage> RocksDBStorage::Create(const string& storageDirectory
rocksdb::Status status;
rocksdb::Slice key(STORAGE_ROCKSDB_VERSION_KEY);
status = rocksdb::DBWithTTL::Open(options, path, &pDb, ttl);
status = rocksdb::DBWithTTL::Open(options, path, &pDb, config.ttl);
if (status.ok())
{
@ -343,7 +343,7 @@ unique_ptr<RocksDBStorage> RocksDBStorage::Create(const string& storageDirectory
unique_ptr<rocksdb::DBWithTTL> sDb(pDb);
sStorage = unique_ptr<RocksDBStorage>(new RocksDBStorage(sDb, zName, path, ttl));
sStorage = unique_ptr<RocksDBStorage>(new RocksDBStorage(zName, config, path, sDb));
}
else
{
@ -420,6 +420,11 @@ cache_result_t RocksDBStorage::GetKey(const char* zDefaultDB, const GWBUF* pQuer
return CACHE_RESULT_OK;
}
void RocksDBStorage::getConfig(CACHE_STORAGE_CONFIG* pConfig)
{
*pConfig = m_config;
}
cache_result_t RocksDBStorage::getInfo(uint32_t what, json_t** ppInfo) const
{
json_t* pInfo = json_object();
@ -461,7 +466,7 @@ cache_result_t RocksDBStorage::getValue(const CACHE_KEY* pKey, uint32_t flags, G
case rocksdb::Status::kOk:
if (value.length() >= RocksDBInternals::TS_LENGTH)
{
bool isStale = RocksDBInternals::IsStale(value, m_ttl, rocksdb::Env::Default());
bool isStale = RocksDBInternals::IsStale(value, m_config.ttl, rocksdb::Env::Default());
if (!isStale || ((flags & CACHE_FLAGS_INCLUDE_STALE) != 0))
{

21
server/modules/filter/cache/storage/storage_rocksdb/rocksdbstorage.hh vendored
View File

@ -25,11 +25,14 @@ public:
static bool Initialize();
static SRocksDBStorage Create(const char* zName, uint32_t ttl, int argc, char* argv[]);
static SRocksDBStorage Create(const char* zName,
const CACHE_STORAGE_CONFIG& config,
int argc, char* argv[]);
~RocksDBStorage();
static cache_result_t GetKey(const char* zDefaultDB, const GWBUF* pQuery, CACHE_KEY* pKey);
void getConfig(CACHE_STORAGE_CONFIG* pConfig);
cache_result_t getInfo(uint32_t flags, json_t** ppInfo) const;
cache_result_t getValue(const CACHE_KEY* pKey, uint32_t flags, GWBUF** ppResult);
cache_result_t putValue(const CACHE_KEY* pKey, const GWBUF* pValue);
@ -41,17 +44,17 @@ public:
cache_result_t getItems(uint64_t* pItems) const;
private:
RocksDBStorage(std::unique_ptr<rocksdb::DBWithTTL>& sDb,
const std::string& name,
RocksDBStorage(const std::string& name,
const CACHE_STORAGE_CONFIG& config,
const std::string& path,
uint32_t ttl);
std::unique_ptr<rocksdb::DBWithTTL>& sDb);
RocksDBStorage(const RocksDBStorage&) = delete;
RocksDBStorage& operator = (const RocksDBStorage&) = delete;
static SRocksDBStorage Create(const std::string& storageDirectory,
const char* zName,
uint32_t ttl,
static SRocksDBStorage Create(const char* zName,
const CACHE_STORAGE_CONFIG& config,
const std::string& storageDirectory,
bool collectStatistics);
static const rocksdb::WriteOptions& writeOptions()
@ -60,10 +63,10 @@ private:
}
private:
std::unique_ptr<rocksdb::DBWithTTL> m_sDb;
std::string m_name;
const CACHE_STORAGE_CONFIG m_config;
std::string m_path;
uint32_t m_ttl;
std::unique_ptr<rocksdb::DBWithTTL> m_sDb;
static rocksdb::WriteOptions s_writeOptions;
};

25
server/modules/filter/cache/storage/storage_rocksdb/storage_rocksdb.cc vendored
View File

@ -29,30 +29,27 @@ bool initialize(uint32_t* pCapabilities)
return RocksDBStorage::Initialize();
}
CACHE_STORAGE* createInstance(cache_thread_model_t, // Ignored, RocksDB always MT safe.
const char* zName,
uint32_t ttl,
uint32_t maxCount,
uint64_t maxSize,
CACHE_STORAGE* createInstance(const char* zName,
const CACHE_STORAGE_CONFIG* pConfig,
int argc, char* argv[])
{
ss_dassert(zName);
if (maxCount != 0)
if (pConfig->max_count != 0)
{
MXS_WARNING("A maximum item count of %u specifed, although 'storage_rocksdb' "
"does not enforce such a limit.", (unsigned int)maxCount);
"does not enforce such a limit.", (unsigned int)pConfig->max_count);
}
if (maxSize != 0)
if (pConfig->max_size != 0)
{
MXS_WARNING("A maximum size of %lu specified, although 'storage_rocksdb' "
"does not enforce such a limit.", (unsigned long)maxSize);
"does not enforce such a limit.", (unsigned long)pConfig->max_size);
}
unique_ptr<RocksDBStorage> sStorage;
MXS_EXCEPTION_GUARD(sStorage = RocksDBStorage::Create(zName, ttl, argc, argv));
MXS_EXCEPTION_GUARD(sStorage = RocksDBStorage::Create(zName, *pConfig, argc, argv));
if (sStorage)
{
@ -67,6 +64,13 @@ void freeInstance(CACHE_STORAGE* pInstance)
MXS_EXCEPTION_GUARD(delete reinterpret_cast<RocksDBStorage*>(pInstance));
}
void getConfig(CACHE_STORAGE* pStorage, CACHE_STORAGE_CONFIG* pConfig)
{
ss_dassert(pStorage);
MXS_EXCEPTION_GUARD(reinterpret_cast<RocksDBStorage*>(pStorage)->getConfig(pConfig));
}
cache_result_t getInfo(CACHE_STORAGE* pStorage,
uint32_t what,
json_t** ppInfo)
@ -204,6 +208,7 @@ CACHE_STORAGE_API* CacheGetStorageAPI()
createInstance,
getKey,
freeInstance,
getConfig,
getInfo,
getValue,
putValue,

43
server/modules/filter/cache/storagefactory.cc vendored
View File

@ -146,19 +146,14 @@ StorageFactory* StorageFactory::Open(const char* zName)
return pFactory;
}
Storage* StorageFactory::createStorage(cache_thread_model_t model,
const char* zName,
uint32_t ttl,
uint64_t max_count,
uint64_t max_size,
Storage* StorageFactory::createStorage(const char* zName,
const CACHE_STORAGE_CONFIG& config,
int argc, char* argv[])
{
ss_dassert(m_handle);
ss_dassert(m_pApi);
cache_thread_model_t used_model;
uint64_t used_max_count;
uint64_t used_max_size;
CacheStorageConfig used_config(config);
uint32_t mask = CACHE_STORAGE_CAP_MAX_COUNT | CACHE_STORAGE_CAP_MAX_SIZE;
@ -167,19 +162,12 @@ Storage* StorageFactory::createStorage(cache_thread_model_t model,
// Since we will wrap the native storage with a LRUStorage, according
// to the used threading model, the storage itself may be single
// threaded. No point in locking twice.
used_model = CACHE_THREAD_MODEL_ST;
used_max_count = 0;
used_max_size = 0;
}
else
{
used_model = model;
used_max_count = max_count;
used_max_size = max_size;
used_config.thread_model = CACHE_THREAD_MODEL_ST;
used_config.max_count = 0;
used_config.max_size = 0;
}
Storage* pStorage = createRawStorage(used_model, zName, ttl, used_max_count, used_max_size,
argc, argv);
Storage* pStorage = createRawStorage(zName, config, argc, argv);
if (pStorage)
{
@ -190,15 +178,15 @@ Storage* StorageFactory::createStorage(cache_thread_model_t model,
LRUStorage *pLruStorage = NULL;
if (model == CACHE_THREAD_MODEL_ST)
if (config.thread_model == CACHE_THREAD_MODEL_ST)
{
pLruStorage = LRUStorageST::create(pStorage, max_count, max_size);
pLruStorage = LRUStorageST::create(config, pStorage);
}
else
{
ss_dassert(model == CACHE_THREAD_MODEL_MT);
ss_dassert(config.thread_model == CACHE_THREAD_MODEL_MT);
pLruStorage = LRUStorageMT::create(pStorage, max_count, max_size);
pLruStorage = LRUStorageMT::create(config, pStorage);
}
if (pLruStorage)
@ -217,11 +205,8 @@ Storage* StorageFactory::createStorage(cache_thread_model_t model,
}
Storage* StorageFactory::createRawStorage(cache_thread_model_t model,
const char* zName,
uint32_t ttl,
uint64_t max_count,
uint64_t max_size,
Storage* StorageFactory::createRawStorage(const char* zName,
const CACHE_STORAGE_CONFIG& config,
int argc, char* argv[])
{
ss_dassert(m_handle);
@ -229,7 +214,7 @@ Storage* StorageFactory::createRawStorage(cache_thread_model_t model,
Storage* pStorage = 0;
CACHE_STORAGE* pRawStorage = m_pApi->createInstance(model, zName, ttl, max_count, max_size, argc, argv);
CACHE_STORAGE* pRawStorage = m_pApi->createInstance(zName, &config, argc, argv);
if (pRawStorage)
{

32
server/modules/filter/cache/storagefactory.hh vendored
View File

@ -49,24 +49,16 @@ public:
* implementation that will be provided on top of what is "natively"
* provided.
*
* @param model The needed thread model.
* @param zName The name of the storage.
* @param ttl Time to live. 0 means no limit.
* @param max_count The maximum number of items in the cache.
* 0 means no limit.
* @param max_size The maximum size of the cache. 0 means
* no limit.
* @param config The storagfe configuration.
* @argc Number of items in argv.
* @argv Storage specific arguments.
*
* @return A storage instance or NULL in case of errors.
*/
Storage* createStorage(cache_thread_model_t model,
const char* zName,
uint32_t ttl,
uint64_t max_count,
uint64_t max_size,
int argc, char* argv[]);
Storage* createStorage(const char* zName,
const CACHE_STORAGE_CONFIG& config,
int argc = 0, char* argv[] = NULL);
/**
* Create raw storage instance.
@ -76,24 +68,16 @@ public:
* arguments (notably max_count and max_size) should be adjusted
* accordingly.
*
* @param model The needed thread model.
* @param zName The name of the storage.
* @param ttl Time to live. 0 means no limit.
* @param max_count The maximum number of items in the cache.
* 0 means no limit.
* @param max_size The maximum size of the cache. 0 means
* no limit.
* @param config The storagfe configuration.
* @argc Number of items in argv.
* @argv Storage specific arguments.
*
* @return A storage instance or NULL in case of errors.
*/
Storage* createRawStorage(cache_thread_model_t model,
const char* zName,
uint32_t ttl,
uint64_t max_count,
uint64_t max_size,
int argc, char* argv[]);
Storage* createRawStorage(const char* zName,
const CACHE_STORAGE_CONFIG& config,
int argc = 0, char* argv[] = NULL);
/**
* Create a key for a GWBUF.

5
server/modules/filter/cache/storagereal.cc vendored
View File

@ -28,6 +28,11 @@ StorageReal::~StorageReal()
m_pApi->freeInstance(m_pStorage);
}
void StorageReal::get_config(CACHE_STORAGE_CONFIG* pConfig)
{
m_pApi->getConfig(m_pStorage, pConfig);
}
cache_result_t StorageReal::get_info(uint32_t flags, json_t** ppInfo) const
{
return m_pApi->getInfo(m_pStorage, flags, ppInfo);

2
server/modules/filter/cache/storagereal.hh vendored
View File

@ -20,6 +20,8 @@ class StorageReal : public Storage
public:
~StorageReal();
void get_config(CACHE_STORAGE_CONFIG* pConfig);
cache_result_t get_info(uint32_t flags,
json_t** ppInfo) const;

2
server/modules/filter/cache/test/CMakeLists.txt vendored
View File

@ -35,4 +35,4 @@ add_test(TestCache_storage_rocksdb testrawstorage storage_rocksdb 0 10 1000 10
#usage: testlrustorage storage-module [threads [time [items [min-size [max-size]]]]]\n"
add_test(TestCache_lru_inmemory testlrustorage storage_inmemory 0 10 1000 1024 1024000)
add_test(TestCache_lru_rocksdb testlrustorage storage_inmemory 0 10 1000 1024 1024000)
add_test(TestCache_lru_rocksdb testlrustorage storage_rocksdb 0 10 1000 1024 1024000)

42
server/modules/filter/cache/test/testerlrustorage.cc vendored
View File

@ -62,12 +62,9 @@ int TesterLRUStorage::test_lru(const CacheItems& cache_items, uint64_t size)
size_t items = cache_items.size() > 100 ? 100 : cache_items.size();
Storage* pStorage = m_factory.createStorage(CACHE_THREAD_MODEL_MT,
"unspecified",
0, // No TTL
0, // No max count
0, // No max size
0, NULL);
CacheStorageConfig config(CACHE_THREAD_MODEL_MT);
Storage* pStorage = m_factory.createStorage("unspecified", config);
if (pStorage)
{
@ -145,6 +142,8 @@ int TesterLRUStorage::test_lru(const CacheItems& cache_items, uint64_t size)
gwbuf_free(pValue);
}
delete pStorage;
}
return rv;
@ -161,12 +160,10 @@ int TesterLRUStorage::test_max_count(size_t n_threads, size_t n_seconds,
out() << "LRU max-count: " << max_count << "\n" << endl;
pStorage = m_factory.createStorage(CACHE_THREAD_MODEL_MT,
"unspecified",
0, // No TTL
max_count,
0, // No max size
0, NULL);
CacheStorageConfig config(CACHE_THREAD_MODEL_MT);
config.max_count = max_count;
pStorage = m_factory.createStorage("unspecified", config);
if (pStorage)
{
@ -200,12 +197,10 @@ int TesterLRUStorage::test_max_size(size_t n_threads, size_t n_seconds,
out() << "LRU max-size: " << max_size << "\n" << endl;
pStorage = m_factory.createStorage(CACHE_THREAD_MODEL_MT,
"unspecified",
0, // No TTL
0, // No max count
max_size,
0, NULL);
CacheStorageConfig config(CACHE_THREAD_MODEL_MT);
config.max_size = max_size;
pStorage = m_factory.createStorage("unspecified", config);
if (pStorage)
{
@ -241,12 +236,11 @@ int TesterLRUStorage::test_max_count_and_size(size_t n_threads, size_t n_seconds
out() << "LRU max-count: " << max_count << "\n" << endl;
out() << "LRU max-size : " << max_size << "\n" << endl;
pStorage = m_factory.createStorage(CACHE_THREAD_MODEL_MT,
"unspecified",
0, // No TTL
max_count,
max_size,
0, NULL);
CacheStorageConfig config(CACHE_THREAD_MODEL_MT);
config.max_count = max_count;
config.max_size = max_size;
pStorage = m_factory.createStorage("unspecified", config);
if (pStorage)
{

9
server/modules/filter/cache/test/testerrawstorage.cc vendored
View File

@ -25,12 +25,9 @@ int TesterRawStorage::execute(size_t n_threads, size_t n_seconds, const CacheIte
{
int rv = EXIT_FAILURE;
Storage* pStorage = m_factory.createRawStorage(CACHE_THREAD_MODEL_MT,
"unspecified",
0, // No TTL
0, // No max count
0, // No max size
0, NULL);
CacheStorageConfig config(CACHE_THREAD_MODEL_MT);
Storage* pStorage = m_factory.createRawStorage("unspecified", config);
if (pStorage)
{