Cache: Add information facilities

Various components of the cache can now provide information in
the form of a JSON object that then can be further processed.
This commit is contained in:
Johan Wikman 2016-12-07 13:04:01 +02:00
parent 8f2170c455
commit e0bedad46e
33 changed files with 346 additions and 18 deletions

View File

@ -76,9 +76,30 @@ bool Cache::Create(const CACHE_CONFIG& config,
void Cache::show(DCB* pDcb) const
{
dcb_printf(pDcb, "Rules:\n");
size_t indent = 2;
m_sRules->print(pDcb, indent);
bool showed = false;
json_t* pInfo = get_info(INFO_ALL);
if (pInfo)
{
size_t flags = JSON_PRESERVE_ORDER;
size_t indent = 2;
char* z = json_dumps(pInfo, JSON_PRESERVE_ORDER | JSON_INDENT(indent));
if (z)
{
dcb_printf(pDcb, "%s\n", z);
free(z);
showed = true;
}
json_decref(pInfo);
}
if (!showed)
{
// So as not to upset anyone expecting a JSON object.
dcb_printf(pDcb, "{\n}\n");
}
}
bool Cache::should_store(const char* zDefaultDb, const GWBUF* pQuery)
@ -90,3 +111,20 @@ bool Cache::should_use(const SESSION* pSession)
{
return m_sRules->should_use(pSession);
}
json_t* Cache::do_get_info(uint32_t what) const
{
json_t* pInfo = json_object();
if (pInfo)
{
if (what & INFO_RULES)
{
json_t* pRules = const_cast<json_t*>(m_sRules->json());
json_object_set(pInfo, "rules", pRules); // Increases ref-count of pRules, we ignore failure.
}
}
return pInfo;
}

View File

@ -26,6 +26,14 @@ class SessionCache;
class Cache
{
public:
enum what_info_t
{
INFO_RULES = 0x01, /*< Include information about the rules. */
INFO_PENDING = 0x02, /*< Include information about any pending items. */
INFO_STORAGE = 0x04, /*< Include information about the storage. */
INFO_ALL = (INFO_RULES | INFO_PENDING | INFO_STORAGE)
};
typedef std::tr1::shared_ptr<CacheRules> SCacheRules;
typedef std::tr1::shared_ptr<StorageFactory> SStorageFactory;
@ -35,6 +43,8 @@ public:
const CACHE_CONFIG& config() const { return m_config; }
virtual json_t* get_info(uint32_t what = INFO_ALL) const = 0;
/**
* Returns whether the results of a particular query should be stored.
*
@ -90,6 +100,8 @@ protected:
CacheRules** ppRules,
StorageFactory** ppFactory);
json_t* do_get_info(uint32_t what) const;
private:
Cache(const Cache&);
Cache& operator = (const Cache&);

View File

@ -16,6 +16,7 @@
#include <stdbool.h>
#include <stdint.h>
#include <jansson.h>
#include <maxscale/buffer.h>
#include <maxscale/protocol/mysql.h>
#include <maxscale/debug.h>
@ -37,6 +38,12 @@ typedef enum cache_flags
CACHE_FLAGS_INCLUDE_STALE = 0x01,
} cache_flags_t;
typedef enum cache_storage_info
{
// TODO: Provide more granularity.
CACHE_STORAGE_INFO_ALL = 0
} cache_storage_info_t;
typedef enum cache_thread_model
{
CACHE_THREAD_MODEL_ST,
@ -120,6 +127,20 @@ typedef struct cache_storage_api
*/
void (*freeInstance)(CACHE_STORAGE* instance);
/**
* Returns information about the storage.
*
* @param storage Pointer to a CACHE_STORAGE.
* @param what Bitmask of cache_storage_info_t values.
* @param info Upon successful return points to json_t object containing
* information. The caller should call @c json_decref on the
* object when it is no longer needed.
*
* @return CACHE_RESULT_OK if a json object could be created.
*/
cache_result_t (*getInfo)(CACHE_STORAGE* storage,
uint32_t what,
json_t** info);
/**
* Create a key for a GWBUF.
*

View File

@ -54,6 +54,13 @@ CacheMT* CacheMT::Create(const std::string& name, const CACHE_CONFIG* pConfig)
return pCache;
}
json_t* CacheMT::get_info(uint32_t flags) const
{
LockGuard guard(&m_lockPending);
return CacheSimple::do_get_info(flags);
}
bool CacheMT::must_refresh(const CACHE_KEY& key, const SessionCache* pSessionCache)
{
LockGuard guard(&m_lockPending);

View File

@ -23,6 +23,8 @@ public:
static CacheMT* Create(const std::string& name, const CACHE_CONFIG* pConfig);
json_t* get_info(uint32_t what) const;
bool must_refresh(const CACHE_KEY& key, const SessionCache* pSessionCache);
void refreshed(const CACHE_KEY& key, const SessionCache* pSessionCache);
@ -44,5 +46,5 @@ private:
CacheMT& operator = (const CacheMT&);
private:
SPINLOCK m_lockPending; // Lock used for protecting 'pending'.
mutable SPINLOCK m_lockPending; // Lock used for protecting 'pending'.
};

View File

@ -91,6 +91,37 @@ void CachePT::refreshed(const CACHE_KEY& key, const SessionCache* pSessionCache
thread_cache().refreshed(key, pSessionCache);
}
json_t* CachePT::get_info(uint32_t what) const
{
json_t* pInfo = Cache::do_get_info(what);
if (pInfo)
{
if (what & (INFO_PENDING | INFO_STORAGE))
{
what &= ~INFO_RULES; // The rules are the same, we don't want them duplicated.
for (size_t i = 0; i < m_caches.size(); ++i)
{
char key[20]; // Surely enough.
sprintf(key, "thread-%u", (unsigned int)i + 1);
SCache sCache = m_caches[i];
json_t* pThreadInfo = sCache->get_info(what);
if (pThreadInfo)
{
json_object_set(pInfo, key, pThreadInfo);
json_decref(pThreadInfo);
}
}
}
}
return pInfo;
}
cache_result_t CachePT::get_key(const char* zDefaultDb, const GWBUF* pQuery, CACHE_KEY* pKey)
{
return thread_cache().get_key(zDefaultDb, pQuery, pKey);

View File

@ -28,6 +28,8 @@ public:
void refreshed(const CACHE_KEY& key, const SessionCache* pSessionCache);
json_t* get_info(uint32_t what) const;
cache_result_t get_key(const char* zDefaultDb, const GWBUF* pQuery, CACHE_KEY* pKey);
cache_result_t get_value(const CACHE_KEY& key, uint32_t flags, GWBUF** ppValue);

View File

@ -75,6 +75,30 @@ cache_result_t CacheSimple::del_value(const CACHE_KEY& key)
return m_pStorage->del_value(key);
}
// protected:
json_t* CacheSimple::do_get_info(uint32_t what) const
{
json_t* pInfo = Cache::do_get_info(what);
if (what & INFO_PENDING)
{
// TODO: Include information about pending items.
}
if (what & INFO_STORAGE)
{
json_t* pStorageInfo;
if (m_pStorage->get_info(Storage::INFO_ALL, &pStorageInfo) == CACHE_RESULT_OK)
{
json_object_set(pInfo, "storage", pStorageInfo);
json_decref(pStorageInfo);
}
}
return pInfo;
}
// protected
bool CacheSimple::do_must_refresh(const CACHE_KEY& key, const SessionCache* pSessionCache)
{

View File

@ -44,6 +44,8 @@ protected:
StorageFactory** ppFactory);
json_t* do_get_info(uint32_t what) const;
bool do_must_refresh(const CACHE_KEY& key, const SessionCache* pSessionCache);
void do_refreshed(const CACHE_KEY& key, const SessionCache* pSessionCache);

View File

@ -65,6 +65,11 @@ CacheST* CacheST::Create(const std::string& name,
return Create(name, pConfig, sRules, sFactory);
}
json_t* CacheST::get_info(uint32_t flags) const
{
return CacheSimple::do_get_info(flags);
}
bool CacheST::must_refresh(const CACHE_KEY& key, const SessionCache* pSessionCache)
{
return CacheSimple::do_must_refresh(key, pSessionCache);

View File

@ -26,6 +26,8 @@ public:
SStorageFactory sFactory,
const CACHE_CONFIG* pConfig);
json_t* get_info(uint32_t what) const;
bool must_refresh(const CACHE_KEY& key, const SessionCache* pSessionCache);
void refreshed(const CACHE_KEY& key, const SessionCache* pSessionCache);

View File

@ -36,6 +36,47 @@ cache_result_t LRUStorage::get_key(const char* zdefault_db,
return pstorage_->get_key(zdefault_db, pquery, pkey);
}
static void set_integer(json_t* pobject, const char* zname, size_t value)
{
json_t* pvalue = json_integer(value);
if (pvalue)
{
json_object_set(pobject, zname, pvalue);
json_decref(pvalue);
}
}
cache_result_t LRUStorage::do_get_info(uint32_t what,
json_t** ppinfo) const
{
*ppinfo = json_object();
if (*ppinfo)
{
json_t* plru = json_object();
if (plru)
{
set_integer(plru, "size", size_);
set_integer(plru, "count", count_);
json_object_set(*ppinfo, "lru", plru);
json_decref(plru);
}
json_t* pstorage_info;
if (pstorage_->get_info(what, &pstorage_info) == CACHE_RESULT_OK)
{
json_object_set(*ppinfo, "real_storage", pstorage_info);
json_decref(pstorage_info);
}
}
return *ppinfo ? CACHE_RESULT_OK : CACHE_RESULT_OUT_OF_RESOURCES;
}
cache_result_t LRUStorage::do_get_value(const CACHE_KEY& key,
uint32_t flags,
GWBUF** ppvalue)

View File

@ -32,6 +32,14 @@ public:
protected:
LRUStorage(Storage* pstorage, size_t max_count, size_t max_size);
/**
* Returns information about the LRU storage and the underlying real
* storage.
*
* @see Storage::get_info
*/
cache_result_t do_get_info(uint32_t what, json_t** ppInfo) const;
/**
* Fetches the value from the underlying storage and, if found, moves the
* entry to the top of the LRU list.

View File

@ -35,6 +35,14 @@ LRUStorageMT* LRUStorageMT::create(Storage* pstorage, size_t max_count, size_t m
return plru_storage;
}
cache_result_t LRUStorageMT::get_info(uint32_t what,
json_t** ppInfo) const
{
LockGuard guard(&lock_);
return LRUStorage::do_get_info(what, ppInfo);
}
cache_result_t LRUStorageMT::get_value(const CACHE_KEY& key,
uint32_t flags,
GWBUF** ppvalue)

View File

@ -23,6 +23,9 @@ public:
static LRUStorageMT* create(Storage* pstorage, size_t max_count, size_t max_size);
cache_result_t get_info(uint32_t what,
json_t** ppInfo) const;
cache_result_t get_value(const CACHE_KEY& key,
uint32_t flags,
GWBUF** ppvalue);
@ -39,5 +42,5 @@ private:
LRUStorageMT& operator = (const LRUStorageMT&);
private:
SPINLOCK lock_;
mutable SPINLOCK lock_;
};

View File

@ -33,6 +33,12 @@ LRUStorageST* LRUStorageST::create(Storage* pstorage, size_t max_count, size_t m
return plru_storage;
}
cache_result_t LRUStorageST::get_info(uint32_t what,
json_t** ppInfo) const
{
return LRUStorage::do_get_info(what, ppInfo);
}
cache_result_t LRUStorageST::get_value(const CACHE_KEY& key,
uint32_t flags,
GWBUF** ppvalue)

View File

@ -22,6 +22,9 @@ public:
static LRUStorageST* create(Storage* pstorage, size_t max_count, size_t max_size);
cache_result_t get_info(uint32_t what,
json_t** ppInfo) const;
cache_result_t get_value(const CACHE_KEY& key,
uint32_t flags,
GWBUF** ppValue);

View File

@ -413,9 +413,9 @@ CacheRules* CacheRules::load(const char *zpath, uint32_t debug)
return pthis;
}
void CacheRules::print(DCB* pdcb, size_t indent) const
const json_t* CacheRules::json() const
{
cache_rules_print(prules_, pdcb, indent);
return prules_->root;
}
bool CacheRules::should_store(const char* zdefault_db, const GWBUF* pquery) const

View File

@ -183,12 +183,14 @@ public:
static CacheRules* load(const char *zpath, uint32_t debug);
/**
* Prints the rules.
* Returns the json rules object.
*
* @param pdcb DCB where the rules should be printed.
* @param indent By how many spaces to indent the output.
* NOTE: The object remains valid only as long as the CacheRules
* object is valid.
*
* @return The rules object.
*/
void print(DCB* pdcb, size_t indent) const;
const json_t* json() const;
/**
* Returns boolean indicating whether the result of the query should be stored.

View File

@ -18,8 +18,16 @@
class Storage
{
public:
enum what_info_t
{
INFO_ALL = CACHE_STORAGE_INFO_ALL
};
virtual ~Storage();
virtual cache_result_t get_info(uint32_t what,
json_t** ppInfo) const = 0;
virtual cache_result_t get_key(const char* zDefaultDb,
const GWBUF* pQuery,
CACHE_KEY* pKey) = 0;

View File

@ -107,6 +107,15 @@ cache_result_t InMemoryStorage::get_key(const char* zdefault_db, const GWBUF* pq
return CACHE_RESULT_OK;
}
cache_result_t InMemoryStorage::do_get_info(uint32_t what, json_t** ppinfo) const
{
*ppinfo = json_object();
// TODO: Fill with unordered_map statistics.
return *ppinfo ? CACHE_RESULT_OK : CACHE_RESULT_OUT_OF_RESOURCES;
}
cache_result_t InMemoryStorage::do_get_value(const CACHE_KEY& key, uint32_t flags, GWBUF** ppresult)
{
cache_result_t result = CACHE_RESULT_NOT_FOUND;

View File

@ -26,6 +26,7 @@ public:
cache_result_t get_key(const char* zdefault_db, const GWBUF* pquery, CACHE_KEY* pkey);
virtual cache_result_t get_info(uint32_t what, json_t** ppInfo) const = 0;
virtual cache_result_t get_value(const CACHE_KEY& key, uint32_t flags, GWBUF** ppresult) = 0;
virtual cache_result_t put_value(const CACHE_KEY& key, const GWBUF* pvalue) = 0;
virtual cache_result_t del_value(const CACHE_KEY& key) = 0;
@ -33,6 +34,7 @@ public:
protected:
InMemoryStorage(const std::string& name, uint32_t ttl);
cache_result_t do_get_info(uint32_t what, json_t** ppInfo) const;
cache_result_t do_get_value(const CACHE_KEY& key, uint32_t flags, GWBUF** ppresult);
cache_result_t do_put_value(const CACHE_KEY& key, const GWBUF* pvalue);
cache_result_t do_del_value(const CACHE_KEY& key);

View File

@ -32,6 +32,13 @@ InMemoryStorageMT* InMemoryStorageMT::create(const std::string& name,
return new InMemoryStorageMT(name, ttl);
}
cache_result_t InMemoryStorageMT::get_info(uint32_t what, json_t** ppInfo) const
{
LockGuard guard(&lock_);
return do_get_info(what, ppInfo);
}
cache_result_t InMemoryStorageMT::get_value(const CACHE_KEY& key, uint32_t flags, GWBUF** ppresult)
{
LockGuard guard(&lock_);

View File

@ -23,6 +23,7 @@ public:
static InMemoryStorageMT* create(const std::string& name, uint32_t ttl, int argc, char* argv[]);
cache_result_t get_info(uint32_t what, json_t** ppInfo) const;
cache_result_t get_value(const CACHE_KEY& key, uint32_t flags, GWBUF** ppresult);
cache_result_t put_value(const CACHE_KEY& key, const GWBUF* pvalue);
cache_result_t del_value(const CACHE_KEY& key);
@ -35,5 +36,5 @@ private:
InMemoryStorageMT& operator = (const InMemoryStorageMT&);
private:
SPINLOCK lock_;
mutable SPINLOCK lock_;
};

View File

@ -31,6 +31,11 @@ InMemoryStorageST* InMemoryStorageST::create(const std::string& name,
return new InMemoryStorageST(name, ttl);
}
cache_result_t InMemoryStorageST::get_info(uint32_t what, json_t** ppinfo) const
{
return do_get_info(what, ppinfo);
}
cache_result_t InMemoryStorageST::get_value(const CACHE_KEY& key, uint32_t flags, GWBUF** ppresult)
{
return do_get_value(key, flags, ppresult);

View File

@ -22,6 +22,7 @@ public:
static InMemoryStorageST* create(const std::string& name, uint32_t ttl, int argc, char* argv[]);
cache_result_t get_info(uint32_t what, json_t** ppinfo) const;
cache_result_t get_value(const CACHE_KEY& key, uint32_t flags, GWBUF** ppresult);
cache_result_t put_value(const CACHE_KEY& key, const GWBUF* pvalue);
cache_result_t del_value(const CACHE_KEY& key);

View File

@ -88,6 +88,35 @@ void freeInstance(CACHE_STORAGE* pinstance)
delete reinterpret_cast<InMemoryStorage*>(pinstance);
}
cache_result_t getInfo(CACHE_STORAGE* pStorage,
uint32_t what,
json_t** ppInfo)
{
ss_dassert(pStorage);
cache_result_t result = CACHE_RESULT_ERROR;
try
{
result = reinterpret_cast<InMemoryStorage*>(pStorage)->get_info(what, ppInfo);
}
catch (const std::bad_alloc&)
{
MXS_OOM();
}
catch (const std::exception& x)
{
MXS_ERROR("Standard exception caught: %s", x.what());
}
catch (...)
{
MXS_ERROR("Unknown exception caught.");
}
return result;
}
cache_result_t getKey(CACHE_STORAGE* pstorage,
const char* zdefault_db,
const GWBUF* pquery,
@ -221,6 +250,7 @@ CACHE_STORAGE_API* CacheGetStorageAPI()
initialize,
createInstance,
freeInstance,
getInfo,
getKey,
getValue,
putValue,

View File

@ -17,7 +17,7 @@ if (ROCKSDB_BUILT)
storage_rocksdb.cc
)
add_dependencies(storage_rocksdb RocksDB)
target_link_libraries(storage_rocksdb maxscale-common ${ROCKSDB_LIB} ${ROCKSDB_LINK_LIBS})
target_link_libraries(storage_rocksdb maxscale-common jansson ${ROCKSDB_LIB} ${ROCKSDB_LINK_LIBS})
set_target_properties(storage_rocksdb PROPERTIES VERSION "1.0.0")
set_target_properties(storage_rocksdb PROPERTIES LINK_FLAGS -Wl,-z,defs)
install_module(storage_rocksdb experimental)

View File

@ -346,6 +346,15 @@ RocksDBStorage* RocksDBStorage::Create(const string& storageDirectory, const cha
return pStorage;
}
cache_result_t RocksDBStorage::getInfo(uint32_t what, json_t** ppInfo) const
{
*ppInfo = json_object();
// TODO: Fill with RocksDB statistics.
return *ppInfo ? CACHE_RESULT_OK : CACHE_RESULT_OUT_OF_RESOURCES;
}
cache_result_t RocksDBStorage::getKey(const char* zDefaultDB, const GWBUF* pQuery, CACHE_KEY* pKey)
{
ss_dassert(GWBUF_IS_CONTIGUOUS(pQuery));

View File

@ -29,6 +29,7 @@ public:
static RocksDBStorage* Create(const char* zName, uint32_t ttl, int argc, char* argv[]);
~RocksDBStorage();
cache_result_t getInfo(uint32_t flags, json_t** ppInfo) const;
cache_result_t getKey(const char* zDefaultDB, const GWBUF* pQuery, CACHE_KEY* pKey);
cache_result_t getValue(const CACHE_KEY* pKey, uint32_t flags, GWBUF** ppResult);
cache_result_t putValue(const CACHE_KEY* pKey, const GWBUF* pValue);

View File

@ -77,6 +77,34 @@ void freeInstance(CACHE_STORAGE* pInstance)
delete reinterpret_cast<RocksDBStorage*>(pInstance);
}
cache_result_t getInfo(CACHE_STORAGE* pStorage,
uint32_t what,
json_t** ppInfo)
{
ss_dassert(pStorage);
cache_result_t result = CACHE_RESULT_ERROR;
try
{
result = reinterpret_cast<RocksDBStorage*>(pStorage)->getInfo(what, ppInfo);
}
catch (const std::bad_alloc&)
{
MXS_OOM();
}
catch (const std::exception& x)
{
MXS_ERROR("Standard exception caught: %s", x.what());
}
catch (...)
{
MXS_ERROR("Unknown exception caught.");
}
return result;
}
cache_result_t getKey(CACHE_STORAGE* pStorage,
const char* zDefaultDB,
const GWBUF* pQuery,
@ -210,6 +238,7 @@ CACHE_STORAGE_API* CacheGetStorageAPI()
initialize,
createInstance,
freeInstance,
getInfo,
getKey,
getValue,
putValue,

View File

@ -27,22 +27,28 @@ StorageReal::~StorageReal()
{
}
cache_result_t StorageReal::get_info(uint32_t flags,
json_t** ppInfo) const
{
return m_pApi->getInfo(m_pStorage, flags, ppInfo);
}
cache_result_t StorageReal::get_key(const char* zDefaultDb,
const GWBUF* pQuery,
CACHE_KEY* pKey)
const GWBUF* pQuery,
CACHE_KEY* pKey)
{
return m_pApi->getKey(m_pStorage, zDefaultDb, pQuery, pKey);
}
cache_result_t StorageReal::get_value(const CACHE_KEY& key,
uint32_t flags,
GWBUF** ppValue)
uint32_t flags,
GWBUF** ppValue)
{
return m_pApi->getValue(m_pStorage, &key, flags, ppValue);
}
cache_result_t StorageReal::put_value(const CACHE_KEY& key,
const GWBUF* pValue)
const GWBUF* pValue)
{
return m_pApi->putValue(m_pStorage, &key, pValue);
}

View File

@ -20,6 +20,9 @@ class StorageReal : public Storage
public:
~StorageReal();
cache_result_t get_info(uint32_t flags,
json_t** ppInfo) const;
cache_result_t get_key(const char* zDefaultDb,
const GWBUF* pQuery,
CACHE_KEY* pKey);