Cache: Prepare for ST, MT, and PT caches

With a small cost it is possible to prepare for a single-thread,
multi-thread and cache-per-thread specific cases.

server/modules/filter/cache/CMakeLists.txt

@@ -1,5 +1,5 @@
 if (JANSSON_FOUND)
-  add_library(cache SHARED cache.cc cachefilter.cc rules.cc sessioncache.cc storage.cc storagefactory.cc)
+  add_library(cache SHARED cache.cc cachefilter.cc cachemt.cc rules.cc sessioncache.cc storage.cc storagefactory.cc)
   target_link_libraries(cache maxscale-common jansson)
   set_target_properties(cache PROPERTIES VERSION "1.0.0")
   set_target_properties(cache PROPERTIES LINK_FLAGS -Wl,-z,defs)

server/modules/filter/cache/cache.cc

@@ -152,6 +152,58 @@ Cache* Cache::Create(const char* zName, CACHE_CONFIG& config)
     return pCache;
 }
 
+//static
+bool Cache::Create(const CACHE_CONFIG& config,
+                   CACHE_RULES**       ppRules,
+                   StorageFactory**    ppFactory,
+                   HASHTABLE**         ppPending)
+{
+    CACHE_RULES* pRules = NULL;
+    HASHTABLE* pPending = NULL;
+    StorageFactory* pFactory = NULL;
+
+    if (config.rules)
+    {
+        pRules = cache_rules_load(config.rules, config.debug);
+    }
+    else
+    {
+        pRules = cache_rules_create(config.debug);
+    }
+
+    if (pRules)
+    {
+        pPending = hashtable_alloc(CACHE_PENDING_ITEMS, hashfn, hashcmp);
+
+        if (pPending)
+        {
+            pFactory = StorageFactory::Open(config.storage);
+
+            if (!pFactory)
+            {
+                MXS_ERROR("Could not open storage factory '%s'.", config.storage);
+            }
+        }
+    }
+
+    bool rv = (pRules && pPending && pFactory);
+
+    if (rv)
+    {
+        *ppRules = pRules;
+        *ppPending = pPending;
+        *ppFactory = pFactory;
+    }
+    else
+    {
+        cache_rules_free(pRules);
+        hashtable_free(pPending);
+        delete pFactory;
+    }
+
+    return rv;
+}
+
 bool Cache::shouldStore(const char* zDefaultDb, const GWBUF* pQuery)
 {
     return cache_rules_should_store(m_pRules, zDefaultDb, pQuery);
@@ -216,3 +268,32 @@ cache_result_t Cache::delValue(const char* pKey)
 {
     return m_pStorage->delValue(pKey);
 }
+
+// protected
+long Cache::hashOfKey(const char* pKey)
+{
+    return hash_of_key(pKey);
+}
+
+// protected
+bool Cache::mustRefresh(long key, const SessionCache* pSessionCache)
+{
+    void *pValue = hashtable_fetch(m_pPending, (void*)key);
+    if (!pValue)
+    {
+        // It's not being fetched, so we make a note that we are.
+        hashtable_add(m_pPending, (void*)key, (void*)pSessionCache);
+    }
+
+    return !pValue;
+}
+
+// protected
+void Cache::refreshed(long key, const SessionCache* pSessionCache)
+{
+    ss_dassert(hashtable_fetch(m_pPending, (void*)key) == pSessionCache);
+    ss_debug(int n =) hashtable_delete(m_pPending, (void*)key);
+    ss_dassert(n == 1);
+}
+
+

server/modules/filter/cache/cache.h

@@ -55,7 +55,7 @@ public:
      *
      * @return True, if the session cache should refresh the data.
      */
-    bool mustRefresh(const char* pKey, const SessionCache* pSessionCache);
+    virtual bool mustRefresh(const char* pKey, const SessionCache* pSessionCache);
 
     /**
      * To inform the cache that a particular item has been updated upon request.
@@ -63,7 +63,7 @@ public:
      * @param pKey           The hashed key for a query.
      * @param pSessionCache  The session cache informing.
      */
-    void refreshed(const char* pKey,  const SessionCache* pSessionCache);
+    virtual void refreshed(const char* pKey,  const SessionCache* pSessionCache);
 
     const CACHE_CONFIG& config() const { return m_config; }
 
@@ -75,7 +75,7 @@ public:
 
     cache_result_t delValue(const char* pKey);
 
-private:
+protected:
     Cache(const char* zName,
           CACHE_CONFIG& config,
           CACHE_RULES* pRules,
@@ -83,11 +83,22 @@ private:
           Storage* pStorage,
           HASHTABLE* pPending);
 
+    static bool Create(const CACHE_CONFIG& config,
+                       CACHE_RULES**       ppRules,
+                       StorageFactory**    ppFactory,
+                       HASHTABLE**         ppPending);
+
+    long hashOfKey(const char* pKey);
+
+    bool mustRefresh(long key, const SessionCache* pSessionCache);
+
+    void refreshed(long key, const SessionCache* pSessionCache);
+
 private:
     Cache(const Cache&);
     Cache& operator = (const Cache&);
 
-private:
+protected:
     const char*     m_zName;       // The name of the instance; the section name in the config.
     CACHE_CONFIG    m_config;      // The configuration of the cache instance.
     CACHE_RULES*    m_pRules;      // The rules of the cache instance.

server/modules/filter/cache/cachefilter.cc

@@ -17,7 +17,7 @@
 #include <maxscale/alloc.h>
 #include <maxscale/filter.h>
 #include <maxscale/gwdirs.h>
-#include "cache.h"
+#include "cachemt.h"
 #include "sessioncache.h"
 
 static char VERSION_STRING[] = "V1.0.0";
@@ -48,11 +48,6 @@ static uint64_t getCapabilities(void);
 
 static bool process_params(char **pzOptions, FILTER_PARAMETER **ppParams, CACHE_CONFIG& config);
 
-#define CPP_GUARD(statement)\
-    do { try { statement; }                                              \
-    catch (const std::exception& x) { MXS_ERROR("Caught standard exception: %s", x.what()); }\
-    catch (...) { MXS_ERROR("Caught unknown exception."); } } while (false)
-
 //
 // Global symbols of the Module
 //
@@ -124,7 +119,7 @@ static FILTER *createInstance(const char* zName, char** pzOptions, FILTER_PARAME
 
     if (process_params(pzOptions, ppParams, config))
     {
-        CPP_GUARD(pCache = Cache::Create(zName, config));
+        CPP_GUARD(pCache = CacheMT::Create(zName, config));
 
         if (!pCache)
         {

server/modules/filter/cache/cachefilter.h

@@ -61,4 +61,9 @@ void cache_config_finish(CACHE_CONFIG& config);
 void cache_config_free(CACHE_CONFIG* pConfig);
 void cache_config_reset(CACHE_CONFIG& config);
 
+#define CPP_GUARD(statement)\
+    do { try { statement; }                                              \
+    catch (const std::exception& x) { MXS_ERROR("Caught standard exception: %s", x.what()); }\
+    catch (...) { MXS_ERROR("Caught unknown exception."); } } while (false)
+
 #endif

server/modules/filter/cache/cachemt.cc

@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2016 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl.
+ *
+ * Change Date: 2019-07-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#include "cachemt.h"
+#include <new>
+#include <maxscale/spinlock.h>
+#include "storage.h"
+#include "storagefactory.h"
+
+CacheMT::CacheMT(const char* zName,
+                 CACHE_CONFIG& config,
+                 CACHE_RULES* pRules,
+                 StorageFactory* pFactory,
+                 Storage* pStorage,
+                 HASHTABLE* pPending)
+    : Cache(zName, config, pRules, pFactory, pStorage, pPending)
+{
+    spinlock_init(&m_lockPending);
+}
+
+CacheMT::~CacheMT()
+{
+}
+
+CacheMT* CacheMT::Create(const char* zName, CACHE_CONFIG& config)
+{
+    CacheMT* pCache = NULL;
+
+    CACHE_RULES* pRules = NULL;
+    HASHTABLE* pPending = NULL;
+    StorageFactory* pFactory = NULL;
+
+    if (Cache::Create(config, &pRules, &pFactory, &pPending))
+    {
+        uint32_t ttl = config.ttl;
+        int argc = config.storage_argc;
+        char** argv = config.storage_argv;
+
+        Storage* pStorage = pFactory->createStorage(zName, ttl, argc, argv);
+
+        if (pStorage)
+        {
+            CPP_GUARD(pCache = new CacheMT(zName,
+                                           config,
+                                           pRules,
+                                           pFactory,
+                                           pStorage,
+                                           pPending));
+
+            if (!pCache)
+            {
+                cache_rules_free(pRules);
+                hashtable_free(pPending);
+                delete pStorage;
+                delete pFactory;
+            }
+        }
+    }
+
+    return pCache;
+}
+
+bool CacheMT::mustRefresh(const char* pKey, const SessionCache* pSessionCache)
+{
+    long key = hashOfKey(pKey);
+
+    spinlock_acquire(&m_lockPending);
+    bool rv = Cache::mustRefresh(key, pSessionCache);
+    spinlock_release(&m_lockPending);
+
+    return rv;
+}
+
+void CacheMT::refreshed(const char* pKey,  const SessionCache* pSessionCache)
+{
+    long key = hashOfKey(pKey);
+
+    spinlock_acquire(&m_lockPending);
+    Cache::refreshed(key, pSessionCache);
+    spinlock_release(&m_lockPending);
+}

server/modules/filter/cache/cachemt.h

@@ -0,0 +1,43 @@
+#pragma once
+/*
+ * Copyright (c) 2016 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl.
+ *
+ * Change Date: 2019-07-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#include <maxscale/cdefs.h>
+#include "cache.h"
+
+class CacheMT : public Cache
+{
+public:
+    ~CacheMT();
+
+    static CacheMT* Create(const char* zName, CACHE_CONFIG& config);
+
+    bool mustRefresh(const char* pKey, const SessionCache* pSessionCache);
+
+    void refreshed(const char* pKey,  const SessionCache* pSessionCache);
+
+private:
+    CacheMT(const char* zName,
+            CACHE_CONFIG& config,
+            CACHE_RULES* pRules,
+            StorageFactory* pFactory,
+            Storage* pStorage,
+            HASHTABLE* pPending);
+
+private:
+    CacheMT(const CacheMT&);
+    CacheMT& operator = (const CacheMT&);
+
+private:
+    SPINLOCK m_lockPending; // Lock used for protecting 'pending'.
+};

server/modules/filter/cache/storage.cc

@@ -23,6 +23,10 @@ Storage::Storage(CACHE_STORAGE_API* pApi, CACHE_STORAGE* pStorage)
     ss_dassert(m_pStorage);
 }
 
+Storage::~Storage()
+{
+}
+
 cache_result_t Storage::getKey(const char* zDefaultDb,
                                const GWBUF* pQuery,
                                char* pKey)