diff --git a/server/core/internal/server.hh b/server/core/internal/server.hh
index c0f98f3a8..9ab603af4 100644
--- a/server/core/internal/server.hh
+++ b/server/core/internal/server.hh
@@ -19,6 +19,7 @@
 #include <maxbase/average.hh>
 #include <maxscale/server.h>
 #include <maxscale/resultset.hh>
+#include <maxscale/routingworker.hh>
 
 std::unique_ptr<ResultSet> serverGetList();
 
@@ -27,19 +28,24 @@ class Server : public SERVER
 {
 public:
 
+    Server()
+        : response_time(maxbase::EMAverage {0.04, 0.35, 500})
+    {
+    }
+
     int response_time_num_samples() const
     {
-        return response_time.num_samples();
+        return response_time->num_samples();
     }
 
     double response_time_average() const
     {
-        return response_time.average();
+        return response_time->average();
     }
 
     void response_time_add(double ave, int num_samples)
     {
-        response_time.add(ave, num_samples);
+        response_time->add(ave, num_samples);
     }
 
 private:
@@ -47,7 +53,7 @@ private:
     // can be calculated at runtime. The "500" or sample_max affects how often a
     // session should updates this stat. sample_max should be slightly lower than max sample
     // rate (which is less than qps due to the noise filter).
-    maxbase::EMAverage response_time = {0.04, 0.35, 500};
+    mxs::rworker_local<maxbase::EMAverage> response_time;
 };
 
 void server_free(Server* server);
diff --git a/server/core/server.cc b/server/core/server.cc
index b8fe4fa4f..2e62c166f 100644
--- a/server/core/server.cc
+++ b/server/core/server.cc
@@ -1535,16 +1535,8 @@ bool server_set_disk_space_threshold(SERVER* server, const char* disk_space_thre
     return rv;
 }
 
-namespace
-{
-// Only need to prevent multiple writes, as long as only the average is read where
-// needed (and not in combination with num_samples), which is by design.
-std::mutex add_response_mutex;
-}
-
 void server_add_response_average(SERVER* srv, double ave, int num_samples)
 {
-    std::lock_guard<std::mutex> lock(add_response_mutex);
     Server* server = static_cast<Server*>(srv);
     server->response_time_add(ave, num_samples);
 }