From 6830f517b4fa1477de0d5afb63c09639487a19c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= <markus.makela@mariadb.com>
Date: Sat, 21 Sep 2019 09:43:12 +0300
Subject: [PATCH 1/2] Improve message queue logging

Logging the pipe buffer size on startup will tell how large it was at the
time when MaxScale read it. If there are some abnormalities in it, this
will make it visible.

Logging the worker ID when the posting of a message fails will tell which
particular worker it was. For example, if the worker in question is the
main worker (i.e. ID 0), we know there's something that's blocking the
processing.
---
 include/maxscale/routingworker.hh          |  2 +-
 maxutils/maxbase/include/maxbase/worker.hh | 10 ++++++++++
 maxutils/maxbase/src/messagequeue.cc       | 22 ++++++++++++++++++----
 3 files changed, 29 insertions(+), 5 deletions(-)

diff --git a/include/maxscale/routingworker.hh b/include/maxscale/routingworker.hh
index 8c7efc074..74100c833 100644
--- a/include/maxscale/routingworker.hh
+++ b/include/maxscale/routingworker.hh
@@ -103,7 +103,7 @@ public:
      *
      * @return The id of the routing worker.
      */
-    int id() const
+    int id() const override
     {
         return m_id;
     }
diff --git a/maxutils/maxbase/include/maxbase/worker.hh b/maxutils/maxbase/include/maxbase/worker.hh
index 88525ea7e..0236e6948 100644
--- a/maxutils/maxbase/include/maxbase/worker.hh
+++ b/maxutils/maxbase/include/maxbase/worker.hh
@@ -334,6 +334,16 @@ public:
 
     virtual ~Worker();
 
+    /**
+     * Returns the id of the worker
+     *
+     * @return The address of the worker cast to an int
+     */
+    virtual int id() const
+    {
+        return (intptr_t)this;
+    }
+
     int load(Load::counter_t counter)
     {
         return m_load.percentage(counter);
diff --git a/maxutils/maxbase/src/messagequeue.cc b/maxutils/maxbase/src/messagequeue.cc
index 0b57fc20e..46b93f809 100644
--- a/maxutils/maxbase/src/messagequeue.cc
+++ b/maxutils/maxbase/src/messagequeue.cc
@@ -17,10 +17,12 @@
 #include <string.h>
 #include <unistd.h>
 #include <fstream>
+#include <mutex>
 #include <maxbase/assert.h>
 #include <maxbase/log.h>
 #include <maxbase/string.h>
 #include <maxbase/worker.hh>
+#include <maxbase/format.hh>
 
 namespace
 {
@@ -147,9 +149,20 @@ MessageQueue* MessageQueue::create(Handler* pHandler)
         if (fcntl(fds[0], F_SETPIPE_SZ, this_unit.pipe_max_size) == -1)
         {
             MXB_WARNING("Failed to increase pipe buffer size to '%d': %d, %s",
-                        this_unit.pipe_max_size,
-                        errno,
-                        mxb_strerror(errno));
+                        this_unit.pipe_max_size, errno, mxb_strerror(errno));
+        }
+        else
+        {
+            static int current_pipe_max_size = 0;
+            static std::mutex pipe_size_lock;
+            std::lock_guard<std::mutex> guard(pipe_size_lock);
+
+            if (current_pipe_max_size == 0)
+            {
+                current_pipe_max_size = this_unit.pipe_max_size;
+                MXB_NOTICE("Worker message queue size: %s",
+                           mxb::to_binary_size(this_unit.pipe_max_size).c_str());
+            }
         }
 #endif
         pThis = new(std::nothrow) MessageQueue(pHandler, read_fd, write_fd);
@@ -219,7 +232,8 @@ bool MessageQueue::post(const Message& message) const
 
         if (n == -1)
         {
-            MXB_ERROR("Failed to write message: %d, %s", errno, mxb_strerror(errno));
+            MXB_ERROR("Failed to write message to worker %d: %d, %s",
+                      m_pWorker->id(), errno, mxb_strerror(errno));
 
             static bool warn_pipe_buffer_size = true;
 

From 8ac11a97c29ac9ce3e4099e77a9aa9475c1002b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= <markus.makela@mariadb.com>
Date: Sat, 21 Sep 2019 10:20:39 +0300
Subject: [PATCH 2/2] Extend pipe buffer size error message

The message now logs the instructions on how to increase the per-process
page limit for pipe buffers. This can happen if fs.pipe-max-size
multiplied by the number of workers exceeds the value of
fs.pipe-user-pages-soft multiplied by 4096.
---
 maxutils/maxbase/src/messagequeue.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/maxutils/maxbase/src/messagequeue.cc b/maxutils/maxbase/src/messagequeue.cc
index 46b93f809..bb39ed010 100644
--- a/maxutils/maxbase/src/messagequeue.cc
+++ b/maxutils/maxbase/src/messagequeue.cc
@@ -148,7 +148,9 @@ MessageQueue* MessageQueue::create(Handler* pHandler)
          */
         if (fcntl(fds[0], F_SETPIPE_SZ, this_unit.pipe_max_size) == -1)
         {
-            MXB_WARNING("Failed to increase pipe buffer size to '%d': %d, %s",
+            MXB_WARNING("Failed to increase pipe buffer size to '%d': %d, %s. "
+                        "Increase pipe-user-pages-soft (sysctl fs.pipe-user-pages-soft) "
+                        "or reduce pipe-max-size (sysctl fs.pipe-max-size).",
                         this_unit.pipe_max_size, errno, mxb_strerror(errno));
         }
         else