MXS-1775 Add functions for checking disk space of servers

Documentation/Getting-Started/Configuration-Guide.md

@@ -1284,7 +1284,8 @@ how full the corresponding disk may be, before action is taken. E.g. an entry li
 specifies that the disk that has been mounted on `/data` may be used until 80%
 of the total space has been consumed. Multiple entries can be specified by
 separating them by a comma. If the path is specified using `*`, then the limit
-applies to all disks.
+applies to all disks. However, the value of `*` is only applied if there is not
+an exact match.
 
 Note that if a particular disk has been mounted on several paths, only one path
 need to be specified. If several are specified, then the one with the smallest
@@ -1294,8 +1295,12 @@ Examples:
 ```
 disk_space_threshold=*:80
 disk_space_threshold=/data:80
-disk_space_threshold=/data1:80,/data2:70,/data3:30
+disk_space_threshold=/data1:80,/data2:60,*:90
 ```
+The last line means that the disk mounted at `/data1` may be used up to
+80%, the disk mounted at `/data2` may be used up to 60% and all other disks
+mounted at any paths may be used up until 90% of maximum capacity, before
+MaxScale starts to warn to take action.
 
 Note that the path to be used, is one of the paths returned by:
 ```

include/maxscale/monitor.h

@@ -230,6 +230,7 @@ typedef struct monitored_server
     uint64_t mon_prev_status;       /**< Status before starting the current monitor loop */
     uint64_t pending_status;        /**< Status during current monitor loop */
     bool new_event;                 /**< Whether an action was taken on the last event */
+    int64_t disk_space_checked;     /**< When was the disk space checked the last time */
     struct monitored_server *next;  /**< The next server in the list */
 } MXS_MONITORED_SERVER;
 

include/maxscale/monitor.hh

@@ -113,6 +113,12 @@ protected:
         return atomic_load_int32(&m_shutdown) != 0;
     }
 
+    bool should_check_disk_space(const MXS_MONITORED_SERVER* pMonitored_server) const;
+
+    void check_disk_space(MXS_MONITORED_SERVER* pMonitored_server);
+
+    static int64_t get_time_ms();
+
     /**
      * @brief Configure the monitor.
      *

server/core/monitor.cc

@@ -31,6 +31,7 @@
 #include <maxscale/clock.h>
 #include <maxscale/json_api.h>
 #include <maxscale/log_manager.h>
+#include <maxscale/mariadb.hh>
 #include <maxscale/mysql_utils.h>
 #include <maxscale/paths.h>
 #include <maxscale/pcre2.h>
@@ -2652,6 +2653,159 @@ bool MonitorInstance::start(const MXS_CONFIG_PARAMETER* pParams)
     return started;
 }
 
+//static
+int64_t MonitorInstance::get_time_ms()
+{
+    timespec t;
+
+    ss_debug(int rv = )clock_gettime(CLOCK_MONOTONIC_COARSE, &t);
+    ss_dassert(rv == 0);
+
+    return t.tv_sec * 1000 + (t.tv_nsec / 1000000);
+}
+
+bool MonitorInstance::should_check_disk_space(const MXS_MONITORED_SERVER* pMs) const
+{
+    bool should_check = false;
+
+    if (m_monitor->disk_space_check_interval &&
+        (m_monitor->disk_space_threshold || pMs->server->disk_space_threshold) &&
+        (pMs->disk_space_checked != -1)) // -1 means disabled
+    {
+        int64_t now = get_time_ms();
+
+        if (now - pMs->disk_space_checked > m_monitor->disk_space_check_interval)
+        {
+            should_check = true;
+        }
+    }
+
+    return should_check;
+}
+
+namespace
+{
+
+bool check_disk_space_exhausted(MXS_MONITORED_SERVER*               pMs,
+                                const std::string&                  path,
+                                const maxscale::disk::SizesAndName& san,
+                                int32_t                             max_percentage)
+{
+    bool disk_space_exhausted = false;
+
+    int32_t used_percentage = ((san.total() - san.available()) / (double)san.total()) * 100;
+
+    if (used_percentage >= max_percentage)
+    {
+        MXS_ERROR("Disk space on %s at %s is exhausted; %d%% of the the disk "
+                  "mounted on the path %s has been used, and the limit it %d%%.",
+                  pMs->server->name, pMs->server->address,
+                  used_percentage, path.c_str(), max_percentage);
+        disk_space_exhausted = true;
+    }
+
+    return disk_space_exhausted;
+}
+
+}
+
+void MonitorInstance::check_disk_space(MXS_MONITORED_SERVER* pMs)
+{
+    std::map<std::string, disk::SizesAndName> info;
+
+    int rv = disk::get_info_by_path(pMs->con, &info);
+
+    if (rv == 0)
+    {
+        bool disk_space_exhausted = false;
+
+        MxsDiskSpaceThreshold* pDst =
+            pMs->server->disk_space_threshold ?
+            pMs->server->disk_space_threshold : m_monitor->disk_space_threshold;
+        ss_dassert(pDst);
+
+        int32_t star_max_percentage = -1;
+
+        std::set<std::string> checked_paths;
+
+        for (auto i = pDst->begin(); i != pDst->end(); ++i)
+        {
+            string path = i->first;
+            int32_t max_percentage = i->second;
+
+            if (path == "*")
+            {
+                star_max_percentage = max_percentage;
+            }
+            else
+            {
+                auto j = info.find(path);
+
+                if (j != info.end())
+                {
+                    const disk::SizesAndName& san = j->second;
+
+                    disk_space_exhausted = check_disk_space_exhausted(pMs, path, san, max_percentage);
+                    checked_paths.insert(path);
+                }
+                else
+                {
+                    MXS_WARNING("Disk space threshold specified for %s even though server %s at %s"
+                                "does not have that.",
+                                path.c_str(), pMs->server->name, pMs->server->address);
+                }
+            }
+        }
+
+        if (star_max_percentage != -1)
+        {
+            for (auto j = info.begin(); j != info.end(); ++j)
+            {
+                string path = j->first;
+
+                if (checked_paths.find(path) == checked_paths.end())
+                {
+                    const disk::SizesAndName& san = j->second;
+
+                    disk_space_exhausted = check_disk_space_exhausted(pMs, path, san, star_max_percentage);
+                }
+            }
+        }
+
+        if (disk_space_exhausted)
+        {
+            pMs->pending_status |= SERVER_DISK_SPACE_EXHAUSTED;
+        }
+        else
+        {
+            pMs->pending_status &= ~SERVER_DISK_SPACE_EXHAUSTED;
+        }
+
+        pMs->disk_space_checked = get_time_ms();
+    }
+    else
+    {
+        SERVER* pServer = pMs->server;
+
+        if (mysql_errno(pMs->con) == ER_UNKNOWN_TABLE)
+        {
+            // Disable disk space checking for this server.
+            pMs->disk_space_checked = -1;
+
+            MXS_ERROR("Disk space cannot be checked for %s at %s, because either the "
+                      "version %s is too old, or the DISKS information schema plugin "
+                      "has not been installed. Disk space checking has been disabled.",
+                      pServer->name, pServer->address, pServer->version_string);
+        }
+        else
+        {
+            MXS_ERROR("Checking the disk space for %s at %s failed due to: (%d) %s",
+                      pServer->name, pServer->address,
+                      mysql_errno(pMs->con), mysql_error(pMs->con));
+        }
+    }
+}
+
 bool MonitorInstance::configure(const MXS_CONFIG_PARAMETER* pParams)
 {
     return true;
@@ -2689,6 +2843,11 @@ void MonitorInstance::tick()
                 monitor_clear_pending_status(pMs, SERVER_AUTH_ERROR);
                 monitor_set_pending_status(pMs, SERVER_RUNNING);
 
+                if (should_check_disk_space(pMs))
+                {
+                    check_disk_space(pMs);
+                }
+
                 update_server_status(pMs);
             }
             else