MXS-1775 Add functions for checking disk space of servers
This commit is contained in:
Johan Wikman
parent
4325ab620b
commit
5172c43e29
@ -1284,7 +1284,8 @@ how full the corresponding disk may be, before action is taken. E.g. an entry li
|
||||
specifies that the disk that has been mounted on `/data` may be used until 80%
|
||||
of the total space has been consumed. Multiple entries can be specified by
|
||||
separating them by a comma. If the path is specified using `*`, then the limit
|
||||
applies to all disks.
|
||||
applies to all disks. However, the value of `*` is only applied if there is not
|
||||
an exact match.
|
||||
|
||||
Note that if a particular disk has been mounted on several paths, only one path
|
||||
need to be specified. If several are specified, then the one with the smallest
|
||||
@ -1294,8 +1295,12 @@ Examples:
|
||||
```
|
||||
disk_space_threshold=*:80
|
||||
disk_space_threshold=/data:80
|
||||
disk_space_threshold=/data1:80,/data2:70,/data3:30
|
||||
disk_space_threshold=/data1:80,/data2:60,*:90
|
||||
```
|
||||
The last line means that the disk mounted at `/data1` may be used up to
|
||||
80%, the disk mounted at `/data2` may be used up to 60% and all other disks
|
||||
mounted at any paths may be used up until 90% of maximum capacity, before
|
||||
MaxScale starts to warn to take action.
|
||||
|
||||
Note that the path to be used, is one of the paths returned by:
|
||||
```
|
||||
|
||||
@ -230,6 +230,7 @@ typedef struct monitored_server
|
||||
uint64_t mon_prev_status; /**< Status before starting the current monitor loop */
|
||||
uint64_t pending_status; /**< Status during current monitor loop */
|
||||
bool new_event; /**< Whether an action was taken on the last event */
|
||||
int64_t disk_space_checked; /**< When was the disk space checked the last time */
|
||||
struct monitored_server *next; /**< The next server in the list */
|
||||
} MXS_MONITORED_SERVER;
|
||||
|
||||
|
||||
@ -113,6 +113,12 @@ protected:
|
||||
return atomic_load_int32(&m_shutdown) != 0;
|
||||
}
|
||||
|
||||
bool should_check_disk_space(const MXS_MONITORED_SERVER* pMonitored_server) const;
|
||||
|
||||
void check_disk_space(MXS_MONITORED_SERVER* pMonitored_server);
|
||||
|
||||
static int64_t get_time_ms();
|
||||
|
||||
/**
|
||||
* @brief Configure the monitor.
|
||||
*
|
||||
|
||||
@ -31,6 +31,7 @@
|
||||
#include <maxscale/clock.h>
|
||||
#include <maxscale/json_api.h>
|
||||
#include <maxscale/log_manager.h>
|
||||
#include <maxscale/mariadb.hh>
|
||||
#include <maxscale/mysql_utils.h>
|
||||
#include <maxscale/paths.h>
|
||||
#include <maxscale/pcre2.h>
|
||||
@ -2652,6 +2653,159 @@ bool MonitorInstance::start(const MXS_CONFIG_PARAMETER* pParams)
|
||||
return started;
|
||||
}
|
||||
|
||||
//static
|
||||
int64_t MonitorInstance::get_time_ms()
|
||||
{
|
||||
timespec t;
|
||||
|
||||
ss_debug(int rv = )clock_gettime(CLOCK_MONOTONIC_COARSE, &t);
|
||||
ss_dassert(rv == 0);
|
||||
|
||||
return t.tv_sec * 1000 + (t.tv_nsec / 1000000);
|
||||
}
|
||||
|
||||
bool MonitorInstance::should_check_disk_space(const MXS_MONITORED_SERVER* pMs) const
|
||||
{
|
||||
bool should_check = false;
|
||||
|
||||
if (m_monitor->disk_space_check_interval &&
|
||||
(m_monitor->disk_space_threshold || pMs->server->disk_space_threshold) &&
|
||||
(pMs->disk_space_checked != -1)) // -1 means disabled
|
||||
{
|
||||
int64_t now = get_time_ms();
|
||||
|
||||
if (now - pMs->disk_space_checked > m_monitor->disk_space_check_interval)
|
||||
{
|
||||
should_check = true;
|
||||
}
|
||||
}
|
||||
|
||||
return should_check;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
bool check_disk_space_exhausted(MXS_MONITORED_SERVER* pMs,
|
||||
const std::string& path,
|
||||
const maxscale::disk::SizesAndName& san,
|
||||
int32_t max_percentage)
|
||||
{
|
||||
bool disk_space_exhausted = false;
|
||||
|
||||
int32_t used_percentage = ((san.total() - san.available()) / (double)san.total()) * 100;
|
||||
|
||||
if (used_percentage >= max_percentage)
|
||||
{
|
||||
MXS_ERROR("Disk space on %s at %s is exhausted; %d%% of the the disk "
|
||||
"mounted on the path %s has been used, and the limit it %d%%.",
|
||||
pMs->server->name, pMs->server->address,
|
||||
used_percentage, path.c_str(), max_percentage);
|
||||
disk_space_exhausted = true;
|
||||
}
|
||||
|
||||
return disk_space_exhausted;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void MonitorInstance::check_disk_space(MXS_MONITORED_SERVER* pMs)
|
||||
{
|
||||
std::map<std::string, disk::SizesAndName> info;
|
||||
|
||||
int rv = disk::get_info_by_path(pMs->con, &info);
|
||||
|
||||
if (rv == 0)
|
||||
{
|
||||
bool disk_space_exhausted = false;
|
||||
|
||||
MxsDiskSpaceThreshold* pDst =
|
||||
pMs->server->disk_space_threshold ?
|
||||
pMs->server->disk_space_threshold : m_monitor->disk_space_threshold;
|
||||
ss_dassert(pDst);
|
||||
|
||||
int32_t star_max_percentage = -1;
|
||||
|
||||
std::set<std::string> checked_paths;
|
||||
|
||||
for (auto i = pDst->begin(); i != pDst->end(); ++i)
|
||||
{
|
||||
string path = i->first;
|
||||
int32_t max_percentage = i->second;
|
||||
|
||||
if (path == "*")
|
||||
{
|
||||
star_max_percentage = max_percentage;
|
||||
}
|
||||
else
|
||||
{
|
||||
auto j = info.find(path);
|
||||
|
||||
if (j != info.end())
|
||||
{
|
||||
const disk::SizesAndName& san = j->second;
|
||||
|
||||
disk_space_exhausted = check_disk_space_exhausted(pMs, path, san, max_percentage);
|
||||
checked_paths.insert(path);
|
||||
}
|
||||
else
|
||||
{
|
||||
MXS_WARNING("Disk space threshold specified for %s even though server %s at %s"
|
||||
"does not have that.",
|
||||
path.c_str(), pMs->server->name, pMs->server->address);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (star_max_percentage != -1)
|
||||
{
|
||||
for (auto j = info.begin(); j != info.end(); ++j)
|
||||
{
|
||||
string path = j->first;
|
||||
|
||||
if (checked_paths.find(path) == checked_paths.end())
|
||||
{
|
||||
const disk::SizesAndName& san = j->second;
|
||||
|
||||
disk_space_exhausted = check_disk_space_exhausted(pMs, path, san, star_max_percentage);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (disk_space_exhausted)
|
||||
{
|
||||
pMs->pending_status |= SERVER_DISK_SPACE_EXHAUSTED;
|
||||
}
|
||||
else
|
||||
{
|
||||
pMs->pending_status &= ~SERVER_DISK_SPACE_EXHAUSTED;
|
||||
}
|
||||
|
||||
pMs->disk_space_checked = get_time_ms();
|
||||
}
|
||||
else
|
||||
{
|
||||
SERVER* pServer = pMs->server;
|
||||
|
||||
if (mysql_errno(pMs->con) == ER_UNKNOWN_TABLE)
|
||||
{
|
||||
// Disable disk space checking for this server.
|
||||
pMs->disk_space_checked = -1;
|
||||
|
||||
MXS_ERROR("Disk space cannot be checked for %s at %s, because either the "
|
||||
"version %s is too old, or the DISKS information schema plugin "
|
||||
"has not been installed. Disk space checking has been disabled.",
|
||||
pServer->name, pServer->address, pServer->version_string);
|
||||
}
|
||||
else
|
||||
{
|
||||
MXS_ERROR("Checking the disk space for %s at %s failed due to: (%d) %s",
|
||||
pServer->name, pServer->address,
|
||||
mysql_errno(pMs->con), mysql_error(pMs->con));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool MonitorInstance::configure(const MXS_CONFIG_PARAMETER* pParams)
|
||||
{
|
||||
return true;
|
||||
@ -2689,6 +2843,11 @@ void MonitorInstance::tick()
|
||||
monitor_clear_pending_status(pMs, SERVER_AUTH_ERROR);
|
||||
monitor_set_pending_status(pMs, SERVER_RUNNING);
|
||||
|
||||
if (should_check_disk_space(pMs))
|
||||
{
|
||||
check_disk_space(pMs);
|
||||
}
|
||||
|
||||
update_server_status(pMs);
|
||||
}
|
||||
else
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user