From 6b556859ce31a6a565d6df7d1e6ec569b7aa330f Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Thu, 17 Jan 2019 16:58:05 +0200 Subject: [PATCH] MXS-2219 Use system.membership as primary table From system.membership we can find out what server exist in the cluster while system.nodeinfo contains information about those servers. If a node goes down, it will disappear from system.nodeinfo, but not from system.membership. Consequently, we must start from system.membership and then fetch more information from system.nodeinfo. Incidentally, a query like SELECT ms.nid, ni.iface_ip FROM system.membership AS ms LEFT JOIN system.nodeinfo AS ni ON ms.nid=ni.nodeid; should provide all information in one go, but it seems that such joins are not supported on the system tables. --- .../monitor/clustrixmon/CMakeLists.txt | 1 + .../modules/monitor/clustrixmon/clustrix.cc | 88 ++++++++ .../modules/monitor/clustrixmon/clustrix.hh | 40 ++++ .../monitor/clustrixmon/clustrixmembership.hh | 83 ++++++++ .../monitor/clustrixmon/clustrixmonitor.cc | 193 ++++++++++++++---- .../monitor/clustrixmon/clustrixmonitor.hh | 23 ++- .../monitor/clustrixmon/clustrixnode.hh | 182 +++++++++++++++++ .../monitor/clustrixmon/clustrixnodeinfo.hh | 117 ----------- 8 files changed, 564 insertions(+), 163 deletions(-) create mode 100644 server/modules/monitor/clustrixmon/clustrix.cc create mode 100644 server/modules/monitor/clustrixmon/clustrix.hh create mode 100644 server/modules/monitor/clustrixmon/clustrixmembership.hh create mode 100644 server/modules/monitor/clustrixmon/clustrixnode.hh delete mode 100644 server/modules/monitor/clustrixmon/clustrixnodeinfo.hh diff --git a/server/modules/monitor/clustrixmon/CMakeLists.txt b/server/modules/monitor/clustrixmon/CMakeLists.txt index 72527477d..81d9d228d 100644 --- a/server/modules/monitor/clustrixmon/CMakeLists.txt +++ b/server/modules/monitor/clustrixmon/CMakeLists.txt @@ -1,4 +1,5 @@ add_library(clustrixmon SHARED + clustrix.cc clustrixmon.cc clustrixmonitor.cc ) diff --git a/server/modules/monitor/clustrixmon/clustrix.cc b/server/modules/monitor/clustrixmon/clustrix.cc new file mode 100644 index 000000000..a50a0d3a5 --- /dev/null +++ b/server/modules/monitor/clustrixmon/clustrix.cc @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2018 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl11. + * + * Change Date: 2022-01-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ + +#include "clustrix.hh" +#include + +namespace +{ + +const char CN_NORMAL[] = "normal"; +const char CN_QUORUM[] = "quorum"; +const char CN_STATIC[] = "static"; +const char CN_UNKNOWN[] = "unknown"; + +} + +std::string Clustrix::to_string(Clustrix::Status status) +{ + switch (status) + { + case Status::QUORUM: + return CN_QUORUM; + + case Status::STATIC: + return CN_STATIC; + + case Status::UNKNOWN: + return CN_UNKNOWN; + } + + mxb_assert(!true); + return CN_UNKNOWN; +} + +Clustrix::Status Clustrix::status_from_string(const std::string& status) +{ + if (status == CN_QUORUM) + { + return Status::QUORUM; + } + else if (status == CN_STATIC) + { + return Status::STATIC; + } + else + { + MXB_WARNING("'%s' is an unknown status for a Clustrix node.", status.c_str()); + return Status::UNKNOWN; + } +} + +std::string Clustrix::to_string(Clustrix::SubState substate) +{ + switch (substate) + { + case SubState::NORMAL: + return CN_NORMAL; + + case SubState::UNKNOWN: + return CN_UNKNOWN; + } + + mxb_assert(!true); + return CN_UNKNOWN; +} + +Clustrix::SubState Clustrix::substate_from_string(const std::string& substate) +{ + if (substate == CN_NORMAL) + { + return SubState::NORMAL; + } + else + { + MXB_WARNING("'%s' is an unknown sub-state for a Clustrix node.", substate.c_str()); + return SubState::UNKNOWN; + } +} diff --git a/server/modules/monitor/clustrixmon/clustrix.hh b/server/modules/monitor/clustrixmon/clustrix.hh new file mode 100644 index 000000000..f66e7959e --- /dev/null +++ b/server/modules/monitor/clustrixmon/clustrix.hh @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2018 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl11. + * + * Change Date: 2022-01-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ +#pragma once + +#include "clustrixmon.hh" +#include + +namespace Clustrix +{ + +enum class Status +{ + QUORUM, + STATIC, + UNKNOWN +}; + +Status status_from_string(const std::string& status); +std::string to_string(Status status); + +enum class SubState +{ + NORMAL, + UNKNOWN +}; + +SubState substate_from_string(const std::string& substate); +std::string to_string(SubState sub_state); + +} diff --git a/server/modules/monitor/clustrixmon/clustrixmembership.hh b/server/modules/monitor/clustrixmon/clustrixmembership.hh new file mode 100644 index 000000000..e5bb7aa69 --- /dev/null +++ b/server/modules/monitor/clustrixmon/clustrixmembership.hh @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2018 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl11. + * + * Change Date: 2022-01-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ +#pragma once + +#include "clustrixmon.hh" +#include +#include +#include +#include "clustrix.hh" + +class ClustrixMembership +{ +public: + ClustrixMembership(int id, + Clustrix::Status status, + Clustrix::SubState substate, + int instance) + : m_id(id) + , m_status(status) + , m_substate(substate) + , m_instance(instance) + { + } + + int id() const + { + return m_id; + } + + Clustrix::Status status() const + { + return m_status; + } + + Clustrix::SubState substate() const + { + return m_substate; + } + + int instance() const + { + return m_instance; + } + + std::string to_string() const + { + std::stringstream ss; + ss << "{" + << m_id << ", " + << Clustrix::to_string(m_status) << ", " + << Clustrix::to_string(m_substate) << ", " + << m_instance + << "}"; + return ss.str(); + } + + void print(std::ostream& o) const + { + o << to_string(); + } + +private: + int m_id; + Clustrix::Status m_status; + Clustrix::SubState m_substate; + int m_instance; +}; + +inline std::ostream& operator << (std::ostream& out, const ClustrixMembership& x) +{ + x.print(out); + return out; +} diff --git a/server/modules/monitor/clustrixmon/clustrixmonitor.cc b/server/modules/monitor/clustrixmon/clustrixmonitor.cc index 0c36b8536..e44c1bff0 100644 --- a/server/modules/monitor/clustrixmon/clustrixmonitor.cc +++ b/server/modules/monitor/clustrixmon/clustrixmonitor.cc @@ -45,7 +45,7 @@ ClustrixMonitor* ClustrixMonitor::create(MXS_MONITOR* pMonitor) bool ClustrixMonitor::configure(const MXS_CONFIG_PARAMETER* pParams) { m_health_urls.clear(); - m_node_infos.clear(); + m_nodes.clear(); m_config.set_cluster_monitor_interval(config_get_integer(pParams, CLUSTER_MONITOR_INTERVAL_NAME)); m_config.set_health_check_threshold(config_get_integer(pParams, HEALTH_CHECK_THRESHOLD_NAME)); @@ -97,7 +97,7 @@ void ClustrixMonitor::tick() } } -void ClustrixMonitor::fetch_cluster_nodes() +void ClustrixMonitor::update_cluster_nodes() { auto b = begin(*(m_monitor->monitored_servers)); auto e = end(*(m_monitor->monitored_servers)); @@ -112,7 +112,7 @@ void ClustrixMonitor::fetch_cluster_nodes() if (it != e) { MXS_MONITORED_SERVER& ms = *it; - fetch_cluster_nodes_from(ms); + update_cluster_nodes(ms); m_pMonitored_server = &ms; } @@ -122,10 +122,14 @@ void ClustrixMonitor::fetch_cluster_nodes() } } -void ClustrixMonitor::fetch_cluster_nodes_from(MXS_MONITORED_SERVER& ms) +void ClustrixMonitor::update_cluster_nodes(MXS_MONITORED_SERVER& ms) { mxb_assert(ms.con); + map memberships; + + check_cluster_membership(ms, &memberships); + const char ZQUERY[] = "SELECT nodeid, iface_ip, mysql_port, healthmon_port FROM system.nodeinfo"; if (mysql_query(ms.con, ZQUERY) == 0) @@ -136,14 +140,13 @@ void ClustrixMonitor::fetch_cluster_nodes_from(MXS_MONITORED_SERVER& ms) { mxb_assert(mysql_field_count(ms.con) == 4); - MYSQL_ROW row; - set nids; - for_each(m_node_infos.begin(), m_node_infos.end(), - [&nids](const pair& element) { + for_each(m_nodes.begin(), m_nodes.end(), + [&nids](const pair& element) { nids.insert(element.first); }); + MYSQL_ROW row; while ((row = mysql_fetch_row(pResult)) != nullptr) { if (row[0] && row[1]) @@ -152,14 +155,39 @@ void ClustrixMonitor::fetch_cluster_nodes_from(MXS_MONITORED_SERVER& ms) string ip = row[1]; int mysql_port = row[2] ? atoi(row[2]) : DEFAULT_MYSQL_PORT; int health_port = row[3] ? atoi(row[3]) : DEFAULT_HEALTH_PORT; - int health_check_threshold = m_config.health_check_threshold(); string name = "@Clustrix-Server-" + std::to_string(id); - auto it = m_node_infos.find(id); + auto nit = m_nodes.find(id); + auto mit = memberships.find(id); - if (it == m_node_infos.end()) + if (nit != m_nodes.end()) { + // Existing node. + mxb_assert(SERVER::find_by_unique_name(name)); + + ClustrixNode& node = nit->second; + + if (node.ip() != ip) + { + node.set_ip(ip); + } + + if (node.mysql_port() != mysql_port) + { + node.set_mysql_port(mysql_port); + } + + if (node.health_port() != health_port) + { + node.set_health_port(health_port); + } + + nids.erase(id); + } + else if (mit != memberships.end()) + { + // New node. mxb_assert(!SERVER::find_by_unique_name(name)); if (runtime_create_server(name.c_str(), @@ -172,23 +200,28 @@ void ClustrixMonitor::fetch_cluster_nodes_from(MXS_MONITORED_SERVER& ms) SERVER* pServer = SERVER::find_by_unique_name(name); mxb_assert(pServer); - ClustrixNodeInfo info(id, ip, mysql_port, health_port, health_check_threshold, pServer); + const ClustrixMembership& membership = mit->second; + int health_check_threshold = m_config.health_check_threshold(); - m_node_infos.insert(make_pair(id, info)); + ClustrixNode node(membership, ip, mysql_port, health_port, health_check_threshold, pServer); + + m_nodes.insert(make_pair(id, node)); } else { MXS_ERROR("Could not create server %s at %s:%d.", name.c_str(), ip.c_str(), mysql_port); } + + memberships.erase(mit); } else { - mxb_assert(SERVER::find_by_unique_name(name)); + // Node found in system.node_info but not in system.membership + MXS_ERROR("Node %d at %s:%d,%d found in system.node_info " + "but not in system.membership.", + id, ip.c_str(), mysql_port, health_port); - auto it = nids.find(id); - mxb_assert(it != nids.end()); - nids.erase(it); } } else @@ -201,19 +234,18 @@ void ClustrixMonitor::fetch_cluster_nodes_from(MXS_MONITORED_SERVER& ms) for_each(nids.begin(), nids.end(), [this](int nid) { - auto it = m_node_infos.find(nid); - mxb_assert(it != m_node_infos.end()); + auto it = m_nodes.find(nid); + mxb_assert(it != m_nodes.end()); - ClustrixNodeInfo& info = it->second; - info.deactivate_server(); - m_node_infos.erase(it); + ClustrixNode& node = it->second; + node.set_running(false, ClustrixNode::APPROACH_OVERRIDE); }); vector health_urls; - for_each(m_node_infos.begin(), m_node_infos.end(), - [&health_urls](const pair& element) { - const ClustrixNodeInfo& info = element.second; - string url = "http://" + info.ip() + ":" + std::to_string(info.health_port()); + for_each(m_nodes.begin(), m_nodes.end(), + [&health_urls](const pair& element) { + const ClustrixNode& node = element.second; + string url = "http://" + node.ip() + ":" + std::to_string(node.health_port()); health_urls.push_back(url); }); @@ -242,22 +274,113 @@ void ClustrixMonitor::refresh_cluster_nodes() if (mon_connection_is_ok(rv)) { - fetch_cluster_nodes_from(*m_pMonitored_server); + update_cluster_nodes(*m_pMonitored_server); } else { mysql_close(m_pMonitored_server->con); m_pMonitored_server->con = nullptr; - fetch_cluster_nodes(); + update_cluster_nodes(); } } else if (m_monitor->monitored_servers) { - fetch_cluster_nodes(); + update_cluster_nodes(); } } +bool ClustrixMonitor::check_cluster_membership(MXS_MONITORED_SERVER& ms, + std::map* pMemberships) +{ + mxb_assert(ms.con); + mxb_assert(pMemberships); + + bool rv = false; + + const char ZQUERY[] = "SELECT nid, status, instance, substate FROM system.membership"; + + if (mysql_query(ms.con, ZQUERY) == 0) + { + MYSQL_RES* pResult = mysql_store_result(ms.con); + + if (pResult) + { + mxb_assert(mysql_field_count(ms.con) == 4); + + set nids; + for_each(m_nodes.begin(), m_nodes.end(), + [&nids](const pair& element) { + nids.insert(element.first); + }); + + MYSQL_ROW row; + while ((row = mysql_fetch_row(pResult)) != nullptr) + { + if (row[0]) + { + int nid = atoi(row[0]); + string status = row[1] ? row[1] : "unknown"; + int instance = row[2] ? atoi(row[2]) : -1; + string substate = row[3] ? row[3] : "unknown"; + + auto it = m_nodes.find(nid); + + if (it != m_nodes.end()) + { + ClustrixNode& node = it->second; + + node.update(Clustrix::status_from_string(status), + Clustrix::substate_from_string(substate), + instance); + + nids.erase(node.id()); + } + else + { + ClustrixMembership membership(nid, + Clustrix::status_from_string(status), + Clustrix::substate_from_string(substate), + instance); + + pMemberships->insert(make_pair(nid, membership)); + } + } + else + { + MXS_WARNING("No node id returned in row for '%s'.", ZQUERY); + } + } + + mysql_free_result(pResult); + + // Deactivate all servers that are no longer members. + for_each(nids.begin(), nids.end(), + [this](int nid) { + auto it = m_nodes.find(nid); + mxb_assert(it != m_nodes.end()); + + ClustrixNode& node = it->second; + node.deactivate_server(); + m_nodes.erase(it); + }); + + rv = true; + } + else + { + MXS_WARNING("No result returned for '%s'.", ZQUERY); + } + } + else + { + MXS_ERROR("Could not execute '%s' on %s: %s", + ZQUERY, ms.server->address, mysql_error(ms.con)); + } + + return rv; +} + void ClustrixMonitor::update_server_statuses() { mxb_assert(m_monitor->monitored_servers); @@ -269,15 +392,15 @@ void ClustrixMonitor::update_server_statuses() [this](MXS_MONITORED_SERVER& ms) { monitor_stash_current_status(&ms); - auto it = find_if(m_node_infos.begin(), m_node_infos.end(), - [&ms](const std::pair& element) -> bool { - const ClustrixNodeInfo& info = element.second; + auto it = find_if(m_nodes.begin(), m_nodes.end(), + [&ms](const std::pair& element) -> bool { + const ClustrixNode& info = element.second; return ms.server->address == info.ip(); }); - if (it != m_node_infos.end()) + if (it != m_nodes.end()) { - const ClustrixNodeInfo& info = it->second; + const ClustrixNode& info = it->second; if (info.is_running()) { @@ -350,7 +473,7 @@ bool ClustrixMonitor::check_http(Call::action_t action) { const vector& results = m_http.results(); - auto it = m_node_infos.begin(); + auto it = m_nodes.begin(); for_each(results.begin(), results.end(), [&it](const http::Result& result) { diff --git a/server/modules/monitor/clustrixmon/clustrixmonitor.hh b/server/modules/monitor/clustrixmon/clustrixmonitor.hh index ff08dcbbc..525c6801a 100644 --- a/server/modules/monitor/clustrixmon/clustrixmonitor.hh +++ b/server/modules/monitor/clustrixmon/clustrixmonitor.hh @@ -16,7 +16,8 @@ #include #include #include -#include "clustrixnodeinfo.hh" +#include "clustrixmembership.hh" +#include "clustrixnode.hh" class ClustrixMonitor : public maxscale::MonitorInstance { @@ -71,9 +72,10 @@ private: void tick(); - void fetch_cluster_nodes(); - void fetch_cluster_nodes_from(MXS_MONITORED_SERVER& ms); + void update_cluster_nodes(); + void update_cluster_nodes(MXS_MONITORED_SERVER& ms); void refresh_cluster_nodes(); + bool check_cluster_membership(MXS_MONITORED_SERVER& ms, std::map* pMemberships); void update_server_statuses(); void make_health_check(); @@ -86,12 +88,11 @@ private: } private: - Config m_config; - std::vector m_config_servers; - std::map m_node_infos; - std::vector m_health_urls; - mxb::http::Async m_http; - uint32_t m_delayed_http_check_id { 0 }; - long m_last_cluster_check { 0 }; - MXS_MONITORED_SERVER* m_pMonitored_server { nullptr }; + Config m_config; + std::map m_nodes; + std::vector m_health_urls; + mxb::http::Async m_http; + uint32_t m_delayed_http_check_id { 0 }; + long m_last_cluster_check { 0 }; + MXS_MONITORED_SERVER* m_pMonitored_server { nullptr }; }; diff --git a/server/modules/monitor/clustrixmon/clustrixnode.hh b/server/modules/monitor/clustrixmon/clustrixnode.hh new file mode 100644 index 000000000..a2d29fb77 --- /dev/null +++ b/server/modules/monitor/clustrixmon/clustrixnode.hh @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2018 MariaDB Corporation Ab + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file and at www.mariadb.com/bsl11. + * + * Change Date: 2022-01-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2 or later of the General + * Public License. + */ +#pragma once + +#include "clustrixmon.hh" +#include +#include +#include +#include "clustrix.hh" + +class ClustrixNode +{ +public: + enum + { + DEFAULT_MYSQL_PORT = 3306, + DEFAULT_HEALTH_PORT = 3581, + }; + + enum approach_t + { + APPROACH_OVERRIDE, + APPROACH_DEFAULT + }; + + ClustrixNode(const ClustrixMembership& membership, + const std::string& ip, + int mysql_port, + int health_port, + int health_check_threshold, + SERVER* pServer) + : m_id(membership.id()) + , m_status(membership.status()) + , m_substate(membership.substate()) + , m_instance(membership.instance()) + , m_ip(ip) + , m_mysql_port(mysql_port) + , m_health_port(health_port) + , m_health_check_threshold(health_check_threshold) + , m_nRunning(m_health_check_threshold) + , m_pServer(pServer) + { + } + + int id() const + { + return m_id; + } + + Clustrix::Status status() const + { + return m_status; + } + + Clustrix::SubState substate() const + { + return m_substate; + } + + int instance() const + { + return m_instance; + } + + const std::string& ip() const + { + return m_ip; + } + + void set_ip(const std::string& ip) + { + m_ip = ip; + m_pServer->server_update_address(ip); + } + + int mysql_port() const + { + return m_mysql_port; + } + + void set_mysql_port(int port) + { + m_mysql_port = port; + m_pServer->update_port(port); + } + + int health_port() const + { + return m_health_port; + } + + void set_health_port(int port) + { + m_health_port = port; + } + + bool is_running() const + { + return m_nRunning > 0; + } + + void set_running(bool running, approach_t approach = APPROACH_DEFAULT) + { + if (running) + { + m_nRunning = m_health_check_threshold; + + m_pServer->set_status(SERVER_RUNNING); + } + else + { + if (m_nRunning > 0) + { + if (approach == APPROACH_OVERRIDE) + { + m_nRunning = 0; + } + else + { + --m_nRunning; + } + + if (m_nRunning == 0) + { + m_pServer->clear_status(SERVER_RUNNING); + } + } + } + } + + void update(Clustrix::Status status, Clustrix::SubState substate, int instance) + { + m_status = status; + m_substate = substate; + m_instance = instance; + } + + void deactivate_server() + { + m_pServer->is_active = false; + } + + std::string to_string() const + { + std::stringstream ss; + ss << "{" << m_id << ", " << m_ip << ", " << m_mysql_port << ", " << m_health_port << "}"; + return ss.str(); + } + + void print(std::ostream& o) const + { + o << to_string(); + } + +private: + int m_id; + Clustrix::Status m_status; + Clustrix::SubState m_substate; + int m_instance; + std::string m_ip; + int m_mysql_port { DEFAULT_MYSQL_PORT }; + int m_health_port { DEFAULT_HEALTH_PORT }; + int m_health_check_threshold { DEFAULT_HEALTH_CHECK_THRESHOLD_VALUE }; + int m_nRunning { 0 }; + SERVER* m_pServer { nullptr }; +}; + +inline std::ostream& operator << (std::ostream& out, const ClustrixNode& x) +{ + x.print(out); + return out; +} diff --git a/server/modules/monitor/clustrixmon/clustrixnodeinfo.hh b/server/modules/monitor/clustrixmon/clustrixnodeinfo.hh deleted file mode 100644 index 5f9aafca6..000000000 --- a/server/modules/monitor/clustrixmon/clustrixnodeinfo.hh +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright (c) 2018 MariaDB Corporation Ab - * - * Use of this software is governed by the Business Source License included - * in the LICENSE.TXT file and at www.mariadb.com/bsl11. - * - * Change Date: 2022-01-01 - * - * On the date above, in accordance with the Business Source License, use - * of this software will be governed by version 2 or later of the General - * Public License. - */ -#pragma once - -#include "clustrixmon.hh" -#include -#include -#include - -class ClustrixNodeInfo -{ -public: - ClustrixNodeInfo(int id, - const std::string& ip, - int mysql_port, - int health_port, - int health_check_threshold, - SERVER* pServer) - : m_id(id) - , m_ip(ip) - , m_mysql_port(mysql_port) - , m_health_port(health_port) - , m_health_check_threshold(health_check_threshold) - , m_nRunning(m_health_check_threshold) - , m_pServer(pServer) - { - } - - int id() const - { - return m_id; - } - - const std::string& ip() const - { - return m_ip; - } - - int mysql_port() const - { - return m_mysql_port; - } - - int health_port() const - { - return m_health_port; - } - - bool is_running() const - { - return m_nRunning > 0; - } - - void set_running(bool running) - { - if (running) - { - m_nRunning = m_health_check_threshold; - - m_pServer->set_status(SERVER_RUNNING); - } - else - { - if (m_nRunning > 0) - { - --m_nRunning; - - if (m_nRunning == 0) - { - m_pServer->clear_status(SERVER_RUNNING); - } - } - } - } - - void deactivate_server() - { - m_pServer->is_active = false; - } - - std::string to_string() const - { - std::stringstream ss; - ss << "{" << m_id << ", " << m_ip << ", " << m_mysql_port << ", " << m_health_port << "}"; - return ss.str(); - } - - void print(std::ostream& o) const - { - o << to_string(); - } - -private: - int m_id; - std::string m_ip; - int m_mysql_port; - int m_health_port; - int m_health_check_threshold; - int m_nRunning; - SERVER* m_pServer; -}; - -inline std::ostream& operator << (std::ostream& out, const ClustrixNodeInfo& x) -{ - x.print(out); - return out; -}