From c0bd5ca3a1a12bea57daccdb9a65f040e34e3be8 Mon Sep 17 00:00:00 2001 From: Esa Korhonen Date: Wed, 1 Aug 2018 10:39:30 +0300 Subject: [PATCH] MXS-1905 Switchover if master is low on disk space Required quite a bit of refactoring. --- include/maxscale/server.h | 7 +- .../mariadbmon/cluster_manipulation.cc | 564 +++++++++++------- .../modules/monitor/mariadbmon/mariadbmon.cc | 53 +- .../modules/monitor/mariadbmon/mariadbmon.hh | 15 +- .../monitor/mariadbmon/mariadbmon_common.cc | 12 + .../monitor/mariadbmon/mariadbmon_common.hh | 24 + .../monitor/mariadbmon/mariadbserver.cc | 111 +++- .../monitor/mariadbmon/mariadbserver.hh | 46 +- 8 files changed, 577 insertions(+), 255 deletions(-) diff --git a/include/maxscale/server.h b/include/maxscale/server.h index 0054c5bc8..a2289cde7 100644 --- a/include/maxscale/server.h +++ b/include/maxscale/server.h @@ -350,9 +350,14 @@ inline bool server_is_slave_of_ext_master(const SERVER* server) (SERVER_RUNNING | SERVER_SLAVE_OF_EXT_MASTER)); } +inline bool status_is_disk_space_exhausted(uint64_t status) +{ + return (status & SERVER_DISK_SPACE_EXHAUSTED); +} + inline bool server_is_disk_space_exhausted(const SERVER* server) { - return (server->status & SERVER_DISK_SPACE_EXHAUSTED); + return status_is_disk_space_exhausted(server->status); } /** diff --git a/server/modules/monitor/mariadbmon/cluster_manipulation.cc b/server/modules/monitor/mariadbmon/cluster_manipulation.cc index 2876e0f01..38c80c55c 100644 --- a/server/modules/monitor/mariadbmon/cluster_manipulation.cc +++ b/server/modules/monitor/mariadbmon/cluster_manipulation.cc @@ -21,45 +21,60 @@ using std::string; using maxscale::string_printf; +static const char RE_ENABLE_FMT[] = "To re-enable automatic %s, manually set '%s' to 'true' " + "for monitor '%s' via MaxAdmin or the REST API, or restart MaxScale."; + static void print_redirect_errors(MariaDBServer* first_server, const ServerArray& servers, json_t** err_out); -bool MariaDBMonitor::manual_switchover(SERVER* new_master, SERVER* current_master, json_t** error_out) +/** + * Run a manual switchover, promoting a new master server and demoting the existing master. + * + * @param promotion_server The server which should be promoted. If null, monitor will autoselect. + * @param demotion_server The server which should be demoted. Can be null for autoselect, in which case + * monitor will select the cluster master server. Otherwise must be a valid master server or a relay. + * @param error_out Error output + * @return True, if switchover was performed successfully + */ +bool MariaDBMonitor::manual_switchover(SERVER* promotion_server, SERVER* demotion_server, json_t** error_out) { - /* It's possible for either current_master, or both new_master & current_master to be NULL, which means - * autoselect. Only autoselecting new_master is not possible. Autoselection will happen at the actual - * switchover function. */ - MariaDBServer *found_new_master = NULL, *found_curr_master = NULL; - auto ok_to_switch = switchover_check(new_master, current_master, &found_new_master, &found_curr_master, - error_out); + /* The server parameters may be null, in which case the monitor will autoselect. + * + * Manual commands (as well as automatic ones) are ran at the end of a normal monitor loop, + * so server states can be assumed to be up-to-date. + */ + MariaDBServer* promotion_target = NULL; + MariaDBServer* demotion_target = NULL; + auto ok_to_switch = switchover_prepare(promotion_server, demotion_server, + &promotion_target, &demotion_target, error_out); bool rval = false; if (ok_to_switch) { - bool switched = do_switchover(&found_curr_master, &found_new_master, error_out); - const char AUTOSELECT[] = ""; - const char* curr_master_name = found_curr_master ? found_curr_master->name() : AUTOSELECT; - const char* new_master_name = found_new_master ? found_new_master->name() : AUTOSELECT; - + bool switched = do_switchover(demotion_target, promotion_target, error_out); if (switched) { - MXS_NOTICE("Switchover %s -> %s performed.", curr_master_name, new_master_name); + MXS_NOTICE("Switchover %s -> %s performed.", demotion_target->name(), promotion_target->name()); rval = true; } else { - string format = "Switchover %s -> %s failed"; + string msg = string_printf("Switchover %s -> %s failed", + demotion_target->name(), promotion_target->name()); bool failover_setting = config_get_bool(m_monitor->parameters, CN_AUTO_FAILOVER); if (failover_setting) { disable_setting(CN_AUTO_FAILOVER); - format += ", automatic failover has been disabled."; + msg += ", automatic failover has been disabled"; } - format += "."; - PRINT_MXS_JSON_ERROR(error_out, format.c_str(), curr_master_name, new_master_name); + msg += "."; + PRINT_MXS_JSON_ERROR(error_out, "%s", msg.c_str()); } } - + else + { + PRINT_MXS_JSON_ERROR(error_out, "Switchover cancelled."); + } return rval; } @@ -446,96 +461,35 @@ bool MariaDBMonitor::server_is_rejoin_suspect(MariaDBServer* rejoin_cand, json_t /** * Performs switchover for a simple topology (1 master, N slaves, no intermediate masters). If an - * intermediate step fails, the cluster may be left without a master. + * intermediate step fails, the cluster may be left without a master and manual intervention is + * required to fix things. * - * @param current_master Handle to current master server. If null, the autoselected server is written here. - * @param new_master Handle to slave which should be promoted. If null, the autoselected server is written - * here. + * @param demotion_target Server to demote + * @param promotion_target Server to promote * @param err_out json object for error printing. Can be NULL. - * @return True if successful. If false, the cluster can be in various situations depending on which step - * failed. In practice, manual intervention is usually required on failure. + * @return True if successful. If false, replication may be broken. */ -bool MariaDBMonitor::do_switchover(MariaDBServer** current_master, MariaDBServer** new_master, +bool MariaDBMonitor::do_switchover(MariaDBServer* demotion_target, MariaDBServer* promotion_target, json_t** err_out) { - ss_dassert(current_master && new_master); - MariaDBServer* demotion_target = NULL; - if (*current_master == NULL) - { - // Autoselect current master. - if (m_master && m_master->is_master()) - { - demotion_target = m_master; - *current_master = demotion_target; - } - else - { - PRINT_MXS_JSON_ERROR(err_out, "Could not autoselect current master for switchover. Cluster does " - "not have a master or master is in maintenance."); - return false; - } - } - else - { - // No need to check a given current master, it has already been checked. - demotion_target = *current_master; - } - - if (m_master_gtid_domain == GTID_DOMAIN_UNKNOWN) - { - PRINT_MXS_JSON_ERROR(err_out, "Cluster gtid domain is unknown. Cannot switchover."); - return false; - } + ss_dassert(demotion_target && promotion_target); // Total time limit on how long this operation may take. Checked and modified after significant steps are // completed. int seconds_remaining = m_switchover_timeout; time_t start_time = time(NULL); - // Step 1: Save all slaves except promotion target to an array. If we have a - // user-defined master candidate, check it. Otherwise, autoselect. - MariaDBServer* promotion_target = NULL; + // Step 1: Save all slaves except promotion target to an array. ServerArray redirectable_slaves; - if (*new_master == NULL) + for (MariaDBServer* redirectable : demotion_target->m_node.children) { - // Autoselect new master. - promotion_target = select_new_master(&redirectable_slaves, err_out); - if (promotion_target) + // TODO: Again check valid replication here + if (redirectable != promotion_target && redirectable->is_replicating_from(demotion_target) && + redirectable->uses_gtid()) { - *new_master = promotion_target; - } - else - { - PRINT_MXS_JSON_ERROR(err_out, "Could not autoselect new master for switchover."); - return false; + redirectable_slaves.push_back(redirectable); } } - else - { - // Check user-given new master. Some checks have already been performed but more is needed. - if (switchover_check_preferred_master(*new_master, err_out)) - { - promotion_target = *new_master; - /* User-given candidate is good. Update info on all slave servers. - * The update_slave_info()-call is not strictly necessary here, but it should be ran to keep this - * path analogous with failover_select_new_master(). The later functions can then assume that - * slave server info is up to date. If the master is replicating from external master, it is - * updated by update_slave_info() but not added to array. */ - for (auto iter = m_servers.begin(); iter != m_servers.end(); iter++) - { - MariaDBServer* server = *iter; - if (server != promotion_target && server->update_slave_info() && server != demotion_target) - { - redirectable_slaves.push_back(server); - } - } - } - else - { - return false; - } - } - ss_dassert(demotion_target && promotion_target); bool rval = false; // Step 2: Set read-only to on, flush logs, update master gtid:s @@ -1058,11 +1012,16 @@ MariaDBServer* MariaDBMonitor::select_new_master(ServerArray* slaves_out, json_t { // If no new master yet, accept any valid candidate. Otherwise check. if (current_best == NULL || - is_candidate_better(current_best, cand, m_master_gtid_domain, ¤t_best_reason)) + is_candidate_better(cand, current_best, m_master_gtid_domain, ¤t_best_reason)) { // The server has been selected for promotion, for now. current_best = cand; master_vector_index = slaves_out->size() - 1; + if (!current_best_reason.empty()) + { + current_best_reason = string_printf("Selected '%s' because %s", current_best->name(), + current_best_reason.c_str()); + } } } } @@ -1088,7 +1047,7 @@ MariaDBServer* MariaDBMonitor::select_new_master(ServerArray* slaves_out, json_t MXS_WARNING(EXCLUDED_ONLY_CAND, excluded_name); break; } - else if (is_candidate_better(current_best, excluded_info, m_master_gtid_domain)) + else if (is_candidate_better(excluded_info, current_best, m_master_gtid_domain)) { // Print a warning if this server is actually a better candidate than the previous best. const char EXCLUDED_CAND[] = "Server '%s' is superior to current best candidate '%s', " @@ -1112,6 +1071,110 @@ MariaDBServer* MariaDBMonitor::select_new_master(ServerArray* slaves_out, json_t return current_best; } + +MariaDBServer* MariaDBMonitor::switchover_select_promotion(MariaDBServer* demotion_target, json_t** err_out) +{ + /* Select a new master candidate. Selects the one with the latest event in relay log. + * If multiple slaves have same number of events, select the one with most processed events. */ + + if (!demotion_target->m_node.children.empty()) + { + MXS_NOTICE("Selecting a server to promote and replace '%s'. Candidates are: %s.", + demotion_target->name(), + monitored_servers_to_string(demotion_target->m_node.children).c_str()); + } + else + { + PRINT_MXS_JSON_ERROR(err_out, "'%s' does not have any slaves to promote.", demotion_target->name()); + return NULL; + } + + // Servers that cannot be selected because of exclusion, but seem otherwise ok. + ServerArray valid_but_excluded; + + string all_reasons; + DelimitedPrinter printer("\n"); + // The valid promotion candidates are the slaves replicating directly from the demotion target. + ServerArray candidates; + for (MariaDBServer* cand : demotion_target->m_node.children) + { + string reason; + if (!cand->can_be_promoted(demotion_target, &reason)) + { + string msg = string_printf("'%s' cannot be selected because %s", cand->name(), reason.c_str()); + printer.cat(all_reasons, msg); + } + else if (server_is_excluded(cand)) + { + valid_but_excluded.push_back(cand); + string msg = string_printf("'%s' cannot be selected because it is excluded.", cand->name()); + printer.cat(all_reasons, msg); + } + else + { + candidates.push_back(cand); + } + } + + MariaDBServer* current_best = NULL; + if (candidates.empty()) + { + PRINT_MXS_JSON_ERROR(err_out, "No suitable promotion candidate found:\n%s", all_reasons.c_str()); + } + else + { + current_best = candidates.front(); + candidates.erase(candidates.begin()); + if (!all_reasons.empty()) + { + MXS_WARNING("Some servers were disqualified for promotion:\n%s", all_reasons.c_str()); + } + } + + // Check which candidate is best + string current_best_reason; + for (MariaDBServer* cand : candidates) + { + if (is_candidate_better(cand, current_best, m_master_gtid_domain, ¤t_best_reason)) + { + // Select the server for promotion, for now. + current_best = cand; + } + } + + // Check if any of the excluded servers would be better than the best candidate. Only print one item. + for (MariaDBServer* excluded_info : valid_but_excluded) + { + const char* excluded_name = excluded_info->name(); + if (current_best == NULL) + { + const char EXCLUDED_ONLY_CAND[] = "Server '%s' is a viable choice for new master, " + "but cannot be selected as it's excluded."; + MXS_WARNING(EXCLUDED_ONLY_CAND, excluded_name); + break; + } + else if (is_candidate_better(excluded_info, current_best, m_master_gtid_domain)) + { + // Print a warning if this server is actually a better candidate than the previous best. + const char EXCLUDED_CAND[] = "Server '%s' is superior to current best candidate '%s', " + "but cannot be selected as it's excluded. This may lead to " + "loss of data if '%s' is ahead of other servers."; + MXS_WARNING(EXCLUDED_CAND, excluded_name, current_best->name(), excluded_name); + break; + } + } + + if (current_best) + { + // If there was a specific reason this server was selected, print it now. If the first candidate + // was chosen (likely all servers were equally good), do not print. + string msg = string_printf("Selected '%s'", current_best->name()); + msg += current_best_reason.empty() ? "." : (" because " + current_best_reason); + MXS_NOTICE("%s", msg.c_str()); + } + return current_best; +} + /** * Is the server in the excluded list * @@ -1133,17 +1196,16 @@ bool MariaDBMonitor::server_is_excluded(const MariaDBServer* server) /** * Is the candidate a better choice for master than the previous best? * - * @param current_best_info Server info of current best choice * @param candidate_info Server info of new candidate + * @param current_best_info Server info of current best choice * @param gtid_domain Which domain to compare * @param reason_out Why is the candidate better than current_best * @return True if candidate is better */ -bool MariaDBMonitor::is_candidate_better(const MariaDBServer* current_best, const MariaDBServer* candidate, +bool MariaDBMonitor::is_candidate_better(const MariaDBServer* candidate, const MariaDBServer* current_best, uint32_t gtid_domain, std::string* reason_out) { - string reason = string("'") + candidate->name() + "' is the best candidate because it "; - + string reason; bool is_better = false; uint64_t cand_io = candidate->m_slave_status[0].gtid_io_pos.get_gtid(gtid_domain).m_sequence; uint64_t curr_io = current_best->m_slave_status[0].gtid_io_pos.get_gtid(gtid_domain).m_sequence; @@ -1151,7 +1213,7 @@ bool MariaDBMonitor::is_candidate_better(const MariaDBServer* current_best, cons if (cand_io > curr_io) { is_better = true; - reason += "has received more events."; + reason = "it has received more events."; } // If io sequences are identical ... else if (cand_io == curr_io) @@ -1162,7 +1224,7 @@ bool MariaDBMonitor::is_candidate_better(const MariaDBServer* current_best, cons if (cand_processed > curr_processed) { is_better = true; - reason += "has processed more events."; + reason = "it has processed more events."; } // If gtid positions are identical ... else if (cand_processed == curr_processed) @@ -1173,7 +1235,7 @@ bool MariaDBMonitor::is_candidate_better(const MariaDBServer* current_best, cons if (cand_updates && !curr_updates) { is_better = true; - reason += "has 'log_slave_updates' on."; + reason = "it has 'log_slave_updates' on."; } // If both have log_slave_updates on ... else if (cand_updates && curr_updates) @@ -1184,7 +1246,7 @@ bool MariaDBMonitor::is_candidate_better(const MariaDBServer* current_best, cons if (cand_disk_ok && !curr_disk_ok) { is_better = true; - reason += "is not low on disk space."; + reason = "it is not low on disk space."; } } } @@ -1197,77 +1259,6 @@ bool MariaDBMonitor::is_candidate_better(const MariaDBServer* current_best, cons return is_better; } -/** - * Check that the given server is a master and it's the only master. - * - * @param suggested_curr_master The server to check, given by user. - * @param error_out On output, error object if function failed. - * @return True if current master seems ok. False, if there is some error with the - * specified current master. - */ -bool MariaDBMonitor::switchover_check_current(const MariaDBServer* suggested_curr_master, - json_t** error_out) const -{ - ss_dassert(suggested_curr_master); - bool server_is_master = false; - MariaDBServer* extra_master = NULL; // A master server which is not the suggested one - - for (auto iter = m_servers.begin(); iter != m_servers.end() && extra_master == NULL; iter++) - { - MariaDBServer* server = *iter; - if (server->is_master()) - { - if (server == suggested_curr_master) - { - server_is_master = true; - } - else - { - extra_master = server; - } - } - } - - if (!server_is_master) - { - PRINT_MXS_JSON_ERROR(error_out, "Server '%s' is not the current master or it's in maintenance.", - suggested_curr_master->name()); - } - else if (extra_master) - { - PRINT_MXS_JSON_ERROR(error_out, "Cluster has an additional master server '%s'.", - extra_master->name()); - } - return server_is_master && !extra_master; -} - -/** - * Check whether specified new master is acceptable. - * - * @param monitored_server The server to check against. - * @param error On output, error object if function failed. - * - * @return True, if suggested new master is a viable promotion candidate. - */ -bool MariaDBMonitor::switchover_check_new(const MariaDBServer* new_master_cand, json_t** error) -{ - bool is_master = new_master_cand->is_master(); - bool is_slave = new_master_cand->is_slave(); - - if (is_master) - { - const char IS_MASTER[] = "Specified new master '%s' is already the current master."; - PRINT_MXS_JSON_ERROR(error, IS_MASTER, new_master_cand->name()); - } - else if (!is_slave) - { - const char NOT_SLAVE[] = "Specified new master '%s' is not a slave."; - PRINT_MXS_JSON_ERROR(error, NOT_SLAVE, new_master_cand->name()); - } - - return !is_master && is_slave; -} - /** * Check that preconditions for a failover are met. * @@ -1345,9 +1336,6 @@ bool MariaDBMonitor::failover_check(string* error_out) */ void MariaDBMonitor::handle_auto_failover() { - const char RE_ENABLE_FMT[] = "To re-enable automatic failover, manually set '%s' to 'true' for monitor " - "'%s' via MaxAdmin or the REST API, or restart MaxScale."; - string cluster_issues; // TODO: Only check this when topology has changed. if (!cluster_supports_failover(&cluster_issues)) @@ -1357,8 +1345,9 @@ void MariaDBMonitor::handle_auto_failover() "%s\n\n" "Automatic failover has been disabled. It should only be enabled after the " "above issues have been resolved."; - string problems = string_printf(PROBLEMS, cluster_issues.c_str()); - string total_msg = problems + " " + string_printf(RE_ENABLE_FMT, CN_AUTO_FAILOVER, m_monitor->name); + string p1 = string_printf(PROBLEMS, cluster_issues.c_str()); + string p2 = string_printf(RE_ENABLE_FMT, "failover", CN_AUTO_FAILOVER, m_monitor->name); + string total_msg = p1 + " " + p2; MXS_ERROR("%s", total_msg.c_str()); m_auto_failover = false; disable_setting(CN_AUTO_FAILOVER); @@ -1426,12 +1415,7 @@ void MariaDBMonitor::handle_auto_failover() failed_master->m_server_base->new_event = false; if (!do_failover(NULL)) { - const string FAILED = "Automatic failover failed, disabling automatic failover."; - string error_msg = FAILED + " " + - string_printf(RE_ENABLE_FMT, CN_AUTO_FAILOVER, m_monitor->name); - MXS_ERROR("%s", error_msg.c_str()); - m_auto_failover = false; - disable_setting(CN_AUTO_FAILOVER); + report_and_disable("failover", CN_AUTO_FAILOVER, &m_auto_failover); } } else @@ -1441,7 +1425,7 @@ void MariaDBMonitor::handle_auto_failover() if (m_warn_failover_precond) { MXS_WARNING("Not performing automatic failover. Will keep retrying with this message " - "suppressed. Errors: \n%s", error_msg.c_str()); + "suppressed. Errors: \n%s", error_msg.c_str()); m_warn_failover_precond = false; } } @@ -1570,71 +1554,130 @@ static void print_redirect_errors(MariaDBServer* first_server, const ServerArray /** * Check cluster and parameters for suitability to switchover. Also writes found servers to output pointers. - * If a server parameter is NULL, the corresponding output parameter is not written to. * - * @param new_master New master requested by the user. Can be null for autoselect. - * @param current_master Current master given by the user. Can be null for autoselect. - * @param new_master_out Where to write found new master. - * @param current_master_out Where to write found current master. - * @param error_out Error output, can be null. - * @return True if cluster is suitable and server parameters were valid and found. + * @param promotion_server The server which should be promoted. If null, monitor will autoselect. + * @param demotion_server The server which should be demoted. Can be null for autoselect. + * @param promotion_target_out Output for promotion target + * @param demotion_target_out Output for demotion target + * @param error_out Error output + * @return True if cluster is suitable and server parameters were valid */ -bool MariaDBMonitor::switchover_check(SERVER* new_master, SERVER* current_master, - MariaDBServer** new_master_out, MariaDBServer** current_master_out, - json_t** error_out) +bool MariaDBMonitor::switchover_prepare(SERVER* promotion_server, SERVER* demotion_server, + MariaDBServer** promotion_target_out, + MariaDBServer** demotion_target_out, + json_t** error_out) { - bool new_master_ok = true, current_master_ok = true; + // Check that both servers are ok if specified, or autoselect them. Demotion target must be checked + // first since the promotion target depends on it. + ss_dassert(promotion_target_out && demotion_target_out && + !*promotion_target_out && !*demotion_target_out); const char NO_SERVER[] = "Server '%s' is not a member of monitor '%s'."; - // Check that both servers are ok if specified. Null is a valid value. - if (new_master) + + MariaDBServer* demotion_target = NULL; + string demotion_msg; + if (demotion_server) { - auto mon_new_master = mon_get_monitored_server(m_monitor, new_master); - if (mon_new_master == NULL) + // Manual select. + MariaDBServer* demotion_candidate = get_server(demotion_server); + if (demotion_candidate == NULL) { - new_master_ok = false; - PRINT_MXS_JSON_ERROR(error_out, NO_SERVER, new_master->name, m_monitor->name); + PRINT_MXS_JSON_ERROR(error_out, NO_SERVER, demotion_server->name, m_monitor->name); + } + else if (!demotion_candidate->can_be_demoted(&demotion_msg)) + { + PRINT_MXS_JSON_ERROR(error_out, "'%s' is not a valid demotion target for switchover: %s", + demotion_candidate->name(), demotion_msg.c_str()); } else { - MariaDBServer* found_new_master = get_server_info(mon_new_master); - if (switchover_check_new(found_new_master, error_out)) + demotion_target = demotion_candidate; + } + } + else + { + // Autoselect current master as demotion target. + if (m_master == NULL) + { + const char msg[] = "Can not autoselect a demotion target for switchover: cluster does " + "not have a master."; + PRINT_MXS_JSON_ERROR(error_out, msg); + } + else if (!m_master->can_be_demoted(&demotion_msg)) + { + const char msg[] = "Can not autoselect '%s' as a demotion target for switchover: %s"; + PRINT_MXS_JSON_ERROR(error_out, msg, m_master->name(), demotion_msg.c_str()); + } + else + { + demotion_target = m_master; + } + } + + MariaDBServer* promotion_target = NULL; + if (demotion_target) + { + string promotion_msg; + if (promotion_server) + { + // Manual select. + MariaDBServer* promotion_candidate = get_server(promotion_server); + if (promotion_candidate == NULL) { - *new_master_out = found_new_master; + PRINT_MXS_JSON_ERROR(error_out, NO_SERVER, promotion_server->name, m_monitor->name); + } + else if (!promotion_candidate->can_be_promoted(demotion_target, &promotion_msg)) + { + const char msg[] = "'%s' is not a valid promotion target for switchover: %s"; + PRINT_MXS_JSON_ERROR(error_out, msg, promotion_candidate->name(), promotion_msg.c_str()); } else { - new_master_ok = false; + promotion_target = promotion_candidate; + } + } + else + { + // Autoselect. More involved than the autoselecting the demotion target. + MariaDBServer* promotion_candidate = switchover_select_promotion(demotion_target, error_out); + if (promotion_candidate) + { + promotion_target = promotion_candidate; + } + else + { + PRINT_MXS_JSON_ERROR(error_out, "Could not autoselect promotion target for switchover."); } } } - if (current_master) + bool gtid_domain_ok = false; + if (m_master_gtid_domain == GTID_DOMAIN_UNKNOWN) { - auto mon_curr_master = mon_get_monitored_server(m_monitor, current_master); - if (mon_curr_master == NULL) - { - current_master_ok = false; - PRINT_MXS_JSON_ERROR(error_out, NO_SERVER, current_master->name, m_monitor->name); - } - else - { - MariaDBServer* found_curr_master = get_server_info(mon_curr_master); - if (switchover_check_current(found_curr_master, error_out)) - { - *current_master_out = get_server_info(mon_curr_master); - } - else - { - current_master_ok = false; - } - } + PRINT_MXS_JSON_ERROR(error_out, "Cluster gtid domain is unknown. Cannot switchover."); + } + else + { + gtid_domain_ok = true; } // Check that all slaves are using gtid-replication. - bool gtid_ok = true; - for (auto iter = m_servers.begin(); iter != m_servers.end(); iter++) + bool gtid_ok = slaves_using_gtid(error_out); + + if (demotion_target && promotion_target && gtid_domain_ok && gtid_ok) + { + *demotion_target_out = demotion_target; + *promotion_target_out = promotion_target; + return true; + } + return false; +} + +bool MariaDBMonitor::slaves_using_gtid(json_t** error_out) +{ + // Check that all slaves are using gtid-replication. + bool gtid_ok = true; + for (MariaDBServer* server : m_servers) { - MariaDBServer* server = *iter; string gtid_error; if (server->is_slave() && !server->uses_gtid(>id_error)) { @@ -1642,8 +1685,7 @@ bool MariaDBMonitor::switchover_check(SERVER* new_master, SERVER* current_master PRINT_MXS_JSON_ERROR(error_out, "%s", gtid_error.c_str()); } } - - return new_master_ok && current_master_ok && gtid_ok; + return gtid_ok; } void MariaDBMonitor::enforce_read_only_on_slaves() @@ -1677,6 +1719,68 @@ void MariaDBMonitor::set_low_disk_slaves_maintenance() !server->is_master() && !server->is_relay_master()) { server->set_status(SERVER_MAINT); + m_cluster_modified = true; } } } + +void MariaDBMonitor::handle_low_disk_space_master() +{ + if (m_master && m_master->is_master() && m_master->is_low_on_disk_space()) + { + if (m_warn_switchover_precond) + { + MXS_WARNING("Master server '%s' is low on disk space. Attempting to switch it with a slave.", + m_master->name()); + } + + // Looks like the master should be swapped out. Before trying it, check if there is even + // a likely valid slave to swap to. + MariaDBServer* demotion_target = NULL; + MariaDBServer* promotion_target = NULL; + auto ok_to_switch = switchover_prepare(NULL, m_master->m_server_base->server, + &promotion_target, &demotion_target, NULL); + + if (ok_to_switch) + { + m_warn_switchover_precond = true; + bool switched = do_switchover(demotion_target, promotion_target, NULL); + if (switched) + { + MXS_NOTICE("Switchover %s -> %s performed.", demotion_target->name(), promotion_target->name()); + } + else + { + report_and_disable("switchover", CN_SWITCHOVER_ON_LOW_DISK_SPACE, + &m_switchover_on_low_disk_space); + } + } + else + { + // Switchover was not attempted because of errors, however these errors are not permanent. + // Servers were not modified, so it's ok to try this again. + if (m_warn_switchover_precond) + { + MXS_WARNING("Not performing automatic switchover. Will keep retrying with this message " + "suppressed."); + m_warn_switchover_precond = false; + } + } + } + else + { + m_warn_switchover_precond = true; + } +} + +void MariaDBMonitor::report_and_disable(const string& operation, const string& setting_name, + bool* setting_var) +{ + string p1 = string_printf("Automatic %s failed, disabling automatic %s.", operation.c_str(), + operation.c_str()); + string p2 = string_printf(RE_ENABLE_FMT, operation.c_str(), setting_name.c_str(), m_monitor->name); + string error_msg = p1 + " " + p2; + MXS_ERROR("%s", error_msg.c_str()); + *setting_var = false; + disable_setting(setting_name.c_str()); +} \ No newline at end of file diff --git a/server/modules/monitor/mariadbmon/mariadbmon.cc b/server/modules/monitor/mariadbmon/mariadbmon.cc index f4355004c..b946dc30f 100644 --- a/server/modules/monitor/mariadbmon/mariadbmon.cc +++ b/server/modules/monitor/mariadbmon/mariadbmon.cc @@ -31,9 +31,11 @@ #include "../../../core/internal/monitor.h" using std::string; +using maxscale::string_printf; // Config parameter names const char * const CN_AUTO_FAILOVER = "auto_failover"; +const char * const CN_SWITCHOVER_ON_LOW_DISK_SPACE = "switchover_on_low_disk_space"; const char * const CN_PROMOTION_SQL_FILE = "promotion_sql_file"; const char * const CN_DEMOTION_SQL_FILE = "demotion_sql_file"; @@ -42,7 +44,6 @@ static const char CN_FAILCOUNT[] = "failcount"; static const char CN_ENFORCE_READONLY[] = "enforce_read_only_slaves"; static const char CN_NO_PROMOTE_SERVERS[] = "servers_no_promotion"; static const char CN_FAILOVER_TIMEOUT[] = "failover_timeout"; -static const char CN_SWITCHOVER_ON_LOW_DISK_SPACE[] = "switchover_on_low_disk_space"; static const char CN_SWITCHOVER_TIMEOUT[] = "switchover_timeout"; static const char CN_DETECT_STANDALONE_MASTER[] = "detect_standalone_master"; static const char CN_MAINTENANCE_ON_LOW_DISK_SPACE[] = "maintenance_on_low_disk_space"; @@ -63,9 +64,9 @@ MariaDBMonitor::MariaDBMonitor(MXS_MONITOR* monitor) , m_external_master_port(PORT_UNKNOWN) , m_cluster_topology_changed(true) , m_cluster_modified(false) - , m_switchover_on_low_disk_space(false) , m_log_no_master(true) , m_warn_failover_precond(true) + , m_warn_switchover_precond(true) , m_warn_cannot_rejoin(true) , m_warn_current_master_invalid(true) , m_warn_have_better_master(true) @@ -143,6 +144,23 @@ MariaDBServer* MariaDBMonitor::get_server(int64_t id) return (found != m_servers_by_id.end()) ? (*found).second : NULL; } +/** + * Get the equivalent MariaDBServer. + * + * @param server Which server to search for + * @return MariaDBServer if found, NULL otherwise + */ +MariaDBServer* MariaDBMonitor::get_server(SERVER* server) +{ + MariaDBServer* found = NULL; + auto mon_server = mon_get_monitored_server(m_monitor, server); + if (mon_server) + { + found = get_server_info(mon_server); + } + return found; +} + bool MariaDBMonitor::set_replication_credentials(const MXS_CONFIG_PARAMETER* params) { bool rval = false; @@ -486,11 +504,6 @@ void MariaDBMonitor::tick() measure_replication_lag(); } - if (m_maintenance_on_low_disk_space) - { - set_low_disk_slaves_maintenance(); - } - // Update shared status. The next functions read the shared status. TODO: change the following // functions to read "pending_status" instead. for (auto mon_srv = m_monitor->monitored_servers; mon_srv; mon_srv = mon_srv->next) @@ -555,6 +568,19 @@ void MariaDBMonitor::process_state_changes() { enforce_read_only_on_slaves(); } + + /* Set low disk space slaves to maintenance. + */ + if (m_maintenance_on_low_disk_space && !m_cluster_modified) + { + set_low_disk_slaves_maintenance(); + } + + /* Check if the master server is on low disk space and act on it. */ + if (m_switchover_on_low_disk_space && !m_cluster_modified) + { + handle_low_disk_space_master(); + } } } @@ -1037,12 +1063,13 @@ bool MariaDBMonitor::execute_manual_command(std::function command, return rval; } -bool MariaDBMonitor::run_manual_switchover(SERVER* new_master, SERVER* current_master, json_t** error_out) +bool MariaDBMonitor::run_manual_switchover(SERVER* promotion_server, SERVER* demotion_server, + json_t** error_out) { bool rval = false; - bool send_ok = execute_manual_command([this, &rval, new_master, current_master, error_out]() + bool send_ok = execute_manual_command([this, &rval, promotion_server, demotion_server, error_out]() { - rval = manual_switchover(new_master, current_master, error_out); + rval = manual_switchover(promotion_server, demotion_server, error_out); }, error_out); return send_ok && rval; } @@ -1092,9 +1119,9 @@ bool handle_manual_switchover(const MODULECMD_ARG* args, json_t** error_out) { MXS_MONITOR* mon = args->argv[0].value.monitor; auto handle = static_cast(mon->instance); - SERVER* new_master = (args->argc >= 2) ? args->argv[1].value.server : NULL; - SERVER* current_master = (args->argc == 3) ? args->argv[2].value.server : NULL; - rval = handle->run_manual_switchover(new_master, current_master, error_out); + SERVER* promotion_server = (args->argc >= 2) ? args->argv[1].value.server : NULL; + SERVER* demotion_server = (args->argc == 3) ? args->argv[2].value.server : NULL; + rval = handle->run_manual_switchover(promotion_server, demotion_server, error_out); } return rval; } diff --git a/server/modules/monitor/mariadbmon/mariadbmon.hh b/server/modules/monitor/mariadbmon/mariadbmon.hh index 59b154b61..39af85fa8 100644 --- a/server/modules/monitor/mariadbmon/mariadbmon.hh +++ b/server/modules/monitor/mariadbmon/mariadbmon.hh @@ -25,6 +25,7 @@ #include "mariadbserver.hh" extern const char * const CN_AUTO_FAILOVER; +extern const char * const CN_SWITCHOVER_ON_LOW_DISK_SPACE; extern const char * const CN_PROMOTION_SQL_FILE; extern const char * const CN_DEMOTION_SQL_FILE; @@ -177,6 +178,7 @@ private: bool m_warn_no_valid_outside_cycle; /**< Log a warning when a replication topology has no valid master * outside of a cycle. */ bool m_warn_failover_precond; /**< Print failover preconditions error message? */ + bool m_warn_switchover_precond; /**< Print switchover preconditions error message? */ bool m_warn_cannot_rejoin; /**< Print warning if auto_rejoin fails because of invalid gtid:s? */ bool m_warn_current_master_invalid; /**< Print warning if current master is not valid? */ bool m_warn_have_better_master; /**< Print warning if the current master is not the best one? */ @@ -190,6 +192,7 @@ private: bool set_replication_credentials(const MXS_CONFIG_PARAMETER* params); MariaDBServer* get_server_info(MXS_MONITORED_SERVER* db); MariaDBServer* get_server(int64_t id); + MariaDBServer* get_server(SERVER* server); bool execute_manual_command(GenericFunction command, json_t** error_out); std::string diagnostics_to_string() const; json_t* diagnostics_to_json() const; @@ -218,21 +221,21 @@ private: void update_master_cycle_info(); void set_low_disk_slaves_maintenance(); void assign_new_master(MariaDBServer* new_master); + bool slaves_using_gtid(json_t** error_out); // Switchover methods bool manual_switchover(SERVER* new_master, SERVER* current_master, json_t** error_out); - bool switchover_check(SERVER* new_master, SERVER* current_master, + bool switchover_prepare(SERVER* new_master, SERVER* current_master, MariaDBServer** new_master_out, MariaDBServer** current_master_out, json_t** error_out); - bool switchover_check_new(const MariaDBServer* new_master_cand, json_t** error); - bool switchover_check_current(const MariaDBServer* suggested_curr_master, json_t** error_out) const; - bool do_switchover(MariaDBServer** current_master, MariaDBServer** new_master, json_t** err_out); + bool do_switchover(MariaDBServer* demotion_target, MariaDBServer* promotion_target, json_t** error_out); bool switchover_check_preferred_master(MariaDBServer* preferred, json_t** err_out); bool switchover_demote_master(MariaDBServer* current_master, json_t** err_out); bool switchover_wait_slaves_catchup(const ServerArray& slaves, const GtidList& gtid, int total_timeout, json_t** err_out); bool switchover_start_slave(MariaDBServer* old_master, MariaDBServer* new_master); + void handle_low_disk_space_master(); // Failover methods bool manual_failover(json_t** output); @@ -252,8 +255,9 @@ private: // Methods common to failover/switchover/rejoin MariaDBServer* select_new_master(ServerArray* slaves_out, json_t** err_out); + MariaDBServer* switchover_select_promotion(MariaDBServer* current_master, json_t** err_out); bool server_is_excluded(const MariaDBServer* server); - bool is_candidate_better(const MariaDBServer* current_best, const MariaDBServer* candidate, + bool is_candidate_better(const MariaDBServer* candidate, const MariaDBServer* current_best, uint32_t gtid_domain, std::string* reason_out = NULL); bool promote_new_master(MariaDBServer* new_master, json_t** err_out); int redirect_slaves(MariaDBServer* new_master, const ServerArray& slaves, @@ -262,6 +266,7 @@ private: bool start_external_replication(MariaDBServer* new_master, json_t** err_out); bool wait_cluster_stabilization(MariaDBServer* new_master, const ServerArray& slaves, int seconds_remaining); + void report_and_disable(const std::string& operation, const std::string& setting_name, bool* setting_var); // Other methods void disable_setting(const std::string& setting); diff --git a/server/modules/monitor/mariadbmon/mariadbmon_common.cc b/server/modules/monitor/mariadbmon/mariadbmon_common.cc index d2fbcf5f1..10a3a6070 100644 --- a/server/modules/monitor/mariadbmon/mariadbmon_common.cc +++ b/server/modules/monitor/mariadbmon/mariadbmon_common.cc @@ -19,3 +19,15 @@ const int64_t SERVER_ID_UNKNOWN = -1; const int64_t GTID_DOMAIN_UNKNOWN = -1; /** Default port */ const int PORT_UNKNOWN = 0; + +using std::string; + +DelimitedPrinter::DelimitedPrinter(const string& separator) + : m_separator(separator) +{} + +void DelimitedPrinter::cat(string& target, const string& addition) +{ + target += m_current_separator + addition; + m_current_separator = m_separator; +} diff --git a/server/modules/monitor/mariadbmon/mariadbmon_common.hh b/server/modules/monitor/mariadbmon/mariadbmon_common.hh index 24e350617..be2e76cee 100644 --- a/server/modules/monitor/mariadbmon/mariadbmon_common.hh +++ b/server/modules/monitor/mariadbmon/mariadbmon_common.hh @@ -20,6 +20,8 @@ #define MXS_MODULE_NAME "mariadbmon" #include + +#include #include /** Utility macro for printing both MXS_ERROR and json error */ @@ -35,3 +37,25 @@ extern const int64_t SERVER_ID_UNKNOWN; extern const int64_t GTID_DOMAIN_UNKNOWN; extern const int PORT_UNKNOWN; + +// Helper class for concatenating strings with a delimiter. +class DelimitedPrinter +{ +private: + DelimitedPrinter(const DelimitedPrinter&) = delete; + DelimitedPrinter& operator = (const DelimitedPrinter&) = delete; + DelimitedPrinter() = delete; +public: + DelimitedPrinter(const std::string& separator); + + /** + * Add to string. + * + * @param target String to modify + * @param addition String to add. The delimiter is printed before the addition. + */ + void cat(std::string& target, const std::string& addition); +private: + const std::string m_separator; + std::string m_current_separator; +}; diff --git a/server/modules/monitor/mariadbmon/mariadbserver.cc b/server/modules/monitor/mariadbmon/mariadbserver.cc index 8b29daaa6..d14051496 100644 --- a/server/modules/monitor/mariadbmon/mariadbserver.cc +++ b/server/modules/monitor/mariadbmon/mariadbserver.cc @@ -276,11 +276,12 @@ bool MariaDBServer::update_gtids(string* errmsg_out) return rval; } -bool MariaDBServer::update_replication_settings() +bool MariaDBServer::update_replication_settings(std::string* error_out) { static const string query = "SELECT @@gtid_strict_mode, @@log_bin, @@log_slave_updates;"; + string query_error; bool rval = false; - auto result = execute_query(query); + auto result = execute_query(query, &query_error); if (result.get() != NULL && result->next_row()) { m_rpl_settings.gtid_strict_mode = result->get_bool(0); @@ -288,6 +289,10 @@ bool MariaDBServer::update_replication_settings() m_rpl_settings.log_slave_updates = result->get_bool(2); rval = true; } + else if (error_out) + { + *error_out = query_error; + } return rval; } @@ -349,7 +354,7 @@ bool MariaDBServer::read_server_variables(string* errmsg_out) return rval; } -bool MariaDBServer::check_replication_settings(print_repl_warnings_t print_warnings) +bool MariaDBServer::check_replication_settings(print_repl_warnings_t print_warnings) const { bool rval = true; const char* servername = name(); @@ -433,6 +438,11 @@ bool MariaDBServer::wait_until_gtid(const GtidList& target, int timeout, json_t* return gtid_reached; } +bool MariaDBServer::binlog_on() const +{ + return m_rpl_settings.log_bin; +} + bool MariaDBServer::is_master() const { return status_is_master(m_server_base->pending_status); @@ -468,6 +478,11 @@ bool MariaDBServer::is_relay_master() const return status_is_relay(m_server_base->pending_status); } +bool MariaDBServer::is_low_on_disk_space() const +{ + return status_is_disk_space_exhausted(m_server_base->pending_status); +} + bool MariaDBServer::has_status(uint64_t bits) const { return (m_server_base->pending_status & bits) == bits; @@ -945,6 +960,96 @@ bool MariaDBServer::sstatus_arrays_topology_equal(const SlaveStatusArray& lhs, c return rval; } +/** + * Check if the server can be demoted. + * + * @param reason_out Output for the reason server cannot be demoted + * @return True, if suggested new master is a viable demotion candidate + */ +bool MariaDBServer::can_be_demoted(string* reason_out) +{ + bool demotable = false; + string reason; + string query_error; + + // TODO: Add relay server support + if (!is_master()) + { + reason = "it is not the current master or it is in maintenance."; + } + else if (!update_replication_settings(&query_error)) + { + reason = string_printf("it could not be queried: '%s'.", query_error.c_str()); + } + else if (!binlog_on()) + { + reason = "its binary log is disabled."; + } + else if (m_gtid_binlog_pos.empty()) + { + reason = "it does not have a 'gtid_binlog_pos'."; + } + else + { + demotable = true; + } + + if (!demotable && reason_out) + { + *reason_out = reason; + } + return demotable; +} + +/** + * Check if the server can be promoted. + * + * @param demotion_target The server this should be promoted to + * @param reason_out Output for the reason server cannot be promoted + * @return True, if suggested new master is a viable promotion candidate + */ +bool MariaDBServer::can_be_promoted(const MariaDBServer* demotion_target, std::string* reason_out) +{ + bool promotable = false; + string reason; + string query_error; + + if (is_master()) + { + reason = "it is already the master."; + } + // TODO: Check that the correct slave connection is working properly in case of switchover. + // For failover the connection may be in CONNECTING-stage. + else if (!is_replicating_from(demotion_target)) + { + reason = string_printf("it is not replicating from '%s'.", demotion_target->name()); + } + else if (!update_replication_settings(&query_error)) + { + string_printf("it could not be queried: '%s'.", query_error.c_str()); + } + else if (!binlog_on()) + { + reason = "its binary log is disabled."; + } + else + { + promotable = true; + } + + if (!promotable && reason_out) + { + *reason_out = reason; + } + return promotable; +} + +bool MariaDBServer::is_replicating_from(const MariaDBServer* target) +{ + // Not properly implemented yet, TODO + return is_slave(); +} + string SlaveStatus::to_string() const { using std::setw; diff --git a/server/modules/monitor/mariadbmon/mariadbserver.hh b/server/modules/monitor/mariadbmon/mariadbserver.hh index 54fbae524..c792db6d5 100644 --- a/server/modules/monitor/mariadbmon/mariadbserver.hh +++ b/server/modules/monitor/mariadbmon/mariadbserver.hh @@ -149,7 +149,9 @@ public: * read-only. */ NodeData m_node; /**< Replication topology data */ SlaveStatusArray m_slave_status; /**< Data returned from SHOW SLAVE STATUS */ - ReplicationSettings m_rpl_settings; /**< Miscellaneous replication related settings */ + ReplicationSettings m_rpl_settings; /**< Miscellaneous replication related settings. These are not + * normally queried from the server, call + * 'update_replication_settings' before use. */ bool m_print_update_errormsg;/**< Should an update error be printed. */ MariaDBServer(MXS_MONITORED_SERVER* monitored_server, int config_index); @@ -196,9 +198,10 @@ public: /** * Query a few miscellaneous replication settings. * + * @param error_out Query error output * @return True on success */ - bool update_replication_settings(); + bool update_replication_settings(std::string* error_out = NULL); /** * Query and save server_id, read_only and (if 10.X) gtid_domain_id. @@ -214,7 +217,7 @@ public: * @param print_on Print warnings or not * @return True if log_bin is on */ - bool check_replication_settings(print_repl_warnings_t print_warnings = WARNINGS_ON); + bool check_replication_settings(print_repl_warnings_t print_warnings = WARNINGS_ON) const; /** * Wait until server catches up to the target gtid. Only considers gtid domains common to this server @@ -228,6 +231,21 @@ public: */ bool wait_until_gtid(const GtidList& target, int timeout, json_t** err_out); + /** + * Is the server replicating (or trying to) from the target server. + * + * @param target Immediate master or relay server + * @return True if replicating + */ + bool is_replicating_from(const MariaDBServer* target); + + /** + * Is binary log on? 'update_replication_settings' should be ran before this function to query the data. + * + * @return True if server has binary log enabled + */ + bool binlog_on() const; + /** * Check if server is a running master. * @@ -273,6 +291,11 @@ public: */ bool is_relay_master() const; + /** + * Is the server low on disk space? + */ + bool is_low_on_disk_space() const; + /** * Check if server has the given bits on in 'pending_status'. * @@ -370,6 +393,23 @@ public: */ bool failover_wait_relay_log(int seconds_remaining, json_t** err_out); + /** + * Is the server a valid demotion target? + * + * @param reason_out Output explaining why server cannot be demoted + * @return True if server can be demoted by switchover + */ + bool can_be_demoted(std::string* reason_out); + + /** + * Is the server a valid promotion target? + * + * @param demotion_target Which server would be demoted + * @param reason_out Output explaining why server cannot be promoted + * @return True if server can be promoted by switchover + */ + bool can_be_promoted(const MariaDBServer* demotion_target, std::string* reason_out); + /** * Read the file contents and send them as sql queries to the server. Any data returned by the queries is * discarded.