/* * Copyright (c) 2018 MariaDB Corporation Ab * * Use of this software is governed by the Business Source License included * in the LICENSE.TXT file and at www.mariadb.com/bsl11. * * Change Date: 2020-01-01 * * On the date above, in accordance with the Business Source License, use * of this software will be governed by version 2 or later of the General * Public License. */ #include "mariadbmon.hh" #include #include static int add_slave_to_master(long *slaves_list, int list_size, long node_id); static void read_server_variables(MXS_MONITORED_SERVER* database, MySqlServerInfo* serv_info); static bool report_version_err = true; /** * Build the replication tree for a MySQL 5.1 cluster * * This function queries each server with SHOW SLAVE HOSTS to determine which servers have slaves replicating * from them. * * @return Lowest server ID master in the monitor */ MXS_MONITORED_SERVER* MariaDBMonitor::build_mysql51_replication_tree() { /** Column positions for SHOW SLAVE HOSTS */ const size_t SLAVE_HOSTS_SERVER_ID = 0; const size_t SLAVE_HOSTS_HOSTNAME = 1; const size_t SLAVE_HOSTS_PORT = 2; MXS_MONITORED_SERVER* database = m_monitor_base->monitored_servers; MXS_MONITORED_SERVER *ptr, *rval = NULL; int i; while (database) { bool ismaster = false; MYSQL_RES* result; MYSQL_ROW row; int nslaves = 0; if (database->con) { if (mxs_mysql_query(database->con, "SHOW SLAVE HOSTS") == 0 && (result = mysql_store_result(database->con)) != NULL) { if (mysql_field_count(database->con) < 4) { mysql_free_result(result); MXS_ERROR("\"SHOW SLAVE HOSTS\" " "returned less than the expected amount of columns. " "Expected 4 columns."); return NULL; } if (mysql_num_rows(result) > 0) { ismaster = true; while (nslaves < MAX_NUM_SLAVES && (row = mysql_fetch_row(result))) { /* get Slave_IO_Running and Slave_SQL_Running values*/ database->server->slaves[nslaves] = atol(row[SLAVE_HOSTS_SERVER_ID]); nslaves++; MXS_DEBUG("Found slave at %s:%s", row[SLAVE_HOSTS_HOSTNAME], row[SLAVE_HOSTS_PORT]); } database->server->slaves[nslaves] = 0; } mysql_free_result(result); } else { mon_report_query_error(database); } /* Set the Slave Role */ if (ismaster) { master = database; MXS_DEBUG("Master server found at [%s]:%d with %d slaves", database->server->name, database->server->port, nslaves); monitor_set_pending_status(database, SERVER_MASTER); database->server->depth = 0; // Add Depth 0 for Master if (rval == NULL || rval->server->node_id > database->server->node_id) { rval = database; } } } database = database->next; } database = m_monitor_base->monitored_servers; /** Set master server IDs */ while (database) { ptr = m_monitor_base->monitored_servers; while (ptr) { for (i = 0; ptr->server->slaves[i]; i++) { if (ptr->server->slaves[i] == database->server->node_id) { database->server->master_id = ptr->server->node_id; database->server->depth = 1; // Add Depth 1 for Slave break; } } ptr = ptr->next; } if (SERVER_IS_SLAVE(database->server) && (database->server->master_id <= 0 || database->server->master_id != master->server->node_id)) { monitor_set_pending_status(database, SERVER_SLAVE); monitor_set_pending_status(database, SERVER_SLAVE_OF_EXTERNAL_MASTER); } database = database->next; } return rval; } /** * This function computes the replication tree from a set of monitored servers and returns the root server * with SERVER_MASTER bit. The tree is computed even for servers in 'maintenance' mode. * * @param num_servers The number of servers monitored * @return The server at root level with SERVER_MASTER bit */ MXS_MONITORED_SERVER* MariaDBMonitor::get_replication_tree(int num_servers) { MXS_MONITORED_SERVER *ptr; MXS_MONITORED_SERVER *backend; SERVER *current; int depth = 0; long node_id; int root_level; ptr = m_monitor_base->monitored_servers; root_level = num_servers; while (ptr) { /* The server could be in SERVER_IN_MAINT * that means SERVER_IS_RUNNING returns 0 * Let's check only for SERVER_IS_DOWN: server is not running */ if (SERVER_IS_DOWN(ptr->server)) { ptr = ptr->next; continue; } depth = 0; current = ptr->server; node_id = current->master_id; /** Either this node doesn't replicate from a master or the master * where it replicates from is not configured to this monitor. */ if (node_id < 1 || getServerByNodeId(m_monitor_base->monitored_servers, node_id) == NULL) { MXS_MONITORED_SERVER *find_slave; find_slave = getSlaveOfNodeId(m_monitor_base->monitored_servers, current->node_id, ACCEPT_DOWN); if (find_slave == NULL) { current->depth = -1; ptr = ptr->next; continue; } else { current->depth = 0; } } else { depth++; } while (depth <= num_servers) { /* set the root master at lowest depth level */ if (current->depth > -1 && current->depth < root_level) { root_level = current->depth; master = ptr; } backend = getServerByNodeId(m_monitor_base->monitored_servers, node_id); if (backend) { node_id = backend->server->master_id; } else { node_id = -1; } if (node_id > 0) { current->depth = depth + 1; depth++; } else { MXS_MONITORED_SERVER *master_cand; current->depth = depth; master_cand = getServerByNodeId(m_monitor_base->monitored_servers, current->master_id); if (master_cand && master_cand->server && master_cand->server->node_id > 0) { add_slave_to_master(master_cand->server->slaves, sizeof(master_cand->server->slaves), current->node_id); master_cand->server->depth = current->depth - 1; if (master && master_cand->server->depth < master->server->depth) { /** A master with a lower depth was found, remove the master status from the previous master. */ monitor_clear_pending_status(master, SERVER_MASTER); master = master_cand; } MySqlServerInfo* info = get_server_info(master_cand); if (SERVER_IS_RUNNING(master_cand->server)) { /** Only set the Master status if read_only is disabled */ monitor_set_pending_status(master_cand, info->read_only ? SERVER_SLAVE : SERVER_MASTER); } } else { if (current->master_id > 0) { monitor_set_pending_status(ptr, SERVER_SLAVE); monitor_set_pending_status(ptr, SERVER_SLAVE_OF_EXTERNAL_MASTER); } } break; } } ptr = ptr->next; } /* * Return the root master */ if (master != NULL) { /* If the root master is in MAINT, return NULL */ if (SERVER_IN_MAINT(master->server)) { return NULL; } else { return master; } } else { return NULL; } } /******* * This function add a slave id into the slaves server field * of its master server * * @param slaves_list The slave list array of the master server * @param list_size The size of the slave list * @param node_id The node_id of the slave to be inserted * @return 1 for inserted value and 0 otherwise */ static int add_slave_to_master(long *slaves_list, int list_size, long node_id) { for (int i = 0; i < list_size; i++) { if (slaves_list[i] == 0) { slaves_list[i] = node_id; return 1; } } return 0; } /** * Fetch a node by node_id * * @param ptr The list of servers to monitor * @param node_id The server_id to fetch * * @return The server with the required server_id */ MXS_MONITORED_SERVER* getServerByNodeId(MXS_MONITORED_SERVER *ptr, long node_id) { SERVER *current; while (ptr) { current = ptr->server; if (current->node_id == node_id) { return ptr; } ptr = ptr->next; } return NULL; } /** * Fetch a slave node from a node_id * * @param ptr The list of servers to monitor * @param node_id The server_id to fetch * @param slave_down_setting Whether to accept or reject slaves which are down * @return The slave server of this node_id */ MXS_MONITORED_SERVER* getSlaveOfNodeId(MXS_MONITORED_SERVER *ptr, long node_id, slave_down_setting_t slave_down_setting) { SERVER *current; while (ptr) { current = ptr->server; if (current->master_id == node_id && (slave_down_setting == ACCEPT_DOWN || !SERVER_IS_DOWN(current))) { return ptr; } ptr = ptr->next; } return NULL; } bool MariaDBMonitor::do_show_slave_status(MySqlServerInfo* serv_info, MXS_MONITORED_SERVER* database) { /** Column positions for SHOW SLAVE STATUS */ const size_t MYSQL55_STATUS_MASTER_LOG_POS = 5; const size_t MYSQL55_STATUS_MASTER_LOG_FILE = 6; const size_t MYSQL55_STATUS_IO_RUNNING = 10; const size_t MYSQL55_STATUS_SQL_RUNNING = 11; const size_t MYSQL55_STATUS_MASTER_ID = 39; /** Column positions for SHOW SLAVE STATUS */ const size_t MARIA10_STATUS_MASTER_LOG_FILE = 7; const size_t MARIA10_STATUS_MASTER_LOG_POS = 8; const size_t MARIA10_STATUS_IO_RUNNING = 12; const size_t MARIA10_STATUS_SQL_RUNNING = 13; const size_t MARIA10_STATUS_MASTER_ID = 41; bool rval = true; unsigned int columns; int i_slave_io_running, i_slave_sql_running, i_read_master_log_pos, i_master_server_id, i_master_log_file; const char *query; mysql_server_version server_version = serv_info->version; if (server_version == MYSQL_SERVER_VERSION_100) { columns = 42; query = "SHOW ALL SLAVES STATUS"; i_slave_io_running = MARIA10_STATUS_IO_RUNNING; i_slave_sql_running = MARIA10_STATUS_SQL_RUNNING; i_master_log_file = MARIA10_STATUS_MASTER_LOG_FILE; i_read_master_log_pos = MARIA10_STATUS_MASTER_LOG_POS; i_master_server_id = MARIA10_STATUS_MASTER_ID; } else { columns = server_version == MYSQL_SERVER_VERSION_55 ? 40 : 38; query = "SHOW SLAVE STATUS"; i_slave_io_running = MYSQL55_STATUS_IO_RUNNING; i_slave_sql_running = MYSQL55_STATUS_SQL_RUNNING; i_master_log_file = MYSQL55_STATUS_MASTER_LOG_FILE; i_read_master_log_pos = MYSQL55_STATUS_MASTER_LOG_POS; i_master_server_id = MYSQL55_STATUS_MASTER_ID; } MYSQL_RES* result; int64_t master_server_id = SERVER_ID_UNKNOWN; int nconfigured = 0; int nrunning = 0; if (mxs_mysql_query(database->con, query) == 0 && (result = mysql_store_result(database->con)) != NULL) { if (mysql_field_count(database->con) < columns) { mysql_free_result(result); MXS_ERROR("\"%s\" returned less than the expected amount of columns. " "Expected %u columns.", query, columns); return false; } MYSQL_ROW row = mysql_fetch_row(result); if (row) { serv_info->slave_configured = true; do { /* get Slave_IO_Running and Slave_SQL_Running values*/ serv_info->slave_status.slave_io_running = strncmp(row[i_slave_io_running], "Yes", 3) == 0; serv_info->slave_status.slave_sql_running = strncmp(row[i_slave_sql_running], "Yes", 3) == 0; if (serv_info->slave_status.slave_io_running && serv_info->slave_status.slave_sql_running) { if (nrunning == 0) { /** Only check binlog name for the first running slave */ uint64_t read_master_log_pos = atol(row[i_read_master_log_pos]); char* master_log_file = row[i_master_log_file]; if (serv_info->slave_status.master_log_file != master_log_file || read_master_log_pos != serv_info->slave_status.read_master_log_pos) { // IO thread is reading events from the master serv_info->latest_event = time(NULL); } serv_info->slave_status.master_log_file = master_log_file; serv_info->slave_status.read_master_log_pos = read_master_log_pos; } nrunning++; } /* If Slave_IO_Running = Yes, assign the master_id to current server: this allows building * the replication tree, slaves ids will be added to master(s) and we will have at least the * root master server. * Please note, there could be no slaves at all if Slave_SQL_Running == 'No' */ const char* last_io_errno = mxs_mysql_get_value(result, row, "Last_IO_Errno"); int io_errno = last_io_errno ? atoi(last_io_errno) : 0; const int connection_errno = 2003; if ((io_errno == 0 || io_errno == connection_errno) && server_version != MYSQL_SERVER_VERSION_51) { /* Get Master_Server_Id */ master_server_id = scan_server_id(row[i_master_server_id]); } if (server_version == MYSQL_SERVER_VERSION_100) { const char* beats = mxs_mysql_get_value(result, row, "Slave_received_heartbeats"); const char* period = mxs_mysql_get_value(result, row, "Slave_heartbeat_period"); const char* using_gtid = mxs_mysql_get_value(result, row, "Using_Gtid"); const char* master_host = mxs_mysql_get_value(result, row, "Master_Host"); const char* master_port = mxs_mysql_get_value(result, row, "Master_Port"); const char* last_io_error = mxs_mysql_get_value(result, row, "Last_IO_Error"); const char* last_sql_error = mxs_mysql_get_value(result, row, "Last_SQL_Error"); ss_dassert(beats && period && using_gtid && master_host && master_port && last_io_error && last_sql_error); serv_info->slave_status.master_host = master_host; serv_info->slave_status.master_port = atoi(master_port); serv_info->slave_status.last_error = *last_io_error ? last_io_error : (*last_sql_error ? last_sql_error : ""); int heartbeats = atoi(beats); if (serv_info->slave_heartbeats < heartbeats) { serv_info->latest_event = time(NULL); serv_info->slave_heartbeats = heartbeats; serv_info->heartbeat_period = atof(period); } if (m_master_gtid_domain >= 0 && (strcmp(using_gtid, "Current_Pos") == 0 || strcmp(using_gtid, "Slave_Pos") == 0)) { const char* gtid_io_pos = mxs_mysql_get_value(result, row, "Gtid_IO_Pos"); ss_dassert(gtid_io_pos); serv_info->slave_status.gtid_io_pos = gtid_io_pos[0] != '\0' ? Gtid(gtid_io_pos, m_master_gtid_domain) : Gtid(); } else { serv_info->slave_status.gtid_io_pos = Gtid(); } } nconfigured++; row = mysql_fetch_row(result); } while (row); } else { /** Query returned no rows, replication is not configured */ serv_info->slave_configured = false; serv_info->slave_heartbeats = 0; serv_info->slave_status = SlaveStatusInfo(); } serv_info->slave_status.master_server_id = master_server_id; mysql_free_result(result); } else { mon_report_query_error(database); } serv_info->n_slaves_configured = nconfigured; serv_info->n_slaves_running = nrunning; return rval; } /** * @brief A node in a graph */ struct graph_node { int index; int lowest_index; int cycle; bool active; struct graph_node *parent; MySqlServerInfo *info; MXS_MONITORED_SERVER *db; }; /** * @brief Visit a node in the graph * * This function is the main function used to determine whether the node is a * part of a cycle. It is an implementation of the Tarjan's strongly connected * component algorithm. All one node cycles are ignored since normal * master-slave monitoring handles that. * * Tarjan's strongly connected component algorithm: * * https://en.wikipedia.org/wiki/Tarjan%27s_strongly_connected_components_algorithm */ static void visit_node(struct graph_node *node, struct graph_node **stack, int *stacksize, int *index, int *cycle) { /** Assign an index to this node */ node->lowest_index = node->index = *index; node->active = true; *index += 1; stack[*stacksize] = node; *stacksize += 1; if (node->parent == NULL) { /** This node does not connect to another node, it can't be a part of a cycle */ node->lowest_index = -1; } else if (node->parent->index == 0) { /** Node has not been visited */ visit_node(node->parent, stack, stacksize, index, cycle); if (node->parent->lowest_index < node->lowest_index) { /** The parent connects to a node with a lower index, this node could be a part of a cycle. */ node->lowest_index = node->parent->lowest_index; } } else if (node->parent->active) { /** This node could be a root node of the cycle */ if (node->parent->index < node->lowest_index) { /** Root node found */ node->lowest_index = node->parent->index; } } else { /** Node connects to an already connected cycle, it can't be a part of it */ node->lowest_index = -1; } if (node->active && node->parent && node->lowest_index > 0) { if (node->lowest_index == node->index && node->lowest_index == node->parent->lowest_index) { /** * Found a multi-node cycle from the graph. The cycle is formed from the * nodes with a lowest_index value equal to the lowest_index value of the * current node. Rest of the nodes on the stack are not part of a cycle * and can be discarded. */ *cycle += 1; while (*stacksize > 0) { struct graph_node *top = stack[(*stacksize) - 1]; top->active = false; if (top->lowest_index == node->lowest_index) { top->cycle = *cycle; } *stacksize -= 1; } } } else { /** Pop invalid nodes off the stack */ node->active = false; if (*stacksize > 0) { *stacksize -= 1; } } } /** * @brief Find the strongly connected components in the replication tree graph * * Each replication cluster is a directed graph made out of replication * trees. If this graph has strongly connected components (more generally * cycles), it is considered a multi-master cluster due to the fact that there * are multiple nodes where the data can originate. * * Detecting the cycles in the graph allows this monitor to better understand * the relationships between the nodes. All nodes that are a part of a cycle can * be labeled as master nodes. This information will later be used to choose the * right master where the writes should go. * * This function also populates the MYSQL_SERVER_INFO structures group * member. Nodes in a group get a positive group ID where the nodes not in a * group get a group ID of 0. */ void find_graph_cycles(MariaDBMonitor *handle, MXS_MONITORED_SERVER *database, int nservers) { struct graph_node graph[nservers]; struct graph_node *stack[nservers]; int nodes = 0; for (MXS_MONITORED_SERVER *db = database; db; db = db->next) { graph[nodes].info = handle->get_server_info(db); graph[nodes].db = db; graph[nodes].index = graph[nodes].lowest_index = 0; graph[nodes].cycle = 0; graph[nodes].active = false; graph[nodes].parent = NULL; nodes++; } /** Build the graph */ for (int i = 0; i < nservers; i++) { if (graph[i].info->slave_status.master_server_id > 0) { /** Found a connected node */ for (int k = 0; k < nservers; k++) { if (graph[k].info->server_id == graph[i].info->slave_status.master_server_id) { graph[i].parent = &graph[k]; break; } } } } int index = 1; int cycle = 0; int stacksize = 0; for (int i = 0; i < nservers; i++) { if (graph[i].index == 0) { /** Index is 0, this node has not yet been visited */ visit_node(&graph[i], stack, &stacksize, &index, &cycle); } } for (int i = 0; i < nservers; i++) { graph[i].info->group = graph[i].cycle; if (graph[i].cycle > 0) { /** We have at least one cycle in the graph */ if (graph[i].info->read_only) { monitor_set_pending_status(graph[i].db, SERVER_SLAVE | SERVER_STALE_SLAVE); monitor_clear_pending_status(graph[i].db, SERVER_MASTER); } else { monitor_set_pending_status(graph[i].db, SERVER_MASTER); monitor_clear_pending_status(graph[i].db, SERVER_SLAVE | SERVER_STALE_SLAVE); } } else if (handle->detectStaleMaster && cycle == 0 && graph[i].db->server->status & SERVER_MASTER && (graph[i].db->pending_status & SERVER_MASTER) == 0) { /** * Stale master detection is handled here for multi-master mode. * * If we know that no cycles were found from the graph and that a * server once had the master status, replication has broken * down. These masters are assigned the stale master status allowing * them to be used as masters even if they lose their slaves. A * slave in this case can be either a normal slave or another * master. */ if (graph[i].info->read_only) { /** The master is in read-only mode, set it into Slave state */ monitor_set_pending_status(graph[i].db, SERVER_SLAVE | SERVER_STALE_SLAVE); monitor_clear_pending_status(graph[i].db, SERVER_MASTER | SERVER_STALE_STATUS); } else { monitor_set_pending_status(graph[i].db, SERVER_MASTER | SERVER_STALE_STATUS); monitor_clear_pending_status(graph[i].db, SERVER_SLAVE | SERVER_STALE_SLAVE); } } } } /** * Monitor an individual server. * * @param database The database to probe */ void MariaDBMonitor::monitor_database(MXS_MONITORED_SERVER *database) { /* Don't probe servers in maintenance mode */ if (SERVER_IN_MAINT(database->server)) { return; } /** Store previous status */ database->mon_prev_status = database->server->status; mxs_connect_result_t rval = mon_ping_or_connect_to_db(m_monitor_base, database); if (rval == MONITOR_CONN_OK) { server_clear_status_nolock(database->server, SERVER_AUTH_ERROR); monitor_clear_pending_status(database, SERVER_AUTH_ERROR); } else { /** * The current server is not running. Clear all but the stale master bit * as it is used to detect masters that went down but came up. */ unsigned int all_bits = ~SERVER_STALE_STATUS; server_clear_status_nolock(database->server, all_bits); monitor_clear_pending_status(database, all_bits); if (mysql_errno(database->con) == ER_ACCESS_DENIED_ERROR) { server_set_status_nolock(database->server, SERVER_AUTH_ERROR); monitor_set_pending_status(database, SERVER_AUTH_ERROR); } /* Log connect failure only once */ if (mon_status_changed(database) && mon_print_fail_status(database)) { mon_log_connect_error(database, rval); } return; } /* Store current status in both server and monitor server pending struct */ server_set_status_nolock(database->server, SERVER_RUNNING); monitor_set_pending_status(database, SERVER_RUNNING); MySqlServerInfo *serv_info = get_server_info(database); /* Check whether current server is MaxScale Binlog Server */ MYSQL_RES *result; if (mxs_mysql_query(database->con, "SELECT @@maxscale_version") == 0 && (result = mysql_store_result(database->con)) != NULL) { serv_info->binlog_relay = true; mysql_free_result(result); } else { serv_info->binlog_relay = false; } /* Get server version string, also get/set numeric representation. */ mxs_mysql_set_server_version(database->con, database->server); /* Set monitor version enum. */ uint64_t version_num = server_get_version(database->server); if (version_num >= 100000) { serv_info->version = MYSQL_SERVER_VERSION_100; } else if (version_num >= 5 * 10000 + 5 * 100) { serv_info->version = MYSQL_SERVER_VERSION_55; } else { serv_info->version = MYSQL_SERVER_VERSION_51; } /* Query a few settings. */ read_server_variables(database, serv_info); /* If gtid domain exists and server is 10.0, update gtid:s */ if (m_master_gtid_domain >= 0 && serv_info->version == MYSQL_SERVER_VERSION_100) { update_gtids(database, serv_info); } /* Check for MariaDB 10.x.x and get status for multi-master replication */ if (serv_info->version == MYSQL_SERVER_VERSION_100 || serv_info->version == MYSQL_SERVER_VERSION_55) { monitor_mysql_db(database, serv_info); } else { if (m_mysql51_replication) { monitor_mysql_db(database, serv_info); } else if (report_version_err) { report_version_err = false; MXS_ERROR("MySQL version is lower than 5.5 and 'mysql51_replication' option is " "not enabled, replication tree cannot be resolved. To enable MySQL 5.1 replication " "detection, add 'mysql51_replication=true' to the monitor section."); } } } /** * Read server_id, read_only and (if 10.X) gtid_domain_id. * * @param database Database to update * @param serv_info Where to save results */ static void read_server_variables(MXS_MONITORED_SERVER* database, MySqlServerInfo* serv_info) { string query = "SELECT @@global.server_id, @@read_only;"; int columns = 2; if (serv_info->version == MYSQL_SERVER_VERSION_100) { query.erase(query.end() - 1); query += ", @@gtid_domain_id;"; columns = 3; } int ind_id = 0; int ind_ro = 1; int ind_domain = 2; StringVector row; if (query_one_row(database, query.c_str(), columns, &row)) { int64_t server_id = scan_server_id(row[ind_id].c_str()); database->server->node_id = server_id; serv_info->server_id = server_id; ss_dassert(row[ind_ro] == "0" || row[ind_ro] == "1"); serv_info->read_only = (row[ind_ro] == "1"); if (columns == 3) { uint32_t domain = 0; ss_debug(int rv = ) sscanf(row[ind_domain].c_str(), "%" PRIu32, &domain); ss_dassert(rv == 1); serv_info->gtid_domain_id = domain; } } } /** * Monitor a database with given server info. * * @param database Database to monitor * @param serv_info Server info for database */ void MariaDBMonitor::monitor_mysql_db(MXS_MONITORED_SERVER* database, MySqlServerInfo *serv_info) { /** Clear old states */ monitor_clear_pending_status(database, SERVER_SLAVE | SERVER_MASTER | SERVER_RELAY_MASTER | SERVER_SLAVE_OF_EXTERNAL_MASTER); if (do_show_slave_status(serv_info, database)) { /* If all configured slaves are running set this node as slave */ if (serv_info->slave_configured && serv_info->n_slaves_running > 0 && serv_info->n_slaves_running == serv_info->n_slaves_configured) { monitor_set_pending_status(database, SERVER_SLAVE); } /** Store master_id of current node. For MySQL 5.1 it will be set at a later point. */ database->server->master_id = serv_info->slave_status.master_server_id; } }