From 2fc42fca1e0be8f53c3ff4633e6c9516522dfc61 Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Wed, 11 Dec 2019 13:08:27 +0200 Subject: [PATCH 1/5] Remove Docker installation from build script Installation of Docker is removed from build script because it is done in the not very reliable way and can fail and cause build failure. E.g. this installation causes conflict with existing Docker. --- BUILD/install_build_deps.sh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/BUILD/install_build_deps.sh b/BUILD/install_build_deps.sh index ba35111b3..710dfb4d4 100755 --- a/BUILD/install_build_deps.sh +++ b/BUILD/install_build_deps.sh @@ -131,14 +131,6 @@ then # Enable it by default echo "source /opt/rh/devtoolset-7/enable" >> ~/.bashrc - else - # Installed for REST API and MaxCtrl unit tests - sudo yum -y install docker epel-release - sudo yum -y install docker-compose - sudo groupadd docker - sudo usermod -a -G docker `whoami` - sudo sed -i 's/--selinux-enabled/--selinux-enabled=false/' /etc/sysconfig/docker - sudo systemctl start docker fi fi From ba46cb4434da255bbc7f4708e5a7c0544e105175 Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Wed, 11 Dec 2019 17:35:32 +0200 Subject: [PATCH 2/5] Enable 'PowerTools' yum repo if it is present In CentOS8 some -devel packages are moved to PowerTools repository which is disabled by default. install_build_depends now checks if this repo is in the repolist then add enablerepo=PowerTools to the yum command --- BUILD/install_build_deps.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/BUILD/install_build_deps.sh b/BUILD/install_build_deps.sh index 710dfb4d4..8490be39b 100755 --- a/BUILD/install_build_deps.sh +++ b/BUILD/install_build_deps.sh @@ -103,7 +103,14 @@ then # YUM! sudo yum clean all sudo yum update -y - sudo yum install -y --nogpgcheck gcc gcc-c++ ncurses-devel bison glibc-devel \ + unset enable_power_tools + yum repolist all | grep PowerTools + if [ $? == 0 ] + then + enable_power_tools="enablerepo=PowerTools" + fi + sudo yum install -y --nogpgcheck ${enable_power_tools} \ + gcc gcc-c++ ncurses-devel bison glibc-devel \ libgcc perl make libtool openssl-devel libaio libaio-devel libedit-devel \ libedit-devel systemtap-sdt-devel rpm-sign wget \ gnupg pcre-devel flex rpmdevtools git wget tcl tcl-devel openssl libuuid-devel xz-devel \ From da682b7777bda11a778eacc049f531fe9be50033 Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Thu, 12 Dec 2019 23:31:38 +0200 Subject: [PATCH 3/5] Fix trailing whitespace in install_build_deps.sh --- BUILD/install_build_deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BUILD/install_build_deps.sh b/BUILD/install_build_deps.sh index 8490be39b..999ee3303 100755 --- a/BUILD/install_build_deps.sh +++ b/BUILD/install_build_deps.sh @@ -104,7 +104,7 @@ then sudo yum clean all sudo yum update -y unset enable_power_tools - yum repolist all | grep PowerTools + yum repolist all | grep PowerTools if [ $? == 0 ] then enable_power_tools="enablerepo=PowerTools" From 44664132bc6829e34ba2dd2cf315612d5c9b644a Mon Sep 17 00:00:00 2001 From: Esa Korhonen Date: Wed, 11 Dec 2019 14:54:56 +0200 Subject: [PATCH 4/5] MXS-2792 Improve monitor script documentation Apparently the previous script example was not working. Moved the updated example to Monitor-Common.md. --- Documentation/Monitors/MariaDB-Monitor.md | 35 ------ Documentation/Monitors/Monitor-Common.md | 131 ++++++++++++++++++---- 2 files changed, 110 insertions(+), 56 deletions(-) diff --git a/Documentation/Monitors/MariaDB-Monitor.md b/Documentation/Monitors/MariaDB-Monitor.md index 3f15ab518..87c2f329b 100644 --- a/Documentation/Monitors/MariaDB-Monitor.md +++ b/Documentation/Monitors/MariaDB-Monitor.md @@ -40,7 +40,6 @@ Table of Contents * [Failover/switchover fails](#failoverswitchover-fails) * [Slave detection shows external masters](#slave-detection-shows-external-masters) * [Using the MariaDB Monitor With Binlogrouter](#using-the-mariadb-monitor-with-binlogrouter) - * [Example 1 - Monitor script](#example-1---monitor-script) ## Overview @@ -852,37 +851,3 @@ binlog server to the list of servers only if _master_id_ identity is set. For addition information read the [Replication Proxy](../Tutorials/Replication-Proxy-Binlog-Router-Tutorial.md) tutorial. - -## Example 1 - Monitor script - -Here is an example shell script which sends an email to an admin@my.org -when a server goes down. - -``` -#!/usr/bin/env bash - -#This script assumes that the local mail server is configured properly -#The second argument is the event type -event=${$2/.*=/} -server=${$3/.*=/} -message="A server has gone down at `date`." -echo $message|mail -s "The event was $event for server $server." admin@my.org - -``` - -Here is a monitor configuration that only triggers the script when a master -or a slave server goes down. - -``` -[Database-Monitor] -type=monitor -module=mariadbmon -servers=server1,server2 -script=mail_to_admin.sh -events=master_down,slave_down -``` - -When a master or a slave server goes down, the script is executed, a mail is -sent and the administrator will be immediately notified of any possible -problems. This is just a simple example showing what you can do with MaxScale -and monitor scripts. diff --git a/Documentation/Monitors/Monitor-Common.md b/Documentation/Monitors/Monitor-Common.md index 4fc300823..fb95ccfb8 100644 --- a/Documentation/Monitors/Monitor-Common.md +++ b/Documentation/Monitors/Monitor-Common.md @@ -2,6 +2,8 @@ This document lists optional parameters that all current monitors support. +[TOC] + ## Parameters ### `user` @@ -177,36 +179,45 @@ disk_space_check_interval=10000 ### `script` -This command will be executed when a server changes its state. The parameter should be an absolute path to a command or the command should be in the executable path. The user which is used to run MaxScale should have execution rights to the file itself and the directory it resides in. +This command will be executed on a server state change. The parameter should +be an absolute path to a command or the command should be in the executable +path. The user running MaxScale should have execution rights to the file itself +and the directory it resides in. The script may have placeholders which +MaxScale will substitute with useful information when launching the script. + +The placeholders and their substition results are: + +* `$INITIATOR` -> IP and port of the server which initiated the event +* `$EVENT` -> event description, e.g. "server_up" +* `$LIST` -> list of IPs and ports of all servers +* `$NODELIST` -> list of IPs and ports of all running servers +* `$SLAVELIST` -> list of IPs and ports of all slave servers +* `$MASTERLIST` -> list of IPs and ports of all master servers +* `$SYNCEDLIST` -> list of IPs and ports of all synced Galera nodes +* `$PARENT` -> IP and port of the parent of the server which initiated the event. +For master-slave setups, this will be the master if the initiating server is a +slave. +* `$CHILDREN` -> list of IPs and ports of the child nodes of the server who +initiated the event. For master-slave setups, this will be a list of slave +servers if the initiating server is a master. + +The expanded variable value can be an empty string if no servers match the +variable's requirements. For example, if no masters are available `$MASTERLIST` +will expand into an empty string. The list-type substitutions will only contain +servers monitored by the current monitor. ``` script=/home/user/myscript.sh initiator=$INITIATOR event=$EVENT live_nodes=$NODELIST ``` -The following substitutions will be made to the parameter value: - -* `$INITIATOR` will be replaced with the IP and port of the server who initiated the event -* `$EVENT` will be replaced with the name of the event -* `$LIST` will be replaced with a list of server IPs and ports -* `$NODELIST` will be replaced with a list of server IPs and ports that are running -* `$SLAVELIST` will be replaced with a list of server IPs and ports that are slaves -* `$MASTERLIST` will be replaced with a list of server IPs and ports that are masters -* `$SYNCEDLIST` will be replaced with a list of server IPs and ports that are synced Galera nodes -* `$PARENT` will be replaced with the IP and port of the parent node of the server who initiated - the event. For master-slave setups, this will be the master if the initiating server is a slave. -* `$CHILDREN` will be replaced with the IPs and ports of the child nodes of the server who initiated - the event. For master-slave setups, this will be a list of slave servers if the initiating server is a master. - -The expanded variable value can be an empty string if no servers match the -variable's requirements. For example, if no masters are available `$MASTERLIST` -will expand into an empty string. - -For example, the previous example will be executed as: +The above script could be executed as: ``` /home/user/myscript.sh initiator=[192.168.0.10]:3306 event=master_down live_nodes=[192.168.0.201]:3306,[192.168.0.121]:3306 ``` +See section [Script example](#script-example) below for an example script. + Any output by the executed script will be logged into the MaxScale log. Each outputted line will be logged as a separate log message. @@ -215,7 +226,7 @@ messages. If the first word in the output line is one of `alert:`, `error:`, `warning:`, `notice:`, `info:` or `debug:`, the message will be logged on the corresponding level. If the message is not prefixed with one of the keywords, the message will be logged on the notice level. Whitespace before, after or -inbetween the keyword and the colon is ignored and the matching is +between the keyword and the colon is ignored and the matching is case-insensitive. ### `script_timeout` @@ -282,3 +293,81 @@ name of the monitor section in the configuration file. If MaxScale crashes or is shut down in an uncontrolled fashion, the journal will be read when MaxScale is started. To skip the recovery process, manually delete the journal file before starting MaxScale. + +## Script example + +Below is an example monitor configuration which launches a script with all +supported substitutions. The example script reads the results and prints it to +file and sends it as email. + +``` +[MyMonitor] +type=monitor +module=mariadbmon +servers=C1N1,C1N2,C1N3 +user=maxscale +password=password +monitor_interval=10000 +script=/path/to/maxscale_monitor_alert_script.sh --initiator=$INITIATOR --parent=$PARENT --children=$CHILDREN --event=$EVENT --node_list=$NODELIST --list=$LIST --master_list=$MASTERLIST --slave_list=$SLAVELIST --synced_list=$SYNCEDLIST +``` + +File "maxscale_monitor_alert_script.sh": +``` +#!/usr/bin/env bash + +initiator="" +parent="" +children="" +event="" +node_list="" +list="" +master_list="" +slave_list="" +synced_list="" + +process_arguments() +{ + while [ "$1" != "" ]; do + if [[ "$1" =~ ^--initiator=.* ]]; then + initiator=${1#'--initiator='} + elif [[ "$1" =~ ^--parent.* ]]; then + parent=${1#'--parent='} + elif [[ "$1" =~ ^--children.* ]]; then + children=${1#'--children='} + elif [[ "$1" =~ ^--event.* ]]; then + event=${1#'--event='} + elif [[ "$1" =~ ^--node_list.* ]]; then + node_list=${1#'--node_list='} + elif [[ "$1" =~ ^--list.* ]]; then + list=${1#'--list='} + elif [[ "$1" =~ ^--master_list.* ]]; then + master_list=${1#'--master_list='} + elif [[ "$1" =~ ^--slave_list.* ]]; then + slave_list=${1#'--slave_list='} + elif [[ "$1" =~ ^--synced_list.* ]]; then + synced_list=${1#'--synced_list='} + fi + shift + done +} + +process_arguments $@ +read -r -d '' MESSAGE << EOM +A server has changed state. The following information was provided: + +Initiator: $initiator +Parent: $parent +Children: $children +Event: $event +Node list: $node_list +List: $list +Master list: $master_list +Slave list: $slave_list +Synced list: $synced_list +EOM + +# print message to file +echo "$MESSAGE" > /path/to/script_output.txt +# email the message +echo "$MESSAGE" | mail -s "MaxScale received $event event for initiator $initiator." mariadb_admin@domain.com +``` From e057c751cdee68164e75b8a4347659965ce00312 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 12 Dec 2019 21:12:11 +0200 Subject: [PATCH 5/5] Improve handleError error messages The hangup and error handlers now have unique messages. Although the behavior in the handlers is practically the same in both cases, the cause of the error is not the same. If a socket error is present, it is added to the error message. If an error is present, it should clearly show the reason why the TCP socket was closed. The is_fake_event boolean helps distinguish fake events from real ones. This makes figuring out the real source of hangup events easier. --- include/maxscale/dcb.h | 3 ++- server/core/dcb.cc | 5 ++++ .../MySQL/mariadbbackend/mysql_backend.cc | 26 ++++++++++++++++--- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/include/maxscale/dcb.h b/include/maxscale/dcb.h index 21933b795..411ab8bb3 100644 --- a/include/maxscale/dcb.h +++ b/include/maxscale/dcb.h @@ -229,7 +229,8 @@ typedef struct dcb uint32_t n_close; /** How many times dcb_close has been called. */ char* path; /** If a Unix socket, the path it was bound to. */ - uint64_t m_uid; /**< Unique identifier for this DCB */ + uint64_t m_uid; /**< Unique identifier for this DCB */ + bool is_fake_event; } DCB; /** diff --git a/server/core/dcb.cc b/server/core/dcb.cc index 456b081cf..d6828cd3f 100644 --- a/server/core/dcb.cc +++ b/server/core/dcb.cc @@ -204,6 +204,7 @@ DCB* dcb_alloc(dcb_role_t role, SERV_LISTENER* listener) newdcb->low_water = config_writeq_low_water(); newdcb->high_water = config_writeq_high_water(); newdcb->m_uid = this_unit.uid_generator.fetch_add(1, std::memory_order_relaxed); + newdcb->is_fake_event = false; if (role == DCB_ROLE_SERVICE_LISTENER) { @@ -2062,7 +2063,9 @@ static void dcb_hangup_foreach_worker(MXB_WORKER* worker, struct server* server) { this_thread.current_dcb = dcb; dcb->flags |= DCBF_HUNG; + dcb->is_fake_event = true; dcb->func.hangup(dcb); + dcb->is_fake_event = false; } } @@ -3249,7 +3252,9 @@ static uint32_t dcb_handler(DCB* dcb, uint32_t events) events = dcb->fake_event; dcb->fake_event = 0; + dcb->is_fake_event = true; rv |= dcb_process_poll_events(dcb, events); + dcb->is_fake_event = false; } this_thread.current_dcb = NULL; diff --git a/server/modules/protocol/MySQL/mariadbbackend/mysql_backend.cc b/server/modules/protocol/MySQL/mariadbbackend/mysql_backend.cc index d00eef6b7..ef904c4f4 100644 --- a/server/modules/protocol/MySQL/mariadbbackend/mysql_backend.cc +++ b/server/modules/protocol/MySQL/mariadbbackend/mysql_backend.cc @@ -25,6 +25,8 @@ #include #include +#include + /* * MySQL Protocol module for handling the protocol between the gateway * and the backend MySQL database. @@ -585,6 +587,23 @@ static int gw_read_backend_event(DCB* dcb) return rc; } +static std::string get_detailed_error(DCB* dcb) +{ + std::ostringstream ss; + + if (int err = gw_getsockerrno(dcb->fd)) + { + ss << " (" << err << ", " << mxs_strerror(err) << ")"; + } + else if (dcb->is_fake_event) + { + // Fake events should not have TCP socket errors + ss << " (Generated event)"; + } + + return ss.str(); +} + static void do_handle_error(DCB* dcb, mxs_error_action_t action, const char* errmsg) { bool succp = true; @@ -592,7 +611,7 @@ static void do_handle_error(DCB* dcb, mxs_error_action_t action, const char* err if (!dcb->dcb_errhandle_called) { - GWBUF* errbuf = mysql_create_custom_error(1, 0, errmsg); + GWBUF* errbuf = mysql_create_custom_error(1, 0, (errmsg + get_detailed_error(dcb)).c_str()); MXS_ROUTER_SESSION* rsession = static_cast(session->router_session); MXS_ROUTER_OBJECT* router = session->service->router; MXS_ROUTER* router_instance = session->service->router_instance; @@ -1340,7 +1359,7 @@ static int gw_error_backend_event(DCB* dcb) } else { - do_handle_error(dcb, ERRACT_NEW_CONNECTION, "Lost connection to backend server."); + do_handle_error(dcb, ERRACT_NEW_CONNECTION, "Lost connection to backend server: network error"); } return 1; @@ -1381,7 +1400,8 @@ static int gw_backend_hangup(DCB* dcb) } else { - do_handle_error(dcb, ERRACT_NEW_CONNECTION, "Lost connection to backend server."); + do_handle_error(dcb, ERRACT_NEW_CONNECTION, + "Lost connection to backend server: connection closed by peer"); } return 1;