hcq/MaxScale
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

2.1 fix restore (#151)

Browse Source 
* refactor test backend fixing

* return comatibility with 5.5 for backend restore

* remove backend configuration scripts
This commit is contained in:
Timofey Turenko
2017-12-18 23:47:02 +02:00
committed by GitHub
parent ba24330d0f
commit 1a63e5ec7e
11 changed files with 449 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files

362
maxscale-system-test/mariadb_nodes.cpp
View File

@ -43,25 +43,34 @@ Mariadb_nodes::~Mariadb_nodes()
}
}
int Mariadb_nodes::connect(int i)
{
if (nodes[i] == NULL || mysql_ping(nodes[i]) != 0)
{
if (nodes[i])
{
mysql_close(nodes[i]);
}
nodes[i] = open_conn_db_timeout(port[i], IP[i], "test", user_name, password, 50, ssl);
}
if ((nodes[i] != NULL) && (mysql_errno(nodes[i]) != 0))
{
return 1;
}
else
{
return 0;
}
}
int Mariadb_nodes::connect()
{
int res = 0;
for (int i = 0; i < N; i++)
{
if (nodes[i] == NULL || mysql_ping(nodes[i]) != 0)
{
if (nodes[i])
{
mysql_close(nodes[i]);
}
nodes[i] = open_conn_db_timeout(port[i], IP[i], "test", user_name, password, 50, ssl);
}
if ((nodes[i] != NULL) && (mysql_errno(nodes[i]) != 0))
{
res++;
}
res += connect(i);
}
return res;
@ -230,7 +239,7 @@ int Mariadb_nodes::read_env()
}
else
{
sprintf(cleanup_db_command[i], " ");
sprintf(cleanup_db_command[i], "rm -rf /var/lib/mysql/*; killall -9 mysqld");
}
sprintf(env_name, "%s_%03d_whoami", prefix, i);
@ -290,9 +299,9 @@ int Mariadb_nodes::find_master()
while ((found == 0) && (i < N))
{
if (find_field(
nodes[i], "show slave status;",
"Master_Host", &str[0]
) == 0)
nodes[i], "show slave status;",
"Master_Host", &str[0]
) == 0)
{
found = 1;
strcpy(master_IP, str);
@ -425,10 +434,35 @@ int Mariadb_nodes::start_replication()
// Start all nodes
for (int i = 0; i < N; i++)
{
local_result += start_node(i, "");
ssh_node(i, true,
"mysql --force -u root %s -e \"STOP SLAVE; STOP ALL SLAVES; RESET SLAVE; RESET SLAVE ALL; RESET MASTER; SET GLOBAL read_only=OFF;\"",
socket_cmd[i]);
if (start_node(i, ""))
{
printf("Start of node %d failed, trying to cleanup and re-initialize node\n", i);
cleanup_db_node(i);
prepare_server(i);
local_result += start_node(i, "");
}
printf("trying to get version\n");
if (connect(i))
{
printf("Connect attempt to node %d failed\n", i);
}
get_version(i);
close_connections();
printf("Node %d: Version is %s\n", i, version_major[i]);
if (strcmp(version_major[i], "5.5") == 0)
{
ssh_node(i, true,
"mysql --force -u root %s -e \"STOP SLAVE; RESET SLAVE; RESET MASTER; SET GLOBAL read_only=OFF;\"",
socket_cmd[i]);
}
else
{
ssh_node(i, true,
"mysql --force -u root %s -e \"STOP SLAVE; STOP ALL SLAVES; RESET SLAVE; RESET SLAVE ALL; RESET MASTER; SET GLOBAL read_only=OFF;\"",
socket_cmd[i]);
}
ssh_node(i, true, "sudo rm -f /etc/my.cnf.d/kerb.cnf");
ssh_node(i, true,
"for i in `mysql -ss --force -u root %s -e \"SHOW DATABASES\"|grep -iv 'mysql\\|information_schema\\|performance_schema'`; "
@ -443,20 +477,33 @@ int Mariadb_nodes::start_replication()
user_name, password, access_homedir[0], socket_cmd[0]);
// Create a database dump from the master and distribute it to the slaves
ssh_node(0, true, "mysql --force -u root %s -e \"CREATE DATABASE test\"; "
"mysqldump --all-databases --add-drop-database --flush-privileges --master-data=1 --gtid %s > /tmp/master_backup.sql",
socket_cmd[0], socket_cmd[0]);
if (version_major[0][0] == '5')
{
printf("Version 5 on master detected, do not use --gtid flag for mysqldump\n");
ssh_node(0, true, "mysql --force -u root %s -e \"CREATE DATABASE test\"; "
"mysqldump --all-databases --add-drop-database --flush-privileges --master-data=1 %s > /tmp/master_backup.sql",
socket_cmd[0], socket_cmd[0]);
}
else
{
ssh_node(0, true, "mysql --force -u root %s -e \"CREATE DATABASE test\"; "
"mysqldump --all-databases --add-drop-database --flush-privileges --master-data=1 --gtid %s > /tmp/master_backup.sql",
socket_cmd[0], socket_cmd[0]);
}
sprintf(str, "%s/master_backup.sql", test_dir);
copy_from_node("/tmp/master_backup.sql", str, 0);
for (int i = 1; i < N; i++)
{
// Reset all nodes by first loading the dump and then starting the replication
printf("Starting node %d\n", i);
printf("Setting node %d\n", i);
fflush(stdout);
copy_to_node(str, "/tmp/master_backup.sql", i);
ssh_node(i, true, "mysql --force -u root %s -e \"STOP SLAVE;\"",
socket_cmd[i]);
ssh_node(i, true, "mysql --force -u root %s < /tmp/master_backup.sql",
socket_cmd[i]);
printf("change master to...\n");
ssh_node(i, true, "mysql --force -u root %s -e \"CHANGE MASTER TO MASTER_HOST=\\\"%s\\\", MASTER_PORT=%d, "
"MASTER_USER=\\\"repl\\\", MASTER_PASSWORD=\\\"repl\\\";"
"START SLAVE;\"", socket_cmd[i], IP_private[0], port[0]);
@ -467,7 +514,6 @@ int Mariadb_nodes::start_replication()
int Galera_nodes::start_galera()
{
char sys1[4096];
char str[1024];
int i;
int local_result = 0;
@ -481,7 +527,13 @@ int Galera_nodes::start_galera()
ssh_node(0, false, "echo [mysqld] > cluster_address.cnf");
ssh_node(0, false, "echo wsrep_cluster_address=gcomm:// >> cluster_address.cnf");
ssh_node(0, true, "cp cluster_address.cnf /etc/my.cnf.d/");
local_result += start_node(0, " --wsrep-cluster-address=gcomm://");
if (start_node(0, " --wsrep-cluster-address=gcomm://"))
{
cleanup_db_node(i);
prepare_server(i);
local_result += start_node(0, " --wsrep-cluster-address=gcomm://");
}
sprintf(str, "%s/create_user_galera.sh", test_dir);
copy_to_node(str, "~/", 0);
@ -655,11 +707,19 @@ bool Mariadb_nodes::check_master_node(MYSQL *conn)
return rval;
}
/**
* @brief bad_slave_thread_status Check if filed in the slave status outpur is not 'yes'
* @param conn MYSQL struct (connection have to be open)
* @param field Filed to check
* @param node Node index
* @return false if requested filed is 'Yes'
*/
static bool bad_slave_thread_status(MYSQL *conn, const char *field, int node)
{
char str[1024] = "";
bool rval = false;
// Doing 3 attempts to check status
for (int i = 0; i < 2; i++)
{
if (find_field(conn, "SHOW SLAVE STATUS;", field, str) != 0)
@ -670,9 +730,10 @@ static bool bad_slave_thread_status(MYSQL *conn, const char *field, int node)
}
else if (strcmp(str, "Yes") == 0 || strcmp(str, "No") == 0)
{
printf("Node %d: filed %s is %s\n", node, field, str);
break;
}
printf("Node %d: filed %s is %s\n", node, field, str);
/** Any other state is transient and we should try again */
sleep(1);
}
@ -683,16 +744,23 @@ static bool bad_slave_thread_status(MYSQL *conn, const char *field, int node)
fflush(stdout);
rval = true;
}
return rval;
}
/**
* @brief multi_source_replication Check if slave is connected to more then one master
* @param conn MYSQL struct (have to be open)
* @param node Node index
* @return false if multisource replication is not detected
*/
static bool multi_source_replication(MYSQL *conn, int node)
{
bool rval = true;
MYSQL_RES *res;
if (mysql_query(conn, "SHOW ALL SLAVES STATUS") == 0 &&
(res = mysql_store_result(conn)))
(res = mysql_store_result(conn)))
{
if (mysql_num_rows(res) == 1)
{
@ -704,6 +772,12 @@ static bool multi_source_replication(MYSQL *conn, int node)
fflush(stdout);
}
}
else
{
printf("Node %d does not support SHOW ALL SLAVE STATUS, ignoring multi source replication check\n", node);
fflush(stdout);
rval = false;
}
return rval;
}
@ -712,7 +786,6 @@ int Mariadb_nodes::check_replication()
{
int master = 0;
int res = 0;
char str[1024];
if (verbose)
{
@ -734,6 +807,10 @@ int Mariadb_nodes::check_replication()
if (!check_master_node(nodes[i]))
{
res = 1;
if (verbose)
{
printf("Master node check failed for node %d\n", i);
}
}
}
else if (bad_slave_thread_status(nodes[i], "Slave_IO_Running", i) ||
@ -741,14 +818,24 @@ int Mariadb_nodes::check_replication()
multi_source_replication(nodes[i], i))
{
res = 1;
if (verbose)
{
printf("Slave %d check failed\n", i);
}
}
}
if (verbose)
{
printf("Replication check for %s gave code %d\n", prefix, res);
}
return res;
}
bool Mariadb_nodes::fix_replication()
{
verbose = true;
if (check_replication())
{
unblock_all_nodes();
@ -760,7 +847,8 @@ bool Mariadb_nodes::fix_replication()
}
int attempts = 2;
int attempts_with_cleanup = 2;
int attempts_with_cleanup = 1;
int attempts_with_revert = 1;
while (check_replication() && attempts > 0)
{
@ -771,6 +859,7 @@ bool Mariadb_nodes::fix_replication()
start_replication();
close_connections();
check_replication();
attempts--;
@ -781,16 +870,27 @@ bool Mariadb_nodes::fix_replication()
printf("****** BACKEND IS STILL BROKEN! Trying to cleanup all nodes *****\n");
stop_nodes();
cleanup_db_nodes();
prepare_servers();
attempts_with_cleanup--;
attempts = 2;
sleep(30);
sleep(10);
start_replication();
sleep(30);
sleep(10);
}
else
{
printf("****** BACKEND IS STILL BROKEN! Exiting *****\n");
return false;
if (attempts_with_revert > 0)
{
printf("****** BACKEND IS STILL BROKEN! Trying to revert all nodes from snapshot *****\n");
revert_nodes_snapshot();
attempts_with_cleanup = 1;
attempts = 2;
}
else
{
printf("****** BACKEND IS STILL BROKEN! Exiting *****\n");
return false;
}
}
}
}
@ -800,6 +900,21 @@ bool Mariadb_nodes::fix_replication()
return true;
}
bool Mariadb_nodes::revert_nodes_snapshot()
{
char str[1024];
bool rval = true;
for (int i = 0; i < N; i++)
{
sprintf(str, "%s clean --node-name %s_%03d", revert_snapshot_command, prefix, i);
if (system(str))
{
rval = false;
}
}
return rval;
}
int Galera_nodes::check_galera()
{
int res1 = 0;
@ -946,6 +1061,7 @@ char * Mariadb_nodes::ssh_node_output(int node, const char *ssh, bool sudo, int
{
char sys[strlen(ssh) + 1024];
generate_ssh_cmd(sys, node, ssh, sudo);
//printf("%s\n", sys);
FILE *output = popen(sys, "r");
if (output == NULL)
{
@ -1006,6 +1122,7 @@ int Mariadb_nodes::ssh_node(int node, bool sudo, const char *format, ...)
sshkey[node], access_user[node], IP[node]);
}
int rc = 1;
//printf("%s *** %s \n", cmd, sys);
FILE *in = popen(cmd, "w");
if (in)
@ -1100,38 +1217,59 @@ int Mariadb_nodes::execute_query_all_nodes(const char* sql)
return local_result;
}
int Mariadb_nodes::get_version(int i)
{
char * str;
int ec;
int local_result = 0;
if (find_field(nodes[i], "SELECT @@version", "@@version", version[i]))
{
printf("Failed to get version: %s, trying ssh node and use MariaDB client\n", mysql_error(nodes[i]));
str = ssh_node_output(i, "mysql --batch --silent -e \"select @@version\"", true, &ec);
if (ec)
{
local_result++;
printf("Failed to get version, node %d is broken\n", i);
}
else
{
strcpy(version[i], str);
free(str);
}
}
strcpy(version_number[i], version[i]);
str = strchr(version_number[i], '-');
if (str != NULL)
{
str[0] = 0;
}
strcpy(version_major[i], version_number[i]);
if (strstr(version_major[i], "5.") == version_major[i])
{
version_major[i][3] = 0;
}
if (strstr(version_major[i], "10.") == version_major[i])
{
version_major[i][4] = 0;
}
if (verbose)
{
printf("Node %s%d: %s\t %s \t %s\n", prefix, i, version[i], version_number[i], version_major[i]);
}
return local_result;
}
int Mariadb_nodes::get_versions()
{
int local_result = 0;
char * str;
v51 = false;
for (int i = 0; i < N; i++)
{
if ((local_result += find_field(nodes[i], "SELECT @@version", "@@version", version[i])))
{
printf("Failed to get version: %s\n", mysql_error(nodes[i]));
}
strcpy(version_number[i], version[i]);
str = strchr(version_number[i], '-');
if (str != NULL)
{
str[0] = 0;
}
strcpy(version_major[i], version_number[i]);
if (strstr(version_major[i], "5.") == version_major[i])
{
version_major[i][3] = 0;
}
if (strstr(version_major[i], "10.") == version_major[i])
{
version_major[i][4] = 0;
}
if (verbose)
{
printf("Node %s%d: %s\t %s \t %s\n", prefix, i, version[i], version_number[i], version_major[i]);
}
local_result += get_version(i);
}
for (int i = 0; i < N; i++)
@ -1367,7 +1505,7 @@ void Mariadb_nodes::close_active_connections()
}
const char *sql =
"select id from information_schema.processlist where id != @@pseudo_thread_id and user not in ('system user', 'repl')";
"select id from information_schema.processlist where id != @@pseudo_thread_id and user not in ('system user', 'repl')";
for (int i = 0; i < N; i++)
{
@ -1389,3 +1527,105 @@ void Mariadb_nodes::close_active_connections()
}
}
}
/**
* @brief extract_version_from_string Tries to find MariaDB server version number in the output of 'mysqld --version'
* Function does not allocate any memory
* @param version String returned by 'mysqld --version'
* @return pointer to the string with version number
*/
char * extract_version_from_string(char * version)
{
int pos1 = 0;
int pos2 = 0;
int l = strlen(version);
while ((! isdigit(version[pos1])) && (pos1 < l))
{
pos1++;
}
pos2 = pos1;
while (((isdigit(version[pos2]) || version[pos2] == '.')) && (pos2 < l))
{
pos2++;
}
version[pos2] = '\0';
return &version[pos1];
}
int Mariadb_nodes::prepare_server(int i)
{
int ec;
char * version;
char * version_digits;
char * tmp_pass;
char str1[1024];
char str2[1024];
ssh_node(i, true, stop_db_command[i]);
sleep(5);
ssh_node(i, true, "sed -i \"s/bind-address/#bind-address/g\" /etc/mysql/my.cnf.d/*.cnf");
ssh_node(i, true, "ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/usr.sbin.mysqld; sudo service apparmor restart");
version = ssh_node_output(i, "/usr/sbin/mysqld --version", false, &ec);
if (ec == 0)
{
version_digits = extract_version_from_string(version);
printf("Detected server version on node %d is %s\n", i, version_digits);
if (memcmp(version_digits, "5.", 2) == 0)
{
ssh_node(i, true, "sed -i \"s/binlog_row_image=full//\" /etc/my.cnf.d/*.cnf");
}
if (memcmp(version_digits, "5.7", 3) == 0)
{
// Disable 'validate_password' plugin, searach for random temporal
// password in the log and reseting passord to empty string
ssh_node(i, true, "/usr/sbin/mysqld --initialize; sudo chown -R mysql:mysql /var/lib/mysql");
ssh_node(i, true, start_db_command[i]);
tmp_pass = ssh_node_output(i, "cat /var/log/mysqld.log | grep \"temporary password\" | sed -n -e 's/^.*: //p'", true, &ec);
ssh_node(i, true, "mysqladmin -uroot -p'%s' password '%s'", tmp_pass, tmp_pass);
ssh_node(i, false, "echo \"UNINSTALL PLUGIN validate_password\" | sudo mysql -uroot -p'%s'", tmp_pass);
ssh_node(i, true, stop_db_command[i]);
ssh_node(i, true, start_db_command[i]);
ssh_node(i, true, "mysqladmin -uroot -p'%s' password ''", tmp_pass);
}
else
{
printf("Executing mysql_install_db on node %d\n", i);
ssh_node(i, true, "mysql_install_db; sudo chown -R mysql:mysql /var/lib/mysql");
printf("Starting server on node %d\n", i);
if (ssh_node(i, true, start_db_command[i]))
{
printf("Server start on node %d failed\n", i);
}
}
sleep(15);
sprintf(str1, "%s/mdbci/backend/create_*_user.sql", test_dir);
sprintf(str2, "%s/", access_homedir[i]);
copy_to_node(str1, str2, i);
sprintf(str1, "mysql < %s/create_repl_user.sql", access_homedir[i]);
ssh_node(i, true, str1);
sprintf(str1, "mysql < %s/create_skysql_user.sql", access_homedir[i]);
ssh_node(i, true, str1);
free(version);
return 0;
}
else
{
return 1;
}
}
int Mariadb_nodes::prepare_servers()
{
int rval = 0;
for (int i; i < N; i++)
{
if (prepare_server(i))
{
rval = 1;
}
}
return rval;
}