MXS-2458: add template for Clustrix monitor tests

First implementation of Clustrix setup creation
TODO: move Clustrix installation to MDBCI
This commit is contained in:
Timofey Turenko 2019-05-07 17:17:57 +03:00
parent 4e6ffc0381
commit 56c27afd0a
12 changed files with 656 additions and 8 deletions

View File

@ -28,7 +28,7 @@ add_library(testcore SHARED testconnections.cpp nodes.cpp mariadb_nodes.cpp maxs
sql_t1.cpp test_binlog_fnc.cpp get_my_ip.cpp big_load.cpp get_com_select_insert.cpp
different_size.cpp fw_copy_rules maxinfo_func.cpp config_operations.cpp rds_vpc.cpp execute_cmd.cpp
blob_test.cpp keepalived_func.cpp tcp_connection.cpp base/stopwatch.cpp fw_copy_rules.cpp
labels_table.cpp envv.cpp
labels_table.cpp envv.cpp clustrix_nodes.cpp
# Include the CDC connector in the core library
${CMAKE_SOURCE_DIR}/connectors/cdc-connector/cdc_connector.cpp)
target_link_libraries(testcore ${MARIADB_CONNECTOR_LIBRARIES} ${JANSSON_LIBRARIES} z m pthread ssl dl rt crypto crypt maxbase)
@ -1127,8 +1127,21 @@ set_tests_properties(bug587_1_big PROPERTIES TIMEOUT 3600)
set_tests_properties(bug471_big PROPERTIES TIMEOUT 3600)
############################################
# END: tests with 15 machines backend #
# END: tests with 15 machines backend #
############################################
############################################
# BEGIN: tests for Clustrix monitor #
############################################
# creates Clustrix cluster and connect Maxscale to it
add_test_executable(clustrix_mon.cpp clustrix_mon clustrix_mon LABELS CLUSTRIX_BACKEND UNSTABLE)
set_tests_properties(clustrix_mon PROPERTIES TIMEOUT 7200)
############################################
# END: tests for Clustrix monitor #
############################################
###############################
# DO NOT ADD TESTS AFTER THIS #
###############################

View File

@ -0,0 +1,19 @@
/**
* @file clustrix_mon.cpp - simple Clustrix monitor test
* Just creates Clustrix cluster and connect Maxscale to it
* It can be used as a template for clustrix tests
*
* See Clustrix_nodes.h for details about configiration
*/
#include "testconnections.h"
int main(int argc, char* argv[])
{
int i;
TestConnections* Test = new TestConnections(argc, argv);
int rval = Test->global_result;
delete Test;
return rval;
}

View File

@ -0,0 +1,68 @@
#include <fstream>
#include <iostream>
#include <sstream>
#include "clustrix_nodes.h"
int Clustrix_nodes::install_clustrix(int m)
{
int ec;
char* clustrix_rpm = ssh_node_output(m, "rpm -qa | grep clustrix-clxnode", true, &ec);
if (strstr(clustrix_rpm, "clustrix-clxnode") == NULL)
{
printf("%s\n", ssh_node_output(m, CLUSTRIX_DEPS_YUM, true, &ec));
printf("%s\n", ssh_node_output(m, WGET_CLUSTRIX, false, &ec));
printf("%s\n", ssh_node_output(m, UNPACK_CLUSTRIX, false, &ec));
printf("%s\n", ssh_node_output(m, INSTALL_CLUSTRIX, false, &ec));
create_users(m);
}
return 0;
}
int Clustrix_nodes::start_cluster()
{
for (int i = 0; i < N; i++)
{
install_clustrix(i);
}
std::string lic_filename = std::string(getenv("HOME"))
+ std::string("/.config/mdbci/clustrix_license");
std::ifstream lic_file;
lic_file.open(lic_filename.c_str());
std::stringstream strStream;
strStream << lic_file.rdbuf();
std::string clustrix_license = strStream.str();
lic_file.close();
execute_query_all_nodes(clustrix_license.c_str());
std::string cluster_setup_sql = std::string("ALTER CLUSTER ADD '")
+ std::string(IP_private[1])
+ std::string("'");
for (int i = 2; i < N; i++)
{
cluster_setup_sql += std::string(",'")
+ std::string(IP_private[i])
+ std::string("'");
}
connect();
execute_query(nodes[0], "%s", cluster_setup_sql.c_str());
close_connections();
return 0;
}
std::string Clustrix_nodes::cnf_servers()
{
std::string s;
for (int i = 0; i < N; i++)
{
s += std::string("\\n[")
+ cnf_server_name
+ std::to_string(i + 1)
+ std::string("]\\ntype=server\\naddress=")
+ std::string(IP_private[i])
+ std::string("\\nport=")
+ std::to_string(port[i])
+ std::string("\\nprotocol=MySQLBackend\\n");
}
return s;
}

View File

@ -0,0 +1,51 @@
/**
* @file clustrix_nodes.h - work with Clustrix setup
*
* ~/.config/mdbci/clustrix_license file have to contain SQL
* which setups license to the Clustrix node
*
* TODO: move functionality of install_clustrix() to MDBCI
*/
#pragma once
#include <errno.h>
#include <string>
#include "nodes.h"
#include "mariadb_nodes.h"
#define CLUSTRIX_DEPS_YUM "yum install -y bzip2 wget screen ntp ntpdate vim htop mdadm"
#define WGET_CLUSTRIX "wget http://files.clustrix.com/releases/software/clustrix-9.1.4.el7.tar.bz2"
#define UNPACK_CLUSTRIX "tar xvjf clustrix-9.1.4.el7.tar.bz2"
#define INSTALL_CLUSTRIX "cd clustrix-9.1.4.el7; sudo ./clxnode_install.py --yes --force"
class Clustrix_nodes : public Mariadb_nodes
{
public:
Clustrix_nodes(const char* pref, const char* test_cwd, bool verbose, std::string network_config)
: Mariadb_nodes(pref, test_cwd, verbose, network_config)
{
}
/**
* @brief install_clustrix
* @param m node index
* @return 0 in case of success
*/
int install_clustrix(int m);
/**
* @brief start_cluster Intstalls Clustrix on all nodes, configure license, form cluster
* @return 0 in case of success
*/
int start_cluster();
/**
* @brief cnf_servers Generate Clustrix servers description for maxscale.cnf
* @return text for maxscale.cnf
*/
std::string cnf_servers();
};

View File

@ -0,0 +1,65 @@
[maxscale]
threads=###threads###
#log_info=1
[Clusterix-Monitor]
type=monitor
module=clustrixmon
servers=###clustrix_server_line###
user=maxskysql
password=skysql
[RW-Split-Router]
type=service
router=readwritesplit
servers=###clustrix_server_line###
user=maxskysql
password=skysql
slave_selection_criteria=LEAST_GLOBAL_CONNECTIONS
max_slave_connections=1
[Read-Connection-Router-Slave]
type=service
router=readconnroute
router_options=slave
servers=###clustrix_server_line###
user=maxskysql
password=skysql
[Read-Connection-Router-Master]
type=service
router=readconnroute
router_options=master
servers=###clustrix_server_line###
user=maxskysql
password=skysql
[RW-Split-Listener]
type=listener
service=RW-Split-Router
protocol=MySQLClient
port=4006
[Read-Connection-Listener-Slave]
type=listener
service=Read-Connection-Router-Slave
protocol=MySQLClient
port=4009
[Read-Connection-Listener-Master]
type=listener
service=Read-Connection-Router-Master
protocol=MySQLClient
port=4008
[CLI]
type=service
router=cli
[CLI-Listener]
type=listener
service=CLI
protocol=maxscaled
socket=default
###clustrix_server###

View File

@ -0,0 +1,25 @@
#!/bin/bash
# Start 4 AWS instances and put info into _network_config
# $1 - path to MDBCI configuration
sg=`cat ~/.config/mdbci/config.yaml | grep security_group | sed "s/security_group://" | tr -d " "`
i_ids=`aws ec2 run-instances --image ami-65fa5c16 --instance-type m3.large --security-groups turenko-home_1547635960 --key-name maxscale --count 4 | grep InstanceId | tr -d ' ' | tr -d ',' | tr -d '"' | sed "s/InstanceId://" | sed ':a;N;$!ba;s/\n/ /g'`
#i_ids=`aws ec2 run-instances --image ami-0ff760d16d9497662 --instance-type i3.large --security-groups turenko-home_1547635960 --key-name maxscale --count 4 | grep InstanceId | tr -d ' ' | tr -d ',' | tr -d '"' | sed "s/InstanceId://" | sed ':a;N;$!ba;s/\n/ /g'`
aws ec2 wait instance-running --instance-ids ${i_ids}
j=0
for i in ${i_ids}
do
ip=`aws ec2 describe-instances --instance-id $i | grep "PublicIpAddress" | tr -d ' ' | tr -d ',' | tr -d '"' | sed "s/PublicIpAddress://"`
pip=`aws ec2 describe-instances --instance-id $i | grep "PrivateIpAddress" | tr -d ' ' | tr -d ',' | tr -d '"' | sed "s/PrivateIpAddress://"`
num=`printf "%03d" $j`
j=`expr $j + 1`
echo "clustrix_${num}_network=$ip" >> $1_network_config
echo "clustrix_${num}_private=$pip" >> $1_network_config
echo "clustrix_${num}_keyfile=~/.config/mdbci/maxscale.pem" >> $1_network_config
echo "clustrix_${num}_hostname=clusterix${num}" >> $1_network_config
echo "clustrix_${num}_whoami=ec2-user" >> $1_network_config
done
labels=`cat $1_configured_labels`
echo "$labels,CLUSTRIX_BACKEND" > $1_configured_labels

View File

@ -16,6 +16,7 @@ const labels_table_t labels_table [] __attribute__((unused)) =
{"GALERA_BACKEND", "GALERA_BACKEND"},
{"TWO_MAXSCALES", "SECOND_MAXSCALE"},
{"COLUMNSTORE_BACKEND", "COLUMNSTORE_BACKEND"},
{"CLUSTRIX_BACKEND", "CLUSTRIX_BACKEND"},
};
/**

View File

@ -51,7 +51,7 @@ Mariadb_nodes::Mariadb_nodes(const char *pref, const char *test_cwd, bool verbos
truncate_mariadb_logs();
flush_hosts();
close_active_connections();
cnf_server_name = std::string(prefix);
cnf_server_name = std::string(prefix) + std::string("_server");
if (strcmp(prefix, "node") == 0)
{
cnf_server_name = std::string("server");

View File

@ -490,7 +490,7 @@ public:
* @brief cnf_servers Generates backend servers description for maxscale.cnf
* @return Servers description including IPs, ports
*/
std::string cnf_servers();
virtual std::string cnf_servers();
/**
* @brief cnf_servers_line Generates list of backend servers for serivces definition in maxscale.cnf

View File

@ -0,0 +1,375 @@
{
"node_000" :
{
"hostname" : "node000",
"box" : "centos_7_aws_large",
"memory_size" : "${vm_memory}",
"labels" : [
"REPL_BACKEND"
],
"product" : {
"name": "${product}",
"version": "${version}",
"cnf_template" : "server1.cnf",
"cnf_template_path": "${cnf_path}"
}
},
"node_001" :
{
"hostname" : "node001",
"box" : "centos_7_aws_large",
"memory_size" : "${vm_memory}",
"labels" : [
"REPL_BACKEND"
],
"product" : {
"name": "${product}",
"version": "${version}",
"cnf_template" : "server2.cnf",
"cnf_template_path": "${cnf_path}"
}
},
"node_002" :
{
"hostname" : "node002",
"box" : "centos_7_aws_large",
"memory_size" : "${vm_memory}",
"labels" : [
"REPL_BACKEND"
],
"product" : {
"name": "${product}",
"version": "${version}",
"cnf_template" : "server3.cnf",
"cnf_template_path": "${cnf_path}"
}
},
"node_003" :
{
"hostname" : "node003",
"box" : "centos_7_aws_large",
"memory_size" : "${vm_memory}",
"labels" : [
"REPL_BACKEND"
],
"product" : {
"name": "${product}",
"version": "${version}",
"cnf_template" : "server4.cnf",
"cnf_template_path": "${cnf_path}"
}
},
"node_004" :
{
"hostname" : "node004",
"box" : "centos_7_aws_large",
"memory_size" : "${vm_memory}",
"labels" : [
"BIG_REPL_BACKEND"
],
"product" : {
"name": "${product}",
"version": "${version}",
"cnf_template" : "server5.cnf",
"cnf_template_path": "${cnf_path}"
}
},
"node_005" :
{
"hostname" : "node005",
"box" : "centos_7_aws_large",
"memory_size" : "${vm_memory}",
"labels" : [
"BIG_REPL_BACKEND"
],
"product" : {
"name": "${product}",
"version": "${version}",
"cnf_template" : "server6.cnf",
"cnf_template_path": "${cnf_path}"
}
},
"node_006" :
{
"hostname" : "node006",
"box" : "centos_7_aws_large",
"memory_size" : "${vm_memory}",
"labels" : [
"BIG_REPL_BACKEND"
],
"product" : {
"name": "${product}",
"version": "${version}",
"cnf_template" : "server7.cnf",
"cnf_template_path": "${cnf_path}"
}
},
"node_007" :
{
"hostname" : "node007",
"box" : "centos_7_aws_large",
"memory_size" : "${vm_memory}",
"labels" : [
"BIG_REPL_BACKEND"
],
"product" : {
"name": "${product}",
"version": "${version}",
"cnf_template" : "server8.cnf",
"cnf_template_path": "${cnf_path}"
}
},
"node_008" :
{
"hostname" : "node008",
"box" : "centos_7_aws_large",
"memory_size" : "${vm_memory}",
"labels" : [
"BIG_REPL_BACKEND"
],
"product" : {
"name": "${product}",
"version": "${version}",
"cnf_template" : "server9.cnf",
"cnf_template_path": "${cnf_path}"
}
},
"node_009" :
{
"hostname" : "node009",
"box" : "centos_7_aws_large",
"memory_size" : "${vm_memory}",
"labels" : [
"BIG_REPL_BACKEND"
],
"product" : {
"name": "${product}",
"version": "${version}",
"cnf_template" : "server10.cnf",
"cnf_template_path": "${cnf_path}"
}
},
"node_010" :
{
"hostname" : "node010",
"box" : "centos_7_aws_large",
"memory_size" : "${vm_memory}",
"labels" : [
"BIG_REPL_BACKEND"
],
"product" : {
"name": "${product}",
"version": "${version}",
"cnf_template" : "server11.cnf",
"cnf_template_path": "${cnf_path}"
}
},
"node_011" :
{
"hostname" : "node011",
"box" : "centos_7_aws_large",
"memory_size" : "${vm_memory}",
"labels" : [
"BIG_REPL_BACKEND"
],
"product" : {
"name": "${product}",
"version": "${version}",
"cnf_template" : "server12.cnf",
"cnf_template_path": "${cnf_path}"
}
},
"node_012" :
{
"hostname" : "node012",
"box" : "centos_7_aws_large",
"memory_size" : "${vm_memory}",
"labels" : [
"BIG_REPL_BACKEND"
],
"product" : {
"name": "${product}",
"version": "${version}",
"cnf_template" : "server13.cnf",
"cnf_template_path": "${cnf_path}"
}
},
"node_013" :
{
"hostname" : "node013",
"box" : "centos_7_aws_large",
"memory_size" : "${vm_memory}",
"labels" : [
"BIG_REPL_BACKEND"
],
"product" : {
"name": "${product}",
"version": "${version}",
"cnf_template" : "server14.cnf",
"cnf_template_path": "${cnf_path}"
}
},
"node_014" :
{
"hostname" : "node013",
"box" : "centos_7_aws_large",
"memory_size" : "${vm_memory}",
"labels" : [
"BIG_REPL_BACKEND"
],
"product" : {
"name": "${product}",
"version": "${version}",
"cnf_template" : "server15.cnf",
"cnf_template_path": "${cnf_path}"
}
},
"galera_000" :
{
"hostname" : "galera000",
"box" : "centos_7_aws_large",
"memory_size" : "${vm_memory}",
"labels" : [
"GALERA_BACKEND"
],
"product" : {
"name": "galera",
"version": "${galera_version}",
"cnf_template" : "galera_server1.cnf",
"cnf_template_path": "${cnf_path}"
}
},
"galera_001" :
{
"hostname" : "galera001",
"box" : "centos_7_aws_large",
"memory_size" : "${vm_memory}",
"labels" : [
"GALERA_BACKEND"
],
"product" : {
"name": "galera",
"version": "${galera_version}",
"cnf_template" : "galera_server2.cnf",
"cnf_template_path": "${cnf_path}"
}
},
"galera_002" :
{
"hostname" : "galera002",
"box" : "centos_7_aws_large",
"memory_size" : "${vm_memory}",
"labels" : [
"GALERA_BACKEND"
],
"product" : {
"name": "galera",
"version": "${galera_version}",
"cnf_template" : "galera_server3.cnf",
"cnf_template_path": "${cnf_path}"
}
},
"galera_003" :
{
"hostname" : "galera003",
"box" : "centos_7_aws_large",
"memory_size" : "${vm_memory}",
"labels" : [
"GALERA_BACKEND"
],
"product" : {
"name": "galera",
"version": "${galera_version}",
"cnf_template" : "galera_server4.cnf",
"cnf_template_path": "${cnf_path}"
}
},
"maxscale_000" :
{
"hostname" : "maxscale",
"box" : "centos_7_aws_large",
"memory_size" : "${vm_memory}",
"labels" : [
"MAXSCALE"
],
"product" : {
"name" : "maxscale_ci",
"version" : "${target}"
}
},
"maxscale_001" :
{
"hostname" : "maxscale2",
"box" : "centos_7_aws_large",
"memory_size" : "${vm_memory}",
"labels" : [
"SECOND_MAXSCALE"
],
"product" : {
"name" : "maxscale_ci",
"version" : "${target}"
}
},
"clustrix_000" :
{
"hostname" : "clustrix000",
"box" : "centos_7_aws_large",
"memory_size" : "16192",
"labels" : [
"CLUSTRIX_BACKEND"
]
},
"clustrix_001" :
{
"hostname" : "clustrix001",
"box" : "centos_7_aws_large",
"memory_size" : "16192",
"labels" : [
"CLUSTRIX_BACKEND"
]
},
"clustrix_002" :
{
"hostname" : "clustrix002",
"box" : "centos_7_aws_large",
"memory_size" : "16192",
"labels" : [
"CLUSTRIX_BACKEND"
]
},
"clustrix_003" :
{
"hostname" : "clustrix003",
"memory_size" : "16192",
"box" : "centos_7_aws_large",
"labels" : [
"CLUSTRIX_BACKEND"
]
}
}

View File

@ -136,6 +136,7 @@ TestConnections::TestConnections(int argc, char* argv[])
, binlog_master_gtid(false)
, binlog_slave_gtid(false)
, no_galera(false)
, no_clustrix(false)
, no_vm_revert(true)
, threads(4)
, use_ipv6(false)
@ -320,7 +321,6 @@ TestConnections::TestConnections(int argc, char* argv[])
{
exit(MDBCI_FAUILT);
}
}
if (mdbci_labels.find(std::string("REPL_BACKEND")) == std::string::npos)
@ -341,6 +341,15 @@ TestConnections::TestConnections(int argc, char* argv[])
}
}
if (mdbci_labels.find(std::string("CLUSTRIX_BACKEND")) == std::string::npos)
{
no_clustrix = true;
if (verbose)
{
tprintf("No need to use Clustrix");
}
}
get_logs_command = (char *) malloc(strlen(test_dir) + 14);
sprintf(get_logs_command, "%s/get_logs.sh", test_dir);
@ -386,6 +395,20 @@ TestConnections::TestConnections(int argc, char* argv[])
galera = NULL;
}
if (!no_clustrix)
{
clustrix = new Clustrix_nodes("clustrix", test_dir, verbose, network_config);
//galera->use_ipv6 = use_ipv6;
clustrix->use_ipv6 = false;
clustrix->take_snapshot_command = take_snapshot_command;
clustrix->revert_snapshot_command = revert_snapshot_command;
clustrix->start_cluster();
}
else
{
clustrix = NULL;
}
maxscales = new Maxscales("maxscale", test_dir, verbose, network_config);
bool maxscale_ok = maxscales->check_nodes();
@ -757,12 +780,13 @@ void TestConnections::process_template(int m, const char* template_name, const c
sprintf(str, "sed -i \"s/###threads###/%d/\" maxscale.cnf", threads);
system(str);
Mariadb_nodes * mdn[2];
Mariadb_nodes * mdn[3];
char * IPcnf;
mdn[0] = repl;
mdn[1] = galera;
mdn[2] = clustrix;
int i, j;
int mdn_n = galera ? 2 : 1;
int mdn_n = 3;
for (j = 0; j < mdn_n; j++)
{
@ -2220,7 +2244,6 @@ int TestConnections::call_mdbci(const char * options)
team_keys +
std::string(" ") +
std::string(mdbci_config_name)).c_str() );
read_env();
if (repl)
{

View File

@ -1,6 +1,7 @@
#pragma once
#include "mariadb_nodes.h"
#include "clustrix_nodes.h"
#include "maxscales.h"
#include "templates.h"
#include <fcntl.h>
@ -104,6 +105,8 @@ public:
*/
Mariadb_nodes* repl;
Clustrix_nodes * clustrix;
/**
* @brief maxscales Maxscale object containing referebces to all Maxscale machines
*/
@ -230,6 +233,11 @@ public:
*/
bool no_galera;
/**
* @brief no_clustrix Do not check, restart and use Clustrix setup
*/
bool no_clustrix;
/**
* @brief no_vm_revert If true tests do not revert VMs after the test even if test failed
* (use it for debugging)