Merge branch '2.3' into 2.4

maxscale-system-test/maxtest/CMakeLists.txt

@@ -0,0 +1,2 @@
+include_directories(include/maxtest)
+add_subdirectory(src)

maxscale-system-test/maxtest/include/maxtest/appexception.h

@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2018 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2024-02-10
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#pragma once
+
+#include <stdexcept>
+#include <sstream>
+
+namespace base
+{
+
+// TODO Add back trace.
+class AppException : public std::runtime_error
+{
+public:
+    AppException(const std::string& msg,
+                 const std::string& file,
+                 int line)
+        : std::runtime_error(msg)
+        , m_file(file)
+        , m_line(line)
+    {
+    }
+private:
+    std::string m_file;
+    int         m_line;
+};
+}   // base
+
+#define DEFINE_EXCEPTION(Type) \
+    struct Type : public base::AppException { \
+        Type(const std::string& msg, \
+             const char* file, \
+             int line)   \
+            : AppException(msg, file, line) {} }
+
+#define THROW(Type, msg_str) \
+    do { \
+        std::ostringstream os; \
+        os << __FILE__ << ':' << __LINE__ << '\n' << msg_str; \
+        throw Type(os.str(), __FILE__, __LINE__);} while (false)

maxscale-system-test/maxtest/include/maxtest/big_load.h

@@ -0,0 +1,45 @@
+#pragma once
+
+#include "testconnections.h"
+#include "sql_t1.h"
+#include "get_com_select_insert.h"
+
+// pthread_mutex_t mutex1 = PTHREAD_MUTEX_INITIALIZER;
+typedef struct
+{
+    int              exit_flag;
+    long             i1;
+    long             i2;
+    int              rwsplit_only;
+    TestConnections* Test;
+} thread_data;
+
+void* query_thread1(void* ptr);
+void* query_thread2(void* ptr);
+
+/**
+ * @brief load Creates load on Maxscale routers
+ * @param new_inserts COM_INSERT variable values array for all nodes after test
+ * @param new_selects COM_SELECT variable values array for all nodes after test
+ * @param selects COM_SELECT variable values array for all nodes before test
+ * @param inserts COM_INSERT variable values array for all nodes before test
+ * @param threads_num Number of load threads
+ * @param Test TestConnections object
+ * @param i1 Number of queries executed by "fast" threads (no wating between queries)
+ * @param i2 Number of queries executed by "slow" threads (sleep 1 second between queries)
+ * @param rwsplit_only if 1 create load only on RWSplit router, do not load ReadConn router
+ * @param galera if true use Galera backend (Test->galera instead of Test->repl)
+ * @param report_errors if true call add_result() in case of query failure
+ */
+
+void load(long* new_inserts,
+          long* new_selects,
+          long* selects,
+          long* inserts,
+          int   threads_num,
+          TestConnections* Test,
+          long* i1,
+          long* i2,
+          int   rwsplit_only,
+          bool  galera,
+          bool  report_errors);

maxscale-system-test/maxtest/include/maxtest/big_transaction.h

@@ -0,0 +1,14 @@
+#pragma once
+
+#include <mysql.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include "sql_t1.h"
+
+/**
+ * @brief big_transaction Executes big transaction (includes N INSERTs of 10000 rows)
+ * @param conn MYSQL connection handler
+ * @param N Number of INSERTs
+ * @return 0 if success
+ */
+int big_transaction(MYSQL* conn, int N);

maxscale-system-test/maxtest/include/maxtest/blob_test.h

@@ -0,0 +1,37 @@
+#pragma once
+
+#include "testconnections.h"
+
+/**
+ * @brief test_longblob INSERT big amount of data into lobg_blob_table
+ * @param Test TestConnection object
+ * @param conn MYSQL connection handler
+ * @param blob_name blob type (LONGBLOB; MEDIUMBLOB or BLOB)
+ * @param chunk_size size of one data chunk (in sizeof(long usingned))
+ * @param chunks number of chunks to INSERT
+ * @param rows number of rows to INSERT (executes INSERT stetament 'rows' times)
+ * @return 0 in case of success
+ */
+int test_longblob(TestConnections* Test,
+                  MYSQL* conn,
+                  char*  blob_name,
+                  unsigned long chunk_size,
+                  int chunks,
+                  int rows);
+
+
+/**
+ * @brief check_longblob_data Does SELECT against table created by test_longblob() and cheks that data are
+ * correct
+ * @param Test TestConnection object
+ * @param conn MYSQL connection handler
+ * @param chunk_size size of one data chunk (in sizeof(long usingned))
+ * @param chunks number of chunks in the table
+ * @param rows number of rows in the table
+ * @return 0 in case of success
+ */
+int check_longblob_data(TestConnections* Test,
+                        MYSQL* conn,
+                        unsigned long chunk_size,
+                        int chunks,
+                        int rows);

maxscale-system-test/maxtest/include/maxtest/clustrix_nodes.h

@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2016 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2024-02-10
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+#pragma once
+
+/**
+ * @file clustrix_nodes.h - work with Clustrix setup
+ *
+ * ~/.config/mdbci/clustrix_license file have to contain SQL
+ * which setups license to the Clustrix node
+ *
+ * TODO: move functionality of install_clustrix() to MDBCI
+ */
+
+#include <errno.h>
+#include <string>
+#include "nodes.h"
+#include "mariadb_nodes.h"
+
+#define CLUSTRIX_DEPS_YUM "yum install -y bzip2 wget screen ntp ntpdate vim htop mdadm"
+#define WGET_CLUSTRIX     "wget http://files.clustrix.com/releases/software/clustrix-9.1.4.el7.tar.bz2"
+#define UNPACK_CLUSTRIX   "tar xvjf clustrix-9.1.4.el7.tar.bz2"
+#define INSTALL_CLUSTRIX  "cd clustrix-9.1.4.el7; sudo ./clxnode_install.py --yes --force"
+
+class Clustrix_nodes : public Mariadb_nodes
+{
+public:
+
+    Clustrix_nodes(const char* pref, const char* test_cwd, bool verbose, std::string network_config)
+        : Mariadb_nodes(pref, test_cwd, verbose, network_config)
+    {
+    }
+
+    /**
+     * @brief start_cluster Intstalls Clustrix on all nodes, configure license, form cluster
+     * @return 0 in case of success
+     */
+    int start_replication();
+
+    /**
+     * @brief cnf_servers Generate Clustrix servers description for maxscale.cnf
+     * @return text for maxscale.cnf
+     */
+    std::string cnf_servers();
+
+    /**
+     * @brief check_replication Checks if Clustrix Cluster is up and running
+     * @return 0 if Clustrix Cluster is ok
+     */
+    int check_replication();
+
+    /**
+     * @brief install_clustrix
+     * @param m node index
+     * @return 0 in case of success
+     */
+    int prepare_server(int i);
+
+    std::string block_command(int node) const override;
+    std::string unblock_command(int node) const override;
+};

maxscale-system-test/maxtest/include/maxtest/config_operations.h

@@ -0,0 +1,142 @@
+#pragma once
+
+#include "testconnections.h"
+#include <string>
+#include <set>
+
+class Config
+{
+public:
+    Config(TestConnections* parent);
+    ~Config();
+
+    /**
+     * Service identifiers for listener creation
+     */
+    enum Service
+    {
+        SERVICE_RWSPLIT      = 0,
+        SERVICE_RCONN_SLAVE  = 1,
+        SERVICE_RCONN_MASTER = 2
+    };
+
+    /**
+     * Add a server to all services and monitors
+     *
+     * @param num Backend number
+     */
+    void add_server(int num);
+
+    /**
+     * Add all created servers to an object
+     *
+     * @param object Object to add servers to
+     */
+    void add_created_servers(const char* object);
+
+    /**
+     * Remove a server
+     *
+     * @param num Backend number
+     */
+    void remove_server(int num);
+
+    /**
+     * Create a new server
+     *
+     * @param num Backend number
+     */
+    void create_server(int num);
+
+    /**
+     * Alter a server
+     *
+     * @param num Backend number
+     * @param key Key to alter
+     * @oaram value Value for @c key, empty string for no value
+     */
+    void alter_server(int num, const char* key, const char* value);
+    void alter_server(int num, const char* key, int value);
+    void alter_server(int num, const char* key, float value);
+
+    /**
+     * Destroy a server
+     * @param num Backend number
+     */
+    void destroy_server(int num);
+
+    /**
+     * Test that server count is at the expected amount
+     * @param expected How many servers are expected to exist
+     * @return True if the number of servers is @c expected
+     */
+    bool check_server_count(int expected);
+
+    /**
+     * Create the monitor
+     * @param type The name of the monitor module to use
+     * @param interval Monitoring interval
+     */
+    void create_monitor(const char* name, const char* module, int interval = 1000);
+
+    /**
+     * Start the created monitor
+     */
+    void start_monitor(const char* name);
+
+    /**
+     * Alter a monitor
+     * @param key Key to alter
+     * @oaram value Value for @c key, empty string for no value
+     */
+    void alter_monitor(const char* name, const char* key, const char* value);
+    void alter_monitor(const char* name, const char* key, int value);
+    void alter_monitor(const char* name, const char* key, float value);
+
+    /**
+     * Destroy the monitor
+     */
+    void destroy_monitor(const char* name);
+
+    /**
+     * Restart all created monitors
+     */
+    void restart_monitors();
+
+    /**
+     * Create a listener
+     *
+     * @param service Service where listener is created
+     */
+    void create_listener(Service service);
+
+
+    /**
+     * Create a listener with SSL enabled
+     *
+     * @param service Service where SSL listener is created
+     */
+    void create_ssl_listener(Service service);
+
+    /**
+     * Destroy a listener
+     *
+     * @param service Service whose listener is destroyed
+     */
+    void destroy_listener(Service service);
+
+    /**
+     * Create all basic listeners
+     */
+    void create_all_listeners();
+
+    /**
+     * Reset the configuration to a standard state
+     */
+    void reset();
+
+private:
+    TestConnections*      test_;
+    std::set<int>         created_servers_;
+    std::set<std::string> created_monitors_;
+};

maxscale-system-test/maxtest/include/maxtest/different_size.h

@@ -0,0 +1,36 @@
+#pragma once
+
+#include <iostream>
+#include <unistd.h>
+#include "testconnections.h"
+
+/**
+ * @brief create_event_size Creates SQL query to generate even of given size
+ * @param size desired size of event
+ * @return SQL query string
+ */
+char* create_event_size(unsigned long size);
+
+/**
+ * @brief connect_to_serv Open connection
+ * @param Test TestConnections object
+ * @param binlog if true - connects to Master, otherwise - to RWSplit router
+ * @return MYSQL handler
+ */
+MYSQL* connect_to_serv(TestConnections* Test, bool binlog);
+
+/**
+ * @brief set_max_packet Executes 'cmd' on Master of RWSplit ('cmd' should be 'set global
+ * max_paxket_size=...')
+ * @param Test TestConnections object
+ * @param binlog if true - connects to Master, otherwise - to RWSplit router
+ * @param cmd command to execute
+ */
+void set_max_packet(TestConnections* Test, bool binlog, char* cmd);
+
+/**
+ * @brief different_packet_size Tries INSERTs with size close to 0x0ffffff * N (N is 1, 2 and 3)
+ * @param Test TestConnections object
+ * @param binlog if true - connects to Master, otherwise - to RWSplit router
+ */
+void different_packet_size(TestConnections* Test, bool binlog);

maxscale-system-test/maxtest/include/maxtest/env.h

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2018 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2024-02-10
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+#pragma once
+
+#include <string>
+
+namespace base
+{
+
+/* Environment variable. Usage:
+ * Env user{"USER"};
+ * std::string home = Env{"HOME"};
+ * An environment variable can be empty() but
+ * still is_defined().
+ */
+class Env : public std::string
+{
+public:
+    Env(const std::string& name) : m_is_defined(false)
+    {
+        if (const char* var = getenv(name.c_str()))
+        {
+            m_is_defined = true;
+            this->assign(var);
+        }
+    }
+
+    bool is_defined() const
+    {
+        return m_is_defined;
+    }
+private:
+    bool m_is_defined;
+};
+}   // base

maxscale-system-test/maxtest/include/maxtest/envv.h

@@ -0,0 +1,30 @@
+#pragma once
+
+#include <stdio.h>
+#include <stdarg.h>
+#include <stdlib.h>
+
+/**
+ * @brief readenv Read enviromental variable, if emtpy - set dafault
+ * @param name Name of the variable
+ * @param format Default value
+ * @return Enviromental variable value
+ */
+char * readenv(const char * name, const char *format, ...);
+
+/**
+ * @brief readenv_int Read integer value of enviromental variable, if empty - set dafault
+ * @param name Name of the variable
+ * @param def Default value
+ * @return Enviromental variable value converted to int
+ */
+int readenv_int(const char * name, int def);
+
+/**
+ * @brief readenv_int Read boolean value of enviromental variable, if empty - set dafault
+ * Values 'yes', 'y', 'true' (case independedant) are interpreted as TRUE, everything else - as FALSE
+ * @param name Name of the variable
+ * @param def Default value
+ * @return Enviromental variable value converted to bool
+ */
+bool readenv_bool(const char * name, bool def);

maxscale-system-test/maxtest/include/maxtest/execute_cmd.h

@@ -0,0 +1,14 @@
+#pragma once
+
+#include <iostream>
+#include <unistd.h>
+
+using namespace std;
+
+/**
+ * @brief execute_cmd Execute shell command
+ * @param cmd Command line
+ * @param res Pointer to variable that will contain command console output (stdout)
+ * @return Process exit code
+ */
+int execute_cmd(char * cmd, char ** res);

maxscale-system-test/maxtest/include/maxtest/fw_copy_rules.h

@@ -0,0 +1,11 @@
+#pragma once
+
+#include "testconnections.h"
+
+/**
+ * @brief copy_rules Copy rules file for firewall filter to Maxscale machine
+ * @param Test TestConnections object
+ * @param rules_name Name of file to be copied
+ * @param rules_dir Directory where file is located
+ */
+void copy_rules(TestConnections* Test, const char* rules_name, const char* rules_dir);

maxscale-system-test/maxtest/include/maxtest/get_com_select_insert.h

@@ -0,0 +1,29 @@
+#pragma once
+
+#include "testconnections.h"
+
+/**
+ * @brief get_global_status_allnodes Reads COM_SELECT and COM_INSERT variables from all nodes and stores into
+ *'selects' and 'inserts'
+ * @param selects pointer to array to store COM_SELECT for all nodes
+ * @param inserts pointer to array to store COM_INSERT for all nodes
+ * @param nodes Mariadb_nodes object that contains information about nodes
+ * @param silent if 1 do not print anything
+ * @return 0 in case of success
+ */
+int get_global_status_allnodes(long int* selects, long int* inserts, Mariadb_nodes* nodes, int silent);
+
+/**
+ * @brief print_delta Prints difference in COM_SELECT and COM_INSERT
+ * @param new_selects pointer to array to store COM_SELECT for all nodes after test
+ * @param new_inserts pointer to array to store COM_INSERT for all nodes after test
+ * @param selects pointer to array to store COM_SELECT for all nodes before test
+ * @param inserts pointer to array to store COM_INSERT for all nodes before test
+ * @param NodesNum Number of nodes
+ * @return
+ */
+int print_delta(long int* new_selects,
+                long int* new_inserts,
+                long int* selects,
+                long int* inserts,
+                int nodes_num);

maxscale-system-test/maxtest/include/maxtest/get_my_ip.h

@@ -0,0 +1,10 @@
+#pragma once
+
+/**
+ * @brief get_my_ip Get IP address of machine where this code is executed as it is visible from remote machine
+ * Connects to DNS port 53 of remote machine and gets own IP from socket info
+ * @param remote_ip IP of remote machine
+ * @param my_ip Pointer to result (own IP string)
+ * @return 0 in case of success
+ */
+int get_my_ip(char * remote_ip, char *my_ip );

maxscale-system-test/maxtest/include/maxtest/keepalived_func.h

@@ -0,0 +1,10 @@
+#pragma once
+
+#include "testconnections.h"
+
+#define FAILOVER_WAIT_TIME 20
+
+char virtual_ip[27];
+char* print_version_string(TestConnections* Test);
+void  configure_keepalived(TestConnections* Test, char* keepalived_file);
+void  stop_keepalived(TestConnections* Test);

maxscale-system-test/maxtest/include/maxtest/labels_table.h

@@ -0,0 +1,39 @@
+#pragma once
+
+#include <string>
+
+struct labels_table_t
+{
+    std::string test_label;
+    std::string mdbci_label;
+
+};
+
+const labels_table_t labels_table [] __attribute__((unused)) =
+{
+    {"REPL_BACKEND", "REPL_BACKEND"},
+    {"BIG_REPL_BACKEND", "BIG_REPL_BACKEND"},
+    {"GALERA_BACKEND", "GALERA_BACKEND"},
+    {"TWO_MAXSCALES", "SECOND_MAXSCALE"},
+    {"COLUMNSTORE_BACKEND", "COLUMNSTORE_BACKEND"},
+    {"CLUSTRIX_BACKEND", "CLUSTRIX_BACKEND"},
+};
+
+/**
+ * @brief get_mdbci_lables Finds all MDBCI labels which are needed by test
+ * Every test has a number of labels defined in the CMakeLists.txt,
+ * some of these lables defines which nodes (virtual machines) are needed
+ * for this particular test. Function finds such labels and forms labels string
+ * in the 'mdbci up' command format
+ * @param labels_string All lables from CMakeLists.txt
+ * @return Labels string in the 'mdbci up' --labels parameter format
+ */
+std::string get_mdbci_lables(const char * labels_string);
+
+/**
+ * @brief check_label Checks if givel lable belogs to current test labels
+ * @param labels String with all labels of the test
+ * @param label Labels to find
+ * @return true if label present
+ */
+bool has_label(std::string labels, std::string label);

maxscale-system-test/maxtest/include/maxtest/mariadb_func.h

@@ -0,0 +1,369 @@
+#pragma once
+
+/**
+ * @file mariadb_func.h - basic DB interaction routines
+ *
+ * @verbatim
+ * Revision History
+ *
+ * Date     Who     Description
+ * 17/11/14 Timofey Turenko Initial implementation
+ *
+ * @endverbatim
+ */
+
+#include <mysql.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <math.h>
+#include <time.h>
+#include <stdarg.h>
+#include <errno.h>
+#include <string>
+#include <vector>
+
+#include <maxbase/ccdefs.hh>
+
+typedef std::vector<std::string> Row;
+typedef std::vector<Row>         Result;
+
+/**
+ * Opens connection to DB: wropper over mysql_real_connect
+ *
+ * @param port  DB server port
+ * @param ip    DB server IP address
+ * @param db    name of DB to connect
+ * @param user  user name
+ * @param password  password
+ * @param flag  Connections flags
+ * @param ssl   true if ssl should be used
+ *
+ * @return MYSQL struct
+ */
+MYSQL* open_conn_db_flags(int port,
+                          std::string ip,
+                          std::string db,
+                          std::string user,
+                          std::string password,
+                          unsigned long flag,
+                          bool ssl);
+
+
+/**
+ * Opens connection to DB: wropper over mysql_real_connect
+ *
+ * @param port  DB server port
+ * @param ip    DB server IP address
+ * @param db    name of DB to connect
+ * @param user  user name
+ * @param password  password
+ * @param timeout  timeout on seconds
+ * @param ssl   true if ssl should be used
+ *
+ * @return MYSQL struct
+ */
+MYSQL* open_conn_db_timeout(int port,
+                            std::string ip,
+                            std::string db,
+                            std::string user,
+                            std::string password,
+                            unsigned int timeout,
+                            bool ssl);
+
+/**
+ * Opens connection to DB with default flags
+ *
+ * @param port  DB server port
+ * @param ip    DB server IP address
+ * @param db    name of DB to connect
+ * @param user  user name
+ * @param password  password
+ * @param ssl   true if ssl should be used
+ *
+ * @return MYSQL struct
+ */
+static MYSQL* open_conn_db(int port,
+                           std::string ip,
+                           std::string db,
+                           std::string user,
+                           std::string password,
+                           bool ssl = false)
+{
+    return open_conn_db_flags(port, ip, db, user, password, CLIENT_MULTI_STATEMENTS, ssl);
+}
+
+/**
+ * Opens connection to 'test' with default flags
+ *
+ * @param port  DB server port
+ * @param ip    DB server IP address
+ * @param user  user name
+ * @param password  password
+ * @param ssl   true if ssl should be used
+ *
+ * @return MYSQL struct
+ */
+static MYSQL* open_conn(int port, std::string ip, std::string user, std::string password, bool ssl = false)
+{
+    return open_conn_db(port, ip.c_str(), "test", user.c_str(), password.c_str(), ssl);
+}
+
+/**
+ * Opens connection to with default flags without defning DB name (just conecto server)
+ *
+ * @param port  DB server port
+ * @param ip    DB server IP address
+ * @param user  user name
+ * @param password  password
+ * @param ssl   true if ssl should be used
+ *
+ * @return MYSQL struct
+ */
+static MYSQL* open_conn_no_db(int port,
+                              std::string ip,
+                              std::string user,
+                              std::string password,
+                              bool ssl = false)
+{
+    return open_conn_db_flags(port, ip, "", user, password, CLIENT_MULTI_STATEMENTS, ssl);
+}
+
+/**
+ * @brief Executes SQL query. Function also executes mysql_store_result() and mysql_free_result() to clean up
+ * returns
+ * @param conn      MYSQL connection
+ * @param format    SQL string with printf style formatting
+ * @param ...       Parameters for @c format
+ * @return 0 in case of success
+ */
+int execute_query(MYSQL* conn, const char* format, ...) mxb_attribute((format(printf, 2, 3)));
+
+/**
+ * @brief execute_query_from_file Read a line from a file, trim leading and trailing whitespace and execute
+ * it.
+ * @param conn MYSQL handler
+ * @param file file handler
+ * @return 0 in case of success
+ */
+int execute_query_from_file(MYSQL* conn, FILE* file);
+
+/**
+ * @brief Executes SQL query. Function also executes mysql_store_result() and mysql_free_result() to clean up
+ * returns
+ * @param conn MYSQL connection struct
+ * @param sql   SQL string
+ * @return 0 in case of success
+ */
+int execute_query_silent(MYSQL* conn, const char* sql, bool silent = true);
+
+/**
+ * @brief Executes SQL query and store 'affected rows' number in affectet_rows parameter
+ * @param conn MYSQL    connection struct
+ * @param sql   SQL string
+ * @param affected_rows pointer to variabe to store number of affected rows
+ * @return 0 in case of success
+ */
+int execute_query_affected_rows(MYSQL* conn, const char* sql, my_ulonglong* affected_rows);
+
+/**
+ * @brief A more convenient form of execute_query_affected_rows()
+ *
+ * @param conn Connection to use for the query
+ * @param sql  The SQL statement to execute
+ * @return Number of rows or -1 on error
+ */
+int execute_query_count_rows(MYSQL* conn, const char* sql);
+
+/**
+ * @brief Executes SQL query and get number of rows in the result
+ * This function does not check boudaries of 'num_of_rows' array. This
+ * array have to be big enough to store all results
+ * @param conn MYSQL    connection struct
+ * @param sql   SQL string
+ * @param num_of_rows pointer to array to store number of result rows
+ * @param i pointer to variable to store number of result sets
+ * @return 0 in case of success
+ */
+int execute_query_num_of_rows(MYSQL* conn,
+                              const char* sql,
+                              my_ulonglong* num_of_rows,
+                              unsigned long long* i);
+
+/**
+ * @brief Executes perared statement and get number of rows in the result
+ * This function does not check boudaries of 'num_of_rows' array. This
+ * array have to be big enough to store all results
+ * @param stmt MYSQL_STMT statetement struct (from mysql_stmt_init())
+ * @param num_of_rows pointer to array to store number of result rows
+ * @param i pointer to variable to store number of result sets
+ * @return 0 in case of success
+ */
+int execute_stmt_num_of_rows(MYSQL_STMT* stmt, my_ulonglong* num_of_rows, unsigned long long* i);
+
+/**
+ * @brief execute_query_check_one Executes query and check if first field of first row is equal to 'expected'
+ * @param conn MYSQL handler
+ * @param sql query SQL query to execute
+ * @param expected Expected result
+ * @return 0 in case of success
+ */
+int execute_query_check_one(MYSQL* conn, const char* sql, const char* expected);
+
+/**
+ * @brief Executes 'show processlist' and calculates number of connections from defined host to defined DB
+ * @param conn MYSQL    connection struct
+ * @param ip    connections from this IP address are counted
+ * @param db    name of DB to which connections are counted
+ * @return number of connections
+ */
+int get_conn_num(MYSQL* conn, std::string ip, std::string hostname, std::string db);
+
+/**
+ * @brief Find given filed in the SQL query reply
+ * Function checks only firs row from the table
+ * @param conn MYSQL    connection struct
+ * @param sql   SQL query to execute
+ * @param filed_name    name of field to find
+ * @param value pointer to variable to store value of found field
+ * @return 0 in case of success
+ */
+int find_field(MYSQL* conn, const char* sql, const char* field_name, char* value);
+
+/**
+ * Execute a query and return the first row
+ *
+ * @param conn The connection to use
+ * @param sql  The query to execute
+ *
+ * @return The first row as a list of strings
+ */
+Row get_row(MYSQL* conn, std::string sql);
+
+/**
+ * Execute a query and return the result
+ *
+ * @param conn The connection to use
+ * @param sql  The query to execute
+ *
+ * @return The result as a list of rows
+ */
+Result get_result(MYSQL* conn, std::string sql);
+
+int get_int_version(std::string version);
+
+// Helper class for performing queries
+class Connection
+{
+public:
+    Connection(Connection&) = delete;
+    Connection& operator=(Connection&) = delete;
+
+    Connection(std::string host,
+               int port,
+               std::string user,
+               std::string password,
+               std::string db = "",
+               bool ssl = false)
+        : m_host(host)
+        , m_port(port)
+        , m_user(user)
+        , m_pw(password)
+        , m_db(db)
+        , m_ssl(ssl)
+    {
+    }
+
+    Connection(Connection&& rhs)
+        : m_host(rhs.m_host)
+        , m_port(rhs.m_port)
+        , m_user(rhs.m_user)
+        , m_pw(rhs.m_pw)
+        , m_db(rhs.m_db)
+        , m_ssl(rhs.m_ssl)
+        , m_conn(rhs.m_conn)
+    {
+        rhs.m_conn = nullptr;
+    }
+
+    virtual ~Connection()
+    {
+        mysql_close(m_conn);
+    }
+
+    bool connect()
+    {
+        mysql_close(m_conn);
+        m_conn = open_conn_db(m_port, m_host, m_db, m_user, m_pw, m_ssl);
+        return m_conn != nullptr && mysql_errno(m_conn) == 0;
+    }
+
+    void disconnect()
+    {
+        mysql_close(m_conn);
+        m_conn = nullptr;
+    }
+
+    bool query(std::string q)
+    {
+        return execute_query_silent(m_conn, q.c_str()) == 0;
+    }
+
+    bool check(std::string q, std::string res)
+    {
+        Row row = get_row(m_conn, q);
+        return !row.empty() && row[0] == res;
+    }
+
+    Row row(std::string q)
+    {
+        return get_row(m_conn, q);
+    }
+
+    Result rows(const std::string& q) const
+    {
+        return get_result(m_conn, q);
+    }
+
+    std::string field(std::string q, int idx = 0)
+    {
+        Row r = get_row(m_conn, q);
+        return r.empty() ? std::string() : r[idx];
+    }
+
+    const char* error() const
+    {
+        return mysql_error(m_conn);
+    }
+
+    bool change_user(std::string user, std::string pw, std::string db = "test")
+    {
+        return mysql_change_user(m_conn, user.c_str(), pw.c_str(), db.c_str()) == 0;
+    }
+
+    bool reset_connection()
+    {
+        return change_user(m_user, m_pw, m_db);
+    }
+
+    void set_credentials(const std::string& user, const std::string pw)
+    {
+        m_user = user;
+        m_pw = pw;
+    }
+
+    uint32_t thread_id() const
+    {
+        return mysql_thread_id(m_conn);
+    }
+
+private:
+    std::string m_host;
+    int         m_port;
+    std::string m_user;
+    std::string m_pw;
+    std::string m_db;
+    bool        m_ssl;
+    MYSQL*      m_conn = nullptr;
+};

maxscale-system-test/maxtest/include/maxtest/mariadb_nodes.h

@@ -0,0 +1,553 @@
+#pragma once
+
+/**
+ * @file mariadb_nodes.h - backend nodes routines
+ *
+ * @verbatim
+ * Revision History
+ *
+ * Date     Who     Description
+ * 17/11/14 Timofey Turenko Initial implementation
+ *
+ * @endverbatim
+ */
+
+
+#include "mariadb_func.h"
+#include <errno.h>
+#include <string>
+#include "nodes.h"
+
+/**
+ * @brief A class to handle backend nodes
+ * Contains references up to 256 nodes, info about IP, port, ssh key, use name and password for each node
+ * Node parameters should be defined in the enviromental variables in the follwing way:
+ * prefix_N - N number of nodes in the setup
+ * prefix_NNN - IP adress of the node (NNN 3 digits node index)
+ * prefix_port_NNN - MariaDB port number of the node
+ * prefix_User - User name to access backend setup (should have full access to 'test' DB with GRANT OPTION)
+ * prefix_Password - Password to access backend setup
+ */
+class Mariadb_nodes : public Nodes
+{
+public:
+    /**
+     * @brief Constructor
+     * @param pref  name of backend setup (like 'repl' or 'galera')
+     */
+    Mariadb_nodes(const char *pref, const char *test_cwd, bool verbose, const std::string& network_config);
+
+    bool setup() override;
+
+    virtual ~Mariadb_nodes();
+
+    /**
+     * @brief  MYSQL structs for every backend node
+     */
+    MYSQL* nodes[256];
+    /**
+     * @brief  IP address strings for every backend node
+     */
+
+    /**
+     * @brief  MariaDB port for every backend node
+     */
+    int port[256];
+    /**
+     * @brief Unix socket to connecto to MariaDB
+     */
+    char * socket[256];
+    /**
+     * @brief 'socket=$socket' line
+     */
+    char * socket_cmd[256];
+
+    /**
+     * @brief   User name to access backend nodes
+     */
+    char * user_name;
+    /**
+     * @brief   Password to access backend nodes
+     */
+    char * password;
+    /**
+     * @brief master index of node which was last configured to be Master
+     */
+    int master;
+
+    /**
+     * @brief start_db_command Command to start DB server
+     */
+    char * start_db_command[256];
+
+    /**
+     * @brief stop_db_command Command to start DB server
+     */
+    char * stop_db_command[256];
+
+    /**
+     * @brief cleanup_db_command Command to remove all
+     * data files and re-install DB with mysql_install_db
+     */
+    char * cleanup_db_command[256];
+
+    /**
+     * @brief ssl if true ssl will be used
+     */
+    int ssl;
+
+    /**
+     * @brief no_set_pos if set to true setup_binlog function do not set log position
+     */
+    bool no_set_pos;
+
+    /**
+     * @brief version Value of @@version
+     */
+    char version[256][256];
+
+    /**
+     * @brief version major part of number value of @@version
+     */
+    char version_major[256][256];
+
+    /**
+     * @brief version Number part of @@version
+     */
+    char version_number[256][256];
+
+    /**
+     * @brief connect open connections to all nodes
+     * @return 0  in case of success
+     */
+
+    /**
+     * @brief v51 true indicates that one backed is 5.1
+     */
+    bool v51;
+
+    /**
+     * @brief test_dir path to test application
+     */
+    char test_dir[4096];
+
+    /**
+     * @brief List of blocked nodes
+     */
+    bool blocked[256];
+
+    /**
+     * @brief  Open connctions to all backend nodes (to 'test' DB)
+     * @return 0 in case of success
+     */
+
+    /**
+     * @brief make_snapshot_command Command line to create a snapshot of all VMs
+     */
+    char* take_snapshot_command;
+
+    /**
+     * @brief revert_snapshot_command Command line to revert a snapshot of all VMs
+     */
+    char* revert_snapshot_command;
+
+    int connect(int i, const std::string& db = "test");
+    int connect(const std::string& db = "test");
+
+    /**
+     * Get a Connection to a node
+     */
+    Connection get_connection(int i, const std::string& db = "test")
+    {
+        return Connection(IP[i], port[i], user_name, password, db, ssl);
+    }
+
+    /**
+     * Repeatedly try to connect with one second sleep in between attempts
+     *
+     * @return True on success
+     */
+    bool robust_connect(int n);
+
+    /**
+     * @brief Close connections opened by connect()
+     *
+     * This sets the values of used @c nodes to NULL.
+     */
+    void close_connections();
+
+    // Alias for close_connections()
+    void disconnect()
+    {
+        close_connections();
+    }
+
+    /**
+     * @brief reads IP, Ports, sshkeys for every node from enviromental variables as well as number of nodes
+     *(N) and  User/Password
+     */
+    void read_env();
+    /**
+     * @brief  prints all nodes information
+     * @return 0
+     */
+    void print_env();
+
+    /**
+     * @brief find_master Tries to find Master node
+     * @return Index of Master node
+     */
+    int find_master();
+    /**
+     * @brief change_master set a new master node for Master/Slave setup
+     * @param NewMaster index of new Master node
+     * @param OldMaster index of current Master node
+     */
+    void change_master(int NewMaster, int OldMaster);
+
+    /**
+     * @brief stop_nodes stops mysqld on all nodes
+     * @return  0 in case of success
+     */
+    int stop_nodes();
+
+    /**
+     * @brief stop_slaves isues 'stop slave;' to all nodes
+     * @return  0 in case of success
+     */
+    int stop_slaves();
+
+    /**
+     * @brief cleanup_db_node Removes all data files and reinstall DB
+     * with mysql_install_db
+     * @param node
+     * @return 0 in case of success
+     */
+    int cleanup_db_node(int node);
+
+    /**
+     * @brief cleanup_db_node Removes all data files and reinstall DB
+     * with mysql_install_db for all nodes
+     * @param node
+     * @return 0 in case of success
+     */
+    int cleanup_db_nodes();
+
+    /**
+     * @brief configures nodes and starts Master/Slave replication
+     * @return  0 in case of success
+     */
+    virtual int start_replication();
+
+    // Create the default users used by all tests
+    void create_users(int node);
+
+    /**
+     * @param node Index of node to block.
+     * @return The command used for blocking a node.
+     */
+    virtual std::string block_command(int node) const;
+
+    /**
+     * @param node Index of node to unblock.
+     * @return The command used for unblocking a node.
+     */
+    virtual std::string unblock_command(int node) const;
+
+    /**
+     * @brif BlockNode setup firewall on a backend node to block MariaDB port
+     * @param node Index of node to block
+     * @return 0 in case of success
+     */
+    int block_node(int node);
+
+    /**
+     * @brief UnblockNode setup firewall on a backend node to unblock MariaDB port
+     * @param node Index of node to unblock
+     * @return 0 in case of success
+     */
+    int unblock_node(int node);
+
+
+    /**
+     * @brief Unblock all nodes for this cluster
+     * @return 0 in case of success
+     */
+    int unblock_all_nodes();
+
+    /**
+     * @brief clean_iptables removes all itables rules connected to MariaDB port to avoid duplicates
+     * @param node Index of node to clean
+     * @return 0 in case of success
+     */
+    int clean_iptables(int node);
+
+    /**
+     * @brief Stop DB server on the node
+     * @param node Node index
+     * @return 0 if success
+     */
+    int stop_node(int node);
+
+    /**
+     * @brief Start DB server on the node
+     * @param node Node index
+     * @param param command line parameters for DB server start command
+     * @return 0 if success
+     */
+    int start_node(int node, const char* param = "");
+
+    /**
+     * @brief Check if all slaves have "Slave_IO_Running" set to "Yes" and master has N-1 slaves
+     * @param master Index of master node
+     * @return 0 if everything is ok
+     */
+    virtual int check_replication();
+
+    /**
+     * @brief executes 'CHANGE MASTER TO ..' and 'START SLAVE'
+     * @param MYSQL conn struct of slave node
+     * @param master_host IP address of master node
+     * @param master_port port of master node
+     * @param log_file name of log file
+     * @param log_pos initial position
+     * @return 0 if everything is ok
+     */
+    int set_slave(MYSQL* conn, char master_host[], int master_port, char log_file[], char log_pos[]);
+
+    /**
+     * @brief Creates 'repl' user on all nodes
+     * @return 0 if everything is ok
+     */
+    int set_repl_user();
+
+    /**
+     * @brief Get the server_id of the node
+     * @param index The index of the node whose server_id to retrieve
+     * @return Node id of the server or -1 on error
+     */
+    int         get_server_id(int index);
+    std::string get_server_id_str(int index);
+
+    /**
+     * Get server IDs of all servers
+     *
+     * @return List of server IDs
+     */
+    std::vector<int> get_all_server_ids();
+    std::vector<std::string> get_all_server_ids_str();
+
+    /**
+     * @brief Execute 'mysqladmin flush-hosts' on all nodes
+     * @return 0 in case of success
+     */
+    int flush_hosts();
+
+    /**
+     * @brief Execute query on all nodes
+     * @param sql query to execute
+     * @return 0 in case of success
+     */
+    int execute_query_all_nodes(const char* sql);
+
+    /**
+     * @brief execute 'SELECT @@version' against one node and store result in 'version' field
+     * @param i Node index
+     * @return 0 in case of success
+     */
+    int get_version(int i);
+
+    /**
+     * @brief execute 'SELECT @@version' against all nodes and store result in 'version' field
+     * @return 0 in case of success
+     */
+    int get_versions();
+
+    /**
+     * @brief Return lowest server version in the cluster
+     * @return The version string of the server with the lowest version number
+     */
+    std::string get_lowest_version();
+
+    /**
+     * @brief truncate_mariadb_logs clean ups MariaDB logs on backend nodes
+     * @return 0 if success
+     */
+    int truncate_mariadb_logs();
+
+    /**
+     * @brief configure_ssl Modifies my.cnf in order to enable ssl, redefine access user to require ssl
+     * @return 0 if success
+     */
+    int configure_ssl(bool require);
+
+    /**
+     * @brief disable_ssl Modifies my.cnf in order to get rid of ssl, redefine access user to allow
+     * connections without ssl
+     * @return 0 if success
+     */
+    int disable_ssl();
+
+    /**
+     * @brief Synchronize slaves with the master
+     *
+     * Only works with master-slave replication and should not be used with Galera clusters.
+     * The function expects that the first node, @c nodes[0], is the master.
+     */
+    virtual void sync_slaves(int node = 0);
+
+    /**
+     * @brief Close all connections to this node
+     *
+     * This will kill all connections that have been created to this node.
+     */
+    void close_active_connections();
+
+    /**
+     * @brief Check and fix replication
+     */
+    bool fix_replication();
+
+    /**
+     * Copy current server settings to a backup directory. Any old backups are overwritten.
+     *
+     * @param node Node to modify
+     */
+    void stash_server_settings(int node);
+
+    /**
+     * Restore server settings from a backup directory. Current settings files are overwritten and
+     * backup settings files are removed.
+     *
+     * @param node Node to modify
+     */
+    void restore_server_settings(int node);
+
+    /**
+     * Comment any line starting with the given setting name in server settings files.
+     *
+     * @param node Node to modify
+     * @param setting Setting to remove
+     */
+    void disable_server_setting(int node, const char* setting);
+
+    /**
+     * Add the following lines to the /etc/mysql.cnf.d/server.cnf-file:
+     * [server]
+     * parameter
+     *
+     * @param node Node to modify
+     * @param setting Line to add
+     */
+    void add_server_setting(int node, const char* setting);
+
+    /**
+     * Get the configuration file name for a particular node
+     *
+     * @param node Node number for which the configuration is requested
+     *
+     * @return The name of the configuration file
+     */
+    virtual std::string get_config_name(int node);
+
+    /**
+     * Restore the original configuration for all servers
+     */
+    void reset_server_settings();
+    // Same but for an individual server
+    void reset_server_settings(int node);
+
+    /**
+     * @brief revert_nodes_snapshot Execute MDBCI snapshot revert command for all nodes
+     * @return true in case of success
+     */
+    bool revert_nodes_snapshot();
+
+    /**
+     * @brief prepare_server Initialize MariaDB setup (run mysql_install_db) and create test users
+     * Tries to detect Mysql 5.7 installation and disable 'validate_password' pluging
+     * @param i Node index
+     * @return 0 in case of success
+     */
+    virtual int prepare_server(int i);
+    int         prepare_servers();
+
+    /**
+     * Static functions
+     */
+
+    /** Whether to require GTID based replication, defaults to false */
+    static void require_gtid(bool value);
+
+    /**
+     * Configure a server as a slave of another server
+     *
+     * The servers are configured with GTID replicating using the configured
+     * GTID position, either slave_pos or current_pos.
+     *
+     * @param slave  The node index to assign as slave
+     * @param master The node index of the master
+     * @param type   Replication type
+     */
+    void replicate_from(int slave, int master, const char* type = "current_pos");
+
+    // Replicates from a host and a port instead of a known server
+    void replicate_from(int slave, const std::string& host, uint16_t port, const char* type = "current_pos");
+
+    /**
+     * @brief limit_nodes Restart replication for only new_N nodes
+     * @param new_N new number of nodes in replication
+     */
+    void limit_nodes(int new_N);
+
+    /**
+     * @brief cnf_servers Generates backend servers description for maxscale.cnf
+     * @return Servers description including IPs, ports
+     */
+    virtual std::string cnf_servers();
+
+    /**
+     * @brief cnf_servers_line Generates list of backend servers for serivces definition in maxscale.cnf
+     * @return List of servers, e.g server1,server2,server3,...
+     */
+    std::string cnf_servers_line();
+
+    /**
+     * @brief cnf_server_name Prefix for backend server name ('server', 'gserver')
+     */
+    std::string cnf_server_name;
+
+private:
+
+    bool check_master_node(MYSQL* conn);
+    bool bad_slave_thread_status(MYSQL* conn, const char* field, int node);
+};
+
+class Galera_nodes : public Mariadb_nodes
+{
+public:
+
+    Galera_nodes(const char *pref, const char *test_cwd, bool verbose, const std::string& network_config)
+        : Mariadb_nodes(pref, test_cwd, verbose, network_config) { }
+
+    int start_galera();
+
+    virtual int start_replication()
+    {
+        return start_galera();
+    }
+
+    int check_galera();
+
+    virtual int check_replication()
+    {
+        return check_galera();
+    }
+
+    std::string get_config_name(int node) override;
+
+    virtual void sync_slaves(int node = 0)
+    {
+        sleep(10);
+    }
+};

maxscale-system-test/maxtest/include/maxtest/maxadmin_operations.h

@@ -0,0 +1,105 @@
+#pragma once
+
+#include <stdio.h>
+#include <string.h>
+#include <signal.h>
+#include <sys/wait.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <netdb.h>
+#include <ctype.h>
+#include <stdlib.h>
+#include <termios.h>
+#include <unistd.h>
+#include <dirent.h>
+#include <locale.h>
+#include <errno.h>
+#include <getopt.h>
+
+
+/**
+ * @brief Connect to the MaxScale server
+ *
+ * @param hostname  The hostname to connect to
+ * @param port      The port to use for the connection
+ * @return      The connected socket or -1 on error
+ */
+int connectMaxScale(char* hostname, char* port);
+
+/**
+ * @brief Set IP address in socket structure in_addr
+ *
+ * @param a Pointer to a struct in_addr into which the address is written
+ * @param p The hostname to lookup
+ * @return  1 on success, 0 on failure
+ */
+int setipaddress(struct in_addr* a, char* p);
+
+
+/**
+ * @brief Perform authentication using the maxscaled protocol conventions
+ *
+ * @param so        The socket connected to MaxScale
+ * @param user      The username to authenticate
+ * @param password  The password to authenticate with
+ * @return      Non-zero of succesful authentication
+ */
+int authMaxScale(int so, char* user, char* password);
+
+/**
+ * @brief Send a comamnd using the MaxScaled protocol, display the return data on standard output.
+ *
+ * Input terminates with a lien containing just the text OK
+ *
+ * @param so    The socket connect to MaxScale
+ * @param cmd   The command to send
+ * @return  0 if the connection was closed
+ */
+int sendCommand(int so, char* cmd, char* buf);
+
+
+/**
+ * @brief Send a comamnd using the MaxScaled protocol, search for certain numeric parameter in MaxScaled
+ * output.
+ *
+ * Input terminates with a lien containing just the text OK
+ *
+ * @param user      The username to authenticate
+ * @param password  The password to authenticate with
+ * @param cmd       The command to send
+ * @param param     Parameter to find
+ * @param result    Value of found parameter
+ * @return  0 if parameter is found
+ */
+int get_maxadmin_param_tcp(char* hostname,
+                           char* user,
+                           char* password,
+                           char* command,
+                           char* param,
+                           char* result);
+
+/**
+ * @brief Send a comamnd using the MaxScaled protocol
+ *
+ * Input terminates with a line containing just the text OK
+ *
+ * @param user      The username to authenticate
+ * @param password  The password to authenticate with
+ * @param cmd       The command to send
+ * @return  0 if parameter is found
+ */
+int execute_maxadmin_command_tcp(char* hostname, char* user, char* password, char* cmd);
+
+/**
+ * @brief Send a comamnd using the MaxScaled protocol, print results of stdout
+ *
+ * Input terminates with a line containing just the text OK
+ *
+ * @param user      The username to authenticate
+ * @param password  The password to authenticate with
+ * @param cmd       The command to send
+ * @return  0 if parameter is found
+ */
+int execute_maxadmin_command_print_pcp(char* hostname, char* user, char* password, char* cmd);

maxscale-system-test/maxtest/include/maxtest/maxinfo_func.h

@@ -0,0 +1,21 @@
+#pragma once
+
+int   create_tcp_socket();
+char* get_ip(char* host);
+char* build_get_query(char* host, const char* page);
+
+/**
+ * @brief get_maxinfo does request to Maxinfo service and return response JSON
+ * @param page retrived info name
+ * @param Test TestConnection object
+ * @return response from Maxinfo
+ */
+char* get_maxinfo(const char* page, TestConnections* Test);
+
+char* read_sc(int sock);
+int   send_so(int sock, char* data);
+static char hexconvtab[] __attribute__ ((unused)) = "0123456789abcdef";
+static char* bin2hex(const unsigned char* old, const size_t oldlen);
+char*        cdc_auth_srt(char* user, char* password);
+int          setnonblocking(int sock);
+int          get_x_fl_from_json(char* line, long long int* x1, long long int* fl);

maxscale-system-test/maxtest/include/maxtest/maxrest.hh

@@ -0,0 +1,245 @@
+/*
+ * Copyright (c) 2016 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2024-02-10
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+#pragma once
+
+#include "testconnections.h"
+#include <maxscale/jansson.hh>
+
+/**
+ * @class MaxRest
+ *
+ * MaxRest is a class that (eventually) provides the same functionality as
+ * the command line program maxctrl, but for use in programs.
+ */
+class MaxRest
+{
+public:
+    MaxRest(const MaxRest&) = delete;
+    MaxRest& operator=(const MaxRest&) = delete;
+
+    /**
+     * A class corresponding to a row in the output of 'maxctrl list servers'
+     */
+    struct Server
+    {
+        Server() = default;
+        Server(const MaxRest& maxrest, json_t* pObject);
+
+        std::string name;
+        std::string address;
+        int64_t     port;
+        int64_t     connections;
+        std::string state;
+    };
+
+    /**
+     * Constructor
+     *
+     * @param pTest  The TestConnections instance. Must exist for the lifetime
+     *               of the MaxRest instance.
+     */
+    MaxRest(TestConnections* pTest);
+
+    /**
+     * @return  The TestConnections instance used by this instance.
+     */
+    TestConnections& test() const
+    {
+        return m_test;
+    }
+
+    /**
+     * @return The JSON object corresponding to /v1/servers/:id:
+     */
+    std::unique_ptr<json_t> v1_servers(const std::string& id) const;
+
+    /**
+     * @return The JSON object corresponding to /v1/servers.
+     */
+    std::unique_ptr<json_t> v1_servers() const;
+
+    /**
+     * POST request to /v1/maxscale/modules/:module:/:command:?instance[&param...]
+     *
+     * @param module    Module name.
+     * @param command   The command.
+     * @param instance  The object instance to execute it on.
+     * @param params    Optional arguments.
+     */
+    void v1_maxscale_modules(const std::string& module,
+                             const std::string& command,
+                             const std::string& instance,
+                             const std::vector<std::string>& params = std::vector<std::string>()) const;
+
+    /**
+     * Call a module command.
+     *
+     * @param module    Module name.
+     * @param command   The command.
+     * @param instance  The object instance to execute it on.
+     * @param params    Optional arguments.
+     */
+    void call_command(const std::string& module,
+                      const std::string& command,
+                      const std::string& instance,
+                      const std::vector<std::string>& params = std::vector<std::string>()) const
+    {
+        return v1_maxscale_modules(module, command, instance, params);
+    }
+
+    /**
+     * The equivalent of 'maxctrl list servers'
+     *
+     * @return The JSON resource /v1/servers as a vector of Server objects.
+     */
+    std::vector<Server> list_servers() const;
+
+    /**
+     * The equivalent of 'maxctrl show server'
+     *
+     * @return The JSON resource /v1/servers/:id: as a Server object.
+     */
+    Server show_server(const std::string& id) const;
+
+    enum class Presence
+    {
+        OPTIONAL,
+        MANDATORY
+    };
+
+    /**
+     * Turns a JSON array at a specific path into a vector of desired type.
+     *
+     * @param pObject   The JSON object containing the JSON array.
+     * @param path      The path of the resource, e.g. "a/b/c"
+     * @param presence  Whether the path must exist or not. Note that if it is
+     *                  a true path "a/b/c", onlt the leaf may be optional, the
+     *                  components leading to the leaf must be present.
+     *
+     * @return Vector of object of desired type.
+     */
+    template<class T>
+    std::vector<T> get_array(json_t* pObject, const std::string& path, Presence presence) const
+    {
+        std::vector<T> rv;
+        pObject = get_leaf_object(pObject, path, presence);
+
+        if (pObject)
+        {
+            if (!json_is_array(pObject))
+            {
+                raise("'" + path + "' exists, but is not an array.");
+            }
+
+            size_t size = json_array_size(pObject);
+
+            for (size_t i = 0; i < size; ++i)
+            {
+                json_t* pElement = json_array_get(pObject, i);
+
+                rv.push_back(T(*this, pElement));
+            }
+        }
+
+        return rv;
+    }
+
+    /**
+     * Get JSON object at specific key.
+     *
+     * @param pObject   The object containing the path.
+     * @param key       The key of the resource. May *not* be a path.
+     * @param presence  Whether the key must exist or not.
+     *
+     * @return The desired object, or NULL if it does not exist and @c presence is OPTIONAL.
+     */
+    json_t* get_object(json_t* pObject, const std::string& path, Presence presence) const;
+
+    /**
+     * Get JSON object at specific path.
+     *
+     * @param pObject   The object containing the path.
+     * @param path      The path of the resource, e.g. "a/b/c"
+     * @param presence  Whether the leaf must exist or not. Note that if it is
+     *                  a true path "a/b/c", only the leaf may be optional, the
+     *                  components leading to the leaf must be present.
+     *
+     * @return The desired object, or NULL if it does not exist and @c presence is OPTIONAL.
+     */
+    json_t* get_leaf_object(json_t* pObject, const std::string& path, Presence presence) const;
+
+    /**
+     * Get a JSON value as a particular C++ type.
+     *
+     * @param pObject   The object containing the path.
+     * @param path      The path of the resource, e.g. "a/b/c"
+     * @param presence  Whether the leaf must exist or not. Note that if it is
+     *                  a true path "a/b/c", onlt the leaf may be optional, the
+     *                  components leading to the leaf must be present.
+     *
+     * @return The desired object, or NULL if it does not exist and @c presence is OPTIONAL.
+     */
+    template<class T>
+    T get(json_t* pObject, const std::string& path, Presence presence = Presence::OPTIONAL) const;
+
+    /**
+     * Parse a JSON object in a string.
+     *
+     * @return  The corresponding json_t object.
+     */
+    std::unique_ptr<json_t> parse(const std::string& json) const;
+
+    /**
+     * Issue a curl GET to the REST-API endpoint of the MaxScale running on
+     * the maxscale 0 VM instance.
+     *
+     * The path will be appended to "http://127.0.0.1:8989/v1/".
+     *
+     * @param path  The path of the resource.
+     *
+     * @return  The corresponding json_t object.
+     */
+    std::unique_ptr<json_t> curl_get(const std::string& path) const;
+
+    /**
+     * Issue a curl POST to the REST-API endpoint of the MaxScale running on
+     * the maxscale 0 VM instance.
+     *
+     * The path will be appended to "http://127.0.0.1:8989/v1/".
+     *
+     * @param path  The path of the resource.
+     *
+     * @return  The corresponding json_t object.
+     */
+    std::unique_ptr<json_t> curl_post(const std::string& path) const;
+
+    void raise(const std::string& message) const;
+
+private:
+    enum Command
+    {
+        GET,
+        POST
+    };
+
+    std::unique_ptr<json_t> curl(Command command, const std::string& path) const;
+
+private:
+    TestConnections& m_test;
+};
+
+template<>
+std::string MaxRest::get<std::string>(json_t* pObject, const std::string& path, Presence presence) const;
+
+template<>
+int64_t MaxRest::get<int64_t>(json_t* pObject, const std::string& path, Presence presence) const;

maxscale-system-test/maxtest/include/maxtest/maxscales.h

@@ -0,0 +1,354 @@
+#pragma once
+
+#include <string>
+#include "nodes.h"
+#include "mariadb_func.h"
+#include "mariadb_nodes.h"
+
+#define DEFAULT_MAXSCALE_CNF "/etc/maxscale.cnf"
+#define DEFAULT_MAXSCALE_LOG_DIR "/var/log/maxscale/"
+#define DEFAULT_MAXSCALE_BINLOG_DIR "/var/lib/maxscale/Binlog_Service/"
+#define DEFAULT_MAXADMIN_PASSWORD "mariadb"
+
+class Maxscales: public Nodes
+{
+public:
+    enum service
+    {
+        RWSPLIT,
+        READCONN_MASTER,
+        READCONN_SLAVE
+    };
+
+    Maxscales(const char *pref, const char *test_cwd, bool verbose,
+              const std::string& network_config);
+
+    bool setup() override;
+
+    int read_env();
+
+    /**
+     * @brief rwsplit_port RWSplit service port
+     */
+    int rwsplit_port[256];
+
+    /**
+     * @brief readconn_master_port ReadConnection in master mode service port
+     */
+    int readconn_master_port[256];
+
+    /**
+     * @brief readconn_slave_port ReadConnection in slave mode service port
+     */
+    int readconn_slave_port[256];
+
+    /**
+     * @brief Get port number of a MaxScale service
+     *
+     * @param type Type of service
+     * @param m    MaxScale instance to use
+     *
+     * @return Port number of the service
+     */
+    int port(enum service type = RWSPLIT, int m = 0) const;
+
+    /**
+     * @brief binlog_port binlog router service port
+     */
+    int binlog_port[256];
+
+    /**
+     * @brief conn_rwsplit  MYSQL connection struct to RWSplit service
+     */
+    MYSQL* conn_rwsplit[256];
+
+    /**
+     * @brief conn_master   MYSQL connection struct to ReadConnection in master mode service
+     */
+    MYSQL* conn_master[256];
+
+    /**
+     * @brief conn_slave MYSQL connection struct to ReadConnection in slave mode service
+     */
+    MYSQL* conn_slave[256];
+
+    /**
+     * @brief routers Array of 3 MYSQL handlers which contains copies of conn_rwsplit, conn_master, conn_slave
+     */
+    MYSQL* routers[256][3];
+
+    /**
+     * @brief ports of 3 int which contains copies of rwsplit_port, readconn_master_port, readconn_slave_port
+     */
+    int ports[256][3];
+
+    /**
+     * @brief maxadmin_Password Password to access Maxadmin tool
+     */
+    char * maxadmin_password[256];
+
+    /**
+      * @brief maxscale_cnf full name of Maxscale configuration file
+      */
+    char * maxscale_cnf[256];
+
+    /**
+      * @brief maxscale_log_dir name of log files directory
+      */
+    char * maxscale_log_dir[256];
+
+    /**
+      * @brief maxscale_lbinog_dir name of binlog files (for binlog router) directory
+      */
+    char * maxscale_binlog_dir[256];
+
+    /**
+     * @brief N_ports Default number of routers
+     */
+    int N_ports[256];
+
+    /**
+     * @brief test_dir path to test application
+     */
+    char test_dir[4096];
+
+    bool ssl;
+
+    /**
+     * @brief ConnectMaxscale   Opens connections to RWSplit, ReadConn master and ReadConn slave Maxscale
+     * services
+     * Opens connections to RWSplit, ReadConn master and ReadConn slave Maxscale services
+     * Connections stored in maxscales->conn_rwsplit[0], maxscales->conn_master[0] and
+     * maxscales->conn_slave[0] MYSQL structs
+     * @return 0 in case of success
+     */
+    int connect_maxscale(int m = 0, const std::string& db = "test");
+    int connect(int m = 0, const std::string& db = "test")
+    {
+        return connect_maxscale(m, db);
+    }
+
+    /**
+     * @brief CloseMaxscaleConn Closes connection that were opened by ConnectMaxscale()
+     * @return 0
+     */
+    int close_maxscale_connections(int m = 0);
+    int disconnect(int m = 0)
+    {
+        return close_maxscale_connections(m);
+    }
+
+    /**
+     * @brief ConnectRWSplit    Opens connections to RWSplit and store MYSQL struct in
+     * maxscales->conn_rwsplit[0]
+     * @return 0 in case of success
+     */
+    int connect_rwsplit(int m = 0, const std::string& db = "test");
+
+    /**
+     * @brief ConnectReadMaster Opens connections to ReadConn master and store MYSQL struct in
+     * maxscales->conn_master[0]
+     * @return 0 in case of success
+     */
+    int connect_readconn_master(int m = 0, const std::string& db = "test");
+
+    /**
+     * @brief ConnectReadSlave Opens connections to ReadConn slave and store MYSQL struct in
+     * maxscales->conn_slave[0]
+     * @return 0 in case of success
+     */
+    int connect_readconn_slave(int m = 0, const std::string& db = "test");
+
+    /**
+     * @brief OpenRWSplitConn   Opens new connections to RWSplit and returns MYSQL struct
+     * To close connection mysql_close() have to be called
+     * @return MYSQL struct
+     */
+    MYSQL* open_rwsplit_connection(int m = 0, const std::string& db = "test")
+    {
+        return open_conn(rwsplit_port[m], IP[m], user_name, password, ssl);
+    }
+
+    /**
+     * Get a readwritesplit Connection
+     */
+    Connection rwsplit(int m = 0, const std::string& db = "test")
+    {
+        return Connection(IP[m], rwsplit_port[m], user_name, password, db, ssl);
+    }
+
+    /**
+     * @brief OpenReadMasterConn    Opens new connections to ReadConn master and returns MYSQL struct
+     * To close connection mysql_close() have to be called
+     * @return MYSQL struct
+     */
+    MYSQL* open_readconn_master_connection(int m = 0)
+    {
+        return open_conn(readconn_master_port[m],
+                         IP[m],
+                         user_name,
+                         password,
+                         ssl);
+    }
+
+    /**
+     * Get a readconnroute master Connection
+     */
+    Connection readconn_master(int m = 0, const std::string& db = "test")
+    {
+        return Connection(IP[m], readconn_master_port[m], user_name, password, db, ssl);
+    }
+
+    /**
+     * @brief OpenReadSlaveConn    Opens new connections to ReadConn slave and returns MYSQL struct
+     * To close connection mysql_close() have to be called
+     * @return  MYSQL struct
+     */
+    MYSQL* open_readconn_slave_connection(int m = 0)
+    {
+        return open_conn(readconn_slave_port[m],
+                         IP[m],
+                         user_name,
+                         password,
+                         ssl);
+    }
+
+    /**
+     * Get a readconnroute slave Connection
+     */
+    Connection readconn_slave(int m = 0, const std::string& db = "test")
+    {
+        return Connection(IP[m], readconn_slave_port[m], user_name, password, db, ssl);
+    }
+
+    /**
+     * @brief CloseRWSplit Closes RWplit connections stored in maxscales->conn_rwsplit[0]
+     */
+    void close_rwsplit(int m = 0)
+    {
+        mysql_close(conn_rwsplit[m]);
+        conn_rwsplit[m] = NULL;
+    }
+
+    /**
+     * @brief CloseReadMaster Closes ReadConn master connections stored in maxscales->conn_master[0]
+     */
+    void close_readconn_master(int m = 0)
+    {
+        mysql_close(conn_master[m]);
+        conn_master[m] = NULL;
+    }
+
+    /**
+     * @brief CloseReadSlave Closes ReadConn slave connections stored in maxscales->conn_slave[0]
+     */
+    void close_readconn_slave(int m = 0)
+    {
+        mysql_close(conn_slave[m]);
+        conn_slave[m] = NULL;
+    }
+
+    /**
+     * @brief restart_maxscale Issues 'service maxscale restart' command
+     */
+    int restart_maxscale(int m = 0);
+    int restart(int m = 0)
+    {
+        return restart_maxscale(m);
+    }
+
+    /**
+     * @brief alias for restart_maxscale
+     */
+    int start_maxscale(int m = 0);
+
+    int start(int m = 0)
+    {
+        return start_maxscale(m);
+    }
+
+    /**
+     * @brief stop_maxscale Issues 'service maxscale stop' command
+     */
+    int stop_maxscale(int m = 0);
+    int stop(int m = 0)
+    {
+        return stop_maxscale(m);
+    }
+
+    // Helper for stopping all maxscales
+    void stop_all()
+    {
+        for (int i = 0; i < N; i++)
+        {
+            stop(i);
+        }
+    }
+
+    int execute_maxadmin_command(int m, const char* cmd);
+    int execute_maxadmin_command_print(int m, const char* cmd);
+    int check_maxadmin_param(int m, const char* command, const char* param, const char* value);
+    int get_maxadmin_param(int m, const char* command, const char* param, char* result);
+
+    /**
+     * @brief get_backend_servers_num Gets number of backend servers configure for service
+     * @param m Number of Maxscale node
+     * @param service Name of service to ask
+     * @return number of backend servers
+     */
+    int get_backend_servers_num(int m, const char* service);
+
+    /**
+     * @brief get_maxscale_memsize Gets size of the memory consumed by Maxscale process
+     * @param m Number of Maxscale node
+     * @return memory size in kilobytes
+     */
+    long unsigned get_maxscale_memsize(int m = 0);
+
+    /**
+     * @brief find_master_maxadmin Tries to find node with 'Master' status using Maxadmin connand 'show
+     * server'
+     * @param nodes Mariadb_nodes object
+     * @return node index if one master found, -1 if no master found or several masters found
+     */
+    int find_master_maxadmin(Mariadb_nodes* nodes, int m = 0);
+    int find_slave_maxadmin(Mariadb_nodes* nodes, int m = 0);
+
+    /**
+     * @brief Get the set of labels that are assigned to server @c name
+     *
+     * @param name The name of the server that must be present in the output `maxadmin list servers`
+     *
+     * @param m Number of Maxscale node
+     *
+     * @return A set of string labels assigned to this server
+     */
+    StringSet get_server_status(const char* name, int m = 0);
+
+    /**
+     * Wait until the monitors have performed at least one monitoring operation
+     *
+     * The function waits until all monitors have performed at least one monitoring cycle.
+     *
+     * @param intervals The number of monitor intervals to wait
+     * @param m Number of Maxscale node
+     */
+    void wait_for_monitor(int intervals = 2, int m = 0);
+
+    /**
+     * @brief use_valrind if true Maxscale will be executed under Valgrind
+     */
+    bool use_valgrind;
+
+    /**
+     * @brief use_callgrind if true Maxscale will be executed under Valgrind with
+     * --callgrind option
+     */
+    bool use_callgrind;
+
+    /**
+     * @brief valgring_log_num Counter for Maxscale restarts to avoid Valgrind log overwriting
+     */
+    int valgring_log_num;
+
+};

maxscale-system-test/maxtest/include/maxtest/nodes.h

@@ -0,0 +1,220 @@
+#pragma once
+
+#include <errno.h>
+#include <string>
+#include "mariadb_func.h"
+#include <set>
+#include <string>
+
+#include <maxbase/ccdefs.hh>
+
+typedef std::set<std::string> StringSet;
+
+
+class Nodes
+{
+public:
+    /**
+     * Sets up the nodes. *Must* be called before instance is used.
+     *
+     * @return True, if the instance could be setup, false otherwise.
+     */
+    virtual bool setup() = 0;
+
+    char * IP[256];
+    /**
+     * @brief  private IP address strings for every backend node (for AWS)
+     */
+
+    char * IP_private[256];
+    /**
+     * @brief  IP address strings for every backend node (IPv6)
+     */
+    char * IP6[256];
+
+    /**
+     * @brief use_ipv6 If true IPv6 addresses will be used to connect Maxscale and backed
+     * Also IPv6 addresses go to maxscale.cnf
+     */
+    bool use_ipv6 = false;
+
+    /**
+     * @brief  Path to ssh key for every backend node
+     */
+    char * sshkey[256];
+
+    /**
+     * @brief Number of backend nodes
+     */
+    int N;
+
+    /**
+     * @brief     name of backend setup (like 'repl' or 'galera')
+     */
+    char prefix[16];
+
+    /**
+     * @brief access_user Unix users name to access nodes via ssh
+     */
+    char * access_user[256];
+
+    /**
+     * @brief access_sudo empty if sudo is not needed or "sudo " if sudo is needed.
+     */
+    char * access_sudo[256];
+
+    /**
+     * @brief access_homedir home directory of access_user
+     */
+    char * access_homedir[256];
+
+    char * hostname[256];
+
+    /**
+     * @brief stop_vm_command Command to suspend VM
+     */
+    char * stop_vm_command[256];
+    /**
+     *
+     * @brief start_vm_command Command to resume VM
+     */
+    char * start_vm_command[256];
+
+    /**
+     * @brief   User name to access backend nodes
+     */
+    char * user_name;
+    /**
+     * @brief   Password to access backend nodes
+     */
+    char * password;
+
+    /**
+     * @brief network_config Content of MDBCI network_config file
+     */
+    std::string network_config;
+
+    /**
+     * @brief Verbose command output
+     */
+    bool verbose;
+
+    /**
+     * @brief Get IP address
+     *
+     * @return The current IP address
+     */
+    const char* ip(int i = 0) const;
+
+    /**
+     * @brief Generate command line to execute command on the node via ssh
+     * @param cmd result
+     * @param index index number of the node (index)
+     * @param ssh command to execute
+     * @param sudo if true the command is executed with root privelegues
+     */
+    void generate_ssh_cmd(char* cmd, int node, const char* ssh, bool sudo);
+
+    /**
+     * @brief executes shell command on the node using ssh
+     * @param index number of the node (index)
+     * @param ssh command to execute
+     * @param sudo if true the command is executed with root privelegues
+     * @param pointer to variable to store process exit code
+     * @return output of the command
+     */
+    char* ssh_node_output_f(int node,
+                            bool sudo,
+                            int* exit_code,
+                            const char* format,
+                            ...) mxb_attribute((format(printf, 5, 6)));
+    char* ssh_node_output(int node, const char* ssh, bool sudo, int* exit_code);
+
+    // Simplified C++ version
+    std::pair<int, std::string> ssh_output(std::string ssh, int node = 0, bool sudo = true)
+    {
+        int rc;
+        char* out = ssh_node_output(node, ssh.c_str(), sudo, &rc);
+        std::string rval(out);
+        free(out);
+        return {rc, rval};
+    }
+
+    /**
+     * @brief executes shell command on the node using ssh
+     * @param index number of the node (index)
+     * @param ssh command to execute
+     * @param sudo if true the command is executed with root privelegues
+     * @return exit code of the coomand
+     */
+    int ssh_node(int node, const char* ssh, bool sudo);
+    int ssh_node_f(int node, bool sudo, const char* format, ...) mxb_attribute((format(printf, 4, 5)));
+
+    /**
+     * @brief Copy a local file to the Node i machine
+     * @param src Source file on the local filesystem
+     * @param dest Destination file on the remote file system
+     * @param i Node index
+     * @return exit code of the system command or 1 in case of i > N
+     */
+    int copy_to_node_legacy(const char* src, const char* dest, int i = 0);
+    int copy_to_node(int i, const char* src, const char* dest);
+
+    /**
+     * @brief Copy a local file to the Node i machine
+     * @param src Source file on the remote filesystem
+     * @param dest Destination file on the local file system
+     * @param i Node index
+     * @return exit code of the system command or 1 in case of i > N
+     */
+    int copy_from_node_legacy(const char* src, const char* dest, int i);
+    int copy_from_node(int i, const char* src, const char* dest);
+
+    /**
+     * @brief Check node via ssh and restart it if it is not resposible
+     * @param node Node index
+     * @return True if node is ok, false if start failed
+     */
+    bool check_nodes();
+
+    /**
+     * @brief read_basic_env Read IP, sshkey, etc - common parameters for all kinds of nodes
+     * @return 0 in case of success
+     */
+    int read_basic_env();
+
+    /**
+     * @brief get_nc_item Find variable in the MDBCI network_config file
+     * @param item_name Name of the variable
+     * @return value of variable or empty value if not found
+     */
+    std::string get_nc_item(const char* item_name);
+
+    /**
+     * @brief get_N Calculate the number of nodes discribed in the _netoek_config file
+     * @return Number of nodes
+     */
+    int get_N();
+
+    /**
+     * @brief start_vm Start virtual machine
+     * @param node Node number
+     * @return 0 in case of success
+     */
+    int start_vm(int node);
+
+    /**
+     * @brief stop_vm Stop virtual machine
+     * @param node Node number
+     * @return 0 in case of success
+     */
+    int stop_vm(int node);
+
+protected:
+    Nodes(const char* pref,
+          const std::string& network_config,
+          bool verbose);
+
+private:
+    bool check_node_ssh(int node);
+};

maxscale-system-test/maxtest/include/maxtest/rds_vpc.h

@@ -0,0 +1,260 @@
+#pragma once
+
+#include <iostream>
+#include <unistd.h>
+#include "testconnections.h"
+#include <jansson.h>
+
+using namespace std;
+
+class RDS
+{
+public:
+    /**
+     * @brief RDS Constructor
+     * @param cluster Name of cluster to create/destroy
+     */
+    RDS(char* cluster);
+
+    const char* get_instance_name(json_t* instance);
+
+    /**
+     * @brief get_cluster Executes 'rds describe-bd-clusters' and creates json object with info on cluster
+     * Finds cluster with ID 'cluster_name_intern'.
+     * Does not set any internal variables
+     * @return JSON describption of cluster
+     */
+    json_t* get_cluster();
+
+    /**
+     * @brief get_cluster_descr Creates JSON cluster describtion from string representation
+     * @param json String representation of cluster description
+     * Does not set any internal variables
+     * @return JSON describption of cluster
+     */
+    json_t* get_cluster_descr(char* json);
+
+    /**
+     * @brief get_subnets_group_descr
+     * @param json String representation of subnets grop description
+     * Does not set any internal variables
+     * @return JSON description of subnets group
+     */
+    json_t* get_subnets_group_descr(char* json);
+
+    /**
+     * @brief get_cluster_nodes Extract list of nodes names from cluster JSON description
+     * Uses 'cluster_intern'
+     * Does not set any internal variables
+     * @return JSON array of node names strings
+     */
+    json_t* get_cluster_nodes();
+
+
+    /**
+     * @brief get_cluster_nodes Extract list of nodes names from cluster JSON description
+     * Does not set any internal variables
+     * @param cluster JSON cluster description
+     * @return JSON array of nodes names strings
+     */
+    json_t* get_cluster_nodes(json_t* cluster);
+
+    /**
+     * @brief get_endpoints Gets list of endpoint (URLs) of cluster nodes
+     * Sets 'cluster_intern'
+     * @return JSON array of nodes endpoints (objects contaning Address and Port)
+     */
+    json_t* get_endpoints();
+
+    /**
+     * @brief get_subnets Extracts subnets IDs from subnets group
+     * Uses 'subnets_group_name_intern'
+     * Sets 'vpc_id_intern' and 'subnets_intern'
+     * @return JSON array of node names strings
+     */
+    json_t* get_subnets();
+
+    /**
+     * @brief get_subnetgroup_name Extracts subnets grop ID from cluster description
+     * Uses 'cluster_intern'
+     * Sets 'subnets_group_name_intern'
+     * If 'cluster_intern' is NULL function returns 'subnets_group_name_intern' value
+     * @return name of subnets group
+     */
+    const char* get_subnetgroup_name();
+
+    /**
+     * @brief destroy_nodes Destroys nodes
+     * @param node_names JSON array with nodes names
+     * @return 0 in case of success
+     */
+    int destroy_nodes(json_t* node_names);
+
+    /**
+     * @brief destroy_subnets Destoys subnets
+     * Uses 'subnets_intern' to get subnets list
+     * If 'subnets_intern' is not set it is needed to run:
+     *   - clustr_intern=get_cluster()
+     *   - get_subnetgroup_name()
+     *   - get_subnets()
+     * @return 0 in case of success
+     */
+    int destroy_subnets();
+
+    /**
+     * @brief destroy_subnets_group Destroys subnets group
+     * Uses 'subnets_group_name_intern'
+     * If 'subnets_group_name_intern'  it is needed to run:
+     *   - clustr_intern=get_cluster()
+     *   - get_subnetgroup_name()
+     * @return 0 in case of success
+     */
+    int destroy_subnets_group();
+
+    /**
+     * @brief destroy_route_tables Destroys route tabele
+     * Not needed to executed directly, route table is destroyed by destroy_vpc
+     * Uses 'vpc_id_intern'
+     * If 'vpc_id_intern' is not set it is needed to run:
+     *   - clustr_intern=get_cluster()
+     *   - get_subnetgroup_name()
+     *   - get_subnets()
+     * @return  0 in case of success
+     */
+    int destroy_route_tables();     // is needed?
+
+    /**
+     * @brief destroy_vpc Destroys VPC
+     * Uses 'vpc_id_intern'
+     * if 'vpc_id_intern' is not set it is needed to run:
+     *   - clustr_intern=get_cluster()
+     *   - get_subnetgroup_name()
+     *   - get_subnets()
+     * @return 0 in case of success
+     */
+    int destroy_vpc();
+
+    /**
+     * @brief destroy_cluster Destroys RDS cluster
+     * Uses 'cluster_name_intern'
+     * @return 0 in case of success
+     */
+    int destroy_cluster();
+
+    /**
+     * @brief detach_and_destroy_gw Finds, detach and destroys internet gateways attached to VPC
+     * Uses 'vpc_id_intern'
+     * if 'vpc_id_intern' is not set it is needed to run:
+     *   - clustr_intern=get_cluster()
+     *   - get_subnetgroup_name()
+     *   - get_subnets()
+     * @return 0 in case of success
+     */
+    int detach_and_destroy_gw();
+
+    /**
+     * @brief create_vpc Creates VPC
+     * Sets 'vpc_id_intern'
+     * @param vpc_id Pointer to variable to place VpcID
+     * @return 0 in case of success
+     */
+    int create_vpc(const char** vpc_id);
+
+    /**
+     * @brief create_subnet Creates subnet inside VPC
+     * Adds element to 'subnets_intern' JSON array (creates it if it does not exist)
+     * @param az Availability zone ID (e.g. 'eu-west-1a')
+     * @param cidr CIDR block (e.g. '172.30.1.0/24')
+     * @param subnet_id Pointer to variable to place SubnetID
+     * @return 0 in case of success
+     */
+    int create_subnet(const char* az, const char* cidr, const char** subnet_id);
+
+    /**
+     * @brief create_subnet_group Creates subnets group for RDS
+     * Uses 'subnets_intern'
+     * Sets 'subnets_group_name_intern'
+     * @return 0 in case of success
+     */
+    int create_subnet_group();
+
+    /**
+     * @brief create_gw Creates internet gateway for vpc_id_intern
+     * Uses 'vpc_id_intern'
+     * Sests 'gw_intern'
+     * @param gw_id Pointer to variable to place gateway ID
+     * @return 0 in case of success
+     */
+    int create_gw(const char** gw_id);
+
+    /**
+     * @brief configure_route_table Adds route to route tabele attched to VPC
+     * Finds route table attached to VPC and adds route from internet to internet gateway
+     * Uses 'vpc_id_intern' and 'gw_intern'
+     * @param rt Pointer to variable to place route table ID which was found ond modified
+     * @return 0 in case of success
+     */
+    int configure_route_table(const char** rt);
+
+    /**
+     * @brief create_cluster Creates RDS cluster and instances
+     * Also configures security group (opens port 3306)
+     * Uses 'cluster_name_intern', 'N_intern'
+     * Sets cluster_intern, sg_intern
+     * @return 0 in case of success
+     */
+    int create_cluster();
+
+    /**
+     * @brief get_writer Find instance which have 'write' attribute
+     * Uses cluster_name_intern
+     * Calls 'aws rds describe-db-clusters' and does not use 'cluster_intern'
+     * (but does not update 'cluster_intern')
+     * @param writer_name Pointer to variable to place name of writer node
+     * @return 0 in case of success
+     */
+    int get_writer(const char** writer_name);
+
+    /**
+     * @brief create_rds_db Creates RDS DB cluster and all needed stuff (vpc, subnets, gateway, route table,
+     *...)
+     * If case of error tries to destry all created stuff
+     * @param cluster_name Name of DB cluster
+     * @param N Number of nodes
+     * @return 0 in case if success
+     */
+    int create_rds_db(int N);
+
+    /**
+     * @brief delete_rds_cluster Destroys RDS cluster, instances and VPC in which RDS cluster was located
+     * Uses 'cluster_name_intern'
+     * Tries to get all items IDs
+     * @return 0 in case if success
+     */
+    int delete_rds_cluster();
+
+    /**
+     * @brief wait_for_nodes Waits until N nodes are in 'avalable' state
+     * Uses 'cluster_name_intern'
+     * Sets 'cluster_intern'
+     * @param N Number of nodes expected to be active (can be less than number of nodes in cluster)
+     * @return 0 in case if success
+     */
+    int wait_for_nodes(size_t N);
+
+    /**
+     * @brief do_failover Does failover for RDS cluster
+     * @return 0 in case if success
+     */
+    int do_failover();
+
+    const char* cluster_name_intern;
+    size_t      N_intern;
+    json_t*     cluster_intern;
+    const char* vpc_id_intern;
+    json_t*     subnets_intern;
+    const char* subnets_group_name_intern;
+    const char* rt_intern;
+    const char* gw_intern;
+    const char* sg_intern;
+};

maxscale-system-test/maxtest/include/maxtest/sql_const.h

@@ -0,0 +1,25 @@
+#ifndef SQL_CONST_H
+#define SQL_CONST_H
+
+const char* create_repl_user =
+    "grant replication slave on *.* to repl@'%%' identified by 'repl'; "
+    "FLUSH PRIVILEGES";
+const char* setup_slave =
+    "change master to MASTER_HOST='%s', "
+    "MASTER_USER='repl', "
+    "MASTER_PASSWORD='repl', "
+    "MASTER_LOG_FILE='%s', "
+    "MASTER_LOG_POS=%s, "
+    "MASTER_PORT=%d; "
+    "start slave;";
+
+const char* setup_slave_no_pos =
+    "change master to MASTER_HOST='%s', "
+    "MASTER_USER='repl', "
+    "MASTER_PASSWORD='repl', "
+    "MASTER_LOG_FILE='mar-bin.000001', "
+    "MASTER_LOG_POS=4, "
+    "MASTER_PORT=%d";
+
+
+#endif      // SQL_CONST_H

maxscale-system-test/maxtest/include/maxtest/sql_t1.h

@@ -0,0 +1,74 @@
+#pragma once
+
+#include "mariadb_func.h"
+#include "testconnections.h"
+
+/**
+ * @brief execute_select_query_and_check Execute query and check that result contains expected number of rows
+ * @param conn MYSQL handler
+ * @param sql Query
+ * @param rows Expected number of rows
+ * @return 0 in case of success
+ */
+int execute_select_query_and_check(MYSQL* conn, const char* sql, unsigned long long int rows);
+
+/**
+ * @brief create_t1 Create t1 table, fileds: (x1 int, fl int)
+ * @param conn MYSQL handler
+ * @return 0 in case of success
+ */
+int create_t1(MYSQL* conn);
+
+
+/**
+ * @brief create_t1 Create t2 table, fileds: (x1 int, fl int)
+ * @param conn MYSQL handler
+ * @return 0 in case of success
+ */
+int create_t2(MYSQL* conn);
+
+/**
+ * @brief create_insert_string Create SQL query string to insert N rows into t1
+ * fl is equal to given value and x1 is incrementing value (row index)
+ * @param sql pointer to buffer to put result
+ * @param N Number of rows to insert
+ * @param fl value to fill 'fl' field
+ * @return 0
+ */
+int create_insert_string(char* sql, int N, int fl);
+
+/**
+ * @brief create_insert_string Create SQL query string to insert N rows into t1
+ * fl is equal to given value and x1 is incrementing value (row index)
+ * (same as create_insert_string(), but allocates buffer for SQL string by itself)
+ * @param sql pointer to buffer to put result
+ * @param N Number of rows to insert
+ * @param fl value to fill 'fl' field
+ * @return pointer to insert SQL string
+ */
+char* allocate_insert_string(int fl, int N);
+
+/**
+ * @brief insert_into_t1 Insert N blocks of 16^i rows into t1
+ * first block has fl=0, second - fl=1, ..., N-block fl=N-1
+ * first block has 16 row, second - 256, ..., N-block 16^N rows
+ * @param conn MYSQL handler
+ * @param N Number of blocks to insert
+ * @return 0 in case of success
+ */
+int insert_into_t1(MYSQL* conn, int N);
+
+/**
+ * @brief select_from_t1 Check that t1 contains data as inserted by insert_into_t1()
+ * @param conn MYSQL handler
+ * @param N Number of blocks to insert
+ * @return 0 in case of success
+ */
+int select_from_t1(MYSQL* conn, int N);
+
+/**
+ * @brief check_if_t1_exists
+ * @param conn MYSQL handler
+ * @return 0 if content of t1 is ok
+ */
+int check_if_t1_exists(MYSQL *conn);

maxscale-system-test/maxtest/include/maxtest/stopwatch.h

@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2018 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2024-02-10
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+#pragma once
+
+#include <chrono>
+#include <iosfwd>
+#include <string>
+
+// Same StopWatch as in maxscale, but dropping support for gcc 4.4
+namespace base
+{
+using Clock = std::chrono::steady_clock;
+
+struct Duration : public Clock::duration    // for ADL
+{
+    using Clock::duration::duration;
+    Duration() = default;
+    Duration(Clock::duration d) : Clock::duration(d)
+    {
+    }
+};
+
+using TimePoint = std::chrono::time_point<Clock, Duration>;
+
+class StopWatch
+{
+public:
+    // Starts the stopwatch, which is always running.
+    StopWatch();
+    // Get elapsed time.
+    Duration lap() const;
+    // Get elapsed time, restart StopWatch.
+    Duration restart();
+private:
+    TimePoint m_start;
+};
+
+// Returns the value as a double and string adjusted to a suffix like ms for milliseconds.
+std::pair<double, std::string> dur_to_human_readable(Duration dur);
+
+// Human readable output. No standard library for it yet.
+std::ostream& operator<<(std::ostream&, Duration dur);
+
+// TimePoint
+std::string   time_point_to_string(TimePoint tp, const std::string& fmt = "%F %T");
+std::ostream& operator<<(std::ostream&, TimePoint tp);
+}   // base

maxscale-system-test/maxtest/include/maxtest/tcp_connection.h

@@ -0,0 +1,48 @@
+#pragma once
+
+#include <cstdint>
+#include <cstddef>
+
+namespace tcp
+{
+
+// A raw TCP connection
+class Connection
+{
+public:
+    ~Connection();
+
+    /**
+     * Connect to the target server
+     *
+     * @param host Server hostname
+     * @param port Server port
+     *
+     * @return True if connection was successfully created
+     */
+    bool connect(const char* host, uint16_t port);
+
+    /**
+     * Write to socket
+     *
+     * @param buf  Buffer to read from
+     * @param size Number of bytes to read from @c buf
+     *
+     * @return Number of written bytes or -1 on error
+     */
+    int write(void* buf, size_t size);
+
+    /**
+     * Read from socket
+     *
+     * @param buf  Buffer to write to
+     * @param size Size of @c buf
+     *
+     * @return Number of read bytes or -1 on error
+     */
+    int read(void* buf, size_t size);
+
+private:
+    int m_so = -1;
+};
+}

maxscale-system-test/maxtest/include/maxtest/test_binlog_fnc.h

@@ -0,0 +1,27 @@
+#pragma once
+
+#include <iostream>
+#include "testconnections.h"
+#include "maxadmin_operations.h"
+#include "sql_t1.h"
+
+/**
+ * @brief check_sha1 Check that checksum of binlog files on Maxscale machines and all backends are equal
+ * @param Test TestConnections object
+ * @return 0 if binlog files checksums are identical
+ */
+int check_sha1(TestConnections* Test);
+
+/**
+ * @brief start_transaction Template for test transaction (used by test_binlog()
+ * @param Test TestConnections object
+ * @return 0 in case of success
+ */
+int start_transaction(TestConnections* Test);
+
+/**
+ * @brief test_binlog Execute a number of tests for check if binlog router is ok
+ * (see test description in setup_binlog.cpp)
+ * @param Test TestConnections object
+ */
+void test_binlog(TestConnections* Test);

maxscale-system-test/maxtest/include/maxtest/testconnections.h

@@ -0,0 +1,760 @@
+#pragma once
+
+#include "mariadb_nodes.h"
+#include "clustrix_nodes.h"
+#include "maxscales.h"
+#include "templates.h"
+#include <fcntl.h>
+#include <pthread.h>
+#include <sys/time.h>
+#include <set>
+#include <string>
+#include <vector>
+#include <thread>
+#include <functional>
+
+#include <maxbase/ccdefs.hh>
+
+typedef std::set<std::string> StringSet;
+
+#define MDBCI_FAUILT 200 // Exit code for the case when failure caused by MDBCI non-zero exit
+#define BROKEN_VM_FAUILT 201 // Exit code for the case when failure caused by screwed VMs
+
+/**
+ * @brief Class contains references to Master/Slave and Galera test setups
+ * Test setup should consist of two setups: one Master/Slave and one Galera.
+ *
+ * Maxscale should be configured separatelly for every test.
+ *
+ * Test setup should be described by enviromental variables:
+ * - Maxscale_IP - IP adress of Maxscale machine
+ * - Maxscale_User - User name to access Maxscale services
+ * - Maxscale_Password - Password to access Maxscale services
+ * - Maxscale_sshkey - ssh key for Maxscale machine
+ * - maxscale_cnf - name of maxscale .cnf file (full)
+ * - KillVMCommand - Command to kill a node (should handle one parameter: IP address of virtual machine to
+ * kill)
+ * - StartVMCommand - Command to restart virtual machine (should handle one parameter: IP address of virtual
+ * machine to kill)
+ * - GetLogsCommand - Command to copy log files from node virtual machines (should handle one parameter: IP
+ * address of virtual machine to kill)
+ * - SysbenchDir - path to SysBench directory (sysbanch should be >= 0.5)
+ * - node_N - Number of Master/Slave setup nodes
+ * - node_NNN - IP address of node NNN (NNN - 3 digits node index starting from 000)
+ * - node_port_NNN - MariaDB port for node NNN
+ * - node_sshkey_NNN - ssh key to access node NNN (should be sutable for 'root' and 'ec2-user')
+ * - node_User - User name to access Master/Slav setup
+ * - node_Password - Password to access Master/Slave setup
+ * - galera_N, galera_NNN, galera_port_NNN, galera_sshkey_NNN, galera_User, galera_Password - same for Galera
+ * setup
+ *
+ */
+class TestConnections
+{
+private:
+    /** Whether timeouts are enabled or not */
+    bool enable_timeouts;
+    bool log_matches(int m, const char* pattern);
+public:
+    /**
+     * @brief TestConnections constructor: reads environmental variables, copies MaxScale.cnf for MaxScale
+     * machine
+     * @param test_exec_name Path to currect executable
+     */
+    TestConnections(int argc, char* argv[]);
+
+    ~TestConnections();
+
+    /**
+     * @brief Is the test still ok?
+     *
+     * @return True, if no errors have occurred, false otherwise.
+     */
+    bool ok() const
+    {
+        return global_result == 0;
+    }
+
+    /**
+     * @brief Has the test failed?
+     *
+     * @return True, if errors have occurred, false otherwise.
+     */
+    bool failed() const
+    {
+        return global_result != 0;
+    }
+
+    /**
+     * @brief global_result Result of test, 0 if PASSED
+     */
+    int global_result;
+
+    /**
+     * @brief test_name Neme of the test
+     */
+    char* test_name;
+
+    /**
+     * @brief galera Mariadb_nodes object containing references to Galera setuo
+     */
+    Galera_nodes * galera;
+
+    /**
+     * @brief repl Mariadb_nodes object containing references to Master/Slave setuo
+     */
+    Mariadb_nodes* repl;
+
+    Clustrix_nodes * clustrix;
+
+    /**
+     * @brief maxscales Maxscale object containing referebces to all Maxscale machines
+     */
+    Maxscales* maxscales;
+
+    /**
+     * @brief mdbci_config_name Name of MDBCI VMs set
+     */
+    char * mdbci_config_name;
+
+    /**
+     * @brief mdbci_vm_path Path to directory with MDBCI VMs descriptions
+     */
+    char * mdbci_vm_path;
+
+    /**
+     * @brief mdbci_temlate Name of mdbci VMs tempate file
+     */
+    char * mdbci_template;
+
+    /**
+     * @brief target Name of Maxscale repository in the CI
+     */
+    char * target;
+
+    /**
+     * @brief GetLogsCommand Command to copy log files from node virtual machines (should handle one
+     * parameter: IP address of virtual machine to kill)
+     */
+    char * get_logs_command;
+
+    /**
+     * @brief make_snapshot_command Command line to create a snapshot of all VMs
+     */
+    char * take_snapshot_command;
+
+    /**
+     * @brief revert_snapshot_command Command line to revert a snapshot of all VMs
+     */
+    char * revert_snapshot_command;
+
+    /**
+     * @brief use_snapshots if TRUE every test is trying to revert snapshot before running the test
+     */
+    bool use_snapshots;
+
+    /**
+     * @brief SysbenchDir   path to SysBench directory (sysbanch should be >= 0.5)
+     */
+    char sysbench_dir[4096];
+
+    /**
+     * @brief copy_mariadb_logs copies MariaDB logs from backend
+     * @param repl Mariadb_nodes object
+     * @param prefix file name prefix
+     * @return 0 if success
+     */
+    int copy_mariadb_logs(Mariadb_nodes* nrepl, const char* prefix, std::vector<std::thread>& threads);
+
+    /**
+     * @brief MaxScale runs locally, specified using -l.
+     */
+    bool local_maxscale;
+
+    /**
+     * @brief network_config Content of MDBCI network_config file
+     */
+    std::string network_config;
+
+    /**
+     * @brief no_backend_log_copy if true logs from backends are not copied
+     *        (needed if case of Aurora RDS backend or similar)
+     */
+    bool no_backend_log_copy;
+
+    /**
+     * @brief Do not download MaxScale logs.
+     */
+    bool no_maxscale_log_copy;
+    /**
+     * @brief verbose if true more printing activated
+     */
+    static bool verbose;
+
+    /**
+     * @brief smoke if true all tests are executed in quick mode
+     */
+    bool smoke;
+
+    /**
+     * @brief binlog_cmd_option index of mariadb start option
+     */
+    int binlog_cmd_option;
+
+    /**
+     * @brief ssl if true ssl will be used
+     */
+    int ssl;
+
+    /**
+     * @brief backend_ssl if true ssl configuratio for all servers will be added
+     */
+    bool backend_ssl;
+
+    /**
+     * @brief binlog_master_gtid If true start_binlog() function configures Maxscale
+     * binlog router to use GTID to connect to Master
+     */
+    bool binlog_master_gtid;
+
+    /**
+     * @brief binlog_slave_gtid If true start_binlog() function configures slaves
+     * to use GTID to connect to Maxscale binlog router
+     */
+    bool binlog_slave_gtid;
+
+    /**
+     * @brief no_repl Do not check, restart and use Maxster/Slave setup;
+     */
+    bool no_repl;
+
+    /**
+     * @brief no_galera Do not check, restart and use Galera setup; all Galera tests will fail
+     */
+    bool no_galera;
+
+    /**
+     * @brief no_clustrix Do not check, restart and use Clustrix setup
+     */
+    bool no_clustrix;
+
+    /**
+     * @brief no_vm_revert If true tests do not revert VMs after the test even if test failed
+     * (use it for debugging)
+     */
+    bool no_vm_revert;
+
+    /**
+     * @brief ssl_options string with ssl configuration for command line client
+     */
+    char ssl_options[1024];
+
+    /**
+     * @brief threads Number of Maxscale threads
+     */
+    int threads;
+
+    /**
+     * @brief timeout seconds until test termination
+     */
+    long int timeout;
+
+    /**
+     * @brief log_copy_interval seconds between log copying
+     */
+    long int log_copy_interval;
+
+    /**
+     * @brief log_copy_interval seconds until next log copying
+     */
+    long int log_copy_to_go;
+
+    /**
+     * @brief timeout_thread_p pointer to timeout thread
+     */
+    pthread_t timeout_thread_p;
+
+    /**
+     * @brief log_copy_thread_p pointer to log copying thread
+     */
+    pthread_t log_copy_thread_p;
+
+    /**
+     * @brief start_time time when test was started (used by printf to print Timestamp)
+     */
+    timeval start_time;
+
+    /**
+     * @brief use_ipv6 If true IPv6 addresses will be used to connect Maxscale and backed
+     * Also IPv6 addresses go to maxscale.cnf
+     */
+    bool use_ipv6;
+
+    /**
+     * @brief template_name Name of maxscale.cnf template
+     */
+    const char * template_name;
+
+    /**
+     * @brief labels 'LABELS' string from CMakeLists.txt
+     */
+    const char * labels;
+
+    /**
+     * @brief mdbci_labels labels to be passed to MDBCI
+     */
+    std::string mdbci_labels;
+
+    /**
+     * @brief configured_labels List of lables for which nodes are configured
+     */
+    std::string configured_labels;
+
+    /**
+    * @brief vm_path Path to the VM Vagrant directory
+    */
+    std::string vm_path;
+
+    /**
+     * @brief reinstall_maxscale Flag that is set when 'reinstall_maxscale'
+     * option is provided;
+     * if true Maxscale will be removed and re-installed on all Maxscale nodes
+     * Used for 'run_test_snapshot'
+     */
+    bool reinstall_maxscale;
+
+    /** Check whether all nodes are in a valid state */
+    static void check_nodes(bool value);
+
+    /** Skip initial start of MaxScale */
+    static void skip_maxscale_start(bool value);
+
+    /** Prepare multiple maxscale instances */
+    static void multiple_maxscales(bool value);
+
+    /** Test requires a certain backend version  */
+    static void require_repl_version(const char* version);
+    static void require_galera_version(const char* version);
+
+    /** Require that galera is present*/
+    static void require_galera(bool value);
+
+    /** Require that columnstore is present*/
+    static void require_columnstore(bool value);
+
+    /**
+     * @brief Specify whether galera should be restarted.
+     *
+     * @param value If true, galera should be restarted.
+     *
+     * @note Even if set to false (which is also the default), '-g' or '--restart-galera' at
+     *       the command line will still cause a restart, unless '-y' or '--no-galera' has
+     *       been specified. '-y' will prevent galera from being restarted even if the value
+     *       has been set to true.
+     */
+    static void restart_galera(bool value);
+
+    /**
+     * @brief add_result adds result to global_result and prints error message if result is not 0
+     * @param result 0 if step PASSED
+     * @param format ... message to pring if result is not 0
+     */
+    void add_result(bool result, const char* format, ...) __attribute__ ((format(printf, 3, 4)));
+
+    /** Same as add_result() but inverted */
+    void expect(bool result, const char* format, ...) __attribute__ ((format(printf, 3, 4)));
+
+    /**
+     * @brief read_mdbci_info Reads name of MDBCI config and tryes to load all network info
+     */
+    void read_mdbci_info();
+
+    /**
+     * @brief ReadEnv Reads all Maxscale and Master/Slave and Galera setups info from environmental variables
+     */
+    void read_env();
+
+    /**
+     * @brief PrintIP   Prints all Maxscale and Master/Slave and Galera setups info
+     */
+    void print_env();
+
+    /**
+     * @brief InitMaxscale  Copies MaxSclae.cnf and start MaxScale
+     * @param m Number of Maxscale node
+     */
+    void init_maxscale(int m = 0);
+
+    /**
+     * @brief InitMaxscale  Copies MaxSclae.cnf and start MaxScale on all Maxscale nodes
+     */
+    void init_maxscales();
+
+    /**
+     * @brief start_binlog configure first node as Master, Second as slave connected to Master and others as
+     * slave connected to MaxScale binlog router
+     * @return  0 in case of success
+     */
+    int start_binlog(int m = 0);
+
+    /**
+     * @brief Start binlogrouter replication from master
+     */
+    bool replicate_from_master(int m = 0);
+
+    /**
+     * @brief Stop binlogrouter replication from master
+     */
+    void revert_replicate_from_master();
+
+    /**
+     * @brief prepare_binlog clean up binlog directory, set proper access rights to it
+     * @return 0
+     */
+    int prepare_binlog(int m = 0);
+
+    /**
+     * @brief start_mm configure first node as Master for second, Second as Master for first
+     * @return  0 in case of success
+     */
+    int start_mm(int m = 0);
+
+    /**
+     * @brief copy_all_logs Copies all MaxScale logs and (if happens) core to current workspace
+     */
+    int copy_all_logs();
+
+    /**
+     * @brief copy_all_logs_periodic Copies all MaxScale logs and (if happens) core to current workspace and
+     * sends time stemp to log copying script
+     */
+    int copy_all_logs_periodic();
+
+    /**
+     * @brief copy_maxscale_logs Copies logs from all Maxscale nodes
+     * @param timestamp
+     * @return 0
+     */
+    int copy_maxscale_logs(double timestamp);
+
+    /**
+     * @brief Test that connections to MaxScale are in the expected state
+     * @param rw_split State of the MaxScale connection to Readwritesplit. True for working connection, false
+     * for no connection.
+     * @param rc_master State of the MaxScale connection to Readconnroute Master. True for working connection,
+     * false for no connection.
+     * @param rc_slave State of the MaxScale connection to Readconnroute Slave. True for working connection,
+     * false for no connection.
+     * @return  0 if connections are in the expected state
+     */
+    int test_maxscale_connections(int m,
+                                  bool rw_split,
+                                  bool rc_master,
+                                  bool rc_slave);
+
+    /**
+     * @brief Create a number of connections to all services, run simple query, close all connections
+     * @param conn_N number of connections
+     * @param rwsplit_flag if true connections to RWSplit router will be created, if false - no connections to
+     * RWSplit
+     * @param master_flag if true connections to ReadConn master router will be created, if false - no
+     * connections to ReadConn master
+     * @param slave_flag if true connections to ReadConn slave router will be created, if false - no
+     * connections to ReadConn slave
+     * @param galera_flag if true connections to RWSplit router with Galera backend will be created, if false
+     *- no connections to RWSplit with Galera backend
+     * @return  0 in case of success
+     */
+    int create_connections(int m,
+                           int conn_N,
+                           bool rwsplit_flag,
+                           bool master_flag,
+                           bool slave_flag,
+                           bool galera_flag);
+
+    /**
+     * Trying to get client IP address by connection to DB via RWSplit and execution 'show processlist'
+     *
+     * @param ip client IP address as it visible by Maxscale
+     * @return 0 in case of success
+     */
+    int get_client_ip(int m, char* ip);
+
+    /**
+     * @brief set_timeout startes timeout thread which terminates test application after timeout_seconds
+     * @param timeout_seconds timeout time
+     * @return 0 if success
+     */
+    int set_timeout(long int timeout_seconds);
+
+    /**
+     * @brief set_log_copy_interval sets interval for periodic log copying
+     * @param interval_seconds interval in seconds
+     * @return 0 if success
+     */
+    int set_log_copy_interval(long int interval_seconds);
+
+    /**
+     * @brief stop_timeout stops timeout thread
+     * @return 0
+     */
+    int stop_timeout();
+
+    /**
+     * @brief printf with automatic timestamps
+     */
+    void tprintf(const char* format, ...);
+
+    /**
+     * @brief injects a message into maxscale.log
+     */
+    void log_printf(const char* format, ...) mxb_attribute((format(printf, 2, 3)));
+
+    /**
+     * @brief Creats t1 table, insert data into it and checks if data can be correctly read from all Maxscale
+     * services
+     * @param Test Pointer to TestConnections object that contains references to test setup
+     * @param N number of INSERTs; every next INSERT is longer 16 times in compare with previous one: for N=4
+     * last INSERT is about 700kb long
+     * @return 0 in case of no error and all checks are ok
+     */
+    int insert_select(int m, int N);
+
+    /**
+     * @brief Executes USE command for all Maxscale service and all Master/Slave backend nodes
+     * @param Test Pointer to TestConnections object that contains references to test setup
+     * @param db Name of DB in 'USE' command
+     * @return 0 in case of success
+     */
+    int use_db(int m, char* db);
+
+    /**
+     * @brief Checks if table t1 exists in DB
+     * @param presence expected result
+     * @param db DB name
+     * @return 0 if (t1 table exists AND presence=TRUE) OR (t1 table does not exist AND presence=false)
+     */
+
+    int check_t1_table(int m, bool presence, char* db);
+
+    /**
+     * @brief Check whether logs match a pattern
+     *
+     * The patterns are interpreted as `grep` compatible patterns (BRE regular expressions). If the
+     * log file does not match the pattern, it is considered an error.
+     */
+    void log_includes(int m, const char* pattern);
+
+    /**
+     * @brief Check whether logs do not match a pattern
+     *
+     * The patterns are interpreted as `grep` compatible patterns (BRE regular expressions). If the
+     * log file match the pattern, it is considered an error.
+     */
+    void log_excludes(int m, const char* pattern);
+
+    /**
+     * @brief FindConnectedSlave Finds slave node which has connections from MaxScale
+     * @param Test TestConnections object which contains info about test setup
+     * @param global_result pointer to variable which is increased in case of error
+     * @return index of found slave node
+     */
+    int find_connected_slave(int m, int* global_result);
+
+    /**
+     * @brief FindConnectedSlave1 same as FindConnectedSlave() but does not increase global_result
+     * @param Test  TestConnections object which contains info about test setup
+     * @return index of found slave node
+     */
+    int find_connected_slave1(int m = 0);
+
+    /**
+     * @brief CheckMaxscaleAlive Checks if MaxScale is alive
+     * Reads test setup info from enviromental variables and tries to connect to all Maxscale services to
+     * check if i is alive.
+     * Also 'show processlist' query is executed using all services
+     * @return 0 in case if success
+     */
+    int check_maxscale_alive(int m = 0);
+
+    /**
+     * @brief try_query Executes SQL query and repors error
+     * @param conn MYSQL struct
+     * @param sql SQL string
+     * @return 0 if ok
+     */
+    int try_query(MYSQL* conn, const char* sql, ...) mxb_attribute((format(printf, 3, 4)));
+
+    /**
+     * @brief try_query_all Executes SQL query on all MaxScale connections
+     * @param sql SQL string
+     * @return 0 if ok
+     */
+    int try_query_all(int m, const char* sql);
+
+    /**
+     * @brief Get the set of labels that are assigned to server @c name
+     *
+     * @param name The name of the server that must be present in the output `maxadmin list servers`
+     *
+     * @return A set of string labels assigned to this server
+     */
+    StringSet get_server_status(const char* name);
+
+    /**
+     * @brief check_maxscale_processes Check if number of running Maxscale processes is equal to 'expected'
+     * @param expected expected number of Maxscale processes
+     * @return 0 if check is done
+     */
+    int check_maxscale_processes(int m, int expected);
+
+    /**
+     * @brief list_dirs Execute 'ls' on binlog directory on all repl nodes and on Maxscale node
+     * @return 0
+     */
+    int list_dirs(int m = 0);
+
+    /**
+     * @brief make_snapshot Makes a snapshot for all running VMs
+     * @param snapshot_name name of created snapshot
+     * @return 0 in case of success or mdbci error code in case of error
+     */
+    int take_snapshot(char* snapshot_name);
+
+    /**
+     * @brief revert_snapshot Revert snapshot for all running VMs
+     * @param snapshot_name name of snapshot to revert
+     * @return 0 in case of success or mdbci error code in case of error
+     */
+    int revert_snapshot(char* snapshot_name);
+
+    /**
+     * @brief Test a bad configuration
+     * @param config Name of the config template
+     * @return Always false, the test will time out if the loading is successful
+     */
+    bool test_bad_config(int m, const char* config);
+
+    /**
+     * @brief Process a template configuration file
+     *
+     * @param dest Destination file name for actual configuration file
+     */
+    void process_template(int m, const char* src, const char* dest = "/etc/maxscale.cnf");
+
+    /**
+     * Execute a MaxCtrl command
+     *
+     * @param cmd  Command to execute, without the `maxctrl` part
+     * @param m    MaxScale node to execute the command on
+     * @param sudo Run the command as root
+     *
+     * @return The exit code and output of MaxCtrl
+     */
+    std::pair<int, std::string> maxctrl(std::string cmd, int m = 0, bool sudo = true)
+    {
+        return maxscales->ssh_output("maxctrl " + cmd, m, sudo);
+    }
+
+    void check_maxctrl(std::string cmd, int m = 0, bool sudo = true)
+    {
+        auto result = maxctrl(cmd, m, sudo);
+        expect(result.first == 0, "Command '%s' should work: %s", cmd.c_str(), result.second.c_str());
+    }
+
+    void check_current_operations(int m, int value);
+    void check_current_connections(int m, int value);
+    int  stop_maxscale(int m = 0);
+    int  start_maxscale(int m = 0);
+    void process_template(const char* src, const char* dest = "/etc/maxscale.cnf");
+
+    /**
+     * Get the current master server id from the cluster, as seen by rwsplit.
+     *
+     * @param m MaxScale node index
+     * @return Server id of the master
+     */
+    int get_master_server_id(int m = 0);
+
+    /**
+     * Add a callback that is called when the test ends
+     *
+     * @param func Function to call
+     */
+    void on_destroy(std::function<void (void)> func)
+    {
+        m_on_destroy.push_back(func);
+    }
+
+    /**
+     * @brief process_mdbci_template Read template file from maxscale-system-test/mdbci/templates
+     * and replace all placeholders with acutal values
+     * @return 0 in case of success
+     */
+    int process_mdbci_template();
+
+    /**
+     * @brief call_mdbci Execute MDBCI to bring up nodes
+     * @return 0 if success
+     */
+    int call_mdbci(const char *options);
+
+    /**
+     * @brief resinstall_maxscales Remove Maxscale form all nodes and installs new ones
+     * (to be used for run_test_snapshot)
+     * @return 0 in case of success
+     */
+    int reinstall_maxscales();
+
+private:
+    void report_result(const char* format, va_list argp);
+    void copy_one_mariadb_log(Mariadb_nodes* nrepl, int i, std::string filename);
+
+    bool too_many_maxscales() const
+    {
+        return maxscales->N < 2
+               && mdbci_labels.find("SECOND_MAXSCALE") != std::string::npos;
+    }
+
+    std::vector<std::function<void(void)>> m_on_destroy;
+};
+
+/**
+ * @brief timeout_thread Thread which terminates test application after 'timeout' milliseconds
+ * @param ptr pointer to TestConnections object
+ * @return void
+ */
+void* timeout_thread(void* ptr);
+
+/**
+ * @brief log_copy_thread Thread which peridically copies logs from Maxscale machine
+ * @param ptr pointer to TestConnections object
+ * @return void
+ */
+void* log_copy_thread(void* ptr);
+
+/**
+ * Dump two server status sets as strings
+ *
+ * @param current  The current status
+ * @param expected The expected status
+ *
+ * @return String form comparison of status sets
+ */
+std::string dump_status(const StringSet& current, const StringSet& expected);
+
+/**
+ * @brief get_template_name Returns the name of maxscale.cnf template to use for given test
+ * @param test_name Name of the test
+ * @param labels pointer to string for storing all test labels
+ * @return Name of maxscale.cnf file template
+ */
+const char *get_template_name(char * test_name, const char **labels);
+
+/**
+ * @brief readenv_and_set_default Read enviromental variable and set default values if
+ * variable is not defined
+ * @param name Name of the environmental variable
+ * @param defaultenv Default values to be set
+ * @return Envaronmental variable value
+ */

maxscale-system-test/maxtest/src/CMakeLists.txt

@@ -0,0 +1,34 @@
+add_library(maxtest SHARED
+        big_load.cpp
+        big_transaction.cpp
+        blob_test.cpp
+        clustrix_nodes.cpp
+        config_operations.cpp
+        different_size.cpp
+        envv.cpp
+        execute_cmd.cpp
+        fw_copy_rules.cpp
+        get_com_select_insert.cpp
+        get_my_ip.cpp
+        keepalived_func.cpp
+        labels_table.cpp
+        mariadb_func.cpp
+        mariadb_nodes.cpp
+        maxadmin_operations.cpp
+        maxinfo_func.cpp
+        maxrest.cc
+        maxscales.cpp
+        nodes.cpp
+        rds_vpc.cpp
+        sql_t1.cpp
+        stopwatch.cpp
+        tcp_connection.cpp
+        test_binlog_fnc.cpp
+        testconnections.cpp
+        # Include the CDC connector in the core library
+        ${CMAKE_SOURCE_DIR}/connectors/cdc-connector/cdc_connector.cpp)
+
+target_link_libraries(maxtest ${MARIADB_CONNECTOR_LIBRARIES} ${JANSSON_LIBRARIES} maxbase maxsql z m pthread ssl dl rt crypto crypt)
+set_target_properties(maxtest PROPERTIES VERSION "1.0.0" LINK_FLAGS -Wl,-z,defs)
+install(TARGETS maxtest DESTINATION system-test)
+add_dependencies(maxtest connector-c jansson maxbase)

maxscale-system-test/maxtest/src/big_load.cpp

@@ -0,0 +1,234 @@
+#include "big_load.h"
+#include <pthread.h>
+
+void load(long int* new_inserts,
+          long int* new_selects,
+          long int* selects,
+          long int* inserts,
+          int threads_num,
+          TestConnections* Test,
+          long int* i1,
+          long int* i2,
+          int rwsplit_only,
+          bool galera,
+          bool report_errors)
+{
+    char sql[1000000];
+    thread_data data;
+    Mariadb_nodes* nodes;
+    if (galera)
+    {
+        nodes = Test->galera;
+    }
+    else
+    {
+        nodes = Test->repl;
+    }
+
+    int sql_l = 20000;
+    int run_time = 100;
+    if (Test->smoke)
+    {
+        sql_l = 500;
+        run_time = 10;
+    }
+
+    nodes->connect();
+    Test->maxscales->connect_rwsplit(0);
+
+    data.i1 = 0;
+    data.i2 = 0;
+    data.exit_flag = 0;
+    data.Test = Test;
+    data.rwsplit_only = rwsplit_only;
+    // connect to the MaxScale server (rwsplit)
+
+    if (Test->maxscales->conn_rwsplit[0] == NULL)
+    {
+        if (report_errors)
+        {
+            Test->add_result(1, "Can't connect to MaxScale\n");
+        }
+        // Test->copy_all_logs();
+        exit(1);
+    }
+    else
+    {
+        create_t1(Test->maxscales->conn_rwsplit[0]);
+        create_insert_string(sql, sql_l, 1);
+
+        if ((execute_query(Test->maxscales->conn_rwsplit[0], "%s", sql) != 0) && (report_errors))
+        {
+            Test->add_result(1, "Query %s failed\n", sql);
+        }
+        // close connections
+        Test->maxscales->close_rwsplit(0);
+
+        Test->tprintf("Waiting for the table to replicate\n");
+        nodes->sync_slaves();
+
+        pthread_t thread1[threads_num];
+        pthread_t thread2[threads_num];
+
+        Test->tprintf("COM_INSERT and COM_SELECT before executing test\n");
+
+        Test->add_result(get_global_status_allnodes(&selects[0], &inserts[0], nodes, 0),
+                         "get_global_status_allnodes failed\n");
+
+        data.exit_flag = 0;
+        /* Create independent threads each of them will execute function */
+        for (int i = 0; i < threads_num; i++)
+        {
+            pthread_create(&thread1[i], NULL, query_thread1, &data);
+            pthread_create(&thread2[i], NULL, query_thread2, &data);
+        }
+        Test->tprintf("Threads are running %d seconds \n", run_time);
+        sleep(run_time);
+        data.exit_flag = 1;
+        Test->tprintf("Waiting for all threads to exit\n");
+        Test->set_timeout(100);
+        for (int i = 0; i < threads_num; i++)
+        {
+            pthread_join(thread1[i], NULL);
+            pthread_join(thread2[i], NULL);
+        }
+        sleep(1);
+
+        Test->tprintf("COM_INSERT and COM_SELECT after executing test\n");
+        get_global_status_allnodes(&new_selects[0], &new_inserts[0], nodes, 0);
+        print_delta(&new_selects[0], &new_inserts[0], &selects[0], &inserts[0], nodes->N);
+        Test->tprintf("First group of threads did %d queries, second - %d \n", data.i1, data.i2);
+    }
+    nodes->close_connections();
+    *i1 = data.i1;
+    *i2 = data.i2;
+}
+
+void* query_thread1(void* ptr)
+{
+    MYSQL* conn1;
+    MYSQL* conn2;
+    MYSQL* conn3;
+    int conn_err = 0;
+    thread_data* data = (thread_data*) ptr;
+    conn1 = open_conn_db_timeout(data->Test->maxscales->rwsplit_port[0],
+                                 data->Test->maxscales->IP[0],
+                                 (char*) "test",
+                                 data->Test->maxscales->user_name,
+                                 data->Test->maxscales->password,
+                                 20,
+                                 data->Test->ssl);
+    // conn1 = data->Test->maxscales->open_rwsplit_connection(0);
+    if (mysql_errno(conn1) != 0)
+    {
+        conn_err++;
+    }
+    if (data->rwsplit_only == 0)
+    {
+        // conn2 = data->Test->maxscales->open_readconn_master_connection(0);
+        conn2 = open_conn_db_timeout(data->Test->maxscales->readconn_master_port[0],
+                                     data->Test->maxscales->IP[0],
+                                     (char*) "test",
+                                     data->Test->maxscales->user_name,
+                                     data->Test->maxscales->password,
+                                     20,
+                                     data->Test->ssl);
+        if (mysql_errno(conn2) != 0)
+        {
+            conn_err++;
+        }
+        // conn3 = data->Test->maxscales->open_readconn_slave_connection(0);
+        conn3 = open_conn_db_timeout(data->Test->maxscales->readconn_slave_port[0],
+                                     data->Test->maxscales->IP[0],
+                                     (char*) "test",
+                                     data->Test->maxscales->user_name,
+                                     data->Test->maxscales->password,
+                                     20,
+                                     data->Test->ssl);
+        if (mysql_errno(conn3) != 0)
+        {
+            conn_err++;
+        }
+    }
+    if (conn_err == 0)
+    {
+        while (data->exit_flag == 0)
+        {
+            if (execute_query_silent(conn1, (char*) "SELECT * FROM t1;") == 0)
+            {
+                __sync_fetch_and_add(&data->i1, 1);
+            }
+
+            if (data->rwsplit_only == 0)
+            {
+                execute_query_silent(conn2, (char*) "SELECT * FROM t1;");
+                execute_query_silent(conn3, (char*) "SELECT * FROM t1;");
+            }
+        }
+        mysql_close(conn1);
+        if (data->rwsplit_only == 0)
+        {
+            mysql_close(conn2);
+            mysql_close(conn3);
+        }
+    }
+    return NULL;
+}
+
+void* query_thread2(void* ptr)
+{
+    MYSQL* conn1;
+    MYSQL* conn2;
+    MYSQL* conn3;
+    thread_data* data = (thread_data*) ptr;
+    // conn1 = data->Test->maxscales->open_rwsplit_connection(0);
+    conn1 = open_conn_db_timeout(data->Test->maxscales->rwsplit_port[0],
+                                 data->Test->maxscales->IP[0],
+                                 (char*) "test",
+                                 data->Test->maxscales->user_name,
+                                 data->Test->maxscales->password,
+                                 20,
+                                 data->Test->ssl);
+    if (data->rwsplit_only == 0)
+    {
+        // conn2 = data->Test->maxscales->open_readconn_master_connection(0);
+        // conn3 = data->Test->maxscales->open_readconn_slave_connection(0);
+
+        conn2 = open_conn_db_timeout(data->Test->maxscales->readconn_master_port[0],
+                                     data->Test->maxscales->IP[0],
+                                     (char*) "test",
+                                     data->Test->maxscales->user_name,
+                                     data->Test->maxscales->password,
+                                     20,
+                                     data->Test->ssl);
+        // if (mysql_errno(conn2) != 0) { conn_err++; }
+        conn3 = open_conn_db_timeout(data->Test->maxscales->readconn_slave_port[0],
+                                     data->Test->maxscales->IP[0],
+                                     (char*) "test",
+                                     data->Test->maxscales->user_name,
+                                     data->Test->maxscales->password,
+                                     20,
+                                     data->Test->ssl);
+        // if (mysql_errno(conn3) != 0) { conn_err++; }
+    }
+    while (data->exit_flag == 0)
+    {
+        sleep(1);
+        if (execute_query_silent(conn1, (char*) "SELECT * FROM t1;") == 0)
+        {
+            __sync_fetch_and_add(&data->i2, 1);
+        }
+        if (data->rwsplit_only == 0)
+        {
+            execute_query_silent(conn2, (char*) "SELECT * FROM t1;");
+            execute_query_silent(conn3, (char*) "SELECT * FROM t1;");
+        }
+    }
+    mysql_close(conn1);
+    if (data->rwsplit_only == 0)
+    {
+        mysql_close(conn2);
+        mysql_close(conn3);
+    }
+    return NULL;
+}

maxscale-system-test/maxtest/src/big_transaction.cpp

@@ -0,0 +1,23 @@
+#include "big_transaction.h"
+
+int big_transaction(MYSQL* conn, int N)
+{
+    int local_result = 0;
+    char sql[1000000];
+    local_result += create_t1(conn);
+    local_result += execute_query(conn, (char*) "START TRANSACTION");
+    local_result += execute_query(conn, (char*) "SET autocommit = 0");
+
+    for (int i = 0; i < N; i++)
+    {
+        create_insert_string(sql, 10000, i);
+        local_result += execute_query(conn, "%s", sql);
+        local_result += execute_query(conn, "CREATE TABLE t2(id int);");
+        local_result += execute_query(conn, "%s", sql);
+        local_result += execute_query(conn, "DROP TABLE t2;");
+        local_result += execute_query(conn, "%s", sql);
+    }
+
+    local_result += execute_query(conn, (char*) "COMMIT");
+    return local_result;
+}

maxscale-system-test/maxtest/src/blob_test.cpp

@@ -0,0 +1,211 @@
+#include "blob_test.h"
+
+int test_longblob(TestConnections* Test,
+                  MYSQL* conn,
+                  char*  blob_name,
+                  unsigned long chunk_size,
+                  int chunks,
+                  int rows)
+{
+    int size = chunk_size;
+    unsigned long* data;
+    int i, j;
+    MYSQL_BIND param[1];
+    char sql[256];
+    int global_res = Test->global_result;
+    // Test->tprintf("chunk size %lu chunks %d inserts %d\n", chunk_size, chunks, rows);
+
+    char* insert_stmt = (char*) "INSERT INTO long_blob_table(x, b) VALUES(1, ?)";
+
+    Test->tprintf("Creating table with %s\n", blob_name);
+    Test->try_query(conn, (char*) "DROP TABLE IF EXISTS long_blob_table");
+    sprintf(sql,
+            "CREATE TABLE long_blob_table(id int NOT NULL AUTO_INCREMENT, x INT, b %s, PRIMARY KEY (id))",
+            blob_name);
+    Test->try_query(conn, "%s", sql);
+
+    for (int k = 0; k < rows; k++)
+    {
+        Test->tprintf("Preparintg INSERT stmt\n");
+        MYSQL_STMT* stmt = mysql_stmt_init(conn);
+        if (stmt == NULL)
+        {
+            Test->add_result(1, "stmt init error: %s\n", mysql_error(conn));
+        }
+
+        Test->add_result(mysql_stmt_prepare(stmt, insert_stmt, strlen(insert_stmt)),
+                         "Error preparing stmt: %s\n",
+                         mysql_stmt_error(stmt));
+
+        param[0].buffer_type = MYSQL_TYPE_STRING;
+        param[0].is_null = 0;
+
+        Test->tprintf("Binding parameter\n");
+        Test->add_result(mysql_stmt_bind_param(stmt, param),
+                         "Error parameter binding: %s\n",
+                         mysql_stmt_error(stmt));
+
+        Test->tprintf("Filling buffer\n");
+        data = (unsigned long*) malloc(size * sizeof(long int));
+
+        if (data == NULL)
+        {
+            Test->add_result(1, "Memory allocation error\n");
+        }
+
+
+
+        Test->tprintf("Sending data in %d bytes chunks, total size is %d\n",
+                      size * sizeof(unsigned long),
+                      (size * sizeof(unsigned long)) * chunks);
+        for (i = 0; i < chunks; i++)
+        {
+            for (j = 0; j < size; j++)
+            {
+                data[j] = j + i * size;
+            }
+            Test->set_timeout(300);
+            Test->tprintf("Chunk #%d\n", i);
+            if (mysql_stmt_send_long_data(stmt, 0, (char*) data, size * sizeof(unsigned long)) != 0)
+            {
+                Test->add_result(1,
+                                 "Error inserting data, iteration %d, error %s\n",
+                                 i,
+                                 mysql_stmt_error(stmt));
+                return 1;
+            }
+        }
+
+        // for (int k = 0; k < rows; k++)
+        // {
+        Test->tprintf("Executing statement: %02d\n", k);
+        Test->set_timeout(3000);
+        Test->add_result(mysql_stmt_execute(stmt),
+                         "INSERT Statement with %s failed, error is %s\n",
+                         blob_name,
+                         mysql_stmt_error(stmt));
+        // }
+        Test->add_result(mysql_stmt_close(stmt), "Error closing stmt\n");
+    }
+
+    if (global_res == Test->global_result)
+    {
+        Test->tprintf("%s is OK\n", blob_name);
+    }
+    else
+    {
+        Test->tprintf("%s FAILED\n", blob_name);
+    }
+
+    return 0;
+}
+
+int check_longblob_data(TestConnections* Test,
+                        MYSQL* conn,
+                        unsigned long chunk_size,
+                        int chunks,
+                        int rows)
+{
+    // char *select_stmt = (char *) "SELECT id, x, b FROM long_blob_table WHERE id = ?";
+    char* select_stmt = (char*) "SELECT id, x, b FROM long_blob_table ";
+    MYSQL_STMT* stmt = mysql_stmt_init(Test->maxscales->conn_rwsplit[0]);
+    if (stmt == NULL)
+    {
+        Test->add_result(1, "stmt init error: %s\n", mysql_error(Test->maxscales->conn_rwsplit[0]));
+    }
+
+    Test->add_result(mysql_stmt_prepare(stmt, select_stmt, strlen(select_stmt)),
+                     "Error preparing stmt: %s\n",
+                     mysql_stmt_error(stmt));
+
+    MYSQL_BIND param[1], result[3];
+    int id = 1;
+
+    memset(param, 0, sizeof(param));
+    memset(result, 0, sizeof(result));
+
+    param[0].buffer_type = MYSQL_TYPE_LONG;
+    param[0].buffer = &id;
+
+    unsigned long* data = (unsigned long*) malloc(chunk_size * chunks * sizeof(long int));
+
+
+    int r_id;
+    int r_x;
+    unsigned long l_id;
+    unsigned long l_x;
+    my_bool b_id;
+    my_bool b_x;
+    my_bool e_id;
+    my_bool e_x;
+
+    result[0].buffer_type = MYSQL_TYPE_LONG;
+    result[0].buffer = &r_id;
+    result[0].buffer_length = 0;
+    result[0].length = &l_id;
+    result[0].is_null = &b_id;
+    result[0].error = &e_id;
+
+    result[1].buffer_type = MYSQL_TYPE_LONG;
+    result[1].buffer = &r_x;
+    result[1].buffer_length = 0;
+    result[1].length = &l_x;
+    result[1].is_null = &b_x;
+    result[1].error = &e_x;
+
+    result[2].buffer_type = MYSQL_TYPE_LONG_BLOB;
+    result[2].buffer = data;
+    result[2].buffer_length = chunk_size * chunks * sizeof(long int);
+
+    /*
+     *   if (mysql_stmt_bind_param(stmt, param) != 0)
+     *   {
+     *       printf("Could not bind parameters\n");
+     *       return 1;
+     *   }
+     */
+    if (mysql_stmt_bind_result(stmt, result) != 0)
+    {
+        printf("Could not bind results: %s\n", mysql_stmt_error(stmt));
+        return 1;
+    }
+
+    if (mysql_stmt_execute(stmt) != 0)
+    {
+        Test->tprintf("Error executing stmt %s\n", mysql_error(Test->maxscales->conn_rwsplit[0]));
+    }
+
+    if (mysql_stmt_store_result(stmt) != 0)
+    {
+        printf("Could not buffer result set: %s\n", mysql_stmt_error(stmt));
+        return 1;
+    }
+
+    int row = 0;
+    while (!mysql_stmt_fetch(stmt))
+    {
+        Test->tprintf("id=%d\tx=%d\n", r_id, r_x);
+        if (r_id != row + 1)
+        {
+            Test->add_result(1, "id field is wrong! Expected %d, but it is %d\n", row + 1, r_id);
+        }
+        for (int y = 0; y < (int)chunk_size * chunks; y++)
+        {
+            if ((int)data[y] != y)
+            {
+                Test->add_result(1, "expected %lu, got %d", data[y], y);
+                break;
+            }
+        }
+        row++;
+    }
+    if (row != rows)
+    {
+        Test->add_result(1, "Wrong number of rows in the table! Expected %d, but it is %d\n", rows, row);
+    }
+    mysql_stmt_free_result(stmt);
+
+    mysql_stmt_close(stmt);
+
+    return 0;
+}

maxscale-system-test/maxtest/src/clustrix_nodes.cpp

@@ -0,0 +1,365 @@
+/*
+ * Copyright (c) 2016 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2024-02-10
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#include <fstream>
+#include <iostream>
+#include <regex>
+#include <sstream>
+#include "clustrix_nodes.h"
+
+int Clustrix_nodes::prepare_server(int m)
+{
+    int rv = 1;
+    int ec;
+    char* clustrix_rpm = ssh_node_output(m, "rpm -qa | grep clustrix-clxnode", true, &ec);
+    if (strstr(clustrix_rpm, "clustrix-clxnode") == NULL)
+    {
+        char* str1 = nullptr;
+        char* str2 = nullptr;
+        char* str3 = nullptr;
+        char* str4 = nullptr;
+
+        str1 = ssh_node_output(m, CLUSTRIX_DEPS_YUM, true, &ec);
+        if (ec == 0)
+        {
+            printf("Installed clustrix dependencies on node %d.\n", m);
+            str2 = ssh_node_output(m, WGET_CLUSTRIX, false, &ec);
+            if (ec == 0)
+            {
+                printf("Wgot Clustrix installation package on node %d.\n", m);
+                str3 = ssh_node_output(m, UNPACK_CLUSTRIX, false, &ec);
+                if (ec == 0)
+                {
+                    printf("Unpacked Clustrix package on node %d.\n", m);
+                    str4 = ssh_node_output(m, INSTALL_CLUSTRIX, false, &ec);
+                    if (ec == 0)
+                    {
+                        printf("Successfully installed Clustrix on node %d.\n", m);
+                    }
+                    else
+                    {
+                        printf("Error: Could not install Clustrix package on node %d: %s\n", m, str4);
+                    }
+                }
+                else
+                {
+                    printf("Error: Could not unpack Clustrix package on node %d: %s\n", m, str3);
+                }
+            }
+            else
+            {
+                printf("Error: Could not wget Clustrix installation package on node %d: %s\n", m, str2);
+            }
+        }
+        else
+        {
+            printf("Error: Could not install Clustrix dependencies on node %d: %s\n", m, str1);
+        }
+
+        free(str4);
+        free(str3);
+        free(str2);
+        free(str1);
+    }
+
+    free(clustrix_rpm);
+
+    bool running = false;
+
+    ec = ssh_node(m, "systemctl status clustrix", true);
+
+    if (ec == 0)
+    {
+        printf("Clustrix running on node %d.\n", m);
+
+        ec = ssh_node(m, "mysql -e 'SELECT @@server_id'", true);
+        if (ec == 0)
+        {
+            running = true;
+        }
+        else
+        {
+            printf("Could not connect as root to Clustrix on node %d, restarting.\n", m);
+
+            ec = ssh_node(m, "systemctl restart clustrix", true);
+
+            if (ec == 0)
+            {
+                printf("Successfully restarted Clustrix on node %d.\n", m);
+                running = true;
+            }
+            else
+            {
+                printf("Could not restart Clustrix on node %d.\n", m);
+            }
+        }
+    }
+    else
+    {
+        printf("Clustrix not running on node %d, starting.\n", m);
+
+        ec = ssh_node(m, "systemctl start clustrix", true);
+
+        if (ec == 0)
+        {
+            printf("Successfully started Clustrix on node %d.\n", m);
+            running = true;
+        }
+        else
+        {
+            printf("Could not start Clustrix on node %d.\n", m);
+        }
+    }
+
+    bool check_users = false;
+
+    if (running)
+    {
+        int start = time(NULL);
+        int now;
+
+        do
+        {
+            ec = ssh_node(m, "mysql -e 'SELECT @@server_id'", true);
+            now = time(NULL);
+
+            if (ec != 0)
+            {
+                printf("Could not connect to Clustrix as root on node %d, "
+                       "sleeping a while (totally at most ~1 minute) and retrying.\n", m);
+                sleep(10);
+            }
+        }
+        while (ec != 0 && now - start < 60);
+
+        if (ec == 0)
+        {
+            printf("Could connect as root to Clustrix on node %d.\n", m);
+            check_users = true;
+        }
+        else
+        {
+            printf("Could not connect as root to Clustrix on node %d within given timeframe.\n", m);
+        }
+    }
+
+    if (check_users)
+    {
+        std::string command("mysql ");
+        command += "-u ";
+        command += this->user_name;
+        command += " ";
+        command += "-p";
+        command += this->password;
+
+        ec = ssh_node(m, command.c_str(), false);
+
+        if (ec == 0)
+        {
+            printf("Can access Clustrix using user '%s'.\n", this->user_name);
+            rv = 0;
+        }
+        else
+        {
+            printf("Cannot access Clustrix using user '%s', creating users.\n", this->user_name);
+            // TODO: We need an return code here.
+            create_users(m);
+            rv = 0;
+        }
+    }
+
+    return rv;
+}
+
+namespace
+{
+
+using std::string;
+
+bool license_is_valid(const std::string& license)
+{
+    static const std::regex regex("\"expiration\":\"[^\"]+\"");
+
+    bool is_valid = false;
+
+    std::smatch match;
+
+    if (std::regex_search(license, match, regex))
+    {
+        if (match.size() == 1)
+        {
+            string s = match[0].str();
+
+            s = s.substr(14, 10); // We expect something like '"expiration":"2019-08-21 00:00:00"'
+
+            if (s.length() == 10) // '2019-08-21' (excluding quotes)
+            {
+                int year = atoi(s.substr(0, 4).c_str());  // 2019
+                int month = atoi(s.substr(5, 2).c_str()); // 08
+                int day = atoi(s.substr(8, 2).c_str());   // 21
+
+                time_t timestamp = time(NULL);
+                struct tm now;
+                localtime_r(&timestamp, &now);
+
+                now.tm_year += 1900;
+                now.tm_mon += 1;
+
+                if (year >= now.tm_year
+                    && (month > now.tm_mon || (month == now.tm_mon && day >= now.tm_mday)))
+                {
+                    is_valid = true;
+                }
+                else
+                {
+                    printf("ERROR: The date is %d-%d-%d, but the license in the license file "
+                           "is valid only until %d-%d-%d.\n",
+                           now.tm_year, now.tm_mon, now.tm_mday,
+                           year, month, day);
+                }
+            }
+            else
+            {
+                printf("ERROR: The value of the key 'expiration' does not appear to be valid.\n");
+            }
+        }
+        else
+        {
+            printf("ERROR: The license in the license file either does not contain an "
+                   "'expiration' key or then it contains several.\n");
+        }
+    }
+    else
+    {
+        printf("ERROR: The license file does not seem to contain a valid license.\n");
+    }
+
+    return is_valid;
+}
+
+}
+
+int Clustrix_nodes::start_replication()
+{
+    int rv = 1;
+
+    std::string lic_filename = std::string(getenv("HOME"))
+            + std::string("/.config/mdbci/clustrix_license");
+    std::ifstream lic_file;
+    lic_file.open(lic_filename.c_str());
+
+    if (lic_file.is_open())
+    {
+        printf("Using license file '%s'.\n", lic_filename.c_str());
+
+        std::stringstream ss;
+        ss << lic_file.rdbuf();
+        std::string clustrix_license = ss.str();
+        lic_file.close();
+
+        if (license_is_valid(clustrix_license))
+        {
+            execute_query_all_nodes(clustrix_license.c_str());
+
+            std::string cluster_setup_sql = std::string("ALTER CLUSTER ADD '")
+                + std::string(IP_private[1])
+                + std::string("'");
+            for (int i = 2; i < N; i++)
+            {
+                cluster_setup_sql += std::string(",'")
+                    + std::string(IP_private[i])
+                    + std::string("'");
+            }
+            connect();
+            execute_query(nodes[0], "%s", cluster_setup_sql.c_str());
+            close_connections();
+
+            rv = 0;
+        }
+    }
+    else
+    {
+        printf("ERROR: The Clustrix license file '%s' does not exist. "
+               "It must contain a string \"set global license='{...}';\" using which the "
+               "Clustrix license can be set.\n",
+               lic_filename.c_str());
+    }
+
+    return rv;
+}
+
+std::string Clustrix_nodes::cnf_servers()
+{
+    std::string s;
+    for (int i = 0; i < N; i++)
+    {
+        s += std::string("\\n[")
+                + cnf_server_name
+                + std::to_string(i + 1)
+                + std::string("]\\ntype=server\\naddress=")
+                + std::string(IP_private[i])
+                + std::string("\\nport=")
+                + std::to_string(port[i])
+                + std::string("\\nprotocol=MySQLBackend\\n");
+    }
+    return s;
+}
+
+int Clustrix_nodes::check_replication()
+{
+    int res = 0;
+    if (connect() == 0)
+    {
+        for (int i = 0; i < N; i++)
+        {
+            if (execute_query_count_rows(nodes[i], "select * from system.nodeinfo") != N)
+            {
+                res = 1;
+            }
+        }
+    }
+    else
+    {
+        res = 1;
+    }
+
+    close_connections(); // Some might have been created by connect().
+
+    return res;
+}
+
+std::string Clustrix_nodes::block_command(int node) const
+{
+    std::string command = Mariadb_nodes::block_command(node);
+
+    // Block health-check port as well.
+    command += ";";
+    command += "iptables -I INPUT -p tcp --dport 3581 -j REJECT";
+    command += ";";
+    command += "ip6tables -I INPUT -p tcp --dport 3581 -j REJECT";
+
+    return command;
+}
+
+std::string Clustrix_nodes::unblock_command(int node) const
+{
+    std::string command = Mariadb_nodes::unblock_command(node);
+
+    // Unblock health-check port as well.
+    command += ";";
+    command += "iptables -I INPUT -p tcp --dport 3581 -j ACCEPT";
+    command += ";";
+    command += "ip6tables -I INPUT -p tcp --dport 3581 -j ACCEPT";
+
+    return command;
+}

maxscale-system-test/maxtest/src/config_operations.cpp

@@ -0,0 +1,252 @@
+#include "config_operations.h"
+
+// The configuration should use these names for the services, listeners and monitors
+#define SERVICE_NAME1  "rwsplit-service"
+#define SERVICE_NAME2  "read-connection-router-master"
+#define SERVICE_NAME3  "read-connection-router-slave"
+#define LISTENER_NAME1 "rwsplit-service-listener"
+#define LISTENER_NAME2 "read-connection-router-master-listener"
+#define LISTENER_NAME3 "read-connection-router-slave-listener"
+
+struct
+{
+    const char* service;
+    const char* listener;
+    int         port;
+} services[]
+{
+    {SERVICE_NAME1, LISTENER_NAME1, 4006},
+    {SERVICE_NAME2, LISTENER_NAME2, 4008},
+    {SERVICE_NAME3, LISTENER_NAME3, 4009}
+};
+
+Config::Config(TestConnections* parent)
+    : test_(parent)
+{
+}
+
+Config::~Config()
+{
+}
+
+void Config::add_server(int num)
+{
+    test_->tprintf("Adding the servers");
+    test_->set_timeout(120);
+    test_->maxscales->ssh_node_f(0, true, "maxadmin add server server%d " SERVICE_NAME1, num);
+    test_->maxscales->ssh_node_f(0, true, "maxadmin add server server%d " SERVICE_NAME2, num);
+    test_->maxscales->ssh_node_f(0, true, "maxadmin add server server%d " SERVICE_NAME3, num);
+
+    for (auto& a : created_monitors_)
+    {
+        test_->maxscales->ssh_node_f(0, true, "maxadmin add server server%d %s", num, a.c_str());
+    }
+
+    test_->stop_timeout();
+}
+
+void Config::remove_server(int num)
+{
+    test_->set_timeout(120);
+    test_->maxscales->ssh_node_f(0, true, "maxadmin remove server server%d " SERVICE_NAME1, num);
+    test_->maxscales->ssh_node_f(0, true, "maxadmin remove server server%d " SERVICE_NAME2, num);
+    test_->maxscales->ssh_node_f(0, true, "maxadmin remove server server%d " SERVICE_NAME3, num);
+
+    for (auto& a : created_monitors_)
+    {
+        test_->maxscales->ssh_node_f(0, true, "maxadmin remove server server%d %s", num, a.c_str());
+    }
+
+    test_->stop_timeout();
+}
+
+void Config::add_created_servers(const char* object)
+{
+    for (auto a : created_servers_)
+    {
+        test_->maxscales->ssh_node_f(0, true, "maxadmin add server server%d %s", a, object);
+    }
+}
+
+void Config::destroy_server(int num)
+{
+    test_->set_timeout(120);
+    test_->maxscales->ssh_node_f(0, true, "maxadmin destroy server server%d", num);
+    created_servers_.erase(num);
+    test_->stop_timeout();
+}
+
+void Config::create_server(int num)
+{
+    test_->set_timeout(120);
+    char ssl_line[200 + 3 * strlen(test_->maxscales->access_homedir[0])] = "";
+    if (test_->backend_ssl)
+    {
+        sprintf(ssl_line,
+                " --tls-key=/%s/certs/client-key.pem "
+                " --tls-cert=/%s/certs/client-cert.pem "
+                " --tls-ca-cert=/%s/certs/ca.pem "
+                " --tls-version=MAX "
+                " --tls-cert-verify-depth=9",
+                test_->maxscales->access_homedir[0],
+                test_->maxscales->access_homedir[0],
+                test_->maxscales->access_homedir[0]);
+    }
+    test_->maxscales->ssh_node_f(0,
+                                 true,
+                                 "maxctrl create server server%d %s %d %s",
+                                 num,
+                                 test_->repl->IP_private[num],
+                                 test_->repl->port[num],
+                                 ssl_line);
+    created_servers_.insert(num);
+    test_->stop_timeout();
+}
+
+void Config::alter_server(int num, const char* key, const char* value)
+{
+    test_->maxscales->ssh_node_f(0, true, "maxadmin alter server server%d %s=%s", num, key, value);
+}
+
+void Config::alter_server(int num, const char* key, int value)
+{
+    test_->maxscales->ssh_node_f(0, true, "maxadmin alter server server%d %s=%d", num, key, value);
+}
+
+void Config::alter_server(int num, const char* key, float value)
+{
+    test_->maxscales->ssh_node_f(0, true, "maxadmin alter server server%d %s=%f", num, key, value);
+}
+
+void Config::create_monitor(const char* name, const char* module, int interval)
+{
+    test_->set_timeout(120);
+    test_->maxscales->ssh_node_f(0, true, "maxadmin create monitor %s %s", name, module);
+    alter_monitor(name, "monitor_interval", interval);
+    alter_monitor(name, "user", test_->maxscales->user_name);
+    alter_monitor(name, "password", test_->maxscales->password);
+    test_->maxscales->ssh_node_f(0, true, "maxadmin restart monitor %s", name);
+    test_->stop_timeout();
+
+    created_monitors_.insert(std::string(name));
+}
+
+void Config::alter_monitor(const char* name, const char* key, const char* value)
+{
+    test_->maxscales->ssh_node_f(0, true, "maxadmin alter monitor %s %s=%s", name, key, value);
+}
+
+void Config::alter_monitor(const char* name, const char* key, int value)
+{
+    test_->maxscales->ssh_node_f(0, true, "maxadmin alter monitor %s %s=%d", name, key, value);
+}
+
+void Config::alter_monitor(const char* name, const char* key, float value)
+{
+    test_->maxscales->ssh_node_f(0, true, "maxadmin alter monitor %s %s=%f", name, key, value);
+}
+
+void Config::start_monitor(const char* name)
+{
+    test_->maxscales->ssh_node_f(0, true, "maxadmin restart monitor %s", name);
+}
+
+void Config::destroy_monitor(const char* name)
+{
+    test_->set_timeout(120);
+    test_->maxscales->ssh_node_f(0, true, "maxadmin destroy monitor %s", name);
+    test_->stop_timeout();
+    created_monitors_.erase(std::string(name));
+}
+
+void Config::restart_monitors()
+{
+    for (auto& a : created_monitors_)
+    {
+        test_->maxscales->ssh_node_f(0, true, "maxadmin shutdown monitor \"%s\"", a.c_str());
+        test_->maxscales->ssh_node_f(0, true, "maxadmin restart monitor \"%s\"", a.c_str());
+    }
+}
+
+void Config::create_listener(Config::Service service)
+{
+    int i = static_cast<int>(service);
+
+    test_->set_timeout(120);
+    test_->maxscales->ssh_node_f(0,
+                                 true,
+                                 "maxadmin create listener %s %s default %d",
+                                 services[i].service,
+                                 services[i].listener,
+                                 services[i].port);
+    test_->stop_timeout();
+}
+
+void Config::create_ssl_listener(Config::Service service)
+{
+    int i = static_cast<int>(service);
+
+    test_->set_timeout(120);
+    test_->maxscales->ssh_node_f(0,
+                                 true,
+                                 "maxadmin create listener %s %s default %d default default default "
+                                 "/%s/certs/server-key.pem "
+                                 "/%s/certs/server-cert.pem "
+                                 "/%s/certs/ca.pem ",
+                                 services[i].service,
+                                 services[i].listener,
+                                 services[i].port,
+                                 test_->maxscales->access_homedir[0],
+                                 test_->maxscales->access_homedir[0],
+                                 test_->maxscales->access_homedir[0]);
+    test_->stop_timeout();
+}
+
+void Config::destroy_listener(Config::Service service)
+{
+    int i = static_cast<int>(service);
+
+    test_->set_timeout(120);
+    test_->maxscales->ssh_node_f(0,
+                                 true,
+                                 "maxadmin destroy listener %s %s",
+                                 services[i].service,
+                                 services[i].listener);
+    test_->stop_timeout();
+}
+
+void Config::create_all_listeners()
+{
+    create_listener(SERVICE_RWSPLIT);
+    create_listener(SERVICE_RCONN_SLAVE);
+    create_listener(SERVICE_RCONN_MASTER);
+}
+
+void Config::reset()
+{
+    /** Make sure the servers exist before checking that connectivity is OK */
+    for (int i = 0; i < test_->repl->N; i++)
+    {
+        if (created_servers_.find(i) == created_servers_.end())
+        {
+            create_server(i);
+            add_server(i);
+        }
+    }
+}
+
+bool Config::check_server_count(int expected)
+{
+    bool rval = true;
+
+    if (test_->maxscales->ssh_node_f(0,
+                                     true,
+                                     "test \"`maxadmin list servers|grep 'server[0-9]'|wc -l`\" == \"%d\"",
+                                     expected))
+    {
+        test_->add_result(1, "Number of servers is not %d.", expected);
+        rval = false;
+    }
+
+    return rval;
+}

maxscale-system-test/maxtest/src/different_size.cpp

@@ -0,0 +1,107 @@
+#include <iostream>
+#include <unistd.h>
+#include "testconnections.h"
+
+using namespace std;
+
+char* create_event_size(unsigned long size)
+{
+    char* prefix = (char*) "insert into test.large_event values (1, '";
+    unsigned long prefix_size = strlen(prefix);
+    char* postfix = (char*) "');";
+    char* event = (char*)malloc(size + 1);
+    strcpy(event, prefix);
+
+    unsigned long max = size - 55 - 45;
+
+
+    // printf("BLOB data size %lu\n", max);
+
+    for (unsigned long i = 0; i < max; i++)
+    {
+        event[i + prefix_size] = 'a';
+    }
+
+    strcpy((char*) event + max + prefix_size, postfix);
+    return event;
+}
+
+MYSQL* connect_to_serv(TestConnections* Test, bool binlog)
+{
+    MYSQL* conn;
+    if (binlog)
+    {
+        conn = open_conn(Test->repl->port[0],
+                         Test->repl->IP[0],
+                         Test->repl->user_name,
+                         Test->repl->password,
+                         Test->ssl);
+    }
+    else
+    {
+        conn = Test->maxscales->open_rwsplit_connection(0);
+    }
+    return conn;
+}
+
+void set_max_packet(TestConnections* Test, bool binlog, char* cmd)
+{
+    Test->tprintf("Setting maximum packet size ...");
+    if (binlog)
+    {
+        Test->repl->connect();
+        Test->try_query(Test->repl->nodes[0], "%s", cmd);
+        Test->repl->close_connections();
+    }
+    else
+    {
+        Test->maxscales->connect_maxscale(0);
+        Test->try_query(Test->maxscales->conn_rwsplit[0], "%s", cmd);
+        Test->maxscales->close_maxscale_connections(0);
+    }
+    Test->tprintf(".. done\n");
+}
+
+void different_packet_size(TestConnections* Test, bool binlog)
+{
+    Test->set_timeout(180);
+    Test->tprintf("Set big max_allowed_packet\n");
+    set_max_packet(Test, binlog, (char*) "set global max_allowed_packet = 200000000;");
+
+    Test->set_timeout(120);
+    Test->tprintf("Create table\n");
+    MYSQL* conn = connect_to_serv(Test, binlog);
+    Test->try_query(conn,
+                    "DROP TABLE IF EXISTS test.large_event;"
+                    "CREATE TABLE test.large_event(id INT, data LONGBLOB);");
+    mysql_close(conn);
+
+    const int loops = 3;
+    const int range = 2;
+
+    for (int i = 1; i <= loops; i++)
+    {
+        for (int j = -range; j <= range; j++)
+        {
+            size_t size = 0x0ffffff * i + j;
+            Test->tprintf("Trying event app. %lu bytes", size);
+            Test->set_timeout(1000);
+
+            char* event = create_event_size(size);
+            conn = connect_to_serv(Test, binlog);
+            Test->expect(execute_query_silent(conn, event) == 0, "Query should succeed");
+            free(event);
+            execute_query_silent(conn, (char*) "DELETE FROM test.large_event");
+            mysql_close(conn);
+        }
+    }
+
+    Test->set_timeout(120);
+    Test->tprintf("Restoring max_allowed_packet");
+    set_max_packet(Test, binlog, (char*) "set global max_allowed_packet = 1048576;");
+
+    Test->set_timeout(1000);
+    conn = connect_to_serv(Test, binlog);
+    Test->try_query(conn, "DROP TABLE test.large_event");
+    mysql_close(conn);
+}

maxscale-system-test/maxtest/src/envv.cpp

@@ -0,0 +1,61 @@
+#include <string.h>
+#include <string>
+#include "envv.h"
+
+char * readenv(const char * name, const char *format, ...)
+{
+    char * env = getenv(name);
+    if (!env)
+    {
+        va_list valist;
+
+        va_start(valist, format);
+        int message_len = vsnprintf(NULL, 0, format, valist);
+        va_end(valist);
+
+        if (message_len < 0)
+        {
+            return NULL;
+        }
+
+        env = (char*)malloc(message_len + 1);
+
+        va_start(valist, format);
+        vsnprintf(env, message_len + 1, format, valist);
+        va_end(valist);
+        setenv(name, env, 1);
+    }
+    return env;
+}
+
+int readenv_int(const char * name, int def)
+{
+    int x;
+    char * env = getenv(name);
+    if (env)
+    {
+        sscanf(env, "%d", &x);
+    }
+    else
+    {
+        x = def;
+        setenv(name, (std::to_string(x).c_str()), 1);
+    }
+    return x;
+}
+
+bool readenv_bool(const char * name, bool def)
+{
+    char * env = getenv(name);
+    if (env)
+    {
+        return ((strcasecmp(env, "yes") == 0) ||
+                (strcasecmp(env, "y") == 0) ||
+                (strcasecmp(env, "true") == 0));
+    }
+    else
+    {
+        setenv(name, def ? "true" : "false", 1);
+        return def;
+    }
+}

maxscale-system-test/maxtest/src/execute_cmd.cpp

@@ -0,0 +1,42 @@
+#include <iostream>
+#include <unistd.h>
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include "execute_cmd.h"
+
+using namespace std;
+
+
+int execute_cmd(char* cmd, char** res)
+{
+    char* result;
+    FILE* output = popen(cmd, "r");
+    if (output == NULL)
+    {
+        printf("Error opening ssh %s\n", strerror(errno));
+        return -1;
+    }
+    char buffer[10240];
+    size_t rsize = sizeof(buffer);
+    result = (char*)calloc(rsize, sizeof(char));
+
+    while (fgets(buffer, sizeof(buffer), output))
+    {
+        result = (char*)realloc(result, sizeof(buffer) + rsize);
+        rsize += sizeof(buffer);
+        strcat(result, buffer);
+    }
+
+    * res = result;
+
+    int return_code = pclose(output);
+    if (WIFEXITED(return_code))
+    {
+        return WEXITSTATUS(return_code);
+    }
+    else
+    {
+        return -1;
+    }
+}

maxscale-system-test/maxtest/src/fw_copy_rules.cpp

@@ -0,0 +1,29 @@
+#include "fw_copy_rules.h"
+#include <sstream>
+
+void copy_rules(TestConnections* Test, const char* rules_name, const char* rules_dir)
+{
+    std::stringstream src;
+    std::stringstream dest;
+
+    Test->maxscales->ssh_node_f(0,
+                                true,
+                                "cd %s;"
+                                "rm -rf rules;"
+                                "mkdir rules;"
+                                "chown %s:%s rules",
+                                Test->maxscales->access_homedir[0],
+                                Test->maxscales->access_user[0],
+                                Test->maxscales->access_user[0]);
+
+    src << rules_dir << "/" << rules_name;
+    dest << Test->maxscales->access_homedir[0] << "/rules/rules.txt";
+
+    Test->set_timeout(30);
+    Test->maxscales->copy_to_node_legacy(src.str().c_str(), dest.str().c_str(), 0);
+    Test->maxscales->ssh_node_f(0,
+                                true,
+                                "chmod a+r %s",
+                                dest.str().c_str());
+    Test->stop_timeout();
+}

maxscale-system-test/maxtest/src/get_com_select_insert.cpp

@@ -0,0 +1,105 @@
+#include "testconnections.h"
+
+/**
+ *  Reads COM_SELECT and COM_INSERT variables from all nodes and stores into 'selects' and 'inserts'
+ */
+int get_global_status_allnodes(long int* selects, long int* inserts, Mariadb_nodes* nodes, int silent)
+{
+    int i;
+    MYSQL_RES* res;
+    MYSQL_ROW row;
+
+    for (i = 0; i < nodes->N; i++)
+    {
+        if (nodes->nodes[i] != NULL)
+        {
+
+            if (mysql_query(nodes->nodes[i], "show global status like 'COM_SELECT';") != 0)
+            {
+                printf("Error: can't execute SQL-query\n");
+                printf("%s\n", mysql_error(nodes->nodes[i]));
+                return 1;
+            }
+
+            res = mysql_store_result(nodes->nodes[i]);
+            if (res == NULL)
+            {
+                printf("Error: can't get the result description\n");
+                return 1;
+            }
+
+            if (mysql_num_rows(res) > 0)
+            {
+                while ((row = mysql_fetch_row(res)) != NULL)
+                {
+                    if (silent == 0)
+                    {
+                        printf("Node %d COM_SELECT=%s\n", i, row[1]);
+                    }
+                    sscanf(row[1], "%ld", &selects[i]);
+                }
+            }
+
+            mysql_free_result(res);
+            while (mysql_next_result(nodes->nodes[i]) == 0)
+            {
+                res = mysql_store_result(nodes->nodes[i]);
+                mysql_free_result(res);
+            }
+
+            if (mysql_query(nodes->nodes[i], "show global status like 'COM_INSERT';") != 0)
+            {
+                printf("Error: can't execute SQL-query\n");
+            }
+
+            res = mysql_store_result(nodes->nodes[i]);
+            if (res == NULL)
+            {
+                printf("Error: can't get the result description\n");
+            }
+
+            if (mysql_num_rows(res) > 0)
+            {
+                while ((row = mysql_fetch_row(res)) != NULL)
+                {
+                    if (silent == 0)
+                    {
+                        printf("Node %d COM_INSERT=%s\n", i, row[1]);
+                    }
+                    sscanf(row[1], "%ld", &inserts[i]);
+                }
+            }
+
+            mysql_free_result(res);
+            while (mysql_next_result(nodes->nodes[i]) == 0)
+            {
+                res = mysql_store_result(nodes->nodes[i]);
+                mysql_free_result(res);
+            }
+        }
+        else
+        {
+            selects[i] = 0;
+            inserts[i] = 0;
+        }
+    }
+    return 0;
+}
+
+/**
+ *  Prints difference in COM_SELECT and COM_INSERT
+ */
+int print_delta(long int* new_selects,
+                long int* new_inserts,
+                long int* selects,
+                long int* inserts,
+                int nodes_num)
+{
+    int i;
+    for (i = 0; i < nodes_num; i++)
+    {
+        printf("COM_SELECT increase on node %d is %ld\n", i, new_selects[i] - selects[i]);
+        printf("COM_INSERT increase on node %d is %ld\n", i, new_inserts[i] - inserts[i]);
+    }
+    return 0;
+}

maxscale-system-test/maxtest/src/get_my_ip.cpp

@@ -0,0 +1,60 @@
+/*
+ * Find local ip used as source ip in ip packets.
+ * Use getsockname and a udp connection
+ */
+
+#include <stdio.h>      // printf
+#include <string.h>     // memset
+#include <errno.h>      // errno
+#include <sys/socket.h> // socket
+#include <netinet/in.h> // sockaddr_in
+#include <arpa/inet.h>  // getsockname
+#include <unistd.h>     // close
+
+#include "get_my_ip.h"
+
+int get_my_ip(char* remote_ip, char* my_ip)
+{
+
+    int dns_port = 53;
+
+    struct sockaddr_in serv;
+
+    int sock = socket (AF_INET, SOCK_DGRAM, 0);
+
+    // Socket could not be created
+    if (sock < 0)
+    {
+        return 1;
+    }
+
+    memset(&serv, 0, sizeof(serv));
+    serv.sin_family = AF_INET;
+    serv.sin_addr.s_addr = inet_addr(remote_ip);
+    serv.sin_port = htons(dns_port);
+
+
+    connect(sock, (const struct sockaddr*) &serv, sizeof(serv));
+
+    struct sockaddr_in name;
+    socklen_t namelen = sizeof(name);
+    getsockname(sock, (struct sockaddr*) &name, &namelen);
+
+    char buffer[100];
+    const char* p = inet_ntop(AF_INET, &name.sin_addr, buffer, 100);
+
+    if (p != NULL)
+    {
+        // printf("Local ip is : %s \n" , buffer);
+        strcpy(my_ip, buffer);
+        close(sock);
+        return 0;
+    }
+    else
+    {
+        // Some error
+        printf ("Error number : %d . Error message : %s \n", errno, strerror(errno));
+        close(sock);
+        return 2;
+    }
+}

maxscale-system-test/maxtest/src/keepalived_func.cpp

@@ -0,0 +1,76 @@
+#include "keepalived_func.h"
+#include "get_my_ip.h"
+
+char* print_version_string(TestConnections* Test)
+{
+    MYSQL* keepalived_conn = open_conn(Test->maxscales->rwsplit_port[0],
+                                       virtual_ip,
+                                       Test->maxscales->user_name,
+                                       Test->maxscales->password,
+                                       Test->ssl);
+    const char* version_string;
+    mariadb_get_info(keepalived_conn, MARIADB_CONNECTION_SERVER_VERSION, (void*)&version_string);
+    Test->tprintf("%s\n", version_string);
+    mysql_close(keepalived_conn);
+    return (char*) version_string;
+}
+
+void configure_keepalived(TestConnections* Test, char* keepalived_file)
+{
+    int i;
+    char client_ip[24];
+    char* last_dot;
+    // Test->get_client_ip(0, client_ip);
+    get_my_ip(Test->maxscales->IP[0], client_ip);
+    last_dot = client_ip;
+    Test->tprintf("My IP is %s\n", client_ip);
+    for (i = 0; i < 3; i++)
+    {
+        last_dot = strstr(last_dot, ".");
+        last_dot = &last_dot[1];
+    }
+    last_dot[0] = '\0';
+    Test->tprintf("First part of IP is %s\n", client_ip);
+
+    sprintf(virtual_ip, "%s253", client_ip);
+
+
+    for (i = 0; i < Test->maxscales->N; i++)
+    {
+        std::string src = std::string(test_dir)
+            + "/keepalived_cnf/"
+            + std::string(keepalived_file)
+            + std::to_string(i + 1)
+            + ".conf";
+        std::string cp_cmd = "cp "
+            + std::string(Test->maxscales->access_homedir[i])
+            + std::string(keepalived_file)
+            + std::to_string(i + 1) + ".conf "
+            + " /etc/keepalived/keepalived.conf";
+        Test->tprintf("%s\n", src.c_str());
+        Test->tprintf("%s\n", cp_cmd.c_str());
+        Test->maxscales->ssh_node(i, "yum install -y keepalived", true);
+        Test->maxscales->ssh_node(i, "service iptables stop", true);
+        Test->maxscales->copy_to_node(i, src.c_str(), Test->maxscales->access_homedir[i]);
+        Test->maxscales->ssh_node(i, cp_cmd.c_str(), true);
+        Test->maxscales->ssh_node_f(i,
+                                    true,
+                                    "sed -i \"s/###virtual_ip###/%s/\" /etc/keepalived/keepalived.conf",
+                                    virtual_ip);
+        std::string script_src = std::string(test_dir) + "/keepalived_cnf/*.sh";
+        std::string script_cp_cmd = "cp " + std::string(Test->maxscales->access_homedir[i])
+            + "*.sh /usr/bin/";
+        Test->maxscales->copy_to_node(i, script_src.c_str(), Test->maxscales->access_homedir[i]);
+        Test->maxscales->ssh_node(i, script_cp_cmd.c_str(), true);
+        Test->maxscales->ssh_node(i, "sudo service keepalived restart", true);
+    }
+}
+
+void stop_keepalived(TestConnections* Test)
+{
+    for (int i = 0; i < Test->maxscales->N; i++)
+    {
+        Test->maxscales->ssh_node(i, "sudo service keepalived stop", true);
+        Test->maxscales->ssh_node(i, "killall -9 keepalived", true);
+    }
+}

maxscale-system-test/maxtest/src/labels_table.cpp

@@ -0,0 +1,31 @@
+#include <cstring>
+#include <string>
+#include <stdio.h>
+#include "labels_table.h"
+#include "testconnections.h"
+
+std::string get_mdbci_lables(const char *labels_string)
+{
+    std::string mdbci_labels("MAXSCALE");
+    for (size_t i = 0; i < sizeof(labels_table) / sizeof(labels_table_t); i++)
+    {
+        std::string test_label = std::string(";") + labels_table[i].test_label;
+        if (strstr(labels_string, test_label.c_str()))
+        {
+            mdbci_labels += "," + labels_table[i].mdbci_label;
+        }
+    }
+
+    if (TestConnections::verbose)
+    {
+        printf("mdbci labels %s\n", mdbci_labels.c_str());
+    }
+    return mdbci_labels;
+}
+
+bool has_label(std::string labels, std::string label)
+{
+    std::string labels_ext = ";" + labels + ";";
+    std::string label_ext = std::string(";") + label + std::string(";");
+    return (labels_ext.find(label_ext, 0) != std::string::npos);
+}

maxscale-system-test/maxtest/src/mariadb_func.cpp

@@ -0,0 +1,606 @@
+/**
+ * @file mariadb_func.cpp - basic DB interaction routines
+ *
+ * @verbatim
+ * Revision History
+ *
+ * Date     Who     Description
+ * 17/11/14 Timofey Turenko Initial implementation
+ *
+ * @endverbatim
+ */
+
+
+#include "mariadb_func.h"
+#include "templates.h"
+#include <ctype.h>
+#include <sstream>
+
+int set_ssl(MYSQL* conn)
+{
+    char client_key[1024];
+    char client_cert[1024];
+    char ca[1024];
+
+    sprintf(client_key, "%s/ssl-cert/client-key.pem", test_dir);
+    sprintf(client_cert, "%s/ssl-cert/client-cert.pem", test_dir);
+    sprintf(ca, "%s/ssl-cert/ca.pem", test_dir);
+
+    return mysql_ssl_set(conn, client_key, client_cert, ca, NULL, NULL);
+}
+
+MYSQL* open_conn_db_flags(int port,
+                          std::string ip,
+                          std::string db,
+                          std::string user,
+                          std::string password,
+                          unsigned long flag,
+                          bool ssl)
+{
+    MYSQL* conn = mysql_init(NULL);
+
+    if (conn == NULL)
+    {
+        fprintf(stdout, "Error: can't create MySQL-descriptor\n");
+        return NULL;
+    }
+
+    if (ssl)
+    {
+        set_ssl(conn);
+    }
+
+    // MXS-2568: This fixes mxs1828_double_local_infile
+    mysql_optionsv(conn, MYSQL_OPT_LOCAL_INFILE, (void*)"1");
+
+    if (!mysql_real_connect(conn,
+                            ip.c_str(),
+                            user.c_str(),
+                            password.c_str(),
+                            db.c_str(),
+                            port,
+                            NULL,
+                            flag))
+    {
+        fprintf(stdout,
+                "Could not connect to %s:%d with user '%s' and password '%s', "
+                "and default database '%s': %s\n",
+                ip.c_str(), port, user.c_str(), password.c_str(), db.c_str(), mysql_error(conn));
+    }
+    return conn;
+}
+
+MYSQL* open_conn_db_timeout(int port,
+                            std::string ip,
+                            std::string db,
+                            std::string user,
+                            std::string password,
+                            unsigned int timeout,
+                            bool ssl)
+{
+    MYSQL* conn = mysql_init(NULL);
+
+    if (conn == NULL)
+    {
+        fprintf(stdout, "Error: can't create MySQL-descriptor\n");
+        return NULL;
+    }
+
+    mysql_options(conn, MYSQL_OPT_CONNECT_TIMEOUT, &timeout);
+    mysql_options(conn, MYSQL_OPT_READ_TIMEOUT, &timeout);
+    mysql_options(conn, MYSQL_OPT_WRITE_TIMEOUT, &timeout);
+
+    // MXS-2568: This fixes mxs1828_double_local_infile
+    mysql_optionsv(conn, MYSQL_OPT_LOCAL_INFILE, (void*)"1");
+
+    if (ssl)
+    {
+        set_ssl(conn);
+    }
+
+    if (!mysql_real_connect(conn,
+                            ip.c_str(),
+                            user.c_str(),
+                            password.c_str(),
+                            db.c_str(),
+                            port,
+                            NULL,
+                            CLIENT_MULTI_STATEMENTS))
+    {
+        fprintf(stdout,
+                "Could not connect to %s:%d with user '%s' and password '%s', "
+                "and default database '%s': %s\n",
+                ip.c_str(), port, user.c_str(), password.c_str(), db.c_str(), mysql_error(conn));
+    }
+    return conn;
+}
+
+int execute_query(MYSQL* conn, const char* format, ...)
+{
+    va_list valist;
+
+    va_start(valist, format);
+    int message_len = vsnprintf(NULL, 0, format, valist);
+    va_end(valist);
+
+    char sql[message_len + 1];
+
+    va_start(valist, format);
+    vsnprintf(sql, sizeof(sql), format, valist);
+    va_end(valist);
+
+    return execute_query_silent(conn, sql, false);
+}
+
+int execute_query_from_file(MYSQL* conn, FILE* file)
+{
+    int rc = -1;
+    char buf[4096];
+
+    if (fgets(buf, sizeof(buf), file))
+    {
+        char* nul = strchr(buf, '\0') - 1;
+
+        while (isspace(*nul))
+        {
+            *nul-- = '\0';
+        }
+
+        char* ptr = buf;
+
+        while (isspace(*ptr))
+        {
+            ptr++;
+        }
+
+        if (*ptr)
+        {
+            rc = execute_query_silent(conn, buf, false);
+        }
+    }
+    else if (!feof(file))
+    {
+        printf("Failed to read file: %d, %s", errno, strerror(errno));
+        rc = 1;
+    }
+
+    return rc;
+}
+
+int execute_query_silent(MYSQL* conn, const char* sql, bool silent)
+{
+    MYSQL_RES* res;
+    if (conn != NULL)
+    {
+        if (mysql_query(conn, sql) != 0)
+        {
+            if (!silent)
+            {
+                int len = strlen(sql);
+                printf("Error: can't execute SQL-query: %.*s\n", len < 60 ? len : 60, sql);
+                printf("%s\n\n", mysql_error(conn));
+            }
+            return 1;
+        }
+        else
+        {
+            do
+            {
+                res = mysql_store_result(conn);
+                mysql_free_result(res);
+            }
+            while (mysql_next_result(conn) == 0);
+            return 0;
+        }
+    }
+    else
+    {
+        if (!silent)
+        {
+            printf("Connection is broken\n");
+        }
+        return 1;
+    }
+}
+
+int execute_query_check_one(MYSQL* conn, const char* sql, const char* expected)
+{
+    int r = 1;
+
+    if (conn != NULL)
+    {
+        const int n_attempts = 3;
+
+        for (int i = 0; i < n_attempts && r != 0; i++)
+        {
+            if (i > 0)
+            {
+                sleep(1);
+            }
+
+            if (mysql_query(conn, sql) != 0)
+            {
+                printf("Error: can't execute SQL-query: %s\n", sql);
+                printf("%s\n\n", mysql_error(conn));
+                break;
+            }
+            else
+            {
+                do
+                {
+                    MYSQL_RES* res = mysql_store_result(conn);
+
+                    if (res)
+                    {
+                        if (mysql_num_rows(res) == 1)
+                        {
+                            MYSQL_ROW row = mysql_fetch_row(res);
+
+                            if (row[0] != NULL)
+                            {
+                                if (strcmp(row[0], expected) == 0)
+                                {
+                                    r = 0;
+                                    printf("First field is '%s' as expected\n", row[0]);
+                                }
+                                else
+                                {
+                                    printf("First field is '%s', but expected '%s'\n", row[0], expected);
+                                }
+                            }
+                            else
+                            {
+                                printf("First field is NULL\n");
+                            }
+                        }
+                        else
+                        {
+                            printf("Number of rows is not 1, it is %llu\n", mysql_num_rows(res));
+                        }
+
+                        mysql_free_result(res);
+                    }
+                }
+                while (mysql_next_result(conn) == 0);
+            }
+        }
+    }
+    else
+    {
+        printf("Connection is broken\n");
+    }
+
+    return r;
+}
+
+int execute_query_affected_rows(MYSQL* conn, const char* sql, my_ulonglong* affected_rows)
+{
+    MYSQL_RES* res;
+    if (conn != NULL)
+    {
+        if (mysql_query(conn, sql) != 0)
+        {
+            printf("Error: can't execute SQL-query: %s\n", sql);
+            printf("%s\n\n", mysql_error(conn));
+            return 1;
+        }
+        else
+        {
+            do
+            {
+                *affected_rows = mysql_affected_rows(conn);
+                res = mysql_store_result(conn);
+                mysql_free_result(res);
+            }
+            while (mysql_next_result(conn) == 0);
+            return 0;
+        }
+    }
+    else
+    {
+        printf("Connection is broken\n");
+        return 1;
+    }
+}
+
+int execute_query_num_of_rows(MYSQL* conn,
+                              const char* sql,
+                              my_ulonglong* num_of_rows,
+                              unsigned long long* i)
+{
+    MYSQL_RES* res;
+    my_ulonglong N;
+
+
+    printf("%s\n", sql);
+    if (conn != NULL)
+    {
+        if (mysql_query(conn, sql) != 0)
+        {
+            printf("Error: can't execute SQL-query: %s\n", sql);
+            printf("%s\n\n", mysql_error(conn));
+            * i = 0;
+            return 1;
+        }
+        else
+        {
+            *i = 0;
+            do
+            {
+                res = mysql_store_result(conn);
+                if (res != NULL)
+                {
+                    N = mysql_num_rows(res);
+                    mysql_free_result(res);
+                }
+                else
+                {
+                    N = 0;
+                }
+                num_of_rows[*i] = N;
+                *i = *i + 1;
+            }
+            while (mysql_next_result(conn) == 0);
+            return 0;
+        }
+    }
+    else
+    {
+        printf("Connection is broken\n");
+        * i = 0;
+        return 1;
+    }
+}
+
+int execute_stmt_num_of_rows(MYSQL_STMT* stmt, my_ulonglong* num_of_rows, unsigned long long* i)
+{
+    my_ulonglong N;
+
+    /* This is debug hack; compatible only with t1 from t1_sql.h
+     *  my_ulonglong k;
+     *  MYSQL_BIND bind[2];
+     *  my_ulonglong x1;
+     *  my_ulonglong fl;
+     *
+     *  unsigned long length[2];
+     *  my_bool       is_null[2];
+     *  my_bool       error[2];
+     *
+     *  memset(bind, 0, sizeof(bind));
+     *  bind[0].buffer = &x1;
+     *  bind[0].buffer_type = MYSQL_TYPE_LONG;
+     *  bind[0].length = &length[0];
+     *  bind[0].is_null = &is_null[0];
+     *  bind[0].error = &error[0];
+     *
+     *  bind[1].buffer = &fl;
+     *  bind[1].buffer_type = MYSQL_TYPE_LONG;
+     *  bind[1].length = &length[0];
+     *  bind[1].is_null = &is_null[0];
+     *  bind[1].error = &error[0];
+     */
+
+    if (mysql_stmt_execute(stmt) != 0)
+    {
+        printf("Error: can't execute prepared statement\n");
+        printf("%s\n\n", mysql_stmt_error(stmt));
+        * i = 0;
+        return 1;
+    }
+    else
+    {
+        *i = 0;
+        do
+        {
+            mysql_stmt_store_result(stmt);
+            N = mysql_stmt_num_rows(stmt);
+            /* This is debug hack; compatible only with t1 from t1_sql.h
+             *  mysql_stmt_bind_result(stmt, bind);
+             *  for (k = 0; k < N; k++)
+             *  {
+             *   mysql_stmt_fetch(stmt);
+             *   printf("%04llu: x1 %llu, fl %llu\n", k, x1, fl);
+             *  }
+             */
+            num_of_rows[*i] = N;
+            *i = *i + 1;
+        }
+        while (mysql_stmt_next_result(stmt) == 0);
+        return 0;
+    }
+    return 1;
+}
+
+int execute_query_count_rows(MYSQL* conn, const char* sql)
+{
+    int rval = -1;
+
+    unsigned long long num_of_rows[1024];
+    unsigned long long total;
+
+    if (execute_query_num_of_rows(conn, sql, num_of_rows, &total) == 0)
+    {
+        rval = 0;
+
+        for (unsigned int i = 0; i < total && i < 1024; i++)
+        {
+            rval += num_of_rows[i];
+        }
+    }
+
+    return rval;
+}
+
+int get_conn_num(MYSQL* conn, std::string ip, std::string hostname, std::string db)
+{
+    MYSQL_RES* res;
+    MYSQL_ROW row;
+    unsigned long long int rows;
+    unsigned long long int i;
+    unsigned int conn_num = 0;
+    const char* hostname_internal;
+
+    if (ip == "127.0.0.1")
+    {
+        hostname_internal = "localhost";
+    }
+    else
+    {
+        hostname_internal = hostname.c_str();
+    }
+
+    if (conn != NULL)
+    {
+        if (mysql_query(conn, "show processlist;") != 0)
+        {
+            printf("Error: can't execute SQL-query: show processlist\n");
+            printf("%s\n\n", mysql_error(conn));
+            conn_num = 0;
+        }
+        else
+        {
+            res = mysql_store_result(conn);
+            if (res == NULL)
+            {
+                printf("Error: can't get the result description\n");
+                conn_num = -1;
+            }
+            else
+            {
+                mysql_num_fields(res);
+                rows = mysql_num_rows(res);
+                for (i = 0; i < rows; i++)
+                {
+                    row = mysql_fetch_row(res);
+                    if ((row[2] != NULL ) && (row[3] != NULL))
+                    {
+                        if ((strcmp(strtok(row[2], ":"), ip.c_str()) == 0) && strstr(row[3], db.c_str()))
+                        {
+                            conn_num++;
+                        }
+                        else if (strstr(row[2], hostname_internal) && strstr(row[3], db.c_str()))
+                        {
+                            conn_num++;
+                        }
+                    }
+                }
+            }
+            mysql_free_result(res);
+        }
+    }
+    if (ip == "127.0.0.1")
+    {
+        // one extra connection is visible in the process list
+        // output in case of local test
+        // (when MaxScale is on the same machine as backends)
+        conn_num--;
+    }
+    return conn_num;
+}
+
+int find_field(MYSQL* conn, const char* sql, const char* field_name, char* value)
+{
+    MYSQL_RES* res;
+    MYSQL_ROW row;
+    MYSQL_FIELD* field;
+    unsigned int ret = 1;
+    unsigned long long int filed_i = 0;
+    unsigned long long int i = 0;
+    if (conn != NULL)
+    {
+        if (mysql_query(conn, sql) != 0)
+        {
+            printf("Error: can't execute SQL-query: %s\n", sql);
+            printf("%s\n\n", mysql_error(conn));
+        }
+        else
+        {
+            res = mysql_store_result(conn);
+            if (res == NULL)
+            {
+                printf("Error: can't get the result description\n");
+            }
+            else
+            {
+                mysql_num_fields(res);
+                while ((field = mysql_fetch_field(res)) && ret != 0)
+                {
+                    if (strstr(field->name, field_name) != NULL)
+                    {
+                        filed_i = i;
+                        ret = 0;
+                    }
+                    i++;
+                }
+                if (mysql_num_rows(res) > 0)
+                {
+                    row = mysql_fetch_row(res);
+                    sprintf(value, "%s", row[filed_i]);
+                }
+                else
+                {
+                    sprintf(value, "%s", "");
+                    ret = 1;
+                }
+            }
+            mysql_free_result(res);
+            do
+            {
+                res = mysql_store_result(conn);
+                mysql_free_result(res);
+            }
+            while (mysql_next_result(conn) == 0);
+        }
+    }
+    return ret;
+}
+
+Result get_result(MYSQL* conn, std::string sql)
+{
+    Result rval;
+    MYSQL_RES* res;
+
+    if (mysql_query(conn, sql.c_str()) == 0 && (res = mysql_store_result(conn)))
+    {
+        MYSQL_ROW row = mysql_fetch_row(res);
+
+        while (row)
+        {
+            std::vector<std::string> tmp;
+            int n = mysql_num_fields(res);
+            for (int i = 0; i < n; ++i)
+            {
+                tmp.push_back(row[i] ? row[i] : "");
+            }
+            rval.push_back(tmp);
+
+            row = mysql_fetch_row(res);
+        }
+        mysql_free_result(res);
+    }
+    else
+    {
+        printf("Error: Query failed: %s\n", mysql_error(conn));
+    }
+
+    return rval;
+}
+
+Row get_row(MYSQL* conn, std::string sql)
+{
+    Result res = get_result(conn, sql);
+    return res.empty() ? Row {} :
+           res[0];
+}
+
+int get_int_version(std::string version)
+{
+    std::istringstream str(version);
+    int major = 0;
+    int minor = 0;
+    int patch = 0;
+    char dot;
+
+    str >> major >> dot >> minor >> dot >> patch;
+    return major * 10000 + minor * 100 + patch;
+}

maxscale-system-test/maxtest/src/maxadmin_operations.cpp

@@ -0,0 +1,289 @@
+/*
+ * This file is distributed as part of MaxScale.  It is free
+ * software: you can redistribute it and/or modify it under the terms of the
+ * GNU General Public License as published by the Free Software Foundation,
+ * version 2.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 51
+ * Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Copyright MariaDB Corporation Ab 2014
+ */
+
+#include "maxadmin_operations.h"
+
+int connectMaxScale(char* hostname, char* port)
+{
+    struct sockaddr_in addr;
+    int so;
+    int keepalive = 1;
+
+    if ((so = socket(AF_INET, SOCK_STREAM, 0)) < 0)
+    {
+        fprintf(stderr,
+                "Unable to create socket: %s\n",
+                strerror(errno));
+        return -1;
+    }
+    memset(&addr, 0, sizeof addr);
+    addr.sin_family = AF_INET;
+    setipaddress(&addr.sin_addr, hostname);
+    addr.sin_port = htons(atoi(port));
+    if (connect(so, (struct sockaddr*)&addr, sizeof(addr)) < 0)
+    {
+        fprintf(stderr,
+                "Unable to connect to MaxScale at %s, %s: %s\n",
+                hostname,
+                port,
+                strerror(errno));
+        close(so);
+        return -1;
+    }
+    if (setsockopt(so,
+                   SOL_SOCKET,
+                   SO_KEEPALIVE,
+                   &keepalive,
+                   sizeof(keepalive )))
+    {
+        perror("setsockopt");
+    }
+
+    return so;
+}
+
+
+int setipaddress(struct in_addr* a, char* p)
+{
+#ifdef __USE_POSIX
+    struct addrinfo* ai = NULL, hint;
+    int rc;
+    struct sockaddr_in* res_addr;
+    memset(&hint, 0, sizeof(hint));
+
+    hint.ai_socktype = SOCK_STREAM;
+    hint.ai_flags = AI_CANONNAME;
+    hint.ai_family = AF_INET;
+
+    if ((rc = getaddrinfo(p, NULL, &hint, &ai)) != 0)
+    {
+        return 0;
+    }
+
+    /* take the first one */
+    if (ai != NULL)
+    {
+        res_addr = (struct sockaddr_in*)(ai->ai_addr);
+        memcpy(a, &res_addr->sin_addr, sizeof(struct in_addr));
+
+        freeaddrinfo(ai);
+
+        return 1;
+    }
+#else
+    struct hostent* h;
+
+    spinlock_acquire(&tmplock);
+    h = gethostbyname(p);
+    spinlock_release(&tmplock);
+
+    if (h == NULL)
+    {
+        if ((a->s_addr = inet_addr(p)) == -1)
+        {
+            return 0;
+        }
+    }
+    else
+    {
+        /* take the first one */
+        memcpy(a, h->h_addr, h->h_length);
+
+        return 1;
+    }
+#endif
+    return 0;
+}
+
+int authMaxScale(int so, char* user, char* password)
+{
+    char buf[20];
+
+    if (read(so, buf, 4) != 4)
+    {
+        return 0;
+    }
+    int len;
+    len = strlen(user);
+    if (write(so, user, len) != len)
+    {
+        return 0;
+    }
+    if (read(so, buf, 8) != 8)
+    {
+        return 0;
+    }
+    len = strlen(password);
+    if (write(so, password, len) != len)
+    {
+        return 0;
+    }
+    if (read(so, buf, 6) != 6)
+    {
+        return 0;
+    }
+
+    return strncmp(buf, "FAILED", 6);
+}
+
+int sendCommand(int so, char* cmd, char* buf)
+{
+    char buf1[80];
+    int i, j, newline = 1;
+    int k = 0;
+
+    if (write(so, cmd, strlen(cmd)) == -1)
+    {
+        return 0;
+    }
+    while (1)
+    {
+        if ((i = read(so, buf1, 80)) <= 0)
+        {
+            return 0;
+        }
+        for (j = 0; j < i; j++)
+        {
+            if (newline == 1 && buf1[j] == 'O')
+            {
+                newline = 2;
+            }
+            else if (newline == 2 && buf1[j] == 'K' && j == i - 1)
+            {
+                return 1;
+            }
+            else if (newline == 2)
+            {
+                buf[k] = 'O';
+                k++;
+                buf[k] = buf1[j];
+                k++;
+                newline = 0;
+            }
+            else if (buf1[j] == '\n' || buf1[j] == '\r')
+            {
+                buf[k] = buf1[j];
+                k++;
+                newline = 1;
+            }
+            else
+            {
+                buf[k] = buf1[j];
+                k++;
+                newline = 0;
+            }
+        }
+    }
+    return 1;
+}
+
+int get_maxadmin_param_tcp(char* hostname, char* user, char* password, char* cmd, char* param, char* result)
+{
+
+    char buf[10240];
+    char* port = (char*) "6603";
+    int so;
+
+    if ((so = connectMaxScale(hostname, port)) == -1)
+    {
+        return 1;
+    }
+    if (!authMaxScale(so, user, password))
+    {
+        fprintf(stderr,
+                "Failed to connect to MaxScale. "
+                "Incorrect username or password.\n");
+        close(so);
+        return 1;
+    }
+
+    sendCommand(so, cmd, buf);
+
+    // printf("%s\n", buf);
+
+    char* x = strstr(buf, param);
+    if (x == NULL)
+    {
+        return 1;
+    }
+    // char f_field[100];
+    int param_len = strlen(param);
+    int cnt = 0;
+    while (x[cnt + param_len] != '\n')
+    {
+        result[cnt] = x[cnt + param_len];
+        cnt++;
+    }
+    result[cnt] = '\0';
+    // sprintf(f_field, "%s %%s", param);
+    // sscanf(x, f_field, result);
+    close(so);
+    return 0;
+}
+
+int execute_maxadmin_command_tcp(char* hostname, char* user, char* password, char* cmd)
+{
+
+    char buf[10240];
+    char* port = (char*) "6603";
+    int so;
+
+    if ((so = connectMaxScale(hostname, port)) == -1)
+    {
+        return 1;
+    }
+    if (!authMaxScale(so, user, password))
+    {
+        fprintf(stderr,
+                "Failed to connect to MaxScale. "
+                "Incorrect username or password.\n");
+        close(so);
+        return 1;
+    }
+
+    sendCommand(so, cmd, buf);
+
+    close(so);
+    return 0;
+}
+
+int execute_maxadmin_command_print_tcp(char* hostname, char* user, char* password, char* cmd)
+{
+
+    char buf[10240];
+    char* port = (char*) "6603";
+    int so;
+
+    if ((so = connectMaxScale(hostname, port)) == -1)
+    {
+        return 1;
+    }
+    if (!authMaxScale(so, user, password))
+    {
+        fprintf(stderr,
+                "Failed to connect to MaxScale. "
+                "Incorrect username or password.\n");
+        close(so);
+        return 1;
+    }
+
+    sendCommand(so, cmd, buf);
+    printf("%s\n", buf);
+    close(so);
+    return 0;
+}

maxscale-system-test/maxtest/src/maxinfo_func.cpp

@@ -0,0 +1,300 @@
+#include <iostream>
+#include <unistd.h>
+#include "testconnections.h"
+
+#include <stdio.h>
+#include <sys/socket.h>
+#include <arpa/inet.h>
+#include <stdlib.h>
+#include <netdb.h>
+#include <string.h>
+
+#include <openssl/sha.h>
+#include "maxinfo_func.h"
+#include <sys/epoll.h>
+#include <jansson.h>
+#include <fcntl.h>
+
+using namespace std;
+
+#define PORT      8080
+#define USERAGENT "HTMLGET 1.1"
+
+int create_tcp_socket()
+{
+    int sock;
+    if ((sock = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0)
+    {
+        perror("Can't create TCP socket");
+        return 0;
+    }
+    return sock;
+}
+
+char* get_ip(char* host)
+{
+    struct hostent* hent;
+    int iplen = 16;     // XXX.XXX.XXX.XXX
+    char* ip = (char*)malloc(iplen + 1);
+    memset(ip, 0, iplen + 1);
+    if ((hent = gethostbyname(host)) == NULL)
+    {
+        herror("Can't get IP");
+        return NULL;
+    }
+    if (inet_ntop(AF_INET, (void*)hent->h_addr_list[0], ip, iplen) == NULL)
+    {
+        perror("Can't resolve host");
+        return NULL;
+    }
+    return ip;
+}
+
+char* build_get_query(char* host, const char* page)
+{
+    char* query;
+    const char* getpage = page;
+    char* tpl = (char*) "GET /%s HTTP/1.1\r\nHost: %s\r\nUser-Agent: %s\r\n\r\n";
+    if (getpage[0] == '/')
+    {
+        getpage = getpage + 1;
+        fprintf(stderr, "Removing leading \"/\", converting %s to %s\n", page, getpage);
+    }
+    // -5 is to consider the %s %s %s in tpl and the ending \0
+    query = (char*)malloc(strlen(host) + strlen(getpage) + strlen(USERAGENT) + strlen(tpl) - 5);
+    sprintf(query, tpl, getpage, host, USERAGENT);
+    return query;
+}
+
+char* get_maxinfo(const char* page, TestConnections* Test)
+{
+    struct sockaddr_in* remote;
+    int sock;
+    int tmpres;
+    char* ip;
+    char* get;
+    char buf[BUFSIZ + 1];
+
+    sock = create_tcp_socket();
+    ip = get_ip(Test->maxscales->IP[0]);
+    if (ip == NULL)
+    {
+        Test->add_result(1, "Can't get IP\n");
+        return NULL;
+    }
+    remote = (struct sockaddr_in*)malloc(sizeof(struct sockaddr_in*));
+    remote->sin_family = AF_INET;
+    tmpres = inet_pton(AF_INET, ip, (void*)(&(remote->sin_addr.s_addr)));
+    if (tmpres < 0)
+    {
+        Test->add_result(1, "Can't set remote->sin_addr.s_addr\n");
+        return NULL;
+    }
+    else if (tmpres == 0)
+    {
+        Test->add_result(1, "%s is not a valid IP address\n", ip);
+        return NULL;
+    }
+    remote->sin_port = htons(PORT);
+
+    if (connect(sock, (struct sockaddr*)remote, sizeof(struct sockaddr)) < 0)
+    {
+        Test->add_result(1, "Could not connect\n");
+        return NULL;
+    }
+    get = build_get_query(Test->maxscales->IP[0], page);
+    // Test->tprintf("Query is:\n<<START>>\n%s<<END>>\n", get);
+
+    // Send the query to the server
+    size_t sent = 0;
+    while (sent < strlen(get))
+    {
+        tmpres = send(sock, get + sent, strlen(get) - sent, 0);
+        if (tmpres == -1)
+        {
+            Test->add_result(1, "Can't send query\n");
+            return NULL;
+        }
+        sent += tmpres;
+    }
+    // now it is time to receive the page
+    memset(buf, 0, sizeof(buf));
+
+    char* result = (char*)calloc(BUFSIZ, sizeof(char));
+    size_t rsize = sizeof(buf);
+    while ((tmpres = recv(sock, buf, BUFSIZ, MSG_WAITALL)) > 0)
+    {
+        result = (char*)realloc(result, tmpres + rsize);
+        rsize += tmpres;
+        strcat(result, buf);
+        memset(buf, 0, tmpres);
+    }
+    if (tmpres < 0)
+    {
+        Test->add_result(1, "Error receiving data\n");
+        return NULL;
+    }
+
+    free(get);
+    free(remote);
+    free(ip);
+    close(sock);
+
+    char* content = strstr(result, "[");
+    if (content == NULL)
+    {
+        Test->add_result(1, "Content not found\n");
+        free(result);
+        return NULL;
+    }
+
+    char* ret_content = (char*) calloc(strlen(content) + 1, sizeof(char));
+    mempcpy(ret_content, content, strlen(content));
+    free(result);
+
+    return ret_content;
+    // return(result);
+}
+
+char* read_sc(int sock)
+{
+    char buf[BUFSIZ + 1];
+    int tmpres;
+    memset(buf, 0, sizeof(buf));
+
+    char* result = (char*)calloc(BUFSIZ, sizeof(char));
+    size_t rsize = sizeof(buf);
+    while ((tmpres = recv(sock, buf, BUFSIZ, 0)) > 0)
+    {
+        result = (char*)realloc(result, tmpres + rsize);
+        rsize += tmpres;
+        // printf("%s", buf);
+        strcat(result, buf);
+        memset(buf, 0, tmpres);
+    }
+    return result;
+}
+int send_so(int sock, char* data)
+{
+    int tmpres;
+    size_t sent = 0;
+    while (sent < strlen(data))
+    {
+        tmpres = send(sock, data + sent, strlen(data) - sent, 0);
+        if (tmpres == -1)
+        {
+            return -1;
+        }
+        sent += tmpres;
+    }
+    return 0;
+}
+
+static char* bin2hex(const unsigned char* old, const size_t oldlen)
+{
+    char* result = (char*) malloc(oldlen * 2 + 1);
+    size_t i, j;
+
+    for (i = j = 0; i < oldlen; i++)
+    {
+        result[j++] = hexconvtab[old[i] >> 4];
+        result[j++] = hexconvtab[old[i] & 15];
+    }
+    result[j] = '\0';
+    return result;
+}
+
+char* cdc_auth_srt(char* user, char* password)
+{
+    unsigned char sha1pass[20];
+    char* str;
+    str = (char*) malloc(42 + strlen(user) * 2);
+
+    unsigned char* password_u;
+    unsigned char* user_u;
+    password_u = (unsigned char*) malloc(strlen(password));
+    user_u = (unsigned char*) malloc(strlen(user));
+    memcpy((void*)password_u, (void*)password, strlen(password));
+    memcpy((void*)user_u, (void*)user, strlen(user));
+
+    SHA1(password_u, strlen(password), sha1pass);
+
+    // char * sha1pass_hex = (char *) "454ac34c2999aacfebc6bf5fe9fa1db9b596f625";
+
+    char* sha1pass_hex = bin2hex(sha1pass, 20);
+    printf("password %s, len %lu, password sha1: %s\n", password, strlen(password), sha1pass_hex);
+
+
+    char* user_hex = bin2hex(user_u, strlen(user));
+    char* clmn_hex = bin2hex((unsigned char*)":", 1);
+
+    sprintf(str, "%s%s%s", user_hex, clmn_hex, sha1pass_hex);
+
+    free(clmn_hex);
+    free(user_hex);
+    free(sha1pass_hex);
+    free(user_u);
+    free(password_u);
+
+    printf("%s\n", str);
+    return str;
+}
+
+int setnonblocking(int sock)
+{
+    int opts;
+    opts = fcntl(sock, F_GETFL);
+    if (opts < 0)
+    {
+        return -1;
+    }
+    opts = (opts | O_NONBLOCK);
+    if (fcntl(sock, F_SETFL, opts) < 0)
+    {
+        return -1;
+    }
+    return 0;
+}
+
+
+int get_x_fl_from_json(char* line, long long int* x1, long long int* fl)
+{
+    json_t* root;
+    json_error_t error;
+
+    root = json_loads(line, 0, &error);
+    if (!root)
+    {
+        fprintf(stderr, "error: on line %d: %s\n", error.line, error.text);
+        return 1;
+    }
+
+    json_t* x_json = json_object_get(root, "x1");
+    if (x_json == NULL)
+    {
+        return 1;
+    }
+    if (!json_is_integer(x_json))
+    {
+        printf("x1 is not int, type is %d\n", json_typeof(x_json));
+        return 1;
+    }
+
+    *x1 = json_integer_value(x_json);
+    json_t* fl_json = json_object_get(root, "fl");
+    if (fl_json == NULL)
+    {
+        return 1;
+    }
+    if (!json_is_integer(fl_json))
+    {
+        printf("fl is not int\n");
+        return 1;
+    }
+
+    *fl = json_integer_value(fl_json);
+    json_decref(x_json);
+    json_decref(fl_json);
+    json_decref(root);
+    return 0;
+}

maxscale-system-test/maxtest/src/maxrest.cc

@@ -0,0 +1,204 @@
+/*
+ * Copyright (c) 2016 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2024-02-10
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#include "maxrest.hh"
+
+using namespace std;
+
+MaxRest::Server::Server(const MaxRest& maxrest, json_t* pObject)
+    : name       (maxrest.get<string> (pObject, "id", Presence::MANDATORY))
+    , address    (maxrest.get<string> (pObject, "attributes/parameters/address"))
+    , port       (maxrest.get<int64_t>(pObject, "attributes/parameters/port"))
+    , connections(maxrest.get<int64_t>(pObject, "attributes/statistics/connections"))
+    , state      (maxrest.get<string> (pObject, "attributes/state"))
+{
+}
+
+MaxRest::MaxRest(TestConnections* pTest)
+    : m_test(*pTest)
+{
+}
+
+unique_ptr<json_t> MaxRest::v1_servers(const string& id) const
+{
+    string path("servers");
+    path += "/";
+    path += id;
+
+    return curl_get(path);
+}
+
+unique_ptr<json_t> MaxRest::v1_servers() const
+{
+    return curl_get("servers");
+}
+
+void MaxRest::v1_maxscale_modules(const string& module,
+                                  const string& command,
+                                  const string& instance,
+                                  const std::vector<string>& params) const
+{
+    string path("maxscale/modules");
+
+    path += "/";
+    path += module;
+    path += "/";
+    path += command;
+    path += "?";
+    path += instance;
+
+    if (!params.empty())
+    {
+        for (const auto& param : params)
+        {
+            path += "\\&";
+            path += param;
+        }
+    }
+
+    curl_post(path);
+}
+
+MaxRest::Server MaxRest::show_server(const std::string& id) const
+{
+    unique_ptr<json_t> sObject = v1_servers(id);
+    json_t* pData = get_object(sObject.get(), "data", Presence::MANDATORY);
+    return Server(*this, pData);
+}
+
+vector<MaxRest::Server> MaxRest::list_servers() const
+{
+    return get_array<Server>(v1_servers().get(), "data", Presence::MANDATORY);
+}
+
+json_t* MaxRest::get_object(json_t* pObject, const string& key, Presence presence) const
+{
+    json_t* pValue = json_object_get(pObject, key.c_str());
+
+    if (!pValue && (presence == Presence::MANDATORY))
+    {
+        raise("Mandatory key '" + key + "' not present.");
+    }
+
+    return pValue;
+}
+
+json_t* MaxRest::get_leaf_object(json_t* pObject, const string& key, Presence presence) const
+{
+    auto i = key.find("/");
+
+    if (i == string::npos)
+    {
+        pObject = get_object(pObject, key, presence);
+    }
+    else
+    {
+        string head = key.substr(0, i);
+        string tail = key.substr(i + 1);
+
+        pObject = get_object(pObject, head, Presence::MANDATORY);
+        pObject = get_leaf_object(pObject, tail, presence);
+    }
+
+    return pObject;
+}
+
+unique_ptr<json_t> MaxRest::parse(const string& json) const
+{
+    json_error_t error;
+    unique_ptr<json_t> sRoot(json_loads(json.c_str(), 0, &error));
+
+    if (!sRoot)
+    {
+        raise("JSON parsing failed: " + string(error.text));
+    }
+
+    return sRoot;
+}
+
+unique_ptr<json_t> MaxRest::curl_get(const string& path) const
+{
+    return curl(GET, path);
+}
+
+unique_ptr<json_t> MaxRest::curl_post(const string& path) const
+{
+    return curl(POST, path);
+}
+
+unique_ptr<json_t> MaxRest::curl(Command command, const string& path) const
+{
+    string url = "http://127.0.0.1:8989/v1/" + path;
+    string curl_command = "curl -u admin:mariadb ";
+
+    switch (command)
+    {
+    case GET:
+        curl_command += "-X GET ";
+        break;
+
+    case POST:
+        curl_command += "-X POST ";
+        break;
+    }
+
+    curl_command += url;
+
+    auto result = m_test.maxscales->ssh_output(curl_command.c_str(), 0, false);
+
+    if (result.first != 0)
+    {
+        raise("Invocation of curl failed: " + to_string(result.first));
+    }
+
+    unique_ptr<json_t> sRv;
+
+    if (!result.second.empty())
+    {
+        sRv = parse(result.second);
+    }
+
+    return sRv;
+}
+
+void MaxRest::raise(const std::string& message) const
+{
+    ++m_test.global_result;
+    throw runtime_error(message);
+}
+
+template<>
+string MaxRest::get<string>(json_t* pObject, const string& key, Presence presence) const
+{
+    json_t* pValue = get_leaf_object(pObject, key, presence);
+
+    if (pValue && !json_is_string(pValue))
+    {
+        raise("Key '" + key + "' is present, but value is not a string.");
+    }
+
+    return pValue ? json_string_value(pValue) : "";
+}
+
+template<>
+int64_t MaxRest::get<int64_t>(json_t* pObject, const string& key, Presence presence) const
+{
+    json_t* pValue = get_leaf_object(pObject, key, presence);
+
+    if (pValue && !json_is_integer(pValue))
+    {
+        raise("Key '" + key + "' is present, but value is not an integer.");
+    }
+
+    return pValue ? json_integer_value(pValue) : 0;
+}

maxscale-system-test/maxtest/src/maxscales.cpp

@@ -0,0 +1,497 @@
+#include "maxscales.h"
+#include <sstream>
+#include <unordered_map>
+#include <string>
+#include "envv.h"
+
+Maxscales::Maxscales(const char *pref,
+                     const char *test_cwd,
+                     bool verbose,
+                     const std::string& network_config)
+    : Nodes(pref, network_config, verbose)
+    , valgring_log_num(0)
+{
+    strcpy(this->test_dir, test_cwd);
+}
+
+bool Maxscales::setup()
+{
+    read_env(); // Sets e.g. use_valgrind.
+    if (this->use_valgrind)
+    {
+        for (int i = 0; i < N; i++)
+        {
+            ssh_node_f(i, true, "yum install -y valgrind gdb 2>&1");
+            ssh_node_f(i, true, "apt install -y --force-yes valgrind gdb 2>&1");
+            ssh_node_f(i, true, "zypper -n install valgrind gdb 2>&1");
+            ssh_node_f(i, true, "rm -rf /var/cache/maxscale/maxscale.lock");
+        }
+    }
+    return true;
+}
+
+int Maxscales::read_env()
+{
+    char env_name[64];
+
+    read_basic_env();
+
+    if ((N > 0) && (N < 255))
+    {
+        for (int i = 0; i < N; i++)
+        {
+            sprintf(env_name, "%s_%03d_cnf", prefix, i);
+            maxscale_cnf[i] = readenv(env_name, DEFAULT_MAXSCALE_CNF);
+
+            sprintf(env_name, "%s_%03d_log_dir", prefix, i);
+            maxscale_log_dir[i] = readenv(env_name, DEFAULT_MAXSCALE_LOG_DIR);
+
+            sprintf(env_name, "%s_%03d_binlog_dir", prefix, i);
+            maxscale_binlog_dir[i] = readenv(env_name, DEFAULT_MAXSCALE_BINLOG_DIR);
+
+            sprintf(env_name, "%s_%03d_maxadmin_password", prefix, i);
+            maxadmin_password[i] = readenv(env_name, DEFAULT_MAXADMIN_PASSWORD);
+
+            rwsplit_port[i] = 4006;
+            readconn_master_port[i] = 4008;
+            readconn_slave_port[i] = 4009;
+            binlog_port[i] = 5306;
+
+            ports[i][0] = rwsplit_port[i];
+            ports[i][1] = readconn_master_port[i];
+            ports[i][2] = readconn_slave_port[i];
+
+            N_ports[0] = 3;
+        }
+    }
+
+    use_valgrind = readenv_bool("use_valgrind", false);
+    use_callgrind = readenv_bool("use_callgrind", false);
+    if (use_callgrind)
+    {
+        use_valgrind = true;
+    }
+
+    return 0;
+}
+
+int Maxscales::connect_rwsplit(int m, const std::string& db)
+{
+    if (use_ipv6)
+    {
+        conn_rwsplit[m] = open_conn_db(rwsplit_port[m],
+                                    IP6[m],
+                                    db,
+                                    user_name,
+                                    password,
+                                    ssl);
+    }
+    else
+    {
+        conn_rwsplit[m] = open_conn_db(rwsplit_port[m],
+                                    IP[m],
+                                    db,
+                                    user_name,
+                                    password,
+                                    ssl);
+    }
+    routers[m][0] = conn_rwsplit[m];
+
+    int rc = 0;
+    int my_errno = mysql_errno(conn_rwsplit[m]);
+
+    if (my_errno)
+    {
+        if (verbose)
+        {
+            printf("Failed to connect to readwritesplit: %d, %s\n", my_errno, mysql_error(conn_rwsplit[m]));
+        }
+        rc = my_errno;
+    }
+
+    return rc;
+}
+
+int Maxscales::connect_readconn_master(int m, const std::string& db)
+{
+    if (use_ipv6)
+    {
+        conn_master[m] = open_conn_db(readconn_master_port[m],
+                                   IP6[m],
+                                   db,
+                                   user_name,
+                                   password,
+                                   ssl);
+    }
+    else
+    {
+        conn_master[m] = open_conn_db(readconn_master_port[m],
+                                   IP[m],
+                                   db,
+                                   user_name,
+                                   password,
+                                   ssl);
+    }
+    routers[m][1] = conn_master[m];
+
+    int rc = 0;
+    int my_errno = mysql_errno(conn_master[m]);
+
+    if (my_errno)
+    {
+        if (verbose)
+        {
+            printf("Failed to connect to readwritesplit: %d, %s\n", my_errno, mysql_error(conn_master[m]));
+        }
+        rc = my_errno;
+    }
+
+    return rc;
+}
+
+int Maxscales::connect_readconn_slave(int m, const std::string& db)
+{
+    if (use_ipv6)
+    {
+        conn_slave[m] = open_conn_db(readconn_slave_port[m],
+                                  IP6[m],
+                                  db,
+                                  user_name,
+                                  password,
+                                  ssl);
+    }
+    else
+    {
+        conn_slave[m] = open_conn_db(readconn_slave_port[m],
+                                  IP[m],
+                                  db,
+                                  user_name,
+                                  password,
+                                  ssl);
+    }
+    routers[m][2] = conn_slave[m];
+
+    int rc = 0;
+    int my_errno = mysql_errno(conn_slave[m]);
+
+    if (my_errno)
+    {
+        if (verbose)
+        {
+            printf("Failed to connect to readwritesplit: %d, %s\n", my_errno, mysql_error(conn_slave[m]));
+        }
+        rc = my_errno;
+    }
+
+    return rc;
+}
+
+int Maxscales::connect_maxscale(int m, const std::string& db)
+{
+    return connect_rwsplit(m, db)
+           + connect_readconn_master(m, db)
+           + connect_readconn_slave(m, db);
+}
+
+int Maxscales::close_maxscale_connections(int m)
+{
+    mysql_close(conn_master[m]);
+    mysql_close(conn_slave[m]);
+    mysql_close(conn_rwsplit[m]);
+    return 0;
+}
+
+int Maxscales::restart_maxscale(int m)
+{
+    int res;
+    if (use_valgrind)
+    {
+        res = stop_maxscale(m);
+        res += start_maxscale(m);
+    }
+    else
+    {
+        res = ssh_node(m, "service maxscale restart", true);
+    }
+    fflush(stdout);
+    return res;
+}
+
+int Maxscales::start_maxscale(int m)
+{
+    int res;
+    if (use_valgrind)
+    {
+        if (use_callgrind)
+        {
+            res = ssh_node_f(m, false,
+                             "sudo --user=maxscale valgrind -d "
+                             "--log-file=/%s/valgrind%02d.log --trace-children=yes "
+                             " --tool=callgrind --callgrind-out-file=/%s/callgrind%02d.log "
+                             " /usr/bin/maxscale",
+                             maxscale_log_dir[m], valgring_log_num,
+                             maxscale_log_dir[m], valgring_log_num);
+        }
+        else
+        {
+            res = ssh_node_f(m, false,
+                             "sudo --user=maxscale valgrind --leak-check=full --show-leak-kinds=all "
+                             "--log-file=/%s/valgrind%02d.log --trace-children=yes "
+                             "--track-origins=yes /usr/bin/maxscale", maxscale_log_dir[m], valgring_log_num);
+        }
+        valgring_log_num++;
+    }
+    else
+    {
+        res =ssh_node(m, "service maxscale restart", true);
+    }
+    fflush(stdout);
+    return res;
+}
+
+int Maxscales::stop_maxscale(int m)
+{
+    int res;
+    if (use_valgrind)
+    {
+        res = ssh_node_f(m, true, "sudo kill $(pidof valgrind) 2>&1 > /dev/null");
+        if ((res != 0) || atoi(ssh_node_output(m, "pidof valgrind", true, &res)) > 0)
+        {
+            res = ssh_node_f(m, true, "sudo kill -9 $(pidof valgrind) 2>&1 > /dev/null");
+        }
+    }
+    else
+    {
+        res = ssh_node(m, "service maxscale stop", true);
+    }
+    fflush(stdout);
+    return res;
+}
+
+int Maxscales::execute_maxadmin_command(int m, const char* cmd)
+{
+    return ssh_node_f(m, true, "maxadmin %s", cmd);
+}
+int Maxscales::execute_maxadmin_command_print(int m, const char* cmd)
+{
+    int exit_code;
+    printf("%s\n", ssh_node_output_f(m, true, &exit_code, "maxadmin %s", cmd));
+    return exit_code;
+}
+
+int Maxscales::check_maxadmin_param(int m, const char* command, const char* param, const char* value)
+{
+    char result[1024];
+    int rval = 1;
+
+    if (get_maxadmin_param(m, (char*)command, (char*)param, (char*)result) == 0)
+    {
+        char* end = strchr(result, '\0') - 1;
+
+        while (isspace(*end))
+        {
+            *end-- = '\0';
+        }
+
+        char* start = result;
+
+        while (isspace(*start))
+        {
+            start++;
+        }
+
+        if (strcmp(start, value) == 0)
+        {
+            rval = 0;
+        }
+        else
+        {
+            printf("Expected %s, got %s\n", value, start);
+        }
+    }
+
+    return rval;
+}
+
+int Maxscales::get_maxadmin_param(int m, const char* command, const char* param, char* result)
+{
+    char* buf;
+    int exit_code;
+
+    buf = ssh_node_output_f(m, true, &exit_code, "maxadmin %s", command);
+
+    // printf("%s\n", buf);
+
+    char* x = strstr(buf, param);
+
+    if (x == NULL)
+    {
+        return 1;
+    }
+
+    x += strlen(param);
+
+    // Skip any trailing parts of the parameter name
+    while (!isspace(*x))
+    {
+        x++;
+    }
+
+    // Trim leading whitespace
+    while (!isspace(*x))
+    {
+        x++;
+    }
+
+    char* end = strchr(x, '\n');
+
+    // Trim trailing whitespace
+    while (isspace(*end))
+    {
+        *end-- = '\0';
+    }
+
+    strcpy(result, x);
+
+    return exit_code;
+}
+
+int Maxscales::get_backend_servers_num(int m, const char* service)
+{
+    char* buf;
+    int exit_code;
+    int i = 0;
+
+    buf = ssh_node_output_f(m, true, &exit_code, "maxadmin show service %s | grep Name: | grep Protocol: | wc -l", service);
+    if (buf && !exit_code)
+    {
+        sscanf(buf, "%d", &i);
+    }
+    return i;
+}
+
+
+
+long unsigned Maxscales::get_maxscale_memsize(int m)
+{
+    int exit_code;
+    char* ps_out = ssh_node_output(m, "ps -e -o pid,vsz,comm= | grep maxscale", false, &exit_code);
+    long unsigned mem = 0;
+    pid_t pid;
+    sscanf(ps_out, "%d %lu", &pid, &mem);
+    return mem;
+}
+
+
+int Maxscales::find_master_maxadmin(Mariadb_nodes* nodes, int m)
+{
+    bool found = false;
+    int master = -1;
+
+    for (int i = 0; i < nodes->N; i++)
+    {
+        char show_server[256];
+        char res[256];
+        sprintf(show_server, "show server server%d", i + 1);
+        get_maxadmin_param(m, show_server, (char*) "Status", res);
+
+        if (strstr(res, "Master"))
+        {
+            if (found)
+            {
+                master = -1;
+            }
+            else
+            {
+                master = i;
+                found = true;
+            }
+        }
+    }
+
+    return master;
+}
+
+int Maxscales::find_slave_maxadmin(Mariadb_nodes* nodes, int m)
+{
+    int slave = -1;
+
+    for (int i = 0; i < nodes->N; i++)
+    {
+        char show_server[256];
+        char res[256];
+        sprintf(show_server, "show server server%d", i + 1);
+        get_maxadmin_param(m, show_server, (char*) "Status", res);
+
+        if (strstr(res, "Slave"))
+        {
+            slave = i;
+        }
+    }
+
+    return slave;
+}
+
+StringSet Maxscales::get_server_status(const char* name, int m)
+{
+    std::set<std::string> rval;
+    int exit_code;
+    char* res = ssh_node_output_f(m, true, &exit_code, "maxadmin list servers|grep \'%s\'", name);
+    char* pipe = strrchr(res, '|');
+
+    if (res && pipe)
+    {
+        pipe++;
+        char* tok = strtok(pipe, ",");
+
+        while (tok)
+        {
+            char* p = tok;
+            char* end = strchr(tok, '\n');
+            if (!end)
+            {
+                end = strchr(tok, '\0');
+            }
+
+            // Trim leading whitespace
+            while (p < end && isspace(*p))
+            {
+                p++;
+            }
+
+            // Trim trailing whitespace
+            while (end > tok && isspace(*end))
+            {
+                *end-- = '\0';
+            }
+
+            rval.insert(p);
+            tok = strtok(NULL, ",\n");
+        }
+
+        free(res);
+    }
+
+    return rval;
+}
+
+int Maxscales::port(enum service type, int m) const
+{
+    switch (type)
+    {
+    case RWSPLIT:
+        return rwsplit_port[m];
+
+    case READCONN_MASTER:
+        return readconn_master_port[m];
+
+    case READCONN_SLAVE:
+        return readconn_slave_port[m];
+    }
+    return -1;
+}
+
+void Maxscales::wait_for_monitor(int intervals, int m)
+{
+    ssh_node_f(m, false, "for ((i=0;i<%d;i++)); do maxctrl api get maxscale/debug/monitor_wait; done", intervals);
+}

maxscale-system-test/maxtest/src/nodes.cpp

@@ -0,0 +1,454 @@
+#include "nodes.h"
+#include <string>
+#include <cstring>
+#include <iostream>
+#include <future>
+#include <functional>
+#include <algorithm>
+#include <signal.h>
+
+#include "envv.h"
+
+Nodes::Nodes(const char* pref,
+             const std::string& network_config,
+             bool verbose)
+    : network_config(network_config)
+    , verbose(verbose)
+{
+    strcpy(this->prefix, pref);
+}
+
+bool Nodes::check_node_ssh(int node)
+{
+    bool res = true;
+
+    if (ssh_node(node, "ls > /dev/null", false) != 0)
+    {
+        std::cout << "Node " << node << " is not available" << std::endl;
+        res = false;
+    }
+
+    return res;
+}
+
+bool Nodes::check_nodes()
+{
+    std::vector<std::future<bool>> f;
+
+    for (int i = 0; i < N; i++)
+    {
+        f.push_back(std::async(std::launch::async, &Nodes::check_node_ssh, this, i));
+    }
+
+    return std::all_of(f.begin(), f.end(), std::mem_fn(&std::future<bool>::get));
+}
+
+void Nodes::generate_ssh_cmd(char* cmd, int node, const char* ssh, bool sudo)
+{
+    if (strcmp(IP[node], "127.0.0.1") == 0)
+    {
+        if (sudo)
+        {
+            sprintf(cmd,
+                    "%s %s",
+                    access_sudo[node],
+                    ssh);
+        }
+        else
+        {
+            sprintf(cmd, "%s", ssh);
+        }
+    }
+    else
+    {
+
+        if (sudo)
+        {
+            sprintf(cmd,
+                    "ssh -i %s -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no  -o LogLevel=quiet %s@%s '%s %s\'",
+                    sshkey[node],
+                    access_user[node],
+                    IP[node],
+                    access_sudo[node],
+                    ssh);
+        }
+        else
+        {
+            sprintf(cmd,
+                    "ssh -i %s -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no  -o LogLevel=quiet %s@%s '%s'",
+                    sshkey[node],
+                    access_user[node],
+                    IP[node],
+                    ssh);
+        }
+    }
+}
+
+
+char* Nodes::ssh_node_output_f(int node, bool sudo, int* exit_code, const char* format, ...)
+{
+    va_list valist;
+
+    va_start(valist, format);
+    int message_len = vsnprintf(NULL, 0, format, valist);
+    va_end(valist);
+
+    if (message_len < 0)
+    {
+        return NULL;
+    }
+
+    char* sys = (char*)malloc(message_len + 1);
+
+    va_start(valist, format);
+    vsnprintf(sys, message_len + 1, format, valist);
+    va_end(valist);
+
+    char* result = ssh_node_output(node, sys, sudo, exit_code);
+    free(sys);
+
+    return result;
+}
+
+
+char* Nodes::ssh_node_output(int node, const char* ssh, bool sudo, int* exit_code)
+{
+    char* cmd = (char*)malloc(strlen(ssh) + 1024);
+
+    generate_ssh_cmd(cmd, node, ssh, sudo);
+
+    FILE* output = popen(cmd, "r");
+
+    if (output == NULL)
+    {
+        printf("Error opening ssh %s\n", strerror(errno));
+        return NULL;
+    }
+    char buffer[1024];
+    size_t rsize = sizeof(buffer);
+    char* result = (char*)calloc(rsize, sizeof(char));
+
+    while (fgets(buffer, sizeof(buffer), output))
+    {
+        result = (char*)realloc(result, sizeof(buffer) + rsize);
+        rsize += sizeof(buffer);
+        strcat(result, buffer);
+    }
+
+    free(cmd);
+    int code = pclose(output);
+    if (WIFEXITED(code))
+    {
+        * exit_code = WEXITSTATUS(code);
+    }
+    else
+    {
+        * exit_code = 256;
+    }
+    return result;
+}
+
+
+int Nodes::ssh_node(int node, const char* ssh, bool sudo)
+{
+    char* cmd = (char*)malloc(strlen(ssh) + 1024);
+
+    if (strcmp(IP[node], "127.0.0.1") == 0)
+    {
+        printf("starting bash\n");
+        sprintf(cmd, "bash");
+    }
+    else
+    {
+        sprintf(cmd,
+                "ssh -i %s -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o LogLevel=quiet %s@%s%s",
+                sshkey[node],
+                access_user[node],
+                IP[node],
+                verbose ? "" :  " > /dev/null");
+    }
+
+    if (verbose)
+    {
+        std::cout << ssh << std::endl;
+    }
+
+    int rc = 1;
+    FILE* in = popen(cmd, "w");
+
+    if (in)
+    {
+        if (sudo)
+        {
+            fprintf(in, "sudo su -\n");
+            fprintf(in, "cd /home/%s\n", access_user[node]);
+        }
+
+        fprintf(in, "%s\n", ssh);
+        rc = pclose(in);
+    }
+
+
+    free(cmd);
+
+    if (WIFEXITED(rc))
+    {
+        return WEXITSTATUS(rc);
+    }
+    else if (WIFSIGNALED(rc) && WTERMSIG(rc) == SIGHUP)
+    {
+        // SIGHUP appears to happen for SSH connections
+        return 0;
+    }
+    else
+    {
+        std::cout << strerror(errno) << std::endl;
+        return 256;
+    }
+}
+
+int Nodes::ssh_node_f(int node, bool sudo, const char* format, ...)
+{
+    va_list valist;
+
+    va_start(valist, format);
+    int message_len = vsnprintf(NULL, 0, format, valist);
+    va_end(valist);
+
+    if (message_len < 0)
+    {
+        return -1;
+    }
+
+    char* sys = (char*)malloc(message_len + 1);
+
+    va_start(valist, format);
+    vsnprintf(sys, message_len + 1, format, valist);
+    va_end(valist);
+    int result = ssh_node(node, sys, sudo);
+    free(sys);
+    return result;
+}
+
+int Nodes::copy_to_node(int i, const char* src, const char* dest)
+{
+    if (i >= N)
+    {
+        return 1;
+    }
+    char sys[strlen(src) + strlen(dest) + 1024];
+
+    if (strcmp(IP[i], "127.0.0.1") == 0)
+    {
+        sprintf(sys,
+                "cp %s %s",
+                src,
+                dest);
+    }
+    else
+    {
+        sprintf(sys,
+                "scp -q -r -i %s -o UserKnownHostsFile=/dev/null "
+                "-o StrictHostKeyChecking=no -o LogLevel=quiet %s %s@%s:%s",
+                sshkey[i],
+                src,
+                access_user[i],
+                IP[i],
+                dest);
+    }
+    if (verbose)
+    {
+        printf("%s\n", sys);
+    }
+
+    return system(sys);
+}
+
+
+int Nodes::copy_to_node_legacy(const char* src, const char* dest, int i)
+{
+
+    return copy_to_node(i, src, dest);
+}
+
+int Nodes::copy_from_node(int i, const char* src, const char* dest)
+{
+    if (i >= N)
+    {
+        return 1;
+    }
+    char sys[strlen(src) + strlen(dest) + 1024];
+    if (strcmp(IP[i], "127.0.0.1") == 0)
+    {
+        sprintf(sys,
+                "cp %s %s",
+                src,
+                dest);
+    }
+    else
+    {
+        sprintf(sys,
+                "scp -q -r -i %s -o UserKnownHostsFile=/dev/null "
+                "-o StrictHostKeyChecking=no -o LogLevel=quiet %s@%s:%s %s",
+                sshkey[i],
+                access_user[i],
+                IP[i],
+                src,
+                dest);
+    }
+    if (verbose)
+    {
+        printf("%s\n", sys);
+    }
+
+    return system(sys);
+}
+
+int Nodes::copy_from_node_legacy(const char* src, const char* dest, int i)
+{
+    return copy_from_node(i, src, dest);
+}
+
+int Nodes::read_basic_env()
+{
+    char env_name[64];
+
+    sprintf(env_name, "%s_user", prefix);
+    user_name = readenv(env_name, "skysql");
+
+    sprintf(env_name, "%s_password", prefix);
+    password = readenv(env_name, "skysql");
+
+    N = get_N();
+
+    if ((N > 0) && (N < 255))
+    {
+        for (int i = 0; i < N; i++)
+        {
+            // reading IPs
+            sprintf(env_name, "%s_%03d_network", prefix, i);
+            IP[i] = strdup(get_nc_item(env_name).c_str());
+
+            // reading private IPs
+            sprintf(env_name, "%s_%03d_private_ip", prefix, i);
+            IP_private[i] = strdup(get_nc_item(env_name).c_str());
+            if (IP_private[i] == NULL)
+            {
+                IP_private[i] = IP[i];
+            }
+            setenv(env_name, IP_private[i], 1);
+
+            // reading IPv6
+            sprintf(env_name, "%s_%03d_network6", prefix, i);
+            IP6[i] = strdup(get_nc_item(env_name).c_str());
+            if (IP6[i] == NULL)
+            {
+                IP6[i] = IP[i];
+            }
+            setenv(env_name, IP6[i], 1);
+
+            //reading sshkey
+            sprintf(env_name, "%s_%03d_keyfile", prefix, i);
+            sshkey[i] = strdup(get_nc_item(env_name).c_str());
+
+
+            sprintf(env_name, "%s_%03d_whoami", prefix, i);
+            access_user[i] = strdup(get_nc_item(env_name).c_str());
+            if (access_user[i] == NULL)
+            {
+                access_user[i] = (char *) "vagrant";
+            }
+            setenv(env_name, access_user[i], 1);
+
+            sprintf(env_name, "%s_%03d_access_sudo", prefix, i);
+            access_sudo[i] = readenv(env_name, " sudo ");
+
+            if (strcmp(access_user[i], "root") == 0)
+            {
+                access_homedir[i] = (char *) "/root/";
+            }
+            else
+            {
+                access_homedir[i] = (char *) malloc(strlen(access_user[i]) + 9);
+                sprintf(access_homedir[i], "/home/%s/", access_user[i]);
+            }
+
+            sprintf(env_name, "%s_%03d_hostname", prefix, i);
+            hostname[i] = strdup(get_nc_item(env_name).c_str());
+            if ((hostname[i] == NULL) || (strcmp(hostname[i], "") == 0))
+            {
+                hostname[i] = IP_private[i];
+            }
+            setenv(env_name, hostname[i], 1);
+
+            sprintf(env_name, "%s_%03d_start_vm_command", prefix, i);
+            start_vm_command[i] = readenv(env_name, "curr_dir=`pwd`; cd %s/%s;vagrant resume %s_%03d ; cd $curr_dir",
+                                          getenv("MDBCI_VM_PATH"), getenv("name"), prefix, i);
+            setenv(env_name, start_vm_command[i], 1);
+
+            sprintf(env_name, "%s_%03d_stop_vm_command", prefix, i);
+            stop_vm_command[i] = readenv(env_name, "curr_dir=`pwd`; cd %s/%s;vagrant suspend %s_%03d ; cd $curr_dir",
+                                         getenv("MDBCI_VM_PATH"), getenv("name"), prefix, i);
+            setenv(env_name, stop_vm_command[i], 1);
+        }
+    }
+
+    return 0;
+}
+
+const char* Nodes::ip(int i) const
+{
+    return use_ipv6 ?  IP6[i] : IP[i];
+}
+
+std::string Nodes::get_nc_item(const char* item_name)
+{
+    size_t start = network_config.find(item_name);
+    if (start == std::string::npos)
+    {
+        return "";
+    }
+
+    size_t end = network_config.find("\n", start);
+    size_t equal = network_config.find("=", start);
+    if (end == std::string::npos)
+    {
+        end = network_config.length();
+    }
+    if (equal == std::string::npos)
+    {
+        return "";
+    }
+
+    std::string str = network_config.substr(equal + 1, end - equal - 1);
+    str.erase(remove(str.begin(), str.end(), ' '), str.end());
+
+    setenv(item_name, str.c_str(), 1);
+
+    return str;
+}
+
+int Nodes::get_N()
+{
+    int N = 0;
+    char item[strlen(prefix) + 13];
+    do
+    {
+        sprintf(item, "%s_%03d_network", prefix, N);
+        N++;
+    }
+    while (network_config.find(item) != std::string::npos);
+    sprintf(item, "%s_N", prefix);
+    setenv(item, std::to_string(N).c_str(), 1);
+    return N - 1 ;
+}
+
+int Nodes::start_vm(int node)
+{
+    return (system(start_vm_command[node]));
+}
+
+int Nodes::stop_vm(int node)
+{
+    return (system(stop_vm_command[node]));
+}

maxscale-system-test/maxtest/src/rds_vpc.cpp

@@ -0,0 +1,817 @@
+#include "execute_cmd.h"
+#include "rds_vpc.h"
+
+RDS::RDS(char* cluster)
+{
+    cluster_name_intern = cluster;
+    subnets_intern = NULL;
+    N_intern = 0;
+}
+
+const char* RDS::get_instance_name(json_t* instance)
+{
+    json_t* instance_name = json_object_get(instance, "DBInstanceIdentifier");
+    return json_string_value(instance_name);
+}
+
+json_t* RDS::get_cluster_descr(char* json)
+{
+    json_t* root;
+    json_error_t error;
+
+    root = json_loads(json, 0, &error);
+    if (!root)
+    {
+        fprintf(stderr, "error: on line %d: %s\n", error.line, error.text);
+        return NULL;
+    }
+
+    json_t* clusters = json_object_get(root, "DBClusters");
+    // cluster_intern =
+    return json_array_get(clusters, 0);
+}
+
+json_t* RDS::get_subnets_group_descr(char* json)
+{
+    json_t* root;
+    json_error_t error;
+
+    root = json_loads(json, 0, &error);
+    if (!root)
+    {
+        fprintf(stderr, "error: on line %d: %s\n", error.line, error.text);
+        return NULL;
+    }
+
+    json_t* subnets = json_object_get(root, "DBSubnetGroups");
+    return json_array_get(subnets, 0);
+}
+
+json_t* RDS::get_cluster_nodes()
+{
+    return get_cluster_nodes(cluster_intern);
+}
+
+json_t* RDS::get_cluster_nodes(json_t* cluster)
+{
+    json_t* members = json_object_get(cluster, "DBClusterMembers");
+    size_t members_N = json_array_size(members);
+    json_t* member;
+    json_t* node_names = json_array();
+
+    for (size_t i = 0; i < members_N; i++)
+    {
+        member = json_array_get(members, i);
+        json_array_append(node_names, json_string(get_instance_name(member)));
+    }
+    return node_names;
+}
+
+json_t* RDS::get_subnets()
+{
+    char cmd[1024];
+    char* result;
+    sprintf(cmd, "aws rds describe-db-subnet-groups --db-subnet-group-name %s", subnets_group_name_intern);
+    if (execute_cmd(cmd, &result) != 0)
+    {
+        return NULL;
+    }
+
+    json_t* subnets_group = get_subnets_group_descr(result);
+
+    json_t* members = json_object_get(subnets_group, "Subnets");
+    vpc_id_intern = json_string_value(json_object_get(subnets_group, "VpcId"));
+    size_t members_N = json_array_size(members);
+    json_t* member;
+    json_t* subnets_names = json_array();
+
+    for (size_t i = 0; i < members_N; i++)
+    {
+        member = json_array_get(members, i);
+        json_array_append(subnets_names, json_object_get(member, "SubnetIdentifier"));
+    }
+    subnets_intern = subnets_names;
+    return subnets_names;
+}
+
+const char* RDS::get_subnetgroup_name()
+{
+    if (cluster_intern != NULL)
+    {
+        subnets_group_name_intern = json_string_value(json_object_get(cluster_intern, "DBSubnetGroup"));
+    }
+    else
+    {
+        subnets_group_name_intern = cluster_name_intern;
+    }
+
+    return subnets_group_name_intern;
+}
+
+json_t* RDS::get_cluster()
+{
+    char cmd[1024];
+    char* result;
+    sprintf(cmd, "aws rds describe-db-clusters --db-cluster-identifier=%s", cluster_name_intern);
+    execute_cmd(cmd, &result);
+    return get_cluster_descr(result);
+}
+
+int RDS::destroy_nodes(json_t* node_names)
+{
+    size_t N = json_array_size(node_names);
+
+    char cmd[1024];
+    char* res;
+    json_t* node;
+    int err = 0;
+    for (size_t i = 0; i < N; i++)
+    {
+        node = json_array_get(node_names, i);
+        sprintf(cmd,
+                "aws rds delete-db-instance --skip-final-snapshot --db-instance-identifier=%s",
+                json_string_value(node));
+        printf("%s\n", cmd);
+        if (execute_cmd(cmd, &res) != 0)
+        {
+            err = -1;
+            fprintf(stderr, "error: can not delete node %s\n", json_string_value(node));
+        }
+    }
+    return err;
+}
+
+int RDS::destroy_subnets()
+{
+    size_t N = json_array_size(subnets_intern);
+
+    char cmd[1024];
+    char* res;
+    json_t* subnet;
+    int err = 0;
+    for (size_t i = 0; i < N; i++)
+    {
+        subnet = json_array_get(subnets_intern, i);
+        sprintf(cmd, "aws ec2 delete-subnet --subnet-id=%s", json_string_value(subnet));
+        printf("%s\n", cmd);
+        execute_cmd(cmd, &res);
+        if (execute_cmd(cmd, &res) != 0)
+        {
+            err = -1;
+            fprintf(stderr, "error: can not delete subnet %s\n", json_string_value(subnet));
+        }
+    }
+    return err;
+}
+
+int RDS::destroy_route_tables()
+{
+    json_t* root;
+    char cmd[1024];
+    char* json;
+    int res = 0;
+
+    sprintf(cmd, "aws ec2 describe-vpcs --vpc-ids=%s", vpc_id_intern);
+    if (execute_cmd(cmd, &json))
+    {
+        fprintf(stderr, "error: can not get internet gateways description\n");
+        return -1;
+    }
+
+    root = get_cluster_descr(json);
+    if (!root)
+    {
+        fprintf(stderr, "error: can not get cluster description\n");
+        return -1;
+    }
+
+    json_t* route_tables = json_object_get(root, "RouteTables");
+
+    size_t i;
+    json_t* route_table;
+    const char* rt_id;
+    const char* vpc_id;
+    json_array_foreach(route_tables, i, route_table)
+    {
+        rt_id = json_string_value(json_object_get(route_table, "RouteTableId"));
+        vpc_id = json_string_value(json_object_get(route_table, "VpcId"));
+        if (strcmp(vpc_id_intern, vpc_id) == 0)
+        {
+            sprintf(cmd, "aws ec2 delete-route-table --route-table-id %s", rt_id);
+            res += system(cmd);
+        }
+    }
+
+    return res;
+}
+
+int RDS::detach_and_destroy_gw()
+{
+    json_t* root;
+    json_error_t error;
+    char cmd[1024];
+    char* json;
+
+    sprintf(cmd,
+            "aws ec2 describe-internet-gateways --filters Name=attachment.vpc-id,Values=%s",
+            vpc_id_intern);
+    if (execute_cmd(cmd, &json))
+    {
+        fprintf(stderr, "error: can not get internet gateways description\n");
+        return -1;
+    }
+
+    root = json_loads(json, 0, &error);
+    if (!root)
+    {
+        fprintf(stderr, "error: on line %d: %s\n", error.line, error.text);
+        return -1;
+    }
+
+    json_t* gws = json_object_get(root, "InternetGateways");
+    if (gws == NULL)
+    {
+        fprintf(stderr, "error: can not parse internet gateways description\n");
+        return -1;
+    }
+    size_t i;
+    json_t* gw;
+    const char* gw_id;
+    json_array_foreach(gws, i, gw)
+    {
+        gw_id = json_string_value(json_object_get(gw, "InternetGatewayId"));
+        sprintf(cmd,
+                "aws ec2 detach-internet-gateway --internet-gateway-id=%s --vpc-id=%s",
+                gw_id,
+                vpc_id_intern);
+        printf("%s\n", cmd);
+        if (system(cmd) != 0)
+        {
+            fprintf(stderr, "error: can not detach gateway %s from vpc %s\n", gw_id, vpc_id_intern);
+            return -1;
+        }
+        sprintf(cmd, "aws ec2 delete-internet-gateway --internet-gateway-id=%s", gw_id);
+        printf("%s\n", cmd);
+        if (system(cmd) != 0)
+        {
+            fprintf(stderr, "error: can not delete gateway %s\n", gw_id);
+            return -1;
+        }
+    }
+    return 0;
+}
+
+int RDS::create_vpc(const char** vpc_id)
+{
+    json_t* root;
+    json_error_t error;
+    char* result;
+    char cmd[1024];
+
+    if (execute_cmd((char*) "aws ec2 create-vpc --cidr-block 172.30.0.0/16", &result) != 0)
+    {
+        fprintf(stderr, "error: can not create VPC\n");
+        return -1;
+    }
+    root = json_loads(result, 0, &error);
+    if (!root)
+    {
+        fprintf(stderr, "error: on line %d: %s\n", error.line, error.text);
+        return -1;
+    }
+    *vpc_id = json_string_value(json_object_get(json_object_get(root, "Vpc"), "VpcId"));
+    if (*vpc_id == NULL)
+    {
+        fprintf(stderr, "error: can not parse output of create-vpc command\n");
+        return -1;
+    }
+    vpc_id_intern = * vpc_id;
+
+    sprintf(cmd, "aws ec2 modify-vpc-attribute --enable-dns-support --vpc-id %s", *vpc_id);
+    if (system(cmd) != 0)
+    {
+        fprintf(stderr, "error: can not enable dns support\n");
+        return -1;
+    }
+    sprintf(cmd, "aws ec2 modify-vpc-attribute --enable-dns-hostnames --vpc-id %s", *vpc_id);
+    if (system(cmd) != 0)
+    {
+        fprintf(stderr, "error: can not enable dns hostnames\n");
+        return -1;
+    }
+
+    return 0;
+}
+
+int RDS::create_subnet(const char* az, const char* cidr, const char** subnet_id)
+{
+    json_t* root;
+    json_error_t error;
+    char* result;
+    char cmd[1024];
+
+    *subnet_id = NULL;
+    sprintf(cmd,
+            "aws ec2 create-subnet --cidr-block %s --availability-zone %s --vpc-id %s",
+            cidr,
+            az,
+            vpc_id_intern);
+    puts(cmd);
+    if (execute_cmd(cmd, &result) != 0)
+    {
+        fprintf(stderr, "error: can not create subnet\n");
+        return -1;
+    }
+    root = json_loads(result, 0, &error);
+    if (!root)
+    {
+        fprintf(stderr, "error: on line %d: %s\n", error.line, error.text);
+        return -1;
+    }
+    *subnet_id = json_string_value(json_object_get(json_object_get(root, "Subnet"), "SubnetId"));
+    if (*subnet_id == NULL)
+    {
+        fprintf(stderr, "error: can not parse output of create-vpc command\n");
+        return -1;
+    }
+
+    if (subnets_intern == NULL)
+    {
+        subnets_intern = json_array();
+    }
+    json_array_append(subnets_intern, json_string(*subnet_id));
+
+    sprintf(cmd, "aws ec2 modify-subnet-attribute --map-public-ip-on-launch --subnet-id %s", *subnet_id);
+    if (system(cmd) != 0)
+    {
+        fprintf(stderr, "error: can not modify subnet attribute\n");
+        return -1;
+    }
+
+    return 0;
+}
+
+int RDS::create_subnet_group()
+{
+    char cmd[1024];
+    size_t i;
+    json_t* subnet;
+
+    sprintf(cmd,
+            "aws rds create-db-subnet-group --db-subnet-group-name %s --db-subnet-group-description maxscale --subnet-ids",
+            cluster_name_intern);
+    json_array_foreach(subnets_intern, i, subnet)
+    {
+        strcat(cmd, " ");
+        strcat(cmd, json_string_value(subnet));
+    }
+    subnets_group_name_intern = cluster_name_intern;
+    if (system(cmd) != 0)
+    {
+        fprintf(stderr, "error: can not create subnets group\n");
+        return -1;
+    }
+
+    return 0;
+}
+
+int RDS::create_gw(const char** gw_id)
+{
+    char* result;
+    char cmd[1024];
+    json_error_t error;
+
+    *gw_id = NULL;
+    gw_intern = NULL;
+    if (execute_cmd((char*) "aws ec2 create-internet-gateway", &result) != 0)
+    {
+        fprintf(stderr, "error: can not create internet gateway\n");
+        return -1;
+    }
+    json_t* root = json_loads(result, 0, &error);
+    if (!root)
+    {
+        fprintf(stderr, "error: on line %d: %s\n", error.line, error.text);
+        return -1;
+    }
+
+    *gw_id =
+        json_string_value(json_object_get(json_object_get(root, "InternetGateway"), "InternetGatewayId"));
+    if (*gw_id == NULL)
+    {
+        fprintf(stderr, "error: can not parse output of create-internet-gateway command\n");
+        return -1;
+    }
+    gw_intern = *gw_id;
+
+    sprintf(cmd,
+            "aws ec2 attach-internet-gateway --internet-gateway-id %s --vpc-id %s",
+            *gw_id,
+            vpc_id_intern);
+    if (system(cmd) != 0)
+    {
+        fprintf(stderr, "error: can not attach gateway to VPC\n");
+        return -1;
+    }
+
+    return 0;
+}
+
+int RDS::configure_route_table(const char** rt)
+{
+    char* result;
+    char cmd[1024];
+    json_error_t error;
+
+    *rt = NULL;
+    if (execute_cmd((char*) "aws ec2 describe-route-tables", &result) != 0)
+    {
+        fprintf(stderr, "error: can not get route tables description\n");
+        return -1;
+    }
+    json_t* root = json_loads(result, 0, &error);
+    if (!root)
+    {
+        fprintf(stderr, "error: on line %d: %s\n", error.line, error.text);
+        return -1;
+    }
+
+    json_t* route_tables = json_object_get(root, "RouteTables");
+    if (route_tables == NULL)
+    {
+        fprintf(stderr, "error: can not parse route tables description\n");
+        return -1;
+    }
+    size_t i;
+    json_t* rtb;
+    const char* rt_vpc;
+
+    json_array_foreach(route_tables, i, rtb)
+    {
+        rt_vpc = json_string_value(json_object_get(rtb, "VpcId"));
+        if (strcmp(vpc_id_intern, rt_vpc) == 0)
+        {
+            // add route to route table which belongs to give VPC
+            *rt = json_string_value(json_object_get(rtb, "RouteTableId"));
+            sprintf(cmd,
+                    "aws ec2 create-route --route-table-id %s --gateway-id %s --destination-cidr-block 0.0.0.0/0",
+                    *rt,
+                    gw_intern);
+            if (system(cmd) != 0)
+            {
+                fprintf(stderr, "error: can not create route\n");
+                return -1;
+            }
+        }
+    }
+    if (*rt == NULL)
+    {
+        fprintf(stderr, "error: can not find route table\n");
+        return -1;
+    }
+    return 0;
+}
+
+int RDS::create_cluster()
+{
+    char cmd[1024];
+    char* result;
+    json_error_t error;
+    size_t i;
+    int res = 0;
+
+    sprintf(cmd,
+            "aws rds create-db-cluster --database-name=test --engine=aurora --master-username=skysql --master-user-password=skysqlrds --db-cluster-identifier=%s --db-subnet-group-name=%s",
+            cluster_name_intern,
+            cluster_name_intern);
+
+    execute_cmd(cmd, &result);
+    json_t* root = json_loads(result, 0, &error);
+    if (!root)
+    {
+        fprintf(stderr, "error: on line %d: %s\n", error.line, error.text);
+        return -1;
+    }
+    json_t* cluster = json_object_get(root, "DBCluster");
+    cluster_intern = cluster;
+    json_t* security_groups = json_object_get(cluster, "VpcSecurityGroups");
+    json_t* sg;
+    const char* sg_id;
+
+    json_array_foreach(security_groups, i, sg)
+    {
+        sg_id = json_string_value(json_object_get(sg, "VpcSecurityGroupId"));
+        printf("Security group %s\n", sg_id);
+        sprintf(cmd,
+                "aws ec2 authorize-security-group-ingress --group-id %s --protocol tcp --port 3306 --cidr 0.0.0.0/0",
+                sg_id);
+        res += system(cmd);
+    }
+    sg_intern = sg_id;
+
+    for (size_t i = 0; i < N_intern; i++)
+    {
+        sprintf(cmd,
+                "aws rds create-db-instance --db-cluster-identifier=%s --engine=aurora --db-instance-class=db.t2.medium --publicly-accessible --db-instance-identifier=node%03lu",
+                cluster_name_intern,
+                i);
+        printf("%s\n", cmd);
+        res += system(cmd);
+    }
+    return res;
+}
+
+int RDS::get_writer(const char** writer_name)
+{
+    char* json;
+    char cmd[1024];
+    sprintf(cmd, "aws rds describe-db-clusters --db-cluster-identifier=%s", cluster_name_intern);
+    execute_cmd(cmd, &json);
+    json_t* cluster = get_cluster_descr(json);
+    json_t* nodes = json_object_get(cluster, "DBClusterMembers");
+
+    // char * s = json_dumps(nodes, JSON_INDENT(4));
+    // puts(s);
+
+    bool writer;
+    json_t* node;
+    size_t i = 0;
+
+    do
+    {
+        node = json_array_get(nodes, i);
+        writer = json_is_true(json_object_get(node, "IsClusterWriter"));
+        i++;
+    }
+    while (!writer);
+    * writer_name = json_string_value(json_object_get(node, "DBInstanceIdentifier"));
+
+    return 0;
+}
+
+int RDS::destroy_vpc()
+{
+    char cmd[1024];
+    sprintf(cmd, "aws ec2 delete-vpc --vpc-id=%s", vpc_id_intern);
+    return system(cmd);
+}
+
+int RDS::destroy_cluster()
+{
+    char cmd[1024];
+    char* result;
+    sprintf(cmd,
+            "aws rds delete-db-cluster --db-cluster-identifier=%s --skip-final-snapshot",
+            cluster_name_intern);
+    return execute_cmd(cmd, &result);
+}
+
+int RDS::destroy_subnets_group()
+{
+    char cmd[1024];
+    char* result;
+    sprintf(cmd, "aws rds delete-db-subnet-group --db-subnet-group-name %s", get_subnetgroup_name());
+    puts(cmd);
+    return execute_cmd(cmd, &result);
+}
+
+int RDS::create_rds_db(int N)
+{
+    const char* vpc;
+    const char* subnet1;
+    const char* subnet2;
+    const char* gw;
+    const char* rt;
+
+    N_intern = N;
+
+    printf("Create VPC\n");
+    if (create_vpc(&vpc) != 0)
+    {
+        fprintf(stderr, "error: can not create VPC\n");
+        destroy_vpc();
+        return -1;
+    }
+    printf("vpc id: %s\n", vpc);
+
+    printf("Create subnets\n");
+    create_subnet("eu-west-1b", "172.30.0.0/24", &subnet1);
+    create_subnet("eu-west-1a", "172.30.1.0/24", &subnet2);
+
+    printf("Create subnets group\n");
+    if (create_subnet_group() != 0)
+    {
+        destroy_subnets();
+        destroy_subnets_group();
+        destroy_vpc();
+        return -1;
+    }
+
+    printf("Create internet gateway\n");
+    if (create_gw(&gw) != 0)
+    {
+        detach_and_destroy_gw();
+        destroy_subnets();
+        destroy_subnets_group();
+        destroy_vpc();
+        return -1;
+    }
+    printf("Gateway: %s\n", gw);
+
+    printf("Configure route table\n");
+    if (configure_route_table(&rt) != 0)
+    {
+        detach_and_destroy_gw();
+        destroy_subnets();
+        destroy_subnets_group();
+        destroy_vpc();
+        return -1;
+    }
+    printf("Route table: %s\n", rt);
+
+    printf("Create RDS cluster\n");
+    if (create_cluster() != 0)
+    {
+        destroy_nodes(get_cluster_nodes());
+        destroy_cluster();
+        detach_and_destroy_gw();
+        destroy_subnets();
+        destroy_subnets_group();
+        destroy_vpc();
+        return -1;
+    }
+    return 0;
+}
+
+int RDS::delete_rds_cluster()
+{
+    char* result;
+    char cmd[1024];
+    json_t* current_cluster;
+
+    printf("Get cluster\n");
+    cluster_intern = get_cluster();
+    printf("Get cluster NODES\n");
+    json_t* nodes = get_cluster_nodes();
+
+    printf("Get subnets group: %s\n", get_subnetgroup_name());
+
+    printf("Get subnets\n");
+    get_subnets();
+
+    printf("Get VPC: %s\n", vpc_id_intern);
+
+    size_t alive_nodes = json_array_size(nodes);
+
+    printf("Destroy nodes\n");
+    destroy_nodes(nodes);
+
+    do
+    {
+        printf("Waiting for nodes to be deleted, now %lu nodes are still alive\n", alive_nodes);
+        sleep(5);
+        current_cluster = get_cluster();
+        nodes = get_cluster_nodes(current_cluster);
+        alive_nodes = json_array_size(nodes);
+    }
+    while (alive_nodes > 0);
+
+    printf("Destroy cluster\n");
+    destroy_cluster();
+
+    do
+    {
+        printf("Waiting for cluster to be deleted\n");
+        sleep(5);
+        sprintf(cmd, "aws rds describe-db-clusters --db-cluster-identifier=%s", cluster_name_intern);
+        execute_cmd(cmd, &result);
+    }
+    while (get_cluster_descr(result) != NULL);
+
+    printf("Destroy subnets\n");
+    destroy_subnets();
+
+    printf("Destroy subnet group\n");
+    destroy_subnets_group();
+
+    printf("Get and destroy Internet Gateways\n");
+    detach_and_destroy_gw();
+
+    printf("Destroy vpc\n");
+    return destroy_vpc();
+}
+
+int RDS::wait_for_nodes(size_t N)
+{
+    char* result;
+    size_t active_nodes = 0;
+    size_t i = 0;
+    json_t* node;
+    char cmd[1024];
+    json_t* nodes;
+    json_t* instances;
+    json_t* instance;
+    json_error_t error;
+
+    do
+    {
+        printf("Waiting for nodes to be active, now %lu are active\n", active_nodes);
+        sleep(5);
+        cluster_intern = get_cluster();
+        nodes = get_cluster_nodes();
+
+        active_nodes = 0;
+        json_array_foreach(nodes, i, node)
+        {
+            sprintf(cmd,
+                    "aws rds describe-db-instances --db-instance-identifier=%s",
+                    json_string_value(node));
+            execute_cmd(cmd, &result);
+            instances = json_loads(result, 0, &error);
+            if (!instances)
+            {
+                fprintf(stderr, "error: on line %d: %s\n", error.line, error.text);
+                return -1;
+            }
+            instance = json_array_get(json_object_get(instances, "DBInstances"), 0);
+            // puts(json_dumps(instance, JSON_INDENT(4)));
+            if (strcmp(json_string_value(json_object_get(instance, "DBInstanceStatus")), "available") == 0)
+            {
+                active_nodes++;
+            }
+        }
+    }
+    while (active_nodes != N);
+    return 0;
+}
+
+int RDS::do_failover()
+{
+    char* result;
+    const char* writer;
+    const char* new_writer;
+    char cmd[1024];
+    if (get_writer(&writer) != 0)
+    {
+        return -1;
+    }
+
+    sprintf(cmd, "aws rds failover-db-cluster --db-cluster-identifier=%s", cluster_name_intern);
+    if (execute_cmd(cmd, &result) != 0)
+    {
+        return -1;
+    }
+    do
+    {
+        if (get_writer(&new_writer) != 0)
+        {
+            return -1;
+        }
+        printf("writer: %s\n", new_writer);
+        sleep(5);
+    }
+    while (strcmp(writer, new_writer) == 0);
+    return 0;
+}
+
+json_t* RDS::get_endpoints()
+{
+    char cmd[1024];
+    char* result;
+
+    json_t* root;
+    json_error_t error;
+
+    json_t* node;
+    json_t* node_json;
+    json_t* endpoint;
+
+    json_t* endpoints;
+
+    endpoints = json_array();
+
+    cluster_intern = get_cluster();
+    json_t* nodes = get_cluster_nodes();
+    // puts(json_dumps(nodes, JSON_INDENT(4)));
+
+    size_t i;
+    json_array_foreach(nodes, i, node)
+    {
+        sprintf(cmd, "aws rds describe-db-instances --db-instance-identifier=%s", json_string_value(node));
+        if (execute_cmd(cmd, &result) != 0)
+        {
+            fprintf(stderr, "error: executing aws rds describe-db-instances\n");
+            return NULL;
+        }
+        root = json_loads(result, 0, &error);
+        if (!root)
+        {
+            fprintf(stderr, "error: on line %d: %s\n", error.line, error.text);
+            return NULL;
+        }
+        node_json = json_array_get(json_object_get(root, "DBInstances"), 0);
+        endpoint = json_object_get(node_json, "Endpoint");
+        json_array_append(endpoints, endpoint);
+    }
+    return endpoints;
+}

maxscale-system-test/maxtest/src/sql_t1.cpp

@@ -0,0 +1,257 @@
+#include "sql_t1.h"
+
+#include <pthread.h>
+
+pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
+static char** sql = NULL;
+static size_t sql_size = 0;
+
+int execute_select_query_and_check(MYSQL* conn, const char* sql, unsigned long long int rows)
+{
+    MYSQL_RES* res;
+    MYSQL_ROW row;
+    unsigned long long int i;
+    unsigned long long int num_fields;
+    unsigned long long int int_res;
+    unsigned long long int row_i = 0;
+    int test_result = 0;
+    unsigned long long int rows_from_select = 0;
+    int wait_i = 0;
+
+    printf("Trying SELECT, num_of_rows=%llu\n", rows);
+    int res_alloc = 0;
+    if (conn != NULL)
+    {
+        rows_from_select = 0;
+        wait_i = 0;
+        while ((rows_from_select != rows) && (wait_i < 100))
+        {
+            if (mysql_query(conn, sql) != 0)
+            {
+                printf("Error: can't execute SQL-query: %s\n", mysql_error(conn));
+            }
+
+            res = mysql_store_result(conn);
+            res_alloc = 1;
+            if (res == NULL)
+            {
+                printf("Error: can't get the result description\n");
+                test_result = 1;
+                mysql_free_result(res);
+                res_alloc = 0;
+                wait_i++;
+                sleep(1);
+            }
+            else
+            {
+                rows_from_select = mysql_num_rows(res);
+                printf("rows=%llu\n", rows_from_select);
+                wait_i++;
+                if (rows_from_select != rows)
+                {
+                    printf("Waiting 1 second and trying again...\n");
+                    mysql_free_result(res);
+                    res_alloc = 0;
+                    sleep(1);
+                }
+            }
+        }
+
+        if (rows_from_select != rows)
+        {
+            printf("SELECT returned %llu rows instead of %llu!\n", rows_from_select, rows);
+            test_result = 1;
+            printf("sql was %s\n", sql);
+        }
+        else
+        {
+            num_fields = mysql_num_fields(res);
+            if (num_fields != 2)
+            {
+                printf("SELECT returned %llu fileds instead of 2!\n", num_fields);
+                test_result = 1;
+            }
+            if (mysql_num_rows(res) > 0)
+            {
+                while ((row = mysql_fetch_row(res)) != NULL)
+                {
+                    for (i = 0; i < num_fields; i++)
+                    {
+                        sscanf(row[i], "%llu", &int_res);
+                        if ((i == 0 ) && (int_res != row_i))
+                        {
+                            printf("SELECT returned wrong result! %llu instead of expected %llu\n",
+                                   int_res,
+                                   row_i);
+                            test_result = 1;
+                            printf("sql was %s\n", sql);
+                        }
+                    }
+                    row_i++;
+                }
+            }
+        }
+        if (res_alloc != 0)
+        {
+            mysql_free_result(res);
+        }
+    }
+    else
+    {
+        printf("FAILED: broken connection\n");
+        test_result = 1;
+    }
+
+    return test_result;
+}
+int create_t1(MYSQL* conn)
+{
+    int result = 0;
+    result += execute_query(conn, "DROP TABLE IF EXISTS t1;");
+    printf("Creating test table\n");
+    result += execute_query(conn, "CREATE TABLE t1 (x1 int, fl int);");
+    return result;
+}
+
+int create_t2(MYSQL* conn)
+{
+    int result = 0;
+    result += execute_query(conn, "DROP TABLE IF EXISTS t2;");
+    printf("Creating test table\n");
+    result += execute_query(conn, "CREATE TABLE t2 (x1 int, fl int);");
+    return result;
+}
+
+static const char ins1[] = "INSERT INTO t1 (x1, fl) VALUES ";
+
+int create_insert_string(char* sql, int N, int fl)
+{
+    char* wptr = sql;
+
+    strcpy(wptr, ins1);
+    for (int i = 0; i < N; i++)
+    {
+        wptr = strchr(wptr, '\0');
+        sprintf(wptr, "(%d, %d),", i, fl);
+    }
+    wptr = strrchr(wptr, ',');
+    sprintf(wptr, ";");
+    return 0;
+}
+
+char* allocate_insert_string(int fl, int N)
+{
+    char* rval = NULL;
+
+    pthread_mutex_lock(&mutex);
+
+    if (sql == NULL)
+    {
+        sql = (char**)calloc(16, sizeof(char*));
+        sql_size = 16;
+    }
+
+    if ((size_t)fl >= sql_size)
+    {
+        fprintf(stderr, "Insert index %d is too large, setting it to %lu", fl, sql_size - 1);
+        fl = sql_size - 1;
+    }
+
+    if (sql[fl] == NULL)
+    {
+        char tmpstr[256];
+        sprintf(tmpstr, "(%d, %d),", N, fl);
+        sql[fl] = (char*)malloc(sizeof(ins1) + N * strlen(tmpstr) + 60);
+        create_insert_string(sql[fl], N, fl);
+    }
+
+    rval = sql[fl];
+    pthread_mutex_unlock(&mutex);
+
+    return rval;
+}
+
+int insert_into_t1(MYSQL* conn, int N)
+{
+
+    int x = 16;
+    int result = 0;
+
+    printf("Generating long INSERTs\n");
+    for (int i = 0; i < N; i++)
+    {
+        printf("sql %d, rows=%d\n", i, x);
+        char* sqlstr = allocate_insert_string(i, x);
+        printf("INSERT: rwsplitter\n");
+        printf("Trying INSERT, len=%d\n", x);
+        fflush(stdout);
+        result += execute_query(conn, "%s", sqlstr);
+        fflush(stdout);
+        x *= 16;
+    }
+    return result;
+}
+
+int select_from_t1(MYSQL* conn, int N)
+{
+    int x = 16;
+    int result = 0;
+    int i;
+    char sq[100];
+
+    for (i = 0; i < N; i++)
+    {
+        sprintf(&sq[0], "select * from t1 where fl=%d;", i);
+        result += execute_select_query_and_check(conn, sq, x);
+        x = x * 16;
+    }
+    return result;
+}
+
+// 0 - if it does not exist
+// -1 - in case of error
+int check_if_t1_exists(MYSQL* conn)
+{
+    MYSQL_RES* res;
+    MYSQL_ROW row;
+
+    int t1 = 0;
+    if (conn != NULL)
+    {
+        if (mysql_query(conn, "show tables;") != 0)
+        {
+            printf("Error: can't execute SQL-query: %s\n", mysql_error(conn));
+            t1 = 0;
+        }
+        else
+        {
+            res = mysql_store_result(conn);
+            if (res == NULL)
+            {
+                printf("Error: can't get the result description\n");
+                t1 = -1;
+            }
+            else
+            {
+                mysql_num_fields(res);
+                if (mysql_num_rows(res) > 0)
+                {
+                    while ((row = mysql_fetch_row(res)) != NULL)
+                    {
+                        if ((row[0] != NULL ) && (strcmp(row[0], "t1") == 0 ))
+                        {
+                            t1 = 1;
+                        }
+                    }
+                }
+            }
+            mysql_free_result(res);
+        }
+    }
+    else
+    {
+        printf("FAILED: broken connection\n");
+        t1 = -1;
+    }
+    return t1;
+}

maxscale-system-test/maxtest/src/stopwatch.cpp

@@ -0,0 +1,147 @@
+/*
+ * Copyright (c) 2018 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2024-02-10
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#include "stopwatch.h"
+
+#include <iomanip>
+#include <iostream>
+#include <sstream>
+#include <ctime>
+
+namespace base
+{
+
+StopWatch::StopWatch()
+{
+    restart();
+}
+
+Duration StopWatch::lap() const
+{
+    return {Clock::now() - m_start};
+}
+
+Duration StopWatch::restart()
+{
+    TimePoint now = Clock::now();
+    Duration lap = now - m_start;
+    m_start = now;
+    return lap;
+}
+}   // base
+
+/********** OUTPUT ***********/
+namespace
+{
+using namespace base;
+struct TimeConvert
+{
+    double      div;        // divide the value of the previous unit by this
+    std::string suffix;     // milliseconds, hours etc.
+    double      max_visual; // threashold to switch to the next unit
+};
+// Will never get to centuries because the duration is a long carrying nanoseconds
+TimeConvert convert[]
+{
+    {1, "ns", 1000}, {1000, "us", 1000}, {1000, "ms", 1000},
+    {1000, "s", 60}, {60, "min", 60}, {60, "hours", 24},
+    {24, "days", 365.25}, {365.25, "years", 10000},
+    {100, "centuries", std::numeric_limits<double>::max()}
+};
+
+int convert_size = sizeof(convert) / sizeof(convert[0]);
+}
+
+namespace base
+{
+std::pair<double, std::string> dur_to_human_readable(Duration dur)
+{
+    using namespace std::chrono;
+    double time = duration_cast<nanoseconds>(dur).count();
+    bool negative = (time < 0) ? time = -time, true : false;
+
+    for (int i = 0; i <= convert_size; ++i)
+    {
+        if (i == convert_size)
+        {
+            return std::make_pair(negative ? -time : time,
+                                  convert[convert_size - 1].suffix);
+        }
+
+        time /= convert[i].div;
+
+        if (time < convert[i].max_visual)
+        {
+            return std::make_pair(negative ? -time : time, convert[i].suffix);
+        }
+    }
+
+    abort();    // should never get here
+}
+
+std::ostream& operator<<(std::ostream& os, Duration dur)
+{
+    auto p = dur_to_human_readable(dur);
+    os << p.first << p.second;
+
+    return os;
+}
+
+// TODO: this will require some thought. time_point_to_string() for a system_clock is
+// obvious, but not so for a steady_clock. Maybe TimePoint belongs to a system clock
+// and sould be called something else here, and live in a time_measuring namespace.
+std::string time_point_to_string(TimePoint tp, const std::string& fmt)
+{
+    using namespace std::chrono;
+    std::time_t timet = system_clock::to_time_t(system_clock::now()
+                                                + (tp - Clock::now()));
+    struct tm* ptm;
+    ptm = gmtime(&timet);
+    const int sz = 1024;
+    char buf[sz];
+    strftime(buf, sz, fmt.c_str(), ptm);
+    return buf;
+}
+
+std::ostream& operator<<(std::ostream& os, TimePoint tp)
+{
+    os << time_point_to_string(tp);
+    return os;
+}
+
+void test_stopwatch_output(std::ostream& os)
+{
+    long long dur[] =
+    {
+        400,                                // 400ns
+        5 * 1000,                           // 5us
+        500 * 1000,                         // 500us
+        1 * 1000000,                        // 1ms
+        700 * 1000000LL,                    // 700ms
+        5 * 1000000000LL,                   // 5s
+        200 * 1000000000LL,                 // 200s
+        5 * 60 * 1000000000LL,              // 5m
+        45 * 60 * 1000000000LL,             // 45m
+        130 * 60 * 1000000000LL,            // 130m
+        24 * 60 * 60 * 1000000000LL,        // 24 hours
+        3 * 24 * 60 * 60 * 1000000000LL,    // 72 hours
+        180 * 24 * 60 * 60 * 1000000000LL,  // 180 days
+        1000 * 24 * 60 * 60 * 1000000000LL  // 1000 days
+    };
+
+    for (unsigned i = 0; i < sizeof(dur) / sizeof(dur[0]); ++i)
+    {
+        os << Duration(dur[i]) << std::endl;
+    }
+}
+}   // base

maxscale-system-test/maxtest/src/tcp_connection.cpp

@@ -0,0 +1,87 @@
+#include "tcp_connection.h"
+
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <unistd.h>
+#include <string.h>
+#include <netdb.h>
+
+
+namespace
+{
+
+static void set_port(struct sockaddr_storage* addr, uint16_t port)
+{
+    if (addr->ss_family == AF_INET)
+    {
+        struct sockaddr_in* ip = (struct sockaddr_in*)addr;
+        ip->sin_port = htons(port);
+    }
+    else if (addr->ss_family == AF_INET6)
+    {
+        struct sockaddr_in6* ip = (struct sockaddr_in6*)addr;
+        ip->sin6_port = htons(port);
+    }
+}
+
+int open_network_socket(struct sockaddr_storage* addr, const char* host, uint16_t port)
+{
+    struct addrinfo* ai = NULL, hint = {};
+    int so = -1;
+    hint.ai_socktype = SOCK_STREAM;
+    hint.ai_family = AF_UNSPEC;
+    hint.ai_flags = AI_ALL;
+
+    /* Take the first one */
+    if (getaddrinfo(host, NULL, &hint, &ai) == 0 && ai)
+    {
+        if ((so = socket(ai->ai_family, SOCK_STREAM, 0)) != -1)
+        {
+            memcpy(addr, ai->ai_addr, ai->ai_addrlen);
+            set_port(addr, port);
+            freeaddrinfo(ai);
+        }
+    }
+
+    return so;
+}
+}
+
+namespace tcp
+{
+
+Connection::~Connection()
+{
+    if (m_so != -1)
+    {
+        close(m_so);
+    }
+}
+
+bool Connection::connect(const char* host, uint16_t port)
+{
+    struct sockaddr_storage addr;
+
+    if ((m_so = open_network_socket(&addr, host, port)) != -1)
+    {
+        if (::connect(m_so, (struct sockaddr*)&addr, sizeof(addr)) != 0)
+        {
+            close(m_so);
+            m_so = -1;
+        }
+    }
+
+    return m_so != -1;
+}
+
+int Connection::write(void* buf, size_t size)
+{
+    return ::write(m_so, buf, size);
+}
+
+int Connection::read(void* buf, size_t size)
+{
+    return ::read(m_so, buf, size);
+}
+}

maxscale-system-test/maxtest/src/test_binlog_fnc.cpp

@@ -0,0 +1,279 @@
+#include <iostream>
+#include "testconnections.h"
+#include "maxadmin_operations.h"
+#include "sql_t1.h"
+
+#include "test_binlog_fnc.h"
+
+int check_sha1(TestConnections* Test)
+{
+    if (Test->binlog_master_gtid || Test->binlog_slave_gtid)
+    {
+        Test->tprintf("GTID is in use, do not check sha1\n");
+        return 0;
+    }
+    else
+    {
+        char sys[1024];
+        char* x;
+        int local_result = 0;
+        int i;
+        int exit_code;
+
+        char* s_maxscale;
+        char* s;
+
+        Test->set_timeout(50);
+        Test->tprintf("ls before FLUSH LOGS");
+        Test->tprintf("Maxscale");
+        Test->maxscales->ssh_node_f(0,
+                                    true,
+                                    "ls -la %s/mar-bin.0000*",
+                                    Test->maxscales->maxscale_binlog_dir[0]);
+        Test->tprintf("Master");
+        Test->set_timeout(50);
+        Test->maxscales->ssh_node(0, "ls -la /var/lib/mysql/mar-bin.0000*", false);
+
+        Test->tprintf("FLUSH LOGS");
+        Test->set_timeout(100);
+        local_result += execute_query(Test->repl->nodes[0], (char*) "FLUSH LOGS");
+        Test->tprintf("Logs flushed");
+        Test->set_timeout(100);
+        Test->repl->sync_slaves();
+        Test->tprintf("ls after first FLUSH LOGS");
+        Test->tprintf("Maxscale");
+        Test->set_timeout(50);
+        Test->maxscales->ssh_node_f(0,
+                                    true,
+                                    "ls -la %s/mar-bin.0000*",
+                                    Test->maxscales->maxscale_binlog_dir[0]);
+
+        Test->tprintf("Master");
+        Test->set_timeout(50);
+        Test->maxscales->ssh_node(0, "ls -la /var/lib/mysql/mar-bin.0000*", false);
+
+        Test->set_timeout(100);
+        Test->tprintf("FLUSH LOGS");
+        local_result += execute_query(Test->repl->nodes[0], (char*) "FLUSH LOGS");
+        Test->tprintf("Logs flushed");
+
+        Test->set_timeout(50);
+        Test->repl->sync_slaves();
+        Test->set_timeout(50);
+        Test->tprintf("ls before FLUSH LOGS");
+        Test->tprintf("Maxscale");
+
+        Test->maxscales->ssh_node_f(0,
+                                    true,
+                                    "ls -la %s/mar-bin.0000*",
+                                    Test->maxscales->maxscale_binlog_dir[0]);
+
+        Test->tprintf("Master");
+        Test->set_timeout(50);
+        Test->maxscales->ssh_node(0, "ls -la /var/lib/mysql/mar-bin.0000*", false);
+
+
+        for (i = 1; i < 3; i++)
+        {
+            Test->tprintf("FILE: 000000%d", i);
+            Test->set_timeout(50);
+            s_maxscale = Test->maxscales->ssh_node_output_f(0,
+                                                            true,
+                                                            &exit_code,
+                                                            "sha1sum %s/mar-bin.00000%d",
+                                                            Test->maxscales->maxscale_binlog_dir[0],
+                                                            i);
+            if (s_maxscale != NULL)
+            {
+                x = strchr(s_maxscale, ' ');
+                if (x != NULL)
+                {
+                    x[0] = 0;
+                }
+                Test->tprintf("Binlog checksum from Maxscale %s", s_maxscale);
+            }
+
+            sprintf(sys, "sha1sum /var/lib/mysql/mar-bin.00000%d", i);
+            Test->set_timeout(50);
+            s = Test->repl->ssh_node_output(0, sys, true, &exit_code);
+            if (s != NULL)
+            {
+                x = strchr(s, ' ');
+                if (x != NULL)
+                {
+                    x[0] = 0;
+                }
+                Test->tprintf("Binlog checksum from master %s", s);
+            }
+            if (strcmp(s_maxscale, s) != 0)
+            {
+                Test->tprintf("Binlog from master checksum is not equal to binlog checksum from Maxscale node");
+                local_result++;
+            }
+        }
+        return local_result;
+    }
+}
+
+int start_transaction(TestConnections* Test)
+{
+    int local_result = 0;
+    Test->tprintf("Transaction test");
+    Test->tprintf("Start transaction");
+    execute_query(Test->repl->nodes[0], (char*) "DELETE FROM t1 WHERE fl=10;");
+    local_result += execute_query(Test->repl->nodes[0], (char*) "START TRANSACTION");
+    local_result += execute_query(Test->repl->nodes[0], (char*) "SET autocommit = 0");
+    Test->tprintf("INSERT data");
+    local_result += execute_query(Test->repl->nodes[0], (char*) "INSERT INTO t1 VALUES(111, 10)");
+    Test->set_timeout(120);
+    Test->repl->sync_slaves();
+
+    return local_result;
+}
+
+void test_binlog(TestConnections* Test)
+{
+    int i;
+    MYSQL* binlog;
+    Test->repl->connect();
+
+    Test->set_timeout(100);
+    Test->try_query(Test->repl->nodes[0], (char*) "SET NAMES utf8mb4");
+    Test->try_query(Test->repl->nodes[0], (char*) "set autocommit=1");
+    Test->try_query(Test->repl->nodes[0], (char*) "select USER()");
+
+    Test->set_timeout(100);
+    create_t1(Test->repl->nodes[0]);
+    Test->add_result(insert_into_t1(Test->repl->nodes[0], 4), "Data inserting to t1 failed");
+    Test->stop_timeout();
+    Test->tprintf("Waiting for replication to catch up");
+    Row row = get_row(Test->repl->nodes[0], "SELECT @@gtid_current_pos");
+
+    for (int i = 1; i < Test->repl->N; i++)
+    {
+        std::string query = "SELECT MASTER_GTID_WAIT('" + row[0] + "', 120)";
+        get_row(Test->repl->nodes[i], query);
+    }
+
+    Test->repl->disconnect();
+    Test->repl->connect();
+
+    for (i = 0; i < Test->repl->N; i++)
+    {
+        Test->tprintf("Checking data from node %d (%s)", i, Test->repl->IP[i]);
+        Test->set_timeout(100);
+        Test->add_result(select_from_t1(Test->repl->nodes[i], 4), "Selecting from t1 failed");
+        Test->stop_timeout();
+    }
+
+    Test->set_timeout(10);
+    Test->tprintf("First transaction test (with ROLLBACK)");
+    start_transaction(Test);
+
+    Test->set_timeout(50);
+
+    Test->tprintf("SELECT * FROM t1 WHERE fl=10, checking inserted values");
+    Test->add_result(execute_query_check_one(Test->repl->nodes[0],
+                                             (char*) "SELECT * FROM t1 WHERE fl=10",
+                                             "111"),
+                     "SELECT check failed");
+
+
+    Test->tprintf("ROLLBACK");
+    Test->try_query(Test->repl->nodes[0], (char*) "ROLLBACK");
+    Test->tprintf("INSERT INTO t1 VALUES(112, 10)");
+    Test->try_query(Test->repl->nodes[0], (char*) "INSERT INTO t1 VALUES(112, 10)");
+    Test->try_query(Test->repl->nodes[0], (char*) "COMMIT");
+    Test->set_timeout(120);
+    Test->repl->sync_slaves();
+
+    Test->set_timeout(20);
+    Test->tprintf("SELECT * FROM t1 WHERE fl=10, checking inserted values");
+    Test->add_result(execute_query_check_one(Test->repl->nodes[0],
+                                             (char*) "SELECT * FROM t1 WHERE fl=10",
+                                             "112"),
+                     "SELECT check failed");
+
+    Test->tprintf("SELECT * FROM t1 WHERE fl=10, checking inserted values from slave");
+    Test->add_result(execute_query_check_one(Test->repl->nodes[2],
+                                             (char*) "SELECT * FROM t1 WHERE fl=10",
+                                             "112"),
+                     "SELECT check failed");
+    Test->tprintf("DELETE FROM t1 WHERE fl=10");
+    Test->try_query(Test->repl->nodes[0], (char*) "DELETE FROM t1 WHERE fl=10");
+    Test->tprintf("Checking t1");
+    Test->add_result(select_from_t1(Test->repl->nodes[0], 4), "SELECT from t1 failed");
+
+    Test->tprintf("Second transaction test (with COMMIT)");
+    start_transaction(Test);
+
+    Test->tprintf("COMMIT");
+    Test->try_query(Test->repl->nodes[0], (char*) "COMMIT");
+
+    Test->tprintf("SELECT, checking inserted values");
+    Test->add_result(execute_query_check_one(Test->repl->nodes[0],
+                                             (char*) "SELECT * FROM t1 WHERE fl=10",
+                                             "111"),
+                     "SELECT check failed");
+
+    Test->tprintf("SELECT, checking inserted values from slave");
+    Test->add_result(execute_query_check_one(Test->repl->nodes[2],
+                                             (char*) "SELECT * FROM t1 WHERE fl=10",
+                                             "111"),
+                     "SELECT check failed");
+    Test->tprintf("DELETE FROM t1 WHERE fl=10");
+    Test->try_query(Test->repl->nodes[0], (char*) "DELETE FROM t1 WHERE fl=10");
+
+    Test->stop_timeout();
+
+    Test->set_timeout(50);
+    Test->add_result(check_sha1(Test), "sha1 check failed");
+    Test->repl->close_connections();
+
+    Test->stop_timeout();
+
+    // test SLAVE STOP/START
+    Test->tprintf("test SLAVE STOP/START");
+    Test->set_timeout(100);
+    Test->repl->connect();
+
+    Test->tprintf("Dropping and re-creating t1");
+    Test->try_query(Test->repl->nodes[0], (char*) "DROP TABLE IF EXISTS t1");
+    create_t1(Test->repl->nodes[0]);
+
+    Test->tprintf("Connecting to MaxScale binlog router");
+    binlog = open_conn(Test->maxscales->binlog_port[0],
+                       Test->maxscales->IP[0],
+                       Test->repl->user_name,
+                       Test->repl->password,
+                       Test->ssl);
+
+    Test->tprintf("STOP SLAVE against Maxscale binlog");
+    execute_query(binlog, (char*) "STOP SLAVE");
+
+    Test->tprintf("FLUSH LOGS on master");
+    execute_query(Test->repl->nodes[0], (char*) "FLUSH LOGS");
+    execute_query(Test->repl->nodes[0], (char*) "FLUSH LOGS");
+    execute_query(Test->repl->nodes[0], (char*) "FLUSH LOGS");
+    execute_query(Test->repl->nodes[0], (char*) "FLUSH LOGS");
+
+    Test->add_result(insert_into_t1(Test->repl->nodes[0], 4), "INSERT into t1 failed");
+
+    Test->tprintf("START SLAVE against Maxscale binlog");
+    Test->try_query(binlog, (char*) "START SLAVE");
+
+    Test->set_timeout(120);
+    Test->repl->sync_slaves();
+
+    for (i = 0; i < Test->repl->N; i++)
+    {
+        Test->set_timeout(50);
+        Test->tprintf("Checking data from node %d (%s)", i, Test->repl->IP[i]);
+        Test->add_result(select_from_t1(Test->repl->nodes[i], 4), "SELECT from t1 failed");
+    }
+
+    Test->set_timeout(100);
+    Test->add_result(check_sha1(Test), "sha1 check failed");
+    Test->repl->close_connections();
+    Test->stop_timeout();
+}