MXS-2518 Check connection distribution with Clustrix

Readconnroute, with default 'router_options', as well as readwritesplit
 should cause the connections to be distributed evenly across all nodes.

maxscale-system-test/CMakeLists.txt

@@ -1165,6 +1165,8 @@ set_tests_properties(clustrix_basics PROPERTIES TIMEOUT 7200)
 add_test_executable(clustrix_transaction_replay.cpp clustrix_transaction_replay clustrix_transaction_replay LABELS CLUSTRIX_BACKEND)
 set_tests_properties(clustrix_transaction_replay PROPERTIES TIMEOUT 7200)
 
+add_test_executable(clustrix_distribution.cpp clustrix_distribution clustrix_distribution LABELS CLUSTRIX_BACKEND)
+
 ############################################
 # END: tests for Clustrix monitor          #
 ############################################

maxscale-system-test/clustrix_distribution.cpp

@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) 2019 MariaDB Corporation Ab
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file and at www.mariadb.com/bsl11.
+ *
+ * Change Date: 2022-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2 or later of the General
+ * Public License.
+ */
+
+#include <iostream>
+#include <map>
+#include <vector>
+#include "testconnections.h"
+#include "maxrest.hh"
+
+using namespace std;
+
+namespace
+{
+
+const int N = 60;
+
+Connection create_rcr_connection(TestConnections& test)
+{
+    return test.maxscales->readconn_master();
+}
+
+Connection create_rws_connection(TestConnections& test)
+{
+    return test.maxscales->rwsplit();
+}
+
+void run_test(TestConnections& test,
+              const std::string& router,
+              Connection (*create_connection)(TestConnections& test))
+{
+    vector<Connection> connections;
+
+    cout << "Creating " << N << " connections: " << flush;
+
+    for (int i = 1; i <= N; ++i)
+    {
+        cout << i << " " << flush;
+        Connection c = create_connection(test);
+        test.expect(c.connect(), "Could not connect to %s.", router.c_str());
+
+        connections.emplace_back(std::move(c));
+    }
+
+    cout << endl;
+
+    MaxRest maxrest(&test);
+
+    auto servers = maxrest.list_servers();
+
+    map<string, int> connections_by_server;
+
+    for (auto server : servers)
+    {
+        if (server.name.front() == '@') // A dynamic server
+        {
+            connections_by_server.insert(std::make_pair(server.name, server.connections));
+        }
+    }
+
+    int n = N / connections_by_server.size();
+    int lb = (n * 90) / 100;
+    int ub = (n * 110) / 100 + 1;
+
+    for (auto kv : connections_by_server)
+    {
+        bool acceptable = (kv.second >= lb && kv.second <= ub);
+
+        cout << kv.first << ": " << kv.second << " connections, which is ";
+        if (!acceptable)
+        {
+            cout << "NOT ";
+        }
+        cout << "within the accepted range [" << lb << ", " << ub << "]." << endl;
+
+        test.expect(acceptable,
+                    "%s has %d connections, accepted range: [%d, %d].",
+                    kv.first.c_str(), kv.second, lb, ub);
+    }
+}
+
+}
+
+int main(int argc, char* argv[])
+{
+    TestConnections test(argc, argv);
+
+    cout << "\nTesting RCR" << endl;
+    run_test(test, "RCR", create_rcr_connection);
+
+    cout << "\nTesting RWS" << endl;
+    run_test(test, "RWS", create_rws_connection);
+
+    return test.global_result;
+}

maxscale-system-test/cnf/maxscale.cnf.template.clustrix_distribution

@@ -0,0 +1,50 @@
+[maxscale]
+threads=###threads###
+log_info=1
+
+###clustrix_server###
+
+[Clustrix-Monitor]
+type=monitor
+module=clustrixmon
+servers=###clustrix_server_line###
+user=maxskysql
+password=skysql
+
+[RWS]
+type=service
+router=readwritesplit
+cluster=Clustrix-Monitor
+user=maxskysql
+password=skysql
+slave_selection_criteria=LEAST_GLOBAL_CONNECTIONS
+
+[RCR]
+type=service
+router=readconnroute
+#router_options=running
+cluster=Clustrix-Monitor
+user=maxskysql
+password=skysql
+
+[RWS-Listener]
+type=listener
+service=RWS
+protocol=MySQLClient
+port=4006
+
+[RCR-Listener]
+type=listener
+service=RCR
+protocol=MySQLClient
+port=4008
+
+[CLI]
+type=service
+router=cli
+
+[CLI-Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=default