MXS-1300: Add more MaxCtrl cluster tests

Added scripts that start two MaxScale on the same machine. This allows the
cluster synchronization commands to be tested locally.

CMakeLists.txt

@@ -129,6 +129,7 @@ configure_file(${CMAKE_SOURCE_DIR}/etc/postinst.in ${CMAKE_BINARY_DIR}/postinst
 configure_file(${CMAKE_SOURCE_DIR}/etc/postrm.in ${CMAKE_BINARY_DIR}/postrm @ONLY)
 configure_file(${CMAKE_SOURCE_DIR}/etc/upstart/maxscale.conf.in ${CMAKE_BINARY_DIR}/upstart/maxscale.conf @ONLY)
 configure_file(${CMAKE_SOURCE_DIR}/test/maxscale_test.cnf ${CMAKE_BINARY_DIR}/maxscale.cnf @ONLY)
+configure_file(${CMAKE_SOURCE_DIR}/test/maxscale_test_secondary.cnf ${CMAKE_BINARY_DIR}/maxscale_secondary.cnf @ONLY)
 
 set(FLAGS "-Wall -Wno-unused-variable -Wno-unused-function -Werror -fPIC" CACHE STRING "Compilation flags")
 set(DEBUG_FLAGS "-ggdb -pthread -pipe -Wformat -fstack-protector --param=ssp-buffer-size=4" CACHE STRING "Debug compilation flags")

maxctrl/test/cluster.js

@@ -1,7 +1,7 @@
 require('../test_utils.js')()
 var cluster = require('../lib/cluster.js')
 
-describe('Cluster Commands', function() {
+describe('Cluster Command Internals', function() {
     before(startMaxScale)
 
     it('detect added and removed objects', function() {
@@ -151,3 +151,79 @@ describe('Cluster Commands', function() {
 
     after(stopMaxScale)
 });
+
+
+describe('Cluster Commands', function() {
+    before(startDoubleMaxScale)
+
+    it('sync after server creation', function() {
+        return doCommand('create server server5 127.0.0.1 3003 --hosts 127.0.0.1:8990')
+            .then(() => verifyCommand('cluster sync 127.0.0.1:8990 --hosts 127.0.0.1:8989',
+                                      'servers/server5'))
+            .should.be.resolved
+    })
+
+    it('sync after server alteration', function() {
+        return doCommand('alter server server2 port 3000 --hosts 127.0.0.1:8990')
+            .then(() => verifyCommand('cluster sync 127.0.0.1:8990 --hosts 127.0.0.1:8989',
+                                      'servers/server2'))
+            .then(function(res) {
+                res.data.attributes.parameters.port.should.equal(3000)
+            })
+    })
+
+    it('sync after server deletion', function() {
+        return doCommand('destroy server server5 --hosts 127.0.0.1:8990')
+            .then(() => verifyCommand('cluster sync 127.0.0.1:8990 --hosts 127.0.0.1:8989',
+                                      'servers/server5'))
+            .should.be.rejected
+    })
+
+    it('sync after monitor creation', function() {
+        return doCommand('create monitor my-monitor-2 mysqlmon --hosts 127.0.0.1:8990')
+            .then(() => verifyCommand('cluster sync 127.0.0.1:8990 --hosts 127.0.0.1:8989',
+                                      'monitors/my-monitor2'))
+            .should.be.resolved
+    })
+
+    it('sync after monitor alteration', function() {
+        return doCommand('alter monitor MySQL-Monitor monitor_interval 12345 --hosts 127.0.0.1:8990')
+            .then(() => verifyCommand('cluster sync 127.0.0.1:8990 --hosts 127.0.0.1:8989',
+                                      'monitors/MySQL-Monitor'))
+            .then(function(res) {
+                res.data.attributes.parameters.monitor_interval.should.equal(12345)
+            })
+    })
+
+    it('sync after monitor deletion', function() {
+        return doCommand('destroy monitor my-monitor-2 --hosts 127.0.0.1:8990')
+            .then(() => verifyCommand('cluster sync 127.0.0.1:8990 --hosts 127.0.0.1:8989',
+                                      'monitors/my-monitor-2'))
+            .should.be.rejected
+    })
+
+    it('sync listener creation', function() {
+        return doCommand('create listener RW-Split-Router my-listener-2 5999 --hosts 127.0.0.1:8990')
+            .then(() => verifyCommand('cluster sync 127.0.0.1:8990 --hosts 127.0.0.1:8989',
+                                      'services/RW-Split-Router/listeners/my-listener-2'))
+            .should.be.resolved
+    })
+
+    it('sync after service alteration', function() {
+        return doCommand('alter service RW-Split-Router enable_root_user true --hosts 127.0.0.1:8990')
+            .then(() => verifyCommand('cluster sync 127.0.0.1:8990 --hosts 127.0.0.1:8989',
+                                      'services/RW-Split-Router'))
+            .then(function(res) {
+                res.data.attributes.parameters.enable_root_user.should.be.true
+            })
+    })
+
+    it('sync after listener deletion', function() {
+        return doCommand('destroy listener RW-Split-Router my-listener-2 --hosts 127.0.0.1:8990')
+            .then(() => verifyCommand('cluster sync 127.0.0.1:8990 --hosts 127.0.0.1:8989',
+                                      'services/RW-Split-Router/listeners/my-listener-2'))
+            .should.be.rejected
+    })
+
+    after(stopDoubleMaxScale)
+})

maxctrl/test_utils.js

@@ -28,6 +28,19 @@ module.exports = function() {
         })
     };
 
+    // Start two MaxScales
+    this.startDoubleMaxScale = function() {
+        return new Promise(function(resolve, reject) {
+            child_process.execFile("./start_double_maxscale.sh", function(err, stdout, stderr) {
+                if (err) {
+                    reject()
+                } else {
+                    resolve()
+                }
+            })
+        })
+    };
+
     // Stop MaxScale, this should be called in the `after` handler of each test unit
     this.stopMaxScale = function() {
         return new Promise(function(resolve, reject) {
@@ -41,6 +54,19 @@ module.exports = function() {
         })
     };
 
+    // Stop two MaxScales
+    this.stopDoubleMaxScale = function() {
+        return new Promise(function(resolve, reject) {
+            child_process.execFile("./stop_double_maxscale.sh", function(err, stdout, stderr) {
+                if (err) {
+                    reject()
+                } else {
+                    resolve()
+                }
+            })
+        })
+    };
+
     // Execute a single MaxCtrl command, returns a Promise
     this.doCommand = function(command) {
         var ctrl = require('./lib/core.js')

test/maxscale_test.cnf

@@ -68,10 +68,6 @@ type=filter
 module=tee
 service=RW Split Hint Router
 
-[Debug Interface]
-type=service
-router=debugcli
-
 [CLI]
 type=service
 router=cli
@@ -100,12 +96,6 @@ service=RW Split Hint Router
 protocol=MySQLClient
 port=4009
 
-[Debug Listener]
-type=listener
-service=Debug Interface
-protocol=telnetd
-port=4442
-
 [CLI Listener]
 type=listener
 service=CLI

test/maxscale_test_secondary.cnf

@@ -0,0 +1,129 @@
+[maxscale]
+threads=4
+log_info=1
+libdir=@CMAKE_INSTALL_PREFIX@/@MAXSCALE_LIBDIR@
+logdir=@CMAKE_INSTALL_PREFIX@/log/maxscale/
+datadir=@CMAKE_INSTALL_PREFIX@/lib/maxscale
+cachedir=@CMAKE_INSTALL_PREFIX@/cache/maxscale
+language=@CMAKE_INSTALL_PREFIX@/lib/maxscale/
+piddir=/tmp/
+admin_auth=false
+admin_port=8990
+
+[MySQL Monitor]
+type=monitor
+module=mysqlmon
+servers=server1,server2,server3,server4
+user=maxuser
+passwd=maxpwd
+monitor_interval=10000
+
+[RW Split Router]
+type=service
+router=readwritesplit
+servers=server1,server2,server3,server4
+user=maxuser
+passwd=maxpwd
+max_slave_connections=100%
+
+[SchemaRouter Router]
+type=service
+router=schemarouter
+servers=server1,server2,server3,server4
+user=maxuser
+passwd=maxpwd
+auth_all_servers=1
+
+[RW Split Hint Router]
+type=service
+router=readwritesplit
+servers=server1,server2,server3,server4
+user=maxuser
+passwd=maxpwd
+max_slave_connections=100%
+filters=Hint
+
+[Read Connection Router]
+type=service
+router=readconnroute
+router_options=master
+servers=server1
+user=maxuser
+passwd=maxpwd
+
+[Hint]
+type=filter
+module=hintfilter
+
+[recurse3]
+type=filter
+module=tee
+service=RW Split Router
+
+[recurse2]
+type=filter
+module=tee
+service=Read Connection Router
+
+[recurse1]
+type=filter
+module=tee
+service=RW Split Hint Router
+
+[CLI]
+type=service
+router=cli
+
+[Read Connection Listener]
+type=listener
+service=Read Connection Router
+protocol=MySQLClient
+port=5008
+
+[RW Split Listener]
+type=listener
+service=RW Split Router
+protocol=MySQLClient
+port=5006
+
+[SchemaRouter Listener]
+type=listener
+service=SchemaRouter Router
+protocol=MySQLClient
+port=5010
+
+[RW Split Hint Listener]
+type=listener
+service=RW Split Hint Router
+protocol=MySQLClient
+port=5009
+
+[CLI Listener]
+type=listener
+service=CLI
+protocol=maxscaled
+socket=/tmp/maxadmin2.sock
+
+[server1]
+type=server
+address=127.0.0.1
+port=3000
+protocol=MySQLBackend
+
+[server2]
+type=server
+address=127.0.0.1
+port=3001
+protocol=MySQLBackend
+
+[server3]
+type=server
+address=127.0.0.1
+port=3002
+protocol=MySQLBackend
+
+[server4]
+type=server
+address=127.0.0.1
+port=3003
+protocol=MySQLBackend

test/start_double_maxscale.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+#
+# This script is run before each test block. It starts two MaxScales and waits
+# for them to become responsive.
+#
+
+maxscaledir=$MAXSCALE_DIR
+
+test -z "$MAXSCALE_DIR" && exit 1
+
+# Start MaxScale
+$maxscaledir/bin/maxscale -lstdout -df $maxscaledir/maxscale.cnf >& $maxscaledir/maxscale1.output &
+
+# Wait for the first MaxScale to start
+for ((i=0;i<60;i++))
+do
+    $maxscaledir/bin/maxadmin help >& /dev/null && break
+    sleep 0.1
+done
+
+# Start a second maxscale
+$maxscaledir/bin/maxscale -lstdout -df $maxscaledir/maxscale_secondary.cnf >& $maxscaledir/maxscale2.output &
+
+# Wait for the second MaxScale to start
+for ((i=0;i<60;i++))
+do
+    $maxscaledir/bin/maxadmin -S /tmp/maxadmin2.sock help >& /dev/null && break
+    sleep 0.1
+done
+
+# Give MaxScale some time to settle
+sleep 1

test/stop_double_maxscale.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+#
+# This script is run after each test block. It kills the two MaxScale processes
+# and cleans up the directories that contain generated files.
+#
+
+maxscaledir=$MAXSCALE_DIR
+
+test -z "$MAXSCALE_DIR" && exit 1
+
+for ((i=0;i<20;i++))
+do
+    pkill '^maxscale$' || break
+    sleep 0.5
+done
+
+# If it wasn't dead before, now it is
+pkill -9 '^maxscale$'
+
+rm -r $maxscaledir/lib/maxscale
+rm -r $maxscaledir/cache/maxscale
+rm -r $maxscaledir/run/maxscale
+test -f /tmp/maxadmin.sock && rm /tmp/maxadmin.sock
+test -f /tmp/maxadmin2.sock && rm /tmp/maxadmin2.sock
+
+mkdir -m 0755 -p $maxscaledir/lib/maxscale
+mkdir -m 0755 -p $maxscaledir/cache/maxscale
+mkdir -m 0755 -p $maxscaledir/run/maxscale