diff --git a/CMakeLists.txt b/CMakeLists.txt
index 743c52736..a6acec667 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -185,9 +185,7 @@ add_custom_target(testall
   COMMAND ${CMAKE_COMMAND} -DDEPS_OK=Y -DBUILD_TESTS=Y -DBUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR} -DINSTALL_SYSTEM_FILES=N ${CMAKE_SOURCE_DIR}
   COMMAND make install
   COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_SOURCE_DIR}/server/test/MaxScale_test.cnf ${CMAKE_BINARY_DIR}/etc/MaxScale.cnf
-  COMMAND /bin/sh -c "${CMAKE_BINARY_DIR}/bin/maxscale -c ${CMAKE_BINARY_DIR} &>/dev/null"
-  COMMAND /bin/sh -c "make test || echo \"Test results written to: ${CMAKE_BINARY_DIR}/Testing/Temporary/\""
-  COMMAND killall maxscale
+  COMMAND ${CMAKE_COMMAND} -P ${CMAKE_SOURCE_DIR}/testall.cmake
   COMMENT "Running full test suite..." VERBATIM)
 
 # uninstall target
diff --git a/testall.cmake b/testall.cmake
new file mode 100644
index 000000000..2c6868faa
--- /dev/null
+++ b/testall.cmake
@@ -0,0 +1,8 @@
+execute_process(COMMAND /bin/sh -c "${CMAKE_BINARY_DIR}/bin/maxscale -c ${CMAKE_BINARY_DIR} &>/dev/null")
+execute_process(COMMAND make test RESULT_VARIABLE RVAL)
+execute_process(COMMAND killall maxscale)
+if(NOT RVAL EQUAL 0)
+  message(FATAL_ERROR "Test suite failed with status: ${RVAL}")
+else()
+  message(STATUS "Test exited with status: ${RVAL}")
+endif()