Merge branch 'blr' of https://github.com/skysql/MaxScale into blr

CMakeLists.txt

@@ -16,9 +16,9 @@ check_dirs()
 set(CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_RPATH}:${CMAKE_INSTALL_PREFIX}/lib:${CMAKE_INSTALL_PREFIX}/modules)
 
 configure_file(${CMAKE_SOURCE_DIR}/server/include/version.h.in ${CMAKE_SOURCE_DIR}/server/include/version.h)
-configure_file(${CMAKE_SOURCE_DIR}/maxscale.conf.in ${CMAKE_SOURCE_DIR}/maxscale.conf @ONLY)
+configure_file(${CMAKE_SOURCE_DIR}/maxscale.conf.in ${CMAKE_SOURCE_DIR}/maxscale.conf.prep @ONLY)
-configure_file(${CMAKE_SOURCE_DIR}/etc/init.d/maxscale.in ${CMAKE_SOURCE_DIR}/etc/init.d/maxscale @ONLY)
+configure_file(${CMAKE_SOURCE_DIR}/etc/init.d/maxscale.in ${CMAKE_SOURCE_DIR}/etc/init.d/maxscale.prep @ONLY)
-configure_file(${CMAKE_SOURCE_DIR}/etc/ubuntu/init.d/maxscale.in ${CMAKE_SOURCE_DIR}/etc/ubuntu/init.d/maxscale @ONLY)
+configure_file(${CMAKE_SOURCE_DIR}/etc/ubuntu/init.d/maxscale.in ${CMAKE_SOURCE_DIR}/etc/ubuntu/init.d/maxscale.prep @ONLY)
 
 
 set(CMAKE_C_FLAGS "-Wall -fPIC")
@@ -55,9 +55,11 @@ if(GCOV)
 endif()
 
 
-include_directories(${MYSQL_DIR})
+subdirs(MYSQL_DIR_ALL ${MYSQL_DIR})
-include_directories(${MYSQL_DIR}/private)
+foreach(DIR ${MYSQL_DIR_ALL})
-include_directories(${MYSQL_DIR}/extra)
+  include_directories(${DIR})
+endforeach()
+
 include_directories(${MYSQL_DIR}/..)
 include_directories(utils)
 include_directories(log_manager)
@@ -75,11 +77,12 @@ add_subdirectory(client)
 
 # Install startup scripts and ldconfig files
 if( NOT ( (DEFINED INSTALL_SYSTEM_FILES) AND ( NOT ( INSTALL_SYSTEM_FILES ) ) ) )
-  install(FILES maxscale.conf DESTINATION /etc/ld.so.conf.d/ PERMISSIONS WORLD_EXECUTE WORLD_READ)
+
+  install(FILES maxscale.conf.prep RENAME maxscale.conf DESTINATION /etc/ld.so.conf.d/ PERMISSIONS WORLD_EXECUTE WORLD_READ)
   if(DEB_BASED)
-    install(FILES etc/ubuntu/init.d/maxscale DESTINATION /etc/init.d/ PERMISSIONS WORLD_EXECUTE)
+    install(FILES etc/ubuntu/init.d/maxscale.prep RENAME maxscale  DESTINATION /etc/init.d/ PERMISSIONS WORLD_EXECUTE)
   else()
-    install(FILES etc/init.d/maxscale DESTINATION /etc/init.d/ PERMISSIONS WORLD_EXECUTE)
+    install(FILES etc/init.d/maxscale.prep RENAME maxscale DESTINATION /etc/init.d/ PERMISSIONS WORLD_EXECUTE)
   endif()
   message(STATUS "Installing maxscale.conf to: /etc/ld.so.conf.d")
   message(STATUS "Installing startup scripts to: /etc/init.d")

VERSION

@@ -1 +1 @@
-1.0.0-beta
+1.0.1-beta

client/test/maxadmin_stress.sh

@@ -0,0 +1,53 @@
+#!/bin/sh
+failure=0
+passed=0
+
+clients=20
+cmdcnt=1000
+
+echo Running $clients parallel iterations of $cmdcnt commands
+
+for ((cnt=0; cnt<$clients; cnt++ )); do
+	for ((i=0; i<$cmdcnt; i++ )); do
+		maxadmin -pskysql show services;
+	done >/dev/null &
+done >& /dev/null
+
+peak=0
+while [ "`jobs -p`" != "" ]; do
+	jobs >& /dev/null
+	zombies=`maxadmin -pskysql list dcbs | grep -ci zombies`
+	if [ $zombies -gt $peak ] ; then
+		peak=$zombies
+	fi
+	sleep 1
+done
+if [ $peak -gt 10 ]; then
+	echo "High peak zombie count ($peak):			Failed"
+	failure=`expr $failure + 1`
+else
+	passed=`expr $passed + 1`
+	echo "Zombie collection ($peak):			Passed"
+fi
+zombies=`maxadmin -pskysql list dcbs | grep -ci zombies`
+if [ $zombies != "0" ]; then
+	echo "Residual zombie DCBs:			Failed"
+	failure=`expr $failure + 1`
+else
+	passed=`expr $passed + 1`
+	echo "Residual zombie DCBs:			Passed"
+fi
+sessions=`maxadmin -pskysql list services | awk -F\| '/ cli/ { print $3 }'`
+if [ $sessions -gt 3 ]; then
+	echo "Session shutdown, $sessions:			Failed"
+	failure=`expr $failure + 1`
+else
+	passed=`expr $passed + 1`
+	echo "Session shutdown:			Passed"
+fi
+
+sessions=`maxadmin -pskysql list services | awk -F\| '/ cli/ { print $4 }'`
+
+echo "Test run complete. $passed passes, $failure failures"
+echo "$sessions CLI sessions executed"
+exit $failure

macros.cmake

@@ -3,7 +3,8 @@ macro(set_maxscale_version)
   #MaxScale version number
   set(MAXSCALE_VERSION_MAJOR "1")
   set(MAXSCALE_VERSION_MINOR "0")
-  set(MAXSCALE_VERSION_PATCH "0")
+  set(MAXSCALE_VERSION_PATCH "1")
+  set(MAXSCALE_VERSION_NUMERIC "${MAXSCALE_VERSION_MAJOR}.${MAXSCALE_VERSION_MINOR}.${MAXSCALE_VERSION_PATCH}")
   set(MAXSCALE_VERSION "${MAXSCALE_VERSION_MAJOR}.${MAXSCALE_VERSION_MINOR}.${MAXSCALE_VERSION_PATCH}-beta")
 
 endmacro()
@@ -208,3 +209,19 @@ macro(check_dirs)
   endif()
 
 endmacro()
+
+function(subdirs VAR DIRPATH)
+
+if(${CMAKE_VERSION} VERSION_LESS 2.12 )
+set(COMP_VAR PATH)
+else()
+set(COMP_VAR DIRECTORY)
+endif()
+  file(GLOB_RECURSE SDIR ${DIRPATH}/*)
+  foreach(LOOP ${SDIR})
+	get_filename_component(LOOP ${LOOP} ${COMP_VAR})
+	list(APPEND ALLDIRS ${LOOP})
+  endforeach()
+  list(REMOVE_DUPLICATES ALLDIRS)
+ set(${VAR} "${ALLDIRS}" CACHE PATH " " FORCE)
+endfunction()

server/core/gateway.c

@@ -1646,6 +1646,7 @@ void
         shutdown_server()
 {
         poll_shutdown();
+	hkshutdown();
         log_flush_shutdown();
 }
 

server/core/housekeeper.c

@@ -42,6 +42,8 @@ static HKTASK	*tasks = NULL;
  */
 static SPINLOCK	tasklock = SPINLOCK_INIT;
 
+static int	do_shutdown = 0;
+
 static	void	hkthread(void *);
 
 /**
@@ -172,6 +174,8 @@ void	*taskdata;
 
 	for (;;)
 	{
+		if (do_shutdown)
+			return;
 		thread_millisleep(1000);
 		now = time(0);
 		spinlock_acquire(&tasklock);
@@ -194,3 +198,13 @@ void	*taskdata;
 		spinlock_release(&tasklock);
 	}
 }
+
+/**
+ * Called to shutdown the housekeeper
+ *
+ */
+void
+hkshutdown()
+{
+	do_shutdown = 1;
+}

server/core/monitor.c

@@ -234,17 +234,17 @@ MONITOR	*ptr;
 
 	spinlock_acquire(&monLock);
 	ptr = allMonitors;
-	dcb_printf(dcb, "+----------------------+---------------------\n");
+	dcb_printf(dcb, "---------------------+---------------------\n");
-	dcb_printf(dcb, "| %-20s | Status\n", "Monitor");
+	dcb_printf(dcb, "%-20s | Status\n", "Monitor");
-	dcb_printf(dcb, "+----------------------+---------------------\n");
+	dcb_printf(dcb, "---------------------+---------------------\n");
 	while (ptr)
 	{
-		dcb_printf(dcb, "| %-20s | %s\n", ptr->name,
+		dcb_printf(dcb, "%-20s | %s\n", ptr->name,
 			ptr->state & MONITOR_STATE_RUNNING
 					? "Running" : "Stopped");
 		ptr = ptr->next;
 	}
-	dcb_printf(dcb, "+----------------------+---------------------\n");
+	dcb_printf(dcb, "---------------------+---------------------\n");
 	spinlock_release(&monLock);
 }
 

server/core/poll.c

@@ -362,7 +362,7 @@ return_rc:
  * The events are now recieved via the epoll_wait call, a queue of DCB's that have
  * events pending is maintained and as new events arrive the DCB is added to the end
  * of this queue. If an eent arrives for a DCB alreayd in the queue, then the event
- * bits are added to the DCB but the DCB mantains the same point inthe queue unless
+ * bits are added to the DCB but the DCB mantains the same point in the queue unless
  * the original events are already being processed. If they are being processed then
  * the DCB is moved to the back of the queue, this means that a DCB that is receiving
  * events at a high rate will not block the execution of events for other DCB's and
@@ -391,7 +391,7 @@ DCB                *zombies = NULL;
 	while (1)
 	{
 		/* Process of the queue of waiting requests */
-		while (process_pollq(thread_id))
+		while (do_shutdown == 0 && process_pollq(thread_id))
 		{
 			if (thread_data)
 				thread_data[thread_id].state = THREAD_ZPROCESSING;
@@ -885,6 +885,20 @@ poll_bitmask()
 	return &poll_mask;
 }
 
+/**
+ * Display an entry from the spinlock statistics data
+ *
+ * @param       dcb     The DCB to print to
+ * @param       desc    Description of the statistic
+ * @param       value   The statistic value
+ */
+static void
+spin_reporter(void *dcb, char *desc, int value)
+{
+	dcb_printf((DCB *)dcb, "\t%-40s  %d\n", desc, value);
+}
+
+
 /**
  * Debug routine to print the polling statistics
  *
@@ -922,6 +936,11 @@ int	i;
 	}
 	dcb_printf(dcb, "\t>= %d\t\t\t%d\n", MAXNFDS,
 					pollStats.n_fds[MAXNFDS-1]);
+
+#if SPINLOCK_PROFILE
+	dcb_printf(dcb, "Event queue lock statistics:\n");
+	spinlock_stats(&pollqlock, spin_reporter, dcb);
+#endif
 }
 
 /**

server/include/housekeeper.h

@@ -47,4 +47,5 @@ typedef struct hktask {
 extern void hkinit();
 extern int  hktask_add(char *name, void (*task)(void *), void *data, int frequency);
 extern int  hktask_remove(char *name);
+extern void hkshutdown();
 #endif

server/include/spinlock.h

@@ -31,7 +31,7 @@
 #include <thread.h>
 #include <stdbool.h>
 
-#define	SPINLOCK_PROFILE		0
+#define	SPINLOCK_PROFILE		1
 
 /**
  * The spinlock structure.

server/modules/include/blr.h

@@ -43,8 +43,8 @@
  * High and Low water marks for the slave dcb. These values can be overriden
  * by the router options highwater and lowwater.
  */
-#define DEF_LOW_WATER		20000
+#define DEF_LOW_WATER		2000
-#define	DEF_HIGH_WATER		300000
+#define	DEF_HIGH_WATER		30000
 
 /**
  * Some useful macros for examining the MySQL Response packets

server/modules/routing/CMakeLists.txt

@@ -15,4 +15,4 @@ target_link_libraries(cli log_manager utils)
 install(TARGETS cli DESTINATION modules)
 
 add_subdirectory(readwritesplit)
-add_subdirectory(binlog)
+

server/modules/routing/Makefile

@@ -47,12 +47,12 @@ CLIOBJ=$(CLISRCS:.c=.o)
 SRCS=$(TESTSRCS) $(READCONSRCS) $(DEBUGCLISRCS) cli.c
 OBJ=$(SRCS:.c=.o)
 LIBS=$(UTILSPATH)/skygw_utils.o -lssl -llog_manager
-MODULES= libdebugcli.so libreadconnroute.so libtestroute.so libcli.so \
+MODULES= libdebugcli.so libreadconnroute.so libtestroute.so libcli.so libbinlogrouter.so
-	libbinlogrouter.so
 
 
 all:	$(MODULES)
 	(cd readwritesplit; make)
+	(cd binlog; make)
 
 libtestroute.so: $(TESTOBJ)
 	$(CC) $(LDFLAGS) $(TESTOBJ) $(LIBS) -o $@
@@ -71,6 +71,7 @@ libreadwritesplit.so:
 
 libbinlogrouter.so:
 	(cd binlog; touch depend.mk ; make; cp $@ ..)
+
 .c.o:
 	$(CC) $(CFLAGS) $< -o $@
 
@@ -82,7 +83,6 @@ clean:
 tags:
 	ctags $(SRCS) $(HDRS)
 	(cd readwritesplit; make tags)
-	(cd binlog; make tags)
 
 depend:
 	@$(DEL) depend.mk
@@ -93,7 +93,6 @@ depend:
 install: $(MODULES)
 	install -D $(MODULES) $(DEST)/modules
 	(cd readwritesplit; make DEST=$(DEST) install)
-	(cd binlog; make DEST=$(DEST) install)
 
 cleantests:
 	$(MAKE) -C test cleantests

server/modules/routing/binlog/blr_master.c

@@ -55,21 +55,57 @@
 #include <skygw_utils.h>
 #include <log_manager.h>
 
+#include <rdtsc.h>
+
 /* Temporary requirement for auth data */
 #include <mysql_client_server_protocol.h>
 
+#define	SAMPLE_COUNT	10000
+CYCLES	samples[10][SAMPLE_COUNT];
+int	sample_index[10] = { 0, 0, 0 };
+
+#define	LOGD_SLAVE_CATCHUP1	0
+#define	LOGD_SLAVE_CATCHUP2	1
+#define	LOGD_DISTRIBUTE		2
+#define	LOGD_FILE_FLUSH		3
+
+SPINLOCK logspin = SPINLOCK_INIT;
+
+void
+log_duration(int sample, CYCLES duration)
+{
+char	fname[100];
+int	i;
+FILE	*fp;
+
+	spinlock_acquire(&logspin);
+	samples[sample][sample_index[sample]++] = duration;
+	if (sample_index[sample] == SAMPLE_COUNT)
+	{
+		sprintf(fname, "binlog_profile.%d", sample);
+		if ((fp = fopen(fname, "a")) != NULL)
+		{
+			for (i = 0; i < SAMPLE_COUNT; i++)
+				fprintf(fp, "%ld\n", samples[sample][i]);
+			fclose(fp);
+		}
+		sample_index[sample] = 0;
+	}
+	spinlock_release(&logspin);
+}
+
 extern int lm_enabled_logfiles_bitmask;
 
 static GWBUF *blr_make_query(char *statement);
 static GWBUF *blr_make_registration(ROUTER_INSTANCE *router);
 static GWBUF *blr_make_binlog_dump(ROUTER_INSTANCE *router);
-static void encode_value(unsigned char *data, unsigned int value, int len);
+void encode_value(unsigned char *data, unsigned int value, int len);
-static void blr_handle_binlog_record(ROUTER_INSTANCE *router, GWBUF *pkt);
+void blr_handle_binlog_record(ROUTER_INSTANCE *router, GWBUF *pkt);
 static void blr_rotate_event(ROUTER_INSTANCE *router, uint8_t *pkt, REP_HEADER *hdr);
-static void blr_distribute_binlog_record(ROUTER_INSTANCE *router, REP_HEADER *hdr, uint8_t *ptr);
+void blr_distribute_binlog_record(ROUTER_INSTANCE *router, REP_HEADER *hdr, uint8_t *ptr);
 static void *CreateMySQLAuthData(char *username, char *password, char *database);
-static void blr_extract_header(uint8_t *pkt, REP_HEADER *hdr);
+void blr_extract_header(uint8_t *pkt, REP_HEADER *hdr);
-static uint32_t extract_field(uint8_t *src, int bits);
+inline uint32_t extract_field(uint8_t *src, int bits);
 static void blr_log_packet(logfile_id_t file, char *msg, uint8_t *ptr, int len);
 
 static int keepalive = 1;
@@ -460,7 +496,7 @@ int		len = 0x1b;
  * @param	value	The value to pack
  * @param	len	Number of bits to encode value into
  */
-static void
+void
 encode_value(unsigned char *data, unsigned int value, int len)
 {
 	while (len > 0)
@@ -478,7 +514,7 @@ encode_value(unsigned char *data, unsigned int value, int len)
  * @param router	The router instance
  * @param pkt		The binlog records
  */
-static void
+void
 blr_handle_binlog_record(ROUTER_INSTANCE *router, GWBUF *pkt)
 {
 uint8_t			*msg = NULL, *ptr, *pdata;
@@ -766,7 +802,9 @@ static REP_HEADER	phdr;
 	{
 		ss_dassert(pkt_length == 0);
 	}
+{ CYCLES start = rdtsc(); 
 	blr_file_flush(router);
+log_duration(LOGD_FILE_FLUSH, rdtsc() - start); }
 }
 
 /**
@@ -775,7 +813,7 @@ static REP_HEADER	phdr;
  * @param pkt	The incoming packet in a GWBUF chain
  * @param hdr	The packet header to populate
  */
-static void
+void
 blr_extract_header(uint8_t *ptr, REP_HEADER *hdr)
 {
 
@@ -796,10 +834,10 @@ blr_extract_header(uint8_t *ptr, REP_HEADER *hdr)
  * @param src	The raw packet source
  * @param birs	The number of bits to extract (multiple of 8)
  */
-static uint32_t
+inline uint32_t
-extract_field(uint8_t *src, int bits)
+extract_field(register uint8_t *src, int bits)
 {
-uint32_t	rval = 0, shift = 0;
+register uint32_t	rval = 0, shift = 0;
 
 	while (bits > 0)
 	{
@@ -884,14 +922,16 @@ MYSQL_session	*auth_info;
  * @param	hdr		The replication event header
  * @param	ptr		The raw replication event data
  */
-static void
+void
 blr_distribute_binlog_record(ROUTER_INSTANCE *router, REP_HEADER *hdr, uint8_t *ptr)
 {
 GWBUF		*pkt;
 uint8_t		*buf;
 ROUTER_SLAVE	*slave;
 int		action;
+CYCLES		entry;
 
+	entry = rdtsc();
 	spinlock_acquire(&router->lock);
 	slave = router->slaves;
 	while (slave)
@@ -945,12 +985,16 @@ int		action;
 				spinlock_acquire(&slave->catch_lock);
 				if (slave->overrun)
 				{
+CYCLES	cycle_start, cycles;
 					slave->stats.n_overrun++;
 					slave->overrun = 0;
 					spinlock_release(&router->lock);
 					slave->cstate &= ~(CS_UPTODATE|CS_DIST);
 					spinlock_release(&slave->catch_lock);
+cycle_start = rdtsc();
 					blr_slave_catchup(router, slave);
+cycles = rdtsc() - cycle_start;
+log_duration(LOGD_SLAVE_CATCHUP2, cycles);
 					spinlock_acquire(&router->lock);
 					slave = router->slaves;
 					if (slave)
@@ -983,6 +1027,7 @@ int		action;
 				 */
 				if (slave->cstate & CS_UPTODATE)
 				{
+CYCLES	cycle_start, cycles;
 					spinlock_release(&router->lock);
 					LOGIF(LD, (skygw_log_write_flush(LOGFILE_DEBUG,
 						"Force slave %d into catchup mode %s@%d\n",
@@ -991,7 +1036,10 @@ int		action;
 					spinlock_acquire(&slave->catch_lock);
 					slave->cstate &= ~(CS_UPTODATE|CS_DIST);
 					spinlock_release(&slave->catch_lock);
+cycle_start = rdtsc();
 					blr_slave_catchup(router, slave);
+cycles = rdtsc() - cycle_start;
+log_duration(LOGD_SLAVE_CATCHUP1, cycles);
 					spinlock_acquire(&router->lock);
 					slave = router->slaves;
 					if (slave)
@@ -1005,6 +1053,7 @@ int		action;
 		slave = slave->next;
 	}
 	spinlock_release(&router->lock);
+	log_duration(LOGD_DISTRIBUTE, rdtsc() - entry);
 }
 
 static void