filter testing harness

server/modules/filter/test/Makefile

@@ -0,0 +1,77 @@
+# This file is distributed as part of MaxScale form SkySQL.  It is free
+# software: you can redistribute it and/or modify it under the terms of the
+# GNU General Public License as published by the Free Software Foundation,
+# version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright SkySQL Ab 2014
+
+include ../../../../build_gateway.inc
+
+LOGPATH 	:= $(ROOT_PATH)/log_manager
+UTILSPATH 	:= $(ROOT_PATH)/utils
+QCLASSPATH 	:= $(ROOT_PATH)/query_classifier
+COREPATH 	:= $(ROOT_PATH)/server/core
+
+CC=cc
+CFLAGS=-c -fPIC -I/usr/include -I../../include -I../../../include \
+	-I$(LOGPATH) -I$(UTILSPATH) -I./ -I$(ROOT_PATH)/server/inih -I$(QCLASSPATH) \
+	$(MYSQL_HEADERS) -Wall -g
+
+include ../../../../makefile.inc
+
+LDFLAGS=-rdynamic -L$(LOGPATH) -L$(UTILSPATH) -L$(EMBEDDED_LIB) \
+	-Wl,-rpath,$(DEST)/lib \
+	-Wl,-rpath,$(LOGPATH) -Wl,-rpath,$(UTILSPATH) \
+	-Wl,-rpath,$(EMBEDDED_LIB)
+
+SRCS=harness.c 
+OBJ=$(SRCS:.c=.o)
+COREOBJ=$(COREPATH)/load_utils.o $(COREPATH)/dcb.o $(COREPATH)/utils.o \
+	$(COREPATH)/gw_utils.o $(COREPATH)/buffer.o $(COREPATH)/poll.o \
+	$(COREPATH)/spinlock.o	$(COREPATH)/gwbitmask.o $(COREPATH)/session.o \
+	$(COREPATH)/atomic.o $(COREPATH)/hashtable.o $(COREPATH)/filter.o $(COREPATH)/modutil.o $(ROOT_PATH)/server/inih/ini.o \
+	$(COREPATH)/hint.o
+LIBS= $(UTILSPATH)/skygw_utils.o -lssl -pthread -llog_manager -lmysqld -ldl -lcrypto
+MODULES := $(wildcard ../*.so)
+
+all: build
+
+build:$(OBJ)
+	$(CC) $(OBJ) $(COREOBJ) $(LDFLAGS) $(LIBS) -o harness
+	$(MAKE) -C ../
+	cp ../*.so ./
+
+%.o: %.c
+	$(CC) $(CFLAGS) $< -o $@
+
+clean:
+	-rm -f *.o
+	-rm -f *.so
+	-rm -f harness
+cleantests:clean
+	-rm *.output
+buildtests:build
+
+testall: 
+	$(MAKE) cleantests
+	$(MAKE) buildtests
+	$(MAKE) runtests
+
+runtests:
+	@echo ""
+	@echo "-------------------------------"
+	@echo "$(shell date)"
+	@echo "Test Filter harness"
+	@echo "-------------------------------"
+	@echo "Testing hints... "
+	@./hint_tests.sh
+	@echo ""

server/modules/filter/test/README

@@ -0,0 +1,20 @@
+Filter Test Harness
+
+For a more detailed description of the filter harness, either generate the documentation or read the harness.h file.
+
+Running the program without arguments enters the interactive mode. Type 'help' for a list of all commands.
+
+The default values for threads and sessions are stored in the 'harness.cnf' file
+
+Mandatory parameters for the command line mode are -c and -i.
+
+Parameters for the command line:
+
+	-h	Display this information
+	-c	Path to the MaxScale configuration file to parse for filters
+	-i	Name of the input file for buffers
+	-o	Name of the output file for results
+	-q	Suppress printing to stdout
+	-t	Number of threads
+	-s	Number of sessions
+	-d	Routing delay

server/modules/filter/test/harness.cnf

@@ -0,0 +1,2 @@
+threads=2
+sessions=4

server/modules/filter/test/harness.h

@@ -0,0 +1,173 @@
+#ifndef _FILTER_HARNESS_H
+#define _FILTER_HARNESS_H
+/*
+ * This file is distributed as part of the SkySQL Gateway.  It is free
+ * software: you can redistribute it and/or modify it under the terms of the
+ * GNU General Public License as published by the Free Software Foundation,
+ * version 2.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 51
+ * Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Copyright SkySQL Ab 2013
+ */
+
+/**
+ * @file harness.h Test harness for independent testing of filters
+ *
+ * A test harness that feeds a GWBUF to a chain of filters and prints the results
+ * either into a file or to the standard output. 
+ *
+ * The contents of the GWBUF and the filter parameters are either manually set through
+ * the command line or read from a file.
+ *
+ * @verbatim
+ * Revision History
+ *
+ * Date		Who			Description
+ * 01/07/14	Markus Makela		Initial implementation
+ *
+ * @endverbatim
+ */
+
+#include <unistd.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <filter.h>
+#include <buffer.h>
+#include <modules.h>
+#include <modutil.h>
+#include <skygw_utils.h>
+#include <log_manager.h>
+#include <atomic.h>
+#include <ini.h>
+#include <hint.h>
+
+/**
+ * A single name-value pair and a link to the next item in the 
+ * configuration.
+ */
+typedef struct CONFIG_ITEM_T
+{
+  char* name;
+  char* value;
+  struct CONFIG_ITEM_T* next;
+}CONFIG_ITEM;
+
+/**
+ *A simplified version of a MaxScale configuration context used to load filters
+ * and their options.
+ */
+typedef struct CONFIG_T
+{
+  char* section;
+  CONFIG_ITEM* item;
+  struct CONFIG_T* next;
+  
+}CONFIG;
+
+/**
+ *A structure that holds all the necessary information to emulate a working
+ * filter environment.
+ */
+struct FILTERCHAIN_T
+{
+  FILTER* filter; /**An instance of a particular filter*/
+  FILTER_OBJECT* instance; /**Dynamically loaded module*/
+  SESSION** session; /**A list of sessions*/
+  DOWNSTREAM** down; /** A list of next filters downstreams*/
+  UPSTREAM** up; /** A list of next filters upstreams*/
+  char* name; /**Module name*/
+  struct FILTERCHAIN_T* next;
+};
+
+typedef struct FILTERCHAIN_T FILTERCHAIN;
+
+/**
+ * A container for all the filters, query buffers and user specified parameters
+ */
+typedef struct
+{
+  int running;
+  int verbose; /**Whether to print to stdout*/
+  int infile; /**A file where the queries are loaded from*/
+  char* infile_name;
+  int outfile; /**A file where the output of the filters is logged*/
+  char* outfile_name;
+  FILTERCHAIN* head; /**The head of the filter chain*/
+  FILTERCHAIN* tail; /**The tail of the filter chain*/
+  GWBUF** buffer; /**Buffers that are fed to the filter chain*/
+  int buffer_count;
+  int session_count;
+  DOWNSTREAM dummyrouter; /**Dummy downstream router for data extraction*/
+  UPSTREAM dummyclient; /**Dummy downstream router for data extraction*/
+  CONFIG* conf; /**Configurations loaded from a file*/
+  pthread_mutex_t work_mtx; /**Mutex for buffer routing*/
+  int buff_ind; /**Index of first unrouted buffer*/
+  int sess_ind;/**Index of first unused session*/
+  int last_ind; /**Index of last used session*/
+  pthread_t* thrpool;
+  int thrcount; /**Number of active threads*/
+  int rt_delay; /**Delay each thread waits after routing a query, in milliseconds*/
+}HARNESS_INSTANCE;
+
+static HARNESS_INSTANCE instance;
+
+/**
+ *A list of available actions.
+ */
+
+typedef enum 
+  {
+    UNDEFINED,
+    RUNFILTERS,
+    LOAD_FILTER,
+    DELETE_FILTER,
+    LOAD_CONFIG,
+    SET_INFILE,
+    SET_OUTFILE,
+    THR_COUNT,
+    SESS_COUNT,
+    OK,
+    QUIT
+  } operation_t;
+
+typedef enum
+  {
+    PACKET_OK,
+    PACKET_ERROR,
+    PACKET_RESULT_SET
+  } packet_t;
+
+typedef packet_t PACKET;
+
+void free_buffers();
+void free_filters();
+operation_t user_input(char*);
+void print_help();
+void print_status();
+int open_file(char* str, unsigned int write);
+FILTER_PARAMETER** read_params(int*);
+int routeQuery(void* instance, void* session, GWBUF* queue);
+void manual_query();
+int load_query();
+static int handler(void* user, const char* section, const char* name,const char* value);
+CONFIG* process_config(CONFIG*);
+FILTERCHAIN* load_filter_module(char* str);
+int load_filter(FILTERCHAIN*, CONFIG*);
+int load_config(char* fname);
+void route_buffers();
+void work_buffer(void*);
+GWBUF* gen_packet(PACKET);
+int process_opts(int argc, char** argv);
+
+#endif

server/modules/filter/test/hint_testing.cnf

@@ -0,0 +1,165 @@
+#
+# Example MaxScale.cnf configuration file
+#
+#
+#
+# Number of server threads
+# Valid options are:
+# 	threads=<number of threads>
+
+[maxscale]
+threads=1
+
+# Define a monitor that can be used to determine the state and role of
+# the servers.
+#
+# Valid options are:
+#
+# 	module=<name of module to load>
+# 	servers=<server name>,<server name>,...
+# 	user =<user name - must have slave replication and 
+#                          slave client privileges>
+# 	passwd=<password of the above user, plain text currently>
+#	monitor_interval=<sampling interval in milliseconds,
+#                          default value is 10000>
+
+[MySQL Monitor]
+type=monitor
+module=mysqlmon
+servers=server1,server2,server3,server4
+user=maxuser
+passwd=maxpwd
+
+# A series of service definition
+#
+# Valid options are:
+#
+# 	router=<name of router module>
+# 	servers=<server name>,<server name>,...
+# 	user=<User to fetch password inforamtion with>
+# 	passwd=<Password of the user, plain text currently>
+#	enable_root_user=<0 or 1, default is 0>
+#	version_string=<specific string for server handshake,
+#		default is the MariaDB embedded library version>
+#
+# Valid router modules currently are:
+# 	readwritesplit, readconnroute and debugcli
+
+
+[RW Split Router]
+type=service
+router=readwritesplit
+servers=server1,server2,server3,server4
+max_slave_connections=90%
+user=maxuser
+passwd=maxpwd
+#filters=MQ
+
+[RW Split Hint Router]
+type=service
+router=readwritesplit
+servers=server1,server2,server3,server4
+max_slave_connections=90%
+user=maxuser
+passwd=maxpwd
+filters=Hint
+
+
+[Read Connection Router]
+type=service
+router=readconnroute
+router_options=master
+servers=server1
+user=maxuser
+passwd=maxpwd
+
+
+[HTTPD Router]
+type=service
+router=testroute
+servers=server1,server2,server3
+
+[Debug Interface]
+type=service
+router=debugcli
+
+
+[Hint]
+type=filter
+module=hintfilter
+
+#[MQ]
+#type=filter
+#module=mqfilter
+#exchange=x1
+#key=k1
+#queue=q1
+#port=5673
+
+# Listener definitions for the services
+#
+# Valid options are:
+#
+# 	service=<name of service defined elsewhere>
+# 	protocol=<name of protocol module with which to listen>
+# 	port=<Listening port>
+#	address=<Address to bind to>
+#	socket=<Listening socket>
+
+[RW Split Listener]
+type=listener
+service=RW Split Router
+protocol=MySQLClient
+port=4006
+
+[RW Split Hint Listener]
+type=listener
+service=RW Split Hint Router
+protocol=MySQLClient
+port=4009
+
+[Read Connection Listener]
+type=listener
+service=Read Connection Router
+protocol=MySQLClient
+port=4008
+#socket=/tmp/readconn.sock
+
+[Debug Listener]
+type=listener
+service=Debug Interface
+protocol=telnetd
+port=4442
+#address=127.0.0.1
+
+[HTTPD Listener]
+type=listener
+service=HTTPD Router
+protocol=HTTPD
+port=6444
+
+# Definition of the servers
+
+[server1]
+type=server
+address=127.0.0.1
+port=3000
+protocol=MySQLBackend
+
+[server2]
+type=server
+address=127.0.0.1
+port=3001
+protocol=MySQLBackend
+
+[server3]
+type=server
+address=127.0.0.1
+port=3002
+protocol=MySQLBackend
+
+[server4]
+type=server
+address=127.0.0.1
+port=3003
+protocol=MySQLBackend

server/modules/filter/test/hint_testing.expected

@@ -0,0 +1,48 @@
+select @@server_id; -- maxscale begin route to master
+select @@server_id;
+select @@server_id; -- maxscale route to server server3|HINT_ROUTE_TO_NAMED_SERVER|server3
+select @@server_id;
+select @@server_id; -- maxscale end
+select @@server_id; -- maxscale named1 prepare route to master
+select @@server_id; -- maxscale named1 begin|HINT_ROUTE_TO_MASTER
+select @@server_id;|HINT_ROUTE_TO_MASTER
+select @@server_id; -- maxscale route to server server3|HINT_ROUTE_TO_NAMED_SERVER|server3
+select @@server_id;|HINT_ROUTE_TO_MASTER
+select @@server_id; -- maxscale end
+select @@server_id; -- maxscale shorthand1 begin route to server server2|HINT_ROUTE_TO_NAMED_SERVER|server2
+select @@server_id;|HINT_ROUTE_TO_NAMED_SERVER|server2
+select @@server_id; -- maxscale route to server server3|HINT_ROUTE_TO_NAMED_SERVER|server3
+select @@server_id;|HINT_ROUTE_TO_NAMED_SERVER|server2
+select @@server_id; -- maxscale end
+select @@server_id; # maxscale begin route to master
+select @@server_id;
+select @@server_id; # maxscale route to server server3|HINT_ROUTE_TO_NAMED_SERVER|server3
+select @@server_id;
+select @@server_id; # maxscale end
+select @@server_id; # maxscale named2 prepare route to master
+select @@server_id; # maxscale named2 begin|HINT_ROUTE_TO_MASTER
+select @@server_id;|HINT_ROUTE_TO_MASTER
+select @@server_id; # maxscale route to server server3|HINT_ROUTE_TO_NAMED_SERVER|server3
+select @@server_id;|HINT_ROUTE_TO_MASTER
+select @@server_id; # maxscale end
+select @@server_id; # maxscale shorthand2 begin route to server server2|HINT_ROUTE_TO_NAMED_SERVER|server2
+select @@server_id;|HINT_ROUTE_TO_NAMED_SERVER|server2
+select @@server_id; # maxscale route to server server3|HINT_ROUTE_TO_NAMED_SERVER|server3
+select @@server_id;|HINT_ROUTE_TO_NAMED_SERVER|server2
+select @@server_id; # maxscale end
+select @@server_id/* maxscale begin route to master */;
+select @@server_id;
+select @@server_id/* maxscale route to server server3 */;|HINT_ROUTE_TO_NAMED_SERVER|*/;
+select @@server_id;
+select @@server_id/* maxscale end */;
+select @@server_id/* maxscale named3 prepare route to master */;|HINT_ROUTE_TO_MASTER
+select @@server_id/* maxscale named3 begin */;
+select @@server_id;
+select @@server_id/* maxscale route to server server3 */;|HINT_ROUTE_TO_NAMED_SERVER|*/;
+select @@server_id;
+select @@server_id/* maxscale end */;
+select @@server_id/* maxscale shorthand3 begin route to server server2 */; |HINT_ROUTE_TO_NAMED_SERVER|*/;
+select @@server_id;|HINT_ROUTE_TO_NAMED_SERVER|*/;
+select @@server_id/* maxscale route to server server3 */;|HINT_ROUTE_TO_NAMED_SERVER|*/;
+select @@server_id;|HINT_ROUTE_TO_NAMED_SERVER|*/;
+select @@server_id/* maxscale end */;

server/modules/filter/test/hint_testing.input

@@ -0,0 +1,48 @@
+select @@server_id; -- maxscale begin route to master
+select @@server_id;
+select @@server_id; -- maxscale route to server server3
+select @@server_id;
+select @@server_id; -- maxscale end
+select @@server_id; -- maxscale named1 prepare route to master
+select @@server_id; -- maxscale named1 begin
+select @@server_id;
+select @@server_id; -- maxscale route to server server3
+select @@server_id;
+select @@server_id; -- maxscale end
+select @@server_id; -- maxscale shorthand1 begin route to server server2
+select @@server_id;
+select @@server_id; -- maxscale route to server server3
+select @@server_id;
+select @@server_id; -- maxscale end
+select @@server_id; # maxscale begin route to master
+select @@server_id;
+select @@server_id; # maxscale route to server server3
+select @@server_id;
+select @@server_id; # maxscale end
+select @@server_id; # maxscale named2 prepare route to master
+select @@server_id; # maxscale named2 begin
+select @@server_id;
+select @@server_id; # maxscale route to server server3
+select @@server_id;
+select @@server_id; # maxscale end
+select @@server_id; # maxscale shorthand2 begin route to server server2
+select @@server_id;
+select @@server_id; # maxscale route to server server3
+select @@server_id;
+select @@server_id; # maxscale end
+select @@server_id/* maxscale begin route to master */;
+select @@server_id;
+select @@server_id/* maxscale route to server server3 */;
+select @@server_id;
+select @@server_id/* maxscale end */;
+select @@server_id/* maxscale named3 prepare route to master */;
+select @@server_id/* maxscale named3 begin */;
+select @@server_id;
+select @@server_id/* maxscale route to server server3 */;
+select @@server_id;
+select @@server_id/* maxscale end */;
+select @@server_id/* maxscale shorthand3 begin route to server server2 */; 
+select @@server_id;
+select @@server_id/* maxscale route to server server3 */;
+select @@server_id;
+select @@server_id/* maxscale end */;

server/modules/filter/test/hint_tests.sh

@@ -0,0 +1,9 @@
+#! /bin/bash
+./harness -q -i hint_testing.input -c hint_testing.cnf -o hint_testing.output -t 1 -s 1 -q &>/dev/null
+diff hint_testing.expected hint_testing.output &>/dev/null
+if [[ "$?" == "0"  ]]
+then
+    echo "PASSED"
+else
+    echo "FAILED"
+fi

server/modules/filter/test/querysmall

@@ -0,0 +1 @@
+SELECT * FROM test_table;