hcq/MaxScale
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'release-1.0beta-refresh' into filter_harness

Browse Source
This commit is contained in:
Markus Makela
2014-09-17 11:07:12 +03:00
parent 6b54927638 c77807971d
commit 530d29f96d
158 changed files with 10243 additions and 1001 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
.gitignore vendored
View File

@ -1,4 +1,35 @@
server/core/tags
server/core/maxscale
server/core/maxkeys
server/core/maxpasswd
# Object files
*.o
*.ko
*.lo
# Libraries
*.lib
*.a
*.la
# Shared objects (inc. Windows DLLs)
*.dll
*.so
*.so.*
*.dylib
# Executables
*.exe
*.out
*.app
# log files (from testing etc.)
*.log
# "make depend" generated stuff
depend
depend.mk
# various auto-backup stuff
*~
*#
.#*
# Vi swap files
.*.swp

BIN
Documentation/MaxScale Configuration And Usage Scenarios-Z2.pdf
View File

Binary file not shown.

BIN
Documentation/MaxScale Configuration And Usage Scenarios.pdf Normal file
View File

Binary file not shown.

BIN
Documentation/MaxScale Debug And Diagnostic Support-Z2.pdf → Documentation/MaxScale Debug And Diagnostic Support.pdf
View File

Binary file not shown.

0
Documentation/MaxScale MySQL Cluster setup-Z2.pdf → Documentation/MaxScale MySQL Cluster setup.pdf
View File

BIN
Documentation/RabbitMQ Setup And MaxScale Integration.pdf Normal file
View File

Binary file not shown.

2
Makefile
View File

@ -42,13 +42,13 @@ all:
(cd client; make)
clean:
echo '#define MAXSCALE_VERSION "'`cat $(ROOT_PATH)/VERSION`'"' > $(ROOT_PATH)/server/include/version.h
(cd log_manager; make clean)
(cd query_classifier; make clean)
(cd server; make clean)
(cd client; touch depend.mk; make clean)
depend:
echo '#define MAXSCALE_VERSION "'`cat $(ROOT_PATH)/VERSION`'"' > $(ROOT_PATH)/server/include/version.h
(cd log_manager; make depend)
(cd query_classifier; make depend)
(cd server; make depend)

14
build_gateway.inc
View File

@ -52,3 +52,17 @@ endif
#
ERRMSG := $(HOME)/usr/share/mysql
#
# Build a binary that produces profile data
#
PROFILE := N
#
# Build a binary that produces code coverage data
#
GCOV := N
# Build optional RabbitMQ filter
# Requires librabbitmq-devel
#
BUILD_RABBITMQ := N

2
client/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
# binaries generated here
maxadmin

23
client/maxadmin.c
View File

@ -58,6 +58,7 @@ static int authMaxScale(int so, char *user, char *password);
static int sendCommand(int so, char *cmd);
static void DoSource(int so, char *cmd);
static void DoUsage();
static int isquit(char *buf);
#ifdef HISTORY
static char *
@ -289,7 +290,7 @@ int argno = 0;
history(hist, &ev, H_ENTER, buf);
#endif
if (!strcasecmp(buf, "quit"))
if (isquit(buf))
{
break;
}
@ -552,3 +553,23 @@ DoUsage()
printf("Any remaining arguments are treated as MaxScale commands or a file\n");
printf("containing commands to execute.\n");
}
/**
* Check command to see if it is a quit command
*
* @param buf The command buffer
* @return Non-zero if the command should cause maxadmin to quit
*/
static int
isquit(char *buf)
{
char *ptr = buf;
if (!buf)
return 0;
while (*ptr && isspace(*ptr))
ptr++;
if (strncasecmp(ptr, "quit", 4) == 0 || strncasecmp(ptr, "exit", 4) == 0)
return 1;
return 0;
}

5
log_manager/.gitignore vendored
View File

@ -1,5 +0,0 @@
*.o
*.so
*.so.*
depend.mk

142
log_manager/log_manager.cc
View File

@ -45,6 +45,12 @@
extern char *program_invocation_name;
extern char *program_invocation_short_name;
#if defined(SS_DEBUG)
static int write_index;
static int block_start_index;
static int prevval;
static simple_mutex_t msg_mutex;
#endif
/**
* Variable holding the enabled logfiles information.
* Used from log users to check enabled logs prior calling
@ -113,7 +119,7 @@ typedef struct blockbuf_st {
skygw_chk_t bb_chk_top;
#endif
logfile_id_t bb_fileid;
bool bb_isfull; /**< closed for disk write */
blockbuf_state_t bb_state; /**State of the block buffer*/
simple_mutex_t bb_mutex; /**< bb_buf_used, bb_isfull */
int bb_refcount; /**< protected by list mutex. #of clients */
// int bb_blankcount; /**< # of blanks used btw strings */
@ -333,6 +339,10 @@ static bool logmanager_init_nomutex(
#if defined(SS_DEBUG)
lm->lm_chk_top = CHK_NUM_LOGMANAGER;
lm->lm_chk_tail = CHK_NUM_LOGMANAGER;
write_index = 0;
block_start_index = 0;
prevval = -1;
simple_mutex_init(&msg_mutex, "Message mutex");
#endif
lm->lm_clientmes = skygw_message_init();
lm->lm_logmes = skygw_message_init();
@ -659,20 +669,66 @@ static int logmanager_write_log(
{
safe_str_len = timestamp_len-1+str_len;
}
/**
* Seek write position and register to block buffer.
* Then print formatted string to write position.
*/
#if defined (SS_LOG_DEBUG)
{
char *copy,*tok;
int tokval;
simple_mutex_lock(&msg_mutex,true);
copy = strdup(str);
tok = strtok(copy,"|");
tok = strtok(NULL,"|");
if(strstr(str,"message|") && tok){
tokval = atoi(tok);
if(prevval > 0){
ss_dassert(tokval == (prevval + 1));
}
prevval = tokval;
}
free(copy);
simple_mutex_unlock(&msg_mutex);
}
#endif
wp = blockbuf_get_writepos(&bb,
id,
safe_str_len,
flush);
#if defined (SS_LOG_DEBUG)
{
sprintf(wp,"[msg:%d]",atomic_add(&write_index,1));
safe_str_len -= strlen(wp);
wp += strlen(wp);
}
#endif
/**
* Write timestamp with at most <timestamp_len> characters
* to wp.
* Returned timestamp_len doesn't include terminating null.
*/
timestamp_len = snprint_timestamp(wp, timestamp_len);
/**
* Write next string to overwrite terminating null character
* of the timestamp string.
@ -753,7 +809,7 @@ static int logmanager_write_log(
wp_c[timestamp_len-1+str_len-1]='\n';
/** lock-free unregistration, includes flush if
* bb_isfull */
* bb_state == BB_FULL */
blockbuf_unregister(bb_c);
}
} /* if (spread_down) */
@ -784,7 +840,7 @@ static void blockbuf_unregister(
/**
* if this is the last client in a full buffer, send write request.
*/
if (atomic_add(&bb->bb_refcount, -1) == 1 && bb->bb_isfull) {
if (atomic_add(&bb->bb_refcount, -1) == 1 && bb->bb_state == BB_FULL) {
skygw_message_send(lf->lf_logmes);
}
ss_dassert(bb->bb_refcount >= 0);
@ -839,6 +895,7 @@ static char* blockbuf_get_writepos(
*/
node = bb_list->mlist_first;
/** Loop over blockbuf list to find write position */
while (true) {
CHK_MLIST_NODE(node);
@ -852,14 +909,16 @@ static char* blockbuf_get_writepos(
/** Lock buffer */
simple_mutex_lock(&bb->bb_mutex, true);
if (bb->bb_isfull || bb->bb_buf_left < str_len) {
if (bb->bb_state == BB_FULL || bb->bb_buf_left < str_len) {
/**
* This block buffer is too full.
* Send flush request to file writer thread. This causes
* flushing all buffers, and (eventually) frees buffer space.
*/
blockbuf_register(bb);
bb->bb_isfull = true;
bb->bb_state = BB_FULL;
blockbuf_unregister(bb);
/** Unlock buffer */
@ -868,6 +927,7 @@ static char* blockbuf_get_writepos(
/** Lock list */
simple_mutex_lock(&bb_list->mlist_mutex, true);
/**
* If next node exists move forward. Else check if there is
* space for a new block buffer on the list.
@ -916,7 +976,28 @@ static char* blockbuf_get_writepos(
node = bb_list->mlist_first;
continue;
}
} else {
}else if(bb->bb_state == BB_CLEARED){
/**
*Move the full buffer to the end of the list
*/
simple_mutex_unlock(&bb->bb_mutex);
simple_mutex_lock(&bb_list->mlist_mutex, true);
if(node->mlnode_next){
bb_list->mlist_first = node->mlnode_next;
bb_list->mlist_last->mlnode_next = node;
node->mlnode_next = NULL;
bb_list->mlist_last = node;
node = bb_list->mlist_first;
}
bb->bb_state = BB_READY;
}else if (bb->bb_state == BB_READY){
/**
* There is space for new log string.
*/
@ -924,9 +1005,11 @@ static char* blockbuf_get_writepos(
}
} /** while (true) */
} else {
/**
* Create the first block buffer to logfile's blockbuf list.
*/
bb = blockbuf_init(id);
CHK_BLOCKBUF(bb);
@ -952,7 +1035,7 @@ static char* blockbuf_get_writepos(
} /* if (bb_list->mlist_nodecount > 0) */
ss_dassert(pos == NULL);
ss_dassert(!(bb->bb_isfull || bb->bb_buf_left < str_len));
ss_dassert(!(bb->bb_state == BB_FULL || bb->bb_buf_left < str_len));
ss_dassert(bb_list->mlist_nodecount <= bb_list->mlist_nodecount_max);
/**
@ -991,7 +1074,7 @@ static char* blockbuf_get_writepos(
* If flush flag is set, set buffer full. As a consequence, no-one
* can write to it before it is flushed to disk.
*/
bb->bb_isfull = (flush == true ? true : bb->bb_isfull);
bb->bb_state = (flush == true ? BB_FULL : bb->bb_state);
/** Unlock buffer */
simple_mutex_unlock(&bb->bb_mutex);
@ -1021,6 +1104,12 @@ static blockbuf_t* blockbuf_init(
bb->bb_buf_left = MAX_LOGSTRLEN;
bb->bb_buf_size = MAX_LOGSTRLEN;
#if defined(SS_LOG_DEBUG)
sprintf(bb->bb_buf,"[block:%d]",atomic_add(&block_start_index,1));
bb->bb_buf_used += strlen(bb->bb_buf);
bb->bb_buf_left -= strlen(bb->bb_buf);
#endif
CHK_BLOCKBUF(bb);
return bb;
}
@ -1170,6 +1259,9 @@ int skygw_log_write_flush(
/**
* Find out the length of log string (to be formatted str).
*/
va_start(valist, str);
len = vsnprintf(NULL, 0, str, valist);
va_end(valist);
@ -1220,6 +1312,9 @@ int skygw_log_write(
err = 1;
goto return_unregister;
}
/**
* Find out the length of log string (to be formatted str).
*/
@ -1233,6 +1328,7 @@ int skygw_log_write(
/**
* Write log string to buffer and add to file write list.
*/
va_start(valist, str);
err = logmanager_write_log(id, false, true, true, len, str, valist);
va_end(valist);
@ -1507,7 +1603,7 @@ static bool fnames_conf_init(
ss_dfprintf(stderr, "%s ", argv[i]);
}
ss_dfprintf(stderr, "\n");*/
#if defined(NOT_USED)
fprintf(stderr,
"Error log :\t%s/%s1%s\n"
"Message log :\t%s/%s1%s\n"
@ -1525,7 +1621,7 @@ static bool fnames_conf_init(
fn->fn_logpath,
fn->fn_debug_prefix,
fn->fn_debug_suffix);
#endif
succp = true;
fn->fn_state = RUN;
CHK_FNAMES_CONF(fn);
@ -1997,6 +2093,16 @@ static bool logfile_init(
form_full_file_name(strparts,
logfile->lf_name_seqno,
2);
fprintf(stderr, "%s\t: %s->%s\n",
STRLOGNAME(logfile_id),
logfile->lf_full_link_name,
logfile->lf_full_file_name);
}
else
{
fprintf(stderr, "%s\t: %s\n",
STRLOGNAME(logfile_id),
logfile->lf_full_file_name);
}
/**
* At least one of the files couldn't be created. Increase
@ -2257,7 +2363,7 @@ static void filewriter_done(
* lists of each logfile object.
*
* Block buffer is written to log file if
* 1. bb_isfull == true,
* 1. bb_state == true,
* 2. logfile object's lf_flushflag == true, or
* 3. skygw_thread_must_exit returns true.
*
@ -2297,7 +2403,7 @@ static void* thr_filewriter_fun(
blockbuf_t* bb;
mlist_node_t* node;
int i;
bool flush_blockbuf; /**< flush single block buffer. */
blockbuf_state_t flush_blockbuf; /**< flush single block buffer. */
bool flush_logfile; /**< flush logfile */
bool flushall_logfiles;/**< flush all logfiles */
size_t vn1;
@ -2356,10 +2462,10 @@ static void* thr_filewriter_fun(
/** Lock block buffer */
simple_mutex_lock(&bb->bb_mutex, true);
flush_blockbuf = bb->bb_isfull;
flush_blockbuf = bb->bb_state;
if (bb->bb_buf_used != 0 &&
(flush_blockbuf ||
(flush_blockbuf == BB_FULL ||
flush_logfile ||
flushall_logfiles))
{
@ -2387,7 +2493,13 @@ static void* thr_filewriter_fun(
bb->bb_buf_left = bb->bb_buf_size;
bb->bb_buf_used = 0;
memset(bb->bb_buf, 0, bb->bb_buf_size);
bb->bb_isfull = false;
bb->bb_state = BB_CLEARED;
#if defined(SS_LOG_DEBUG)
sprintf(bb->bb_buf,"[block:%d]",atomic_add(&block_start_index,1));
bb->bb_buf_used += strlen(bb->bb_buf);
bb->bb_buf_left -= strlen(bb->bb_buf);
#endif
}
/** Release lock to block buffer */
simple_mutex_unlock(&bb->bb_mutex);

6
log_manager/log_manager.h
View File

@ -22,6 +22,12 @@ typedef struct logfile_st logfile_t;
typedef struct fnames_conf_st fnames_conf_t;
typedef struct logmanager_st logmanager_t;
typedef enum {
BB_READY = 0x00,
BB_FULL,
BB_CLEARED
} blockbuf_state_t;
typedef enum {
LOGFILE_ERROR = 1,
LOGFILE_FIRST = LOGFILE_ERROR,

5
log_manager/makefile
View File

@ -8,6 +8,11 @@ SRCS := log_manager.cc
UTILS_PATH := $(ROOT_PATH)/utils
CUR_DIR := $(shell pwd)
ifeq ($(ADD_DEBUG_TAGS),Y)
CFLAGS += -DSS_LOG_DEBUG
endif
makeall: clean all
clean:

2
log_manager/test/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
# binaries generated here
testlog

61
log_manager/test/logorder.sh Executable file
View File

@ -0,0 +1,61 @@
#! /bin/bash
if [[ $# -lt 4 ]]
then
echo "Usage: logorder.sh <iterations> <frequency of flushes> <message size>"
echo "To disable log flushing, use 0 for flush frequency"
exit
fi
rm *.log
#Create large messages
$PWD/testorder $1 $2 $3
TESTLOG=$4
MCOUNT=$1
BLOCKS=`cat skygw_err1.log |tr -s ' '|grep -o 'block:[[:digit:]]\+'|cut -d ':' -f 2`
MESSAGES=`cat skygw_err1.log |tr -s ' '|grep -o 'message|[[:digit:]]\+'|cut -d '|' -f 2`
prev=0
error=0
for i in $BLOCKS
do
if [[ $i -le $prev ]]
then
error=1
echo "block mismatch: $i was after $prev." >> $TESTLOG
fi
prev=$i
done
if [[ error -eq 0 ]]
then
echo "Block buffers were in order" >> $TESTLOG
else
echo "Error: block buffers were written in the wrong order" >> $TESTLOG
fi
prev=0
error=0
for i in $MESSAGES
do
if [[ $i -ne $(( prev + 1 )) ]]
then
error=1
echo "message mismatch: $i was after $prev." >> $TESTLOG
fi
prev=$i
done
if [[ error -eq 0 ]]
then
echo "Block buffer messages were in order" >> $TESTLOG
else
echo "Error: block buffer messages were written in the wrong order" >> $TESTLOG
fi

18
log_manager/test/makefile
View File

@ -28,10 +28,13 @@ testall:
cleantests:
- $(DEL) *.o
- $(DEL) *.log
- $(DEL) testlog
- $(DEL) testorder
- $(DEL) *~
buildtests:
$(MAKE) -C $(LOG_MANAGER_PATH) ADD_DEBUG_TAGS=Y
$(CC) $(CFLAGS) \
-L$(LOG_MANAGER_PATH) \
-Wl,-rpath,$(DEST)/lib \
@ -41,6 +44,17 @@ buildtests:
-I$(LOG_MANAGER_PATH) -I$(UTILS_PATH) testlog.c \
-lstdc++ -llog_manager $(LDLIBS) \
$(UTILS_PATH)/skygw_utils.o
$(CC) $(CFLAGS) \
-L$(LOG_MANAGER_PATH) \
-Wl,-rpath,$(DEST)/lib \
-Wl,-rpath,$(LOG_MANAGER_PATH)/ \
-o testorder \
-I$(MARIADB_SRC_PATH)/include \
-I$(LOG_MANAGER_PATH) -I$(UTILS_PATH) testorder.c \
-lstdc++ -llog_manager $(LDLIBS) \
$(UTILS_PATH)/skygw_utils.o
runtests:
@ -61,6 +75,10 @@ runtests:
@echo "Use 16 threads" >> $(TESTLOG)
@echo "" >> $(TESTLOG)
@-$(LAUNCH_DEBUGGER) $(TESTAPP) "-t 16" 2>>$(TESTLOG)
@echo "" >> $(TEST_MAXSCALE_LOG)
@echo "Test Message Order" >> $(TEST_MAXSCALE_LOG)
@echo "" >> $(TEST_MAXSCALE_LOG)
./logorder.sh 500 0 500 $(TEST_MAXSCALE_LOG)
@echo "Log Manager PASSED" >> $(TESTLOG)
@echo "" >> $(TESTLOG)

105
log_manager/test/testorder.c Normal file
View File

@ -0,0 +1,105 @@
/*
* This file is distributed as part of the SkySQL Gateway. It is free
* software: you can redistribute it and/or modify it under the terms of the
* GNU General Public License as published by the Free Software Foundation,
* version 2.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
* FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
* details.
*
* You should have received a copy of the GNU General Public License along with
* this program; if not, write to the Free Software Foundation, Inc., 51
* Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Copyright SkySQL Ab 2013
*/
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <skygw_utils.h>
#include <log_manager.h>
int main(int argc, char** argv)
{
int iterations = 0, i, interval = 10;
int block_size;
int succp = 0, err = 0;
char cwd[1024];
char tmp[2048];
char *message;
char** optstr;
long msg_index = 1;
memset(cwd,0,1024);
if( argc <4){
fprintf(stderr,
"Log Manager Log Order Test\n"
"Writes an ascending number into the error log to determine if log writes are in order.\n"
"Usage:\t testorder <iterations> <frequency of log flushes> <size of message in bytes>\n");
return 1;
}
block_size = atoi(argv[3]);
if(block_size < 1){
fprintf(stderr,"Message size too small, must be at least 1 byte long.");
}
if(getcwd(cwd,sizeof(cwd)) == NULL ||
(optstr = (char**)malloc(sizeof(char*)*4)) == NULL ||
(message = (char*)malloc(sizeof(char)*block_size))== NULL){
fprintf(stderr,"Fatal Error, exiting...");
return 1;
}
memset(tmp,0,1024);
sprintf(tmp,"%s",cwd);
optstr[0] = strdup("log_manager");
optstr[1] = strdup("-j");
optstr[2] = strdup(tmp);
optstr[3] = NULL;
iterations = atoi(argv[1]);
interval = atoi(argv[2]);
succp = skygw_logmanager_init( 3, optstr);
ss_dassert(succp);
skygw_log_disable(LOGFILE_TRACE);
skygw_log_disable(LOGFILE_MESSAGE);
skygw_log_disable(LOGFILE_DEBUG);
for(i = 0;i<iterations;i++){
sprintf(message,"message|%ld",msg_index++);
memset(message + strlen(message),' ',block_size - strlen(message));
memset(message + block_size - 1,'\0',1);
if(interval > 0 && i % interval == 0){
err = skygw_log_write_flush(LOGFILE_ERROR, message);
}else{
err = skygw_log_write(LOGFILE_ERROR, message);
}
if(err){
fprintf(stderr,"Error: log_manager returned %d",err);
break;
}
usleep(100);
//printf("%s\n",message);
}
skygw_log_flush(LOGFILE_ERROR);
skygw_logmanager_done();
free(message);
free(optstr[0]);
free(optstr[1]);
free(optstr[2]);
free(optstr[3]);
free(optstr);
return 0;
}

10
makefile.inc
View File

@ -41,3 +41,13 @@ endif
ifdef PROF
CFLAGS := $(CFLAGS) -DSS_PROF
endif
ifeq "$(PROFILE)" "Y"
CFLAGS += -pg
LDFLAGS += -pg
endif
ifeq "$(GCOV)" "Y"
CFLAGS += -fprofile-arcs -ftest-coverage
LIBS += -lgcov
endif

4
maxscale.spec
View File

@ -16,9 +16,9 @@ Group: Development/Tools
#Requires:
%if 0%{?suse_version}
BuildRequires: gcc gcc-c++ ncurses-devel bison glibc-devel cmake libgcc_s1 perl make libtool libopenssl-devel libaio libaio-devel mariadb libedit-devel
BuildRequires: gcc gcc-c++ ncurses-devel bison glibc-devel cmake libgcc_s1 perl make libtool libopenssl-devel libaio libaio-devel mariadb libedit-devel librabbitmq-devel
%else
BuildRequires: gcc gcc-c++ ncurses-devel bison glibc-devel cmake libgcc perl make libtool openssl-devel libaio libaio-devel
BuildRequires: gcc gcc-c++ ncurses-devel bison glibc-devel cmake libgcc perl make libtool openssl-devel libaio libaio-devel librabbitmq-devel
%if 0%{?rhel} == 6
BuildRequires: libedit-devel
%endif

5
query_classifier/.gitignore vendored
View File

@ -1,5 +0,0 @@
*.o
*.so
*.so.*
depend.mk

327
query_classifier/query_classifier.cc
View File

@ -406,9 +406,9 @@ return_here:
* restrictive, for example, QUERY_TYPE_READ is smaller than QUERY_TYPE_WRITE.
*
*/
static u_int16_t set_query_type(
u_int16_t* qtype,
u_int16_t new_type)
static u_int32_t set_query_type(
u_int32_t* qtype,
u_int32_t new_type)
{
*qtype = MAX(*qtype, new_type);
return *qtype;
@ -434,7 +434,7 @@ static skygw_query_type_t resolve_query_type(
THD* thd)
{
skygw_query_type_t qtype = QUERY_TYPE_UNKNOWN;
u_int16_t type = QUERY_TYPE_UNKNOWN;
u_int32_t type = QUERY_TYPE_UNKNOWN;
int set_autocommit_stmt = -1; /*< -1 no, 0 disable, 1 enable */
LEX* lex;
Item* item;
@ -454,7 +454,7 @@ static skygw_query_type_t resolve_query_type(
/** SELECT ..INTO variable|OUTFILE|DUMPFILE */
if (lex->result != NULL) {
type = QUERY_TYPE_SESSION_WRITE;
type = QUERY_TYPE_GSYSVAR_WRITE;
goto return_qtype;
}
@ -501,25 +501,49 @@ static skygw_query_type_t resolve_query_type(
type |= QUERY_TYPE_DISABLE_AUTOCOMMIT;
type |= QUERY_TYPE_BEGIN_TRX;
}
if (lex->option_type == OPT_GLOBAL)
{
/**
* SHOW syntax http://dev.mysql.com/doc/refman/5.6/en/show.html
*/
if (lex->sql_command == SQLCOM_SHOW_VARIABLES)
{
type |= QUERY_TYPE_GSYSVAR_READ;
}
/**
* SET syntax http://dev.mysql.com/doc/refman/5.6/en/set-statement.html
*/
else if (lex->sql_command == SQLCOM_SET_OPTION)
{
type |= QUERY_TYPE_GSYSVAR_WRITE;
}
/**
* REVOKE ALL, ASSIGN_TO_KEYCACHE,
* PRELOAD_KEYS, FLUSH, RESET, CREATE|ALTER|DROP SERVER
*/
if (lex->option_type == OPT_GLOBAL)
else
{
type |= QUERY_TYPE_GLOBAL_WRITE;
type |= QUERY_TYPE_GSYSVAR_WRITE;
}
goto return_qtype;
}
else if (lex->option_type == OPT_SESSION)
{
/** SHOW commands are all reads to one backend */
/**
* SHOW syntax http://dev.mysql.com/doc/refman/5.6/en/show.html
*/
if (lex->sql_command == SQLCOM_SHOW_VARIABLES)
{
type |= QUERY_TYPE_SESSION_READ;
type |= QUERY_TYPE_SYSVAR_READ;
}
else
/**
* SET syntax http://dev.mysql.com/doc/refman/5.6/en/set-statement.html
*/
else if (lex->sql_command == SQLCOM_SET_OPTION)
{
type |= QUERY_TYPE_SESSION_WRITE;
/** Either user- or system variable write */
type |= QUERY_TYPE_GSYSVAR_WRITE;
}
goto return_qtype;
}
@ -538,31 +562,26 @@ static skygw_query_type_t resolve_query_type(
if (thd->variables.sql_log_bin == 0 &&
force_data_modify_op_replication)
{
/** Not replicated */
type |= QUERY_TYPE_SESSION_WRITE;
}
else
{
type |= QUERY_TYPE_WRITE;
/** Written to binlog, that is, replicated except tmp tables */
type |= QUERY_TYPE_WRITE; /*< to master */
if (lex->create_info.options & HA_LEX_CREATE_TMP_TABLE &&
lex->sql_command == SQLCOM_CREATE_TABLE)
{
type |= QUERY_TYPE_CREATE_TMP_TABLE;
type |= QUERY_TYPE_CREATE_TMP_TABLE; /*< remember in router */
}
}
goto return_qtype;
}
/** Try to catch session modifications here */
switch (lex->sql_command) {
case SQLCOM_SET_OPTION: /*< SET commands. */
if (lex->option_type == OPT_GLOBAL)
{
type |= QUERY_TYPE_GLOBAL_WRITE;
break;
}
/**<! fall through */
/** fallthrough */
case SQLCOM_CHANGE_DB:
case SQLCOM_DEALLOCATE_PREPARE:
type |= QUERY_TYPE_SESSION_WRITE;
@ -599,15 +618,23 @@ static skygw_query_type_t resolve_query_type(
default:
break;
}
if (QTYPE_LESS_RESTRICTIVE_THAN_WRITE(type)) {
#if defined(UPDATE_VAR_SUPPORT)
if (QTYPE_LESS_RESTRICTIVE_THAN_WRITE(type))
#endif
if (QUERY_IS_TYPE(qtype, QUERY_TYPE_UNKNOWN) ||
QUERY_IS_TYPE(qtype, QUERY_TYPE_LOCAL_READ) ||
QUERY_IS_TYPE(qtype, QUERY_TYPE_READ) ||
QUERY_IS_TYPE(qtype, QUERY_TYPE_USERVAR_READ) ||
QUERY_IS_TYPE(qtype, QUERY_TYPE_SYSVAR_READ) ||
QUERY_IS_TYPE(qtype, QUERY_TYPE_GSYSVAR_READ))
{
/**
* These values won't change qtype more restrictive than write.
* UDFs and procedures could possibly cause session-wide write,
* but unless their content is replicated this is a limitation
* of this implementation.
* In other words : UDFs and procedures are not allowed to
* perform writes which are not replicated but nede to repeat
* perform writes which are not replicated but need to repeat
* in every node.
* It is not sure if such statements exist. vraa 25.10.13
*/
@ -628,7 +655,9 @@ static skygw_query_type_t resolve_query_type(
if (itype == Item::SUBSELECT_ITEM) {
continue;
} else if (itype == Item::FUNC_ITEM) {
}
else if (itype == Item::FUNC_ITEM)
{
int func_qtype = QUERY_TYPE_UNKNOWN;
/**
* Item types:
@ -710,23 +739,44 @@ static skygw_query_type_t resolve_query_type(
break;
/** System session variable */
case Item_func::GSYSVAR_FUNC:
/** User-defined variable read */
case Item_func::GUSERVAR_FUNC:
/** User-defined variable modification */
case Item_func::SUSERVAR_FUNC:
func_qtype |= QUERY_TYPE_SESSION_READ;
func_qtype |= QUERY_TYPE_SYSVAR_READ;
LOGIF(LD, (skygw_log_write(
LOGFILE_DEBUG,
"%lu [resolve_query_type] "
"functype SUSERVAR_FUNC, could be "
"executed in MaxScale.",
"functype GSYSVAR_FUNC, system "
"variable read.",
pthread_self())));
break;
/** User-defined variable read */
case Item_func::GUSERVAR_FUNC:
func_qtype |= QUERY_TYPE_USERVAR_READ;
LOGIF(LD, (skygw_log_write(
LOGFILE_DEBUG,
"%lu [resolve_query_type] "
"functype GUSERVAR_FUNC, user "
"variable read.",
pthread_self())));
break;
/** User-defined variable modification */
case Item_func::SUSERVAR_FUNC:
/**
* Really it is user variable but we
* don't separate sql variables atm.
* 15.9.14
*/
func_qtype |= QUERY_TYPE_GSYSVAR_WRITE;
LOGIF(LD, (skygw_log_write(
LOGFILE_DEBUG,
"%lu [resolve_query_type] "
"functype SUSERVAR_FUNC, user "
"variable write.",
pthread_self())));
break;
case Item_func::UNKNOWN_FUNC:
if (item->name != NULL &&
strcmp(item->name, "last_insert_id()") == 0)
{
func_qtype |= QUERY_TYPE_SESSION_READ;
func_qtype |= QUERY_TYPE_MASTER_READ;
}
else
{
@ -757,6 +807,7 @@ static skygw_query_type_t resolve_query_type(
/**< Set new query type */
type |= set_query_type(&type, func_qtype);
}
#if defined(UPDATE_VAR_SUPPORT)
/**
* Write is as restrictive as it gets due functions,
* so break.
@ -764,6 +815,7 @@ static skygw_query_type_t resolve_query_type(
if ((type & QUERY_TYPE_WRITE) == QUERY_TYPE_WRITE) {
break;
}
#endif
} /**< for */
} /**< if */
return_qtype:
@ -890,26 +942,61 @@ char* skygw_query_classifier_get_stmtname(
}
/**
*Returns the LEX struct of the parsed GWBUF
*@param The parsed GWBUF
*@return Pointer to the LEX struct or NULL if an error occurred or the query was not parsed
*/
LEX* get_lex(GWBUF* querybuf)
{
parsing_info_t* pi;
MYSQL* mysql;
THD* thd;
if (!GWBUF_IS_PARSED(querybuf))
{
return NULL;
}
pi = (parsing_info_t *)gwbuf_get_buffer_object_data(querybuf,
GWBUF_PARSING_INFO);
if (pi == NULL)
{
return NULL;
}
if ((mysql = (MYSQL *)pi->pi_handle) == NULL ||
(thd = (THD *)mysql->thd) == NULL)
{
ss_dassert(mysql != NULL &&
thd != NULL);
return NULL;
}
return thd->lex;
}
/**
* Finds the head of the list of tables affected by the current select statement.
* @param thd Pointer to a valid THD
* @return Pointer to the head of the TABLE_LIST chain or NULL in case of an error
*/
void* skygw_get_affected_tables(void* thdp)
void* skygw_get_affected_tables(void* lexptr)
{
THD* thd = (THD*)thdp;
LEX* lex = (LEX*)lexptr;
if(thd == NULL ||
thd->lex == NULL ||
thd->lex->current_select == NULL)
if(lex == NULL ||
lex->current_select == NULL)
{
ss_dassert(thd != NULL &&
thd->lex != NULL &&
thd->lex->current_select != NULL);
ss_dassert(lex != NULL &&
lex->current_select != NULL);
return NULL;
}
return (void*)thd->lex->current_select->table_list.first;
return (void*)lex->current_select->table_list.first;
}
@ -922,45 +1009,25 @@ void* skygw_get_affected_tables(void* thdp)
* @param tblsize Pointer where the number of tables is written
* @return Array of null-terminated strings with the table names
*/
char** skygw_get_table_names(GWBUF* querybuf,int* tblsize)
char** skygw_get_table_names(GWBUF* querybuf,int* tblsize, bool fullnames)
{
parsing_info_t* pi;
MYSQL* mysql;
THD* thd;
LEX* lex;
TABLE_LIST* tbl;
int i = 0, currtblsz = 0;
char**tables,**tmp;
int i = 0,
currtblsz = 0;
char **tables,
**tmp;
if (!GWBUF_IS_PARSED(querybuf))
if((lex = get_lex(querybuf)) == NULL)
{
tables = NULL;
goto retblock;
}
pi = (parsing_info_t *)gwbuf_get_buffer_object_data(querybuf,
GWBUF_PARSING_INFO);
if (pi == NULL)
{
tables = NULL;
goto retblock;
}
if (pi->pi_query_plain_str == NULL ||
(mysql = (MYSQL *)pi->pi_handle) == NULL ||
(thd = (THD *)mysql->thd) == NULL)
{
ss_dassert(pi->pi_query_plain_str != NULL &&
mysql != NULL &&
thd != NULL);
tables = NULL;
goto retblock;
}
lex->current_select = lex->all_selects_list;
thd->lex->current_select = thd->lex->all_selects_list;
while(lex->current_select){
while(thd->lex->current_select){
tbl = (TABLE_LIST*)skygw_get_affected_tables(thd);
tbl = (TABLE_LIST*)skygw_get_affected_tables(lex);
while (tbl)
{
@ -982,57 +1049,96 @@ char** skygw_get_table_names(GWBUF* querybuf,int* tblsize)
}
}
tables[i++] = strdup(tbl->alias);
char *catnm = NULL;
if(fullnames)
{
if(tbl->db && strcmp(tbl->db,"skygw_virtual") != 0)
{
catnm = (char*)calloc(strlen(tbl->db) + strlen(tbl->table_name) + 2,sizeof(char));
strcpy(catnm,tbl->db);
strcat(catnm,".");
strcat(catnm,tbl->table_name);
}
}
if(catnm)
{
tables[i++] = catnm;
}
else
{
tables[i++] = strdup(tbl->table_name);
}
tbl=tbl->next_local;
}
thd->lex->current_select = thd->lex->current_select->next_select_in_list();
lex->current_select = lex->current_select->next_select_in_list();
}
retblock:
*tblsize = i;
return tables;
}
/**
* Extract the name of the created table.
* Extract, allocate memory and copy the name of the created table.
* @param querybuf Buffer to use.
* @return A pointer to the name if a table was created, otherwise NULL
*/
char* skygw_get_created_table_name(GWBUF* querybuf)
{
parsing_info_t* pi;
MYSQL* mysql;
THD* thd;
if (!GWBUF_IS_PARSED(querybuf))
{
return NULL;
}
pi = (parsing_info_t *)gwbuf_get_buffer_object_data(querybuf,
GWBUF_PARSING_INFO);
LEX* lex;
if (pi == NULL)
if((lex = get_lex(querybuf)) == NULL)
{
return NULL;
}
if ((mysql = (MYSQL *)pi->pi_handle) == NULL ||
(thd = (THD *)mysql->thd) == NULL)
{
ss_dassert(mysql != NULL &&
thd != NULL);
return NULL;
}
if(thd->lex->create_last_non_select_table &&
thd->lex->create_last_non_select_table->table_name){
char* name = strdup(thd->lex->create_last_non_select_table->table_name);
if(lex->create_last_non_select_table &&
lex->create_last_non_select_table->table_name){
char* name = strdup(lex->create_last_non_select_table->table_name);
return name;
}else{
return NULL;
}
}
/**
* Checks whether the query is a "real" query ie. SELECT,UPDATE,INSERT,DELETE or any variation of these.
* Queries that affect the underlying database are not considered as real queries and the queries that target
* specific row or variable data are regarded as the real queries.
* @param GWBUF to analyze
* @return true if the query is a real query, otherwise false
*/
bool skygw_is_real_query(GWBUF* querybuf)
{
LEX* lex = get_lex(querybuf);
if(lex){
switch(lex->sql_command){
case SQLCOM_SELECT:
return lex->all_selects_list->table_list.elements > 0;
case SQLCOM_UPDATE:
case SQLCOM_INSERT:
case SQLCOM_INSERT_SELECT:
case SQLCOM_DELETE:
case SQLCOM_TRUNCATE:
case SQLCOM_REPLACE:
case SQLCOM_REPLACE_SELECT:
case SQLCOM_PREPARE:
case SQLCOM_EXECUTE:
return true;
default:
return false;
}
}
return false;
}
/**
* Checks whether the buffer contains a DROP TABLE... query.
* @param querybuf Buffer to inspect
@ -1040,31 +1146,10 @@ char* skygw_get_created_table_name(GWBUF* querybuf)
*/
bool is_drop_table_query(GWBUF* querybuf)
{
parsing_info_t* pi;
MYSQL* mysql;
THD* thd;
LEX* lex;
if (!GWBUF_IS_PARSED(querybuf))
{
return false;
}
pi = (parsing_info_t *)gwbuf_get_buffer_object_data(querybuf,
GWBUF_PARSING_INFO);
if (pi == NULL)
{
return false;
}
if ((mysql = (MYSQL *)pi->pi_handle) == NULL ||
(thd = (THD *)mysql->thd) == NULL)
{
ss_dassert(mysql != NULL &&
thd != NULL);
return false;
}
return thd->lex->sql_command == SQLCOM_DROP_TABLE;
return (lex = get_lex(querybuf)) != NULL &&
lex->sql_command == SQLCOM_DROP_TABLE;
}
/*

46
query_classifier/query_classifier.h
View File

@ -31,23 +31,30 @@ EXTERN_C_BLOCK_BEGIN
* is modified
*/
typedef enum {
QUERY_TYPE_UNKNOWN = 0x0000, /*< Initial value, can't be tested bitwisely */
QUERY_TYPE_LOCAL_READ = 0x0001, /*< Read non-database data, execute in MaxScale */
QUERY_TYPE_READ = 0x0002, /*< No updates */
QUERY_TYPE_WRITE = 0x0004, /*< Master data will be modified */
QUERY_TYPE_SESSION_WRITE = 0x0008, /*< Session data will be modified */
QUERY_TYPE_GLOBAL_WRITE = 0x0010, /*< Global system variable modification */
QUERY_TYPE_BEGIN_TRX = 0x0020, /*< BEGIN or START TRANSACTION */
QUERY_TYPE_ENABLE_AUTOCOMMIT = 0x0040, /*< SET autocommit=1 */
QUERY_TYPE_DISABLE_AUTOCOMMIT = 0x0080, /*< SET autocommit=0 */
QUERY_TYPE_ROLLBACK = 0x0100, /*< ROLLBACK */
QUERY_TYPE_COMMIT = 0x0200, /*< COMMIT */
QUERY_TYPE_PREPARE_NAMED_STMT = 0x0400, /*< Prepared stmt with name from user */
QUERY_TYPE_PREPARE_STMT = 0x0800, /*< Prepared stmt with id provided by server */
QUERY_TYPE_EXEC_STMT = 0x1000, /*< Execute prepared statement */
QUERY_TYPE_SESSION_READ = 0x2000, /*< Read session data (from master 31.8.14) */
QUERY_TYPE_CREATE_TMP_TABLE = 0x4000, /*< Create temporary table */
QUERY_TYPE_READ_TMP_TABLE = 0x8000 /*< Read temporary table */
QUERY_TYPE_UNKNOWN = 0x000000, /*< Initial value, can't be tested bitwisely */
QUERY_TYPE_LOCAL_READ = 0x000001, /*< Read non-database data, execute in MaxScale:any */
QUERY_TYPE_READ = 0x000002, /*< Read database data:any */
QUERY_TYPE_WRITE = 0x000004, /*< Master data will be modified:master */
QUERY_TYPE_MASTER_READ = 0x000008, /*< Read from the master:master */
QUERY_TYPE_SESSION_WRITE = 0x000010, /*< Session data will be modified:master or all */
/** Not implemented yet */
// QUERY_TYPE_USERVAR_WRITE = 0x000020, /*< Write a user variable:master or all */
QUERY_TYPE_USERVAR_READ = 0x000040, /*< Read a user variable:master or any */
QUERY_TYPE_SYSVAR_READ = 0x000080, /*< Read a system variable:master or any */
/** Not implemented yet */
// QUERY_TYPE_SYSVAR_WRITE = 0x000100, /*< Write a system variable:master or all */
QUERY_TYPE_GSYSVAR_READ = 0x000200, /*< Read global system variable:master or any */
QUERY_TYPE_GSYSVAR_WRITE = 0x000400, /*< Write global system variable:master or all */
QUERY_TYPE_BEGIN_TRX = 0x000800, /*< BEGIN or START TRANSACTION */
QUERY_TYPE_ENABLE_AUTOCOMMIT = 0x001000, /*< SET autocommit=1 */
QUERY_TYPE_DISABLE_AUTOCOMMIT = 0x002000, /*< SET autocommit=0 */
QUERY_TYPE_ROLLBACK = 0x004000, /*< ROLLBACK */
QUERY_TYPE_COMMIT = 0x008000, /*< COMMIT */
QUERY_TYPE_PREPARE_NAMED_STMT = 0x010000, /*< Prepared stmt with name from user:all */
QUERY_TYPE_PREPARE_STMT = 0x020000, /*< Prepared stmt with id provided by server:all */
QUERY_TYPE_EXEC_STMT = 0x040000, /*< Execute prepared statement:master or any */
QUERY_TYPE_CREATE_TMP_TABLE = 0x080000, /*< Create temporary table:master (could be all) */
QUERY_TYPE_READ_TMP_TABLE = 0x100000 /*< Read temporary table:master (could be any) */
} skygw_query_type_t;
@ -76,8 +83,9 @@ skygw_query_type_t query_classifier_get_type(GWBUF* querybuf);
char* skygw_query_classifier_get_stmtname(MYSQL* mysql);
char* skygw_get_created_table_name(GWBUF* querybuf);
bool is_drop_table_query(GWBUF* querybuf);
void* skygw_get_affected_tables(void* thdp);
char** skygw_get_table_names(GWBUF* querybuf,int* tblsize);
bool skygw_is_real_query(GWBUF* querybuf);
void* skygw_get_affected_tables(void* lexptr);
char** skygw_get_table_names(GWBUF* querybuf,int* tblsize,bool fullnames);
char* skygw_get_canonical(GWBUF* querybuf);
bool parse_query (GWBUF* querybuf);
parsing_info_t* parsing_info_init(void (*donefun)(void *));

2
query_classifier/test/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
# binaries generated here
testmain

8
query_classifier/test/canonical_tests/Makefile
View File

@ -26,8 +26,8 @@ LDFLAGS=-L$(QUERY_CLASSIFIER_PATH) \
-Wl,-rpath,$(LOG_MANAGER_PATH) \
-Wl,-rpath,$(QUERY_CLASSIFIER_PATH)
LIBS=-lstdc++ -lpthread -lquery_classifier -lz -ldl -lssl -laio -lcrypt -lcrypto -lrt \
-llog_manager $(UTILS_PATH)/skygw_utils.o $(CORE_PATH)/buffer.o $(CORE_PATH)/atomic.o $(CORE_PATH)/spinlock.o
LIBS=-lstdc++ -lpthread -lquery_classifier -lz -ldl -lssl -laio -lcrypt -lcrypto -lrt -lm \
-llog_manager $(UTILS_PATH)/skygw_utils.o $(CORE_PATH)/buffer.o $(CORE_PATH)/atomic.o $(CORE_PATH)/spinlock.o $(CORE_PATH)/hint.o
CFLAGS=-g $(MYSQL_HEADERS) \
-I$(QUERY_CLASSIFIER_PATH) \
@ -35,8 +35,6 @@ CFLAGS=-g $(MYSQL_HEADERS) \
-I$(ROOT_PATH)/server/include \
-I$(UTILS_PATH)
EMBFLAGS=$(shell mysql_config --cflags --libmysqld-libs)
testall:
$(MAKE) cleantests
@ -52,7 +50,7 @@ cleantests:
buildtests: $(OBJS)
cp $(ERRMSG)/errmsg.sys .
$(CC) $(CFLAGS) $(LDFLAGS) $(EMBFLAGS) $(LIBS) canonizer.c -o $(TESTAPP)
$(CC) $(CFLAGS) $(LDFLAGS) $(LIBS) canonizer.c -o $(TESTAPP) $(LDLIBS) $(LDMYSQL)
runtests:
@echo "" > $(TESTLOG)

35
query_classifier/test/canonical_tests/canonizer.c
View File

@ -57,19 +57,33 @@ int main(int argc, char** argv)
}
read(fdin,buffer,fsz);
tok = strpbrk(buffer,"\n");
lines = 1;
while((tok = strpbrk(tok + 1,"\n"))){
lines++;
}
qbuff = malloc(sizeof(GWBUF*)*lines);
i = 0;
int bsz = 4,z=0;
qbuff = calloc(bsz,sizeof(GWBUF*));
tok = strtok(buffer,"\n");
while(tok){
if(i>=bsz){
GWBUF** tmp = calloc(bsz*2,sizeof(GWBUF*));
if(!tmp){
printf("Error: Failed to allocate memory.");
return 1;
}
for(z=0;z<bsz;z++){
tmp[z] = qbuff[z];
}
free(qbuff);
qbuff = tmp;
bsz *= 2;
}
if(strlen(tok) > 0){
qin = strdup(tok);
psize = strlen(qin);
qbuff[i] = gwbuf_alloc(psize + 6);
@ -79,14 +93,16 @@ int main(int argc, char** argv)
*(qbuff[i]->sbuf->data + 4) = 0x03;
memcpy(qbuff[i]->sbuf->data + 5,qin,psize);
*(qbuff[i]->sbuf->data + 5 + psize) = 0x00;
tok = strtok(NULL,"\n");
tok = strtok(NULL,"\n\0");
free(qin);
i++;
}
}
fdout = open(argv[2],O_TRUNC|O_CREAT|O_WRONLY,S_IRWXU|S_IXGRP|S_IXOTH);
for(i = 0;i<lines;i++){
for(i = 0;i<bsz;i++){
if(qbuff[i]){
parse_query(qbuff[i]);
tok = skygw_get_canonical(qbuff[i]);
write(fdout,tok,strlen(tok));
@ -94,6 +110,9 @@ int main(int argc, char** argv)
gwbuf_free(qbuff[i]);
}
}
free(qbuff);
free(buffer);
close(fdin);
close(fdout);

44
rabbitmq_consumer/CMakeLists.txt Normal file
View File

@ -0,0 +1,44 @@
cmake_minimum_required (VERSION 2.6)
set(CMAKE_LIBRARY_PATH ${CMAKE_LIBRARY_PATH} /usr/lib /usr/lib64 /usr/local/lib /usr/local/lib64 /usr/lib/mariadb /usr/lib64/mariadb)
set(CMAKE_INCLUDE_PATH ${CMAKE_INCLUDE_PATH} /usr/include /usr/local/include /usr/include/mysql /usr/local/include/mysql /usr/include/mariadb /usr/local/include/mariadb)
include(InstallRequiredSystemLibraries)
project (consumer)
find_path(MYSQL_INCLUDE_DIRS mysql.h)
find_library(MYSQL_LIBRARIES NAMES mysqlclient)
find_library(RABBITMQ_C_LIBRARIES NAMES rabbitmq)
include_directories(${MYSQL_INCLUDE_DIRS})
include_directories(${RABBITMQ_C_INCLUDE_DIRS})
include_directories(${CMAKE_SOURCE_DIR}/inih)
add_subdirectory (inih)
link_directories(${CMAKE_SOURCE_DIR}/inih)
if(RABBITMQ_C_LIBRARIES AND MYSQL_LIBRARIES AND MYSQL_INCLUDE_DIRS)
add_executable (consumer consumer.c ${MYSQL_LIBRARIES} ${RABBITMQ_C_LIBRARIES})
target_link_libraries(consumer mysqlclient)
target_link_libraries(consumer rabbitmq)
target_link_libraries(consumer inih)
install(TARGETS consumer DESTINATION bin)
install(FILES consumer.cnf DESTINATION share/consumer)
else(RABBITMQ_C_LIBRARIES AND MYSQL_LIBRARIES AND MYSQL_INCLUDE_DIRS)
message(FATAL_ERROR "Error: Can not find requred libraries: libmysqld, librabbitmq.")
endif(RABBITMQ_C_LIBRARIES AND MYSQL_LIBRARIES AND MYSQL_INCLUDE_DIRS)
set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "RabbitMQ Consumer Client")
set(CPACK_PACKAGE_NAME "RabbitMQ Consumer")
set(CPACK_GENERATOR "RPM")
set(CPACK_PACKAGE_VERSION_MAJOR "1")
set(CPACK_PACKAGE_VERSION_MINOR "0")
set(CPACK_RPM_PACKAGE_NAME "rabbitmq-consumer")
set(CPACK_RPM_PACKAGE_VENDOR "SkySQL Ab")
set(CPACK_RPM_PACKAGE_AUTOREQPROV " no")
include(CPack)

15
rabbitmq_consumer/Makefile Normal file
View File

@ -0,0 +1,15 @@
include buildconfig.inc
CC=gcc
CFLAGS=-c -Wall -g -Iinih $(INCLUDE_DIRS)
LDFLAGS= $(LIBRARY_DIRS) -lrabbitmq -lmysqlclient
SRCS= inih/ini.c consumer.c
OBJ=$(SRCS:.c=.o)
all:$(OBJ)
$(CC) $(LDFLAGS) $(OBJ) -o consumer `mysql_config --cflags --libs`
%.o:%.c
$(CC) $(CFLAGS) $< -o $@
clean:
-rm *.o
-rm *~

39
rabbitmq_consumer/README Normal file
View File

@ -0,0 +1,39 @@
This program requires the librabbitmq and libmysqlclient libraries.
librabbitmq-c - https://github.com/alanxz/rabbitmq-c
MariaDB Client Library for C 2.0 Series - https://mariadb.com/kb/en/mariadb/client-libraries/client-library-for-c/
Building with CMake:
'cmake .'
Variables to pass for CMake:
Path to headers -DCMAKE_INCLUDE_PATH=<path to headers>
Path to libraries -DCMAKE_LIBRARY_PATH=<path to libraries>
Install prefix -DCMAKE_INSTALL_PREFIX=<prefix>
Separate multiple folders with colons, for example:
'path1:path2:path3'
After running CMake run 'make' to build the binaries and 'make package' to build RPMs.
To build without CMake, use the provided makefile and update the
include and library directories 'in buildvars.inc'
The configuration for the consumer client are red from 'consumer.cnf'.
Options for the configuration file:
hostname Hostname of the RabbitMQ server
port Port of the RabbitMQ server
vhost Virtual host location of the RabbitMQ server
user Username for the RabbitMQ server
passwd Password for the RabbitMQ server
queue Queue to consume from
dbserver Hostname of the SQL server
dbport Port of the SQL server
dbname Name of the SQL database to use
dbuser Database username
dbpasswd Database passwork
logfile Message log filename

8
rabbitmq_consumer/buildconfig.inc Normal file
View File

@ -0,0 +1,8 @@
#Use the '-I' prefix for include and '-L' for library directories
#You can use multiple library and include directories
#Path to the rabbitmq-c and mysqlclient libraries
LIBRARY_DIRS :=-L/usr/lib64
#path to headers
INCLUDE_DIRS :=-I/usr/include -I/usr/include/mysql

524
rabbitmq_consumer/consumer.c Normal file
View File

@ -0,0 +1,524 @@
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <ini.h>
#include <stdint.h>
#include <amqp_tcp_socket.h>
#include <amqp.h>
#include <amqp_framing.h>
#include <mysql.h>
#include <signal.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
typedef struct delivery_t
{
uint64_t dtag;
amqp_message_t* message;
struct delivery_t *next,*prev;
}DELIVERY;
typedef struct consumer_t
{
char *hostname,*vhost,*user,*passwd,*queue,*dbserver,*dbname,*dbuser,*dbpasswd;
DELIVERY* query_stack;
int port,dbport;
}CONSUMER;
static int all_ok;
static FILE* out_fd;
static CONSUMER* c_inst;
static char* DB_DATABASE = "CREATE DATABASE IF NOT EXISTS %s;";
static char* DB_TABLE = "CREATE TABLE IF NOT EXISTS pairs (tag VARCHAR(64) PRIMARY KEY NOT NULL, query VARCHAR(2048), reply VARCHAR(2048), date_in DATETIME NOT NULL, date_out DATETIME DEFAULT NULL, counter INT DEFAULT 1)";
static char* DB_INSERT = "INSERT INTO pairs(tag, query, date_in) VALUES ('%s','%s',FROM_UNIXTIME(%s))";
static char* DB_UPDATE = "UPDATE pairs SET reply='%s', date_out=FROM_UNIXTIME(%s) WHERE tag='%s'";
static char* DB_INCREMENT = "UPDATE pairs SET counter = counter+1, date_out=FROM_UNIXTIME(%s) WHERE query='%s'";
void sighndl(int signum)
{
if(signum == SIGINT){
all_ok = 0;
alarm(1);
}
}
int handler(void* user, const char* section, const char* name,
const char* value)
{
if(strcmp(section,"consumer") == 0){
if(strcmp(name,"hostname") == 0){
c_inst->hostname = strdup(value);
}else if(strcmp(name,"vhost") == 0){
c_inst->vhost = strdup(value);
}else if(strcmp(name,"port") == 0){
c_inst->port = atoi(value);
}else if(strcmp(name,"user") == 0){
c_inst->user = strdup(value);
}else if(strcmp(name,"passwd") == 0){
c_inst->passwd = strdup(value);
}else if(strcmp(name,"queue") == 0){
c_inst->queue = strdup(value);
}else if(strcmp(name,"dbserver") == 0){
c_inst->dbserver = strdup(value);
}else if(strcmp(name,"dbport") == 0){
c_inst->dbport = atoi(value);
}else if(strcmp(name,"dbname") == 0){
c_inst->dbname = strdup(value);
}else if(strcmp(name,"dbuser") == 0){
c_inst->dbuser = strdup(value);
}else if(strcmp(name,"dbpasswd") == 0){
c_inst->dbpasswd = strdup(value);
}else if(strcmp(name,"logfile") == 0){
out_fd = fopen(value,"ab");
}
}
return 1;
}
int isPair(amqp_message_t* a, amqp_message_t* b)
{
int keylen = a->properties.correlation_id.len >=
b->properties.correlation_id.len ?
a->properties.correlation_id.len :
b->properties.correlation_id.len;
return strncmp(a->properties.correlation_id.bytes,
b->properties.correlation_id.bytes,
keylen) == 0 ? 1 : 0;
}
int connectToServer(MYSQL* server)
{
mysql_init(server);
mysql_options(server,MYSQL_READ_DEFAULT_GROUP,"client");
mysql_options(server,MYSQL_OPT_USE_REMOTE_CONNECTION,0);
my_bool tr = 1;
mysql_options(server,MYSQL_OPT_RECONNECT,&tr);
MYSQL* result = mysql_real_connect(server,
c_inst->dbserver,
c_inst->dbuser,
c_inst->dbpasswd,
NULL,
c_inst->dbport,
NULL,
0);
if(result==NULL){
fprintf(out_fd,"Error: Could not connect to MySQL server: %s\n",mysql_error(server));
return 0;
}
int bsz = 1024;
char *qstr = calloc(bsz,sizeof(char));
if(!qstr){
fprintf(stderr, "Fatal Error: Cannot allocate enough memory.\n");
return 0;
}
/**Connection ok, check that the database and table exist*/
memset(qstr,0,bsz);
sprintf(qstr,DB_DATABASE,c_inst->dbname);
if(mysql_query(server,qstr)){
fprintf(stderr,"Error: Could not send query MySQL server: %s\n",mysql_error(server));
}
memset(qstr,0,bsz);
sprintf(qstr,"USE %s;",c_inst->dbname);
if(mysql_query(server,qstr)){
fprintf(stderr,"Error: Could not send query MySQL server: %s\n",mysql_error(server));
}
memset(qstr,0,bsz);
sprintf(qstr,DB_TABLE);
if(mysql_query(server,qstr)){
fprintf(stderr,"Error: Could not send query MySQL server: %s\n",mysql_error(server));
}
free(qstr);
return 1;
}
int sendMessage(MYSQL* server, amqp_message_t* msg)
{
int buffsz = (int)((msg->body.len + 1)*2+1) +
(int)((msg->properties.correlation_id.len + 1)*2+1) +
strlen(DB_INSERT),
rval = 0;
char *qstr = calloc(buffsz,sizeof(char)),
*rawmsg = calloc((msg->body.len + 1),sizeof(char)),
*clnmsg = calloc(((msg->body.len + 1)*2+1),sizeof(char)),
*rawdate = calloc((msg->body.len + 1),sizeof(char)),
*clndate = calloc(((msg->body.len + 1)*2+1),sizeof(char)),
*rawtag = calloc((msg->properties.correlation_id.len + 1),sizeof(char)),
*clntag = calloc(((msg->properties.correlation_id.len + 1)*2+1),sizeof(char));
sprintf(qstr,"%.*s",(int)msg->body.len,(char *)msg->body.bytes);
fprintf(out_fd,"Received: %s\n",qstr);
char *ptr = strtok(qstr,"|");
sprintf(rawdate,"%s",ptr);
ptr = strtok(NULL,"\n\0");
if(ptr == NULL){
fprintf(out_fd,"Message content not valid.\n");
rval = 1;
goto cleanup;
}
sprintf(rawmsg,"%s",ptr);
sprintf(rawtag,"%.*s",(int)msg->properties.correlation_id.len,(char *)msg->properties.correlation_id.bytes);
memset(qstr,0,buffsz);
mysql_real_escape_string(server,clnmsg,rawmsg,strnlen(rawmsg,msg->body.len + 1));
mysql_real_escape_string(server,clndate,rawdate,strnlen(rawdate,msg->body.len + 1));
mysql_real_escape_string(server,clntag,rawtag,strnlen(rawtag,msg->properties.correlation_id.len + 1));
if(strncmp(msg->properties.message_id.bytes,
"query",msg->properties.message_id.len) == 0)
{
sprintf(qstr,DB_INCREMENT,clndate,clnmsg);
rval = mysql_query(server,qstr);
if(mysql_affected_rows(server) == 0){
memset(qstr,0,buffsz);
sprintf(qstr,DB_INSERT,clntag,clnmsg,clndate);
rval = mysql_query(server,qstr);
}
}else if(strncmp(msg->properties.message_id.bytes,
"reply",msg->properties.message_id.len) == 0){
sprintf(qstr,DB_UPDATE,clnmsg,clndate,clntag);
rval = mysql_query(server,qstr);
}else{
rval = 1;
goto cleanup;
}
if(rval){
fprintf(stderr,"Could not send query to SQL server:%s\n",mysql_error(server));
goto cleanup;
}
cleanup:
free(qstr);
free(rawmsg);
free(clnmsg);
free(rawdate);
free(clndate);
free(rawtag);
free(clntag);
return rval;
}
int sendToServer(MYSQL* server, amqp_message_t* a, amqp_message_t* b){
amqp_message_t *msg, *reply;
int buffsz = 2048;
char *qstr = calloc(buffsz,sizeof(char));
if(!qstr){
fprintf(out_fd, "Fatal Error: Cannot allocate enough memory.\n");
free(qstr);
return 0;
}
if( a->properties.message_id.len == strlen("query") &&
strncmp(a->properties.message_id.bytes,"query",
a->properties.message_id.len) == 0){
msg = a;
reply = b;
}else{
msg = b;
reply = a;
}
printf("pair: %.*s\nquery: %.*s\nreply: %.*s\n",
(int)msg->properties.correlation_id.len,
(char *)msg->properties.correlation_id.bytes,
(int)msg->body.len,
(char *)msg->body.bytes,
(int)reply->body.len,
(char *)reply->body.bytes);
if((int)msg->body.len +
(int)reply->body.len +
(int)msg->properties.correlation_id.len + 50 >= buffsz)
{
char *qtmp = calloc(buffsz*2,sizeof(char));
free(qstr);
if(qtmp){
qstr = qtmp;
buffsz *= 2;
}else{
fprintf(stderr, "Fatal Error: Cannot allocate enough memory.\n");
return 0;
}
}
char *rawmsg = calloc((msg->body.len + 1),sizeof(char)),
*clnmsg = calloc(((msg->body.len + 1)*2+1),sizeof(char)),
*rawrpl = calloc((reply->body.len + 1),sizeof(char)),
*clnrpl = calloc(((reply->body.len + 1)*2+1),sizeof(char)),
*rawtag = calloc((msg->properties.correlation_id.len + 1),sizeof(char)),
*clntag = calloc(((msg->properties.correlation_id.len + 1)*2+1),sizeof(char));
sprintf(rawmsg,"%.*s",(int)msg->body.len,(char *)msg->body.bytes);
sprintf(rawrpl,"%.*s",(int)reply->body.len,(char *)reply->body.bytes);
sprintf(rawtag,"%.*s",(int)msg->properties.correlation_id.len,(char *)msg->properties.correlation_id.bytes);
char *ptr;
while((ptr = strchr(rawmsg,'\n'))){
*ptr = ' ';
}
while((ptr = strchr(rawrpl,'\n'))){
*ptr = ' ';
}
while((ptr = strchr(rawtag,'\n'))){
*ptr = ' ';
}
mysql_real_escape_string(server,clnmsg,rawmsg,strnlen(rawmsg,msg->body.len + 1));
mysql_real_escape_string(server,clnrpl,rawrpl,strnlen(rawrpl,reply->body.len + 1));
mysql_real_escape_string(server,clntag,rawtag,strnlen(rawtag,msg->properties.correlation_id.len + 1));
sprintf(qstr,"INSERT INTO pairs VALUES ('%s','%s','%s');",clnmsg,clnrpl,clntag);
free(rawmsg);
free(clnmsg);
free(rawrpl);
free(clnrpl);
free(rawtag);
free(clntag);
if(mysql_query(server,qstr)){
fprintf(stderr,"Could not send query to SQL server:%s\n",mysql_error(server));
free(qstr);
return 0;
}
free(qstr);
return 1;
}
int main(int argc, char** argv)
{
int channel = 1, status = AMQP_STATUS_OK, cnfnlen;
amqp_socket_t *socket = NULL;
amqp_connection_state_t conn;
amqp_rpc_reply_t ret;
amqp_message_t *reply = NULL;
amqp_frame_t frame;
struct timeval timeout;
MYSQL db_inst;
char ch, *cnfname = NULL, *cnfpath = NULL;
static const char* fname = "consumer.cnf";
if((c_inst = calloc(1,sizeof(CONSUMER))) == NULL){
fprintf(stderr, "Fatal Error: Cannot allocate enough memory.\n");
return 1;
}
if(signal(SIGINT,sighndl) == SIG_IGN){
signal(SIGINT,SIG_IGN);
}
while((ch = getopt(argc,argv,"c:"))!= -1){
switch(ch){
case 'c':
cnfnlen = strlen(optarg);
cnfpath = strdup(optarg);
break;
default:
break;
}
}
cnfname = calloc(cnfnlen + strlen(fname) + 1,sizeof(char));
if(cnfpath){
/**Config file path as argument*/
strcpy(cnfname,cnfpath);
if(cnfpath[cnfnlen-1] != '/'){
strcat(cnfname,"/");
}
}
strcat(cnfname,fname);
timeout.tv_sec = 1;
timeout.tv_usec = 0;
all_ok = 1;
out_fd = NULL;
/**Parse the INI file*/
if(ini_parse(cnfname,handler,NULL) < 0){
/**Try to parse a config in the same directory*/
if(ini_parse(fname,handler,NULL) < 0){
fprintf(stderr, "Fatal Error: Error parsing configuration file!\n");
goto fatal_error;
}
}
if(out_fd == NULL){
out_fd = stdout;
}
fprintf(out_fd,"\n--------------------------------------------------------------\n");
/**Confirm that all parameters were in the configuration file*/
if(!c_inst->hostname||!c_inst->vhost||!c_inst->user||
!c_inst->passwd||!c_inst->dbpasswd||!c_inst->queue||
!c_inst->dbserver||!c_inst->dbname||!c_inst->dbuser){
fprintf(stderr, "Fatal Error: Inadequate configuration file!\n");
goto fatal_error;
}
connectToServer(&db_inst);
if((conn = amqp_new_connection()) == NULL ||
(socket = amqp_tcp_socket_new(conn)) == NULL){
fprintf(stderr, "Fatal Error: Cannot create connection object or socket.\n");
goto fatal_error;
}
if(amqp_socket_open(socket, c_inst->hostname, c_inst->port)){
fprintf(stderr, "RabbitMQ Error: Cannot open socket.\n");
goto error;
}
ret = amqp_login(conn, c_inst->vhost, 0, 131072, 0, AMQP_SASL_METHOD_PLAIN, c_inst->user, c_inst->passwd);
if(ret.reply_type != AMQP_RESPONSE_NORMAL){
fprintf(stderr, "RabbitMQ Error: Cannot login to server.\n");
goto error;
}
amqp_channel_open(conn, channel);
ret = amqp_get_rpc_reply(conn);
if(ret.reply_type != AMQP_RESPONSE_NORMAL){
fprintf(stderr, "RabbitMQ Error: Cannot open channel.\n");
goto error;
}
reply = malloc(sizeof(amqp_message_t));
if(!reply){
fprintf(stderr, "Error: Cannot allocate enough memory.\n");
goto error;
}
amqp_basic_consume(conn,channel,amqp_cstring_bytes(c_inst->queue),amqp_empty_bytes,0,0,0,amqp_empty_table);
while(all_ok){
status = amqp_simple_wait_frame_noblock(conn,&frame,&timeout);
/**No frames to read from server, possibly out of messages*/
if(status == AMQP_STATUS_TIMEOUT){
sleep(timeout.tv_sec);
continue;
}
if(frame.payload.method.id == AMQP_BASIC_DELIVER_METHOD){
amqp_basic_deliver_t* decoded = (amqp_basic_deliver_t*)frame.payload.method.decoded;
amqp_read_message(conn,channel,reply,0);
if(sendMessage(&db_inst,reply)){
fprintf(stderr,"RabbitMQ Error: Received malformed message.\n");
amqp_basic_reject(conn,channel,decoded->delivery_tag,0);
amqp_destroy_message(reply);
}else{
amqp_basic_ack(conn,channel,decoded->delivery_tag,0);
amqp_destroy_message(reply);
}
}else{
fprintf(stderr,"RabbitMQ Error: Received method from server: %s\n",amqp_method_name(frame.payload.method.id));
all_ok = 0;
goto error;
}
}
fprintf(out_fd,"Shutting down...\n");
error:
mysql_close(&db_inst);
mysql_library_end();
if(c_inst && c_inst->query_stack){
while(c_inst->query_stack){
DELIVERY* d = c_inst->query_stack->next;
amqp_destroy_message(c_inst->query_stack->message);
free(c_inst->query_stack);
c_inst->query_stack = d;
}
}
amqp_channel_close(conn, channel, AMQP_REPLY_SUCCESS);
amqp_connection_close(conn, AMQP_REPLY_SUCCESS);
amqp_destroy_connection(conn);
fatal_error:
if(out_fd){
fclose(out_fd);
}
if(c_inst){
free(c_inst->hostname);
free(c_inst->vhost);
free(c_inst->user);
free(c_inst->passwd);
free(c_inst->queue);
free(c_inst->dbserver);
free(c_inst->dbname);
free(c_inst->dbuser);
free(c_inst->dbpasswd);
free(c_inst);
}
return all_ok;
}

28
rabbitmq_consumer/consumer.cnf Normal file
View File

@ -0,0 +1,28 @@
#
#The options for the consumer are:
#hostname RabbitMQ hostname
#port RabbitMQ port
#vhost RabbitMQ virtual host
#user RabbitMQ username
#passwd RabbitMQ password
#queue Name of the queue to use
#dbserver SQL server name
#dbport SQL server port
#dbname Name of the databse to use
#dbuser SQL server username
#dbpasswd SQL server password
#logfile Message log filename
#
[consumer]
hostname=127.0.0.1
port=5673
vhost=/
user=guest
passwd=guest
queue=q1
dbserver=127.0.0.1
dbport=3000
dbname=mqpairs
dbuser=maxuser
dbpasswd=maxpwd
#logfile=consumer.log

BIN
rabbitmq_consumer/inih/._LICENSE.txt Executable file
View File

Binary file not shown.

BIN
rabbitmq_consumer/inih/._README.txt Executable file
View File

Binary file not shown.

BIN
rabbitmq_consumer/inih/._cpp Executable file
View File

Binary file not shown.

BIN
rabbitmq_consumer/inih/._examples Executable file
View File

Binary file not shown.

BIN
rabbitmq_consumer/inih/._extra Executable file
View File

Binary file not shown.

BIN
rabbitmq_consumer/inih/._ini.c Executable file
View File

Binary file not shown.

BIN
rabbitmq_consumer/inih/._ini.h Executable file
View File

Binary file not shown.

BIN
rabbitmq_consumer/inih/._tests Executable file
View File

Binary file not shown.

3
rabbitmq_consumer/inih/.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
*.o
*.a
make.depend

1
rabbitmq_consumer/inih/CMakeLists.txt Normal file
View File

@ -0,0 +1 @@
add_library(inih ini.c)

27
rabbitmq_consumer/inih/LICENSE.txt Executable file
View File

@ -0,0 +1,27 @@
The "inih" library is distributed under the New BSD license:
Copyright (c) 2009, Brush Technology
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
* Neither the name of Brush Technology nor the names of its contributors
may be used to endorse or promote products derived from this software
without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY BRUSH TECHNOLOGY ''AS IS'' AND ANY
EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL BRUSH TECHNOLOGY BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

5
rabbitmq_consumer/inih/README.txt Executable file
View File

@ -0,0 +1,5 @@
inih is a simple .INI file parser written in C, released under the New BSD
license (see LICENSE.txt). Go to the project home page for more info:
http://code.google.com/p/inih/

BIN
rabbitmq_consumer/inih/cpp/._INIReader.cpp Executable file
View File

Binary file not shown.

BIN
rabbitmq_consumer/inih/cpp/._INIReader.h Executable file
View File

Binary file not shown.

BIN
rabbitmq_consumer/inih/cpp/._INIReaderTest.cpp Executable file
View File

Binary file not shown.

67
rabbitmq_consumer/inih/cpp/INIReader.cpp Executable file
View File

@ -0,0 +1,67 @@
// Read an INI file into easy-to-access name/value pairs.
#include <algorithm>
#include <cctype>
#include <cstdlib>
#include "../ini.h"
#include "INIReader.h"
using std::string;
INIReader::INIReader(string filename)
{
_error = ini_parse(filename.c_str(), ValueHandler, this);
}
int INIReader::ParseError()
{
return _error;
}
string INIReader::Get(string section, string name, string default_value)
{
string key = MakeKey(section, name);
return _values.count(key) ? _values[key] : default_value;
}
long INIReader::GetInteger(string section, string name, long default_value)
{
string valstr = Get(section, name, "");
const char* value = valstr.c_str();
char* end;
// This parses "1234" (decimal) and also "0x4D2" (hex)
long n = strtol(value, &end, 0);
return end > value ? n : default_value;
}
bool INIReader::GetBoolean(string section, string name, bool default_value)
{
string valstr = Get(section, name, "");
// Convert to lower case to make string comparisons case-insensitive
std::transform(valstr.begin(), valstr.end(), valstr.begin(), ::tolower);
if (valstr == "true" || valstr == "yes" || valstr == "on" || valstr == "1")
return true;
else if (valstr == "false" || valstr == "no" || valstr == "off" || valstr == "0")
return false;
else
return default_value;
}
string INIReader::MakeKey(string section, string name)
{
string key = section + "." + name;
// Convert to lower case to make section/name lookups case-insensitive
std::transform(key.begin(), key.end(), key.begin(), ::tolower);
return key;
}
int INIReader::ValueHandler(void* user, const char* section, const char* name,
const char* value)
{
INIReader* reader = (INIReader*)user;
string key = MakeKey(section, name);
if (reader->_values[key].size() > 0)
reader->_values[key] += "\n";
reader->_values[key] += value;
return 1;
}

48
rabbitmq_consumer/inih/cpp/INIReader.h Executable file
View File

@ -0,0 +1,48 @@
// Read an INI file into easy-to-access name/value pairs.
// inih and INIReader are released under the New BSD license (see LICENSE.txt).
// Go to the project home page for more info:
//
// http://code.google.com/p/inih/
#ifndef __INIREADER_H__
#define __INIREADER_H__
#include <map>
#include <string>
// Read an INI file into easy-to-access name/value pairs. (Note that I've gone
// for simplicity here rather than speed, but it should be pretty decent.)
class INIReader
{
public:
// Construct INIReader and parse given filename. See ini.h for more info
// about the parsing.
INIReader(std::string filename);
// Return the result of ini_parse(), i.e., 0 on success, line number of
// first error on parse error, or -1 on file open error.
int ParseError();
// Get a string value from INI file, returning default_value if not found.
std::string Get(std::string section, std::string name,
std::string default_value);
// Get an integer (long) value from INI file, returning default_value if
// not found or not a valid integer (decimal "1234", "-1234", or hex "0x4d2").
long GetInteger(std::string section, std::string name, long default_value);
// Get a boolean value from INI file, returning default_value if not found or if
// not a valid true/false value. Valid true values are "true", "yes", "on", "1",
// and valid false values are "false", "no", "off", "0" (not case sensitive).
bool GetBoolean(std::string section, std::string name, bool default_value);
private:
int _error;
std::map<std::string, std::string> _values;
static std::string MakeKey(std::string section, std::string name);
static int ValueHandler(void* user, const char* section, const char* name,
const char* value);
};
#endif // __INIREADER_H__

20
rabbitmq_consumer/inih/cpp/INIReaderTest.cpp Executable file
View File

@ -0,0 +1,20 @@
// Example that shows simple usage of the INIReader class
#include <iostream>
#include "INIReader.h"
int main()
{
INIReader reader("../examples/test.ini");
if (reader.ParseError() < 0) {
std::cout << "Can't load 'test.ini'\n";
return 1;
}
std::cout << "Config loaded from 'test.ini': version="
<< reader.GetInteger("protocol", "version", -1) << ", name="
<< reader.Get("user", "name", "UNKNOWN") << ", email="
<< reader.Get("user", "email", "UNKNOWN") << ", active="
<< reader.GetBoolean("user", "active", true) << "\n";
return 0;
}

BIN
rabbitmq_consumer/inih/examples/._config.def Executable file
View File

Binary file not shown.

BIN
rabbitmq_consumer/inih/examples/._ini_dump.c Executable file
View File

Binary file not shown.

BIN
rabbitmq_consumer/inih/examples/._ini_example.c Executable file
View File

Binary file not shown.

BIN
rabbitmq_consumer/inih/examples/._ini_xmacros.c Executable file
View File

Binary file not shown.

BIN
rabbitmq_consumer/inih/examples/._test.ini Executable file
View File

Binary file not shown.

8
rabbitmq_consumer/inih/examples/config.def Executable file
View File

@ -0,0 +1,8 @@
// CFG(section, name, default)
CFG(protocol, version, "0")
CFG(user, name, "Fatty Lumpkin")
CFG(user, email, "fatty@lumpkin.com")
#undef CFG

40
rabbitmq_consumer/inih/examples/ini_dump.c Executable file
View File

@ -0,0 +1,40 @@
/* ini.h example that simply dumps an INI file without comments */
#include <stdio.h>
#include <string.h>
#include "../ini.h"
static int dumper(void* user, const char* section, const char* name,
const char* value)
{
static char prev_section[50] = "";
if (strcmp(section, prev_section)) {
printf("%s[%s]\n", (prev_section[0] ? "\n" : ""), section);
strncpy(prev_section, section, sizeof(prev_section));
prev_section[sizeof(prev_section) - 1] = '\0';
}
printf("%s = %s\n", name, value);
return 1;
}
int main(int argc, char* argv[])
{
int error;
if (argc <= 1) {
printf("Usage: ini_dump filename.ini\n");
return 1;
}
error = ini_parse(argv[1], dumper, NULL);
if (error < 0) {
printf("Can't read '%s'!\n", argv[1]);
return 2;
}
else if (error) {
printf("Bad config file (first error on line %d)!\n", error);
return 3;
}
return 0;
}

44
rabbitmq_consumer/inih/examples/ini_example.c Executable file
View File

@ -0,0 +1,44 @@
/* Example: parse a simple configuration file */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "../ini.h"
typedef struct
{
int version;
const char* name;
const char* email;
} configuration;
static int handler(void* user, const char* section, const char* name,
const char* value)
{
configuration* pconfig = (configuration*)user;
#define MATCH(s, n) strcmp(section, s) == 0 && strcmp(name, n) == 0
if (MATCH("protocol", "version")) {
pconfig->version = atoi(value);
} else if (MATCH("user", "name")) {
pconfig->name = strdup(value);
} else if (MATCH("user", "email")) {
pconfig->email = strdup(value);
} else {
return 0; /* unknown section/name, error */
}
return 1;
}
int main(int argc, char* argv[])
{
configuration config;
if (ini_parse("test.ini", handler, &config) < 0) {
printf("Can't load 'test.ini'\n");
return 1;
}
printf("Config loaded from 'test.ini': version=%d, name=%s, email=%s\n",
config.version, config.name, config.email);
return 0;
}

46
rabbitmq_consumer/inih/examples/ini_xmacros.c Executable file
View File

@ -0,0 +1,46 @@
/* Parse a configuration file into a struct using X-Macros */
#include <stdio.h>
#include <string.h>
#include "../ini.h"
/* define the config struct type */
typedef struct {
#define CFG(s, n, default) char *s##_##n;
#include "config.def"
} config;
/* create one and fill in its default values */
config Config = {
#define CFG(s, n, default) default,
#include "config.def"
};
/* process a line of the INI file, storing valid values into config struct */
int handler(void *user, const char *section, const char *name,
const char *value)
{
config *cfg = (config *)user;
if (0) ;
#define CFG(s, n, default) else if (strcmp(section, #s)==0 && \
strcmp(name, #n)==0) cfg->s##_##n = strdup(value);
#include "config.def"
return 1;
}
/* print all the variables in the config, one per line */
void dump_config(config *cfg)
{
#define CFG(s, n, default) printf("%s_%s = %s\n", #s, #n, cfg->s##_##n);
#include "config.def"
}
int main(int argc, char* argv[])
{
if (ini_parse("test.ini", handler, &Config) < 0)
printf("Can't load 'test.ini', using defaults\n");
dump_config(&Config);
return 0;
}

9
rabbitmq_consumer/inih/examples/test.ini Executable file
View File

@ -0,0 +1,9 @@
; Test config file for ini_example.c and INIReaderTest.cpp
[protocol] ; Protocol configuration
version=6 ; IPv6
[user]
name = Bob Smith ; Spaces around '=' are stripped
email = bob@smith.com ; And comments (like this) ignored
active = true ; Test a boolean

BIN
rabbitmq_consumer/inih/extra/._Makefile.static Executable file
View File

Binary file not shown.

19
rabbitmq_consumer/inih/extra/Makefile.static Executable file
View File

@ -0,0 +1,19 @@
# Simple makefile to build inih as a static library using g++
SRC = ../ini.c
OBJ = $(SRC:.c=.o)
OUT = libinih.a
INCLUDES = -I..
CCFLAGS = -g -O2
CC = g++
default: $(OUT)
.c.o:
$(CC) $(INCLUDES) $(CCFLAGS) $(EXTRACCFLAGS) -c $< -o $@
$(OUT): $(OBJ)
ar rcs $(OUT) $(OBJ) $(EXTRAARFLAGS)
clean:
rm -f $(OBJ) $(OUT)

176
rabbitmq_consumer/inih/ini.c Executable file
View File

@ -0,0 +1,176 @@
/* inih -- simple .INI file parser
inih is released under the New BSD license (see LICENSE.txt). Go to the project
home page for more info:
http://code.google.com/p/inih/
*/
#include <stdio.h>
#include <ctype.h>
#include <string.h>
#include "ini.h"
#if !INI_USE_STACK
#include <stdlib.h>
#endif
#define MAX_SECTION 50
#define MAX_NAME 50
/* Strip whitespace chars off end of given string, in place. Return s. */
static char* rstrip(char* s)
{
char* p = s + strlen(s);
while (p > s && isspace((unsigned char)(*--p)))
*p = '\0';
return s;
}
/* Return pointer to first non-whitespace char in given string. */
static char* lskip(const char* s)
{
while (*s && isspace((unsigned char)(*s)))
s++;
return (char*)s;
}
/* Return pointer to first char c or ';' comment in given string, or pointer to
null at end of string if neither found. ';' must be prefixed by a whitespace
character to register as a comment. */
static char* find_char_or_comment(const char* s, char c)
{
int was_whitespace = 0;
while (*s && *s != c && !(was_whitespace && *s == ';')) {
was_whitespace = isspace((unsigned char)(*s));
s++;
}
return (char*)s;
}
/* Version of strncpy that ensures dest (size bytes) is null-terminated. */
static char* strncpy0(char* dest, const char* src, size_t size)
{
strncpy(dest, src, size);
dest[size - 1] = '\0';
return dest;
}
/* See documentation in header file. */
int ini_parse_file(FILE* file,
int (*handler)(void*, const char*, const char*,
const char*),
void* user)
{
/* Uses a fair bit of stack (use heap instead if you need to) */
#if INI_USE_STACK
char line[INI_MAX_LINE];
#else
char* line;
#endif
char section[MAX_SECTION] = "";
char prev_name[MAX_NAME] = "";
char* start;
char* end;
char* name;
char* value;
int lineno = 0;
int error = 0;
#if !INI_USE_STACK
line = (char*)malloc(INI_MAX_LINE);
if (!line) {
return -2;
}
#endif
/* Scan through file line by line */
while (fgets(line, INI_MAX_LINE, file) != NULL) {
lineno++;
start = line;
#if INI_ALLOW_BOM
if (lineno == 1 && (unsigned char)start[0] == 0xEF &&
(unsigned char)start[1] == 0xBB &&
(unsigned char)start[2] == 0xBF) {
start += 3;
}
#endif
start = lskip(rstrip(start));
if (*start == ';' || *start == '#') {
/* Per Python ConfigParser, allow '#' comments at start of line */
}
#if INI_ALLOW_MULTILINE
else if (*prev_name && *start && start > line) {
/* Non-black line with leading whitespace, treat as continuation
of previous name's value (as per Python ConfigParser). */
if (!handler(user, section, prev_name, start) && !error)
error = lineno;
}
#endif
else if (*start == '[') {
/* A "[section]" line */
end = find_char_or_comment(start + 1, ']');
if (*end == ']') {
*end = '\0';
strncpy0(section, start + 1, sizeof(section));
*prev_name = '\0';
}
else if (!error) {
/* No ']' found on section line */
error = lineno;
}
}
else if (*start && *start != ';') {
/* Not a comment, must be a name[=:]value pair */
end = find_char_or_comment(start, '=');
if (*end != '=') {
end = find_char_or_comment(start, ':');
}
if (*end == '=' || *end == ':') {
*end = '\0';
name = rstrip(start);
value = lskip(end + 1);
end = find_char_or_comment(value, '\0');
if (*end == ';')
*end = '\0';
rstrip(value);
/* Valid name[=:]value pair found, call handler */
strncpy0(prev_name, name, sizeof(prev_name));
if (!handler(user, section, name, value) && !error)
error = lineno;
}
else if (!error) {
/* No '=' or ':' found on name[=:]value line */
error = lineno;
}
}
}
#if !INI_USE_STACK
free(line);
#endif
return error;
}
/* See documentation in header file. */
int ini_parse(const char* filename,
int (*handler)(void*, const char*, const char*, const char*),
void* user)
{
FILE* file;
int error;
file = fopen(filename, "r");
if (!file)
return -1;
error = ini_parse_file(file, handler, user);
fclose(file);
return error;
}

72
rabbitmq_consumer/inih/ini.h Executable file
View File

@ -0,0 +1,72 @@
/* inih -- simple .INI file parser
inih is released under the New BSD license (see LICENSE.txt). Go to the project
home page for more info:
http://code.google.com/p/inih/
*/
#ifndef __INI_H__
#define __INI_H__
/* Make this header file easier to include in C++ code */
#ifdef __cplusplus
extern "C" {
#endif
#include <stdio.h>
/* Parse given INI-style file. May have [section]s, name=value pairs
(whitespace stripped), and comments starting with ';' (semicolon). Section
is "" if name=value pair parsed before any section heading. name:value
pairs are also supported as a concession to Python's ConfigParser.
For each name=value pair parsed, call handler function with given user
pointer as well as section, name, and value (data only valid for duration
of handler call). Handler should return nonzero on success, zero on error.
Returns 0 on success, line number of first error on parse error (doesn't
stop on first error), -1 on file open error, or -2 on memory allocation
error (only when INI_USE_STACK is zero).
*/
int ini_parse(const char* filename,
int (*handler)(void* user, const char* section,
const char* name, const char* value),
void* user);
/* Same as ini_parse(), but takes a FILE* instead of filename. This doesn't
close the file when it's finished -- the caller must do that. */
int ini_parse_file(FILE* file,
int (*handler)(void* user, const char* section,
const char* name, const char* value),
void* user);
/* Nonzero to allow multi-line value parsing, in the style of Python's
ConfigParser. If allowed, ini_parse() will call the handler with the same
name for each subsequent line parsed. */
#ifndef INI_ALLOW_MULTILINE
#define INI_ALLOW_MULTILINE 1
#endif
/* Nonzero to allow a UTF-8 BOM sequence (0xEF 0xBB 0xBF) at the start of
the file. See http://code.google.com/p/inih/issues/detail?id=21 */
#ifndef INI_ALLOW_BOM
#define INI_ALLOW_BOM 1
#endif
/* Nonzero to use stack, zero to use heap (malloc/free). */
#ifndef INI_USE_STACK
#define INI_USE_STACK 1
#endif
/* Maximum line length for any line in INI file. */
#ifndef INI_MAX_LINE
#define INI_MAX_LINE 200
#endif
#ifdef __cplusplus
}
#endif
#endif /* __INI_H__ */

BIN
rabbitmq_consumer/inih/tests/._bad_comment.ini Executable file
View File

Binary file not shown.

BIN
rabbitmq_consumer/inih/tests/._bad_multi.ini Executable file
View File

Binary file not shown.

BIN
rabbitmq_consumer/inih/tests/._bad_section.ini Executable file
View File

Binary file not shown.

BIN
rabbitmq_consumer/inih/tests/._baseline_multi.txt Executable file
View File

Binary file not shown.

BIN
rabbitmq_consumer/inih/tests/._baseline_single.txt Executable file
View File

Binary file not shown.

BIN
rabbitmq_consumer/inih/tests/._bom.ini Executable file
View File

Binary file not shown.

BIN
rabbitmq_consumer/inih/tests/._multi_line.ini Executable file
View File

Binary file not shown.

BIN
rabbitmq_consumer/inih/tests/._normal.ini Executable file
View File

Binary file not shown.

BIN
rabbitmq_consumer/inih/tests/._unittest.bat Executable file
View File

Binary file not shown.

BIN
rabbitmq_consumer/inih/tests/._unittest.c Executable file
View File

Binary file not shown.

BIN
rabbitmq_consumer/inih/tests/._user_error.ini Executable file
View File

Binary file not shown.

1
rabbitmq_consumer/inih/tests/bad_comment.ini Executable file
View File

@ -0,0 +1 @@
This is an error

1
rabbitmq_consumer/inih/tests/bad_multi.ini Executable file
View File

@ -0,0 +1 @@
indented

5
rabbitmq_consumer/inih/tests/bad_section.ini Executable file
View File

@ -0,0 +1,5 @@
[section1]
name1=value1
[section2
[section3 ; comment ]
name2=value2

47
rabbitmq_consumer/inih/tests/baseline_multi.txt Executable file
View File

@ -0,0 +1,47 @@
no_file.ini: e=-1 user=0
... [section1]
... one=This is a test;
... two=1234;
... [ section 2 ]
... happy=4;
... sad=;
... [comment_test]
... test1=1;2;3;
... test2=2;3;4;this won't be a comment, needs whitespace before ';';
... test;3=345;
... test4=4#5#6;
... [colon_tests]
... Content-Type=text/html;
... foo=bar;
... adams=42;
normal.ini: e=0 user=101
... [section1]
... name1=value1;
... name2=value2;
bad_section.ini: e=3 user=102
bad_comment.ini: e=1 user=102
... [section]
... a=b;
... user=parse_error;
... c=d;
user_error.ini: e=3 user=104
... [section1]
... single1=abc;
... multi=this is a;
... multi=multi-line value;
... single2=xyz;
... [section2]
... multi=a;
... multi=b;
... multi=c;
... [section3]
... single=ghi;
... multi=the quick;
... multi=brown fox;
... name=bob smith;
multi_line.ini: e=0 user=105
bad_multi.ini: e=1 user=105
... [bom_section]
... bom_name=bom_value;
... key“=value“;
bom.ini: e=0 user=107

43
rabbitmq_consumer/inih/tests/baseline_single.txt Executable file
View File

@ -0,0 +1,43 @@
no_file.ini: e=-1 user=0
... [section1]
... one=This is a test;
... two=1234;
... [ section 2 ]
... happy=4;
... sad=;
... [comment_test]
... test1=1;2;3;
... test2=2;3;4;this won't be a comment, needs whitespace before ';';
... test;3=345;
... test4=4#5#6;
... [colon_tests]
... Content-Type=text/html;
... foo=bar;
... adams=42;
normal.ini: e=0 user=101
... [section1]
... name1=value1;
... name2=value2;
bad_section.ini: e=3 user=102
bad_comment.ini: e=1 user=102
... [section]
... a=b;
... user=parse_error;
... c=d;
user_error.ini: e=3 user=104
... [section1]
... single1=abc;
... multi=this is a;
... single2=xyz;
... [section2]
... multi=a;
... [section3]
... single=ghi;
... multi=the quick;
... name=bob smith;
multi_line.ini: e=4 user=105
bad_multi.ini: e=1 user=105
... [bom_section]
... bom_name=bom_value;
... key“=value“;
bom.ini: e=0 user=107

3
rabbitmq_consumer/inih/tests/bom.ini Executable file
View File

@ -0,0 +1,3 @@
[bom_section]
bom_name=bom_value
key“ = value“

15
rabbitmq_consumer/inih/tests/multi_line.ini Executable file
View File

@ -0,0 +1,15 @@
[section1]
single1 = abc
multi = this is a
multi-line value
single2 = xyz
[section2]
multi = a
b
c
[section3]
single: ghi
multi: the quick
brown fox
name = bob smith ; comment line 1
; comment line 2

25
rabbitmq_consumer/inih/tests/normal.ini Executable file
View File

@ -0,0 +1,25 @@
; This is an INI file
[section1] ; section comment
one=This is a test ; name=value comment
two = 1234
; x=y
[ section 2 ]
happy = 4
sad =
[empty]
; do nothing
[comment_test]
test1 = 1;2;3 ; only this will be a comment
test2 = 2;3;4;this won't be a comment, needs whitespace before ';'
test;3 = 345 ; key should be "test;3"
test4 = 4#5#6 ; '#' only starts a comment at start of line
#test5 = 567 ; entire line commented
# test6 = 678 ; entire line commented, except in MULTILINE mode
[colon_tests]
Content-Type: text/html
foo:bar
adams : 42

2
rabbitmq_consumer/inih/tests/unittest.bat Executable file
View File

@ -0,0 +1,2 @@
@call tcc ..\ini.c -I..\ -run unittest.c > baseline_multi.txt
@call tcc ..\ini.c -I..\ -DINI_ALLOW_MULTILINE=0 -run unittest.c > baseline_single.txt

58
rabbitmq_consumer/inih/tests/unittest.c Executable file
View File

@ -0,0 +1,58 @@
/* inih -- unit tests
This works simply by dumping a bunch of info to standard output, which is
redirected to an output file (baseline_*.txt) and checked into the Subversion
repository. This baseline file is the test output, so the idea is to check it
once, and if it changes -- look at the diff and see which tests failed.
Here's how I produced the two baseline files (with Tiny C Compiler):
tcc -DINI_ALLOW_MULTILINE=1 ../ini.c -run unittest.c > baseline_multi.txt
tcc -DINI_ALLOW_MULTILINE=0 ../ini.c -run unittest.c > baseline_single.txt
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "../ini.h"
int User;
char Prev_section[50];
int dumper(void* user, const char* section, const char* name,
const char* value)
{
User = (int)user;
if (strcmp(section, Prev_section)) {
printf("... [%s]\n", section);
strncpy(Prev_section, section, sizeof(Prev_section));
Prev_section[sizeof(Prev_section) - 1] = '\0';
}
printf("... %s=%s;\n", name, value);
return strcmp(name, "user")==0 && strcmp(value, "parse_error")==0 ? 0 : 1;
}
void parse(const char* fname) {
static int u = 100;
int e;
*Prev_section = '\0';
e = ini_parse(fname, dumper, (void*)u);
printf("%s: e=%d user=%d\n", fname, e, User);
u++;
}
int main(void)
{
parse("no_file.ini");
parse("normal.ini");
parse("bad_section.ini");
parse("bad_comment.ini");
parse("user_error.ini");
parse("multi_line.ini");
parse("bad_multi.ini");
parse("bom.ini");
return 0;
}

4
rabbitmq_consumer/inih/tests/user_error.ini Executable file
View File

@ -0,0 +1,4 @@
[section]
a = b
user = parse_error
c = d

55
rabbitmq_consumer/rabbitmq-message-consumer.spec Normal file
View File

@ -0,0 +1,55 @@
%define _topdir %(echo $PWD)/
%define name rabbitmq-message-consumer
%define release beta
%define version 1.0
%define install_path /usr/local/skysql/maxscale/extra/consumer/
BuildRoot: %{buildroot}
Summary: rabbitmq-message-consumer
License: GPL
Name: %{name}
Version: %{version}
Release: %{release}
Source: %{name}-%{version}-%{release}.tar.gz
Prefix: /
Group: Development/Tools
Requires: maxscale
%if 0%{?suse_version}
BuildRequires: gcc gcc-c++ ncurses-devel bison glibc-devel cmake libgcc_s1 perl make libtool libopenssl-devel libaio libaio-devel mariadb libedit-devel librabbitmq-devel MariaDB-shared
%else
BuildRequires: gcc gcc-c++ ncurses-devel bison glibc-devel cmake libgcc perl make libtool openssl-devel libaio libaio-devel librabbitmq-devel MariaDB-shared
%if 0%{?rhel} == 6
BuildRequires: libedit-devel
%endif
%if 0%{?rhel} == 7
BuildRequires: mariadb-devel mariadb-embedded-devel libedit-devel
%else
BuildRequires: MariaDB-devel MariaDB-server
%endif
%endif
%description
rabbitmq-message-consumer
%prep
%setup -q
%build
make clean
make
%install
mkdir -p $RPM_BUILD_ROOT%{install_path}
cp consumer $RPM_BUILD_ROOT%{install_path}
cp consumer.cnf $RPM_BUILD_ROOT%{install_path}
%clean
%files
%defattr(-,root,root)
%{install_path}/consumer
%{install_path}/consumer.cnf
%changelog

14
server/MaxScale_template.cnf
View File

@ -13,7 +13,7 @@ threads=1
# Define a monitor that can be used to determine the state and role of
# the servers.
#
# Valid options are:
# Valid options for all monitors are:
#
# module=<name of module to load>
# servers=<server name>,<server name>,...
@ -29,6 +29,16 @@ module=mysqlmon
servers=server1,server2,server3
user=maxuser
passwd=maxpwd
#
# options for mysql_monitor only
#
# detect_replication_lag=<enable detection of replication slaves lag
# via replication_heartbeat table,
# default value is 0>
# detect_stale_master=<if the replication is stopped or misconfigured
# the previous detected master will be still available
# until monitor or MaxSclale restart,
# default value is 0>
# A series of service definition
#
@ -42,6 +52,7 @@ passwd=maxpwd
# version_string=<specific string for server handshake,
# default is the MariaDB embedded library version>
#
# use_sql_variables_in=[master|all] (default all)
# router_options=<option[=value]>,<option[=value]>,...
# where value=[master|slave|synced]
#
@ -60,6 +71,7 @@ router=readwritesplit
servers=server1,server2,server3
user=maxuser
passwd=maxpwd
use_sql_variables_in=all
max_slave_connections=50%
max_slave_replication_lag=30
router_options=slave_selection_criteria=LEAST_BEHIND_MASTER

5
server/core/.gitignore vendored
View File

@ -1,3 +1,4 @@
*.o
# binaries generated here
maxscale
depend.mk
maxkeys
maxpasswd

22
server/core/Makefile
View File

@ -32,9 +32,10 @@
# are behind SS_DEBUG macros.
# 29/06/13 Vilho Raatikka Reverted Query classifier changes because
# gateway needs mysql client lib, not qc.
# 24/07/13 Mark Riddoch Addition of encryption routines
# 24/07/13 Mark Ridoch Addition of encryption routines
# 30/05/14 Mark Riddoch Filter API added
# 25/07/14 Mark Riddoch Addition of hints
# 29/08/14 Mark Riddoch Added housekeeper
include ../../build_gateway.inc
@ -48,17 +49,23 @@ CFLAGS=-c -I/usr/include -I../include -I../modules/include -I../inih \
-I$(LOGPATH) -I$(UTILSPATH) \
-Wall -g
include ../../makefile.inc
LDFLAGS=-rdynamic -L$(LOGPATH) \
-Wl,-rpath,$(DEST)/lib \
-Wl,-rpath,$(LOGPATH) -Wl,-rpath,$(UTILSPATH) \
-Wl,-rpath,$(EMBEDDED_LIB)
LIBS=-L$(EMBEDDED_LIB) \
-lmysqld \
-lz -lm -lcrypt -lcrypto -ldl -laio -lrt -pthread -llog_manager \
-L../inih/extra -linih -lssl -lstdc++
include ../../makefile.inc
SRCS= atomic.c buffer.c spinlock.c gateway.c \
gw_utils.c utils.c dcb.c load_utils.c session.c service.c server.c \
poll.c config.c users.c hashtable.c dbusers.c thread.c gwbitmask.c \
monitor.c adminusers.c secrets.c filter.c modutil.c hint.c
monitor.c adminusers.c secrets.c filter.c modutil.c hint.c housekeeper.c
HDRS= ../include/atomic.h ../include/buffer.h ../include/dcb.h \
../include/gw.h ../modules/include/mysql_client_server_protocol.h \
@ -66,18 +73,13 @@ HDRS= ../include/atomic.h ../include/buffer.h ../include/dcb.h \
../include/modules.h ../include/poll.h ../include/config.h \
../include/users.h ../include/hashtable.h ../include/gwbitmask.h \
../include/adminusers.h ../include/version.h ../include/maxscale.h \
../include/filter.h modutil.h hint.h
../include/filter.h ../include/modutil.h ../hint.h ../include/housekeeper.h
OBJ=$(SRCS:.c=.o)
KOBJS=maxkeys.o secrets.o utils.o
POBJS=maxpasswd.o secrets.o utils.o
LIBS=-L$(EMBEDDED_LIB) \
-lmysqld \
-lz -lm -lcrypt -lcrypto -ldl -laio -lrt -pthread -llog_manager \
-L../inih/extra -linih -lssl -lstdc++
all: maxscale maxkeys maxpasswd
cleantests:

22
server/core/buffer.c
View File

@ -33,6 +33,8 @@
* 16/07/2013 Massimiliano Pinto Added command type to gwbuf struct
* 24/06/2014 Mark Riddoch Addition of gwbuf_trim
* 15/07/2014 Mark Riddoch Addition of properties
* 28/08/2014 Mark Riddoch Adition of tail pointer to speed
* the gwbuf_append process
*
* @endverbatim
*/
@ -89,6 +91,7 @@ SHARED_BUF *sbuf;
sbuf->refcount = 1;
rval->sbuf = sbuf;
rval->next = NULL;
rval->tail = rval;
rval->hint = NULL;
rval->properties = NULL;
rval->gwbuf_type = GWBUF_TYPE_UNDEFINED;
@ -171,6 +174,7 @@ GWBUF *rval;
rval->gwbuf_info = buf->gwbuf_info;
rval->gwbuf_bufobj = buf->gwbuf_bufobj;
rval->next = NULL;
rval->tail = rval;
CHK_GWBUF(rval);
return rval;
}
@ -201,6 +205,7 @@ GWBUF *gwbuf_clone_portion(
clonebuf->gwbuf_info = buf->gwbuf_info;
clonebuf->gwbuf_bufobj = buf->gwbuf_bufobj;
clonebuf->next = NULL;
clonebuf->tail = clonebuf;
CHK_GWBUF(clonebuf);
return clonebuf;
@ -277,11 +282,8 @@ GWBUF *ptr = head;
if (!head)
return tail;
CHK_GWBUF(head);
while (ptr->next)
{
ptr = ptr->next;
}
ptr->next = tail;
head->tail->next = tail;
head->tail = tail->tail;
return head;
}
@ -306,6 +308,7 @@ GWBUF *
gwbuf_consume(GWBUF *head, unsigned int length)
{
GWBUF *rval = head;
CHK_GWBUF(head);
GWBUF_CONSUME(head, length);
CHK_GWBUF(head);
@ -313,8 +316,13 @@ GWBUF *rval = head;
if (GWBUF_EMPTY(head))
{
rval = head->next;
if (head->next)
head->next->tail = head->tail;
gwbuf_free(head);
}
ss_dassert(rval == NULL || (rval->end > rval->start));
return rval;
}
@ -346,6 +354,8 @@ int rval = 0;
* buffer has n_bytes or less then it will be freed and
* NULL will be returned.
*
* This routine assumes the buffer is not part of a chain
*
* @param buf The buffer to trim
* @param n_bytes The number of bytes to trim off
* @return The buffer chain or NULL if buffer has <= n_bytes
@ -353,6 +363,8 @@ int rval = 0;
GWBUF *
gwbuf_trim(GWBUF *buf, unsigned int n_bytes)
{
ss_dassert(buf->next == NULL);
if (GWBUF_LENGTH(buf) <= n_bytes)
{
gwbuf_consume(buf, GWBUF_LENGTH(buf));

360
server/core/config.c
View File

@ -34,12 +34,17 @@
* 29/05/14 Mark Riddoch Addition of filter definition
* 23/05/14 Massimiliano Pinto Added automatic set of maxscale-id: first listening ipv4_raw + port + pid
* 28/05/14 Massimiliano Pinto Added detect_replication_lag parameter
* 28/08/14 Massimiliano Pinto Added detect_stale_master parameter
* 09/09/14 Massimiliano Pinto Added localhost_match_wildcard_host parameter
* 12/09/14 Mark Riddoch Addition of checks on servers list and
* internal router suppression of messages
*
* @endverbatim
*/
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <ctype.h>
#include <ini.h>
#include <config.h>
#include <service.h>
@ -60,6 +65,7 @@ static int handle_global_item(const char *, const char *);
static void global_defaults();
static void check_config_objects(CONFIG_CONTEXT *context);
static int config_truth_value(char *str);
static int internalService(char *router);
static char *config_file = NULL;
static GATEWAY_CONF gateway;
@ -102,7 +108,7 @@ handler(void *userdata, const char *section, const char *name, const char *value
{
CONFIG_CONTEXT *cntxt = (CONFIG_CONTEXT *)userdata;
CONFIG_CONTEXT *ptr = cntxt;
CONFIG_PARAMETER *param;
CONFIG_PARAMETER *param, *p1;
if (strcmp(section, "gateway") == 0 || strcasecmp(section, "MaxScale") == 0)
{
@ -125,6 +131,23 @@ CONFIG_PARAMETER *param;
ptr->element = NULL;
cntxt->next = ptr;
}
/* Check to see if the paramter already exists for the section */
p1 = ptr->parameters;
while (p1)
{
if (!strcmp(p1->name, name))
{
LOGIF(LE, (skygw_log_write_flush(
LOGFILE_ERROR,
"Error : Configuration object '%s' has multiple "
"parameters names '%s'.",
ptr->object, name)));
return 0;
}
p1 = p1->next;
}
if ((param = (CONFIG_PARAMETER *)malloc(sizeof(CONFIG_PARAMETER))) == NULL)
return 0;
param->name = strdup(name);
@ -246,18 +269,31 @@ int error_count = 0;
{
char* max_slave_conn_str;
char* max_slave_rlag_str;
char *user;
char *auth;
char *enable_root_user;
char *weightby;
char *version_string;
bool is_rwsplit = false;
obj->element = service_alloc(obj->object, router);
char *user =
config_get_value(obj->parameters, "user");
char *auth =
config_get_value(obj->parameters, "passwd");
char *enable_root_user =
config_get_value(obj->parameters, "enable_root_user");
char *weightby =
config_get_value(obj->parameters, "weightby");
user = config_get_value(obj->parameters, "user");
auth = config_get_value(obj->parameters, "passwd");
enable_root_user = config_get_value(
obj->parameters,
"enable_root_user");
weightby = config_get_value(obj->parameters, "weightby");
char *version_string = config_get_value(obj->parameters, "version_string");
version_string = config_get_value(obj->parameters,
"version_string");
/** flag for rwsplit-specific parameters */
if (strncmp(router, "readwritesplit", strlen("readwritesplit")+1) == 0)
{
is_rwsplit = true;
}
char *allow_localhost_match_wildcard_host =
config_get_value(obj->parameters, "localhost_match_wildcard_host");
if (obj->element == NULL) /*< if module load failed */
{
@ -293,6 +329,11 @@ int error_count = 0;
if (weightby)
serviceWeightBy(obj->element, weightby);
if (allow_localhost_match_wildcard_host)
serviceEnableLocalhostMatchWildcardHost(
obj->element,
config_truth_value(allow_localhost_match_wildcard_host));
if (!auth)
auth = config_get_value(obj->parameters,
"auth");
@ -321,12 +362,19 @@ int error_count = 0;
param = config_get_param(obj->parameters,
"max_slave_connections");
if (param == NULL)
{
succp = false;
}
else
{
succp = service_set_param_value(
obj->element,
param,
max_slave_conn_str,
COUNT_ATMOST,
(COUNT_TYPE|PERCENT_TYPE));
}
if (!succp)
{
@ -353,12 +401,19 @@ int error_count = 0;
obj->parameters,
"max_slave_replication_lag");
if (param == NULL)
{
succp = false;
}
else
{
succp = service_set_param_value(
obj->element,
param,
max_slave_rlag_str,
COUNT_ATMOST,
COUNT_TYPE);
}
if (!succp)
{
@ -373,8 +428,52 @@ int error_count = 0;
param->value)));
}
}
/** Parameters for rwsplit router only */
if (is_rwsplit)
{
CONFIG_PARAMETER* param;
char* use_sql_variables_in;
bool succp;
use_sql_variables_in =
config_get_value(obj->parameters,
"use_sql_variables_in");
if (use_sql_variables_in != NULL)
{
param = config_get_param(
obj->parameters,
"use_sql_variables_in");
if (param == NULL)
{
succp = false;
}
else
{
succp = service_set_param_value(obj->element,
param,
use_sql_variables_in,
COUNT_NONE,
SQLVAR_TARGET_TYPE);
}
if (!succp)
{
LOGIF(LM, (skygw_log_write(
LOGFILE_MESSAGE,
"* Warning : invalid value type "
"for parameter \'%s.%s = %s\'\n\tExpected "
"type is [master|all] for "
"use sql variables in.",
((SERVICE*)obj->element)->name,
param->name,
param->value)));
}
}
} /*< if (rw_split) */
} /*< if (router) */
else
{
obj->element = NULL;
LOGIF(LE, (skygw_log_write_flush(
@ -518,36 +617,50 @@ int error_count = 0;
{
char *servers;
char *roptions;
char *router;
char *filters = config_get_value(obj->parameters,
"filters");
servers = config_get_value(obj->parameters, "servers");
roptions = config_get_value(obj->parameters,
"router_options");
router = config_get_value(obj->parameters, "router");
if (servers && obj->element)
{
char *s = strtok(servers, ",");
while (s)
{
CONFIG_CONTEXT *obj1 = context;
int found = 0;
while (obj1)
{
if (strcmp(trim(s), obj1->object) == 0 &&
obj->element && obj1->element)
{
found = 1;
serviceAddBackend(
obj->element,
obj1->element);
}
obj1 = obj1->next;
}
s = strtok(NULL, ",");
}
}
else if (servers == NULL)
if (!found)
{
LOGIF(LE, (skygw_log_write_flush(
LOGFILE_ERROR,
"Error : The service '%s' is missing a "
"Error: Unable to find "
"server '%s' that is "
"configured as part of "
"service '%s'.",
s, obj->object)));
}
s = strtok(NULL, ",");
}
}
else if (servers == NULL && internalService(router) == 0)
{
LOGIF(LE, (skygw_log_write_flush(
LOGFILE_ERROR,
"Warning: The service '%s' is missing a "
"definition of the servers that provide "
"the service.",
obj->object)));
@ -650,6 +763,7 @@ int error_count = 0;
char *passwd;
unsigned long interval = 0;
int replication_heartbeat = 0;
int detect_stale_master = 0;
module = config_get_value(obj->parameters, "module");
servers = config_get_value(obj->parameters, "servers");
@ -663,6 +777,10 @@ int error_count = 0;
replication_heartbeat = atoi(config_get_value(obj->parameters, "detect_replication_lag"));
}
if (config_get_value(obj->parameters, "detect_stale_master")) {
detect_stale_master = atoi(config_get_value(obj->parameters, "detect_stale_master"));
}
if (module)
{
obj->element = monitor_alloc(obj->object, module);
@ -686,22 +804,38 @@ int error_count = 0;
if(replication_heartbeat == 1)
monitorSetReplicationHeartbeat(obj->element, replication_heartbeat);
/* detect stale master */
if(detect_stale_master == 1)
monitorDetectStaleMaster(obj->element, detect_stale_master);
/* get the servers to monitor */
s = strtok(servers, ",");
while (s)
{
CONFIG_CONTEXT *obj1 = context;
int found = 0;
while (obj1)
{
if (strcmp(s, obj1->object) == 0 &&
obj->element && obj1->element)
{
found = 1;
monitorAddServer(
obj->element,
obj1->element);
}
obj1 = obj1->next;
}
if (!found)
LOGIF(LE,
(skygw_log_write_flush(
LOGFILE_ERROR,
"Error: Unable to find "
"server '%s' that is "
"configured in the "
"monitor '%s'.",
s, obj->object)));
s = strtok(NULL, ",");
}
}
@ -803,12 +937,15 @@ config_param_type_t config_get_paramtype(
return param->qfd_param_type;
}
int config_get_valint(
bool config_get_valint(
int* val,
CONFIG_PARAMETER* param,
const char* name, /*< if NULL examine current param only */
config_param_type_t ptype)
{
int val = -1; /*< -1 indicates failure */
bool succp = false;;
ss_dassert((ptype == COUNT_TYPE || ptype == PERCENT_TYPE) && param != NULL);
while (param)
{
@ -816,32 +953,95 @@ int config_get_valint(
{
switch (ptype) {
case COUNT_TYPE:
val = param->qfd.valcount;
goto return_val;
*val = param->qfd.valcount;
succp = true;
goto return_succp;
case PERCENT_TYPE:
val = param->qfd.valpercent;
goto return_val;
case BOOL_TYPE:
val = param->qfd.valbool;
goto return_val;
*val = param->qfd.valpercent;
succp =true;
goto return_succp;
default:
goto return_val;
goto return_succp;
}
}
else if (name == NULL)
{
goto return_val;
}
param = param->next;
}
return_val:
return val;
return_succp:
return succp;
}
bool config_get_valbool(
bool* val,
CONFIG_PARAMETER* param,
const char* name,
config_param_type_t ptype)
{
bool succp;
ss_dassert(ptype == BOOL_TYPE);
ss_dassert(param != NULL);
if (ptype != BOOL_TYPE || param == NULL)
{
succp = false;
goto return_succp;
}
while (param)
{
if (name == NULL || !strncmp(param->name, name, MAX_PARAM_LEN))
{
*val = param->qfd.valbool;
succp = true;
goto return_succp;
}
param = param->next;
}
succp = false;
return_succp:
return succp;
}
bool config_get_valtarget(
target_t* val,
CONFIG_PARAMETER* param,
const char* name,
config_param_type_t ptype)
{
bool succp;
ss_dassert(ptype == SQLVAR_TARGET_TYPE);
ss_dassert(param != NULL);
if (ptype != SQLVAR_TARGET_TYPE || param == NULL)
{
succp = false;
goto return_succp;
}
while (param)
{
if (name == NULL || !strncmp(param->name, name, MAX_PARAM_LEN))
{
*val = param->qfd.valtarget;
succp = true;
goto return_succp;
}
param = param->next;
}
succp = false;
return_succp:
return succp;
}
CONFIG_PARAMETER* config_clone_param(
CONFIG_PARAMETER* param)
{
@ -906,6 +1106,15 @@ config_threadcount()
return gateway.n_threads;
}
static struct {
char *logname;
logfile_id_t logfile;
} lognames[] = {
{ "log_messages", LOGFILE_MESSAGE },
{ "log_trace", LOGFILE_TRACE },
{ "log_debug", LOGFILE_DEBUG },
{ NULL, 0 }
};
/**
* Configuration handler for items in the global [MaxScale] section
*
@ -916,10 +1125,20 @@ config_threadcount()
static int
handle_global_item(const char *name, const char *value)
{
int i;
if (strcmp(name, "threads") == 0) {
gateway.n_threads = atoi(value);
} else {
return 0;
for (i = 0; lognames[i].logname; i++)
{
if (strcasecmp(name, lognames[i].logname) == 0)
{
if (atoi(value))
skygw_log_enable(lognames[i].logfile);
else
skygw_log_disable(lognames[i].logfile);
}
}
}
return 1;
}
@ -981,6 +1200,7 @@ SERVER *server;
char* max_slave_conn_str;
char* max_slave_rlag_str;
char *version_string;
char *allow_localhost_match_wildcard_host;
enable_root_user = config_get_value(obj->parameters, "enable_root_user");
@ -991,6 +1211,8 @@ SERVER *server;
version_string = config_get_value(obj->parameters, "version_string");
allow_localhost_match_wildcard_host = config_get_value(obj->parameters, "localhost_match_wildcard_host");
if (version_string) {
if (service->version_string) {
free(service->version_string);
@ -1005,6 +1227,11 @@ SERVER *server;
if (enable_root_user)
serviceEnableRootUser(service, atoi(enable_root_user));
if (allow_localhost_match_wildcard_host)
serviceEnableLocalhostMatchWildcardHost(
service,
atoi(allow_localhost_match_wildcard_host));
/** Read, validate and set max_slave_connections */
max_slave_conn_str =
config_get_value(
@ -1019,12 +1246,19 @@ SERVER *server;
param = config_get_param(obj->parameters,
"max_slave_connections");
if (param == NULL)
{
succp = false;
}
else
{
succp = service_set_param_value(
service,
param,
max_slave_conn_str,
COUNT_ATMOST,
(PERCENT_TYPE|COUNT_TYPE));
}
if (!succp)
{
@ -1055,12 +1289,19 @@ SERVER *server;
obj->parameters,
"max_slave_replication_lag");
if (param == NULL)
{
succp = false;
}
else
{
succp = service_set_param_value(
service,
param,
max_slave_rlag_str,
COUNT_ATMOST,
COUNT_TYPE);
}
if (!succp)
{
@ -1084,10 +1325,13 @@ SERVER *server;
char *user;
char *auth;
char *enable_root_user;
char *allow_localhost_match_wildcard_host;
enable_root_user =
config_get_value(obj->parameters,
"enable_root_user");
allow_localhost_match_wildcard_host =
config_get_value(obj->parameters, "localhost_match_wildcard_host");
user = config_get_value(obj->parameters,
"user");
@ -1103,6 +1347,11 @@ SERVER *server;
auth);
if (enable_root_user)
serviceEnableRootUser(service, atoi(enable_root_user));
if (allow_localhost_match_wildcard_host)
serviceEnableLocalhostMatchWildcardHost(
service,
atoi(allow_localhost_match_wildcard_host));
}
}
}
@ -1196,11 +1445,13 @@ SERVER *server;
while (s)
{
CONFIG_CONTEXT *obj1 = context;
int found = 0;
while (obj1)
{
if (strcmp(s, obj1->object) == 0 &&
obj->element && obj1->element)
{
found = 1;
if (!serviceHasBackend(obj->element, obj1->element))
{
serviceAddBackend(
@ -1210,6 +1461,16 @@ SERVER *server;
}
obj1 = obj1->next;
}
if (!found)
{
LOGIF(LE, (skygw_log_write_flush(
LOGFILE_ERROR,
"Error: Unable to find "
"server '%s' that is "
"configured as part of "
"service '%s'.",
s, obj->object)));
}
s = strtok(NULL, ",");
}
}
@ -1308,8 +1569,10 @@ static char *service_params[] =
"user",
"passwd",
"enable_root_user",
"localhost_match_wildcard_host",
"max_slave_connections",
"max_slave_replication_lag",
"use_sql_variables_in", /*< rwsplit only */
"version_string",
"filters",
NULL
@ -1346,6 +1609,7 @@ static char *monitor_params[] =
"passwd",
"monitor_interval",
"detect_replication_lag",
"detect_stale_master",
NULL
};
/**
@ -1433,6 +1697,10 @@ bool config_set_qualified_param(
succp = true;
break;
case SQLVAR_TARGET_TYPE:
param->qfd.valtarget = *(target_t *)val;
succp = true;
break;
default:
succp = false;
break;
@ -1466,3 +1734,29 @@ config_truth_value(char *str)
return atoi(str);
}
static char *InternalRouters[] = {
"debugcli",
"cli",
NULL
};
/**
* Determine if the router is one of the special internal services that
* MaxScale offers.
*
* @param router The router name
* @return Non-zero if the router is in the InternalRouters table
*/
static int
internalService(char *router)
{
int i;
if (router)
{
for (i = 0; InternalRouters[i]; i++)
if (strcmp(router, InternalRouters[i]) == 0)
return 1;
}
return 0;
}

5
server/core/dbusers.c
View File

@ -359,10 +359,10 @@ getUsers(SERVICE *service, struct users *users)
row[0],
row[1],
rc == NULL ? "NULL" : ret_ip)));
continue;
}
free(key.user);
} else {
/* setipaddress() failed, skip user add and log this*/
LOGIF(LE, (skygw_log_write_flush(
@ -380,7 +380,6 @@ getUsers(SERVICE *service, struct users *users)
memcpy(users->cksum, hash, SHA_DIGEST_LENGTH);
free(users_data);
free(key.user);
mysql_free_result(result);
mysql_close(con);

218
server/core/dcb.c
View File

@ -89,7 +89,13 @@ static int dcb_null_write(DCB *dcb, GWBUF *buf);
static int dcb_null_close(DCB *dcb);
static int dcb_null_auth(DCB *dcb, SERVER *server, SESSION *session, GWBUF *buf);
DCB* dcb_get_zombies(void)
/**
* Return the pointer to the lsit of zombie DCB's
*
* @return Zombies DCB list
*/
DCB *
dcb_get_zombies(void)
{
return zombies;
}
@ -128,6 +134,12 @@ DCB *rval;
spinlock_init(&rval->delayqlock);
spinlock_init(&rval->authlock);
spinlock_init(&rval->cb_lock);
spinlock_init(&rval->pollinlock);
spinlock_init(&rval->polloutlock);
rval->pollinbusy = 0;
rval->readcheck = 0;
rval->polloutbusy = 0;
rval->writecheck = 0;
rval->fd = -1;
memset(&rval->stats, 0, sizeof(DCBSTATS)); // Zero the statistics
rval->state = DCB_STATE_ALLOC;
@ -376,11 +388,6 @@ DCB_CALLBACK *cb;
}
spinlock_release(&dcb->cb_lock);
if (dcb->dcb_readqueue)
{
GWBUF* queue = dcb->dcb_readqueue;
while ((queue = gwbuf_consume(queue, GWBUF_LENGTH(queue))) != NULL);
}
bitmask_free(&dcb->memdata.bitmask);
simple_mutex_done(&dcb->dcb_read_lock);
simple_mutex_done(&dcb->dcb_write_lock);
@ -399,7 +406,7 @@ DCB_CALLBACK *cb;
*
* @param threadid The thread ID of the caller
*/
DCB*
DCB *
dcb_process_zombies(int threadid)
{
DCB *ptr, *lptr;
@ -1202,6 +1209,19 @@ printDCB(DCB *dcb)
printf("\t\tNo. of Low Water Events: %d\n",
dcb->stats.n_low_water);
}
/**
* Display an entry from the spinlock statistics data
*
* @param dcb The DCB to print to
* @param desc Description of the statistic
* @param value The statistic value
*/
static void
spin_reporter(void *dcb, char *desc, int value)
{
dcb_printf((DCB *)dcb, "\t\t%-35s %d\n", desc, value);
}
/**
* Diagnostic to print all DCB allocated in the system
@ -1231,6 +1251,12 @@ void dprintAllDCBs(DCB *pdcb)
DCB *dcb;
spinlock_acquire(&dcbspin);
#if SPINLOCK_PROFILE
dcb_printf(pdcb, "DCB List Spinlock Statistics:\n");
spinlock_stats(&dcbspin, spin_reporter, pdcb);
dcb_printf(pdcb, "Zombie Queue Lock Statistics:\n");
spinlock_stats(&zombiespin, spin_reporter, pdcb);
#endif
dcb = allDCBs;
while (dcb)
{
@ -1254,6 +1280,10 @@ DCB *dcb;
dcb_printf(pdcb, "\t\tNo. of Writes: %d\n", dcb->stats.n_writes);
dcb_printf(pdcb, "\t\tNo. of Buffered Writes: %d\n", dcb->stats.n_buffered);
dcb_printf(pdcb, "\t\tNo. of Accepts: %d\n", dcb->stats.n_accepts);
dcb_printf(pdcb, "\t\tNo. of busy polls: %d\n", dcb->stats.n_busypolls);
dcb_printf(pdcb, "\t\tNo. of read rechecks: %d\n", dcb->stats.n_readrechecks);
dcb_printf(pdcb, "\t\tNo. of busy write polls: %d\n", dcb->stats.n_busywrpolls);
dcb_printf(pdcb, "\t\tNo. of write rechecks: %d\n", dcb->stats.n_writerechecks);
dcb_printf(pdcb, "\t\tNo. of High Water Events: %d\n", dcb->stats.n_high_water);
dcb_printf(pdcb, "\t\tNo. of Low Water Events: %d\n", dcb->stats.n_low_water);
if (dcb->flags & DCBF_CLONE)
@ -1276,20 +1306,20 @@ DCB *dcb;
spinlock_acquire(&dcbspin);
dcb = allDCBs;
dcb_printf(pdcb, "Descriptor Control Blocks\n");
dcb_printf(pdcb, "------------+----------------------------+----------------------+----------\n");
dcb_printf(pdcb, " %-10s | %-26s | %-20s | %s\n",
dcb_printf(pdcb, "------------------+----------------------------+--------------------+----------\n");
dcb_printf(pdcb, " %-16s | %-26s | %-18s | %s\n",
"DCB", "State", "Service", "Remote");
dcb_printf(pdcb, "------------+----------------------------+----------------------+----------\n");
dcb_printf(pdcb, "------------------+----------------------------+--------------------+----------\n");
while (dcb)
{
dcb_printf(pdcb, " %10p | %-26s | %-20s | %s\n",
dcb_printf(pdcb, " %-16p | %-26s | %-18s | %s\n",
dcb, gw_dcb_state2string(dcb->state),
(dcb->session->service ?
dcb->session->service->name : ""),
((dcb->session && dcb->session->service) ? dcb->session->service->name : ""),
(dcb->remote ? dcb->remote : ""));
dcb = dcb->next;
}
dcb_printf(pdcb, "------------+----------------------------+----------------------+----------\n\n");
dcb_printf(pdcb, "------------------+----------------------------+--------------------+----------\n\n");
spinlock_release(&dcbspin);
}
@ -1306,16 +1336,16 @@ DCB *dcb;
spinlock_acquire(&dcbspin);
dcb = allDCBs;
dcb_printf(pdcb, "Client Connections\n");
dcb_printf(pdcb, "-----------------+------------+----------------------+------------\n");
dcb_printf(pdcb, " %-15s | %-10s | %-20s | %s\n",
dcb_printf(pdcb, "-----------------+------------------+----------------------+------------\n");
dcb_printf(pdcb, " %-15s | %-16s | %-20s | %s\n",
"Client", "DCB", "Service", "Session");
dcb_printf(pdcb, "-----------------+------------+----------------------+------------\n");
dcb_printf(pdcb, "-----------------+------------------+----------------------+------------\n");
while (dcb)
{
if (dcb_isclient(dcb)
&& dcb->dcb_role == DCB_ROLE_REQUEST_HANDLER)
{
dcb_printf(pdcb, " %-15s | %10p | %-20s | %10p\n",
dcb_printf(pdcb, " %-15s | %16p | %-20s | %10p\n",
(dcb->remote ? dcb->remote : ""),
dcb, (dcb->session->service ?
dcb->session->service->name : ""),
@ -1323,7 +1353,7 @@ DCB *dcb;
}
dcb = dcb->next;
}
dcb_printf(pdcb, "-----------------+------------+----------------------+------------\n\n");
dcb_printf(pdcb, "-----------------+------------------+----------------------+------------\n\n");
spinlock_release(&dcbspin);
}
@ -1350,6 +1380,8 @@ dprintDCB(DCB *pdcb, DCB *dcb)
dcb_printf(pdcb, "\tOwning Session: %p\n", dcb->session);
if (dcb->writeq)
dcb_printf(pdcb, "\tQueued write data: %d\n", gwbuf_length(dcb->writeq));
if (dcb->delayq)
dcb_printf(pdcb, "\tDelayed write data: %d\n", gwbuf_length(dcb->delayq));
dcb_printf(pdcb, "\tStatistics:\n");
dcb_printf(pdcb, "\t\tNo. of Reads: %d\n",
dcb->stats.n_reads);
@ -1359,12 +1391,30 @@ dprintDCB(DCB *pdcb, DCB *dcb)
dcb->stats.n_buffered);
dcb_printf(pdcb, "\t\tNo. of Accepts: %d\n",
dcb->stats.n_accepts);
dcb_printf(pdcb, "\t\tNo. of busy polls: %d\n", dcb->stats.n_busypolls);
dcb_printf(pdcb, "\t\tNo. of read rechecks: %d\n", dcb->stats.n_readrechecks);
dcb_printf(pdcb, "\t\tNo. of busy write polls: %d\n", dcb->stats.n_busywrpolls);
dcb_printf(pdcb, "\t\tNo. of write rechecks: %d\n", dcb->stats.n_writerechecks);
dcb_printf(pdcb, "\t\tNo. of High Water Events: %d\n",
dcb->stats.n_high_water);
dcb_printf(pdcb, "\t\tNo. of Low Water Events: %d\n",
dcb->stats.n_low_water);
if (dcb->flags & DCBF_CLONE)
dcb_printf(pdcb, "\t\tDCB is a clone.\n");
#if SPINLOCK_PROFILE
dcb_printf(pdcb, "\tInitlock Statistics:\n");
spinlock_stats(&dcb->dcb_initlock, spin_reporter, pdcb);
dcb_printf(pdcb, "\tWrite Queue Lock Statistics:\n");
spinlock_stats(&dcb->writeqlock, spin_reporter, pdcb);
dcb_printf(pdcb, "\tDelay Queue Lock Statistics:\n");
spinlock_stats(&dcb->delayqlock, spin_reporter, pdcb);
dcb_printf(pdcb, "\tPollin Lock Statistics:\n");
spinlock_stats(&dcb->pollinlock, spin_reporter, pdcb);
dcb_printf(pdcb, "\tPollout Lock Statistics:\n");
spinlock_stats(&dcb->polloutlock, spin_reporter, pdcb);
dcb_printf(pdcb, "\tCallback Lock Statistics:\n");
spinlock_stats(&dcb->cb_lock, spin_reporter, pdcb);
#endif
}
/**
@ -1510,9 +1560,12 @@ static bool dcb_set_state_nomutex(
case DCB_STATE_ALLOC:
switch (new_state) {
case DCB_STATE_POLLING: /*< for client requests */
case DCB_STATE_LISTENING: /*< for connect listeners */
case DCB_STATE_DISCONNECTED: /*< for failed connections */
/** fall through, for client requests */
case DCB_STATE_POLLING:
/** fall through, for connect listeners */
case DCB_STATE_LISTENING:
/** for failed connections */
case DCB_STATE_DISCONNECTED:
dcb->state = new_state;
succp = true;
break;
@ -1548,7 +1601,7 @@ static bool dcb_set_state_nomutex(
case DCB_STATE_NOPOLLING:
switch (new_state) {
case DCB_STATE_ZOMBIE:
case DCB_STATE_ZOMBIE: /*< fall through */
dcb->state = new_state;
case DCB_STATE_POLLING: /*< ok to try but state can't change */
succp = true;
@ -1561,7 +1614,7 @@ static bool dcb_set_state_nomutex(
case DCB_STATE_ZOMBIE:
switch (new_state) {
case DCB_STATE_DISCONNECTED:
case DCB_STATE_DISCONNECTED: /*< fall through */
dcb->state = new_state;
case DCB_STATE_POLLING: /*< ok to try but state can't change */
succp = true;
@ -1717,10 +1770,7 @@ int gw_write(
* @return Non-zero (true) if the callback was added
*/
int
dcb_add_callback(
DCB *dcb,
DCB_REASON reason,
int (*callback)(struct dcb *, DCB_REASON, void *), void *userdata)
dcb_add_callback(DCB *dcb, DCB_REASON reason, int (*callback)(struct dcb *, DCB_REASON, void *), void *userdata)
{
DCB_CALLBACK *cb, *ptr;
int rval = 1;
@ -1752,7 +1802,10 @@ int rval = 1;
return 0;
}
if (cb->next == NULL)
{
cb->next = ptr;
break;
}
cb = cb->next;
}
spinlock_release(&dcb->cb_lock);
@ -1773,7 +1826,7 @@ int rval = 1;
* @return Non-zero (true) if the callback was removed
*/
int
dcb_remove_callback(DCB *dcb, DCB_REASON reason, int (*callback)(struct dcb *, DCB_REASON), void *userdata)
dcb_remove_callback(DCB *dcb, DCB_REASON reason, int (*callback)(struct dcb *, DCB_REASON, void *), void *userdata)
{
DCB_CALLBACK *cb, *pcb = NULL;
int rval = 0;
@ -1866,8 +1919,102 @@ int rval = 0;
return rval;
}
static DCB* dcb_get_next (
DCB* dcb)
/**
* Called by the EPOLLIN event. Take care of calling the protocol
* read entry point and managing multiple threads competing for the DCB
* without blocking those threads.
*
* This mechanism does away with the need for a mutex on the EPOLLIN event
* and instead implements a queuing mechanism in which nested events are
* queued on the DCB such that when the thread processing the first event
* returns it will read the queued event and process it. This allows the
* thread that woudl otherwise have to wait to process the nested event
* to return immediately and and process other events.
*
* @param dcb The DCB that has data available
*/
void
dcb_pollin(DCB *dcb, int thread_id)
{
spinlock_acquire(&dcb->pollinlock);
if (dcb->pollinbusy == 0)
{
dcb->pollinbusy = 1;
do {
if (dcb->readcheck)
{
dcb->stats.n_readrechecks++;
dcb_process_zombies(thread_id);
}
dcb->readcheck = 0;
spinlock_release(&dcb->pollinlock);
dcb->func.read(dcb);
spinlock_acquire(&dcb->pollinlock);
} while (dcb->readcheck);
dcb->pollinbusy = 0;
}
else
{
dcb->stats.n_busypolls++;
dcb->readcheck = 1;
}
spinlock_release(&dcb->pollinlock);
}
/**
* Called by the EPOLLOUT event. Take care of calling the protocol
* write_ready entry point and managing multiple threads competing for the DCB
* without blocking those threads.
*
* This mechanism does away with the need for a mutex on the EPOLLOUT event
* and instead implements a queuing mechanism in which nested events are
* queued on the DCB such that when the thread processing the first event
* returns it will read the queued event and process it. This allows the
* thread that would otherwise have to wait to process the nested event
* to return immediately and and process other events.
*
* @param dcb The DCB thats available for writes
*/
void
dcb_pollout(DCB *dcb, int thread_id)
{
spinlock_acquire(&dcb->polloutlock);
if (dcb->polloutbusy == 0)
{
dcb->polloutbusy = 1;
do {
if (dcb->writecheck)
{
dcb_process_zombies(thread_id);
dcb->stats.n_writerechecks++;
}
dcb->writecheck = 0;
spinlock_release(&dcb->polloutlock);
dcb->func.write_ready(dcb);
spinlock_acquire(&dcb->polloutlock);
} while (dcb->writecheck);
dcb->polloutbusy = 0;
}
else
{
dcb->stats.n_busywrpolls++;
dcb->writecheck = 1;
}
spinlock_release(&dcb->polloutlock);
}
/**
* Get the next DCB in the list of all DCB's
*
* @param dcb The current DCB
* @return The pointer to the next DCB or NULL if this is the last
*/
static DCB *
dcb_get_next (DCB* dcb)
{
DCB* p;
@ -1901,8 +2048,13 @@ static DCB* dcb_get_next (
return dcb;
}
void dcb_call_foreach (
DCB_REASON reason)
/**
* Call all the callbacks on all DCB's that match the reason given
*
* @param reason The DCB_REASON that triggers the callback
*/
void
dcb_call_foreach(DCB_REASON reason)
{
switch (reason) {
case DCB_REASON_CLOSE:

12
server/core/filter.c
View File

@ -311,7 +311,8 @@ int i;
* @param filter The filter to add into the chain
* @param session The client session
* @param downstream The filter downstream of this filter
* @return The downstream component for the next filter
* @return The downstream component for the next filter or NULL
* if the filter could not be created
*/
DOWNSTREAM *
filterApply(FILTER_DEF *filter, SESSION *session, DOWNSTREAM *downstream)
@ -331,8 +332,13 @@ DOWNSTREAM *me;
}
}
if (filter->filter == NULL)
filter->filter = (filter->obj->createInstance)(filter->options,
filter->parameters);
{
if ((filter->filter = (filter->obj->createInstance)(filter->options,
filter->parameters)) == NULL)
{
return NULL;
}
}
if ((me = (DOWNSTREAM *)calloc(1, sizeof(DOWNSTREAM))) == NULL)
{
return NULL;

162
server/core/gateway.c
View File

@ -51,6 +51,7 @@
#include <modules.h>
#include <config.h>
#include <poll.h>
#include <housekeeper.h>
#include <stdlib.h>
#include <unistd.h>
@ -65,6 +66,8 @@
# include <skygw_utils.h>
# include <log_manager.h>
#include <execinfo.h>
/** for procname */
#define _GNU_SOURCE
@ -189,6 +192,53 @@ sigint_handler (int i)
fprintf(stderr, "\n\nShutting down MaxScale\n\n");
}
int fatal_handling = 0;
static int signal_set (int sig, void (*handler)(int));
static void
sigfatal_handler (int i)
{
if (fatal_handling) {
fprintf(stderr, "Fatal signal %d while backtracing\n", i);
_exit(1);
}
fatal_handling = 1;
fprintf(stderr, "\n\nMaxScale received fatal signal %d\n", i);
LOGIF(LE, (skygw_log_write_flush(
LOGFILE_ERROR,
"Fatal: MaxScale received fatal signal %d. Attempting backtrace.", i)));
{
void *addrs[128];
char **strings= NULL;
int n, count = backtrace(addrs, 128);
char** symbols = backtrace_symbols( addrs, count );
if (symbols) {
for( n = 0; n < count; n++ ) {
LOGIF(LE, (skygw_log_write_flush(
LOGFILE_ERROR,
" %s\n", symbols[n])));
}
free(symbols);
} else {
fprintf(stderr, "\nresolving symbols to error log failed, writing call trace to stderr:\n");
backtrace_symbols_fd(addrs, count, fileno(stderr));
}
}
/* re-raise signal to enforce core dump */
fprintf(stderr, "\n\nWriting core dump\n");
signal_set(i, SIG_DFL);
raise(i);
}
/**
* @node Wraps sigaction calls
*
@ -1080,6 +1130,68 @@ int main(int argc, char **argv)
rc = MAXSCALE_INTERNALERROR;
goto return_main;
}
r = sigdelset(&sigset, SIGSEGV);
if (r != 0)
{
char* logerr = "Failed to delete signal SIGSEGV from the "
"signal set of MaxScale. Exiting.";
eno = errno;
errno = 0;
print_log_n_stderr(true, true, fprerr, logerr, eno);
rc = MAXSCALE_INTERNALERROR;
goto return_main;
}
r = sigdelset(&sigset, SIGABRT);
if (r != 0)
{
char* logerr = "Failed to delete signal SIGABRT from the "
"signal set of MaxScale. Exiting.";
eno = errno;
errno = 0;
print_log_n_stderr(true, true, fprerr, logerr, eno);
rc = MAXSCALE_INTERNALERROR;
goto return_main;
}
r = sigdelset(&sigset, SIGILL);
if (r != 0)
{
char* logerr = "Failed to delete signal SIGILL from the "
"signal set of MaxScale. Exiting.";
eno = errno;
errno = 0;
print_log_n_stderr(true, true, fprerr, logerr, eno);
rc = MAXSCALE_INTERNALERROR;
goto return_main;
}
r = sigdelset(&sigset, SIGFPE);
if (r != 0)
{
char* logerr = "Failed to delete signal SIGFPE from the "
"signal set of MaxScale. Exiting.";
eno = errno;
errno = 0;
print_log_n_stderr(true, true, fprerr, logerr, eno);
rc = MAXSCALE_INTERNALERROR;
goto return_main;
}
#ifdef SIGBUS
r = sigdelset(&sigset, SIGBUS);
if (r != 0)
{
char* logerr = "Failed to delete signal SIGBUS from the "
"signal set of MaxScale. Exiting.";
eno = errno;
errno = 0;
print_log_n_stderr(true, true, fprerr, logerr, eno);
rc = MAXSCALE_INTERNALERROR;
goto return_main;
}
#endif
r = sigprocmask(SIG_SETMASK, &sigset, NULL);
if (r != 0) {
@ -1094,7 +1206,7 @@ int main(int argc, char **argv)
gw_daemonize();
}
/*<
* Set signal handlers for SIGHUP, SIGTERM, and SIGINT.
* Set signal handlers for SIGHUP, SIGTERM, SIGINT and critical signals like SIGSEGV.
*/
{
char* fprerr = "Failed to initialize signal handlers. Exiting.";
@ -1123,6 +1235,48 @@ int main(int argc, char **argv)
"SIGINT. Exiting.");
goto sigset_err;
}
l = signal_set(SIGSEGV, sigfatal_handler);
if (l != 0)
{
logerr = strdup("Failed to set signal handler for "
"SIGSEGV. Exiting.");
goto sigset_err;
}
l = signal_set(SIGABRT, sigfatal_handler);
if (l != 0)
{
logerr = strdup("Failed to set signal handler for "
"SIGABRT. Exiting.");
goto sigset_err;
}
l = signal_set(SIGILL, sigfatal_handler);
if (l != 0)
{
logerr = strdup("Failed to set signal handler for "
"SIGILL. Exiting.");
goto sigset_err;
}
l = signal_set(SIGFPE, sigfatal_handler);
if (l != 0)
{
logerr = strdup("Failed to set signal handler for "
"SIGFPE. Exiting.");
goto sigset_err;
}
#ifdef SIGBUS
l = signal_set(SIGBUS, sigfatal_handler);
if (l != 0)
{
logerr = strdup("Failed to set signal handler for "
"SIGBUS. Exiting.");
goto sigset_err;
}
#endif
sigset_err:
if (l != 0)
{
@ -1362,6 +1516,12 @@ int main(int argc, char **argv)
log_flush_thr = thread_start(
log_flush_cb,
(void *)&log_flush_timeout_ms);
/*
* Start the housekeeper thread
*/
hkinit();
/*<
* Start the polling threads, note this is one less than is
* configured as the main thread will also poll.

24
server/core/hashtable.c
View File

@ -28,7 +28,7 @@
* and value and to free them.
*
* The hashtable is arrange as a set of linked lists, the number of linked
* lists beign the hashsize as requested by the user. Entries are hashed by
* lists being the hashsize as requested by the user. Entries are hashed by
* calling the hash function that is passed in by the user, this is used as
* an index into the array of linked lists, usign modulo hashsize.
*
@ -83,9 +83,13 @@ nullfn(void *data)
}
/**
* Allocate a new hash table
* Allocate a new hash table.
*
* @param size The size of the hash table
* The hashtable must have a size of at least one, however to be of any
* practical use a larger size sould be chosen as the size relates to the number
* of has buckets in the table.
*
* @param size The size of the hash table, msut be > 0
* @param hashfn The user supplied hash function
* @param cmpfn The user supplied key comparison function
* @return The hashtable table
@ -130,7 +134,7 @@ hashtable_alloc_real(
rval->ht_chk_top = CHK_NUM_HASHTABLE;
rval->ht_chk_tail = CHK_NUM_HASHTABLE;
#endif
rval->hashsize = size;
rval->hashsize = size > 0 ? size : 1;
rval->hashfn = hashfn;
rval->cmpfn = cmpfn;
rval->kcopyfn = nullfn;
@ -140,12 +144,12 @@ hashtable_alloc_real(
rval->n_readers = 0;
rval->writelock = 0;
spinlock_init(&rval->spin);
if ((rval->entries = (HASHENTRIES **)calloc(size, sizeof(HASHENTRIES *))) == NULL)
if ((rval->entries = (HASHENTRIES **)calloc(rval->hashsize, sizeof(HASHENTRIES *))) == NULL)
{
free(rval);
return NULL;
}
memset(rval->entries, 0, size * sizeof(HASHENTRIES *));
memset(rval->entries, 0, rval->hashsize * sizeof(HASHENTRIES *));
return rval;
}
@ -254,6 +258,7 @@ hashtable_add(HASHTABLE *table, void *key, void *value)
/* check succesfull key copy */
if ( ptr->key == NULL) {
hashtable_write_unlock(table);
return 0;
}
@ -266,6 +271,7 @@ hashtable_add(HASHTABLE *table, void *key, void *value)
table->kfreefn(ptr->key);
/* value not copied, return */
hashtable_write_unlock(table);
return 0;
}
@ -582,10 +588,10 @@ HASHENTRIES *entries;
iter->depth++;
while (iter->chain < iter->table->hashsize)
{
hashtable_read_lock(iter->table);
if ((entries = iter->table->entries[iter->chain]) != NULL)
{
i = 0;
hashtable_read_lock(iter->table);
while (entries && i < iter->depth)
{
entries = entries->next;
@ -595,6 +601,10 @@ HASHENTRIES *entries;
if (entries)
return entries->key;
}
else
{
hashtable_read_unlock(iter->table);
}
iter->depth = 0;
iter->chain++;
}

195
server/core/housekeeper.c Normal file
View File

@ -0,0 +1,195 @@
/*
* This file is distributed as part of the SkySQL Gateway. It is free
* software: you can redistribute it and/or modify it under the terms of the
* GNU General Public License as published by the Free Software Foundation,
* version 2.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
* FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
* details.
*
* You should have received a copy of the GNU General Public License along with
* this program; if not, write to the Free Software Foundation, Inc., 51
* Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Copyright SkySQL Ab 2014
*/
#include <stdlib.h>
#include <housekeeper.h>
#include <thread.h>
#include <spinlock.h>
/**
* @file housekeeper.c Provide a mechanism to run periodic tasks
*
* @verbatim
* Revision History
*
* Date Who Description
* 29/08/14 Mark Riddoch Initial implementation
*
* @endverbatim
*/
/**
* List of all tasks that need to be run
*/
static HKTASK *tasks = NULL;
/**
* Spinlock to protect the tasks list
*/
static SPINLOCK tasklock = SPINLOCK_INIT;
static void hkthread(void *);
/**
* Initialise the housekeeper thread
*/
void
hkinit()
{
thread_start(hkthread, NULL);
}
/**
* Add a new task to the housekeepers lists of tasks that should be
* run periodically.
*
* The task will be first run frequency seconds after this call is
* made and will the be executed repeatedly every frequency seconds
* until the task is removed.
*
* Task names must be unique.
*
* @param name The unique name for this housekeeper task
* @param taskfn The function to call for the task
* @param data Data to pass to the task function
* @param frequency How often to run the task, expressed in seconds
* @return Return the tiem in seconds when the task will be first run if the task was added, otherwise 0
*/
int
hktask_add(char *name, void (*taskfn)(void *), void *data, int frequency)
{
HKTASK *task, *ptr;
if ((task = (HKTASK *)malloc(sizeof(HKTASK))) == NULL)
{
return 0;
}
if ((task->name = strdup(name)) == NULL)
{
free(task);
return 0;
}
task->task = taskfn;
task->data = data;
task->frequency = frequency;
task->nextdue = time(0) + frequency;
task->next = NULL;
spinlock_acquire(&tasklock);
ptr = tasks;
while (ptr && ptr->next)
{
if (strcmp(ptr->name, name) == 0)
{
spinlock_release(&tasklock);
free(task->name);
free(task);
return 0;
}
ptr = ptr->next;
}
if (ptr)
ptr->next = task;
else
tasks = task;
spinlock_release(&tasklock);
return task->nextdue;
}
/**
* Remove a named task from the housekeepers task list
*
* @param name The task name to remove
* @return Returns 0 if the task could not be removed
*/
int
hktask_remove(char *name)
{
HKTASK *ptr, *lptr = NULL;
spinlock_acquire(&tasklock);
ptr = tasks;
while (ptr && strcmp(ptr->name, name) != 0)
{
lptr = ptr;
ptr = ptr->next;
}
if (ptr && lptr)
lptr->next = ptr->next;
else if (ptr)
tasks = ptr->next;
spinlock_release(&tasklock);
if (ptr)
{
free(ptr->name);
free(ptr);
return 1;
}
else
{
return 0;
}
}
/**
* The housekeeper thread implementation.
*
* This function is responsible for executing the housekeeper tasks.
*
* The implementation of the callng of the task functions is such that
* the tasks are called without the tasklock spinlock being held. This
* allows manipulation of the housekeeper task list during execution of
* one of the tasks. The resutl is that upon completion of a task the
* search for tasks to run must restart from the start of the queue.
* It is vital that the task->nextdue tiem is updated before the task
* is run.
*
* @param data Unused, here to satisfy the thread system
*/
void
hkthread(void *data)
{
HKTASK *ptr;
time_t now;
void (*taskfn)(void *);
void *taskdata;
for (;;)
{
thread_millisleep(1000);
now = time(0);
spinlock_acquire(&tasklock);
ptr = tasks;
while (ptr)
{
if (ptr->nextdue <= now)
{
ptr->nextdue = now + ptr->frequency;
taskfn = ptr->task;
taskdata = ptr->data;
spinlock_release(&tasklock);
(*taskfn)(taskdata);
spinlock_acquire(&tasklock);
ptr = tasks;
}
else
ptr = ptr->next;
}
spinlock_release(&tasklock);
}
}

21
server/core/monitor.c
View File

@ -208,7 +208,6 @@ MONITOR *ptr;
* Show a single monitor
*
* @param dcb DCB for printing output
* @param monitor The monitor to print information regarding
*/
void
monitorShow(DCB *dcb, MONITOR *monitor)
@ -304,12 +303,26 @@ monitorSetInterval (MONITOR *mon, unsigned long interval)
* Enable Replication Heartbeat support in monitor.
*
* @param mon The monitor instance
* @param replication_heartbeat The replication heartbeat
* @param enable The enabling value is 1, 0 turns it off
*/
void
monitorSetReplicationHeartbeat(MONITOR *mon, int replication_heartbeat)
monitorSetReplicationHeartbeat(MONITOR *mon, int enable)
{
if (mon->module->replicationHeartbeat != NULL) {
mon->module->replicationHeartbeat(mon->handle, replication_heartbeat);
mon->module->replicationHeartbeat(mon->handle, enable);
}
}
/**
* Enable Stale Master assignement.
*
* @param mon The monitor instance
* @param enable The enabling value is 1, 0 turns it off
*/
void
monitorDetectStaleMaster(MONITOR *mon, int enable)
{
if (mon->module->detectStaleMaster != NULL) {
mon->module->detectStaleMaster(mon->handle, enable);
}
}

375
server/core/poll.c
View File

@ -28,6 +28,8 @@
#include <skygw_utils.h>
#include <log_manager.h>
#include <gw.h>
#include <config.h>
#include <housekeeper.h>
extern int lm_enabled_logfiles_bitmask;
@ -41,6 +43,8 @@ extern int lm_enabled_logfiles_bitmask;
* 19/06/13 Mark Riddoch Initial implementation
* 28/06/13 Mark Riddoch Added poll mask support and DCB
* zombie management
* 29/08/14 Mark Riddoch Addition of thread status data, load average
* etc.
*
* @endverbatim
*/
@ -49,6 +53,53 @@ static int epoll_fd = -1; /*< The epoll file descriptor */
static int do_shutdown = 0; /*< Flag the shutdown of the poll subsystem */
static GWBITMASK poll_mask;
static simple_mutex_t epoll_wait_mutex; /*< serializes calls to epoll_wait */
static int n_waiting = 0; /*< No. of threads in epoll_wait */
/**
* Thread load average, this is the average number of descriptors in each
* poll completion, a value of 1 or less is the ideal.
*/
static double load_average = 0.0;
static int load_samples = 0;
static int load_nfds = 0;
static double current_avg = 0.0;
static double *avg_samples = NULL;
static int next_sample = 0;
static int n_avg_samples;
/* Thread statistics data */
static int n_threads; /*< No. of threads */
/**
* Internal MaxScale thread states
*/
typedef enum { THREAD_STOPPED, THREAD_IDLE,
THREAD_POLLING, THREAD_PROCESSING,
THREAD_ZPROCESSING } THREAD_STATE;
/**
* Thread data used to report the current state and activity related to
* a thread
*/
typedef struct {
THREAD_STATE state; /*< Current thread state */
int n_fds; /*< No. of descriptors thread is processing */
DCB *cur_dcb; /*< Current DCB being processed */
uint32_t event; /*< Current event being processed */
} THREAD_DATA;
static THREAD_DATA *thread_data = NULL; /*< Status of each thread */
/**
* The number of buckets used to gather statistics about how many
* descriptors where processed on each epoll completion.
*
* An array of wakeup counts is created, with the number of descriptors used
* to index that array. Each time a completion occurs the n_fds - 1 value is
* used to index this array and increment the count held there.
* If n_fds - 1 >= MAXFDS then the count at MAXFDS -1 is incremented.
*/
#define MAXNFDS 10
/**
* The polling statistics
@ -60,8 +111,20 @@ static struct {
int n_hup; /*< Number of hangup events */
int n_accept; /*< Number of accept events */
int n_polls; /*< Number of poll cycles */
int n_nothreads; /*< Number of times no threads are polling */
int n_fds[MAXNFDS]; /*< Number of wakeups with particular
n_fds value */
} pollStats;
/**
* How frequently to call the poll_loadav function used to monitor the load
* average of the poll subsystem.
*/
#define POLL_LOAD_FREQ 10
/**
* Periodic function to collect load data for average calculations
*/
static void poll_loadav(void *);
/**
* Initialise the polling system we are using for the gateway.
@ -71,6 +134,8 @@ static struct {
void
poll_init()
{
int i;
if (epoll_fd != -1)
return;
if ((epoll_fd = epoll_create(MAX_EVENTS)) == -1)
@ -80,7 +145,23 @@ poll_init()
}
memset(&pollStats, 0, sizeof(pollStats));
bitmask_init(&poll_mask);
n_threads = config_threadcount();
if ((thread_data =
(THREAD_DATA *)malloc(n_threads * sizeof(THREAD_DATA))) != NULL)
{
for (i = 0; i < n_threads; i++)
{
thread_data[i].state = THREAD_STOPPED;
}
}
simple_mutex_init(&epoll_wait_mutex, "epoll_wait_mutex");
hktask_add("Load Average", poll_loadav, NULL, POLL_LOAD_FREQ);
n_avg_samples = 15 * 60 / POLL_LOAD_FREQ;
avg_samples = (double *)malloc(sizeof(double *) * n_avg_samples);
for (i = 0; i < n_avg_samples; i++)
avg_samples[i] = 0.0;
}
/**
@ -100,7 +181,7 @@ poll_add_dcb(DCB *dcb)
CHK_DCB(dcb);
ev.events = EPOLLIN | EPOLLOUT | EPOLLET;
ev.events = EPOLLIN | EPOLLOUT | EPOLLRDHUP | EPOLLHUP | EPOLLET;
ev.data.ptr = dcb;
/*<
@ -245,22 +326,29 @@ return_rc:
void
poll_waitevents(void *arg)
{
struct epoll_event events[MAX_EVENTS];
int i, nfds;
int thread_id = (int)arg;
bool no_op = false;
static bool process_zombies_only = false; /*< flag for all threads */
DCB *zombies = NULL;
struct epoll_event events[MAX_EVENTS];
int i, nfds;
int thread_id = (int)arg;
bool no_op = false;
static bool process_zombies_only = false; /*< flag for all threads */
DCB *zombies = NULL;
/** Add this thread to the bitmask of running polling threads */
bitmask_set(&poll_mask, thread_id);
if (thread_data)
{
thread_data[thread_id].state = THREAD_IDLE;
}
/** Init mysql thread context for use with a mysql handle and a parser */
mysql_thread_init();
while (1)
{
atomic_add(&n_waiting, 1);
#if BLOCKINGPOLL
nfds = epoll_wait(epoll_fd, events, MAX_EVENTS, -1);
atomic_add(&n_waiting, -1);
#else /* BLOCKINGPOLL */
if (!no_op) {
LOGIF(LD, (skygw_log_write(
@ -274,9 +362,14 @@ poll_waitevents(void *arg)
#if 0
simple_mutex_lock(&epoll_wait_mutex, TRUE);
#endif
if (thread_data)
{
thread_data[thread_id].state = THREAD_POLLING;
}
if ((nfds = epoll_wait(epoll_fd, events, MAX_EVENTS, 0)) == -1)
{
atomic_add(&n_waiting, -1);
int eno = errno;
errno = 0;
LOGIF(LD, (skygw_log_write(
@ -290,6 +383,7 @@ poll_waitevents(void *arg)
}
else if (nfds == 0)
{
atomic_add(&n_waiting, -1);
if (process_zombies_only) {
#if 0
simple_mutex_unlock(&epoll_wait_mutex);
@ -312,6 +406,13 @@ poll_waitevents(void *arg)
}
}
}
else
{
atomic_add(&n_waiting, -1);
}
if (n_waiting == 0)
atomic_add(&pollStats.n_nothreads, 1);
#if 0
simple_mutex_unlock(&epoll_wait_mutex);
#endif
@ -324,6 +425,20 @@ poll_waitevents(void *arg)
pthread_self(),
nfds)));
atomic_add(&pollStats.n_polls, 1);
if (thread_data)
{
thread_data[thread_id].n_fds = nfds;
thread_data[thread_id].cur_dcb = NULL;
thread_data[thread_id].event = 0;
thread_data[thread_id].state = THREAD_PROCESSING;
}
pollStats.n_fds[(nfds < MAXNFDS ? (nfds - 1) : MAXNFDS - 1)]++;
load_average = (load_average * load_samples + nfds)
/ (load_samples + 1);
atomic_add(&load_samples, 1);
atomic_add(&load_nfds, nfds);
for (i = 0; i < nfds; i++)
{
@ -331,6 +446,11 @@ poll_waitevents(void *arg)
__uint32_t ev = events[i].events;
CHK_DCB(dcb);
if (thread_data)
{
thread_data[thread_id].cur_dcb = dcb;
thread_data[thread_id].event = ev;
}
#if defined(SS_DEBUG)
if (dcb_fake_write_ev[dcb->fd] != 0) {
@ -366,6 +486,7 @@ poll_waitevents(void *arg)
eno = gw_getsockerrno(dcb->fd);
if (eno == 0) {
#if MUTEX_BLOCK
simple_mutex_lock(
&dcb->dcb_write_lock,
true);
@ -380,6 +501,11 @@ poll_waitevents(void *arg)
dcb->dcb_write_active = FALSE;
simple_mutex_unlock(
&dcb->dcb_write_lock);
#else
atomic_add(&pollStats.n_write,
1);
dcb_pollout(dcb, thread_id);
#endif
} else {
LOGIF(LD, (skygw_log_write(
LOGFILE_DEBUG,
@ -395,11 +521,13 @@ poll_waitevents(void *arg)
}
if (ev & EPOLLIN)
{
#if MUTEX_BLOCK
simple_mutex_lock(&dcb->dcb_read_lock,
true);
ss_info_dassert(!dcb->dcb_read_active,
"Read already active");
dcb->dcb_read_active = TRUE;
#endif
if (dcb->state == DCB_STATE_LISTENING)
{
@ -423,11 +551,17 @@ poll_waitevents(void *arg)
dcb,
dcb->fd)));
atomic_add(&pollStats.n_read, 1);
#if MUTEX_BLOCK
dcb->func.read(dcb);
#else
dcb_pollin(dcb, thread_id);
#endif
}
#if MUTEX_BLOCK
dcb->dcb_read_active = FALSE;
simple_mutex_unlock(
&dcb->dcb_read_lock);
#endif
}
if (ev & EPOLLERR)
{
@ -477,10 +611,33 @@ poll_waitevents(void *arg)
atomic_add(&pollStats.n_hup, 1);
dcb->func.hangup(dcb);
}
if (ev & EPOLLRDHUP)
{
int eno = 0;
eno = gw_getsockerrno(dcb->fd);
LOGIF(LD, (skygw_log_write(
LOGFILE_DEBUG,
"%lu [poll_waitevents] "
"EPOLLRDHUP on dcb %p, fd %d. "
"Errno %d, %s.",
pthread_self(),
dcb,
dcb->fd,
eno,
strerror(eno))));
atomic_add(&pollStats.n_hup, 1);
dcb->func.hangup(dcb);
}
} /*< for */
no_op = FALSE;
}
process_zombies:
if (thread_data)
{
thread_data[thread_id].state = THREAD_ZPROCESSING;
}
zombies = dcb_process_zombies(thread_id);
if (zombies == NULL) {
@ -493,9 +650,17 @@ poll_waitevents(void *arg)
* Remove the thread from the bitmask of running
* polling threads.
*/
if (thread_data)
{
thread_data[thread_id].state = THREAD_STOPPED;
}
bitmask_clear(&poll_mask, thread_id);
return;
}
if (thread_data)
{
thread_data[thread_id].state = THREAD_IDLE;
}
} /*< while(1) */
/** Release mysql thread context */
mysql_thread_end();
@ -529,10 +694,194 @@ poll_bitmask()
void
dprintPollStats(DCB *dcb)
{
dcb_printf(dcb, "Number of epoll cycles: %d\n", pollStats.n_polls);
dcb_printf(dcb, "Number of read events: %d\n", pollStats.n_read);
dcb_printf(dcb, "Number of write events: %d\n", pollStats.n_write);
dcb_printf(dcb, "Number of error events: %d\n", pollStats.n_error);
dcb_printf(dcb, "Number of hangup events: %d\n", pollStats.n_hup);
dcb_printf(dcb, "Number of accept events: %d\n", pollStats.n_accept);
int i;
dcb_printf(dcb, "Number of epoll cycles: %d\n",
pollStats.n_polls);
dcb_printf(dcb, "Number of read events: %d\n",
pollStats.n_read);
dcb_printf(dcb, "Number of write events: %d\n",
pollStats.n_write);
dcb_printf(dcb, "Number of error events: %d\n",
pollStats.n_error);
dcb_printf(dcb, "Number of hangup events: %d\n",
pollStats.n_hup);
dcb_printf(dcb, "Number of accept events: %d\n",
pollStats.n_accept);
dcb_printf(dcb, "Number of times no threads polling: %d\n",
pollStats.n_nothreads);
dcb_printf(dcb, "No of poll completions with descriptors\n");
dcb_printf(dcb, "\tNo. of descriptors\tNo. of poll completions.\n");
for (i = 0; i < MAXNFDS - 1; i++)
{
dcb_printf(dcb, "\t%2d\t\t\t%d\n", i + 1, pollStats.n_fds[i]);
}
dcb_printf(dcb, "\t>= %d\t\t\t%d\n", MAXNFDS,
pollStats.n_fds[MAXNFDS-1]);
}
/**
* Convert an EPOLL event mask into a printable string
*
* @param event The event mask
* @return A string representation, the caller must free the string
*/
static char *
event_to_string(uint32_t event)
{
char *str;
str = malloc(22); // 22 is max returned string length
if (str == NULL)
return NULL;
*str = 0;
if (event & EPOLLIN)
{
strcat(str, "IN");
}
if (event & EPOLLOUT)
{
if (*str)
strcat(str, "|");
strcat(str, "OUT");
}
if (event & EPOLLERR)
{
if (*str)
strcat(str, "|");
strcat(str, "ERR");
}
if (event & EPOLLHUP)
{
if (*str)
strcat(str, "|");
strcat(str, "HUP");
}
if (event & EPOLLRDHUP)
{
if (*str)
strcat(str, "|");
strcat(str, "RDHUP");
}
return str;
}
/**
* Print the thread status for all the polling threads
*
* @param dcb The DCB to send the thread status data
*/
void
dShowThreads(DCB *dcb)
{
int i, j, n;
char *state;
double avg1 = 0.0, avg5 = 0.0, avg15 = 0.0;
dcb_printf(dcb, "Polling Threads.\n\n");
dcb_printf(dcb, "Historic Thread Load Average: %.2f.\n", load_average);
dcb_printf(dcb, "Current Thread Load Average: %.2f.\n", current_avg);
/* Average all the samples to get the 15 minute average */
for (i = 0; i < n_avg_samples; i++)
avg15 += avg_samples[i];
avg15 = avg15 / n_avg_samples;
/* Average the last third of the samples to get the 5 minute average */
n = 5 * 60 / POLL_LOAD_FREQ;
i = next_sample - (n + 1);
if (i < 0)
i += n_avg_samples;
for (j = i; j < i + n; j++)
avg5 += avg_samples[j % n_avg_samples];
avg5 = (3 * avg5) / (n_avg_samples);
/* Average the last 15th of the samples to get the 1 minute average */
n = 60 / POLL_LOAD_FREQ;
i = next_sample - (n + 1);
if (i < 0)
i += n_avg_samples;
for (j = i; j < i + n; j++)
avg1 += avg_samples[j % n_avg_samples];
avg1 = (15 * avg1) / (n_avg_samples);
dcb_printf(dcb, "15 Minute Average: %.2f, 5 Minute Average: %.2f, "
"1 Minute Average: %.2f\n\n", avg15, avg5, avg1);
if (thread_data == NULL)
return;
dcb_printf(dcb, " ID | State | # fds | Descriptor | Event\n");
dcb_printf(dcb, "----+------------+--------+------------------+---------------\n");
for (i = 0; i < n_threads; i++)
{
switch (thread_data[i].state)
{
case THREAD_STOPPED:
state = "Stopped";
break;
case THREAD_IDLE:
state = "Idle";
break;
case THREAD_POLLING:
state = "Polling";
break;
case THREAD_PROCESSING:
state = "Processing";
break;
case THREAD_ZPROCESSING:
state = "Collecting";
break;
}
if (thread_data[i].state != THREAD_PROCESSING)
dcb_printf(dcb,
" %2d | %-10s | | |\n",
i, state);
else if (thread_data[i].cur_dcb == NULL)
dcb_printf(dcb,
" %2d | %-10s | %6d | |\n",
i, state, thread_data[i].n_fds);
else
{
char *event_string
= event_to_string(thread_data[i].event);
if (event_string == NULL)
event_string = "??";
dcb_printf(dcb,
" %2d | %-10s | %6d | %-16p | %s\n",
i, state, thread_data[i].n_fds,
thread_data[i].cur_dcb, event_string);
free(event_string);
}
}
}
/**
* The function used to calculate time based load data. This is called by the
* housekeeper every POLL_LOAD_FREQ seconds.
*
* @param data Argument required by the housekeeper but not used here
*/
static void
poll_loadav(void *data)
{
static int last_samples = 0, last_nfds = 0;
int new_samples, new_nfds;
new_samples = load_samples - last_samples;
new_nfds = load_nfds - last_nfds;
last_samples = load_samples;
last_nfds = load_nfds;
/* POLL_LOAD_FREQ average is... */
if (new_samples)
current_avg = new_nfds / new_samples;
else
current_avg = 0.0;
avg_samples[next_sample] = current_avg;
next_sample++;
if (next_sample >= n_avg_samples)
next_sample = 0;
}

Some files were not shown because too many files have changed in this diff Show More