hcq/MaxScale
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into MAX-345

Browse Source
This commit is contained in:
MassimilianoPinto
2015-02-05 16:14:36 +01:00
parent 625a7d7a10 db045d16f1
commit b2e3de0362
13 changed files with 43083 additions and 828 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42445
Documentation/Design-Documents/Core Object Models.mdj Normal file
View File

File diff suppressed because it is too large Load Diff

8
Documentation/Documentation-Contents.md
View File

@ -1,3 +1,5 @@
[Search page for MaxScale Documentation]((http://mariadb-corporation.github.io/MaxScale/Search/)
# Contents
## About MaxScale
@ -39,6 +41,11 @@
- [Tee Filter](filters/Tee-Filter.md)
- [Top N Filter](filters/Top-N-Filter.md)
- [Firewall Filter](filters/Firewall-Filter.md)
- [RabbitMQ Filter](filters/RabbitMQ-Filter.md)
## Utilities
- [RabbitMQ Consumer Client](filters/RabbitMQ-Consumer-Client.md)
## Design Documents
@ -53,3 +60,4 @@
- [MaxScale 1.0 Release Notes](Release-Notes/MaxScale-1.0-Release-Notes.md)
- [MaxScale 1.0.1 Release Notes](Release-Notes/MaxScale-1.0.1-Release-Notes.md)
- [MaxScale 1.0.3 Release Notes](Release-Notes/MaxScale-1.0.3-Release-Notes.md)

1133
Documentation/Getting-Started/Configuration-Guide.md
View File

File diff suppressed because it is too large Load Diff

50
Documentation/Getting-Started/Getting-Started-With-MaxScale.md
View File

@ -62,82 +62,46 @@ $ cd build
The next step is to run the cmake command to build the Makefile you need to compile Maxscale. There are a number of options you may give to configure cmake and point it to the various packages it requires. These are documented in the MaxScale README file, in this example we will assume the MariaDB developer packages have been installed in a non-standard location and set all the options required to locate these, along with options to build the unit tests and configure the installation target directory.
$ cmake -DMYSQL_DIR=~/usr/include/mysql \
-DEMBEDDED_LIB=~/usr/lib64/libmysqld.a \
-DMYSQLCLIENT_LIBRARIES=~/usr/lib64/libmysqlclient.so \
-DERRMSG=~/usr/share/mysql/english/errmsg.sys \
-DINSTALL_DIR=/usr/local/maxscale -DBUILD_TESTS=Y \
-DINSTALL_SYSTEM_FILES=N ../MaxScale
$ cmake -DMYSQL\_DIR=/usr/mariadb-5.5.41-linux-x86_64/include/mysql \
-DEMBEDDED\_LIB=/usr/mariadb-5.5.41-linux-x86\_64/lib/libmysqld.a \
-DMYSQLCLIENT\_LIBRARIES=/usr/mariadb-5.5.41-linux-x86_64/lib/libmysqlclient.so \
-DERRMSG=/usr/mariadb-5.5.41-linux-x86\_64/share/english/errmsg.sys \
-DINSTALL\_DIR=/home/maxscale/MaxScale -DBUILD_TESTS=Y \
-DINSTALL\_SYSTEM\_FILES=N \
-DBUILD_BINLOG=Y ../
-- CMake version: 2.8.12.2
-- The C compiler identification is GNU 4.4.7
-- The CXX compiler identification is GNU 4.4.7
-- Check for working C compiler: /usr/bin/cc
-- Check for working C compiler: /usr/bin/cc -- works
-- Detecting C compiler ABI info
-- Detecting C compiler ABI info - done
-- Check for working CXX compiler: /usr/bin/c++
-- Check for working CXX compiler: /usr/bin/c++ -- works
-- Detecting CXX compiler ABI info
-- Detecting CXX compiler ABI info - done
-- Library was found at: /lib64/libaio.so
-- Library was found at: /usr/lib64/libssl.so
-- Library was found at: /usr/lib64/libcrypt.so
-- Library was found at: /usr/lib64/libcrypto.so
-- Library was found at: /usr/lib64/libz.so
-- Library was found at: /usr/lib64/libm.so
-- Library was found at: /usr/lib64/libdl.so
-- Library was found at: /usr/lib64/librt.so
-- Library was found at: /usr/lib64/libpthread.so
-- Using errmsg.sys found at: /home/maxscale/usr/share/mysql/english/errmsg.sys
-- Using embedded library: /home/mpinto/usr/lib64/libmysqld.a
-- Valgrind found: /usr/bin/valgrind
-- Found dynamic MySQL client library: /home/maxscale/usr/lib64/libmysqlclient.so
-- Found static MySQL client library: /usr/lib/libmysqlclient.a
-- C Compiler supports: -Werror=format-security
-- Linking against: /home/mpinto/usr/lib64/libmysqlclient.so
-- Installing MaxScale to: /usr/local/maxscale/
-- Generating RPM packages
-- Found Doxygen: /usr/bin/doxygen (found version "1.6.1")
-- Configuring done
-- Generating done
-- Build files have been written to: /home/maxscale/develop/build
-bash-4.1$ make depend

23
Documentation/Reference/Hint-Syntax.md
View File

@ -1,29 +1,42 @@
Hint Syntax
Use either ’-- ’ (notice the whitespace) or ’#’after the semicolon or ’/* .. */’ before
# Hint Syntax
Use either ’-- ’ (notice the whitespace) or ’#’ after the semicolon or ’/* .. */’ before
the semicolon.
The MySQL manual doesn’t specify if comment blocks, i.e. ’/* .. */’, should contain a w
hitespace character before or after the tags.
All hints must start with the ’maxscale tag’:
-- maxscale <hint>
-- maxscale <hint>
The hints right now have two types, ones that route to a server and others that contain
name-value pairs.
Routing queries to a server:
-- maxscale route to [master | slave | server <server name>]
The name of the server is the same as in MaxScale.cnf
Creating a name-value pair:
-- maxscale <param>=<value>
Currently the only accepted parameter is
’max_slave_replication_lag’
Hints can be either single-use hints, which makes them affect only one query, or named
hints, which can be pushed on and off a stack of active hints.
Defining named hints:
-- maxscale <hint name> prepare <hint content>
Pushing a hint onto the stack:
-- maxscale <hint name> begin
Popping the topmost hint off the stack:
-- maxscale end
You can define and activate a hint in a single command using the following:
-- maxscale <hint name> begin <hint content>
You can also push anonymous hints onto the stack which are only used as long as they ar
e on the stack:
You can also push anonymous hints onto the stack which are only used as long as they are on the stack:
-- maxscale begin <hint content>

4
Documentation/Tutorials/Replication-Proxy-Binlog-Router-Tutorial.md
View File

@ -80,7 +80,7 @@ This parameter is used to provide the stem of the file names that are used to st
### initialfile
This optional parameter allows for the administrator to define the number of the first binlog file to download. In normal circumstances MaxScale will use any existing binlog file to determine what to request from the master. If there are no files it will then ask for the binlog file with the index number defined in the initialfile parameter. If this parameter is not set then MaxScale will ask the master for binlog events from file 1.
This optional parameter allows for the administrator to define the number of the first binlog file to download. If MaxScale has previously received binlogs it will use those existing binlog files to determine what to request from the master. If no files have been downloaded MaxScale will then ask for the binlog file with the index number defined in the initialfile parameter. If this parameter is not set then MaxScale will ask the master for binlog events from file 1.
### binlogdir
@ -99,7 +99,7 @@ A complete example of a service entry for a binlog router service would be as fo
[Replication]
type=service
router=binlogrouter
servers=maserdb
servers=masterdb
version_string=5.6.17-log
router_options=uuid=f12fcb7f-b97b-11e3-bc5e-0401152c4c22,server-id=3,user=repl,password=slavepass,master-id=1,filestem=mybin,heartbeat=30,binlogdir=/home/mriddoch/binlogs
user=maxscale

38
Documentation/filters/RabbitMQ-Consumer-Client.md Normal file
View File

@ -0,0 +1,38 @@
#RabbitMQ Consumer Client
## Overview
This utility tool is used to read messages from a RabbitMQ broker sent by the [RabbitMQ Filter](RabbitMQ-Filter.md) and forward these messages into an SQL database as queries.
## Command Line Arguments
The **RabbitMQ Consumer Client** only has one command line argument.
| Command | Argument |
|---------|-------------------------------------------------|
| -c | Path to the folder containing the configuration file |
## Installation
To install the RabbitMQ Consumer Client you ca either use the provided packages or you can compile it from source code. The source code is included as a part of the MaxScale source code and can be found in the `rabbtmq_consumer` folder. Please refer to the [README](../../rabbitmq_consumer/README) in the folder for more detailed instructions about installation and configuration.
## Configuration
The consumer client requires that the `consumer.cnf` configuration file is either be present in the `etc` folder of the installation directory or in the folder specified by the `-c` argument.
The source broker, the destination database and the message log file can be configured into the separate `consumer.cnf` file.
| Option | Desctiption |
|-----------|---------------------------------------------|
| hostname | Hostname of the RabbitMQ server |
| port | Port of the RabbitMQ server |
| vhost | Virtual host location of the RabbitMQ server |
| user | Username for the RabbitMQ server |
| passwd | Password for the RabbitMQ server |
| queue | Queue to consume from |
| dbserver | Hostname of the SQL server |
| dbport | Port of the SQL server |
| dbname | Name of the SQL database to use |
| dbuser | Database username |
| dbpasswd | Database passwork |
| logfile | Message log filename |

61
Documentation/filters/RabbitMQ-Filter.md Normal file
View File

@ -0,0 +1,61 @@
#RabbitMQ Filter
## Overview
This filter is designed to extract queries and transform them into a canonical form e.g. `INSERT INTO dabata.table VALUES ("John Doe", "Downtown",100,50.0);` turns into `INSERT INTO dabata.table VALUES ("?", "?",?,?);`. The filter pushes these canonized queries and their replies in to a RabbitMQ broker where they can later be retrieved. The retrieval can be done with your own application or the [RabbitMQ Consumer Client](RabbitMQ-Consumer-Client.md) utility tool, which reads the messages from the broker and sends the contents of those messages as SQL queries to a database.
## Configuration
The configuration block for the **mqfilter** filter requires the minimal filter options in it’s section within the MaxScale.cnf file, stored in $MAXSCALE_HOME/etc/MaxScale.cnf. Although the filter will start, it will use the default values which only work with a freshly installed RabbitMQ server and use its default values. This setup is mostly intednded for testing the filter.
The following is an example of a mqfilter configuration in the MaxScale.cnf file used for actual logging of queries to a RabbitMQ broker on a different host.
```
[RabbitMQ]
type=filter
module=mqfilter
hostname=192.168.122.100
port=4000
username=messageuser
password=msgpwd
exchange=msg-ex-1
key=MaxScale
logging_trigger=object,schema,source
logging_strict=false
logging_log_all=false
logging_object=my1
logging_schema=test
logging_source_user=maxtest
```
### Filter Options
The mqfilter filter does not support any filter options.
### Filter Parameters
The RabbitMQ filter has parameters to control which queries are logged based on either the attributes of the user or the query itself. These can be combined to to only log queries targeting a certain table in a certain database from a certain user from a certain network address.
Option | Description | Accepted Values | Default |
--------|-------------|-----------------|-------------
logging_trigger | Set the logging level | `all, source, schema, object` | `all` |
logging_strict | Sets whether to trigger when any of the parameters match or only if all parameters match | `true, false` | `false` |
logging_log_all | Log only SELECT, UPDATE, DELETE and INSERT or all possible queries | `true, false` | `true` |
logging_source_user | Comma-separated list of usernames to log | | |
logging_source_host | Comma-separated list of hostnames to log | | |
logging_schema | Comma-separated list of databases | | |
logging_object | Comma-separated list of database objects |
hostname | The server hostname where the messages are sent | | `localhost` |
port | Port to send the messages to | | `5672` |
username | Server login username | | `guest` |
password | Server login password | | `guest` |
vhost | The virtual host location on the server, where the messages are sent | | `/` |
exchange | The name of the exchange | | `default_exchange` |
exchange_type | The type of the exchange | `direct, fanout, topic, headers` | `direct` |
key | The routing key used when sending messages to the exchange | | `key` |
queue | The queue that will be bound to the used exchange | | |
ssl_CA_cert | Path to the CA certificate in PEM format | | |
ssl_client_cert | Path to the client cerificate in PEM format | | |
ssl_client_key | Path to the client public key in PEM format | | |

2
server/modules/include/blr.h
View File

@ -250,6 +250,8 @@ typedef struct router_instance {
char *user; /*< User name to use with master */
char *password; /*< Password to use with master */
char *fileroot; /*< Root of binlog filename */
bool master_chksum;/*< Does the master provide checksums */
char *master_uuid; /*< UUID of the master */
DCB *master; /*< DCB for master connection */
DCB *client; /*< DCB for dummy client */
SESSION *session; /*< Fake session for master connection */

2
server/modules/routing/binlog/blr.c
View File

@ -181,6 +181,8 @@ unsigned char *defuuid;
spinlock_init(&inst->binlog_lock);
inst->binlog_fd = -1;
inst->master_chksum = true;
inst->master_uuid = NULL;
inst->low_water = DEF_LOW_WATER;
inst->high_water = DEF_HIGH_WATER;

4
server/modules/routing/binlog/blr_file.c
View File

@ -354,6 +354,10 @@ int n;
unsigned long filelen = 0;
struct stat statb;
if (!file)
{
return NULL;
}
if (fstat(file->fd, &statb) == 0)
filelen = statb.st_size;
if (pos >= filelen)

122
server/modules/routing/binlog/blr_master.c
View File

@ -79,6 +79,8 @@ void blr_extract_header(uint8_t *pkt, REP_HEADER *hdr);
inline uint32_t extract_field(uint8_t *src, int bits);
static void blr_log_packet(logfile_id_t file, char *msg, uint8_t *ptr, int len);
static void blr_master_close(ROUTER_INSTANCE *);
static char *blr_extract_column(GWBUF *buf, int col);
static int keepalive = 1;
/**
@ -384,6 +386,9 @@ char query[128];
router->retry_backoff = 1;
break;
case BLRM_SERVERID:
{
char *val = blr_extract_column(buf, 1);
// Response to fetch of master's server-id
if (router->saved_master.server_id)
GWBUF_CONSUME_ALL(router->saved_master.server_id);
@ -398,6 +403,7 @@ char query[128];
router->master_state = BLRM_HBPERIOD;
router->master->func.write(router->master, buf);
break;
}
case BLRM_HBPERIOD:
// Response to set the heartbeat period
if (router->saved_master.heartbeat)
@ -419,6 +425,15 @@ char query[128];
router->master->func.write(router->master, buf);
break;
case BLRM_CHKSUM2:
{
char *val = blr_extract_column(buf, 1);
if (val && strncasecmp(val, "NONE", 4) == 0)
{
router->master_chksum = false;
}
if (val)
free(val);
// Response to the master_binlog_checksum, should be stored
if (router->saved_master.chksum2)
GWBUF_CONSUME_ALL(router->saved_master.chksum2);
@ -428,6 +443,7 @@ char query[128];
router->master_state = BLRM_GTIDMODE;
router->master->func.write(router->master, buf);
break;
}
case BLRM_GTIDMODE:
// Response to the GTID_MODE, should be stored
if (router->saved_master.gtid_mode)
@ -439,6 +455,10 @@ char query[128];
router->master->func.write(router->master, buf);
break;
case BLRM_MUUID:
{
char *val = blr_extract_column(buf, 1);
router->master_uuid = val;
// Response to the SERVER_UUID, should be stored
if (router->saved_master.uuid)
GWBUF_CONSUME_ALL(router->saved_master.uuid);
@ -449,6 +469,7 @@ char query[128];
router->master_state = BLRM_SUUID;
router->master->func.write(router->master, buf);
break;
}
case BLRM_SUUID:
// Response to the SET @server_uuid, should be stored
if (router->saved_master.setslaveuuid)
@ -874,30 +895,33 @@ static REP_HEADER phdr;
* First check that the checksum we calculate matches the
* checksum in the packet we received.
*/
uint32_t chksum, pktsum;
chksum = crc32(0L, NULL, 0);
chksum = crc32(chksum, ptr + 5, hdr.event_size - 4);
pktsum = EXTRACT32(ptr + hdr.event_size + 1);
if (pktsum != chksum)
if (router->master_chksum)
{
router->stats.n_badcrc++;
if (msg)
uint32_t chksum, pktsum;
chksum = crc32(0L, NULL, 0);
chksum = crc32(chksum, ptr + 5, hdr.event_size - 4);
pktsum = EXTRACT32(ptr + hdr.event_size + 1);
if (pktsum != chksum)
{
free(msg);
msg = NULL;
router->stats.n_badcrc++;
if (msg)
{
free(msg);
msg = NULL;
}
LOGIF(LE,(skygw_log_write(LOGFILE_ERROR,
"%s: Checksum error in event "
"from master, "
"binlog %s @ %d. "
"Closing master connection.",
router->service->name,
router->binlog_name,
router->binlog_position)));
blr_master_close(router);
blr_master_delayed_connect(router);
return;
}
LOGIF(LE,(skygw_log_write(LOGFILE_ERROR,
"%s: Checksum error in event "
"from master, "
"binlog %s @ %d. "
"Closing master connection.",
router->service->name,
router->binlog_name,
router->binlog_position)));
blr_master_close(router);
blr_master_delayed_connect(router);
return;
}
router->stats.n_binlogs++;
router->lastEventReceived = hdr.event_type;
@ -1398,3 +1422,59 @@ blr_master_connected(ROUTER_INSTANCE *router)
{
return router->master_state == BLRM_BINLOGDUMP;
}
/**
* Extract a result value from the set of messages that make up a
* MySQL response packet.
*
* @param buf The GWBUF containing the response
* @param col The column number to return
* @return The result form the column or NULL. The caller must free the result
*/
static char *
blr_extract_column(GWBUF *buf, int col)
{
uint8_t *ptr;
int len, ncol, collen;
char *rval;
ptr = (uint8_t *)GWBUF_DATA(buf);
/* First packet should be the column count */
len = EXTRACT24(ptr);
ptr += 3;
if (*ptr != 1) // Check sequence number is 1
return NULL;
ptr++;
ncol = *ptr++;
if (ncol < col) // Not that many column in result
return NULL;
// Now ptr points at the column definition
while (ncol-- > 0)
{
len = EXTRACT24(ptr);
ptr += 4; // Skip to payload
ptr += len; // Skip over payload
}
// Now we should have an EOF packet
len = EXTRACT24(ptr);
ptr += 4; // Skip to payload
if (*ptr != 0xfe)
return NULL;
ptr += len;
// Finally we have reached the row
len = EXTRACT24(ptr);
ptr += 4;
while (--col > 0)
{
collen = *ptr++;
ptr += collen;
}
collen = *ptr++;
if ((rval = malloc(collen + 1)) == NULL)
return NULL;
memcpy(rval, ptr, collen);
rval[collen] = 0; // NULL terminate
return rval;
}

19
server/modules/routing/binlog/blr_slave.c
View File

@ -859,7 +859,8 @@ int len, actual_len, col_len, seqno, ncols, i;
*ptr++ = 0;
sprintf(column, "%s", router->uuid);
sprintf(column, "%s", router->master_uuid ?
router->master_uuid : router->uuid);
col_len = strlen(column);
*ptr++ = col_len; // Length of result string
strncpy((char *)ptr, column, col_len); // Result string
@ -1078,6 +1079,15 @@ uint32_t chksum;
ptr = GWBUF_DATA(queue);
len = extract_field(ptr, 24);
binlognamelen = len - 11;
if (binlognamelen > BINLOG_FNAMELEN)
{
LOGIF(LE, (skygw_log_write(
LOGFILE_ERROR,
"blr_slave_binlog_dump truncating binlog filename "
"from %d to %d",
binlognamelen, BINLOG_FNAMELEN)));
binlognamelen = BINLOG_FNAMELEN;
}
ptr += 4; // Skip length and sequence number
if (*ptr++ != COM_BINLOG_DUMP)
{
@ -1097,6 +1107,13 @@ uint32_t chksum;
strncpy(slave->binlogfile, (char *)ptr, binlognamelen);
slave->binlogfile[binlognamelen] = 0;
LOGIF(LD, (skygw_log_write(
LOGFILE_DEBUG,
"%s: COM_BINLOG_DUMP: binlog name '%s', length %d, "
"from position %d.", router->service->name,
slave->binlogfile, binlognamelen,
slave->binlog_pos)));
slave->seqno = 1;