hcq/MaxScale
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added replicaton listener library from https://github.com/SponsorPay/mysql-replication-listener and first prototype implementation for table replication consistency module

Browse Source
This commit is contained in:
Jan Lindström
2013-06-21 09:11:57 +03:00
parent 0fed3d624e
commit bc3a104e3f
176 changed files with 23519 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
replication_listener/include/access_method_factory.h Normal file
View File

@ -0,0 +1,36 @@
/*
Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights
reserved.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
02110-1301 USA
*/
#ifndef _ACCESS_METHOD_FACTORY_H
#define _ACCESS_METHOD_FACTORY_H
#include "binlog_driver.h"
namespace mysql {
namespace system {
Binary_log_driver *create_transport(const char *url);
Binary_log_driver *parse_mysql_url(char *url, const char
*mysql_access_method);
Binary_log_driver *parse_file_url(char *url, const char
*file_access_method);
}
}
#endif /* _ACCESS_METHOD_FACTORY_H */

81
replication_listener/include/basic_content_handler.h Normal file
View File

@ -0,0 +1,81 @@
/*
Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights
reserved.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
02110-1301 USA
*/
#ifndef BASIC_CONTENT_HANDLER_H
#define BASIC_CONTENT_HANDLER_H
#include "binlog_event.h"
namespace mysql {
class Injection_queue : public std::list<mysql::Binary_log_event * >
{
public:
Injection_queue() : std::list<mysql::Binary_log_event * >() {}
~Injection_queue() {}
};
/**
* A content handler accepts an event and returns the same event,
* a new one or 0 (the event was consumed by the content handler).
* The default behaviour is to return the event unaffected.
* The generic event handler is used for events which aren't routed to
* a dedicated member function, user defined events being the most
* common case.
*/
class Content_handler {
public:
Content_handler();
Content_handler(const mysql::Content_handler& orig);
virtual ~Content_handler();
virtual mysql::Binary_log_event *process_event(mysql::Query_event *ev);
virtual mysql::Binary_log_event *process_event(mysql::Row_event *ev);
virtual mysql::Binary_log_event *process_event(mysql::Table_map_event *ev);
virtual mysql::Binary_log_event *process_event(mysql::Xid *ev);
virtual mysql::Binary_log_event *process_event(mysql::User_var_event *ev);
virtual mysql::Binary_log_event *process_event(mysql::Incident_event *ev);
virtual mysql::Binary_log_event *process_event(mysql::Rotate_event *ev);
virtual mysql::Binary_log_event *process_event(mysql::Int_var_event *ev);
virtual mysql::Binary_log_event *process_event(mysql::Gtid_event *ev);
/**
Process any event which hasn't been registered yet.
*/
virtual mysql::Binary_log_event *process_event(mysql::Binary_log_event *ev);
protected:
/**
* The Injection queue is emptied before any new event is pulled from
* the Binary_log_driver. Injected events will pass through all content
* handlers. The Injection_queue is a derived std::list.
*/
Injection_queue *get_injection_queue();
private:
Injection_queue *m_reinject_queue;
void set_injection_queue(Injection_queue *injection_queue);
mysql::Binary_log_event *internal_process_event(mysql::Binary_log_event *ev);
friend class Binary_log;
};
} // end namespace
#endif /* BASIC_CONTENT_HANDLER_H */

83
replication_listener/include/basic_transaction_parser.h Normal file
View File

@ -0,0 +1,83 @@
/*
Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights
reserved.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
02110-1301 USA
*/
#ifndef _BASIC_TRANSACTION_PARSER_H
#define _BASIC_TRANSACTION_PARSER_H
/*
TODO The Transaction_log_event and Basic_transaction_parser will be removed
from this library and replaced with a table map indexer instead which can be
used to retrive table names.
*/
#include <list>
#include <boost/cstdint.hpp>
#include "binlog_event.h"
#include "basic_content_handler.h"
#include <iostream>
namespace mysql {
typedef std::pair<boost::uint64_t, Binary_log_event *> Event_index_element;
typedef std::map<boost::uint64_t, Binary_log_event *> Int_to_Event_map;
class Transaction_log_event : public Binary_log_event
{
public:
Transaction_log_event() : Binary_log_event() {}
Transaction_log_event(Log_event_header *header) : Binary_log_event(header) {}
virtual ~Transaction_log_event();
Int_to_Event_map &table_map() { return m_table_map; }
/**
* Index for easier table name look up
*/
Int_to_Event_map m_table_map;
std::list<Binary_log_event *> m_events;
};
Transaction_log_event *create_transaction_log_event(void);
class Basic_transaction_parser : public mysql::Content_handler
{
public:
Basic_transaction_parser() : mysql::Content_handler()
{
m_transaction_state= NOT_IN_PROGRESS;
}
mysql::Binary_log_event *process_event(mysql::Query_event *ev);
mysql::Binary_log_event *process_event(mysql::Row_event *ev);
mysql::Binary_log_event *process_event(mysql::Table_map_event *ev);
mysql::Binary_log_event *process_event(mysql::Xid *ev);
mysql::Binary_log_event *process_event(mysql::Binary_log_event *ev) {return ev; }
mysql::Binary_log_event *process_event(mysql::Gtid_event *ev);
private:
boost::uint32_t m_start_time;
enum Transaction_states { STARTING, IN_PROGRESS, COMMITTING, NOT_IN_PROGRESS } ;
enum Transaction_states m_transaction_state;
std::list <mysql::Binary_log_event *> m_event_stack;
mysql::Binary_log_event *process_transaction_state(mysql::Binary_log_event *ev);
};
} // end namespace
#endif /* _BASIC_TRANSACTION_PARSER_H */

179
replication_listener/include/binlog_api.h Normal file
View File

@ -0,0 +1,179 @@
/*
Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights
reserved.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
02110-1301 USA
*/
#ifndef _REPEVENT_H
#define _REPEVENT_H
#include <iosfwd>
#include <boost/iostreams/categories.hpp>
#include <boost/iostreams/positioning.hpp>
#include <boost/iostreams/concepts.hpp>
#include <boost/asio.hpp>
#include <boost/function.hpp>
#include <boost/bind.hpp>
#include <list>
#include <cassert>
#include "binlog_event.h"
#include "binlog_driver.h"
#include "tcp_driver.h"
#include "file_driver.h"
#include "basic_content_handler.h"
#include "basic_transaction_parser.h"
#include "field_iterator.h"
#include "rowset.h"
#include "access_method_factory.h"
#include "gtid.h"
namespace io = boost::iostreams;
namespace mysql
{
/**
* Error codes.
*/
enum Error_code {
ERR_OK = 0, /* All OK */
ERR_EOF, /* End of file */
ERR_FAIL, /* Unspecified failure */
ERROR_CODE_COUNT
};
/**
* Returns true if the event is consumed
*/
typedef boost::function< bool (Binary_log_event *& )> Event_content_handler;
class Dummy_driver : public system::Binary_log_driver
{
public:
Dummy_driver() : Binary_log_driver("", 0) {}
virtual ~Dummy_driver() {}
virtual int connect(Gtid gtid = Gtid()) { return 1; }
virtual int wait_for_next_event(mysql::Binary_log_event **event) {
return ERR_EOF;
}
virtual int set_position(const std::string &str, unsigned long position) {
return ERR_OK;
}
virtual int set_position_gtid(const Gtid gtid) {
return ERR_OK;
}
virtual int get_position(std::string *str, unsigned long *position) {
return ERR_OK;
}
virtual int fetch_server_version(const std::string& user,
const std::string& passwd,
const std::string& host,
long port)
{
return ERR_OK;
}
virtual void shutdown() {}
};
class Content_handler;
typedef std::list<Content_handler *> Content_handler_pipeline;
class Binary_log {
private:
system::Binary_log_driver *m_driver;
Dummy_driver m_dummy_driver;
Content_handler_pipeline m_content_handlers;
unsigned long m_binlog_position;
std::string m_binlog_file;
mysql_server_types m_server_type;
std::string m_uri;
public:
Binary_log(system::Binary_log_driver *drv);
Binary_log(system::Binary_log_driver *drv, std::string);
~Binary_log() {}
int connect(Gtid gtid = Gtid());
/**
* Blocking attempt to get the next binlog event from the stream
*/
int wait_for_next_event(Binary_log_event **event);
/**
* Inserts/removes content handlers in and out of the chain
* The Content_handler_pipeline is a derived std::list
*/
Content_handler_pipeline *content_handler_pipeline();
/**
* Set the binlog position (filename, position)
*
* @return Error_code
* @retval ERR_OK The position is updated.
* @retval ERR_EOF The position is out-of-range
* @retval >= ERR_CODE_COUNT An unspecified error occurred
*/
int set_position(const std::string &filename, unsigned long position);
/**
* Set the binlog position using current filename
* @param position Requested position
*
* @return Error_code
* @retval ERR_OK The position is updated.
* @retval ERR_EOF The position is out-of-range
* @retval >= ERR_CODE_COUNT An unspecified error occurred
*/
int set_position(unsigned long position);
int set_position_gtid(const Gtid gtid);
/**
* Fetch the binlog position for the current file
*/
unsigned long get_position(void);
/**
* Fetch the current active binlog file name.
* @param[out] filename
* TODO replace reference with a pointer.
* @return The file position
*/
unsigned long get_position(std::string &filename);
mysql_server_types get_mysql_server_type() const;
const char *get_mysql_server_type_str() const;
std::string get_url() const {return m_uri; }
void shutdown();
};
}
#endif /* _REPEVENT_H */

104
replication_listener/include/binlog_driver.h Normal file
View File

@ -0,0 +1,104 @@
/*
Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights
reserved.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
02110-1301 USA
*/
#ifndef _BINLOG_DRIVER_H
#define _BINLOG_DRIVER_H
#include "binlog_event.h"
#include "protocol.h"
#include "gtid.h"
namespace mysql {
namespace system {
class Binary_log_driver
{
public:
template <class FilenameT>
Binary_log_driver(const FilenameT& filename = FilenameT(), unsigned int offset = 0)
: m_binlog_file_name(filename), m_binlog_offset(offset), m_server_type(MYSQL_SERVER_TYPE_NA)
{
}
~Binary_log_driver() {}
/**
* Connect to the binary log using previously declared connection parameters
* @return Success or error code
* @retval 0 Success
* @retval >0 Error code (to be specified)
*/
virtual int connect(Gtid gtid = Gtid())= 0;
/**
* Blocking attempt to get the next binlog event from the stream
* @param event [out] Pointer to a binary log event to be fetched.
*/
virtual int wait_for_next_event(mysql::Binary_log_event **event)= 0;
/**
* Set the reader position
* @param str The file name
* @param position The file position
*
* @return False on success and True if an error occurred.
*/
virtual int set_position(const std::string &str, unsigned long position)= 0;
virtual int set_position_gtid(const Gtid gtid) = 0;
/**
* Get the read position.
*
* @param[out] string_ptr Pointer to location where the filename will be stored.
* @param[out] position_ptr Pointer to location where the position will be stored.
*
* @retval 0 Success
* @retval >0 Error code
*/
virtual int get_position(std::string *filename_ptr, unsigned long *position_ptr) = 0;
virtual int fetch_server_version(const std::string& user,
const std::string& passwd,
const std::string& host,
long port) = 0;
virtual void shutdown() = 0;
Binary_log_event* parse_event(std::istream &sbuff, Log_event_header *header);
mysql_server_types get_mysql_server_type() const
{
return m_server_type;
}
protected:
/**
* Used each time the client reconnects to the server to specify an
* offset position.
*/
unsigned long m_binlog_offset;
std::string m_binlog_file_name;
mysql_server_types m_server_type;
};
} // namespace mysql::system
} // namespace mysql
#endif /* _BINLOG_DRIVER_H */

282
replication_listener/include/binlog_event.h Normal file
View File

@ -0,0 +1,282 @@
/*
Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights
reserved.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
02110-1301 USA
*/
#ifndef _BINLOG_EVENT_H
#define _BINLOG_EVENT_H
#include <boost/cstdint.hpp>
#include <list>
#include <boost/asio.hpp>
#include <boost/function.hpp>
#include <vector>
namespace mysql
{
/**
@enum Log_event_type
Enumeration type for the different types of log events.
*/
enum Log_event_type
{
/*
Every time you update this enum (when you add a type), you have to
fix Format_description_log_event::Format_description_log_event().
*/
UNKNOWN_EVENT= 0,
START_EVENT_V3= 1,
QUERY_EVENT= 2,
STOP_EVENT= 3,
ROTATE_EVENT= 4,
INTVAR_EVENT= 5,
LOAD_EVENT= 6,
SLAVE_EVENT= 7,
CREATE_FILE_EVENT= 8,
APPEND_BLOCK_EVENT= 9,
EXEC_LOAD_EVENT= 10,
DELETE_FILE_EVENT= 11,
/*
NEW_LOAD_EVENT is like LOAD_EVENT except that it has a longer
sql_ex, allowing multibyte TERMINATED BY etc; both types share the
same class (Load_log_event)
*/
NEW_LOAD_EVENT= 12,
RAND_EVENT= 13,
USER_VAR_EVENT= 14,
FORMAT_DESCRIPTION_EVENT= 15,
XID_EVENT= 16,
BEGIN_LOAD_QUERY_EVENT= 17,
EXECUTE_LOAD_QUERY_EVENT= 18,
TABLE_MAP_EVENT = 19,
/*
These event numbers were used for 5.1.0 to 5.1.15 and are
therefore obsolete.
*/
PRE_GA_WRITE_ROWS_EVENT = 20,
PRE_GA_UPDATE_ROWS_EVENT = 21,
PRE_GA_DELETE_ROWS_EVENT = 22,
/*
These event numbers are used from 5.1.16 and forward
*/
WRITE_ROWS_EVENT = 23,
UPDATE_ROWS_EVENT = 24,
DELETE_ROWS_EVENT = 25,
/*
Something out of the ordinary happened on the master
*/
INCIDENT_EVENT= 26,
/*
* A user defined event
*/
USER_DEFINED= 27,
/* We have two different implementations of global transaction id */
GTID_EVENT_MYSQL=33,
GTID_EVENT_MARIADB= 162,
/*
Add new events here - right above this comment!
Existing events (except ENUM_END_EVENT) should never change their numbers
*/
ENUM_END_EVENT /* end marker */
};
namespace system {
/**
* Convenience function to get the string representation of a binlog event.
*/
const char* get_event_type_str(Log_event_type type);
} // end namespace system
#define LOG_EVENT_HEADER_SIZE 20
class Log_event_header
{
public:
boost::uint8_t marker; // always 0 or 0xFF
boost::uint32_t timestamp;
boost::uint8_t type_code;
boost::uint32_t server_id;
boost::uint32_t event_length;
boost::uint32_t next_position;
boost::uint16_t flags;
};
class Binary_log_event;
/**
* TODO Base class for events. Implementation is in body()
*/
class Binary_log_event
{
public:
Binary_log_event()
{
/*
An event length of 0 indicates that the header isn't initialized
*/
m_header.event_length= 0;
m_header.type_code= 0;
}
Binary_log_event(Log_event_header *header)
{
m_header= *header;
}
virtual ~Binary_log_event();
/**
* Helper method
*/
enum Log_event_type get_event_type() const
{
return (enum Log_event_type) m_header.type_code;
}
/**
* Return a pointer to the header of the log event
*/
Log_event_header *header() { return &m_header; }
private:
Log_event_header m_header;
};
class Query_event: public Binary_log_event
{
public:
Query_event(Log_event_header *header) : Binary_log_event(header) {}
boost::uint32_t thread_id;
boost::uint32_t exec_time;
boost::uint16_t error_code;
std::vector<boost::uint8_t > variables;
std::string db_name;
std::string query;
};
class Gtid_event: public Binary_log_event
{
public:
Gtid_event(Log_event_header *header) : Binary_log_event(header) {}
boost::uint32_t domain_id;
boost::uint32_t server_id;
boost::uint64_t sequence_number;
};
class Rotate_event: public Binary_log_event
{
public:
Rotate_event(Log_event_header *header) : Binary_log_event(header) {}
std::string binlog_file;
boost::uint64_t binlog_pos;
};
class Format_event: public Binary_log_event
{
public:
Format_event(Log_event_header *header) : Binary_log_event(header) {}
boost::uint16_t binlog_version;
std::string master_version;
boost::uint32_t created_ts;
boost::uint8_t log_header_len;
};
class User_var_event: public Binary_log_event
{
public:
enum Value_type {
STRING_TYPE,
REAL_TYPE,
INT_TYPE,
ROW_TYPE,
DECIMAL_TYPE,
VALUE_TYPE_COUNT
};
User_var_event(Log_event_header *header) : Binary_log_event(header) {}
std::string name;
boost::uint8_t is_null;
boost::uint8_t type;
boost::uint32_t charset; /* charset of the string */
std::string value; /* encoded in binary speak, depends on .type */
};
class Table_map_event: public Binary_log_event
{
public:
Table_map_event(Log_event_header *header) : Binary_log_event(header) {}
boost::uint64_t table_id;
boost::uint16_t flags;
std::string db_name;
std::string table_name;
std::vector<uint8_t> columns;
std::vector<uint8_t> metadata;
std::vector<uint8_t> null_bits;
};
class Row_event: public Binary_log_event
{
public:
Row_event(Log_event_header *header) : Binary_log_event(header) {}
boost::uint64_t table_id;
boost::uint16_t flags;
boost::uint64_t columns_len;
boost::uint32_t null_bits_len;
std::vector<boost::uint8_t> columns_before_image;
std::vector<uint8_t> used_columns;
std::vector<uint8_t> row;
};
class Int_var_event: public Binary_log_event
{
public:
Int_var_event(Log_event_header *header) : Binary_log_event(header) {}
boost::uint8_t type;
boost::uint64_t value;
};
class Incident_event: public Binary_log_event
{
public:
Incident_event() : Binary_log_event() {}
Incident_event(Log_event_header *header) : Binary_log_event(header) {}
boost::uint8_t type;
std::string message;
};
class Xid: public Binary_log_event
{
public:
Xid(Log_event_header *header) : Binary_log_event(header) {}
boost::uint64_t xid_id;
};
Binary_log_event *create_incident_event(unsigned int type, const char *message, unsigned long pos= 0);
} // end namespace mysql
#endif /* _BINLOG_EVENT_H */

91
replication_listener/include/bounded_buffer.h Normal file
View File

@ -0,0 +1,91 @@
/*
Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights
reserved.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
02110-1301 USA
*/
#ifndef _BOUNDED_BUFFER_H
#define _BOUNDED_BUFFER_H
#include <boost/circular_buffer.hpp>
#include <boost/thread/mutex.hpp>
#include <boost/thread/condition.hpp>
#include <boost/thread/thread.hpp>
#include <boost/progress.hpp>
#include <boost/bind.hpp>
template <class T>
class bounded_buffer
{
public:
typedef boost::circular_buffer<T> container_type;
typedef typename container_type::size_type size_type;
typedef typename container_type::value_type value_type;
explicit bounded_buffer(size_type capacity) : m_unread(0), m_container(capacity) {}
void push_front(const value_type& item)
{
boost::mutex::scoped_lock lock(m_mutex);
m_not_full.wait(lock, boost::bind(&bounded_buffer<value_type>::is_not_full, this));
m_container.push_front(item);
++m_unread;
lock.unlock();
m_not_empty.notify_one();
}
void pop_back(value_type* pItem)
{
boost::mutex::scoped_lock lock(m_mutex);
m_not_empty.wait(lock, boost::bind(&bounded_buffer<value_type>::is_not_empty, this));
*pItem = m_container[--m_unread];
lock.unlock();
m_not_full.notify_one();
}
bool has_unread()
{
boost::mutex::scoped_lock lock(m_mutex);
return is_not_empty();
}
void lock()
{
m_mutex.lock();
}
void unlock()
{
m_mutex.unlock();
}
private:
bounded_buffer(const bounded_buffer&); // Disabled copy constructor
bounded_buffer& operator = (const bounded_buffer&); // Disabled assign operator
bool is_not_empty() const { return m_unread > 0; }
bool is_not_full() const { return m_unread < m_container.capacity(); }
size_type m_unread;
container_type m_container;
boost::mutex m_mutex;
boost::condition m_not_empty;
boost::condition m_not_full;
};
#endif /* _BOUNDED_BUFFER_H */

196
replication_listener/include/field_iterator.h Normal file
View File

@ -0,0 +1,196 @@
/*
Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights
reserved.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
02110-1301 USA
*/
#ifndef _FIELD_ITERATOR_H
#define _FIELD_ITERATOR_H
#include "binlog_event.h"
#include "value.h"
#include "row_of_fields.h"
#include <vector>
using namespace mysql;
namespace mysql {
bool is_null(unsigned char *bitmap, int index);
int lookup_metadata_field_size(enum mysql::system::enum_field_types field_type);
boost::uint32_t extract_metadata(const Table_map_event *map, int col_no);
template <class Iterator_value_type >
class Row_event_iterator : public std::iterator<std::forward_iterator_tag,
Iterator_value_type>
{
public:
Row_event_iterator() : m_row_event(0), m_table_map(0),
m_new_field_offset_calculated(0), m_field_offset(0)
{ }
Row_event_iterator(const Row_event *row_event,
const Table_map_event *table_map)
: m_row_event(row_event), m_table_map(table_map),
m_new_field_offset_calculated(0)
{
m_field_offset= 0;
}
Iterator_value_type operator*();
Row_event_iterator& operator++();
Row_event_iterator operator++(int);
bool operator==(const Row_event_iterator& x) const;
bool operator!=(const Row_event_iterator& x) const;
//Row_iterator end() const;
private:
size_t fields(Iterator_value_type& fields_vector );
const Row_event *m_row_event;
const Table_map_event *m_table_map;
unsigned long m_new_field_offset_calculated;
unsigned long m_field_offset;
};
template <class Iterator_value_type>
size_t Row_event_iterator<Iterator_value_type>::fields(Iterator_value_type& fields_vector )
{
size_t field_offset= m_field_offset;
int row_field_col_index= 0;
std::vector<boost::uint8_t> nullbits(m_row_event->null_bits_len);
std::copy(m_row_event->row.begin()+m_field_offset,
m_row_event->row.begin()+(m_field_offset+m_row_event->null_bits_len),
nullbits.begin());
field_offset += m_row_event->null_bits_len;
for(unsigned col_no=0; col_no < m_table_map->columns.size(); ++col_no)
{
++row_field_col_index;
unsigned int type= m_table_map->columns[col_no]&0xFF;
boost::uint32_t metadata= extract_metadata(m_table_map, col_no);
mysql::Value val((enum mysql::system::enum_field_types)type,
metadata,
(const char *)&m_row_event->row[field_offset]);
if (is_null((unsigned char *)&nullbits[0], col_no ))
{
val.is_null(true);
}
else
{
/*
If the value is null it is not in the list of values and thus we won't
increse the offset. TODO what if all values are null?!
*/
field_offset += val.length();
}
fields_vector.push_back(val);
}
return field_offset;
}
template <class Iterator_value_type >
Iterator_value_type Row_event_iterator<Iterator_value_type>::operator*()
{ // dereferencing
Iterator_value_type fields_vector;
/*
* Remember this offset if we need to increate the row pointer
*/
m_new_field_offset_calculated= fields(fields_vector);
return fields_vector;
}
template< class Iterator_value_type >
Row_event_iterator< Iterator_value_type >&
Row_event_iterator< Iterator_value_type >::operator++()
{ // prefix
if (m_field_offset < m_row_event->row.size())
{
/*
* If we requested the fields in a previous operations
* we also calculated the new offset at the same time.
*/
if (m_new_field_offset_calculated != 0)
{
m_field_offset= m_new_field_offset_calculated;
//m_field_offset += m_row_event->null_bits_len;
m_new_field_offset_calculated= 0;
if (m_field_offset >= m_row_event->row.size())
m_field_offset= 0;
return *this;
}
/*
* Advance the field offset to the next row
*/
int row_field_col_index= 0;
std::vector<uint8_t> nullbits(m_row_event->null_bits_len);
std::copy(m_row_event->row.begin()+m_field_offset,
m_row_event->row.begin()+(m_field_offset+m_row_event->null_bits_len),
nullbits.begin());
m_field_offset += m_row_event->null_bits_len;
for(unsigned col_no=0; col_no < m_table_map->columns.size(); ++col_no)
{
++row_field_col_index;
mysql::Value val((enum mysql::system::enum_field_types)m_table_map->columns[col_no],
m_table_map->metadata[col_no],
(const char *)&m_row_event->row[m_field_offset]);
if (!is_null((unsigned char *)&nullbits[0], col_no))
{
m_field_offset += val.length();
}
}
return *this;
}
m_field_offset= 0;
return *this;
}
template <class Iterator_value_type >
Row_event_iterator< Iterator_value_type >
Row_event_iterator< Iterator_value_type >::operator++(int)
{ // postfix
Row_event_iterator temp = *this;
++*this;
return temp;
}
template <class Iterator_value_type >
bool Row_event_iterator< Iterator_value_type >::operator==(const Row_event_iterator& x) const
{
return m_field_offset == x.m_field_offset;
}
template <class Iterator_value_type >
bool Row_event_iterator< Iterator_value_type >::operator!=(const Row_event_iterator& x) const
{
return m_field_offset != x.m_field_offset;
}
}
#endif /* _FIELD_ITERATOR_H */

92
replication_listener/include/file_driver.h Normal file
View File

@ -0,0 +1,92 @@
/*
Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights
reserved.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
02110-1301 USA
*/
#ifndef _FILE_DRIVER_H
#define _FILE_DRIVER_H
#include <iostream>
#include <fstream>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include "binlog_api.h"
#include "binlog_driver.h"
#include "protocol.h"
#define MAGIC_NUMBER_SIZE 4
namespace mysql {
namespace system {
class Binlog_file_driver
: public Binary_log_driver
{
public:
template <class TFilename>
Binlog_file_driver(const TFilename& filename = TFilename(),
unsigned int offset = 0)
: Binary_log_driver(filename, offset)
{
}
int connect(Gtid gtid = Gtid());
int disconnect();
int wait_for_next_event(mysql::Binary_log_event **event);
int set_position(const std::string &str, unsigned long position);
int get_position(std::string *str, unsigned long *position);
int set_position_gtid(const Gtid gtid)
{
return 0; // TODO
}
int fetch_server_version(const std::string& user,
const std::string& passwd,
const std::string& host,
long port)
{
return 0; // TODO
}
void shutdown()
{
// TODO
}
private:
unsigned long m_binlog_file_size;
/*
Bytes that has been read so for from the file.
Updated after every event is read.
*/
unsigned long m_bytes_read;
std::ifstream m_binlog_file;
Log_event_header m_event_log_header;
};
} // namespace mysql::system
} // namespace mysql
#endif /* _FILE_DRIVER_H */

79
replication_listener/include/gtid.h Normal file
View File

@ -0,0 +1,79 @@
/*
Copyright (C) 2013, SkySQL Ab
This file is distributed as part of the SkySQL Gateway. It is free
software: you can redistribute it and/or modify it under the terms of the
GNU General Public License as published by the Free Software Foundation,
version 2.
This program is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
details.
You should have received a copy of the GNU General Public License along with
this program; if not, write to the Free Software Foundation, Inc., 51
Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
Author: Jan Lindström jan.lindstrom@skysql.com
*/
#ifndef REPLICATION_LISTENER_MYSQL_GTID_H
#define REPLICATION_LISTENER_MYSQL_GTID_H
#include <boost/asio.hpp>
#include <boost/function.hpp>
#include <boost/bind.hpp>
namespace mysql
{
enum mysql_server_types {
MYSQL_SERVER_TYPE_NA = 0,
MYSQL_SERVER_TYPE_MARIADB = 1,
MYSQL_SERVER_TYPE_MYSQL = 2
};
class Gtid
{
public:
Gtid()
: m_real_gtid(false), m_domain_id(0), m_server_id(0), m_sequence_number(0), m_mysql_sid(""), m_server_type(MYSQL_SERVER_TYPE_NA)
{}
Gtid(const boost::uint32_t domain_id,
const boost::uint32_t server_id,
const boost::uint64_t sequence_number);
Gtid(const std::string &mysql_sid,
const boost::uint64_t gno);
~Gtid() {}
bool is_real_gtid() const { return m_real_gtid;}
const std::string& get_mysql_gtid() const { return m_mysql_sid;}
std::string get_string() const;
boost::uint32_t get_domain_id() const { return m_domain_id; }
boost::uint32_t get_server_id() const { return m_server_id; }
boost::uint32_t get_sequence_number() const { return m_sequence_number; }
private:
bool m_real_gtid;
mysql_server_types m_server_type;
boost::uint32_t m_domain_id;
boost::uint32_t m_server_id;
boost::uint64_t m_sequence_number;
std::string m_mysql_sid;
};
}
#endif

58
replication_listener/include/listener_exception.h Normal file
View File

@ -0,0 +1,58 @@
/*
Copyright (C) 2013, SkySQL Ab
This file is distributed as part of the SkySQL Gateway. It is free
software: you can redistribute it and/or modify it under the terms of the
GNU General Public License as published by the Free Software Foundation,
version 2.
This program is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
details.
You should have received a copy of the GNU General Public License along with
this program; if not, write to the Free Software Foundation, Inc., 51
Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
Author: Jan Lindström jan.lindstrom@skysql.com
*/
#ifndef REPLICATION_LISTENER_MYSQL_ERROR_EXCEPTION
#define REPLICATION_LISTENER_MYSQL_ERROR_EXCEPTION
namespace mysql
{
// Derive from std::runtime_error rather than std::exception
// runtime_error's constructor can take a string as parameter
// the standard's compliant version of std::exception can not
// (though some compiler provide a non standard constructor).
//
#include <sstream>
#include <boost/system/system_error.hpp>
// Helper function
template <class T>
inline std::string to_string (const T& t)
{
std::stringstream ss;
ss << t;
return ss.str();
}
class ListenerException : public std::runtime_error
{
public:
ListenerException(std::string message, const char *file, int line)
: std::runtime_error(std::string("Exception: ") + message + std::string(" file: ") + std::string(file) + std::string(" line: ") + (to_string(line))) {}
};
}
#endif

437
replication_listener/include/protocol.h Normal file
View File

@ -0,0 +1,437 @@
/*
Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights
reserved.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
02110-1301 USA
*/
#ifndef _PROTOCOL_H
#define _PROTOCOL_H
#include <boost/asio.hpp>
#include <list>
#include "binlog_event.h"
using boost::asio::ip::tcp;
namespace mysql {
namespace system {
/**
Storage structure for the handshake package sent from the server to
the client.
*/
struct st_handshake_package
{
boost::uint8_t protocol_version;
std::string server_version_str;
boost::uint32_t thread_id;
boost::uint8_t scramble_buff[8];
boost::uint16_t server_capabilities;
boost::uint8_t server_language;
boost::uint16_t server_status;
boost::uint8_t scramble_buff2[13];
};
/**
Storage structure for the OK package sent from the server to
the client.
*/
struct st_ok_package
{
boost::uint8_t result_type;
boost::uint64_t affected_rows;
boost::uint64_t insert_id;
boost::uint16_t server_status;
boost::uint16_t warning_count;
std::string message;
};
struct st_eof_package
{
boost::uint16_t warning_count;
boost::uint16_t status_flags;
};
/**
Storage structure for the Error package sent from the server to
the client.
*/
struct st_error_package
{
boost::uint16_t error_code;
boost::uint8_t sql_state[5];
std::string message;
};
#define CLIENT_LONG_PASSWORD 1 /* new more secure passwords */
#define CLIENT_FOUND_ROWS 2 /* Found instead of affected rows */
#define CLIENT_LONG_FLAG 4 /* Get all column flags */
#define CLIENT_CONNECT_WITH_DB 8 /* One can specify db on connect */
#define CLIENT_NO_SCHEMA 16 /* Don't allow database.table.column */
#define CLIENT_COMPRESS 32 /* Can use compression protocol */
#define CLIENT_ODBC 64 /* Odbc client */
#define CLIENT_LOCAL_FILES 128 /* Can use LOAD DATA LOCAL */
#define CLIENT_IGNORE_SPACE 256 /* Ignore spaces before '(' */
#define CLIENT_PROTOCOL_41 512 /* New 4.1 protocol */
#define CLIENT_INTERACTIVE 1024 /* This is an interactive client */
#define CLIENT_SSL 2048 /* Switch to SSL after handshake */
#define CLIENT_IGNORE_SIGPIPE 4096 /* IGNORE sigpipes */
#define CLIENT_TRANSACTIONS 8192 /* Client knows about transactions */
#define CLIENT_RESERVED 16384 /* Old flag for 4.1 protocol */
#define CLIENT_SECURE_CONNECTION 32768 /* New 4.1 authentication */
#define CLIENT_MULTI_STATEMENTS (1UL << 16) /* Enable/disable multi-stmt support */
#define CLIENT_MULTI_RESULTS (1UL << 17) /* Enable/disable multi-results */
#define CLIENT_SSL_VERIFY_SERVER_CERT (1UL << 30)
#define CLIENT_REMEMBER_OPTIONS (1UL << 31)
/* Gather all possible capabilites (flags) supported by the server */
#define CLIENT_ALL_FLAGS (CLIENT_LONG_PASSWORD | \
CLIENT_FOUND_ROWS | \
CLIENT_LONG_FLAG | \
CLIENT_CONNECT_WITH_DB | \
CLIENT_NO_SCHEMA | \
CLIENT_COMPRESS | \
CLIENT_ODBC | \
CLIENT_LOCAL_FILES | \
CLIENT_IGNORE_SPACE | \
CLIENT_PROTOCOL_41 | \
CLIENT_INTERACTIVE | \
CLIENT_SSL | \
CLIENT_IGNORE_SIGPIPE | \
CLIENT_TRANSACTIONS | \
CLIENT_RESERVED | \
CLIENT_SECURE_CONNECTION | \
CLIENT_MULTI_STATEMENTS | \
CLIENT_MULTI_RESULTS | \
CLIENT_SSL_VERIFY_SERVER_CERT | \
CLIENT_REMEMBER_OPTIONS)
/*
Switch off the flags that are optional and depending on build flags
If any of the optional flags is supported by the build it will be switched
on before sending to the client during the connection handshake.
*/
#define CLIENT_BASIC_FLAGS (((CLIENT_ALL_FLAGS & ~CLIENT_SSL) \
& ~CLIENT_COMPRESS) \
& ~CLIENT_SSL_VERIFY_SERVER_CERT)
enum enum_server_command
{
COM_SLEEP, COM_QUIT, COM_INIT_DB, COM_QUERY, COM_FIELD_LIST,
COM_CREATE_DB, COM_DROP_DB, COM_REFRESH, COM_SHUTDOWN, COM_STATISTICS,
COM_PROCESS_INFO, COM_CONNECT, COM_PROCESS_KILL, COM_DEBUG, COM_PING,
COM_TIME, COM_DELAYED_INSERT, COM_CHANGE_USER, COM_BINLOG_DUMP,
COM_TABLE_DUMP, COM_CONNECT_OUT, COM_REGISTER_SLAVE,
COM_STMT_PREPARE, COM_STMT_EXECUTE, COM_STMT_SEND_LONG_DATA, COM_STMT_CLOSE,
COM_STMT_RESET, COM_SET_OPTION, COM_STMT_FETCH, COM_DAEMON,
COM_BINLOG_DUMP_GTID,
/* don't forget to update const char *command_name[] in sql_parse.cc */
/* Must be last */
COM_END
};
enum enum_field_types { MYSQL_TYPE_DECIMAL, MYSQL_TYPE_TINY,
MYSQL_TYPE_SHORT, MYSQL_TYPE_LONG,
MYSQL_TYPE_FLOAT, MYSQL_TYPE_DOUBLE,
MYSQL_TYPE_NULL, MYSQL_TYPE_TIMESTAMP,
MYSQL_TYPE_LONGLONG,MYSQL_TYPE_INT24,
MYSQL_TYPE_DATE, MYSQL_TYPE_TIME,
MYSQL_TYPE_DATETIME, MYSQL_TYPE_YEAR,
MYSQL_TYPE_NEWDATE, MYSQL_TYPE_VARCHAR,
MYSQL_TYPE_BIT,
MYSQL_TYPE_NEWDECIMAL=246,
MYSQL_TYPE_ENUM=247,
MYSQL_TYPE_SET=248,
MYSQL_TYPE_TINY_BLOB=249,
MYSQL_TYPE_MEDIUM_BLOB=250,
MYSQL_TYPE_LONG_BLOB=251,
MYSQL_TYPE_BLOB=252,
MYSQL_TYPE_VAR_STRING=253,
MYSQL_TYPE_STRING=254,
MYSQL_TYPE_GEOMETRY=255
};
#define int3store(T,A) do { *(T)= (unsigned char) ((A));\
*(T+1)=(unsigned char) (((unsigned int) (A) >> 8));\
*(T+2)=(unsigned char) (((A) >> 16)); } while (0)
/*
* Helper functions
*
static void proto_append_int_len(boost::asio::streambuf &buf, unsigned long long num, int len)
{
std::ostream os(&buf);
for (int i= 0; i< len; i++)
{
os << (char) (num & 0xFF);
num= num >> 8;
}
}
static void proto_append_int_len(std::ostream &os, unsigned long long num, int len)
{
for (int i= 0; i< len; i++)
{
unsigned char shift= (num >> i) & 0xFF;
os << shift;
}
}
static void proto_append_int_len(char *buff, unsigned long long num, int len)
{
for (int i= 0; i< len; i++)
{
buff[i]= (unsigned char) (num & 0xFF);
num= num >> 8;
}
}
*/
void write_packet_header(char *buff, boost::uint16_t size, boost::uint8_t packet_no);
class Protocol_validator;
class buffer_source;
/**
* The Protocol interface is used to describe a grammar consisting of
* chunks of bytes that are read or written in consequtive order.
* Example:
* class Protocol_impl : public Protocol;
* Protocol_impl chunk1(chunk_datastore1);
* Protocol_impl chunk2(chunk_datastore2);
* input_stream >> chunk1
* >> chunk2;
*/
class Protocol
{
public:
Protocol() { m_length_encoded_binary= false; }
/**
Return the number of bytes which is read or written by this protocol chunk.
The default size is equal to the underlying storage data type.
*/
virtual unsigned int size()=0;
/** Return a pointer to the first byte in a linear storage buffer */
virtual const char *data()=0;
/**
Change the number of bytes which will be read or written to the storage.
The default size is euqal to the storage type size. This can change if the
protocol is reading a length encoded binary.
The new size must always be less than the size of the underlying storage
type.
*/
virtual void collapse_size(unsigned int new_size)=0;
/**
The first byte will have a special interpretation which will indicate
how many bytes should be read next.
*/
void set_length_encoded_binary(bool bswitch) { m_length_encoded_binary= bswitch; }
bool is_length_encoded_binary(void) { return m_length_encoded_binary; }
private:
bool m_length_encoded_binary;
friend std::istream &operator<<(std::istream &is, Protocol &chunk);
friend std::istream &operator>>(std::istream &is, Protocol &chunk);
friend buffer_source &operator>>(buffer_source &src, Protocol &chunk);
friend std::istream &operator>>(std::istream &is, std::string &str);
};
template<typename T>
class Protocol_chunk : public Protocol
{
public:
Protocol_chunk() : Protocol()
{
m_size= 0;
m_data= 0;
}
Protocol_chunk(T &chunk) : Protocol()
{
m_data= (const char *)&chunk;
m_size= sizeof(T);
}
Protocol_chunk(const T &chunk) : Protocol ()
{
m_data= (const char *) &chunk;
m_size= sizeof(T);
}
/**
* @param buffer A pointer to the storage
* @param size The size of the storage
*
* @note If size == 0 then the chunk is a
* length coded binary.
*/
Protocol_chunk(T *buffer, unsigned long size) : Protocol ()
{
m_data= (const char *)buffer;
m_size= size;
}
virtual unsigned int size() { return m_size; }
virtual const char *data() { return m_data; }
virtual void collapse_size(unsigned int new_size)
{
//assert(new_size <= m_size);
memset((char *)m_data+new_size,'\0', m_size-new_size);
m_size= new_size;
}
private:
const char *m_data;
unsigned long m_size;
};
std::ostream &operator<<(std::ostream &os, Protocol &chunk);
typedef Protocol_chunk<boost::uint8_t> Protocol_chunk_uint8;
class Protocol_chunk_string //: public Protocol_chunk_uint8
{
public:
Protocol_chunk_string(std::string &chunk, unsigned long size) //: Protocol_chunk_uint8()
{
m_str= &chunk;
m_str->assign(size,'*');
}
virtual unsigned int size() const { return m_str->size(); }
virtual const char *data() const { return m_str->data(); }
virtual void collapse_size(unsigned int new_size)
{
m_str->resize(new_size);
}
private:
friend std::istream &operator>>(std::istream &is, Protocol_chunk_string &str);
std::string *m_str;
};
class Protocol_chunk_vector : public Protocol_chunk_uint8
{
public:
Protocol_chunk_vector(std::vector<boost::uint8_t> &chunk, unsigned long size)
: Protocol_chunk_uint8()
{
m_vec= &chunk;
m_vec->reserve(size);
m_size= size;
}
virtual unsigned int size() { return m_vec->size(); }
virtual const char *data() { return reinterpret_cast<const char *>(&*m_vec->begin()); }
virtual void collapse_size(unsigned int new_size)
{
m_vec->resize(new_size);
}
private:
friend std::istream &operator>>(std::istream &is, Protocol_chunk_vector &chunk);
std::vector<boost::uint8_t> *m_vec;
unsigned long m_size;
};
std::istream &operator>>(std::istream &is, Protocol_chunk_vector &chunk);
class buffer_source
{
public:
buffer_source(const char *src, int sz)
{
m_src= src;
m_size= sz;
m_ptr= 0;
}
friend buffer_source &operator>>(buffer_source &src, Protocol &chunk);
private:
const char *m_src;
int m_size;
int m_ptr;
};
class Protocol_chunk_string_len
{
public:
Protocol_chunk_string_len(std::string &str)
{
m_storage= &str;
}
private:
friend std::istream &operator>>(std::istream &is, Protocol_chunk_string_len &lenstr);
std::string *m_storage;
};
buffer_source &operator>>(buffer_source &src, Protocol &chunk);
/** TODO assert that the correct endianess is used */
std::istream &operator>>(std::istream &is, Protocol &chunk);
std::istream &operator>>(std::istream &is, std::string &str);
std::istream &operator>>(std::istream &is, Protocol_chunk_string_len &lenstr);
std::istream &operator>>(std::istream &is, Protocol_chunk_string &str);
int proto_read_package_header(tcp::socket *socket, unsigned long *packet_length, unsigned char *packet_no);
/**
* Read a server package header from a stream buffer
*
* @retval 0 Success
* @retval >0 An error occurred
*/
int proto_read_package_header(tcp::socket *socket, boost::asio::streambuf &buff, unsigned long *packet_length, unsigned char *packet_no);
/**
* Get one complete packet from the server
*
* @param socket Pointer to the active tcp-socket
* @param buff A reference to a stream buffer
* @param packet_no [out] The number of the packet as given by the server
*
* @return the size of the packet or 0 to indicate an error
*/
int proto_get_one_package(tcp::socket *socket, boost::asio::streambuf &buff, boost::uint8_t *packet_no);
void prot_parse_error_message(std::istream &is, struct st_error_package &err, int packet_length);
void prot_parse_ok_message(std::istream &is, struct st_ok_package &ok, int packet_length);
void prot_parse_eof_message(std::istream &is, struct st_eof_package &eof);
void proto_get_handshake_package(std::istream &is, struct st_handshake_package &p, int packet_length);
/**
Allocates a new event and copy the header. The caller must be responsible for
releasing the allocated memory.
*/
Query_event *proto_query_event(std::istream &is, Log_event_header *header);
Rotate_event *proto_rotate_event(std::istream &is, Log_event_header *header);
Incident_event *proto_incident_event(std::istream &is, Log_event_header *header);
Row_event *proto_rows_event(std::istream &is, Log_event_header *header);
Table_map_event *proto_table_map_event(std::istream &is, Log_event_header *header);
Int_var_event *proto_intvar_event(std::istream &is, Log_event_header *header);
User_var_event *proto_uservar_event(std::istream &is, Log_event_header *header);
Gtid_event *proto_gtid_event(std::istream &is, Log_event_header *header);
} // end namespace system
} // end namespace mysql
#endif /* _PROTOCOL_H */

185
replication_listener/include/resultset_iterator.h Normal file
View File

@ -0,0 +1,185 @@
/*
Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights
reserved.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
02110-1301 USA
*/
#ifndef _RESULTSET_ITERATOR_H
#define _RESULTSET_ITERATOR_H
#include <iostream>
// if error; try #include <boost/iterator.hpp>
#include <boost/iterator/iterator_facade.hpp>
#include <boost/asio.hpp>
#include "value.h"
#include "rowset.h"
#include "row_of_fields.h"
using namespace mysql;
namespace mysql
{
struct Field_packet
{
std::string catalog; // Length Coded String
std::string db; // Length Coded String
std::string table; // Length Coded String
std::string org_table;// Length Coded String
std::string name; // Length Coded String
std::string org_name; // Length Coded String
boost::uint8_t marker; // filler
boost::uint16_t charsetnr; // charsetnr
boost::uint32_t length; // length
boost::uint8_t type; // field type
boost::uint16_t flags;
boost::uint8_t decimals;
boost::uint16_t filler; // filler, always 0x00
//boost::uint64_t default_value; // Length coded binary; only in table descr.
};
typedef std::list<std::string > String_storage;
namespace system {
void digest_result_header(std::istream &is, boost::uint64_t &field_count, boost::uint64_t extra);
void digest_field_packet(std::istream &is, Field_packet &field_packet);
void digest_marker(std::istream &is);
void digest_row_content(std::istream &is, int field_count, Row_of_fields &row, String_storage &storage, bool &is_eof);
}
template <class T>
class Result_set_iterator;
class Result_set
{
public:
typedef Result_set_iterator<Row_of_fields > iterator;
typedef Result_set_iterator<Row_of_fields const > const_iterator;
Result_set(tcp::socket *socket) { source(socket); }
void source(tcp::socket *socket) { m_socket= socket; digest_row_set(); }
iterator begin();
iterator end();
const_iterator begin() const;
const_iterator end() const;
private:
void digest_row_set();
friend class Result_set_iterator<Row_of_fields >;
friend class Result_set_iterator<Row_of_fields const>;
std::vector<Field_packet > m_field_types;
int m_row_count;
std::vector<Row_of_fields > m_rows;
String_storage m_storage;
tcp::socket *m_socket;
typedef enum { RESULT_HEADER,
FIELD_PACKETS,
MARKER,
ROW_CONTENTS,
EOF_PACKET
} state_t;
state_t m_current_state;
/**
* The number of fields in the field packets block
*/
boost::uint64_t m_field_count;
/**
* Used for SHOW COLUMNS to return the number of rows in the table
*/
boost::uint64_t m_extra;
};
template <class Iterator_value_type >
class Result_set_iterator :
public boost::iterator_facade<Result_set_iterator<Iterator_value_type >,
Iterator_value_type,
boost::forward_traversal_tag >
{
public:
Result_set_iterator() : m_feeder(0), m_current_row(-1)
{}
explicit Result_set_iterator(Result_set *feeder) : m_feeder(feeder),
m_current_row(-1)
{
increment();
}
private:
friend class boost::iterator_core_access;
void increment()
{
if (++m_current_row >= m_feeder->m_row_count)
m_current_row= -1;
}
bool equal(const Result_set_iterator& other) const
{
if (other.m_feeder == 0 && m_feeder == 0)
return true;
if (other.m_feeder == 0)
{
if (m_current_row == -1)
return true;
else
return false;
}
if (m_feeder == 0)
{
if (other.m_current_row == -1)
return true;
else
return false;
}
if( other.m_feeder->m_field_count != m_feeder->m_field_count)
return false;
Iterator_value_type *row1= &m_feeder->m_rows[m_current_row];
Iterator_value_type *row2= &other.m_feeder->m_rows[m_current_row];
for (unsigned i=0; i< m_feeder->m_field_count; ++i)
{
Value val1= row1->at(i);
Value val2= row2->at(i);
if (val1 != val2)
return false;
}
return true;
}
Iterator_value_type &dereference() const
{
return m_feeder->m_rows[m_current_row];
}
private:
Result_set *m_feeder;
int m_current_row;
};
} // end namespace mysql
#endif /* _RESULTSET_ITERATOR_H */

47
replication_listener/include/row_of_fields.h Normal file
View File

@ -0,0 +1,47 @@
/*
Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights
reserved.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
02110-1301 USA
*/
#ifndef _ROW_OF_FIELDS_H
#define _ROW_OF_FIELDS_H
#include <vector>
#include <iostream>
#include "value.h"
using namespace mysql;
namespace mysql
{
class Row_of_fields : public std::vector<Value >
{
public:
Row_of_fields() : std::vector<Value >(0) { }
Row_of_fields(int field_count) : std::vector<Value >(field_count) {}
virtual ~Row_of_fields() {}
Row_of_fields& operator=(const Row_of_fields &right);
Row_of_fields& operator=(Row_of_fields &right);
private:
};
}
#endif /* _ROW_OF_FIELDS_H */

55
replication_listener/include/rowset.h Normal file
View File

@ -0,0 +1,55 @@
/*
Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights
reserved.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
02110-1301 USA
*/
#ifndef _ROWSET_H
#define _ROWSET_H
#include "field_iterator.h"
#include "resultset_iterator.h"
#include <boost/function.hpp>
#include <boost/iterator.hpp>
using namespace mysql;
namespace mysql {
class Row_event;
class Table_map_event;
class Row_event_set
{
public:
typedef Row_event_iterator<Row_of_fields > iterator;
typedef Row_event_iterator<Row_of_fields const > const_iterator;
Row_event_set(Row_event *arg1, Table_map_event *arg2) { source(arg1, arg2); }
iterator begin() { return iterator(m_row_event, m_table_map_event); }
iterator end() { return iterator(); }
const_iterator begin() const { return const_iterator(m_row_event, m_table_map_event); }
const_iterator end() const { return const_iterator(); }
private:
void source(Row_event *arg1, Table_map_event *arg2) { m_row_event= arg1; m_table_map_event= arg2; }
Row_event *m_row_event;
Table_map_event *m_table_map_event;
};
}
#endif /* _ROWSET_H */

291
replication_listener/include/tcp_driver.h Normal file
View File

@ -0,0 +1,291 @@
/*
Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights
reserved.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
02110-1301 USA
*/
#ifndef _TCP_DRIVER_H
#define _TCP_DRIVER_H
#include "binlog_driver.h"
#include "bounded_buffer.h"
#include "protocol.h"
#include <boost/asio.hpp>
#include <boost/thread.hpp>
#include "gtid.h"
#define MAX_PACKAGE_SIZE 0xffffff
#define GET_NEXT_PACKET_HEADER \
boost::asio::async_read(*m_socket, boost::asio::buffer(m_net_header, 4), \
boost::bind(&Binlog_tcp_driver::handle_net_packet_header, this, \
boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred)) \
using boost::asio::ip::tcp;
namespace mysql { namespace system {
class Binlog_tcp_driver : public Binary_log_driver
{
public:
Binlog_tcp_driver(const std::string& user, const std::string& passwd,
const std::string& host, unsigned long port)
: Binary_log_driver("", 4), m_host(host), m_user(user), m_passwd(passwd),
m_port(port), m_socket(NULL), m_waiting_event(0), m_event_loop(0),
m_total_bytes_transferred(0), m_shutdown(false), m_packet_no(0),
m_event_queue(new bounded_buffer<Binary_log_event*>(50))
{
}
~Binlog_tcp_driver()
{
delete m_event_queue;
delete m_socket;
}
/**
* Connect using previously declared connection parameters.
*/
int connect(Gtid gtid = Gtid());
/**
* Blocking wait for the next binary log event to reach the client
*/
int wait_for_next_event(mysql::Binary_log_event **event);
/**
* Reconnects to the master with a new binlog dump request.
*/
int set_position(const std::string &str, unsigned long position);
/**
* Reconnects to the master with a new binlog dump request.
*/
int set_position_gtid(const Gtid gtid);
int get_position(std::string *str, unsigned long *position);
const std::string& user() const { return m_user; }
const std::string& password() const { return m_passwd; }
const std::string& host() const { return m_host; }
unsigned long port() const { return m_port; }
int fetch_server_version(const std::string& user,
const std::string& passwd,
const std::string& host,
long port);
protected:
/**
* Connects to a mysql server, authenticates and initiates the event
* request loop.
*
* @param user The user account on the server side
* @param passwd The password used to authenticate the user
* @param host The DNS host name or IP of the server
* @param port The service port number to connect to
*
*
* @return Success or failure code
* @retval 0 Successfully established a connection
* @retval >1 An error occurred.
*/
int connect(const std::string& user, const std::string& passwd,
const std::string& host, long port,
const Gtid gtid = Gtid(),
const std::string& binlog_filename="", size_t offset=4);
bool send_client_capabilites(tcp::socket *socket);
bool send_slave_connect_state(tcp::socket *socket,Gtid gtid);
tcp::socket *sync_connect_and_authenticate(boost::asio::io_service &io_service,
const std::string &user,
const std::string &passwd,
const std::string &host,
long port);
int authenticate(tcp::socket *socket,
const std::string& user,
const std::string& passwd,
const st_handshake_package &handshake_package);
bool fetch_master_status(tcp::socket *socket,
std::string *filename,
unsigned long *position);
bool fetch_binlogs_name_and_size(tcp::socket *socket,
std::map<std::string,
unsigned long> &binlog_map);
private:
/**
* Request a binlog dump and starts the event loop in a new thread
* @param binlog_file_name The base name of the binlog files to query
*
*/
void start_binlog_dump(const std::string &binlog_file_name, size_t offset);
void start_binlog_dump(const Gtid gtid);
/**
* Handles a completed mysql server package header and put a
* request for the body in the job queue.
*/
void handle_net_packet_header(const boost::system::error_code& err, std::size_t bytes_transferred);
/**
* Handles a completed network package with the assumption that it contains
* a binlog event.
*
* TODO rename to handle_event_log_packet?
*/
void handle_net_packet(const boost::system::error_code& err, std::size_t bytes_transferred);
/**
* Called from handle_net_packet(). The function handle a stream of bytes
* representing event packets which may or may not be complete.
* It uses m_waiting_event and the size of the stream as parameters
* in a state machine. If there is no m_waiting_event then the event
* header must be parsed for the event packet length. This can only
* be done if the accumulated stream of bytes are more than 19.
* Next, if there is a m_waiting_event, it can only be completed if
* event_length bytes are waiting on the stream.
*
* If none of these conditions are fullfilled, the function exits without
* any action.
*
* @param err Not used
* @param bytes_transferred The number of bytes waiting in the event stream
*
*/
void handle_event_packet(const boost::system::error_code& err, std::size_t bytes_transferred);
/**
* Executes io_service in a loop.
* TODO Checks for connection errors and reconnects to the server
* if necessary.
*/
void start_event_loop(void);
/**
* Reconnect to the server by first calling disconnect and then connect.
*/
void reconnect(Gtid gtid = Gtid());
/**
* Disconnet from the server. The io service must have been stopped before
* this function is called.
* The event queue is emptied.
*/
void disconnect(void);
/**
* Terminates the io service and sets the shudown flag.
* this causes the event loop to terminate.
*/
void shutdown(void);
boost::thread *m_event_loop;
boost::asio::io_service m_io_service;
tcp::socket *m_socket;
bool m_shutdown;
/**
* Temporary storage for a handshake package
*/
st_handshake_package m_handshake_package;
/**
* Temporary storage for an OK package
*/
st_ok_package m_ok_package;
/**
* Temporary storage for an error package
*/
st_error_package m_error_package;
/**
* each bin log event starts with a 19 byte long header
* We use this sturcture every time we initiate an async
* read.
*/
boost::uint8_t m_event_header[19];
/**
*
*/
boost::uint8_t m_net_header[4];
/**
*
*/
boost::uint8_t m_net_packet[MAX_PACKAGE_SIZE];
boost::asio::streambuf m_event_stream_buffer;
char * m_event_packet;
/**
* This pointer points to an object constructed from event
* stream during async communication with
* server. If it is 0 it means that no event has been
* constructed yet.
*/
Log_event_header *m_waiting_event;
Log_event_header m_log_event_header;
/**
* A ring buffer used to dispatch aggregated events to the user application
*/
bounded_buffer<Binary_log_event *> *m_event_queue;
std::string m_user;
std::string m_host;
std::string m_passwd;
long m_port;
boost::uint32_t m_packet_no;
boost::uint64_t m_total_bytes_transferred;
};
/**
* Sends a SHOW MASTER STATUS command to the server and retrieve the
* current binlog position.
*
* @return False if the operation succeeded, true if it failed.
*/
bool fetch_master_status(tcp::socket *socket, std::string *filename, unsigned long *position);
/**
* Sends a SHOW BINARY LOGS command to the server and stores the file
* names and sizes in a map.
*/
bool fetch_binlogs_name_and_size(tcp::socket *socket, std::map<std::string, unsigned long> &binlog_map);
int authenticate(tcp::socket *socket, const std::string& user,
const std::string& passwd,
const st_handshake_package &handshake_package);
tcp::socket *
sync_connect_and_authenticate(boost::asio::io_service &io_service, const std::string &user,
const std::string &passwd, const std::string &host, long port);
} }
#endif /* _TCP_DRIVER_H */

53
replication_listener/include/utilities.h Normal file
View File

@ -0,0 +1,53 @@
/*
Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights
reserved.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
02110-1301 USA
*/
#ifndef _UTILITIES_H
#define _UTILITIES_H
#include "value.h"
#include "protocol.h"
using namespace mysql;
namespace mysql {
typedef enum
{
Q_FLAGS2_CODE= 0,
Q_SQL_MODE_CODE,
Q_CATALOG_CODE,
Q_AUTO_INCREMENT,
Q_CHARSET_CODE,
Q_TIME_ZONE_CODE,
Q_CATALOG_NZ_CODE,
Q_LC_TIME_NAMES_CODE,
Q_CHARSET_DATABASE_CODE,
Q_TABLE_MAP_FOR_UPDATE_CODE,
Q_MASTER_DATA_WRITTEN_CODE,
Q_INVOKER
} enum_var_types;
int server_var_decoder (std::map<std::string, mysql::Value> *my_var_map,
std::vector<boost::uint8_t > variables);
}
#endif /* _UTILITIES_H */

181
replication_listener/include/value.h Normal file
View File

@ -0,0 +1,181 @@
/*
Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights
reserved.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
02110-1301 USA
*/
#ifndef _VALUE_ADAPTER_H
#define _VALUE_ADAPTER_H
#include <boost/cstdint.hpp>
#include "protocol.h"
#include <boost/any.hpp>
#include <iostream>
using namespace mysql;
namespace mysql {
/**
This helper function calculates the size in bytes of a particular field in a
row type event as defined by the field_ptr and metadata_ptr arguments.
@param column_type Field type code
@param field_ptr The field data
@param metadata_ptr The field metadata
@note We need the actual field data because the string field size is not
part of the meta data. :(
@return The size in bytes of a particular field
*/
int calc_field_size(unsigned char column_type, const unsigned char *field_ptr,
boost::uint32_t metadata);
/**
* A value object class which encapsluate a tuple (value type, metadata, storage)
* and provide for views to this storage through a well defined interface.
*
* Can be used with a Converter to convert between different Values.
*/
class Value
{
public:
Value(enum system::enum_field_types type, boost::uint32_t metadata, const char *storage) :
m_type(type), m_storage(storage), m_metadata(metadata), m_is_null(false)
{
m_size= calc_field_size((unsigned char)type,
(const unsigned char*)storage,
metadata);
//std::cout << "TYPE: " << type << " SIZE: " << m_size << std::endl;
};
Value()
{
m_size= 0;
m_storage= 0;
m_metadata= 0;
m_is_null= false;
}
/**
* Copy constructor
*/
Value(const Value& val);
Value &operator=(const Value &val);
bool operator==(const Value &val) const;
bool operator!=(const Value &val) const;
~Value() {}
void is_null(bool s) { m_is_null= s; }
bool is_null(void) const { return m_is_null; }
const char *storage() const { return m_storage; }
/**
* Get the length in bytes of the entire storage (any metadata part +
* atual data)
*/
size_t length() const { return m_size; }
enum system::enum_field_types type() const { return m_type; }
boost::uint32_t metadata() const { return m_metadata; }
/**
* Returns the integer representation of a storage of a pre-specified
* type.
*/
boost::int32_t as_int32() const;
/**
* Returns the integer representation of a storage of pre-specified
* type.
*/
boost::int64_t as_int64() const;
/**
* Returns the integer representation of a storage of pre-specified
* type.
*/
boost::int8_t as_int8() const;
/**
* Returns the integer representation of a storage of pre-specified
* type.
*/
boost::int16_t as_int16() const;
/**
* Returns a pointer to the character data of a string type stored
* in the pre-defined storage.
* @note The position is an offset of the storage pointer determined
* by the metadata and type.
*
* @param[out] size The size in bytes of the character string.
*
*/
char *as_c_str(unsigned long &size) const;
/**
* Returns a pointer to the byte data of a blob type stored in the pre-
* defined storage.
* @note The position is an offset of the storage pointer determined
* by the metadata and type.
*
* @param[out] size The size in bytes of the blob data.
*/
unsigned char *as_blob(unsigned long &size) const;
float as_float() const;
double as_double() const;
private:
enum system::enum_field_types m_type;
size_t m_size;
const char *m_storage;
boost::uint32_t m_metadata;
bool m_is_null;
};
class Converter
{
public:
/**
* Converts and copies the sql value to a std::string object.
* @param[out] str The target string
* @param[in] val The value object to be converted
*/
void to(std::string &str, const Value &val) const;
/**
* Converts and copies the sql value to a long integer.
* @param[out] out The target variable
* @param[in] val The value object to be converted
*/
void to(long &out, const Value &val) const;
/**
* Converts and copies the sql value to a floating point number.
* @param[out] out The target variable
* @param[in] val The value object to be converted
*/
void to(float &out, const Value &val) const;
};
} // end namespace mysql
#endif /* _VALUE_ADAPTER_H */