Merge branch '2.3' into develop
This commit is contained in:
@ -80,3 +80,7 @@ case-insensitive by converting all names into their lowercase form.
|
||||
|
||||
**Note:** The identifier names are converted using an ASCII-only function. This
|
||||
means that non-ASCII characters will retain their case-sensitivity.
|
||||
|
||||
```
|
||||
authenticator_options=lower_case_table_names=false
|
||||
```
|
||||
|
@ -3,141 +3,7 @@
|
||||
Table of Contents
|
||||
=================
|
||||
|
||||
* [Introduction](#introduction)
|
||||
* [Glossary](#glossary)
|
||||
* [Administration](#administration)
|
||||
* [Configuration](#configuration)
|
||||
* [Special Parameter Types](#special-parameter-types)
|
||||
* [Sizes](#sizes)
|
||||
* [Regular Expressions](#regular-expressions)
|
||||
* [Global Settings](#global-settings)
|
||||
* [threads](#threads)
|
||||
* [thread_stack_size](#thread_stack_size)
|
||||
* [auth_connect_timeout](#auth_connect_timeout)
|
||||
* [auth_read_timeout](#auth_read_timeout)
|
||||
* [auth_write_timeout](#auth_write_timeout)
|
||||
* [query_retries](#query_retries)
|
||||
* [query_retry_timeout](#query_retry_timeout)
|
||||
* [passive](#passive)
|
||||
* [ms_timestamp](#ms_timestamp)
|
||||
* [skip_permission_checks](#skip_permission_checks)
|
||||
* [syslog](#syslog)
|
||||
* [maxlog](#maxlog)
|
||||
* [log_to_shm](#log_to_shm)
|
||||
* [log_warning](#log_warning)
|
||||
* [log_notice](#log_notice)
|
||||
* [log_info](#log_info)
|
||||
* [log_debug](#log_debug)
|
||||
* [log_messages](#log_messages)
|
||||
* [log_trace](#log_trace)
|
||||
* [log_augmentation](#log_augmentation)
|
||||
* [log_throttling](#log_throttling)
|
||||
* [logdir](#logdir)
|
||||
* [datadir](#datadir)
|
||||
* [libdir](#libdir)
|
||||
* [cachedir](#cachedir)
|
||||
* [piddir](#piddir)
|
||||
* [execdir](#execdir)
|
||||
* [connector_plugindir](#connector_plugindir)
|
||||
* [persistdir](#persistdir)
|
||||
* [module_configdir](#module_configdir)
|
||||
* [language](#language)
|
||||
* [query_classifier](#query_classifier)
|
||||
* [query_classifier_args](#query_classifier_args)
|
||||
* [query_classifier_cache_size](#query_classifier_cache_size)
|
||||
* [log_unrecognized_statements](#log_unrecognized_statements)
|
||||
* [substitute_variables](#substitute_variables)
|
||||
* [sql_mode](#sql_mode)
|
||||
* [local_address](#local_address)
|
||||
* [users_refresh_time](#users_refresh_time)
|
||||
* [retain_last_statements](#retain_last_statements)
|
||||
* [dump_last_statements](#dump_last_statements)
|
||||
* [writeq_high_water](#writeq_high_water)
|
||||
* [writeq_low_water](#writeq_low_water)
|
||||
* [REST API Configuration](#rest-api-configuration)
|
||||
* [admin_host](#admin_host)
|
||||
* [admin_port](#admin_port)
|
||||
* [admin_auth](#admin_auth)
|
||||
* [admin_ssl_key](#admin_ssl_key)
|
||||
* [admin_ssl_cert](#admin_ssl_cert)
|
||||
* [admin_ssl_ca_cert](#admin_ssl_ca_cert)
|
||||
* [admin_enabled](#admin_enabled)
|
||||
* [admin_log_auth_failures](#admin_log_auth_failures)
|
||||
* [<em>events</em>](#events)
|
||||
* ['authentication_failure'](#authentication_failure)
|
||||
* [Service](#service)
|
||||
* [router](#router)
|
||||
* [router_options](#router_options)
|
||||
* [filters](#filters)
|
||||
* [servers](#servers)
|
||||
* [user](#user)
|
||||
* [password](#password)
|
||||
* [enable_root_user](#enable_root_user)
|
||||
* [localhost_match_wildcard_host](#localhost_match_wildcard_host)
|
||||
* [version_string](#version_string)
|
||||
* [weightby](#weightby)
|
||||
* [auth_all_servers](#auth_all_servers)
|
||||
* [strip_db_esc](#strip_db_esc)
|
||||
* [retry_on_failure](#retry_on_failure)
|
||||
* [log_auth_warnings](#log_auth_warnings)
|
||||
* [connection_timeout](#connection_timeout)
|
||||
* [max_connections](#max_connections)
|
||||
* [max_retry_interval](#max_retry_interval)
|
||||
* [session_track_trx_state](#session_track_trx_state)
|
||||
* [retain_last_statements](#retain_last_statements-1)
|
||||
* [Server](#server)
|
||||
* [address](#address)
|
||||
* [port](#port)
|
||||
* [protocol](#protocol)
|
||||
* [monitoruser](#monitoruser)
|
||||
* [monitorpw](#monitorpw)
|
||||
* [extra_port](#extra_port)
|
||||
* [persistpoolmax](#persistpoolmax)
|
||||
* [persistmaxtime](#persistmaxtime)
|
||||
* [proxy_protocol](#proxy_protocol)
|
||||
* [authenticator](#authenticator)
|
||||
* [authenticator_options](#authenticator_options)
|
||||
* [disk_space_threshold](#disk_space_threshold)
|
||||
* [Listener](#listener)
|
||||
* [service](#service-1)
|
||||
* [protocol](#protocol-1)
|
||||
* [address](#address-1)
|
||||
* [port](#port-1)
|
||||
* [socket](#socket)
|
||||
* [authenticator](#authenticator-1)
|
||||
* [authenticator_options](#authenticator_options-1)
|
||||
* [Available Protocols](#available-protocols)
|
||||
* [MariaDBClient](#mariadbclient)
|
||||
* [MariaDBBackend](#mariadbbackend)
|
||||
* [telnetd](#telnetd)
|
||||
* [maxscaled](#maxscaled)
|
||||
* [HTTPD](#httpd)
|
||||
* [TLS/SSL encryption](#tlsssl-encryption)
|
||||
* [ssl](#ssl)
|
||||
* [ssl_key](#ssl_key)
|
||||
* [ssl_cert](#ssl_cert)
|
||||
* [ssl_ca_cert](#ssl_ca_cert)
|
||||
* [ssl_version](#ssl_version)
|
||||
* [ssl_cert_verify_depth](#ssl_cert_verify_depth)
|
||||
* [ssl_verify_peer_certificate](#ssl_verify_peer_certificate)
|
||||
* [Example SSL enabled server configuration](#example-ssl-enabled-server-configuration)
|
||||
* [Example SSL enabled listener configuration](#example-ssl-enabled-listener-configuration)
|
||||
* [Routing Modules](#routing-modules)
|
||||
* [Diagnostic modules](#diagnostic-modules)
|
||||
* [Monitor Modules](#monitor-modules)
|
||||
* [Filter Modules](#filter-modules)
|
||||
* [Encrypting Passwords](#encrypting-passwords)
|
||||
* [Creating Encrypted Passwords](#creating-encrypted-passwords)
|
||||
* [Runtime Configuration Changes](#runtime-configuration-changes)
|
||||
* [Backing Up Configuration Changes](#backing-up-configuration-changes)
|
||||
* [Reloading Configuration](#reloading-configuration)
|
||||
* [Limitations](#limitations)
|
||||
* [Authentication](#authentication)
|
||||
* [Wildcard Hosts](#wildcard-hosts)
|
||||
* [Limitations](#limitations-1)
|
||||
* [Error Reporting](#error-reporting)
|
||||
* [Troubleshooting](#troubleshooting)
|
||||
* [Systemd Watchdog](#systemd-watchdog)
|
||||
[TOC]
|
||||
|
||||
## Introduction
|
||||
|
||||
@ -146,24 +12,99 @@ possible usage scenarios. MariaDB MaxScale is designed with flexibility in mind,
|
||||
and consists of an event processing core with various support functions and
|
||||
plugin modules that tailor the behavior of the program.
|
||||
|
||||
## Glossary
|
||||
## Concepts
|
||||
|
||||
### Glossary
|
||||
|
||||
Word | Description
|
||||
--------------------|----------------------------------------------------
|
||||
service | A service represents a set of databases with a specific access mechanism that is offered to clients of MariaDB MaxScale. The access mechanism defines the algorithm that MariaDB MaxScale will use to direct particular requests to the individual databases.
|
||||
server | A server represents an individual database server to which a client can be connected via MariaDB MaxScale.
|
||||
router | A router is a module within MariaDB MaxScale that will route client requests to the various database servers which MariaDB MaxScale provides a service interface to.
|
||||
connection routing | Connection routing is a method of handling requests in which MariaDB MaxScale will accept connections from a client and route data on that connection to a single database using a single connection. Connection based routing will not examine individual requests on a connection and it will not move that connection once it is established.
|
||||
statement routing | Statement routing is a method of handling requests in which each request within a connection will be handled individually. Requests may be sent to one or more servers and connections may be dynamically added or removed from the session.
|
||||
protocol | A protocol is a module of software that is used to communicate with another software entity within the system. MariaDB MaxScale supports the dynamic loading of protocol modules to allow for increased flexibility.
|
||||
module | A module is a separate code entity that may be loaded dynamically into MariaDB MaxScale to increase the available functionality. Modules are implemented as run-time loadable shared objects.
|
||||
monitor | A monitor is a module that can be executed within MariaDB MaxScale to monitor the state of a set of database. The use of an internal monitor is optional, monitoring may be performed externally to MariaDB MaxScale.
|
||||
listener | A listener is the network endpoint that is used to listen for connections to MariaDB MaxScale from the client applications. A listener is associated to a single service, however, a service may have many listeners.
|
||||
connection failover | When a connection currently being used between MariaDB MaxScale and the database server fails a replacement will be automatically created to another server by MariaDB MaxScale without client intervention
|
||||
backend database | A term used to refer to a database that sits behind MariaDB MaxScale and is accessed by applications via MariaDB MaxScale.
|
||||
filter | A module that can be placed between the client and the MariaDB MaxScale router module. All client data passes through the filter module and may be examined or modified by the filter modules. Filters may be chained together to form processing pipelines.
|
||||
REST API | HTTP administrative interface
|
||||
|
||||
### Objects
|
||||
|
||||
#### Server
|
||||
A server represents an individual database server to which a client can be
|
||||
connected via MariaDB MaxScale. The status of a server varies during the lifetime
|
||||
of the server and typically the status is updated by some monitor. However, it
|
||||
is also possible to update the status of a server manually.
|
||||
|
||||
Status | Description
|
||||
-------|------------
|
||||
Running | The server is running.
|
||||
Master | The server is the master.
|
||||
Slave | The server is a slave.
|
||||
Maintenance | The server is under maintenance. Typically this status bit is turned on manually using _maxctrl_, but it will also be turned on for a server that for some reason is blocking connections from MaxScale. When a server is in maintenace mode, no connections will be created to it and existing connections will be closed.
|
||||
Slave of External Master | The server is a slave of a master that is not being monitored.
|
||||
|
||||
#### Protocol
|
||||
A protocol module is responsible for the low-level communication between
|
||||
MaxScale and either clients of MaxScale or servers exposed by MaxScale.
|
||||
The most commonly used protocol modules are `mariadbclient` and
|
||||
`mariadbbackend`.
|
||||
|
||||
Protocol modules do not have sections of their own in the MaxScale
|
||||
configuration file, but are referred to from _servers_ and _listeners_.
|
||||
|
||||
#### Monitor
|
||||
A monitor module is capable of monitoring the state of a particular kind
|
||||
of cluster and making that state available to the routers of MaxScale.
|
||||
|
||||
Examples of monitor modules are `mariadbmon` that is capable of monitoring
|
||||
a regular master-slave cluster and in addition of performing both _switchover_
|
||||
and _failover_, `galeramon` that is capable of monitoring a Galera cluster,
|
||||
and `csmon` that is capable of monitoring a Columnstore cluster.
|
||||
|
||||
Monitor modules have sections of their own in the MaxScale configuration
|
||||
file.
|
||||
|
||||
#### Filter
|
||||
A filter module resides in front of routers in the request processing chain
|
||||
of MaxScale. That is, a filter will see a request before it reaches the router
|
||||
and before a response is sent back to the client. This allows filters to
|
||||
reject, handle, alter or log information about a request.
|
||||
|
||||
Examples of filters are `dbfwfilter` that is a configurable firewall, `cache`
|
||||
that provides query caching according to rules, `regexfilter` that can rewrite
|
||||
requests according to regular expressions, and `qlafilter` that logs
|
||||
information about requests.
|
||||
|
||||
Filters have sections of their own in the MaxScale configuration file that are
|
||||
referred to from _services_.
|
||||
|
||||
#### Router
|
||||
A router module is capable of routing requests to backend servers according to
|
||||
the characteristics of a request and/or the algorithm the router
|
||||
implements. Examples of routers are `readconnroute` that provides _connection
|
||||
routing_, that is, the server is chosen according to specified rules when the
|
||||
session is created and all requests are subsequently routed to that server,
|
||||
and `readwritesplit` that provides _statement routing_, that is, each
|
||||
individual request is routed to the most appropriate server.
|
||||
|
||||
Routers do not have sections of their own in the MaxScale configuration file,
|
||||
but are referred to from _services_.
|
||||
|
||||
#### Service
|
||||
A service abstracts a set of databases and makes them appear as a single one
|
||||
to the client. Depending on what router (e.g. `readconnroute` or
|
||||
`readwritesplit`) the service uses, the servers are used in some particular
|
||||
way. If the service uses filters, then all requests will be pre-processed in
|
||||
some way before they reach the router.
|
||||
|
||||
Services have sections of their own in the MaxScale configuration file.
|
||||
|
||||
#### Listener
|
||||
A listener defines a port MaxScale listens on. Connection requests arriving on
|
||||
that port will be forwarded to the service the listener is associated with. A
|
||||
listener may be associated with a single service, but several listeners may be
|
||||
associated with the same service.
|
||||
|
||||
Listeners have sections of their own in the MaxScale configuration file.
|
||||
|
||||
## Administration
|
||||
|
||||
The administation of MaxScale can be divided in two parts:
|
||||
|
@ -65,6 +65,11 @@ For information about common service parameters, refer to the
|
||||
|
||||
The source for the binary logs. This is an optional parameter.
|
||||
|
||||
**Note:** If the `source` parameter is defined the values for `binlogdir` and
|
||||
`filestem` are only read from the source service. This means that the
|
||||
parameters defined for the avrorouter are ignored and the ones in the
|
||||
binlogrouter are used.
|
||||
|
||||
The value of this parameter should be the name of a Binlog Server service.
|
||||
The _filestem_ and _binlogdir_ parameters of this service will be read from
|
||||
the router_options and they will be used as the source for the binary logs. This
|
||||
@ -306,6 +311,47 @@ If the CREATE TABLE statements for the tables aren't present in the current
|
||||
binary logs, the schema files must be generated with a schema file
|
||||
generator. There are currently two methods to generate the .avsc schema files.
|
||||
|
||||
### Simple Schema Generator
|
||||
|
||||
The `cdc_one_schema.py` generates a schema file for a single table by reading a
|
||||
tab separated list of field and type names from the standard input. This is the
|
||||
recommended schema generation tool as it does not directly communicate with the
|
||||
database thus making it more flexible.
|
||||
|
||||
The only requirement to run the script is that a Python interpreter is
|
||||
installed.
|
||||
|
||||
To use this script, pipe the output of the `mysql` command line into the
|
||||
`cdc_one_schema.py` script:
|
||||
|
||||
```
|
||||
mysql -ss -u <user> -p -h <host> -P <port> -e 'DESCRIBE `<database>`.`<table>`'|./cdc_one_schema.py <database> <table>
|
||||
```
|
||||
|
||||
Replace the `<user>`, `<host>`, `<port>`, `<database>` and `<table>` with
|
||||
appropriate values and run the command. Note that the `-ss` parameter is
|
||||
mandatory as that will generate the tab separated output instead of the default
|
||||
pretty-printed output.
|
||||
|
||||
An .avsc file named after the database and table name will be generated in the
|
||||
current working directory. Copy this file to the location pointed by the
|
||||
`avrodir` parameter of the avrorouter.
|
||||
|
||||
Alternatively, you can also copy the output of the `mysql` command to a file and
|
||||
feed it into the script if you cannot execute the SQL command directly:
|
||||
|
||||
```
|
||||
# On the database server
|
||||
mysql -ss -u <user> -p -h <host> -P <port> -e 'DESCRIBE `<database>`.`<table>`' > schema.tsv
|
||||
# On the MaxScale server
|
||||
./cdc_one_schema.py <database> <table> < schema.tsv
|
||||
```
|
||||
|
||||
If you want to use a specific Python interpreter instead of the one found in the
|
||||
search path, you can modify the first line of the script from `#!/usr/bin/env
|
||||
python` to `#!/path/to/python` where `/path/to/python` is the absolute path to
|
||||
the Python interpreter (both Python 2 and Python 3 can be used).
|
||||
|
||||
### Python Schema Generator
|
||||
|
||||
```
|
||||
|
@ -427,6 +427,8 @@ bool Connection::read_schema()
|
||||
{
|
||||
m_error = "Failed to parse JSON: ";
|
||||
m_error += err.text;
|
||||
m_error += ". Data received so far: ";
|
||||
m_error += row;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -99,18 +99,30 @@ static const char* datetime_types[] =
|
||||
"DATETIME(4)",
|
||||
"DATETIME(5)",
|
||||
"DATETIME(6)",
|
||||
// TODO: Fix test setup to use same timezone
|
||||
// "TIMESTAMP",
|
||||
NULL
|
||||
};
|
||||
|
||||
static const char* datetime_values[] =
|
||||
{
|
||||
"'2018-01-01 11:11:11'",
|
||||
"'0-00-00 00:00:00'",
|
||||
"NULL",
|
||||
NULL
|
||||
};
|
||||
|
||||
static const char* timestamp_types[] =
|
||||
{
|
||||
"TIMESTAMP",
|
||||
NULL
|
||||
};
|
||||
|
||||
static const char* timestamp_values[] =
|
||||
{
|
||||
"'2018-01-01 11:11:11'",
|
||||
"'0-00-00 00:00:00'",
|
||||
NULL
|
||||
};
|
||||
|
||||
static const char* date_types[] =
|
||||
{
|
||||
"DATE",
|
||||
@ -120,6 +132,7 @@ static const char* date_types[] =
|
||||
static const char* date_values[] =
|
||||
{
|
||||
"'2018-01-01'",
|
||||
"'0-00-00'",
|
||||
"NULL",
|
||||
NULL
|
||||
};
|
||||
@ -154,6 +167,7 @@ struct
|
||||
{string_types, string_values},
|
||||
{binary_types, binary_values},
|
||||
{datetime_types, datetime_values},
|
||||
{timestamp_types, timestamp_values},
|
||||
{date_types, date_values},
|
||||
{time_types, time_values},
|
||||
{0, 0}
|
||||
|
@ -374,7 +374,16 @@ static void unpack_datetime2(uint8_t* ptr, uint8_t decimals, struct tm* dest)
|
||||
static void unpack_timestamp(uint8_t* ptr, uint8_t decimals, struct tm* dest)
|
||||
{
|
||||
time_t t = unpack4(ptr);
|
||||
localtime_r(&t, dest);
|
||||
|
||||
if (t == 0)
|
||||
{
|
||||
// Use GMT date to detect zero date timestamps
|
||||
gmtime_r(&t, dest);
|
||||
}
|
||||
else
|
||||
{
|
||||
localtime_r(&t, dest);
|
||||
}
|
||||
}
|
||||
|
||||
#define unpack3(data) (data[2] + (data[1] << 8) + (data[0] << 16))
|
||||
@ -574,6 +583,13 @@ size_t unpack_temporal_value(uint8_t type, uint8_t* ptr, uint8_t* metadata, int
|
||||
return temporal_field_size(type, metadata, length);
|
||||
}
|
||||
|
||||
static bool is_zero_date(struct tm* tm)
|
||||
{
|
||||
// Detects 1970-01-01 00:00:00
|
||||
return tm->tm_sec == 0 && tm->tm_min == 0 && tm->tm_hour == 0
|
||||
&& tm->tm_mday == 1 && tm->tm_mon == 0 && tm->tm_year == 70;
|
||||
}
|
||||
|
||||
void format_temporal_value(char* str, size_t size, uint8_t type, struct tm* tm)
|
||||
{
|
||||
const char* format = "";
|
||||
@ -605,7 +621,15 @@ void format_temporal_value(char* str, size_t size, uint8_t type, struct tm* tm)
|
||||
mxb_assert(false);
|
||||
break;
|
||||
}
|
||||
strftime(str, size, format, tm);
|
||||
|
||||
if ((type == TABLE_COL_TYPE_TIMESTAMP || type == TABLE_COL_TYPE_TIMESTAMP2) && is_zero_date(tm))
|
||||
{
|
||||
strcpy(str, "0-00-00 00:00:00");
|
||||
}
|
||||
else
|
||||
{
|
||||
strftime(str, size, format, tm);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -56,8 +56,10 @@ static std::string do_query(MXS_MONITORED_SERVER* srv, const char* query)
|
||||
// Returns a numeric version similar to mysql_get_server_version
|
||||
int get_cs_version(MXS_MONITORED_SERVER* srv)
|
||||
{
|
||||
// GCC 4.8 appears to have a broken std::regex_constants::ECMAScript that doesn't support brackets
|
||||
std::regex re("Columnstore \\([0-9]*\\)[.]\\([0-9]*\\)[.]\\([0-9]*\\)-[0-9]*",
|
||||
std::regex_constants::basic);
|
||||
std::string result = do_query(srv, "SELECT @@version_comment");
|
||||
std::regex re("Columnstore ([0-9]*)[.]([0-9]*)[.]([0-9]*)-[0-9]*");
|
||||
std::smatch match;
|
||||
int rval = 0;
|
||||
|
||||
|
@ -3,4 +3,5 @@ install_script(cdc_users.py core)
|
||||
install_script(cdc_last_transaction.py core)
|
||||
install_script(cdc_kafka_producer.py core)
|
||||
install_script(cdc_schema.py core)
|
||||
install_script(cdc_one_schema.py core)
|
||||
install_file(cdc_schema.go core)
|
||||
|
98
server/modules/protocol/examples/cdc_one_schema.py
Executable file
98
server/modules/protocol/examples/cdc_one_schema.py
Executable file
@ -0,0 +1,98 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Copyright (c) 2016 MariaDB Corporation Ab
|
||||
#
|
||||
# Use of this software is governed by the Business Source License included
|
||||
# in the LICENSE.TXT file and at www.mariadb.com/bsl11.
|
||||
#
|
||||
# Change Date: 2022-01-01
|
||||
#
|
||||
# On the date above, in accordance with the Business Source License, use
|
||||
# of this software will be governed by version 2 or later of the General
|
||||
# Public License.
|
||||
|
||||
import json
|
||||
import sys
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description = """
|
||||
|
||||
This program generates a schema file for a single table by reading a tab
|
||||
separated list of field and type names from the standard input.
|
||||
|
||||
To use this script, pipe the output of the `mysql` command line into the
|
||||
`cdc_one_schema.py` script:
|
||||
|
||||
mysql -ss -u <user> -p -h <host> -P <port> -e 'DESCRIBE `<database>`.`<table>`'|./cdc_one_schema.py <database> <table>
|
||||
|
||||
Replace the <user>, <host>, <port>, <database> and <table> with appropriate
|
||||
values and run the command. Note that the `-ss` parameter is mandatory as that
|
||||
will generate the tab separated output instead of the default pretty-printed
|
||||
output.
|
||||
|
||||
An .avsc file named after the database and table name will be generated in the
|
||||
current working directory. Copy this file to the location pointed by the
|
||||
`avrodir` parameter of the avrorouter.
|
||||
|
||||
Alternatively, you can also copy the output of the `mysql` command to a file and
|
||||
feed it into the script if you cannot execute the SQL command directly:
|
||||
|
||||
# On the database server
|
||||
mysql -ss -u <user> -p -h <host> -P <port> -e 'DESCRIBE `<database>`.`<table>`' > schema.tsv
|
||||
# On the MaxScale server
|
||||
./cdc_one_schema.py <database> <table> < schema.tsv
|
||||
|
||||
""", formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument("DATABASE", help="The database name where the table is located")
|
||||
parser.add_argument("TABLE", help="The name of the table")
|
||||
|
||||
opts = parser.parse_args(sys.argv[1:])
|
||||
|
||||
def parse_field(row):
|
||||
res = dict()
|
||||
parts = row[1].lower().split('(')
|
||||
name = parts[0]
|
||||
|
||||
res["real_type"] = name
|
||||
|
||||
if len(parts) > 1 and name not in ["enum", "set", "decimal"]:
|
||||
res["length"] = int(parts[1].split(')')[0])
|
||||
else:
|
||||
res["length"] = -1
|
||||
|
||||
type = "string"
|
||||
|
||||
if name in ("date", "datetime", "time", "timestamp", "year", "tinytext", "text",
|
||||
"mediumtext", "longtext", "char", "varchar", "enum", "set"):
|
||||
type = "string"
|
||||
elif name in ("tinyblob", "blob", "mediumblob", "longblob", "binary", "varbinary"):
|
||||
type = "bytes"
|
||||
elif name in ("int", "smallint", "mediumint", "integer", "tinyint", "short", "bit"):
|
||||
type = "int"
|
||||
elif name in ("float"):
|
||||
type = "float"
|
||||
elif name in ("double", "decimal"):
|
||||
type = "double"
|
||||
elif name in ("long", "bigint"):
|
||||
type = "long"
|
||||
|
||||
res["type"] = ["null", type]
|
||||
|
||||
res["name"] = row[0].lower()
|
||||
|
||||
return res
|
||||
|
||||
try:
|
||||
schema = dict(namespace="MaxScaleChangeDataSchema.avro", type="record", name="ChangeRecord", fields=[])
|
||||
for line in sys.stdin:
|
||||
schema["fields"].append(parse_field(line.split('\t')))
|
||||
|
||||
print("Creating: {}.{}.000001.avsc".format(opts.DATABASE, opts.TABLE))
|
||||
dest = open("{}.{}.000001.avsc".format(opts.DATABASE, opts.TABLE), 'w')
|
||||
dest.write(json.dumps(schema))
|
||||
dest.close()
|
||||
|
||||
except Exception as e:
|
||||
print(e)
|
||||
exit(1)
|
||||
|
@ -652,17 +652,30 @@ void avro_load_metadata_from_schemas(Avro* router)
|
||||
for (int i = files.gl_pathc - 1; i > -1; i--)
|
||||
{
|
||||
char* dbstart = strrchr(files.gl_pathv[i], '/');
|
||||
mxb_assert(dbstart);
|
||||
|
||||
if (!dbstart)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
dbstart++;
|
||||
|
||||
char* tablestart = strchr(dbstart, '.');
|
||||
mxb_assert(tablestart);
|
||||
|
||||
if (!tablestart)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
snprintf(db, sizeof(db), "%.*s", (int)(tablestart - dbstart), dbstart);
|
||||
tablestart++;
|
||||
|
||||
char* versionstart = strchr(tablestart, '.');
|
||||
mxb_assert(versionstart);
|
||||
|
||||
if (!versionstart)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
snprintf(table, sizeof(table), "%.*s", (int)(versionstart - tablestart), tablestart);
|
||||
versionstart++;
|
||||
|
Reference in New Issue
Block a user