From 3ec91735a83c5f3aac90e728e6d49913d1e9c524 Mon Sep 17 00:00:00 2001 From: Johan Wikman Date: Wed, 28 Jun 2017 20:59:56 +0200 Subject: [PATCH 01/11] Update 2.0 license text --- LICENSE.TXT | 133 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 87 insertions(+), 46 deletions(-) diff --git a/LICENSE.TXT b/LICENSE.TXT index 21734b80a..630351ce3 100644 --- a/LICENSE.TXT +++ b/LICENSE.TXT @@ -1,63 +1,104 @@ -Licensor: MariaDB Corporation Ab +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. -Software: MariaDB MaxScale (TM) v.2.0. - The Software is (c) 2016 MariaDB Corporation Ab +Parameters -Use Limitation: Usage of the software is free when your application uses - the Software with a total of less than three database - server instances for production purposes. +Licensor: MariaDB Corporation Ab +Licensed Work: MariaDB MaxScale (TM) v.2.0. + The Licensed Work is (c) 2017 MariaDB Corporation Ab +Additional Use Grant: You may use the Licensed Work when your application + uses the Licensed Work with a total of less than three + server instances for any purpose. -Change Date: 2019-01-01 +Change Date: 2019-01-01 -Change License: Version 2 or later of the GNU General Public License as - published by the Free Software Foundation. +Change License: Version 2 or later of the GNU General Public License as + published by the Free Software Foundation. For information about alternative licensing arrangements for the Software, please visit: https://mariadb.com/products/mariadb-enterprise --------------------------------------------------------------------------- +Notice -Business Source License 0.9 +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. -You are granted limited license to the Software under this Business Source -License. Please read this Business Source License carefully, particularly -the Use Limitation set forth above. +For more information on the use of the Business Source License for MariaDB +products, please visit the MariaDB Business Source License FAQ at +https://mariadb.com/bsl-faq-mariadb. -Subject to the Use Limitation, Licensor grants you a non-exclusive, -worldwide (subject to applicable laws) license to copy, modify, display, -use, create derivative works, and redistribute the Software until the -Change Date. If your use of the Software exceeds, or will exceed, the -foregoing limitations you MUST obtain alternative licensing terms for the -Software directly from Licensor, its affiliated entities, or authorized -resellers. For the avoidance of doubt, prior to the Change Date, there is -no Use Limitations for non-production purposes. +For more information on the use of the Business Source License generally, +please visit the Adopting and Developing Business Source License FAQ at +https://mariadb.com/bsl-faq-adopting. -After the Change Date, this Business Source License will convert to the -Change License and your use of the Software, including modified versions -of the Software, will be governed by such Change License. +----------------------------------------------------------------------------- -All copies of original and modified Software, and derivative works of the -Software, are subject to this Business Source License. This Business -Source License applies separately for each version of the Software and the -Change Date will vary for each version of the Software released by -Licensor. +Business Source License 1.1 -You must conspicuously display this Business Source License on each -original or modified copy of the Software. If you receive the Software in -original or modified form from a third party, the restrictions set forth -in this Business Source License apply to your use of such Software. +Terms -Any use of the Software in violation of this Business Source License will -automatically terminate your rights under this Business Source License for -the current and all future versions of the Software. +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. -You may not use the marks or logos of Licensor or its affiliates for -commercial purposes without prior written consent from Licensor. +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. -TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE SOFTWARE AND ALL SERVICES -PROVIDED BY LICENSOR OR ITS AFFILIATES UNDER OR IN CONNECTION WITH WITH -THIS BUSINESS SOURCE LICENSE ARE PROVIDED ON AN “AS IS” AND “AS AVAILABLE” -BASIS. YOU EXPRESSLY WAIVE ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING -(WITHOUT LIMITATION) WARRANTIES OF MERCHANTABILITY, FITNESS FOR A -PARTICULAR PURPOSE, NON-INFRINGEMENT, TITLE, SYSTEM INTEGRATION, AND -ACCURACY OF INFORMATIONAL CONTENT. +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. From 927b4addc87c54f505b64cfc56d320705a013aaf Mon Sep 17 00:00:00 2001 From: MassimilianoPinto Date: Fri, 24 Nov 2017 09:38:17 +0100 Subject: [PATCH 02/11] MXS-1530: check ANNOTATE_ROWS flag in connecting slave MXS-1530: check ANNOTATE_ROWS flag in connecting slave. In MariaDB 10.2.4 replicate_annotate_row_events and binlog_annotate_row_events have default to ON: this change checks whether the connecting slave is not has ANNOTATE_ROWS in blr_slave_binlog_dump(), those ANNOTATE_ROWS events can be sent or not to the slave. --- server/modules/routing/binlogrouter/blr.h | 1 + .../modules/routing/binlogrouter/blr_slave.c | 32 +++++++++++++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/server/modules/routing/binlogrouter/blr.h b/server/modules/routing/binlogrouter/blr.h index 029fcb788..1fdc74667 100644 --- a/server/modules/routing/binlogrouter/blr.h +++ b/server/modules/routing/binlogrouter/blr.h @@ -440,6 +440,7 @@ typedef struct router_slave char lsi_binlog_name[BINLOG_FNAMELEN + 1]; /*< Which binlog file */ uint32_t lsi_binlog_pos; /*< What position */ void *encryption_ctx; /*< Encryption context */ + bool annotate_rows; /*< MariaDB 10 Slave requests ANNOTATE_ROWS */ #if defined(SS_DEBUG) skygw_chk_t rses_chk_tail; #endif diff --git a/server/modules/routing/binlogrouter/blr_slave.c b/server/modules/routing/binlogrouter/blr_slave.c index de009b213..f2b34d80b 100644 --- a/server/modules/routing/binlogrouter/blr_slave.c +++ b/server/modules/routing/binlogrouter/blr_slave.c @@ -2033,7 +2033,21 @@ blr_slave_binlog_dump(ROUTER_INSTANCE *router, ROUTER_SLAVE *slave, GWBUF *queue } slave->binlog_pos = extract_field(ptr, 32); + + /* Go ahead: after 4 bytes pos we have 2 bytes flags */ ptr += 4; + + uint16_t flags = gw_mysql_get_byte2(ptr); + /* Check whether connected slave is asking for ANNOTATE_ROWS events */ + if (flags & BLR_REQUEST_ANNOTATE_ROWS_EVENT) + { + slave->annotate_rows = true; + MXS_INFO("Registering slave (server-id %d) asks " + "for ANNOTATE_ROWS events.", + slave->serverid); + } + + /* Go ahead: after 2 bytes flags and 4 bytes serverid */ ptr += 2; ptr += 4; memcpy(slave->binlogfile, (char *)ptr, binlognamelen); @@ -2381,9 +2395,13 @@ blr_slave_catchup(ROUTER_INSTANCE *router, ROUTER_SLAVE *slave, bool large) strcpy(binlog_name, slave->binlogfile); binlog_pos = slave->binlog_pos; - /* Don't sent special events generated by MaxScale */ + /* Don't sent special events generated by MaxScale + * or ANNOTATE_ROWS events if not requested + */ if (hdr.event_type == MARIADB10_START_ENCRYPTION_EVENT || - hdr.event_type == IGNORABLE_EVENT) + hdr.event_type == IGNORABLE_EVENT || + (!slave->annotate_rows && + hdr.event_type == MARIADB_ANNOTATE_ROWS_EVENT)) { /* In case of file rotation or pos = 4 the events are sent from position 4. * new FDE at pos 4 is read. @@ -2416,6 +2434,16 @@ blr_slave_catchup(ROUTER_INSTANCE *router, ROUTER_SLAVE *slave, bool large) slave->binlogfile, (unsigned long) hdr.next_pos); } + /* MARIADB_ANNOTATE_ROWS_EVENT is skipped: just log that */ + else if (hdr.event_type == MARIADB_ANNOTATE_ROWS_EVENT) + { + MXS_INFO("Skipping ANNOTATE_ROWS event [%s] of size %lu while " + "reading binlog %s at %lu", + blr_get_event_description(router, hdr.event_type), + (unsigned long)hdr.event_size, + slave->binlogfile, + (unsigned long)slave->binlog_pos); + } else { MXS_INFO("Found ignorable event [%s] of size %lu while reading binlog %s at %lu", From 7c21eb3ec431050fbba43e51220cd9ff26f228da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 23 Nov 2017 17:26:44 +0200 Subject: [PATCH 03/11] Do not force router_options usage The usage of the router_options has become optional in 2.1. This means that the binlogrouter should not fail to start if no router_options are defined. Also lowered the error about master.ini to a warning as it is expected to happen on a fresh installation. --- server/modules/routing/binlogrouter/blr.c | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/server/modules/routing/binlogrouter/blr.c b/server/modules/routing/binlogrouter/blr.c index e091f188f..890dea8bb 100644 --- a/server/modules/routing/binlogrouter/blr.c +++ b/server/modules/routing/binlogrouter/blr.c @@ -242,13 +242,6 @@ createInstance(SERVICE *service, char **options) return NULL; } - if (options == NULL || options[0] == NULL) - { - MXS_ERROR("%s: Error: No router options supplied for binlogrouter", - service->name); - return NULL; - } - /* * We only support one server behind this router, since the server is * the master from which we replicate binlog records. Therefore check @@ -623,10 +616,6 @@ createInstance(SERVICE *service, char **options) } } } - else - { - MXS_ERROR("%s: Error: No router options supplied for binlogrouter", service->name); - } if (inst->masterid) { @@ -754,10 +743,10 @@ createInstance(SERVICE *service, char **options) { if (rc == -1) { - MXS_ERROR("%s: master.ini file not found in %s." - " Master registration cannot be started." - " Configure with CHANGE MASTER TO ...", - inst->service->name, inst->binlogdir); + MXS_WARNING("%s: master.ini file not found in %s." + " Master registration cannot be started." + " Configure with CHANGE MASTER TO ...", + inst->service->name, inst->binlogdir); } else { From db559a9d618cb651eabc62b77e382edf51310cfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 23 Nov 2017 17:58:52 +0200 Subject: [PATCH 04/11] Update binlogrouter documentation Updated all examples to use parameters instead of router_options. Added a more clear note about router_options being deprecated. Removed unnecessary parameters from all examples. --- Documentation/Routers/Binlogrouter.md | 206 +++++++++++++------------- 1 file changed, 99 insertions(+), 107 deletions(-) diff --git a/Documentation/Routers/Binlogrouter.md b/Documentation/Routers/Binlogrouter.md index 044f18dbb..ae603f146 100644 --- a/Documentation/Routers/Binlogrouter.md +++ b/Documentation/Routers/Binlogrouter.md @@ -13,34 +13,33 @@ replication setup where replication is high-priority. ## Mandatory Router Parameters -The binlogrouter requires the `server`, `user` and `password` parameters. These -should be configured according to the +The binlogrouter requires the `user` and `password` parameters. These should be +configured according to the [Configuration Guide](../Getting-Started/Configuration-Guide.md#service). -In addition to these two parameters, `router_options` needs to be defined. This -is the main way the binlogrouter is configured and it will be covered in detail -in the next section. +In addition to these two parameters, the `server_id` and `binlogdir` parameters needs to be defined. -**Note:** As of version 2.1 of MaxScale, all of the router options can also be -defined as parameters. The values defined in _router_options_ will have priority -over the parameters. +## Router Parameters -## Router Options +The binlogrouter accepts the following parameters. -Binlogrouter is configured with a comma-separated list of key-value pairs. The -following options should be given as a value to the `router_options` parameter. +**Note:** Earlier versions of MaxScale supported the configuration of the +binlogrouter only via `router_options` (a the comma-separated list of key-value +pairs). As of MaxScale 2.1, all of the router options should be defined as +parameters. The values defined in `router_options` will have priority over the +parameters to support legacy configurations. The use of `router_options` is +deprecated. ### `binlogdir` This parameter controls the location where MariaDB MaxScale stores the binary log -files. If this parameter is not set to a directory name then MariaDB -MaxScale will store the binlog files in the directory -`/var/cache/maxscale/` where `` is the name of the -service in the configuration file. The _binlogdir_ also contains the -_cache_ subdirectory which stores data retrieved from the master during the slave -registration phase. The master.ini file also resides in the _binlogdir_. This -file keeps track of the current master configuration and it is updated when a -`CHANGE MASTER TO` query is executed. +files. This is a mandatory parameter. + +The _binlogdir_ also contains the _cache_ subdirectory which stores data +retrieved from the master during the slave registration phase. The +master.ini file also resides in the _binlogdir_. This file keeps track of +the current master configuration and it is updated when a `CHANGE MASTER +TO` query is executed. From 2.1 onwards, the 'cache' directory is stored in the same location as other user credential caches. This means that with the default options, the user @@ -51,17 +50,11 @@ Read the [MySQL Authenticator](../Authenticators/MySQL-Authenticator.md) documentation for instructions on how to define a custom location for the user cache. -### `uuid` - -This is used to set the unique UUID that the binlog router uses when it connects -to the master server. If no explicit value is given for the UUID in the -configuration file then a UUID will be generated. - ### `server_id` -As with UUID, MariaDB MaxScale must have a unique _server_id_. This parameter -configures the value of the _server_id_ that -MariaDB MaxScale will use when connecting to the master. +MariaDB MaxScale must have a unique _server_id_. This parameter configures +the value of the _server_id_ that MariaDB MaxScale will use when +connecting to the master. This is a mandatory parameter. Older versions of MaxScale allowed the ID to be specified using `server-id`. This has been deprecated and will be removed in a future release of MariaDB MaxScale. @@ -69,40 +62,49 @@ This has been deprecated and will be removed in a future release of MariaDB MaxS ### `master_id` The _server_id_ value that MariaDB MaxScale should use to report to the slaves -that connect to MariaDB MaxScale. This may either be the same as the server id -of the real master or can be chosen to be different if the slaves need to be -aware of the proxy layer. The real master server ID will be used if the option -is not set. +that connect to MariaDB MaxScale. + +This may either be the same as the server id of the real master or can be +chosen to be different if the slaves need to be aware of the proxy +layer. The real master server ID will be used if the option is not set. Older versions of MaxScale allowed the ID to be specified using `master-id`. This has been deprecated and will be removed in a future release of MariaDB MaxScale. +### `uuid` + +This is used to set the unique UUID that the binlog router uses when it connects +to the master server. By default the UUID will be generated. + ### `master_uuid` -It is a requirement of replication that each slave has a unique UUID value. The -MariaDB MaxScale router will identify itself to the slaves using the UUID of the -real master if this option is not set. +It is a requirement of replication that each server has a unique UUID value. If +this option is not set, binlogrouter will identify itself to the slaves using +the UUID of the real master. ### `master_version` By default, the router will identify itself to the slaves using the server -version of the real master. This option allows the router to use a custom version string. +version of the real master. This option allows the router to use a custom +version string. ### `master_hostname` -By default, the router will identify itself to the slaves using the -hostname of the real master. This option allows the router to use a custom hostname. +By default, the router will identify itself to the slaves using the hostname of +the real master. This option allows the router to use a custom hostname. ### `slave_hostname` -Since MaxScale 2.1.6 the router can optionally identify itself -to the master using a custom hostname. -The specified hostname can be seen in the master via -`SHOW SLAVE HOSTS` command. -The default is not to send any hostname string during registration. +Since MaxScale 2.1.6 the router can optionally identify itself to the master +using a custom hostname. The specified hostname can be seen in the master via +`SHOW SLAVE HOSTS` command. The default is not to send any hostname string +during registration. ### `user` +*Note:* This is option can only be given to the `router_options` parameter. Use + the `user` parameter of the service instead. + This is the user name that MariaDB MaxScale uses when it connects to the master. This user name must have the rights required for replication as with any other user that a slave uses for replication purposes. If the user parameter is @@ -118,9 +120,8 @@ authenticator module. Read the [MySQL Authenticator](../Authenticators/MySQL-Authenticator.md) documentation for more details. -The user that is used for replication, either defined using the user= option in -the router options or using the username and password defined of the service -must be granted replication privileges on the database server. +The user that is used for replication must be granted replication privileges on +the database server. ``` CREATE USER 'repl'@'maxscalehost' IDENTIFIED by 'password'; @@ -129,60 +130,69 @@ GRANT REPLICATION SLAVE ON *.* TO 'repl'@'maxscalehost'; ### `password` +*Note:* This is option can only be given to the `router_options` parameter. Use + the `password` parameter of the service instead. + The password for the user. If the password is not explicitly given then the password in the service entry will be used. For compatibility with other username and password definitions within the MariaDB MaxScale configuration file -it is also possible to use the parameter passwd=. +it is also possible to use the parameter `passwd`. ### `heartbeat` This defines the value of the heartbeat interval in seconds for the connection -to the master. MariaDB MaxScale requests the master to ensure that a binlog -event is sent at least every heartbeat period. If there are no real binlog -events to send the master will sent a special heartbeat event. The default value -for the heartbeat period is every 5 minutes. The current interval value is +to the master. The default value for the heartbeat period is every 5 minutes. + +MariaDB MaxScale requests the master to ensure that a binlog event is sent at +least every heartbeat period. If there are no real binlog events to send the +master will sent a special heartbeat event. The current interval value is reported in the diagnostic output. ### `burstsize` This parameter is used to define the maximum amount of data that will be sent to -a slave by MariaDB MaxScale when that slave is lagging behind the master. In -this situation the slave is said to be in "catchup mode", this parameter is +a slave by MariaDB MaxScale when that slave is lagging behind the master. The +default value is `1M`. + +The burst size can be provided as specified +[here](../Getting-Started/Configuration-Guide.md#sizes), except that IEC binary +prefixes can be used as suffixes only from MaxScale 2.1 onwards. MaxScale 2.0 +and earlier only support `burstsize` defined in bytes. + +In this situation the slave is said to be in "catchup mode", this parameter is designed to both prevent flooding of that slave and also to prevent threads within MariaDB MaxScale spending disproportionate amounts of time with slaves -that are lagging behind the master. The burst size can be provided as specified -[here](../Getting-Started/Configuration-Guide.md#sizes), except that IEC -binary prefixes can be used as suffixes only from MaxScale 2.1 onwards. -The default value is `1M`, which will be used if `burstsize` is not provided in -the router options. +that are lagging behind the master. ### `mariadb10-compatibility` This parameter allows binlogrouter to replicate from a MariaDB 10.0 master -server. GTID will not be used in the replication. +server. GTID will not be used in the replication. The default value is _false_. ``` # Example -router_options=mariadb10-compatibility=1 +mariadb10-compatibility=1 ``` ### `transaction_safety` This parameter is used to enable/disable incomplete transactions detection in -binlog router. When MariaDB MaxScale starts an error message may appear if -current binlog file is corrupted or an incomplete transaction is found. During -normal operations binlog events are not distributed to the slaves until a COMMIT -is seen. The default value is off, set transaction_safety=on to enable the -incomplete transactions detection. +binlog router. The default value is _off_. + +When MariaDB MaxScale starts an error message may appear if current binlog file +is corrupted or an incomplete transaction is found. During normal operations +binlog events are not distributed to the slaves until a COMMIT is seen. Set +transaction_safety=on to enable detection of incomplete transactions. ### `send_slave_heartbeat` This defines whether MariaDB MaxScale sends the heartbeat packet to the slave -when there are no real binlog events to send. The default value -is 'off' and no heartbeat events are sent to slave servers. If value is 'on' the -interval value (requested by the slave during registration) is reported in the -diagnostic output and the packet is send after the time interval without any -event to send. +when there are no real binlog events to send. The default value is 'off' and no +heartbeat events are sent to slave servers. + +If value is 'on' the interval value (requested by the slave during registration) +is reported in the diagnostic output and the packet is send after the time +interval without any event to send. ### `semisync` @@ -211,14 +221,14 @@ Master communication. This parameter sets the maximum length of the certificate authority chain that will be accepted. Legal values are positive integers. This applies to SSL connection to master server that could be acivated either by writing options in -master.ini or later via CHANGE MASTER TO. This parameter cannot be modified at -runtime, default is 9. +master.ini or later via a _CHANGE MASTER TO_ command. This parameter cannot be +modified at runtime. The default verification depth is 9. ### `encrypt_binlog` -Whether to encrypt binlog files: the default is Off. +Whether to encrypt binlog files: the default is _off_. -When set to On the binlog files will be encrypted using specified AES algorithm +When set to _on_ the binlog files will be encrypted using specified AES algorithm and the KEY in the specified key file. **Note:** binlog encryption must be used while replicating from a MariaDB 10.1 @@ -252,11 +262,11 @@ The KEY must have exact 16, 24 or 32 bytes size and the selected algorithm (aes_ctr or aes_cbc) with 128, 192 or 256 ciphers will be used. **Note:** the key file has the same format as MariaDB 10.1 server so it's -possible to use an existing key file (not ecncrypted) which could contain -several scheme;keys: only key id with value 1 will be parsed, and if not found +possible to use an existing key file (not encrypted) which could contain several +`scheme;key` values: only key id with value 1 will be parsed, and if not found an error will be reported. -Example: +Example key file with multiple keys: ``` # @@ -272,38 +282,20 @@ Example: A complete example of a service entry for a binlog router service would be as follows. ``` - [Replication] - type=service - router=binlogrouter - servers=masterdb - version_string=5.6.17-log - user=maxscale - passwd=Mhu87p2D - router_options=uuid=f12fcb7f-b97b-11e3-bc5e-0401152c4c22, - server_id=3, - user=repl, - password=slavepass, - master_id=32, - heartbeat=30, - binlogdir=/var/binlogs, - transaction_safety=1, - master_version=5.6.19-common, - master_hostname=common_server, - master_uuid=xxx-fff-cccc-common, - mariadb10-compatibility=1, - send_slave_heartbeat=1, - ssl_cert_verification_depth=9, - semisync=1, - encrypt_binlog=1, - encryption_algorithm=aes_ctr, - encryption_key_file=/var/binlogs/enc_key.txt, - slave_hostname=maxscale-blr-1 +[Replication] +type=service +router=binlogrouter +servers=masterdb +user=maxscale +passwd=maxpwd +server_id=3 +binlogdir=/var/lib/maxscale/ +mariadb10-compatibility=1 +encrypt_binlog=1 +encryption_algorithm=aes_ctr +encryption_key_file=/var/binlogs/enc_key.txt ``` -The minimum set of router options that must be given in the configuration are -`server_id` and `master_id` (unless the real master id should be used); default -values may be used for all other options. - ## Examples The [Replication Proxy](../Tutorials/Replication-Proxy-Binlog-Router-Tutorial.md) tutorial will From 098e9311c195093c16f57a052c115ff94f448357 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20M=C3=A4kel=C3=A4?= Date: Thu, 23 Nov 2017 18:45:01 +0200 Subject: [PATCH 05/11] Update binlogrouter tutorial Break lines at 80 characters. Add a Table of Contents and reorder sections to have the actual tutorial part first. Things that still need doing: * Move limitations and other documentation to the actual router documentation * Review and simplify examples --- ...eplication-Proxy-Binlog-Router-Tutorial.md | 835 ++++++++++-------- 1 file changed, 461 insertions(+), 374 deletions(-) diff --git a/Documentation/Tutorials/Replication-Proxy-Binlog-Router-Tutorial.md b/Documentation/Tutorials/Replication-Proxy-Binlog-Router-Tutorial.md index 9fd30cf7d..f89c716b1 100644 --- a/Documentation/Tutorials/Replication-Proxy-Binlog-Router-Tutorial.md +++ b/Documentation/Tutorials/Replication-Proxy-Binlog-Router-Tutorial.md @@ -1,191 +1,503 @@ # MariaDB MaxScale as a Binlog Server -MariaDB MaxScale is a dynamic data routing platform that sits between a database layer and the clients of that database. However, the binlog router described here is somewhat different to that original concept, moving MariaDB MaxScale down to play a role within the database layer itself. -In a traditional MariaDB replication setup a single master server is created and a set of slaves MariaDB instances are configured to pull the binlog files from that master to the slaves. There are some problems, however, in this setup; when the number of slaves grows, an increasing load caused by the serving of binlogs to each slave, is placed on the master. When the master server fails, some action must be performed on every slave server before a new server can become the master server. +# Table of Contents -Introducing a proxy layer between the master server and the slave servers can improve the situation, by reducing the load on the master to simply serving the proxy layer rather than all of the slaves. The slaves only need to be aware of the proxy layer and not of the real master server. Removing the need for the slaves to have knowledge of the actual master, greatly simplifies the process of replacing a failed master within a replication environment. +* [Introduction](#introduction) + * [MariaDB as a Binlog Server](#mariadb-as-a-binlog-server) + * [MariaDB MaxScale's approach](#mariadb-maxscales-approach) +* [Configuring MariaDB MaxScale as a Binlog Server](#configuring-mariadb-maxscale-as-a-binlog-server) + * [Service Configuration](#service-configuration) + * [Listener Configuration](#listener-configuration) + * [Configuring Replication](#configuring-replication) +* [Binlog Router Compatibility](#binlog-router-compatibility) +* [MariaDB MaxScale Replication Diagnostics](#mariadb-maxscale-replication-diagnostics) + +# Introduction + +MariaDB MaxScale is a dynamic data routing platform that sits between a database +layer and the clients of that database. However, the binlog router described +here is somewhat different to that original concept, moving MariaDB MaxScale +down to play a role within the database layer itself. + +In a traditional MariaDB replication setup a single master server is created and +a set of slaves MariaDB instances are configured to pull the binlog files from +that master to the slaves. There are some problems, however, in this setup; when +the number of slaves grows, an increasing load caused by the serving of binlogs +to each slave, is placed on the master. When the master server fails, some +action must be performed on every slave server before a new server can become +the master server. + +Introducing a proxy layer between the master server and the slave servers can +improve the situation, by reducing the load on the master to simply serving the +proxy layer rather than all of the slaves. The slaves only need to be aware of +the proxy layer and not of the real master server. Removing the need for the +slaves to have knowledge of the actual master, greatly simplifies the process of +replacing a failed master within a replication environment. ## MariaDB as a Binlog Server -The most obvious solution to the requirement for a proxy layer within a replication environment is to use a MariaDB or MySQL database instance. The database server is designed to allow this, since a slave server is able to be configured such that it will produce binary logs for updates it has itself received via replication from the master server. This is done with the *log_slave_updates* configuration option of the server. In this case the server is known as an intermediate master, it is simultaneously a slave to the real master and a master to the other slaves in the configuration. -Using an intermediate master does not, however, solve all the problems and introduces some new ones, due to the way replication is implemented. A slave server reads the binary log data and creates a relay log from that binary log. This log provides a source of SQL statements, which are executed within the slave in order to make the same changes to the databases on the slaves as were made on the master. If the *log_slave_updates* option has been enabled, new binary log entries are created for the statements executed from the relay log. +The most obvious solution to the requirement for a proxy layer within a +replication environment is to use a MariaDB or MySQL database instance. The +database server is designed to allow this, since a slave server is able to be +configured such that it will produce binary logs for updates it has itself +received via replication from the master server. This is done with the +*log_slave_updates* configuration option of the server. In this case the server +is known as an intermediate master, it is simultaneously a slave to the real +master and a master to the other slaves in the configuration. -The above means that the data in the binary log of the intermediate master is not a direct copy of the data that was received from the binary log of the real master. The resultant changes to the database will be the same, provided no updates have been performed on the intermediate master that did not originate on the real master, but the steps to achieve those changes may be different. In particular, if group commit functionality is used, to allow multiple transactions to commit in parallel, these may well be different on the intermediate master. This can cause a reduction in the parallelism of the commits and a subsequent reduction in the performance of the slave servers. +Using an intermediate master does not, however, solve all the problems and +introduces some new ones, due to the way replication is implemented. A slave +server reads the binary log data and creates a relay log from that binary +log. This log provides a source of SQL statements, which are executed within the +slave in order to make the same changes to the databases on the slaves as were +made on the master. If the *log_slave_updates* option has been enabled, new +binary log entries are created for the statements executed from the relay log. -This re-execution of the SQL statements also adds latency to the intermediate master solution, since the full process of parsing, optimization and execution must occur for every statement that is replicated from the master to the slaves must be performed in the intermediate master. This latency introduces lag in the replication chain, with a greater delay being introduced from the time a transaction is committed on the master until the data is available on the slaves. +The above means that the data in the binary log of the intermediate master is +not a direct copy of the data that was received from the binary log of the real +master. The resultant changes to the database will be the same, provided no +updates have been performed on the intermediate master that did not originate on +the real master, but the steps to achieve those changes may be different. In +particular, if group commit functionality is used, to allow multiple +transactions to commit in parallel, these may well be different on the +intermediate master. This can cause a reduction in the parallelism of the +commits and a subsequent reduction in the performance of the slave servers. -Use of an intermediate master does improve the process of failover of the master server, since the slaves are only aware of the intermediate master the process of promoting one of the existing slaves to become the new master only involves that slave and the intermediate master. A slave can become the new master as soon as all the changes from the intermediate master have been processed. The intermediate master then needs to be reset to the correct point in the binary log of the new master and replication can continue. +This re-execution of the SQL statements also adds latency to the intermediate +master solution, since the full process of parsing, optimization and execution +must occur for every statement that is replicated from the master to the slaves +must be performed in the intermediate master. This latency introduces lag in the +replication chain, with a greater delay being introduced from the time a +transaction is committed on the master until the data is available on the +slaves. -An added complexity that needs to be dealt with is the failure of the intermediate master itself. If this occurs then the same problem as described earlier exists, all slaves must be updated when a new intermediate master is created. If multiple intermediate masters are used, there is also a restriction that slaves can not be moved from the failed intermediate master to another intermediate master due to the fact that the binlog on the different intermediate nodes are not guaranteed to be the same. +Use of an intermediate master does improve the process of failover of the master +server, since the slaves are only aware of the intermediate master the process +of promoting one of the existing slaves to become the new master only involves +that slave and the intermediate master. A slave can become the new master as +soon as all the changes from the intermediate master have been processed. The +intermediate master then needs to be reset to the correct point in the binary +log of the new master and replication can continue. + +An added complexity that needs to be dealt with is the failure of the +intermediate master itself. If this occurs then the same problem as described +earlier exists, all slaves must be updated when a new intermediate master is +created. If multiple intermediate masters are used, there is also a restriction +that slaves can not be moved from the failed intermediate master to another +intermediate master due to the fact that the binlog on the different +intermediate nodes are not guaranteed to be the same. ## MariaDB MaxScale's approach -MariaDB MaxScale takes a much simpler approach to the process of being a Binlog Server. It acts as a slave to the real master and as a master to the slaves, in the same way as an intermediate master does. However, it does not implement any re-execution of the statements within the binary log. MariaDB MaxScale creates a local cache of the binary logs it receives from the master and will serve binary log events to the slaves from this cache of the master's binary log. This means that the slaves will always get binary log events that have a one-to-one correlation to those written by the master. Parallelism in the binary log events of the master is maintained in the events that are observed by the slaves. -In the MariaDB MaxScale approach, the latency that is introduced is mostly the added network latency associated with adding the extra network hop. There is no appreciable processing performed at the MariaDB MaxScale level, other than for managing the local cache of the binlog files. +MariaDB MaxScale takes a much simpler approach to the process of being a Binlog +Server. It acts as a slave to the real master and as a master to the slaves, in +the same way as an intermediate master does. However, it does not implement any +re-execution of the statements within the binary log. MariaDB MaxScale creates a +local cache of the binary logs it receives from the master and will serve binary +log events to the slaves from this cache of the master's binary log. This means +that the slaves will always get binary log events that have a one-to-one +correlation to those written by the master. Parallelism in the binary log events +of the master is maintained in the events that are observed by the slaves. -In addition, every MariaDB MaxScale that is acting as a proxy of the master will have exactly the same binlog events as the master itself. This means that a slave can be moved between any of the MariaDB MaxScale server or to the real master without a need to perform any special processing. The result is much simpler behavior for failure recovery and the ability to have a very simple, redundant proxy layer with slaves free to both between the proxies. +In the MariaDB MaxScale approach, the latency that is introduced is mostly the +added network latency associated with adding the extra network hop. There is no +appreciable processing performed at the MariaDB MaxScale level, other than for +managing the local cache of the binlog files. + +In addition, every MariaDB MaxScale that is acting as a proxy of the master will +have exactly the same binlog events as the master itself. This means that a +slave can be moved between any of the MariaDB MaxScale server or to the real +master without a need to perform any special processing. The result is much +simpler behavior for failure recovery and the ability to have a very simple, +redundant proxy layer with slaves free to both between the proxies. # Configuring MariaDB MaxScale as a Binlog Server -Using MariaDB MaxScale as a Binlog Server is much the same as using MariaDB MaxScale as a proxy between the clients and the database servers. In this case the master server should be considered as the database backend and the slave servers as the clients of MariaDB MaxScale. + +Using MariaDB MaxScale as a Binlog Server is much the same as using MariaDB +MaxScale as a proxy between the clients and the database servers. In this case +the master server should be considered as the database backend and the slave +servers as the clients of MariaDB MaxScale. ## Service Configuration -As with any MariaDB MaxScale configuration a good starting point is with the service definition with the *maxscale.cnf* file. The service requires a name which is the section name in the ini file, a type parameter with a value of service and the name of the router plugin that should be loaded. In the case of replication proxies this router name is *binlogrouter*. +As with any MariaDB MaxScale configuration a good starting point is with the +service definition with the *maxscale.cnf* file. The service requires a name +which is the section name in the ini file, a type parameter with a value of +service and the name of the router plugin that should be loaded. In the case of +replication proxies this router name is *binlogrouter*. +The minimum set of router options that must be given in the configuration are +are `server_id` and `binlogdir`, default values may be used for all other +options. -The minimum set of router options that must be given in the configuration are are *server-id* and *master-id*, default values may be used for all other options. +All configuration prameters can be found in the [Binlog Router +Documentation](../Routers/Binlogrouter.md). -All configuration prameters can be found in the [Binlog Router Documentation](../Routers/Binlogrouter.md). - -A complete example of a service entry for a binlog router service would be as follows. +A minimal example of a service entry for a binlog router service that is used +with MariaDB 10 would be as follows. ``` - [Replication] - type=service - router=binlogrouter - version_string=5.6.17-log - user=maxscale - passwd=Mhu87p2D - router_options=uuid=f12fcb7f-b97b-11e3-bc5e-0401152c4c22,server-id=3, - user=repl, - password=slavepass, - master-id=1, - heartbeat=30, - binlogdir=/var/binlogs, - transaction_safety=1, - master_version=5.6.19-common, - master_hostname=common_server, - master_uuid=xxx-fff-cccc-common, - master-id=999, - mariadb10-compatibility=1, - ssl_cert_verification_depth=9, - semisync=1, - encrypt_binlog=1, - encryption_algorithm=aes_ctr, - encryption_key_file=/var/binlogs/enc_key.txt, - slave_hostname=maxscale-blr-1 +[Replication] +type=service +router=binlogrouter +user=maxscale +passwd=maxpwd +server_id=1 +mariadb10-compatibility=1 +binlogdir=/var/lib/maxscale/ ``` -## Listener Section +## Listener Configuration -As per any service in MariaDB MaxScale a listener section is required to define the address, port and protocol that is used to listen for incoming connections. Those incoming connections will originate from the slave servers or from a MySQL client in order to administer/configure the master server configuration via SQL commands such as *STOP/START SLAVE* and *CHANGE MASTER TO* ... +As per any service in MariaDB MaxScale, a listener section is required to define +the address, port and protocol that is used to listen for incoming +connections. Those incoming connections will originate from either slave servers +or from a MySQL client. The binlogrouter is administered and configured via SQL +commands on the listener. - [Replication Listener] - type=listener - service=Replication - protocol=MySQLClient - port=5308 - -The protocol used by slaves for connection to MariaDB MaxScale is the same *MySQLClient* protocol that is used for client applications to connect to databases, therefore the same MariaDB MaxScale protocol module can be used. - -It's also possible to enable SSL from clients (MySQL clients or Slave servers) by adding SSL options in the listener, or in a new one: ``` - [Replication Listener_SSL] - type=listener - service=Replication - protocol=MySQLClient - port=5309 - ssl=required - ssl_key=/path_to/key.pem - ssl_cert=/path_to/cert.pem - ssl_ca_cert=/path_to/ca-cert.pem - #ssl_version=TLSv10 +[Replication Listener] +type=listener +service=Replication +protocol=MySQLClient +port=3306 ``` -Check the [Configuration-Guide](../Getting-Started/Configuration-Guide.md) for SSL options details. -# MariaDB MaxScale replication diagnostics +The protocol used by slaves for connection to MariaDB MaxScale is the same +*MySQLClient* protocol that is used for client applications to connect to +databases, therefore the same MariaDB MaxScale protocol module can be used. + +It's also possible to enable client side SSL by adding the required SSL options +in the listener: + +``` +[Replication SSL Listener] +type=listener +service=Replication +protocol=MySQLClient +port=3306 +ssl=required +ssl_key=/path/to/key.pem +ssl_cert=/path/to/cert.pem +ssl_ca_cert=/path/to/ca-cert.pem +``` + +Refer to the [Configuration-Guide](../Getting-Started/Configuration-Guide.md) +for more details about the SSL configuration in MaxScale. + +## Configuring Replication + +When the binlogrouter is started for the first time, it needs to be configured +to replicate from a master. To do this, connect to the binlogrouter listener +that was defined before and execute a normal `CHANGE MASTER TO` command. Use the +credentials defined in `maxscale.cnf` when you connect to MaxScale. Finally, +execute a `START SLAVE` command to start the replication. + +Here is an example SQL command that configures the binlogrouter to replicate +from a MariaDB server and starts replication: + +``` +CHANGE MASTER TO MASTER_HOST='master.example.com', + MASTER_PORT=3306, + MASTER_USER='maxuser', + MASTER_PASSWORD='maxpwd', + MASTER_LOG_FILE='mysql-bin.000001', + MASTER_LOG_POS=4; + +START SLAVE; +``` + +Both the _MASTER_LOG_FILE_ and _MASTER_LOG_POS_ must be defined and the value of +_MASTER_LOG_POS_ must be 4. + +**Note:** Legacy versions defined the server by configuring a separate server +object in `maxscale.cnf`. + +### Stopping and Starting the Replication + +When router is configured and it is properly working it is possible to stop the +replication with `STOP SLAVE` and to resume it with `START SLAVE`. In addition +to this, the `SHOW SLAVE STATUS` command can be used to display information +about the replication configuration. + +Slave connections are not affected by the `STOP SLAVE` and `START SLAVE` +commands. They only control the connection to the master server. + +### Change the Master server configuration + +When router is configured and it is properly working it is possible to change the master parameters. +First step is stop the replication from the master. + +``` +STOP SLAVE; +``` + +Next step is the master configuration + +``` +CHANGE MASTER TO ... +``` + +A successful configuration change results in *master.ini* being updated. Any +error is reported in the MySQL and in log files. + +The supported `CHAGE MASTER TO` options are: + +- `MASTER_HOST` +- `MASTER_PORT` +- `MASTER_USER` +- `MASTER_PASSWORD` +- `MASTER_LOG_FILE` +- `MASTER_LOG_POS` +- `MASTER_SSL` +- `MASTER_SSL_CERT` (path to certificate file) +- `MASTER_SSL_KEY` (path to key file) +- `MASTER_SSL_CA` (path to CA cerificate file) +- `MASTER_TLS_VERSION` (TLS/SSL version) + +Further details about level of encryption or certificates could be found in the +[Configuration Guide](../Getting-Started/Configuration-Guide.md) + +### Slave servers setup + +Examples of *CHANGE MASTER TO* command issued on a slave server that wants to +gets replication events from MariaDB MaxScale binlog router: + +``` +CHANGE MASTER TO MASTER_HOST=‘$maxscale_IP’, MASTER_PORT=5308, MASTER_USER='repl', MASTER_PASSWORD=‘somepasswd’, +MASTER_LOG_FILE=‘mysql-bin.000001' + +CHANGE MASTER TO MASTER_HOST=‘$maxscale_IP’, MASTER_PORT=5308, MASTER_USER='repl', MASTER_PASSWORD=‘somepasswd’, +MASTER_LOG_FILE=‘mysql-bin.000159', MASTER_LOG_POS=245 +``` + +The latter example specifies a *MASTER_LOG_POS* for the selected +*MASTER_LOG_FILE* + +**Note:** + + - *MASTER_LOG_FILE* must be set to one of existing binlog files in MariaDB + MaxScale binlogdir + + - If *MASTER_LOG_POS* is not set with *CHANGE MASTER TO* it defaults to 4 + + - Latest binlog file name and pos in MariaDB MaxScale can be found by executing + `SHOW MASTER STATUS` on MaxScale. + +### Controlling the Binlogrouter + +There are some constraints related to *MASTER_LOG_FILE* and *MASTER_LOG_POS*. +*MASTER_LOG_FILE* can be changed to next binlog in sequence with +*MASTER_LOG_POS=4* or to current one at current position. + +Examples: + +1) Current binlog file is ‘mysql-bin.000003', position 88888 + + MariaDB> CHANGE MASTER TO MASTER_LOG_FILE=‘mysql-bin.000003',MASTER_LOG_POS=8888 + +This could be applied to current master_host/port or a new one. If there is a +master server maintenance and a slave is being promoted as master it should be +checked that binlog file and position are valid: in case of any error +replication stops and errors are reported via *SHOW SLAVE STATUS* and in error +logs. + +2) Current binlog file is ‘mysql-bin.000099', position 1234 + + MariaDB> CHANGE MASTER TO MASTER_LOG_FILE=‘mysql-bin.000100',MASTER_LOG_POS=4 + +This could be applied with current master_host/port or a new one If transaction +safety option is on and the current binlog file contains an incomplete +transaction it will be truncated to the position where transaction started. In +such situation a proper message is reported in MySQL connection and with next +START SLAVE binlog file truncation will occur and MariaDB MaxScale will request +events from the master using the next binlog file at position 4. + +The above scenario might refer to a master crash/failure: the new server that +has just been promoted as master doesn't have last transaction events but it +should have the new binlog file (the next in sequence). Truncating the previous +MariaDB MaxScale binlog is safe as that incomplete transaction is lost. It +should be checked that current master or new one has the new binlog file, in +case of any error replication stops and errors are reported via *SHOW SLAVE +STATUS* and in error logs. + + MariaDB> START SLAVE; + +Check for any error in log files with: + + MariaDB> SHOW SLAVE STATUS; + +In some situations replication state could be *STOPPED* and proper messages are +displayed in error logs and in *SHOW SLAVE STATUS*. In order to resolve any +mistake done with *CHANGE MASTER TO MASTER_LOG_FILE / MASTER_LOG_POS*, another +administrative command can be helpful. + + MariaDB> RESET SLAVE; + +This command removes *master.ini* file, blanks all master configuration in +memory and sets binlog router in unconfigured state: a *CHANGE MASTER TO* +command should be issued for the new configuration. + +**Note:** existing binlog files are not touched by this command. + +Examples with SSL options: + + MySQL [(none)]> CHANGE MASTER TO MASTER_SSL = 1, MASTER_SSL_CERT='/home/maxscale/packages/certificates/client/client-cert.pem', MASTER_SSL_CA='/home/maxscale/packages/certificates/client/ca.pem', MASTER_SSL_KEY='/home/maxscale/packages/certificates/client/client-key.pem', MASTER_TLS_VERSION='TLSv12'; + + MySQL [(none)]> CHANGE MASTER TO MASTER_TLS_VERSION='TLSv12'; + + MySQL [(none)]> CHANGE MASTER TO MASTER_SSL = 0; + + +#### SSL Limitations + + - In order to enable/re-enable Master SSL comunication the MASTER_SSL=1 option + is required and all certificate options must be explicitey set in the same + CHANGE MASTER TO command. + + - New certificate options changes take effect after maxScale restart or after + MASTER_SSL=1 with the new options. + + - SHOW SLAVE STATUS displays all the options but MASTER_TLS_VERSION value. + + - Maxadmin, 'show services' or 'show service $binlog_service' displays all the + options when SSL is on. + + - STOP SLAVE is required for CHANGE MASTER TO command (any option) + + - START SLAVE will use new SSL options for Master SSL communication setup. + +# Binlog Router Compatibility + +Binlog Router Plugin is compatible with MariaDB 5.5, 10.0, 10.1 and 10.2 as well +as MySQL 5.6 and 5.7. + +## Enabling MariaDB 10 compatibility + +MariaDB 10 has different slave registration phase so an extra option is required: + +``` +mariadb10-compatibility=1 +``` + +`version_string` can be modified in order to present MariaDB 10 version when +MariaDB MaxScale sends server handshake packet. + +``` +version_string=10.0.17-log +``` + +## MySQL Limitations + +In order to use it with MySQL 5.6/5.7, the *GTID_MODE* setting must be OFF and +connecting slaves must not use *MASTER_AUTO_POSITION = 1* option. Additionally +with MySQL 5.7 slaves the `send_slave_heartbeat` option must be set to on. + +Binlog Router currently does not work for MySQL 5.5 due to missing +*@@global.binlog_checksum* variable. + +# MariaDB MaxScale Replication Diagnostics + +The binlog router module of MariaDB MaxScale produces diagnostic output that can +be viewed via the `maxadmin` client application. Running the maxadmin command +and issuing a show service command will produce output that will show both the +master connection status and statistics and also a block for each of the slaves +currently connected. -The binlog router module of MariaDB MaxScale produces diagnostic output that can be viewed via the `maxadmin` client application. Running the maxadmin command and issuing a show service command will produce a considerable amount of output that will show both the master connection status and statistics and also a block for each of the slaves currently connected. ``` -bash-4.1$ maxadmin show service Replication Service 0x1567ef0 - Service: Replication - Router: binlogrouter (0x7f4ceb96a820) - State: Started - Master connection DCB: 0x15693c0 - Master connection state: Binlog Dump - Binlog directory: /var/maxscale/binlogs - Heartbeat period (seconds): 200 - Number of master connects: 1 - Number of delayed reconnects: 0 - Current binlog file: mybin.000061 - Current binlog position: 120 - Number of slave servers: 0 - No. of binlog events received this session: 1002705 - Total no. of binlog events received: 2005410 - No. of bad CRC received from master: 0 - Number of binlog events per minute - Current 5 10 15 30 Min Avg - 4 4.0 4.0 4.0 4.0 - Number of fake binlog events: 0 - Number of artificial binlog events: 61 - Number of binlog events in error: 0 - Number of binlog rotate events: 60 - Number of heartbeat events: 69 - Number of packets received: 599 - Number of residual data packets: 379 - Average events per packet 3347.9 - Last event from master at: Thu Jan 29 16:41:53 2015 (10 seconds ago) - Last event from master: 0x1b (Heartbeat Event) - Events received: - Invalid 0 - Start Event V3 0 - Query Event 703307 - Stop Event 55 - Rotate Event 65 - Integer Session Variable 0 - Load Event 0 - Slave Event 0 - Create File Event 0 - Append Block Event 0 - Exec Load Event 0 - Delete File Event 0 - New Load Event 0 - Rand Event 0 - User Variable Event 0 - Format Description Event 61 - Transaction ID Event (2 Phase Commit) 299148 - Begin Load Query Event 0 - Execute Load Query Event 0 - Table Map Event 0 - Write Rows Event (v0) 0 - Update Rows Event (v0) 0 - Delete Rows Event (v0) 0 - Write Rows Event (v1) 0 - Update Rows Event (v1) 0 - Delete Rows Event (v1) 0 - Incident Event 0 - Heartbeat Event 69 - Ignorable Event 0 - Rows Query Event 0 - Write Rows Event (v2) 0 - Update Rows Event (v2) 0 - Delete Rows Event (v2) 0 - GTID Event 0 - Anonymous GTID Event 0 - Previous GTIDS Event 0 - Started: Thu Jan 29 16:06:11 2015 - Root user access: Disabled - Backend databases - 178.62.50.70:3306 Protocol: MySQLBackend - Users data: 0x156c030 - Total connections: 2 - Currently connected: 2 + Service: Replication + Router: binlogrouter (0x7f4ceb96a820) + State: Started + Master connection DCB: 0x15693c0 + Master connection state: Binlog Dump + Binlog directory: /var/maxscale/binlogs + Heartbeat period (seconds): 200 + Number of master connects: 1 + Number of delayed reconnects: 0 + Current binlog file: mybin.000061 + Current binlog position: 120 + Number of slave servers: 0 + No. of binlog events received this session: 1002705 + Total no. of binlog events received: 2005410 + No. of bad CRC received from master: 0 + Number of binlog events per minute + Current 5 10 15 30 Min Avg + 4 4.0 4.0 4.0 4.0 + Number of fake binlog events: 0 + Number of artificial binlog events: 61 + Number of binlog events in error: 0 + Number of binlog rotate events: 60 + Number of heartbeat events: 69 + Number of packets received: 599 + Number of residual data packets: 379 + Average events per packet 3347.9 + Last event from master at: Thu Jan 29 16:41:53 2015 (10 seconds ago) + Last event from master: 0x1b (Heartbeat Event) + Events received: + Invalid 0 + Start Event V3 0 + Query Event 703307 + Stop Event 55 + Rotate Event 65 + Integer Session Variable 0 + Load Event 0 + Slave Event 0 + Create File Event 0 + Append Block Event 0 + Exec Load Event 0 + Delete File Event 0 + New Load Event 0 + Rand Event 0 + User Variable Event 0 + Format Description Event 61 + Transaction ID Event (2 Phase Commit) 299148 + Begin Load Query Event 0 + Execute Load Query Event 0 + Table Map Event 0 + Write Rows Event (v0) 0 + Update Rows Event (v0) 0 + Delete Rows Event (v0) 0 + Write Rows Event (v1) 0 + Update Rows Event (v1) 0 + Delete Rows Event (v1) 0 + Incident Event 0 + Heartbeat Event 69 + Ignorable Event 0 + Rows Query Event 0 + Write Rows Event (v2) 0 + Update Rows Event (v2) 0 + Delete Rows Event (v2) 0 + GTID Event 0 + Anonymous GTID Event 0 + Previous GTIDS Event 0 + Started: Thu Jan 29 16:06:11 2015 + Root user access: Disabled + Backend databases + 178.62.50.70:3306 Protocol: MySQLBackend + Users data: 0x156c030 + Total connections: 2 + Currently connected: 2 ``` -If a slave is connected to MaxScale with SSL, an entry will be present in the Slave report: + +If a slave is connected to MaxScale with SSL, an entry will be present in the +Slave report: + ``` - Slaves: - Server-id: 106 - Hostname: SBslave6 - Slave UUID: 00019686-7777-7777-7777-777777777777 - Slave_host_port: 188.165.213.5:40365 - Username: massi - Slave DCB: 0x7fc01be3ba88 - Slave connected with SSL: Established + Slaves: + Server-id: 106 + Hostname: SBslave6 + Slave UUID: 00019686-7777-7777-7777-777777777777 + Slave_host_port: 188.165.213.5:40365 + Username: massi + Slave DCB: 0x7fc01be3ba88 + Slave connected with SSL: Established ``` -Another Binlog Server diagnostic output comes from SHOW SLAVE STATUS MySQL command +The `SHOW SLAVE STATUS` command provides diagnostic information about the +replication state. ``` MySQL [(none)]> show slave status\G @@ -234,232 +546,7 @@ Master_SSL_Verify_Server_Cert: No Master_Info_File: /home/maxscale/binlog/first/binlogs/master.ini ``` -# Binlog router compatibility - -Binlog Router Plugin is compatible with MariaDB 5.5 and MySQL 5.6/5.7. - -In order to use it with MySQL 5.6/5.7, the *GTID_MODE* setting must be OFF -and connecting slaves must not use *MASTER_AUTO_POSITION = 1* option. -Additionally with MySQL 5.7 slaves the `send_slave_heartbeat` option must be set to on. - -It’s also works with a MariaDB 10.X setup (master and slaves). -Slave connection must not include any GTID feature if MaxScale version is less than 2.2. - -Binlog Router currently does not work for MySQL 5.5 due to missing *@@global.binlog_checksum* variable. - -The default compatibility is MariaDB 10 since MaxScale 2.2. - -# Master server setup/change - -In the MariaDB MaxScale ini file the server section for master is no longer required, same for *servers=master_server* in the service section. The master server setup is currently managed via *CHANGE MASTER TO* command issued in MySQL client connection to MariaDB MaxScale or by providing a proper *master.ini* file in the *binlogdir*. - -If MariaDB MaxScale starts without *master.ini* there is no replication configured to any master and slaves cannot register to the router: the binlog router could be later configured via *CHANGE MASTER TO* and the *master.ini* file will be written. - -Please note that is such condition the only user for MySQL protocol connection to MaxScale Binlog Server is the service user. - -*master.ini* file example: - - [binlog_configuration] - master_host=127.0.0.1 - master_port=3308 - master_user=repl - master_password=somepass - filestem=repl-bin - # Master SSL communication options - master_ssl=0 - master_ssl_key=/home/mpinto/packages/certificates/client/client-key.pem - master_ssl_cert=/home/mpinto/packages/certificates/client/client-cert.pem - master_ssl_ca=/home/mpinto/packages/certificates/client/ca.pem - #master_tls_version=TLSv12 - -Enabling replication from a master server requires: - - CHANGE MASTER TO MASTER_HOST=‘$master_server’, MASTER_PORT=$master_port, MASTER_USER='repl', MASTER_PASSWORD=‘somepasswd’, MASTER_LOG_FILE=‘repl-bin.000159', MASTER_LOG_POS=4 - -It's possible to specify the desired *MASTER_LOG_FILE* but position must be 4 - -The initfile option is no longer available, filestem option too it's no longer available as the stem is automatically set by parsing *MASTER_LOG_FILE*. - -### Stop/start the replication - -When router is configured and it is properly working it is possible to stop/start replication: - - MariaDB> STOP SLAVE; - ... - MariaDB> SHOW SLAVE STATUS; - ... - MariaDB> START SLAVE; - -Connected or new slave connections are not affected: this *STOP/START* only controls the the connection to the master and the binlog events receiving. - -### Change the Master server configuration - -When router is configured and it is properly working it is possible to change the master parameters. -First step is stop the replication from the master. - - MariaDB> STOP SLAVE; - -Next step is the master configuration - - MariaDB> CHANGE MASTER TO ... - -A successful configuration change results in *master.ini* being updated. - -Any error is reported in the MySQL and in log files - -The supported options are: - - MASTER_HOST - MASTER_PORT - MASTER_USER - MASTER_PASSWORD - MASTER_LOG_FILE - MASTER_LOG_POS - - and SSL options as well: - MASTER_SSL (0|1) - MASTER_SSL_CERT (path to certificate file) - MASTER_SSL_KEY (path to key file) - MASTER_SSL_CA (path to CA cerificate file) - MASTER_TLS_VERSION (allowed level of encryption used) - -Further details about level of encryption or certificates could be found here [Configuration Guuide](../Getting-Started/Configuration-Guide.md) - -There are some constraints related to *MASTER_LOG_FILE* and *MASTER_LOG_POS*: - -*MASTER_LOG_FILE* could be changed to next binlog in sequence with *MASTER_LOG_POS=4* or to current one at current position. - -Examples: - -1) Current binlog file is ‘mysql-bin.000003', position 88888 - - MariaDB> CHANGE MASTER TO MASTER_LOG_FILE=‘mysql-bin.000003',MASTER_LOG_POS=8888 - -This could be applied to current master_host/port or a new one. -If there is a master server maintenance and a slave is being promoted as master it should be checked that binlog file and position are valid: in case of any error replication stops and errors are reported via *SHOW SLAVE STATUS* and in error logs. - -2) Current binlog file is ‘mysql-bin.000099', position 1234 - - MariaDB> CHANGE MASTER TO MASTER_LOG_FILE=‘mysql-bin.000100',MASTER_LOG_POS=4 - -This could be applied with current master_host/port or a new one -If transaction safety option is on and the current binlog file contains an incomplete transaction it will be truncated to the position where transaction started. -In such situation a proper message is reported in MySQL connection and with next START SLAVE binlog file truncation will occur and MariaDB MaxScale will request events from the master using the next binlog file at position 4. - -The above scenario might refer to a master crash/failure: - the new server that has just been promoted as master doesn't have last transaction events but it should have the new binlog file (the next in sequence). -Truncating the previous MariaDB MaxScale binlog is safe as that incomplete transaction is lost. -It should be checked that current master or new one has the new binlog file, in case of any error replication stops and errors are reported via *SHOW SLAVE STATUS* and in error logs. - - MariaDB> START SLAVE; - -Check for any error in log files and with - - MariaDB> SHOW SLAVE STATUS; - -In some situations replication state could be *STOPPED* and proper messages are displayed in error logs and in *SHOW SLAVE STATUS*, - -In order to resolve any mistake done with *CHANGE MASTER TO MASTER_LOG_FILE / MASTER_LOG_POS*, another administrative command would be helpful. - - MariaDB> RESET SLAVE; - -This command removes *master.ini* file, blanks all master configuration in memory and sets binlog router in unconfigured state: a *CHANGE MASTER TO* command should be issued for the new configuration. - -Note: existing binlog files are not touched by this command. - -Examples with SSL options: - - MySQL [(none)]> CHANGE MASTER TO MASTER_SSL = 1, MASTER_SSL_CERT='/home/maxscale/packages/certificates/client/client-cert.pem', MASTER_SSL_CA='/home/maxscale/packages/certificates/client/ca.pem', MASTER_SSL_KEY='/home/maxscale/packages/certificates/client/client-key.pem', MASTER_TLS_VERSION='TLSv12'; - - MySQL [(none)]> CHANGE MASTER TO MASTER_TLS_VERSION='TLSv12'; - - MySQL [(none)]> CHANGE MASTER TO MASTER_SSL = 0; - - -#### Some constraints: - - In order to enable/re-enable Master SSL comunication the MASTER_SSL=1 option is required and all certificate options must be explicitey set in the same CHANGE MASTER TO command. - - New certificate options changes take effect after maxScale restart or after MASTER_SSL=1 with the new options. - -Note: - - SHOW SLAVE STATUS displays all the options but MASTER_TLS_VERSION value. - - Maxadmin, 'show services' or 'show service $binlog_service' displays all the options when SSL is on. - - STOP SLAVE is required for CHANGE MASTER TO command (any option) - - START SLAVE will use new SSL options for Master SSL communication setup. - -Examples: - mysql client - - MySQL> SHOW SLAVE STATUS\G - - Master_SSL_Allowed: Yes - Master_SSL_CA_File: /home/mpinto/packages/certificates/client/ca.pem - Master_SSL_CA_Path: - Master_SSL_Cert: /home/mpinto/packages/certificates/client/client-cert.pem - Master_SSL_Cipher: - Master_SSL_Key: /home/mpinto/packages/certificates/client/client-key.pem - - maxadmin client - - MaxScale>'show service BinlogServer' - - Service: BinlogServer - Router: binlogrouter (0x7fd8c3002b40) - State: Started - Master connection DCB: 0x7fd8c8fce228 - Master SSL is ON: - Master SSL CA cert: /home/mpinto/packages/certificates/client/ca.pem - Master SSL Cert: /home/mpinto/packages/certificates/client/client-cert.pem - Master SSL Key: /home/mpinto/packages/certificates/client/client-key.pem - Master SSL tls_ver: TLSv12 - - - -### Slave servers setup - -Examples of *CHANGE MASTER TO* command issued on a slave server that wants to gets replication events from MariaDB MaxScale binlog router: -``` -CHANGE MASTER TO MASTER_HOST=‘$maxscale_IP’, MASTER_PORT=5308, MASTER_USER='repl', MASTER_PASSWORD=‘somepasswd’, -MASTER_LOG_FILE=‘mysql-bin.000001' - -CHANGE MASTER TO MASTER_HOST=‘$maxscale_IP’, MASTER_PORT=5308, MASTER_USER='repl', MASTER_PASSWORD=‘somepasswd’, -MASTER_LOG_FILE=‘mysql-bin.000159', MASTER_LOG_POS=245 -``` -The latter example specifies a *MASTER_LOG_POS* for the selected *MASTER_LOG_FILE* - -Note: - - - *MASTER_LOG_FILE* must be set to one of existing binlog files in MariaDB MaxScale binlogdir - - - If *MASTER_LOG_POS* is not set with *CHANGE MASTER TO* it defaults to 4 - - - Latest binlog file name and pos in MariaDB MaxScale could be find via maxadmin output or from mysql client connected to MariaDB MaxScale: - -Example: -``` --bash-4.1$ mysql -h 127.0.0.1 -P 5308 -u$user -p$pass - - MySQL [(none)]> show master status\G - *************************** 1. row *************************** - File: mysql-bin.000181 - Position: 2569 -``` -# Enabling MariaDB 10 compatibility - -MariaDB 10 has different slave registration phase so an option is required: - -``` -router_options=...., mariadb10-compatibility=1 -``` - -version_string should be modified in order to present MariaDB 10 version when MariaDB MaxScale sends server handshake packet. - -``` -version_string=10.0.17-log -``` - -# New MariaDB events in Diagnostics - -With a MariaDB 10 setups new events are displayed when master server is MariaDB 10. +MariaDB 10 masters display some extra events. ``` MariaDB 10 Annotate Rows Event 0 From b1f4d27eb1c4dafa686816dc8ed4a8c70fb31ccc Mon Sep 17 00:00:00 2001 From: "Goltz, Immo [Extern]" Date: Mon, 20 Nov 2017 15:50:37 +0100 Subject: [PATCH 06/11] Fix check_maxscale_resources.pl listernes parsing, close MXS-1531 --- plugins/nagios/check_maxscale_resources.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/nagios/check_maxscale_resources.pl b/plugins/nagios/check_maxscale_resources.pl index 79fdf6613..e15e845ce 100755 --- a/plugins/nagios/check_maxscale_resources.pl +++ b/plugins/nagios/check_maxscale_resources.pl @@ -118,7 +118,7 @@ chop($resource_type); my $resource_match = ucfirst("$resource_type Name"); if ($resource_type eq "listener") { - $resource_match = "Service Name"; + $resource_match = "Name"; } if ($resource_type eq "filter") { $resource_match = "Filter"; From 6555fc0babc33a75c04f66db1b50432a59f1d5fc Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Fri, 17 Nov 2017 15:33:17 +0200 Subject: [PATCH 07/11] add MDBCI build scripts --- BUILD/mdbci/build.sh | 138 ++++++++++++++++++++++ BUILD/mdbci/copy_repos.sh | 28 +++++ BUILD/mdbci/create_remote_repo.sh | 37 ++++++ BUILD/mdbci/create_repo.sh | 92 +++++++++++++++ BUILD/mdbci/generate_build_info_path.sh | 15 +++ BUILD/mdbci/remote_build.sh | 79 +++++++++++++ BUILD/mdbci/set_build_variables.sh | 73 ++++++++++++ BUILD/mdbci/templates/build.json.template | 8 ++ BUILD/run_test.sh | 6 - 9 files changed, 470 insertions(+), 6 deletions(-) create mode 100755 BUILD/mdbci/build.sh create mode 100755 BUILD/mdbci/copy_repos.sh create mode 100755 BUILD/mdbci/create_remote_repo.sh create mode 100755 BUILD/mdbci/create_repo.sh create mode 100755 BUILD/mdbci/generate_build_info_path.sh create mode 100755 BUILD/mdbci/remote_build.sh create mode 100644 BUILD/mdbci/set_build_variables.sh create mode 100644 BUILD/mdbci/templates/build.json.template delete mode 100644 BUILD/run_test.sh diff --git a/BUILD/mdbci/build.sh b/BUILD/mdbci/build.sh new file mode 100755 index 000000000..8fa5c82d4 --- /dev/null +++ b/BUILD/mdbci/build.sh @@ -0,0 +1,138 @@ +#!/bin/bash + +# $box - Vagrant box to be used for build + +# $target - name of repository to put results + +# $cmake_flags - cmake flags to be used in the build + +# $MDBCI_VM_PATH - path to the MDBCI virtual machies directory + +# $source - reference to the point in the source code repository + +# $do_not_destroy_vm - if "yes" VM stays alive after the build + +# $try_already_running - if "yes" already running VM will be used for build + +# $gpg_keys_path - path to the directory containing GPG keys for repo signing +# directory have to contain only one file *.public and only one *.private + +set -x + +# read the name of build scripts directory +export script_dir="$(dirname $(readlink -f $0))" + +# load all needed variables +. ${script_dir}/set_build_variables.sh + +dist_sfx="$platform"."$platform_version" +export cmake_flags="${cmake_flags} -DPACKAGE=Y -DDISTRIB_SUFFIX=${dist_sfx}" + +# prerare VM +export provider=`${mdbci_dir}/mdbci show provider $box --silent 2> /dev/null` +export name="$box-${JOB_NAME}-${BUILD_NUMBER}" +export name=`echo $name | sed "s|/|-|g"` + +export platform=`${mdbci_dir}/mdbci show boxinfo --box-name=$box --field='platform' --silent` +export platform_version=`${mdbci_dir}/mdbci show boxinfo --box-name=$box --field='platform_version' --silent` + + +if [ "${try_already_running}" == "yes" ]; then + export name=${box} + export snapshot_lock_file=$MDBCI_VM_PATH/${name}_snapshot_lock + while [ -f ${snapshot_lock_file} ] + do + echo "snapshot is locked, waiting ..." + sleep 5 + done + echo ${JOB_NAME}-${BUILD_NUMBER} > ${snapshot_lock_file} + ${mdbci_dir}/mdbci snapshot revert --path-to-nodes $name --snapshot-name clean + if [ $? == 0 ]; then + export already_running="ok" + fi +fi + +if [ "$already_running" != "ok" ]; then + + eval "cat < /dev/null > $MDBCI_VM_PATH/${name}.json + + while [ -f ~/vagrant_lock ] + do + sleep 5 + done + touch ~/vagrant_lock + echo $JOB_NAME-$BUILD_NUMBER >> ~/vagrant_lock + + # destroying existing box + if [ -d "$MDBCI_VM_PATH/${name}" ]; then + cd $MDBCI_VM_PATH/${name} + vagrant destroy -f + cd ${dir} + fi + + # starting VM for build + echo "Generating build VM template" + ${mdbci_dir}/mdbci --override --template $MDBCI_VM_PATH/$name.json generate $name + echo "starting VM for build" + ${mdbci_dir}/mdbci up --attempts=1 $name + if [ $? != 0 ] ; then + echo "Error starting VM" + cd $MDBCI_VM_PATH/${name} + rm ~/vagrant_lock + cd $dir + exit 1 + fi + echo "copying public keys to VM" + cp ~/build-scripts/team_keys . + ${mdbci_dir}/mdbci public_keys --key team_keys --silent $name +fi + +echo "Get VM info" +export sshuser=`${mdbci_dir}/mdbci ssh --command 'whoami' --silent $name/build 2> /dev/null | tr -d '\r'` +export IP=`${mdbci_dir}/mdbci show network $name/build --silent 2> /dev/null` +export sshkey=`${mdbci_dir}/mdbci show keyfile $name/build --silent 2> /dev/null | sed 's/"//g'` +export scpopt="-i $sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ConnectTimeout=120 " +export sshopt="$scpopt $sshuser@$IP" + +echo "Release Vagrant lock" +rm ~/vagrant_lock + +echo "Starting build" +${script_dir}/remote_build.sh +export build_result=$? + +shellcheck `find . | grep "\.sh"` | grep -i "POSIX sh" +if [ $? -eq 0 ] ; then + echo "POSIX sh error are found in the scripts" +# exit 1 +fi + + +${script_dir}/create_remote_repo.sh + +${script_dir}/copy_repos.sh + + +echo "Removing locks and destroying VM" +cd $MDBCI_VM_PATH/$name +if [ "$try_already_running" == "yes" ] ; then + echo "Release lock for already running VM" + rm $snapshot_lock_file +fi +if [[ "$do_not_destroy_vm" != "yes" && "$try_already_running" != "yes" ]] ; then + echo "Destroying VM" + vagrant destroy -f + cd .. + rm -rf $name + rm -rf ${name}.json + rm -rf ${name}_netwotk_config +fi +cd $dir + +if [ $build_result -ne 0 ] ; then + echo "Build FAILED!" + exit $build_result +fi + diff --git a/BUILD/mdbci/copy_repos.sh b/BUILD/mdbci/copy_repos.sh new file mode 100755 index 000000000..84a260546 --- /dev/null +++ b/BUILD/mdbci/copy_repos.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Copyies repo from ${unsorted_repo_dir}/$target/$box to + +dir=`pwd` +if [ "$box_type" == "RPM" ] ; then + export arch=`ssh $sshopt "arch"` + . ${script_dir}/generate_build_info_path.sh + + rm -rf $path_prefix/$platform/$platform_version/$arch/ + mkdir -p $path_prefix/$platform/$platform_version/$arch/ + cp -r ${unsorted_repo_dir}/$repo_name/$box/* $path_prefix/$platform/$platform_version/$arch/ + env > $build_info_path + cd $path_prefix/$platform + ln -s $platform_version "$platform_version"server + ln -s $platform_version "$platform_version"Server + + echo "copying done" +else + export arch=`ssh $sshopt "dpkg --print-architecture"` + . ${script_dir}/generate_build_info_path.sh + rm -rf $path_prefix/$platform_family/dists/$platform_version/main/binary-"$arch" + rm -rf $path_prefix/$platform_family/dists/$platform_version/main/binary-i386 + mkdir -p $path_prefix/$platform_family/ + cp -r ${unsorted_repo_dir}/$repo_name/$box/* $path_prefix/$platform_family/ + env > $build_info_path +fi +cd $dir diff --git a/BUILD/mdbci/create_remote_repo.sh b/BUILD/mdbci/create_remote_repo.sh new file mode 100755 index 000000000..2de76e80f --- /dev/null +++ b/BUILD/mdbci/create_remote_repo.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# Creates RPM or DEB repository for biniries from +# $pre_repo_dir/$target/$box, signs it with keys +# from ${gpg_keys_path} and puts signed repo to + +set -x + +export work_dir="MaxScale" + +echo "creating repository" +echo "cleaning VM" +ssh $sshopt "rm -rf dest; rm -rf src;" + +echo " creating dirs on VM" +ssh $sshopt "mkdir -p dest ; mkdir -p src; mkdir gpg_keys" + +echo "copying stuff to VM" +scp $scpopt $pre_repo_dir/$target/$box/* $sshuser@$IP:src/ + +scp $scpopt -r ${gpg_keys_path}/* $sshuser@$IP:./gpg_keys/ +ssh $sshopt "key=\`ls ~/gpg_keys/*.public -1\` ; gpg --import \$key" +ssh $sshopt "key=\`ls ~/gpg_keys/*.private -1\` ; gpg --allow-secret-key-import --import \$key" + +echo "executing create_repo.sh on VM" +ssh $sshopt "export platform=$platform; export platform_version=$platform_version; $work_dir/BUILD/mdbci/create_repo.sh dest/ src/" +if [ $? != 0 ] ; then + echo "Repo creation failed!" + exit 1 +fi + +echo "cleaning ${unsorted_repo_dir}/$target/$box/" +rm -rf ${unsorted_repo_dir}/$target/$box/* + +echo "copying repo from $box" +mkdir -p ${unsorted_repo_dir}/$target/$box +scp $scpopt -r $sshuser@$IP:dest/* ${unsorted_repo_dir}/$target/$box/ diff --git a/BUILD/mdbci/create_repo.sh b/BUILD/mdbci/create_repo.sh new file mode 100755 index 000000000..8e687adb8 --- /dev/null +++ b/BUILD/mdbci/create_repo.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +set -x + +destdir=$1 +sourcedir=$2 + +#rm -rf $destdir +mkdir -p $destdir/ + +zypper --version +z_res=$? +yum --version +y_res=$? + +if [ $z_res -eq 127 ] && [ $y_res -eq 127 ] ; then +# DEB-based system + arch_name=`dpkg --print-architecture` + arch="binary-$arch_name" + cd $destdir + debian_ver=`cat /etc/debian_version` + echo "Debian version: $debian_ver" + dist_name=$platform_version + + mkdir -p dists/$dist_name/main/$arch/ + + cp ~/$sourcedir/* dists/$dist_name/main/$arch/ + sudo apt-get update + sudo apt-get install -y dpkg-dev + dpkg-scanpackages dists/$dist_name/main/$arch/ /dev/null | gzip -9c > dists/$dist_name/main/$arch/Packages.gz + gunzip -c dists/$dist_name/main/$arch/Packages.gz > dists/$dist_name/main/$arch/Packages +# echo "Archive: main" > dists/$dist_name/main/$arch/Release +# echo "Suite: main" >> dists/$dist_name/main/$arch/Release + echo "Components: main" >> dists/$dist_name/main/$arch/Release + echo "Codename: $dist_name" >> dists/$dist_name/main/$arch/Release + echo "Origin: MariaDB" >> dists/$dist_name/main/$arch/Release + echo "Label: MariaDB Maxscale repository" >> dists/$dist_name/main/$arch/Release + uname -m | grep "x86_64" + if [ $? -eq 0 ] ; then +# echo "Architectures: amd64 i386" >> dists/$dist_name/main/$arch/Release + mkdir -p dists/$dist_name/main/binary-i386/ + dpkg-scanpackages dists/$dist_name/main/binary-i386/ /dev/null | gzip -9c > dists/$dist_name/main/binary-i386/Packages.gz + gunzip -c dists/$dist_name/main/binary-i386/Packages.gz > dists/$dist_name/main/binary-i386/Packages +# else +# echo "Architectures: ppc64el" >> dists/$dist_name/main/$arch/Release + fi + archs=`ls -1 dists/$dist_name/main | sed "s/binary-//" | tr '\n' ' '` + echo "Architectures: $archs" >> dists/$dist_name/main/$arch/Release + echo "Description: MariaDB MaxScale" >> dists/$dist_name/main/$arch/Release + cp dists/$dist_name/main/$arch/Release dists/$dist_name/Release +# cp dists/$dist_name/main/$arch/Packages.gz dists/$dist_name + apt-ftparchive release dists/$dist_name/ >> dists/$dist_name/Release + if [ $? != 0 ] ; then + echo "Repo creation failed!" + exit 1 + fi + + gpg -abs -o dists/$dist_name/Release.gpg dists/$dist_name/Release + if [ $? != 0 ] ; then + echo "Package signing failed!" + exit 1 + fi +else +# RPM-based system + sudo yum install -y createrepo + sudo zypper -n remove patterns-openSUSE-minimal_base-conflicts + sudo zypper -n install createrepo + echo "%_signature gpg" >> ~/.rpmmacros + echo "%_gpg_name MariaDB Maxscale" >> ~/.rpmmacros +# echo "%_gpg_name MariaDBManager" >> ~/.rpmmacros + echo "\r" | setsid rpm --resign $sourcedir/*.rpm + + if [ $? != 0 ] ; then + echo "Package signing failed!" + exit 1 + fi + gpg --output repomd.xml.key --sign $sourcedir/repodata/repomd.xml + cp $sourcedir/* $destdir/ + pushd ${destdir} >/dev/null 2>&1 + createrepo -d -s sha . + if [ $? != 0 ] ; then + echo "Repo creation failed!" + exit 1 + fi + + popd >/dev/null 2>&1 + gpg -a --detach-sign $destdir/repodata/repomd.xml + if [ $? != 0 ] ; then + echo "Package signing failed!" + exit 1 + fi +fi diff --git a/BUILD/mdbci/generate_build_info_path.sh b/BUILD/mdbci/generate_build_info_path.sh new file mode 100755 index 000000000..230986fd1 --- /dev/null +++ b/BUILD/mdbci/generate_build_info_path.sh @@ -0,0 +1,15 @@ +#! /bin/bash + +set -x + +export web_prefix=$(echo $path_prefix | sed "s|${repo_path}/||g") + +if [ "$box_type" == "RPM" ] ; then + export build_info_file="$platform/$platform_version/$arch/build_info" +else + export build_info_file="$platform_family/dists/$platform_version/main/binary-$arch/build_info" +fi + +echo "BUILD_PATH_INFO=$web_prefix/$build_info_file" > $dir/build_info_env_var_$BUILD_ID + +export build_info_path=$path_prefix/$build_info_file diff --git a/BUILD/mdbci/remote_build.sh b/BUILD/mdbci/remote_build.sh new file mode 100755 index 000000000..a91bdbe0b --- /dev/null +++ b/BUILD/mdbci/remote_build.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +# Copyies stuff to VM, run build on VM and copies binaries +# to $pre_repo_dir/$target/$box + +set -x + + +rm -rf $pre_repo_dir/$target/$box +mkdir -p $pre_repo_dir/$target/SRC +mkdir -p $pre_repo_dir/$target/$box + +export work_dir="MaxScale" +export orig_image=$box + +ssh $sshopt "sudo rm -rf $work_dir" +echo "copying stuff to $image machine" +ssh $sshopt "mkdir -p $work_dir" + +rsync -avz --progress --delete -e "ssh $scpopt" ${script_dir}/../../../MaxScale/* $sshuser@$IP:$work_dir/ +if [ $? -ne 0 ] ; then + echo "Error copying stuff to $box machine" + exit 2 +fi + +export install_script="install_build_deps.sh" + +if [ "$box_type" == "RPM" ] ; then + build_script="build_rpm_local.sh" + files="*.rpm" + tars="$product_name*.tar.gz" +else + build_script="build_deb_local.sh" + files="../*.deb" + tars="$product_name*.tar.gz" +fi + +export remote_build_cmd="export already_running=\"$already_running\"; \ + export build_experimental=\"$build_experimental\"; \ + export cmake_flags=\"$cmake_flags\"; \ + export work_dir=\"$work_dir\"; \ + export platform=\"$platform\"; \ + export platform_version=\"$platform_version\"; \ + export source=\"$source\"; \ + export BUILD_TAG=\"$BUILD_TAG\"; \ + " + +if [ "$already_running" != "ok" ] +then + echo "install packages on $image" + ssh $sshopt "$remote_build_cmd ./MaxScale/BUILD/$install_script" + installres=$? + if [ $installres -ne 0 ] + then + exit $installres + fi + + $HOME/mdbci/mdbci snapshot take --path-to-nodes $box --snapshot-name clean + +else + echo "already running VM, not installing deps" +fi + +echo "run build on $box" +ssh $sshopt "$remote_build_cmd ./MaxScale/BUILD/$build_script" +if [ $? -ne 0 ] ; then + echo "Error build on $box" + exit 4 +fi + +echo "copying binaries to the '$pre_repo_dir/$target/$box'" +scp $scpopt $sshuser@$IP:$work_dir/$files $pre_repo_dir/$target/$box/ +scp $scpopt $sshuser@$IP:$work_dir/$tars $pre_repo_dir/$target/$box/ + + +echo "package building for '$target' for '$platform' '$platform_version' done!" + + + diff --git a/BUILD/mdbci/set_build_variables.sh b/BUILD/mdbci/set_build_variables.sh new file mode 100644 index 000000000..95c76332e --- /dev/null +++ b/BUILD/mdbci/set_build_variables.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +# Defines defaults values for all uninitialized environmental variables +# In case of running from Jenkins all values go from Jenkins parameters + +export dir=`pwd` + +export MDBCI_VM_PATH=${MDBCI_VM_PATH:-$HOME/vms} +mkdir -p $MDBCI_VM_PATH +echo "MDBCI_VM_PATH=$MDBCI_VM_PATH" + +export box=${box:-"centos_7_libvirt"} +echo "box=$box" + +# get commit ID +commitID=`git log | head -1 | sed "s/commit //"` +echo "commitID $commitID" + +export branch=`git symbolic-ref --short HEAD` +export curr_date=`date '+%Y-%m-%d_%H-%M'` + +export source=${source:-"$branch"} +echo "source=$source" + +#hack to get rid of Jenkins artifacts +export target=`echo $target | tr -cd "[:print:]" | sed "s/?//g" | sed "s/ //g"` + +export target=${target:-"$source-$curr_date"} +echo "target=$target" + +export product_name=${product_name:-"maxscale"} + +export build_experimental=${build_experimental:-"yes"} + +export gpg_keys_path=${gpg_keys_path:-"$HOME/maxscale_gpg_keys/"} + +export pre_repo_dir=${pre_repo_dir:-"$HOME/pre-repo/"} + +export unsorted_repo_dir=${unsorted_repo_dir:-"$HOME/repo/"} + +export box_type="RPM" +echo $box | grep -i ubuntu +if [ $? == 0 ] ; then + export box_type="DEB" + export platform_family="ubuntu" +fi +echo $box | grep -i deb +if [ $? == 0 ] ; then + export box_type="DEB" + export platform_family="debian" +fi + +export cmake_flags=${cmake_flags:-"-DBUILD_TESTS=Y -DCMAKE_BUILD_TYPE=Debug -DBUILD_MMMON=Y -DBUILD_AVRO=Y -DBUILD_CDC=Y"} +echo "cmake_flags=$cmake_flags" + +export do_not_destroy_vm=${do_not_destroy_vm:-"no"} + +export try_already_running=${try_already_running:-"no"} + +export JOB_NAME=${JOB_NAME:-"local_build"} + +export BUILD_NUMBER=${BUILD_NUMBER:-`date '+%Y%m%d%H%M'`} + +export BUILD_TAG=${BUILD_TAG:-jenkins-${JOB_NAME}-${BUILD_NUMBER}} + +export mdbci_dir=${mdbci_dir:-"$HOME/mdbci/"} + +export repo_name=$target + +export repo_path=${repo_path:-$HOME/repository} + +export path_prefix="$repo_path/$repo_name/mariadb-$product_name/" + diff --git a/BUILD/mdbci/templates/build.json.template b/BUILD/mdbci/templates/build.json.template new file mode 100644 index 000000000..b53ecb321 --- /dev/null +++ b/BUILD/mdbci/templates/build.json.template @@ -0,0 +1,8 @@ +{ + "build" : + { + "hostname" : "default", + "box" : "$box" + } +} + diff --git a/BUILD/run_test.sh b/BUILD/run_test.sh deleted file mode 100644 index 57ea40d3b..000000000 --- a/BUILD/run_test.sh +++ /dev/null @@ -1,6 +0,0 @@ -cd ~/Maxscale/maxscale-system-test - -cmake . -make - -ctest -LE HEAVY -VV From 6541f845ea8e827d09f18febe7cf6dc05cd508e4 Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Mon, 20 Nov 2017 12:24:26 +0200 Subject: [PATCH 08/11] add check for zypper and fix gpg repo sign --- BUILD/mdbci/create_repo.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/BUILD/mdbci/create_repo.sh b/BUILD/mdbci/create_repo.sh index 8e687adb8..f3a565ebf 100755 --- a/BUILD/mdbci/create_repo.sh +++ b/BUILD/mdbci/create_repo.sh @@ -62,9 +62,13 @@ if [ $z_res -eq 127 ] && [ $y_res -eq 127 ] ; then fi else # RPM-based system - sudo yum install -y createrepo - sudo zypper -n remove patterns-openSUSE-minimal_base-conflicts - sudo zypper -n install createrepo + if [ ${y_res} == 0 ]; then + sudo yum install -y createrepo + fi + if [ ${z_res} == 0 ]; then + sudo zypper -n remove patterns-openSUSE-minimal_base-conflicts + sudo zypper -n install createrepo + fi echo "%_signature gpg" >> ~/.rpmmacros echo "%_gpg_name MariaDB Maxscale" >> ~/.rpmmacros # echo "%_gpg_name MariaDBManager" >> ~/.rpmmacros @@ -74,7 +78,7 @@ else echo "Package signing failed!" exit 1 fi - gpg --output repomd.xml.key --sign $sourcedir/repodata/repomd.xml + cp $sourcedir/* $destdir/ pushd ${destdir} >/dev/null 2>&1 createrepo -d -s sha . @@ -84,6 +88,7 @@ else fi popd >/dev/null 2>&1 + gpg --output repomd.xml.key --sign $destdir/repodata/repomd.xml gpg -a --detach-sign $destdir/repodata/repomd.xml if [ $? != 0 ] ; then echo "Package signing failed!" From 0c1b0648b6e3037083dbfb5f430175ce92120323 Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Tue, 21 Nov 2017 12:11:44 +0200 Subject: [PATCH 09/11] add repo config file for mdbci generation --- BUILD/mdbci/copy_repos.sh | 8 ++++++++ BUILD/mdbci/set_build_variables.sh | 5 +++++ .../templates/repository-config/deb.json.template | 10 ++++++++++ .../templates/repository-config/rpm.json.template | 10 ++++++++++ 4 files changed, 33 insertions(+) create mode 100644 BUILD/mdbci/templates/repository-config/deb.json.template create mode 100644 BUILD/mdbci/templates/repository-config/rpm.json.template diff --git a/BUILD/mdbci/copy_repos.sh b/BUILD/mdbci/copy_repos.sh index 84a260546..ef5eca33b 100755 --- a/BUILD/mdbci/copy_repos.sh +++ b/BUILD/mdbci/copy_repos.sh @@ -15,6 +15,11 @@ if [ "$box_type" == "RPM" ] ; then ln -s $platform_version "$platform_version"server ln -s $platform_version "$platform_version"Server + eval "cat < /dev/null > ${path_prefix}/${platform}_${platform_version}.json + + echo "copying done" else export arch=`ssh $sshopt "dpkg --print-architecture"` @@ -24,5 +29,8 @@ else mkdir -p $path_prefix/$platform_family/ cp -r ${unsorted_repo_dir}/$repo_name/$box/* $path_prefix/$platform_family/ env > $build_info_path + eval "cat < /dev/null > ${path_prefix}/${platform}_${platform_version}.json fi cd $dir diff --git a/BUILD/mdbci/set_build_variables.sh b/BUILD/mdbci/set_build_variables.sh index 95c76332e..069d40197 100644 --- a/BUILD/mdbci/set_build_variables.sh +++ b/BUILD/mdbci/set_build_variables.sh @@ -71,3 +71,8 @@ export repo_path=${repo_path:-$HOME/repository} export path_prefix="$repo_path/$repo_name/mariadb-$product_name/" +export ci_url=${ci_url:-"http://max-tst-01.mariadb.com/ci-repository/"} + +export deb_repo_key=${deb_repo_key:-"135659e928c12247"} + +export rpm_repo_key=${rpm_repo_key:-"$ci_url/MariaDBMaxscale-GPG-KEY.public"} diff --git a/BUILD/mdbci/templates/repository-config/deb.json.template b/BUILD/mdbci/templates/repository-config/deb.json.template new file mode 100644 index 000000000..fc2725554 --- /dev/null +++ b/BUILD/mdbci/templates/repository-config/deb.json.template @@ -0,0 +1,10 @@ +[ +{ + "product": "maxscale", + "version": "default", + "repo": "${ci_url}/${web_prefix}/${platform_family} ${platform_version} main", + "repo_key": "$deb_repo_key70E4618A8167EE24", + "platform": "$platform", + "platform_version": "${platform_version}" +} +] diff --git a/BUILD/mdbci/templates/repository-config/rpm.json.template b/BUILD/mdbci/templates/repository-config/rpm.json.template new file mode 100644 index 000000000..075b2bd36 --- /dev/null +++ b/BUILD/mdbci/templates/repository-config/rpm.json.template @@ -0,0 +1,10 @@ +[ +{ + "product": "maxscale", + "version": "default", + "repo": "${ci_url}/$web_prefix/centos/${platform_version}/\$basearch", + "repo_key": "${rpm_repo_key}", + "platform": "$platform", + "platform_version": "${platform_version}" +} +] From e1711889b959ee1f49afd3c03bcda976dc5f3f6b Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Thu, 23 Nov 2017 11:26:18 +0200 Subject: [PATCH 10/11] add upgrade test --- BUILD/mdbci/build.sh | 4 + BUILD/mdbci/cnf/maxscale.cnf.minimum | 15 +++ BUILD/mdbci/cnf/maxscale.cnf.minimum.1.4.4 | 16 +++ BUILD/mdbci/configure_log_dir.sh | 15 +++ BUILD/mdbci/local_rep.sh | 1 + BUILD/mdbci/set_build_variables.sh | 4 + BUILD/mdbci/templates/install.json.template | 11 ++ BUILD/mdbci/upgrade_test.sh | 138 ++++++++++++++++++++ 8 files changed, 204 insertions(+) create mode 100644 BUILD/mdbci/cnf/maxscale.cnf.minimum create mode 100644 BUILD/mdbci/cnf/maxscale.cnf.minimum.1.4.4 create mode 100644 BUILD/mdbci/configure_log_dir.sh create mode 100644 BUILD/mdbci/local_rep.sh create mode 100644 BUILD/mdbci/templates/install.json.template create mode 100755 BUILD/mdbci/upgrade_test.sh diff --git a/BUILD/mdbci/build.sh b/BUILD/mdbci/build.sh index 8fa5c82d4..7437a8ff0 100755 --- a/BUILD/mdbci/build.sh +++ b/BUILD/mdbci/build.sh @@ -136,3 +136,7 @@ if [ $build_result -ne 0 ] ; then exit $build_result fi +if [ ${run_upgrade_test} == "yes" ] ; then + ${script_dir}/upgrade_test.sh +fi + diff --git a/BUILD/mdbci/cnf/maxscale.cnf.minimum b/BUILD/mdbci/cnf/maxscale.cnf.minimum new file mode 100644 index 000000000..7f98211b6 --- /dev/null +++ b/BUILD/mdbci/cnf/maxscale.cnf.minimum @@ -0,0 +1,15 @@ +[maxscale] +threads=4 +log_warning=1 + +[CLI] +type=service +router=cli + +[CLI Listener] +type=listener +service=CLI +protocol=maxscaled +#address=localhost +socket=default + diff --git a/BUILD/mdbci/cnf/maxscale.cnf.minimum.1.4.4 b/BUILD/mdbci/cnf/maxscale.cnf.minimum.1.4.4 new file mode 100644 index 000000000..1ae204cc2 --- /dev/null +++ b/BUILD/mdbci/cnf/maxscale.cnf.minimum.1.4.4 @@ -0,0 +1,16 @@ +[maxscale] +threads=4 +log_warning=1 + +[CLI] +type=service +router=cli + +[CLI Listener] +type=listener +service=CLI +protocol=maxscaled +#address=localhost +port=6603 + + diff --git a/BUILD/mdbci/configure_log_dir.sh b/BUILD/mdbci/configure_log_dir.sh new file mode 100644 index 000000000..568c298b7 --- /dev/null +++ b/BUILD/mdbci/configure_log_dir.sh @@ -0,0 +1,15 @@ +set -x +LOGS_DIR=${logs_dir:-$HOME/LOGS} +echo $JOB_NAME | grep "/" +if [ $? == 0 ] ; then + export job_name_buildID=`echo $JOB_NAME | sed "s|/|-$BUILD_NUMBER/|"` + export logs_publish_dir="${LOGS_DIR}/${job_name_buildID}/" +else + export logs_publish_dir="${LOGS_DIR}/${JOB_NAME}-${BUILD_NUMBER}" +fi + +export job_name_buildID=`echo ${JOB_NAME} | sed "s|/|-${BUILD_NUMBER}/|"` +export logs_publish_dir="${LOGS_DIR}/${job_name_buildID}-${BUILD_NUMBER}" +echo "Logs go to ${logs_publish_dir}" +mkdir -p ${logs_publish_dir} + diff --git a/BUILD/mdbci/local_rep.sh b/BUILD/mdbci/local_rep.sh new file mode 100644 index 000000000..0cb6626aa --- /dev/null +++ b/BUILD/mdbci/local_rep.sh @@ -0,0 +1 @@ +export ci_url="http://192.168.122.1/repository" diff --git a/BUILD/mdbci/set_build_variables.sh b/BUILD/mdbci/set_build_variables.sh index 069d40197..8c4dd8b02 100644 --- a/BUILD/mdbci/set_build_variables.sh +++ b/BUILD/mdbci/set_build_variables.sh @@ -76,3 +76,7 @@ export ci_url=${ci_url:-"http://max-tst-01.mariadb.com/ci-repository/"} export deb_repo_key=${deb_repo_key:-"135659e928c12247"} export rpm_repo_key=${rpm_repo_key:-"$ci_url/MariaDBMaxscale-GPG-KEY.public"} + +export run_upgrade_test=${run_upgrade_test:-"no"} + +export production_url=${production_url:-"https://downloads.mariadb.com/MaxScale/"} diff --git a/BUILD/mdbci/templates/install.json.template b/BUILD/mdbci/templates/install.json.template new file mode 100644 index 000000000..81db5a9de --- /dev/null +++ b/BUILD/mdbci/templates/install.json.template @@ -0,0 +1,11 @@ +{ + "maxscale" : + { + "hostname" : "maxscale", + "box" : "$box", + "product" : { + "name": "maxscale" + } + + } +} diff --git a/BUILD/mdbci/upgrade_test.sh b/BUILD/mdbci/upgrade_test.sh new file mode 100755 index 000000000..c446a0de9 --- /dev/null +++ b/BUILD/mdbci/upgrade_test.sh @@ -0,0 +1,138 @@ +#!/bin/bash + +set -x + +# read the name of build scripts directory +export script_dir="$(dirname $(readlink -f $0))" + +# load all needed variables +. ${script_dir}/set_build_variables.sh + +export maxadmin_command=${maxadmin_command:-"sudo maxadmin show services"} + +export old_target=${old_target:-"2.1.9"} +export old_target=`echo $old_target | sed "s/?//g"` + +provider=`${mdbci_dir}/mdbci show provider $box --silent 2> /dev/null` +name=$box-${JOB_NAME}-${BUILD_NUMBER}_upgradetest +name=`echo $name | sed "s|/|-|g"` + + +cp ${script_dir}/install.json.template ${MDBCI_VM_PATH}/$name.json + + eval "cat < /dev/null > $MDBCI_VM_PATH/${name}.json + + +while [ -f ~/vagrant_lock ] +do + sleep 5 +done +touch ~/vagrant_lock +echo $JOB_NAME-$BUILD_NUMBER >> ~/vagrant_lock + +# destroying existing box +if [ -d "install_$box" ]; then + cd $MDBCI_VM_PATH/$name + vagrant destroy -f + cd $dir +fi + +${mdbci_dir}/repository-config/generate_all.sh repo.d +${mdbci_dir}/repository-config/maxscale-release.sh $old_target repo.d + +# starting VM for build +${mdbci_dir}/mdbci --override --template $MDBCI_VM_PATH/$name.json --repo-dir $dir/repo.d generate $name +${mdbci_dir}/mdbci up $name --attempts=1 +if [ $? != 0 ] ; then + if [ $? != 0 ] ; then + echo "Error starting VM" + cd ${MDBCI_VM_PATH}/$name + if [ "x$do_not_destroy_vm" != "xyes" ] ; then + vagrant destroy -f + fi + cd $dir + rm ~/vagrant_lock + exit 1 + fi +fi + +rm ~/vagrant_lock + +# get VM info +export sshuser=`${mdbci_dir}/mdbci ssh --command 'whoami' --silent $name/maxscale 2> /dev/null | tr -d '\r'` +export IP=`${mdbci_dir}/mdbci show network $name/maxscale --silent 2> /dev/null` +export sshkey=`${mdbci_dir}/mdbci show keyfile $name/maxscale --silent 2> /dev/null | sed 's/"//g'` +export scpopt="-i $sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ConnectTimeout=120 " +export sshopt="$scpopt $sshuser@$IP" + +old_version=`ssh $sshopt "maxscale --version" ` + +rm -rf repo.d +${mdbci_dir}/repository-config/generate_all.sh repo.d +${mdbci_dir}/repository-config/maxscale-ci.sh $target repo.d + +${mdbci_dir}/mdbci setup_repo --product maxscale --repo-dir $dir/repo.d $name/maxscale +${mdbci_dir}/mdbci install_product --product maxscale $name/maxscale + +res=$? + +new_version=`ssh $sshopt "maxscale --version" ` + +echo "old version: '${old_version}', new version: '${new_version}'" +if [ "${old_version}" == "${new_version}" ]; then + echo "Upgrde was not done!" + res=1 +fi + +export cnf_file=${cnf_file:-"maxscale.cnf.minimum"} + +scp $scpopt ${script_dir}/cnf/$cnf_file $sshuser@$IP:~/ + +. ${script_dir}/configure_log_dir.sh + +${mdbci_dir}/mdbci ssh --command 'service --help' $name/maxscale +if [ $? == 0 ] ; then + maxscale_start_cmd="sudo service maxscale start" +else + ${mdbci_dir}/mdbci ssh --command 'echo \"/usr/bin/maxscale 2> /dev/null &\" > maxscale_start.sh; echo \"disown\" >> maxscale_start.sh; chmod a+x maxscale_start.sh' $name/maxscale --silent + maxscale_start_cmd="sudo ./maxscale_start.sh 2> /dev/null &" +fi + + + +ssh $sshopt "sudo cp $cnf_file /etc/maxscale.cnf" +ssh $sshopt "$maxscale_start_cmd" & +pid_to_kill=$! + +sleep 10 + +ssh $sshopt $maxadmin_command +if [ $? != 0 ] ; then + echo "Maxadmin executing error" + res=1 +fi +maxadmin_out=`ssh $sshopt $maxadmin_command` +echo $maxadmin_out | grep "CLI" +if [ $? != 0 ] ; then + echo "CLI service is not found in maxadmin output" + res=1 +fi +echo $maxadmin_out | grep "Started" +if [ $? != 0 ] ; then + echo "'Started' is not found in the CLI service description" + res=1 +fi + +mkdir -p $logs_publish_dir +scp $scpopt $sshuser@$IP:/var/log/maxscale/* $logs_publish_dir +chmod a+r $logs_publish_dir/* + +if [ "x$do_not_destroy_vm" != "xyes" ] ; then + cd $MDBCI_VM_PATH/$name + vagrant destroy -f + cd $dir +fi +kill $pid_to_kill +exit $res From 58984b71d85ac67382442d6b28ac7d73a90783e2 Mon Sep 17 00:00:00 2001 From: Timofey Turenko Date: Thu, 23 Nov 2017 13:01:34 +0200 Subject: [PATCH 11/11] fix paths --- BUILD/mdbci/create_remote_repo.sh | 2 +- BUILD/mdbci/remote_build.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/BUILD/mdbci/create_remote_repo.sh b/BUILD/mdbci/create_remote_repo.sh index 2de76e80f..1d9102cd7 100755 --- a/BUILD/mdbci/create_remote_repo.sh +++ b/BUILD/mdbci/create_remote_repo.sh @@ -23,7 +23,7 @@ ssh $sshopt "key=\`ls ~/gpg_keys/*.public -1\` ; gpg --import \$key" ssh $sshopt "key=\`ls ~/gpg_keys/*.private -1\` ; gpg --allow-secret-key-import --import \$key" echo "executing create_repo.sh on VM" -ssh $sshopt "export platform=$platform; export platform_version=$platform_version; $work_dir/BUILD/mdbci/create_repo.sh dest/ src/" +ssh $sshopt "export platform=$platform; export platform_version=$platform_version; ./$work_dir/BUILD/mdbci/create_repo.sh dest/ src/" if [ $? != 0 ] ; then echo "Repo creation failed!" exit 1 diff --git a/BUILD/mdbci/remote_build.sh b/BUILD/mdbci/remote_build.sh index a91bdbe0b..be02521f2 100755 --- a/BUILD/mdbci/remote_build.sh +++ b/BUILD/mdbci/remote_build.sh @@ -17,7 +17,7 @@ ssh $sshopt "sudo rm -rf $work_dir" echo "copying stuff to $image machine" ssh $sshopt "mkdir -p $work_dir" -rsync -avz --progress --delete -e "ssh $scpopt" ${script_dir}/../../../MaxScale/* $sshuser@$IP:$work_dir/ +rsync -avz --progress --delete -e "ssh $scpopt" ${script_dir}/../../* $sshuser@$IP:./$work_dir/ if [ $? -ne 0 ] ; then echo "Error copying stuff to $box machine" exit 2