hcq/MaxScale
hcq/MaxScale
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch '2.1' into 2.2

Browse Source
This commit is contained in:
Markus Mäkelä 2017-11-27 09:10:33 +02:00
commit a72956e2f3
22 changed files with 1272 additions and 846 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

142
BUILD/mdbci/build.sh Executable file
View File

@ -0,0 +1,142 @@
#!/bin/bash
# $box - Vagrant box to be used for build
# $target - name of repository to put results
# $cmake_flags - cmake flags to be used in the build
# $MDBCI_VM_PATH - path to the MDBCI virtual machies directory
# $source - reference to the point in the source code repository
# $do_not_destroy_vm - if "yes" VM stays alive after the build
# $try_already_running - if "yes" already running VM will be used for build
# $gpg_keys_path - path to the directory containing GPG keys for repo signing
# directory have to contain only one file *.public and only one *.private
set -x
# read the name of build scripts directory
export script_dir="$(dirname $(readlink -f $0))"
# load all needed variables
. ${script_dir}/set_build_variables.sh
dist_sfx="$platform"."$platform_version"
export cmake_flags="${cmake_flags} -DPACKAGE=Y -DDISTRIB_SUFFIX=${dist_sfx}"
# prerare VM
export provider=`${mdbci_dir}/mdbci show provider $box --silent 2> /dev/null`
export name="$box-${JOB_NAME}-${BUILD_NUMBER}"
export name=`echo $name | sed "s|/|-|g"`
export platform=`${mdbci_dir}/mdbci show boxinfo --box-name=$box --field='platform' --silent`
export platform_version=`${mdbci_dir}/mdbci show boxinfo --box-name=$box --field='platform_version' --silent`
if [ "${try_already_running}" == "yes" ]; then
export name=${box}
export snapshot_lock_file=$MDBCI_VM_PATH/${name}_snapshot_lock
while [ -f ${snapshot_lock_file} ]
do
echo "snapshot is locked, waiting ..."
sleep 5
done
echo ${JOB_NAME}-${BUILD_NUMBER} > ${snapshot_lock_file}
${mdbci_dir}/mdbci snapshot revert --path-to-nodes $name --snapshot-name clean
if [ $? == 0 ]; then
export already_running="ok"
fi
fi
if [ "$already_running" != "ok" ]; then
eval "cat <<EOF
$(<${script_dir}/templates/build.json.template)
" 2> /dev/null > $MDBCI_VM_PATH/${name}.json
while [ -f ~/vagrant_lock ]
do
sleep 5
done
touch ~/vagrant_lock
echo $JOB_NAME-$BUILD_NUMBER >> ~/vagrant_lock
# destroying existing box
if [ -d "$MDBCI_VM_PATH/${name}" ]; then
cd $MDBCI_VM_PATH/${name}
vagrant destroy -f
cd ${dir}
fi
# starting VM for build
echo "Generating build VM template"
${mdbci_dir}/mdbci --override --template $MDBCI_VM_PATH/$name.json generate $name
echo "starting VM for build"
${mdbci_dir}/mdbci up --attempts=1 $name
if [ $? != 0 ] ; then
echo "Error starting VM"
cd $MDBCI_VM_PATH/${name}
rm ~/vagrant_lock
cd $dir
exit 1
fi
echo "copying public keys to VM"
cp ~/build-scripts/team_keys .
${mdbci_dir}/mdbci public_keys --key team_keys --silent $name
fi
echo "Get VM info"
export sshuser=`${mdbci_dir}/mdbci ssh --command 'whoami' --silent $name/build 2> /dev/null | tr -d '\r'`
export IP=`${mdbci_dir}/mdbci show network $name/build --silent 2> /dev/null`
export sshkey=`${mdbci_dir}/mdbci show keyfile $name/build --silent 2> /dev/null | sed 's/"//g'`
export scpopt="-i $sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ConnectTimeout=120 "
export sshopt="$scpopt $sshuser@$IP"
echo "Release Vagrant lock"
rm ~/vagrant_lock
echo "Starting build"
${script_dir}/remote_build.sh
export build_result=$?
shellcheck `find . | grep "\.sh"` | grep -i "POSIX sh"
if [ $? -eq 0 ] ; then
echo "POSIX sh error are found in the scripts"
# exit 1
fi
${script_dir}/create_remote_repo.sh
${script_dir}/copy_repos.sh
echo "Removing locks and destroying VM"
cd $MDBCI_VM_PATH/$name
if [ "$try_already_running" == "yes" ] ; then
echo "Release lock for already running VM"
rm $snapshot_lock_file
fi
if [[ "$do_not_destroy_vm" != "yes" && "$try_already_running" != "yes" ]] ; then
echo "Destroying VM"
vagrant destroy -f
cd ..
rm -rf $name
rm -rf ${name}.json
rm -rf ${name}_netwotk_config
fi
cd $dir
if [ $build_result -ne 0 ] ; then
echo "Build FAILED!"
exit $build_result
fi
if [ ${run_upgrade_test} == "yes" ] ; then
${script_dir}/upgrade_test.sh
fi

15
BUILD/mdbci/cnf/maxscale.cnf.minimum Normal file
View File

@ -0,0 +1,15 @@
[maxscale]
threads=4
log_warning=1
[CLI]
type=service
router=cli
[CLI Listener]
type=listener
service=CLI
protocol=maxscaled
#address=localhost
socket=default

16
BUILD/mdbci/cnf/maxscale.cnf.minimum.1.4.4 Normal file
View File

@ -0,0 +1,16 @@
[maxscale]
threads=4
log_warning=1
[CLI]
type=service
router=cli
[CLI Listener]
type=listener
service=CLI
protocol=maxscaled
#address=localhost
port=6603

15
BUILD/mdbci/configure_log_dir.sh Normal file
View File

@ -0,0 +1,15 @@
set -x
LOGS_DIR=${logs_dir:-$HOME/LOGS}
echo $JOB_NAME | grep "/"
if [ $? == 0 ] ; then
export job_name_buildID=`echo $JOB_NAME | sed "s|/|-$BUILD_NUMBER/|"`
export logs_publish_dir="${LOGS_DIR}/${job_name_buildID}/"
else
export logs_publish_dir="${LOGS_DIR}/${JOB_NAME}-${BUILD_NUMBER}"
fi
export job_name_buildID=`echo ${JOB_NAME} | sed "s|/|-${BUILD_NUMBER}/|"`
export logs_publish_dir="${LOGS_DIR}/${job_name_buildID}-${BUILD_NUMBER}"
echo "Logs go to ${logs_publish_dir}"
mkdir -p ${logs_publish_dir}

36
BUILD/mdbci/copy_repos.sh Executable file
View File

@ -0,0 +1,36 @@
#!/bin/bash
# Copyies repo from ${unsorted_repo_dir}/$target/$box to
dir=`pwd`
if [ "$box_type" == "RPM" ] ; then
export arch=`ssh $sshopt "arch"`
. ${script_dir}/generate_build_info_path.sh
rm -rf $path_prefix/$platform/$platform_version/$arch/
mkdir -p $path_prefix/$platform/$platform_version/$arch/
cp -r ${unsorted_repo_dir}/$repo_name/$box/* $path_prefix/$platform/$platform_version/$arch/
env > $build_info_path
cd $path_prefix/$platform
ln -s $platform_version "$platform_version"server
ln -s $platform_version "$platform_version"Server
eval "cat <<EOF
$(<${script_dir}/templates/repository-config/rpm.json.template)
" 2> /dev/null > ${path_prefix}/${platform}_${platform_version}.json
echo "copying done"
else
export arch=`ssh $sshopt "dpkg --print-architecture"`
. ${script_dir}/generate_build_info_path.sh
rm -rf $path_prefix/$platform_family/dists/$platform_version/main/binary-"$arch"
rm -rf $path_prefix/$platform_family/dists/$platform_version/main/binary-i386
mkdir -p $path_prefix/$platform_family/
cp -r ${unsorted_repo_dir}/$repo_name/$box/* $path_prefix/$platform_family/
env > $build_info_path
eval "cat <<EOF
$(<${script_dir}/templates/repository-config/deb.json.template)
" 2> /dev/null > ${path_prefix}/${platform}_${platform_version}.json
fi
cd $dir

37
BUILD/mdbci/create_remote_repo.sh Executable file
View File

@ -0,0 +1,37 @@
#!/bin/bash
# Creates RPM or DEB repository for biniries from
# $pre_repo_dir/$target/$box, signs it with keys
# from ${gpg_keys_path} and puts signed repo to
set -x
export work_dir="MaxScale"
echo "creating repository"
echo "cleaning VM"
ssh $sshopt "rm -rf dest; rm -rf src;"
echo " creating dirs on VM"
ssh $sshopt "mkdir -p dest ; mkdir -p src; mkdir gpg_keys"
echo "copying stuff to VM"
scp $scpopt $pre_repo_dir/$target/$box/* $sshuser@$IP:src/
scp $scpopt -r ${gpg_keys_path}/* $sshuser@$IP:./gpg_keys/
ssh $sshopt "key=\`ls ~/gpg_keys/*.public -1\` ; gpg --import \$key"
ssh $sshopt "key=\`ls ~/gpg_keys/*.private -1\` ; gpg --allow-secret-key-import --import \$key"
echo "executing create_repo.sh on VM"
ssh $sshopt "export platform=$platform; export platform_version=$platform_version; ./$work_dir/BUILD/mdbci/create_repo.sh dest/ src/"
if [ $? != 0 ] ; then
echo "Repo creation failed!"
exit 1
fi
echo "cleaning ${unsorted_repo_dir}/$target/$box/"
rm -rf ${unsorted_repo_dir}/$target/$box/*
echo "copying repo from $box"
mkdir -p ${unsorted_repo_dir}/$target/$box
scp $scpopt -r $sshuser@$IP:dest/* ${unsorted_repo_dir}/$target/$box/

97
BUILD/mdbci/create_repo.sh Executable file
View File

@ -0,0 +1,97 @@
#!/bin/bash
set -x
destdir=$1
sourcedir=$2
#rm -rf $destdir
mkdir -p $destdir/
zypper --version
z_res=$?
yum --version
y_res=$?
if [ $z_res -eq 127 ] && [ $y_res -eq 127 ] ; then
# DEB-based system
arch_name=`dpkg --print-architecture`
arch="binary-$arch_name"
cd $destdir
debian_ver=`cat /etc/debian_version`
echo "Debian version: $debian_ver"
dist_name=$platform_version
mkdir -p dists/$dist_name/main/$arch/
cp ~/$sourcedir/* dists/$dist_name/main/$arch/
sudo apt-get update
sudo apt-get install -y dpkg-dev
dpkg-scanpackages dists/$dist_name/main/$arch/ /dev/null | gzip -9c > dists/$dist_name/main/$arch/Packages.gz
gunzip -c dists/$dist_name/main/$arch/Packages.gz > dists/$dist_name/main/$arch/Packages
# echo "Archive: main" > dists/$dist_name/main/$arch/Release
# echo "Suite: main" >> dists/$dist_name/main/$arch/Release
echo "Components: main" >> dists/$dist_name/main/$arch/Release
echo "Codename: $dist_name" >> dists/$dist_name/main/$arch/Release
echo "Origin: MariaDB" >> dists/$dist_name/main/$arch/Release
echo "Label: MariaDB Maxscale repository" >> dists/$dist_name/main/$arch/Release
uname -m | grep "x86_64"
if [ $? -eq 0 ] ; then
# echo "Architectures: amd64 i386" >> dists/$dist_name/main/$arch/Release
mkdir -p dists/$dist_name/main/binary-i386/
dpkg-scanpackages dists/$dist_name/main/binary-i386/ /dev/null | gzip -9c > dists/$dist_name/main/binary-i386/Packages.gz
gunzip -c dists/$dist_name/main/binary-i386/Packages.gz > dists/$dist_name/main/binary-i386/Packages
# else
# echo "Architectures: ppc64el" >> dists/$dist_name/main/$arch/Release
fi
archs=`ls -1 dists/$dist_name/main | sed "s/binary-//" | tr '\n' ' '`
echo "Architectures: $archs" >> dists/$dist_name/main/$arch/Release
echo "Description: MariaDB MaxScale" >> dists/$dist_name/main/$arch/Release
cp dists/$dist_name/main/$arch/Release dists/$dist_name/Release
# cp dists/$dist_name/main/$arch/Packages.gz dists/$dist_name
apt-ftparchive release dists/$dist_name/ >> dists/$dist_name/Release
if [ $? != 0 ] ; then
echo "Repo creation failed!"
exit 1
fi
gpg -abs -o dists/$dist_name/Release.gpg dists/$dist_name/Release
if [ $? != 0 ] ; then
echo "Package signing failed!"
exit 1
fi
else
# RPM-based system
if [ ${y_res} == 0 ]; then
sudo yum install -y createrepo
fi
if [ ${z_res} == 0 ]; then
sudo zypper -n remove patterns-openSUSE-minimal_base-conflicts
sudo zypper -n install createrepo
fi
echo "%_signature gpg" >> ~/.rpmmacros
echo "%_gpg_name MariaDB Maxscale" >> ~/.rpmmacros
# echo "%_gpg_name MariaDBManager" >> ~/.rpmmacros
echo "\r" | setsid rpm --resign $sourcedir/*.rpm
if [ $? != 0 ] ; then
echo "Package signing failed!"
exit 1
fi
cp $sourcedir/* $destdir/
pushd ${destdir} >/dev/null 2>&1
createrepo -d -s sha .
if [ $? != 0 ] ; then
echo "Repo creation failed!"
exit 1
fi
popd >/dev/null 2>&1
gpg --output repomd.xml.key --sign $destdir/repodata/repomd.xml
gpg -a --detach-sign $destdir/repodata/repomd.xml
if [ $? != 0 ] ; then
echo "Package signing failed!"
exit 1
fi
fi

15
BUILD/mdbci/generate_build_info_path.sh Executable file
View File

@ -0,0 +1,15 @@
#! /bin/bash
set -x
export web_prefix=$(echo $path_prefix | sed "s|${repo_path}/||g")
if [ "$box_type" == "RPM" ] ; then
export build_info_file="$platform/$platform_version/$arch/build_info"
else
export build_info_file="$platform_family/dists/$platform_version/main/binary-$arch/build_info"
fi
echo "BUILD_PATH_INFO=$web_prefix/$build_info_file" > $dir/build_info_env_var_$BUILD_ID
export build_info_path=$path_prefix/$build_info_file

1
BUILD/mdbci/local_rep.sh Normal file
View File

@ -0,0 +1 @@
export ci_url="http://192.168.122.1/repository"

79
BUILD/mdbci/remote_build.sh Executable file
View File

@ -0,0 +1,79 @@
#!/bin/bash
# Copyies stuff to VM, run build on VM and copies binaries
# to $pre_repo_dir/$target/$box
set -x
rm -rf $pre_repo_dir/$target/$box
mkdir -p $pre_repo_dir/$target/SRC
mkdir -p $pre_repo_dir/$target/$box
export work_dir="MaxScale"
export orig_image=$box
ssh $sshopt "sudo rm -rf $work_dir"
echo "copying stuff to $image machine"
ssh $sshopt "mkdir -p $work_dir"
rsync -avz --progress --delete -e "ssh $scpopt" ${script_dir}/../../* $sshuser@$IP:./$work_dir/
if [ $? -ne 0 ] ; then
echo "Error copying stuff to $box machine"
exit 2
fi
export install_script="install_build_deps.sh"
if [ "$box_type" == "RPM" ] ; then
build_script="build_rpm_local.sh"
files="*.rpm"
tars="$product_name*.tar.gz"
else
build_script="build_deb_local.sh"
files="../*.deb"
tars="$product_name*.tar.gz"
fi
export remote_build_cmd="export already_running=\"$already_running\"; \
export build_experimental=\"$build_experimental\"; \
export cmake_flags=\"$cmake_flags\"; \
export work_dir=\"$work_dir\"; \
export platform=\"$platform\"; \
export platform_version=\"$platform_version\"; \
export source=\"$source\"; \
export BUILD_TAG=\"$BUILD_TAG\"; \
"
if [ "$already_running" != "ok" ]
then
echo "install packages on $image"
ssh $sshopt "$remote_build_cmd ./MaxScale/BUILD/$install_script"
installres=$?
if [ $installres -ne 0 ]
then
exit $installres
fi
$HOME/mdbci/mdbci snapshot take --path-to-nodes $box --snapshot-name clean
else
echo "already running VM, not installing deps"
fi
echo "run build on $box"
ssh $sshopt "$remote_build_cmd ./MaxScale/BUILD/$build_script"
if [ $? -ne 0 ] ; then
echo "Error build on $box"
exit 4
fi
echo "copying binaries to the '$pre_repo_dir/$target/$box'"
scp $scpopt $sshuser@$IP:$work_dir/$files $pre_repo_dir/$target/$box/
scp $scpopt $sshuser@$IP:$work_dir/$tars $pre_repo_dir/$target/$box/
echo "package building for '$target' for '$platform' '$platform_version' done!"

82
BUILD/mdbci/set_build_variables.sh Normal file
View File

@ -0,0 +1,82 @@
#!/bin/bash
# Defines defaults values for all uninitialized environmental variables
# In case of running from Jenkins all values go from Jenkins parameters
export dir=`pwd`
export MDBCI_VM_PATH=${MDBCI_VM_PATH:-$HOME/vms}
mkdir -p $MDBCI_VM_PATH
echo "MDBCI_VM_PATH=$MDBCI_VM_PATH"
export box=${box:-"centos_7_libvirt"}
echo "box=$box"
# get commit ID
commitID=`git log | head -1 | sed "s/commit //"`
echo "commitID $commitID"
export branch=`git symbolic-ref --short HEAD`
export curr_date=`date '+%Y-%m-%d_%H-%M'`
export source=${source:-"$branch"}
echo "source=$source"
#hack to get rid of Jenkins artifacts
export target=`echo $target | tr -cd "[:print:]" | sed "s/?//g" | sed "s/ //g"`
export target=${target:-"$source-$curr_date"}
echo "target=$target"
export product_name=${product_name:-"maxscale"}
export build_experimental=${build_experimental:-"yes"}
export gpg_keys_path=${gpg_keys_path:-"$HOME/maxscale_gpg_keys/"}
export pre_repo_dir=${pre_repo_dir:-"$HOME/pre-repo/"}
export unsorted_repo_dir=${unsorted_repo_dir:-"$HOME/repo/"}
export box_type="RPM"
echo $box | grep -i ubuntu
if [ $? == 0 ] ; then
export box_type="DEB"
export platform_family="ubuntu"
fi
echo $box | grep -i deb
if [ $? == 0 ] ; then
export box_type="DEB"
export platform_family="debian"
fi
export cmake_flags=${cmake_flags:-"-DBUILD_TESTS=Y -DCMAKE_BUILD_TYPE=Debug -DBUILD_MMMON=Y -DBUILD_AVRO=Y -DBUILD_CDC=Y"}
echo "cmake_flags=$cmake_flags"
export do_not_destroy_vm=${do_not_destroy_vm:-"no"}
export try_already_running=${try_already_running:-"no"}
export JOB_NAME=${JOB_NAME:-"local_build"}
export BUILD_NUMBER=${BUILD_NUMBER:-`date '+%Y%m%d%H%M'`}
export BUILD_TAG=${BUILD_TAG:-jenkins-${JOB_NAME}-${BUILD_NUMBER}}
export mdbci_dir=${mdbci_dir:-"$HOME/mdbci/"}
export repo_name=$target
export repo_path=${repo_path:-$HOME/repository}
export path_prefix="$repo_path/$repo_name/mariadb-$product_name/"
export ci_url=${ci_url:-"http://max-tst-01.mariadb.com/ci-repository/"}
export deb_repo_key=${deb_repo_key:-"135659e928c12247"}
export rpm_repo_key=${rpm_repo_key:-"$ci_url/MariaDBMaxscale-GPG-KEY.public"}
export run_upgrade_test=${run_upgrade_test:-"no"}
export production_url=${production_url:-"https://downloads.mariadb.com/MaxScale/"}

8
BUILD/mdbci/templates/build.json.template Normal file
View File

@ -0,0 +1,8 @@
{
"build" :
{
"hostname" : "default",
"box" : "$box"
}
}

11
BUILD/mdbci/templates/install.json.template Normal file
View File

@ -0,0 +1,11 @@
{
"maxscale" :
{
"hostname" : "maxscale",
"box" : "$box",
"product" : {
"name": "maxscale"
}
}
}

10
BUILD/mdbci/templates/repository-config/deb.json.template Normal file
View File

@ -0,0 +1,10 @@
[
{
"product": "maxscale",
"version": "default",
"repo": "${ci_url}/${web_prefix}/${platform_family} ${platform_version} main",
"repo_key": "$deb_repo_key70E4618A8167EE24",
"platform": "$platform",
"platform_version": "${platform_version}"
}
]

10
BUILD/mdbci/templates/repository-config/rpm.json.template Normal file
View File

@ -0,0 +1,10 @@
[
{
"product": "maxscale",
"version": "default",
"repo": "${ci_url}/$web_prefix/centos/${platform_version}/\$basearch",
"repo_key": "${rpm_repo_key}",
"platform": "$platform",
"platform_version": "${platform_version}"
}
]

138
BUILD/mdbci/upgrade_test.sh Executable file
View File

@ -0,0 +1,138 @@
#!/bin/bash
set -x
# read the name of build scripts directory
export script_dir="$(dirname $(readlink -f $0))"
# load all needed variables
. ${script_dir}/set_build_variables.sh
export maxadmin_command=${maxadmin_command:-"sudo maxadmin show services"}
export old_target=${old_target:-"2.1.9"}
export old_target=`echo $old_target | sed "s/?//g"`
provider=`${mdbci_dir}/mdbci show provider $box --silent 2> /dev/null`
name=$box-${JOB_NAME}-${BUILD_NUMBER}_upgradetest
name=`echo $name | sed "s|/|-|g"`
cp ${script_dir}/install.json.template ${MDBCI_VM_PATH}/$name.json
eval "cat <<EOF
$(<${script_dir}/templates/install.json.template)
" 2> /dev/null > $MDBCI_VM_PATH/${name}.json
while [ -f ~/vagrant_lock ]
do
sleep 5
done
touch ~/vagrant_lock
echo $JOB_NAME-$BUILD_NUMBER >> ~/vagrant_lock
# destroying existing box
if [ -d "install_$box" ]; then
cd $MDBCI_VM_PATH/$name
vagrant destroy -f
cd $dir
fi
${mdbci_dir}/repository-config/generate_all.sh repo.d
${mdbci_dir}/repository-config/maxscale-release.sh $old_target repo.d
# starting VM for build
${mdbci_dir}/mdbci --override --template $MDBCI_VM_PATH/$name.json --repo-dir $dir/repo.d generate $name
${mdbci_dir}/mdbci up $name --attempts=1
if [ $? != 0 ] ; then
if [ $? != 0 ] ; then
echo "Error starting VM"
cd ${MDBCI_VM_PATH}/$name
if [ "x$do_not_destroy_vm" != "xyes" ] ; then
vagrant destroy -f
fi
cd $dir
rm ~/vagrant_lock
exit 1
fi
fi
rm ~/vagrant_lock
# get VM info
export sshuser=`${mdbci_dir}/mdbci ssh --command 'whoami' --silent $name/maxscale 2> /dev/null | tr -d '\r'`
export IP=`${mdbci_dir}/mdbci show network $name/maxscale --silent 2> /dev/null`
export sshkey=`${mdbci_dir}/mdbci show keyfile $name/maxscale --silent 2> /dev/null | sed 's/"//g'`
export scpopt="-i $sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ConnectTimeout=120 "
export sshopt="$scpopt $sshuser@$IP"
old_version=`ssh $sshopt "maxscale --version" `
rm -rf repo.d
${mdbci_dir}/repository-config/generate_all.sh repo.d
${mdbci_dir}/repository-config/maxscale-ci.sh $target repo.d
${mdbci_dir}/mdbci setup_repo --product maxscale --repo-dir $dir/repo.d $name/maxscale
${mdbci_dir}/mdbci install_product --product maxscale $name/maxscale
res=$?
new_version=`ssh $sshopt "maxscale --version" `
echo "old version: '${old_version}', new version: '${new_version}'"
if [ "${old_version}" == "${new_version}" ]; then
echo "Upgrde was not done!"
res=1
fi
export cnf_file=${cnf_file:-"maxscale.cnf.minimum"}
scp $scpopt ${script_dir}/cnf/$cnf_file $sshuser@$IP:~/
. ${script_dir}/configure_log_dir.sh
${mdbci_dir}/mdbci ssh --command 'service --help' $name/maxscale
if [ $? == 0 ] ; then
maxscale_start_cmd="sudo service maxscale start"
else
${mdbci_dir}/mdbci ssh --command 'echo \"/usr/bin/maxscale 2> /dev/null &\" > maxscale_start.sh; echo \"disown\" >> maxscale_start.sh; chmod a+x maxscale_start.sh' $name/maxscale --silent
maxscale_start_cmd="sudo ./maxscale_start.sh 2> /dev/null &"
fi
ssh $sshopt "sudo cp $cnf_file /etc/maxscale.cnf"
ssh $sshopt "$maxscale_start_cmd" &
pid_to_kill=$!
sleep 10
ssh $sshopt $maxadmin_command
if [ $? != 0 ] ; then
echo "Maxadmin executing error"
res=1
fi
maxadmin_out=`ssh $sshopt $maxadmin_command`
echo $maxadmin_out | grep "CLI"
if [ $? != 0 ] ; then
echo "CLI service is not found in maxadmin output"
res=1
fi
echo $maxadmin_out | grep "Started"
if [ $? != 0 ] ; then
echo "'Started' is not found in the CLI service description"
res=1
fi
mkdir -p $logs_publish_dir
scp $scpopt $sshuser@$IP:/var/log/maxscale/* $logs_publish_dir
chmod a+r $logs_publish_dir/*
if [ "x$do_not_destroy_vm" != "xyes" ] ; then
cd $MDBCI_VM_PATH/$name
vagrant destroy -f
cd $dir
fi
kill $pid_to_kill
exit $res

6
BUILD/run_test.sh
View File

@ -1,6 +0,0 @@
cd ~/Maxscale/maxscale-system-test
cmake .
make
ctest -LE HEAVY -VV

206
Documentation/Routers/Binlogrouter.md
View File

@ -13,34 +13,33 @@ replication setup where replication is high-priority.
## Mandatory Router Parameters
The binlogrouter requires the `server`, `user` and `password` parameters. These
should be configured according to the
The binlogrouter requires the `user` and `password` parameters. These should be
configured according to the
[Configuration Guide](../Getting-Started/Configuration-Guide.md#service).
In addition to these two parameters, `router_options` needs to be defined. This
is the main way the binlogrouter is configured and it will be covered in detail
in the next section.
In addition to these two parameters, the `server_id` and `binlogdir` parameters needs to be defined.
**Note:** As of version 2.1 of MaxScale, all of the router options can also be
defined as parameters. The values defined in _router_options_ will have priority
over the parameters.
## Router Parameters
## Router Options
The binlogrouter accepts the following parameters.
Binlogrouter is configured with a comma-separated list of key-value pairs. The
following options should be given as a value to the `router_options` parameter.
**Note:** Earlier versions of MaxScale supported the configuration of the
binlogrouter only via `router_options` (a the comma-separated list of key-value
pairs). As of MaxScale 2.1, all of the router options should be defined as
parameters. The values defined in `router_options` will have priority over the
parameters to support legacy configurations. The use of `router_options` is
deprecated.
### `binlogdir`
This parameter controls the location where MariaDB MaxScale stores the binary
log files. If this parameter is not set to a directory name then MariaDB
MaxScale will store the binlog files in the directory `/var/lib/maxscale/`.
This parameter controls the location where MariaDB MaxScale stores the binary log
files. This is a mandatory parameter.
The _binlogdir_ also contains the _cache_ subdirectory which stores data
retrieved from the master during the slave registration phase. The master.ini
file also resides in the _binlogdir_. This file keeps track of the current
master configuration and it is updated when a `CHANGE MASTER TO` query is
executed.
retrieved from the master during the slave registration phase. The
master.ini file also resides in the _binlogdir_. This file keeps track of
the current master configuration and it is updated when a `CHANGE MASTER
TO` query is executed.
From 2.1 onwards, the 'cache' directory is stored in the same location as other
user credential caches. This means that with the default options, the user
@ -51,17 +50,11 @@ Read the [MySQL Authenticator](../Authenticators/MySQL-Authenticator.md)
documentation for instructions on how to define a custom location for the user
cache.
### `uuid`
This is used to set the unique UUID that the binlog router uses when it connects
to the master server. If no explicit value is given for the UUID in the
configuration file then a UUID will be generated.
### `server_id`
As with UUID, MariaDB MaxScale must have a unique _server_id_. This parameter
configures the value of the _server_id_ that
MariaDB MaxScale will use when connecting to the master.
MariaDB MaxScale must have a unique _server_id_. This parameter configures
the value of the _server_id_ that MariaDB MaxScale will use when
connecting to the master. This is a mandatory parameter.
Older versions of MaxScale allowed the ID to be specified using `server-id`.
This has been deprecated and will be removed in a future release of MariaDB MaxScale.
@ -69,40 +62,49 @@ This has been deprecated and will be removed in a future release of MariaDB MaxS
### `master_id`
The _server_id_ value that MariaDB MaxScale should use to report to the slaves
that connect to MariaDB MaxScale. This may either be the same as the server id
of the real master or can be chosen to be different if the slaves need to be
aware of the proxy layer. The real master server ID will be used if the option
is not set.
that connect to MariaDB MaxScale.
This may either be the same as the server id of the real master or can be
chosen to be different if the slaves need to be aware of the proxy
layer. The real master server ID will be used if the option is not set.
Older versions of MaxScale allowed the ID to be specified using `master-id`.
This has been deprecated and will be removed in a future release of MariaDB MaxScale.
### `uuid`
This is used to set the unique UUID that the binlog router uses when it connects
to the master server. By default the UUID will be generated.
### `master_uuid`
It is a requirement of replication that each slave has a unique UUID value. The
MariaDB MaxScale router will identify itself to the slaves using the UUID of the
real master if this option is not set.
It is a requirement of replication that each server has a unique UUID value. If
this option is not set, binlogrouter will identify itself to the slaves using
the UUID of the real master.
### `master_version`
By default, the router will identify itself to the slaves using the server
version of the real master. This option allows the router to use a custom version string.
version of the real master. This option allows the router to use a custom
version string.
### `master_hostname`
By default, the router will identify itself to the slaves using the
hostname of the real master. This option allows the router to use a custom hostname.
By default, the router will identify itself to the slaves using the hostname of
the real master. This option allows the router to use a custom hostname.
### `slave_hostname`
Since MaxScale 2.1.6 the router can optionally identify itself
to the master using a custom hostname.
The specified hostname can be seen in the master via
`SHOW SLAVE HOSTS` command.
The default is not to send any hostname string during registration.
Since MaxScale 2.1.6 the router can optionally identify itself to the master
using a custom hostname. The specified hostname can be seen in the master via
`SHOW SLAVE HOSTS` command. The default is not to send any hostname string
during registration.
### `user`
*Note:* This is option can only be given to the `router_options` parameter. Use
the `user` parameter of the service instead.
This is the user name that MariaDB MaxScale uses when it connects to the
master. This user name must have the rights required for replication as with any
other user that a slave uses for replication purposes. If the user parameter is
@ -118,9 +120,8 @@ authenticator module. Read the
[MySQL Authenticator](../Authenticators/MySQL-Authenticator.md)
documentation for more details.
The user that is used for replication, either defined using the user= option in
the router options or using the username and password defined of the service
must be granted replication privileges on the database server.
The user that is used for replication must be granted replication privileges on
the database server.
```
CREATE USER 'repl'@'maxscalehost' IDENTIFIED by 'password';
@ -129,32 +130,39 @@ GRANT REPLICATION SLAVE ON *.* TO 'repl'@'maxscalehost';
### `password`
*Note:* This is option can only be given to the `router_options` parameter. Use
the `password` parameter of the service instead.
The password for the user. If the password is not explicitly given then the
password in the service entry will be used. For compatibility with other
username and password definitions within the MariaDB MaxScale configuration file
it is also possible to use the parameter passwd=.
it is also possible to use the parameter `passwd`.
### `heartbeat`
This defines the value of the heartbeat interval in seconds for the connection
to the master. MariaDB MaxScale requests the master to ensure that a binlog
event is sent at least every heartbeat period. If there are no real binlog
events to send the master will sent a special heartbeat event. The default value
for the heartbeat period is every 5 minutes. The current interval value is
to the master. The default value for the heartbeat period is every 5 minutes.
MariaDB MaxScale requests the master to ensure that a binlog event is sent at
least every heartbeat period. If there are no real binlog events to send the
master will sent a special heartbeat event. The current interval value is
reported in the diagnostic output.
### `burstsize`
This parameter is used to define the maximum amount of data that will be sent to
a slave by MariaDB MaxScale when that slave is lagging behind the master. In
this situation the slave is said to be in "catchup mode", this parameter is
a slave by MariaDB MaxScale when that slave is lagging behind the master. The
default value is `1M`.
The burst size can be provided as specified
[here](../Getting-Started/Configuration-Guide.md#sizes), except that IEC binary
prefixes can be used as suffixes only from MaxScale 2.1 onwards. MaxScale 2.0
and earlier only support `burstsize` defined in bytes.
In this situation the slave is said to be in "catchup mode", this parameter is
designed to both prevent flooding of that slave and also to prevent threads
within MariaDB MaxScale spending disproportionate amounts of time with slaves
that are lagging behind the master. The burst size can be provided as specified
[here](../Getting-Started/Configuration-Guide.md#sizes), except that IEC
binary prefixes can be used as suffixes only from MaxScale 2.1 onwards.
The default value is `1M`, which will be used if `burstsize` is not provided in
the router options.
that are lagging behind the master.
### `mariadb10-compatibility`
@ -164,7 +172,7 @@ In earlier versions the parameter was disabled by default.
```
# Example
router_options=mariadb10-compatibility=1
mariadb10-compatibility=1
```
@ -195,20 +203,22 @@ the value match and following binlog events will be sent.
### `transaction_safety`
This parameter is used to enable/disable incomplete transactions detection in
binlog router. When MariaDB MaxScale starts an error message may appear if
current binlog file is corrupted or an incomplete transaction is found. During
normal operations binlog events are not distributed to the slaves until a COMMIT
is seen. The default value is off, set transaction_safety=on to enable the
incomplete transactions detection.
binlog router. The default value is _off_.
When MariaDB MaxScale starts an error message may appear if current binlog file
is corrupted or an incomplete transaction is found. During normal operations
binlog events are not distributed to the slaves until a COMMIT is seen. Set
transaction_safety=on to enable detection of incomplete transactions.
### `send_slave_heartbeat`
This defines whether MariaDB MaxScale sends the heartbeat packet to the slave
when there are no real binlog events to send. The default value
is 'off' and no heartbeat events are sent to slave servers. If value is 'on' the
interval value (requested by the slave during registration) is reported in the
diagnostic output and the packet is send after the time interval without any
event to send.
when there are no real binlog events to send. The default value is 'off' and no
heartbeat events are sent to slave servers.
If value is 'on' the interval value (requested by the slave during registration)
is reported in the diagnostic output and the packet is send after the time
interval without any event to send.
### `semisync`
@ -237,14 +247,14 @@ Master communication.
This parameter sets the maximum length of the certificate authority chain that
will be accepted. Legal values are positive integers. This applies to SSL
connection to master server that could be acivated either by writing options in
master.ini or later via CHANGE MASTER TO. This parameter cannot be modified at
runtime, default is 9.
master.ini or later via a _CHANGE MASTER TO_ command. This parameter cannot be
modified at runtime. The default verification depth is 9.
### `encrypt_binlog`
Whether to encrypt binlog files: the default is Off.
Whether to encrypt binlog files: the default is _off_.
When set to On the binlog files will be encrypted using specified AES algorithm
When set to _on_ the binlog files will be encrypted using specified AES algorithm
and the KEY in the specified key file.
**Note:** binlog encryption must be used while replicating from a MariaDB 10.1
@ -278,11 +288,11 @@ The KEY must have exact 16, 24 or 32 bytes size and the selected algorithm
(aes_ctr or aes_cbc) with 128, 192 or 256 ciphers will be used.
**Note:** the key file has the same format as MariaDB 10.1 server so it's
possible to use an existing key file (not ecncrypted) which could contain
several scheme;keys: only key id with value 1 will be parsed, and if not found
possible to use an existing key file (not encrypted) which could contain several
`scheme;key` values: only key id with value 1 will be parsed, and if not found
an error will be reported.
Example:
Example key file with multiple keys:
```
#
@ -349,42 +359,20 @@ The option sets the time interval for a new connection retry to master server, d
follows.
```
[Replication]
type=service
router=binlogrouter
servers=masterdb
version_string=5.6.17-log
user=maxscale
passwd=Mhu87p2D
router_options=uuid=f12fcb7f-b97b-11e3-bc5e-0401152c4c22,
server_id=3,
user=repl,
password=slavepass,
master_id=32,
heartbeat=30,
binlogdir=/var/binlogs,
transaction_safety=1,
master_version=5.6.19-common,
master_hostname=common_server,
master_uuid=xxx-fff-cccc-common,
mariadb10-compatibility=1,
send_slave_heartbeat=1,
ssl_cert_verification_depth=9,
semisync=1,
encrypt_binlog=1,
encryption_algorithm=aes_ctr,
encryption_key_file=/var/binlogs/enc_key.txt,
mariadb10_master_gtid=Off,
slave_hostname=maxscale-blr-1,
master_retry_count=1000,
connect_retry=60
[Replication]
type=service
router=binlogrouter
servers=masterdb
user=maxscale
passwd=maxpwd
server_id=3
binlogdir=/var/lib/maxscale/
mariadb10-compatibility=1
encrypt_binlog=1
encryption_algorithm=aes_ctr
encryption_key_file=/var/binlogs/enc_key.txt
```
The minimum set of router options that must be given in the configuration are
`server_id` and `master_id` (unless the real master id should be used); default
values may be used for all other options.
## Examples
The [Replication Proxy](../Tutorials/Replication-Proxy-Binlog-Router-Tutorial.md) tutorial will

1178
Documentation/Tutorials/Replication-Proxy-Binlog-Router-Tutorial.md
View File

File diff suppressed because it is too large Load Diff

2
plugins/nagios/check_maxscale_resources.pl
View File

@ -118,7 +118,7 @@ chop($resource_type);
my $resource_match = ucfirst("$resource_type Name");
if ($resource_type eq "listener") {
$resource_match = "Service Name";
$resource_match = "Name";
}
if ($resource_type eq "filter") {
$resource_match = "Filter";

13
server/modules/routing/binlogrouter/blr.c
View File

@ -646,11 +646,6 @@ createInstance(SERVICE *service, char **options)
}
}
}
else
{
MXS_ERROR("%s: Error: No router options supplied for binlogrouter",
service->name);
}
inst->orig_masterid = 0;
inst->mariadb10_gtid_domain = BLR_DEFAULT_GTID_DOMAIN_ID;
@ -873,10 +868,10 @@ createInstance(SERVICE *service, char **options)
{
if (rc == -1)
{
MXS_ERROR("%s: master.ini file not found in %s."
" Master registration cannot be started."
" Configure with CHANGE MASTER TO ...",
inst->service->name, inst->binlogdir);
MXS_WARNING("%s: master.ini file not found in %s."
" Master registration cannot be started."
" Configure with CHANGE MASTER TO ...",
inst->service->name, inst->binlogdir);
}
else
{

1
server/modules/routing/binlogrouter/blr.h
View File

@ -532,6 +532,7 @@ typedef struct router_slave
char *mariadb_gtid; /*< MariaDB 10 Slave connects with GTID */
sqlite3 *gtid_maps; /*< GTID storage client handle, read only*/
MARIADB_GTID_INFO f_info; /*< GTID info for file name prefix */
bool annotate_rows; /*< MariaDB 10 Slave requests ANNOTATE_ROWS */
#if defined(SS_DEBUG)
skygw_chk_t rses_chk_tail;
#endif