diff --git a/BUILD/mdbci/build.sh b/BUILD/mdbci/build.sh new file mode 100755 index 000000000..8fa5c82d4 --- /dev/null +++ b/BUILD/mdbci/build.sh @@ -0,0 +1,138 @@ +#!/bin/bash + +# $box - Vagrant box to be used for build + +# $target - name of repository to put results + +# $cmake_flags - cmake flags to be used in the build + +# $MDBCI_VM_PATH - path to the MDBCI virtual machies directory + +# $source - reference to the point in the source code repository + +# $do_not_destroy_vm - if "yes" VM stays alive after the build + +# $try_already_running - if "yes" already running VM will be used for build + +# $gpg_keys_path - path to the directory containing GPG keys for repo signing +# directory have to contain only one file *.public and only one *.private + +set -x + +# read the name of build scripts directory +export script_dir="$(dirname $(readlink -f $0))" + +# load all needed variables +. ${script_dir}/set_build_variables.sh + +dist_sfx="$platform"."$platform_version" +export cmake_flags="${cmake_flags} -DPACKAGE=Y -DDISTRIB_SUFFIX=${dist_sfx}" + +# prerare VM +export provider=`${mdbci_dir}/mdbci show provider $box --silent 2> /dev/null` +export name="$box-${JOB_NAME}-${BUILD_NUMBER}" +export name=`echo $name | sed "s|/|-|g"` + +export platform=`${mdbci_dir}/mdbci show boxinfo --box-name=$box --field='platform' --silent` +export platform_version=`${mdbci_dir}/mdbci show boxinfo --box-name=$box --field='platform_version' --silent` + + +if [ "${try_already_running}" == "yes" ]; then + export name=${box} + export snapshot_lock_file=$MDBCI_VM_PATH/${name}_snapshot_lock + while [ -f ${snapshot_lock_file} ] + do + echo "snapshot is locked, waiting ..." + sleep 5 + done + echo ${JOB_NAME}-${BUILD_NUMBER} > ${snapshot_lock_file} + ${mdbci_dir}/mdbci snapshot revert --path-to-nodes $name --snapshot-name clean + if [ $? == 0 ]; then + export already_running="ok" + fi +fi + +if [ "$already_running" != "ok" ]; then + + eval "cat < /dev/null > $MDBCI_VM_PATH/${name}.json + + while [ -f ~/vagrant_lock ] + do + sleep 5 + done + touch ~/vagrant_lock + echo $JOB_NAME-$BUILD_NUMBER >> ~/vagrant_lock + + # destroying existing box + if [ -d "$MDBCI_VM_PATH/${name}" ]; then + cd $MDBCI_VM_PATH/${name} + vagrant destroy -f + cd ${dir} + fi + + # starting VM for build + echo "Generating build VM template" + ${mdbci_dir}/mdbci --override --template $MDBCI_VM_PATH/$name.json generate $name + echo "starting VM for build" + ${mdbci_dir}/mdbci up --attempts=1 $name + if [ $? != 0 ] ; then + echo "Error starting VM" + cd $MDBCI_VM_PATH/${name} + rm ~/vagrant_lock + cd $dir + exit 1 + fi + echo "copying public keys to VM" + cp ~/build-scripts/team_keys . + ${mdbci_dir}/mdbci public_keys --key team_keys --silent $name +fi + +echo "Get VM info" +export sshuser=`${mdbci_dir}/mdbci ssh --command 'whoami' --silent $name/build 2> /dev/null | tr -d '\r'` +export IP=`${mdbci_dir}/mdbci show network $name/build --silent 2> /dev/null` +export sshkey=`${mdbci_dir}/mdbci show keyfile $name/build --silent 2> /dev/null | sed 's/"//g'` +export scpopt="-i $sshkey -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ConnectTimeout=120 " +export sshopt="$scpopt $sshuser@$IP" + +echo "Release Vagrant lock" +rm ~/vagrant_lock + +echo "Starting build" +${script_dir}/remote_build.sh +export build_result=$? + +shellcheck `find . | grep "\.sh"` | grep -i "POSIX sh" +if [ $? -eq 0 ] ; then + echo "POSIX sh error are found in the scripts" +# exit 1 +fi + + +${script_dir}/create_remote_repo.sh + +${script_dir}/copy_repos.sh + + +echo "Removing locks and destroying VM" +cd $MDBCI_VM_PATH/$name +if [ "$try_already_running" == "yes" ] ; then + echo "Release lock for already running VM" + rm $snapshot_lock_file +fi +if [[ "$do_not_destroy_vm" != "yes" && "$try_already_running" != "yes" ]] ; then + echo "Destroying VM" + vagrant destroy -f + cd .. + rm -rf $name + rm -rf ${name}.json + rm -rf ${name}_netwotk_config +fi +cd $dir + +if [ $build_result -ne 0 ] ; then + echo "Build FAILED!" + exit $build_result +fi + diff --git a/BUILD/mdbci/copy_repos.sh b/BUILD/mdbci/copy_repos.sh new file mode 100755 index 000000000..84a260546 --- /dev/null +++ b/BUILD/mdbci/copy_repos.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Copyies repo from ${unsorted_repo_dir}/$target/$box to + +dir=`pwd` +if [ "$box_type" == "RPM" ] ; then + export arch=`ssh $sshopt "arch"` + . ${script_dir}/generate_build_info_path.sh + + rm -rf $path_prefix/$platform/$platform_version/$arch/ + mkdir -p $path_prefix/$platform/$platform_version/$arch/ + cp -r ${unsorted_repo_dir}/$repo_name/$box/* $path_prefix/$platform/$platform_version/$arch/ + env > $build_info_path + cd $path_prefix/$platform + ln -s $platform_version "$platform_version"server + ln -s $platform_version "$platform_version"Server + + echo "copying done" +else + export arch=`ssh $sshopt "dpkg --print-architecture"` + . ${script_dir}/generate_build_info_path.sh + rm -rf $path_prefix/$platform_family/dists/$platform_version/main/binary-"$arch" + rm -rf $path_prefix/$platform_family/dists/$platform_version/main/binary-i386 + mkdir -p $path_prefix/$platform_family/ + cp -r ${unsorted_repo_dir}/$repo_name/$box/* $path_prefix/$platform_family/ + env > $build_info_path +fi +cd $dir diff --git a/BUILD/mdbci/create_remote_repo.sh b/BUILD/mdbci/create_remote_repo.sh new file mode 100755 index 000000000..2de76e80f --- /dev/null +++ b/BUILD/mdbci/create_remote_repo.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# Creates RPM or DEB repository for biniries from +# $pre_repo_dir/$target/$box, signs it with keys +# from ${gpg_keys_path} and puts signed repo to + +set -x + +export work_dir="MaxScale" + +echo "creating repository" +echo "cleaning VM" +ssh $sshopt "rm -rf dest; rm -rf src;" + +echo " creating dirs on VM" +ssh $sshopt "mkdir -p dest ; mkdir -p src; mkdir gpg_keys" + +echo "copying stuff to VM" +scp $scpopt $pre_repo_dir/$target/$box/* $sshuser@$IP:src/ + +scp $scpopt -r ${gpg_keys_path}/* $sshuser@$IP:./gpg_keys/ +ssh $sshopt "key=\`ls ~/gpg_keys/*.public -1\` ; gpg --import \$key" +ssh $sshopt "key=\`ls ~/gpg_keys/*.private -1\` ; gpg --allow-secret-key-import --import \$key" + +echo "executing create_repo.sh on VM" +ssh $sshopt "export platform=$platform; export platform_version=$platform_version; $work_dir/BUILD/mdbci/create_repo.sh dest/ src/" +if [ $? != 0 ] ; then + echo "Repo creation failed!" + exit 1 +fi + +echo "cleaning ${unsorted_repo_dir}/$target/$box/" +rm -rf ${unsorted_repo_dir}/$target/$box/* + +echo "copying repo from $box" +mkdir -p ${unsorted_repo_dir}/$target/$box +scp $scpopt -r $sshuser@$IP:dest/* ${unsorted_repo_dir}/$target/$box/ diff --git a/BUILD/mdbci/create_repo.sh b/BUILD/mdbci/create_repo.sh new file mode 100755 index 000000000..8e687adb8 --- /dev/null +++ b/BUILD/mdbci/create_repo.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +set -x + +destdir=$1 +sourcedir=$2 + +#rm -rf $destdir +mkdir -p $destdir/ + +zypper --version +z_res=$? +yum --version +y_res=$? + +if [ $z_res -eq 127 ] && [ $y_res -eq 127 ] ; then +# DEB-based system + arch_name=`dpkg --print-architecture` + arch="binary-$arch_name" + cd $destdir + debian_ver=`cat /etc/debian_version` + echo "Debian version: $debian_ver" + dist_name=$platform_version + + mkdir -p dists/$dist_name/main/$arch/ + + cp ~/$sourcedir/* dists/$dist_name/main/$arch/ + sudo apt-get update + sudo apt-get install -y dpkg-dev + dpkg-scanpackages dists/$dist_name/main/$arch/ /dev/null | gzip -9c > dists/$dist_name/main/$arch/Packages.gz + gunzip -c dists/$dist_name/main/$arch/Packages.gz > dists/$dist_name/main/$arch/Packages +# echo "Archive: main" > dists/$dist_name/main/$arch/Release +# echo "Suite: main" >> dists/$dist_name/main/$arch/Release + echo "Components: main" >> dists/$dist_name/main/$arch/Release + echo "Codename: $dist_name" >> dists/$dist_name/main/$arch/Release + echo "Origin: MariaDB" >> dists/$dist_name/main/$arch/Release + echo "Label: MariaDB Maxscale repository" >> dists/$dist_name/main/$arch/Release + uname -m | grep "x86_64" + if [ $? -eq 0 ] ; then +# echo "Architectures: amd64 i386" >> dists/$dist_name/main/$arch/Release + mkdir -p dists/$dist_name/main/binary-i386/ + dpkg-scanpackages dists/$dist_name/main/binary-i386/ /dev/null | gzip -9c > dists/$dist_name/main/binary-i386/Packages.gz + gunzip -c dists/$dist_name/main/binary-i386/Packages.gz > dists/$dist_name/main/binary-i386/Packages +# else +# echo "Architectures: ppc64el" >> dists/$dist_name/main/$arch/Release + fi + archs=`ls -1 dists/$dist_name/main | sed "s/binary-//" | tr '\n' ' '` + echo "Architectures: $archs" >> dists/$dist_name/main/$arch/Release + echo "Description: MariaDB MaxScale" >> dists/$dist_name/main/$arch/Release + cp dists/$dist_name/main/$arch/Release dists/$dist_name/Release +# cp dists/$dist_name/main/$arch/Packages.gz dists/$dist_name + apt-ftparchive release dists/$dist_name/ >> dists/$dist_name/Release + if [ $? != 0 ] ; then + echo "Repo creation failed!" + exit 1 + fi + + gpg -abs -o dists/$dist_name/Release.gpg dists/$dist_name/Release + if [ $? != 0 ] ; then + echo "Package signing failed!" + exit 1 + fi +else +# RPM-based system + sudo yum install -y createrepo + sudo zypper -n remove patterns-openSUSE-minimal_base-conflicts + sudo zypper -n install createrepo + echo "%_signature gpg" >> ~/.rpmmacros + echo "%_gpg_name MariaDB Maxscale" >> ~/.rpmmacros +# echo "%_gpg_name MariaDBManager" >> ~/.rpmmacros + echo "\r" | setsid rpm --resign $sourcedir/*.rpm + + if [ $? != 0 ] ; then + echo "Package signing failed!" + exit 1 + fi + gpg --output repomd.xml.key --sign $sourcedir/repodata/repomd.xml + cp $sourcedir/* $destdir/ + pushd ${destdir} >/dev/null 2>&1 + createrepo -d -s sha . + if [ $? != 0 ] ; then + echo "Repo creation failed!" + exit 1 + fi + + popd >/dev/null 2>&1 + gpg -a --detach-sign $destdir/repodata/repomd.xml + if [ $? != 0 ] ; then + echo "Package signing failed!" + exit 1 + fi +fi diff --git a/BUILD/mdbci/generate_build_info_path.sh b/BUILD/mdbci/generate_build_info_path.sh new file mode 100755 index 000000000..230986fd1 --- /dev/null +++ b/BUILD/mdbci/generate_build_info_path.sh @@ -0,0 +1,15 @@ +#! /bin/bash + +set -x + +export web_prefix=$(echo $path_prefix | sed "s|${repo_path}/||g") + +if [ "$box_type" == "RPM" ] ; then + export build_info_file="$platform/$platform_version/$arch/build_info" +else + export build_info_file="$platform_family/dists/$platform_version/main/binary-$arch/build_info" +fi + +echo "BUILD_PATH_INFO=$web_prefix/$build_info_file" > $dir/build_info_env_var_$BUILD_ID + +export build_info_path=$path_prefix/$build_info_file diff --git a/BUILD/mdbci/remote_build.sh b/BUILD/mdbci/remote_build.sh new file mode 100755 index 000000000..a91bdbe0b --- /dev/null +++ b/BUILD/mdbci/remote_build.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +# Copyies stuff to VM, run build on VM and copies binaries +# to $pre_repo_dir/$target/$box + +set -x + + +rm -rf $pre_repo_dir/$target/$box +mkdir -p $pre_repo_dir/$target/SRC +mkdir -p $pre_repo_dir/$target/$box + +export work_dir="MaxScale" +export orig_image=$box + +ssh $sshopt "sudo rm -rf $work_dir" +echo "copying stuff to $image machine" +ssh $sshopt "mkdir -p $work_dir" + +rsync -avz --progress --delete -e "ssh $scpopt" ${script_dir}/../../../MaxScale/* $sshuser@$IP:$work_dir/ +if [ $? -ne 0 ] ; then + echo "Error copying stuff to $box machine" + exit 2 +fi + +export install_script="install_build_deps.sh" + +if [ "$box_type" == "RPM" ] ; then + build_script="build_rpm_local.sh" + files="*.rpm" + tars="$product_name*.tar.gz" +else + build_script="build_deb_local.sh" + files="../*.deb" + tars="$product_name*.tar.gz" +fi + +export remote_build_cmd="export already_running=\"$already_running\"; \ + export build_experimental=\"$build_experimental\"; \ + export cmake_flags=\"$cmake_flags\"; \ + export work_dir=\"$work_dir\"; \ + export platform=\"$platform\"; \ + export platform_version=\"$platform_version\"; \ + export source=\"$source\"; \ + export BUILD_TAG=\"$BUILD_TAG\"; \ + " + +if [ "$already_running" != "ok" ] +then + echo "install packages on $image" + ssh $sshopt "$remote_build_cmd ./MaxScale/BUILD/$install_script" + installres=$? + if [ $installres -ne 0 ] + then + exit $installres + fi + + $HOME/mdbci/mdbci snapshot take --path-to-nodes $box --snapshot-name clean + +else + echo "already running VM, not installing deps" +fi + +echo "run build on $box" +ssh $sshopt "$remote_build_cmd ./MaxScale/BUILD/$build_script" +if [ $? -ne 0 ] ; then + echo "Error build on $box" + exit 4 +fi + +echo "copying binaries to the '$pre_repo_dir/$target/$box'" +scp $scpopt $sshuser@$IP:$work_dir/$files $pre_repo_dir/$target/$box/ +scp $scpopt $sshuser@$IP:$work_dir/$tars $pre_repo_dir/$target/$box/ + + +echo "package building for '$target' for '$platform' '$platform_version' done!" + + + diff --git a/BUILD/mdbci/set_build_variables.sh b/BUILD/mdbci/set_build_variables.sh new file mode 100644 index 000000000..95c76332e --- /dev/null +++ b/BUILD/mdbci/set_build_variables.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +# Defines defaults values for all uninitialized environmental variables +# In case of running from Jenkins all values go from Jenkins parameters + +export dir=`pwd` + +export MDBCI_VM_PATH=${MDBCI_VM_PATH:-$HOME/vms} +mkdir -p $MDBCI_VM_PATH +echo "MDBCI_VM_PATH=$MDBCI_VM_PATH" + +export box=${box:-"centos_7_libvirt"} +echo "box=$box" + +# get commit ID +commitID=`git log | head -1 | sed "s/commit //"` +echo "commitID $commitID" + +export branch=`git symbolic-ref --short HEAD` +export curr_date=`date '+%Y-%m-%d_%H-%M'` + +export source=${source:-"$branch"} +echo "source=$source" + +#hack to get rid of Jenkins artifacts +export target=`echo $target | tr -cd "[:print:]" | sed "s/?//g" | sed "s/ //g"` + +export target=${target:-"$source-$curr_date"} +echo "target=$target" + +export product_name=${product_name:-"maxscale"} + +export build_experimental=${build_experimental:-"yes"} + +export gpg_keys_path=${gpg_keys_path:-"$HOME/maxscale_gpg_keys/"} + +export pre_repo_dir=${pre_repo_dir:-"$HOME/pre-repo/"} + +export unsorted_repo_dir=${unsorted_repo_dir:-"$HOME/repo/"} + +export box_type="RPM" +echo $box | grep -i ubuntu +if [ $? == 0 ] ; then + export box_type="DEB" + export platform_family="ubuntu" +fi +echo $box | grep -i deb +if [ $? == 0 ] ; then + export box_type="DEB" + export platform_family="debian" +fi + +export cmake_flags=${cmake_flags:-"-DBUILD_TESTS=Y -DCMAKE_BUILD_TYPE=Debug -DBUILD_MMMON=Y -DBUILD_AVRO=Y -DBUILD_CDC=Y"} +echo "cmake_flags=$cmake_flags" + +export do_not_destroy_vm=${do_not_destroy_vm:-"no"} + +export try_already_running=${try_already_running:-"no"} + +export JOB_NAME=${JOB_NAME:-"local_build"} + +export BUILD_NUMBER=${BUILD_NUMBER:-`date '+%Y%m%d%H%M'`} + +export BUILD_TAG=${BUILD_TAG:-jenkins-${JOB_NAME}-${BUILD_NUMBER}} + +export mdbci_dir=${mdbci_dir:-"$HOME/mdbci/"} + +export repo_name=$target + +export repo_path=${repo_path:-$HOME/repository} + +export path_prefix="$repo_path/$repo_name/mariadb-$product_name/" + diff --git a/BUILD/mdbci/templates/build.json.template b/BUILD/mdbci/templates/build.json.template new file mode 100644 index 000000000..b53ecb321 --- /dev/null +++ b/BUILD/mdbci/templates/build.json.template @@ -0,0 +1,8 @@ +{ + "build" : + { + "hostname" : "default", + "box" : "$box" + } +} + diff --git a/BUILD/run_test.sh b/BUILD/run_test.sh deleted file mode 100644 index 57ea40d3b..000000000 --- a/BUILD/run_test.sh +++ /dev/null @@ -1,6 +0,0 @@ -cd ~/Maxscale/maxscale-system-test - -cmake . -make - -ctest -LE HEAVY -VV