From 4fa53b4cdbd226ddeb79fdeda3a0e3917888d601 Mon Sep 17 00:00:00 2001 From: Adonis Ling Date: Thu, 18 Aug 2022 16:07:28 +0800 Subject: [PATCH] [chore](workflow) Add shellcheck to check shell scripts (#11744) --- .github/actions/action-sh-checker | 1 + .github/workflows/shellcheck.yml | 38 ++ .gitmodules | 3 + .shellcheckrc | 27 + bin/start_be.sh | 166 +++--- bin/start_fe.sh | 164 +++--- bin/stop_be.sh | 63 +-- bin/stop_fe.sh | 63 +-- build-support/check-format.sh | 13 +- build-support/clang-format.sh | 13 +- build-support/shell-check.sh | 322 +++++++++++ build.sh | 498 ++++++++++-------- build_plugin.sh | 108 ++-- contrib/udf/build_udf.sh | 118 +++-- docs/build_help_zip.sh | 25 +- env.sh | 113 ++-- fe_plugins/auditloader/build.sh | 22 +- .../apache_hdfs_broker/bin/start_broker.sh | 85 +-- .../apache_hdfs_broker/bin/stop_broker.sh | 51 +- fs_brokers/apache_hdfs_broker/build.sh | 39 +- gensrc/script/gen_build_version.sh | 67 ++- run-be-ut.sh | 223 ++++---- run-fe-ut.sh | 67 ++- run-regression-test.sh | 119 +++-- samples/insert/shell/insert_utils.sh | 33 +- thirdparty/build-thirdparty.sh | 125 ++--- thirdparty/download-thirdparty.sh | 326 ++++++------ thirdparty/vars.sh | 134 ++--- tools/find_libjvm.sh | 47 +- tools/restore_tablet_tool.sh | 86 +-- 30 files changed, 1843 insertions(+), 1316 deletions(-) create mode 160000 .github/actions/action-sh-checker create mode 100644 .github/workflows/shellcheck.yml create mode 100644 .shellcheckrc create mode 100755 build-support/shell-check.sh diff --git a/.github/actions/action-sh-checker b/.github/actions/action-sh-checker new file mode 160000 index 0000000000..edd0e45ecf --- /dev/null +++ b/.github/actions/action-sh-checker @@ -0,0 +1 @@ +Subproject commit edd0e45ecff35b05f162052b50df50976c1b74fc diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml new file mode 100644 index 0000000000..c500983541 --- /dev/null +++ b/.github/workflows/shellcheck.yml @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: ShellCheck + +on: [push, pull_request] + +jobs: + shellcheck: + name: ShellCheck + runs-on: ubuntu-latest + steps: + - name: Checkout ${{ github.ref }} ( ${{ github.sha }} ) + uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Run ShellCheck + uses: ./.github/actions/action-sh-checker + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + sh_checker_comment: true + sh_checker_exclude: .git .github ^thirdparty/src ^thirdparty/installed ^ui ^docs/node_modules ^tools/ssb-tools ^tools/tpch-tools ^tools/clickbench-tools ^extension ^output ^fs_brokers/apache_hdfs_broker/output diff --git a/.gitmodules b/.gitmodules index 142ebff986..7f8c88c5ef 100644 --- a/.gitmodules +++ b/.gitmodules @@ -19,3 +19,6 @@ [submodule ".github/actions/ccache-action"] path = .github/actions/ccache-action url = https://github.com/hendrikmuhs/ccache-action +[submodule ".github/actions/action-sh-checker"] + path = .github/actions/action-sh-checker + url = https://github.com/luizm/action-sh-checker diff --git a/.shellcheckrc b/.shellcheckrc new file mode 100644 index 0000000000..2946ee0d34 --- /dev/null +++ b/.shellcheckrc @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +source-path=SCRIPTDIR +source-path=SCRIPTDIR/.. +source-path=SCRIPTDIR/../.. +source-path=thirdparty + +external-sources=true + +enable=all +disable=SC2310 +disable=SC2312 diff --git a/bin/start_be.sh b/bin/start_be.sh index 4a26915451..4210a0e1ef 100755 --- a/bin/start_be.sh +++ b/bin/start_be.sh @@ -16,19 +16,17 @@ # specific language governing permissions and limitations # under the License. -curdir=$(dirname "$0") -curdir=$( - cd "$curdir" - pwd -) +set -eo pipefail -OPTS=$(getopt \ - -n $0 \ +curdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" + +OPTS="$(getopt \ + -n "$0" \ -o '' \ -l 'daemon' \ - -- "$@") + -- "$@")" -eval set -- "$OPTS" +eval set -- "${OPTS}" RUN_DAEMON=0 RUN_IN_AWS=0 @@ -53,69 +51,69 @@ while true; do esac done -export DORIS_HOME=$( - cd "$curdir/.." +DORIS_HOME="$( + cd "${curdir}/.." pwd -) +)" +export DORIS_HOME -MAX_MAP_COUNT=`sysctl -n vm.max_map_count` -if [ $MAX_MAP_COUNT -lt 2000000 ]; then +MAX_MAP_COUNT="$(sysctl -n vm.max_map_count)" +if [[ "${MAX_MAP_COUNT}" -lt 2000000 ]]; then echo "Please set vm.max_map_count to be 2000000. sysctl -w vm.max_map_count=2000000" exit 1 fi # add libs to CLASSPATH -for f in $DORIS_HOME/lib/*.jar; do - if [ ! -n "${DORIS_JNI_CLASSPATH_PARAMETER}" ]; then - export DORIS_JNI_CLASSPATH_PARAMETER=$f - else - export DORIS_JNI_CLASSPATH_PARAMETER=$f:${DORIS_JNI_CLASSPATH_PARAMETER} - fi +for f in "${DORIS_HOME}/lib"/*.jar; do + if [[ -z "${DORIS_JNI_CLASSPATH_PARAMETER}" ]]; then + export DORIS_JNI_CLASSPATH_PARAMETER="${f}" + else + export DORIS_JNI_CLASSPATH_PARAMETER="${f}:${DORIS_JNI_CLASSPATH_PARAMETER}" + fi done # DORIS_JNI_CLASSPATH_PARAMETER is used to configure additional jar path to jvm. e.g. -Djava.class.path=$DORIS_HOME/lib/java-udf.jar export DORIS_JNI_CLASSPATH_PARAMETER="-Djava.class.path=${DORIS_JNI_CLASSPATH_PARAMETER}" jdk_version() { local result - local java_cmd=$JAVA_HOME/bin/java + local java_cmd="${JAVA_HOME:-.}/bin/java" local IFS=$'\n' - # remove \r for Cygwin - local lines=$("$java_cmd" -Xms32M -Xmx32M -version 2>&1 | tr '\r' '\n') - if [[ -z $java_cmd ]] - then + + if [[ -z "${java_cmd}" ]]; then result=no_java + return 1 else - for line in $lines; do - if [[ (-z $result) && ($line = *"version \""*) ]] - then - local ver=$(echo $line | sed -e 's/.*version "\(.*\)"\(.*\)/\1/; 1q') - # on macOS, sed doesn't support '?' - if [[ $ver = "1."* ]] - then - result=$(echo $ver | sed -e 's/1\.\([0-9]*\)\(.*\)/\1/; 1q') - else - result=$(echo $ver | sed -e 's/\([0-9]*\)\(.*\)/\1/; 1q') - fi - fi - done + local version + # remove \r for Cygwin + version="$("${java_cmd}" -Xms32M -Xmx32M -version 2>&1 | tr '\r' '\n' | grep version | awk '{print $3}')" + version="${version//\"/}" + if [[ "${version}" =~ ^1\. ]]; then + result="$(echo "${version}" | awk -F '.' '{print $2}')" + else + result="$(echo "${version}" | awk -F '.' '{print $1}')" + fi fi - echo "$result" + echo "${result}" + return 0 } jvm_arch="amd64" -MACHINE_TYPE=$(uname -m) +MACHINE_TYPE="$(uname -m)" if [[ "${MACHINE_TYPE}" == "aarch64" ]]; then jvm_arch="aarch64" fi -java_version=$(jdk_version) -if [[ $java_version -gt 8 ]]; then - export LD_LIBRARY_PATH=$JAVA_HOME/lib/server:$JAVA_HOME/lib:$LD_LIBRARY_PATH +java_version="$( + set -e + jdk_version +)" +if [[ "${java_version}" -gt 8 ]]; then + export LD_LIBRARY_PATH="${JAVA_HOME}/lib/server:${JAVA_HOME}/lib:${LD_LIBRARY_PATH}" # JAVA_HOME is jdk -elif [[ -d "$JAVA_HOME/jre" ]]; then - export LD_LIBRARY_PATH=$JAVA_HOME/jre/lib/$jvm_arch/server:$JAVA_HOME/jre/lib/$jvm_arch:$LD_LIBRARY_PATH +elif [[ -d "${JAVA_HOME}/jre" ]]; then + export LD_LIBRARY_PATH="${JAVA_HOME}/jre/lib/${jvm_arch}/server:${JAVA_HOME}/jre/lib/${jvm_arch}:${LD_LIBRARY_PATH}" # JAVA_HOME is jre else - export LD_LIBRARY_PATH=$JAVA_HOME/lib/$jvm_arch/server:$JAVA_HOME/lib/$jvm_arch:$LD_LIBRARY_PATH + export LD_LIBRARY_PATH="${JAVA_HOME}/lib/${jvm_arch}/server:${JAVA_HOME}/lib/${jvm_arch}:${LD_LIBRARY_PATH}" fi # export env variables from be.conf @@ -123,75 +121,81 @@ fi # UDF_RUNTIME_DIR # LOG_DIR # PID_DIR -export UDF_RUNTIME_DIR=${DORIS_HOME}/lib/udf-runtime -export LOG_DIR=${DORIS_HOME}/log -export PID_DIR=$( - cd "$curdir" +export UDF_RUNTIME_DIR="${DORIS_HOME}/lib/udf-runtime" +export LOG_DIR="${DORIS_HOME}/log" +PID_DIR="$( + cd "${curdir}" pwd -) +)" +export PID_DIR # set odbc conf path -export ODBCSYSINI=$DORIS_HOME/conf +export ODBCSYSINI="${DORIS_HOME}/conf" # support utf8 for oracle database -export NLS_LANG=AMERICAN_AMERICA.AL32UTF8 +export NLS_LANG='AMERICAN_AMERICA.AL32UTF8' #filter known leak for lsan. -export LSAN_OPTIONS=suppressions=${DORIS_HOME}/conf/asan_suppr.conf +export LSAN_OPTIONS="suppressions=${DORIS_HOME}/conf/asan_suppr.conf" -while read line; do - envline=$(echo $line | sed 's/[[:blank:]]*=[[:blank:]]*/=/g' | sed 's/^[[:blank:]]*//g' | egrep "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*=") - envline=$(eval "echo $envline") - if [[ $envline == *"="* ]]; then - eval 'export "$envline"' +while read -r line; do + envline="$(echo "${line}" | + sed 's/[[:blank:]]*=[[:blank:]]*/=/g' | + sed 's/^[[:blank:]]*//g' | + grep -E "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*=" || + true)" + envline="$(eval "echo ${envline}")" + if [[ "${envline}" == *"="* ]]; then + eval 'export "${envline}"' fi -done < $DORIS_HOME/conf/be.conf +done <"${DORIS_HOME}/conf/be.conf" -if [ -e $DORIS_HOME/bin/palo_env.sh ]; then - source $DORIS_HOME/bin/palo_env.sh +if [[ -e "${DORIS_HOME}/bin/palo_env.sh" ]]; then + # shellcheck disable=1091 + source "${DORIS_HOME}/bin/palo_env.sh" fi -if [ ! -d $LOG_DIR ]; then - mkdir -p $LOG_DIR +if [[ ! -d "${LOG_DIR}" ]]; then + mkdir -p "${LOG_DIR}" fi -if [ ! -d $UDF_RUNTIME_DIR ]; then - mkdir -p ${UDF_RUNTIME_DIR} +if [[ ! -d "${UDF_RUNTIME_DIR}" ]]; then + mkdir -p "${UDF_RUNTIME_DIR}" fi -rm -f ${UDF_RUNTIME_DIR}/* +rm -f "${UDF_RUNTIME_DIR}"/* -pidfile=$PID_DIR/be.pid +pidfile="${PID_DIR}/be.pid" -if [ -f $pidfile ]; then - if kill -0 $(cat $pidfile) > /dev/null 2>&1; then - echo "Backend running as process $(cat $pidfile). Stop it first." +if [[ -f "${pidfile}" ]]; then + if kill -0 "$(cat "${pidfile}")" >/dev/null 2>&1; then + echo "Backend running as process $(cat "${pidfile}"). Stop it first." exit 1 else - rm $pidfile + rm "${pidfile}" fi fi -chmod 755 ${DORIS_HOME}/lib/doris_be -echo "start time: "$(date) >> $LOG_DIR/be.out +chmod 755 "${DORIS_HOME}/lib/doris_be" +echo "start time: $(date)" >>"${LOG_DIR}/be.out" -if [ ! -f /bin/limit3 ]; then - LIMIT= +if [[ ! -f '/bin/limit3' ]]; then + LIMIT='' else LIMIT="/bin/limit3 -c 0 -n 65536" fi ## If you are not running in aws cloud, disable this env since https://github.com/aws/aws-sdk-cpp/issues/1410. -if [ ${RUN_IN_AWS} -eq 0 ]; then +if [[ "${RUN_IN_AWS}" -eq 0 ]]; then export AWS_EC2_METADATA_DISABLED=true fi ## set hdfs conf -export LIBHDFS3_CONF=${DORIS_HOME}/conf/hdfs-site.xml +export LIBHDFS3_CONF="${DORIS_HOME}/conf/hdfs-site.xml" -if [ ${RUN_DAEMON} -eq 1 ]; then - nohup $LIMIT ${DORIS_HOME}/lib/doris_be "$@" >> $LOG_DIR/be.out 2>&1 < /dev/null & +if [[ "${RUN_DAEMON}" -eq 1 ]]; then + nohup ${LIMIT:+${LIMIT}} "${DORIS_HOME}/lib/doris_be" "$@" >>"${LOG_DIR}/be.out" 2>&1 &1 < /dev/null + ${LIMIT:+${LIMIT}} "${DORIS_HOME}/lib/doris_be" "$@" 2>&1 /dev/null && pwd)" + +OPTS="$(getopt \ + -n "$0" \ -o '' \ -l 'daemon' \ -l 'helper:' \ -l 'image:' \ -l 'version' \ - -- "$@") + -- "$@")" -eval set -- "$OPTS" +eval set -- "${OPTS}" RUN_DAEMON=0 -HELPER= -IMAGE_PATH= -IMAGE_TOOL= -OPT_VERSION= +HELPER='' +IMAGE_PATH='' +IMAGE_TOOL='' +OPT_VERSION='' while true; do case "$1" in --daemon) @@ -49,12 +47,12 @@ while true; do shift ;; --helper) - HELPER=$2 + HELPER="$2" shift 2 ;; --image) IMAGE_TOOL=1 - IMAGE_PATH=$2 + IMAGE_PATH="$2" shift 2 ;; --) @@ -68,10 +66,11 @@ while true; do esac done -export DORIS_HOME=$( - cd "$curdir/.." +DORIS_HOME="$( + cd "${curdir}/.." pwd -) +)" +export DORIS_HOME # export env variables from fe.conf # @@ -79,31 +78,37 @@ export DORIS_HOME=$( # LOG_DIR # PID_DIR export JAVA_OPTS="-Xmx1024m" -export LOG_DIR="$DORIS_HOME/log" -export PID_DIR=$( - cd "$curdir" +export LOG_DIR="${DORIS_HOME}/log" +PID_DIR="$( + cd "${curdir}" pwd -) +)" +export PID_DIR -while read line; do - envline=$(echo $line | sed 's/[[:blank:]]*=[[:blank:]]*/=/g' | sed 's/^[[:blank:]]*//g' | egrep "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*=") - envline=$(eval "echo $envline") - if [[ $envline == *"="* ]]; then - eval 'export "$envline"' +while read -r line; do + envline="$(echo "${line}" | + sed 's/[[:blank:]]*=[[:blank:]]*/=/g' | + sed 's/^[[:blank:]]*//g' | + grep -E "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*=" || + true)" + envline="$(eval "echo ${envline}")" + if [[ "${envline}" == *"="* ]]; then + eval 'export "${envline}"' fi -done <$DORIS_HOME/conf/fe.conf +done <"${DORIS_HOME}/conf/fe.conf" -if [ -e $DORIS_HOME/bin/palo_env.sh ]; then - source $DORIS_HOME/bin/palo_env.sh +if [[ -e "${DORIS_HOME}/bin/palo_env.sh" ]]; then + # shellcheck disable=1091 + source "${DORIS_HOME}/bin/palo_env.sh" fi -if [ -z "$JAVA_HOME" ]; then - JAVA=$(which java) +if [[ -z "${JAVA_HOME}" ]]; then + JAVA="$(which java)" else - JAVA="$JAVA_HOME/bin/java" + JAVA="${JAVA_HOME}/bin/java" fi -if [ ! -x "$JAVA" ]; then +if [[ ! -x "${JAVA}" ]]; then echo "The JAVA_HOME environment variable is not defined correctly" echo "This environment variable is needed to run this program" echo "NB: JAVA_HOME should point to a JDK not a JRE" @@ -114,80 +119,87 @@ fi # 1.8 => 8, 13.0 => 13 jdk_version() { local result + local java_cmd="${JAVA_HOME:-.}/bin/java" local IFS=$'\n' - # remove \r for Cygwin - local lines=$("$JAVA" -Xms32M -Xmx32M -version 2>&1 | tr '\r' '\n') - for line in $lines; do - if [[ (-z $result) && ($line = *"version \""*) ]]; then - local ver=$(echo $line | sed -e 's/.*version "\(.*\)"\(.*\)/\1/; 1q') - # on macOS, sed doesn't support '?' - if [[ $ver = "1."* ]]; then - result=$(echo $ver | sed -e 's/1\.\([0-9]*\)\(.*\)/\1/; 1q') - else - result=$(echo $ver | sed -e 's/\([0-9]*\)\(.*\)/\1/; 1q') - fi + + if [[ -z "${java_cmd}" ]]; then + result=no_java + return 1 + else + local version + # remove \r for Cygwin + version="$("${java_cmd}" -Xms32M -Xmx32M -version 2>&1 | tr '\r' '\n' | grep version | awk '{print $3}')" + version="${version//\"/}" + if [[ "${version}" =~ ^1\. ]]; then + result="$(echo "${version}" | awk -F '.' '{print $2}')" + else + result="$(echo "${version}" | awk -F '.' '{print $1}')" fi - done - echo "$result" + fi + echo "${result}" + return 0 } # need check and create if the log directory existed before outing message to the log file. -if [ ! -d $LOG_DIR ]; then - mkdir -p $LOG_DIR +if [[ ! -d "${LOG_DIR}" ]]; then + mkdir -p "${LOG_DIR}" fi # check java version and choose correct JAVA_OPTS -java_version=$(jdk_version) -final_java_opt=$JAVA_OPTS -if [ $java_version -gt 8 ]; then - if [ -z "$JAVA_OPTS_FOR_JDK_9" ]; then - echo "JAVA_OPTS_FOR_JDK_9 is not set in fe.conf" >>$LOG_DIR/fe.out +java_version="$( + set -e + jdk_version +)" +final_java_opt="${JAVA_OPTS}" +if [[ "${java_version}" -gt 8 ]]; then + if [[ -z "${JAVA_OPTS_FOR_JDK_9}" ]]; then + echo "JAVA_OPTS_FOR_JDK_9 is not set in fe.conf" >>"${LOG_DIR}/fe.out" exit 1 fi - final_java_opt=$JAVA_OPTS_FOR_JDK_9 + final_java_opt="${JAVA_OPTS_FOR_JDK_9}" fi -echo "using java version $java_version" >>$LOG_DIR/fe.out -echo $final_java_opt >>$LOG_DIR/fe.out +echo "using java version ${java_version}" >>"${LOG_DIR}/fe.out" +echo "${final_java_opt}" >>"${LOG_DIR}/fe.out" # add libs to CLASSPATH -for f in $DORIS_HOME/lib/*.jar; do - CLASSPATH=$f:${CLASSPATH} +for f in "${DORIS_HOME}/lib"/*.jar; do + CLASSPATH="${f}:${CLASSPATH}" done -export CLASSPATH=${CLASSPATH}:${DORIS_HOME}/lib:${DORIS_HOME}/conf +export CLASSPATH="${CLASSPATH}:${DORIS_HOME}/lib:${DORIS_HOME}/conf" -pidfile=$PID_DIR/fe.pid +pidfile="${PID_DIR}/fe.pid" -if [ -f $pidfile ]; then - if kill -0 $(cat $pidfile) >/dev/null 2>&1; then - echo Frontend running as process $(cat $pidfile). Stop it first. +if [[ -f "${pidfile}" ]]; then + if kill -0 "$(cat "${pidfile}")" >/dev/null 2>&1; then + echo "Frontend running as process $(cat "${pidfile}"). Stop it first." exit 1 fi fi -if [ ! -f /bin/limit ]; then - LIMIT= +if [[ ! -f "/bin/limit" ]]; then + LIMIT='' else LIMIT=/bin/limit fi -echo $(date) >>$LOG_DIR/fe.out +date >>"${LOG_DIR}/fe.out" -if [ x"$HELPER" != x"" ]; then +if [[ "${HELPER}" != "" ]]; then # change it to '-helper' to be compatible with code in Frontend - HELPER="-helper $HELPER" + HELPER="-helper ${HELPER}" fi -if [[ ${IMAGE_TOOL} -eq 1 ]]; then - if [ ! -z ${IMAGE_PATH} ]; then - $LIMIT $JAVA $final_java_opt org.apache.doris.PaloFe -i ${IMAGE_PATH} +if [[ "${IMAGE_TOOL}" -eq 1 ]]; then + if [[ -n "${IMAGE_PATH}" ]]; then + ${LIMIT:+${LIMIT}} "${JAVA}" ${final_java_opt:+${final_java_opt}} org.apache.doris.PaloFe -i "${IMAGE_PATH}" else echo "Internal Error. USE IMAGE_TOOL like : ./start_fe.sh --image image_path" fi -elif [[ ${RUN_DAEMON} -eq 1 ]]; then - nohup $LIMIT $JAVA $final_java_opt -XX:OnOutOfMemoryError="kill -9 %p" org.apache.doris.PaloFe ${HELPER} "$@" >>$LOG_DIR/fe.out 2>&1 >"${LOG_DIR}/fe.out" 2>&1 $pidfile +echo $! >"${pidfile}" diff --git a/bin/stop_be.sh b/bin/stop_be.sh index 91b772b8ec..600e1539a0 100755 --- a/bin/stop_be.sh +++ b/bin/stop_be.sh @@ -16,75 +16,68 @@ # specific language governing permissions and limitations # under the License. -curdir=$(dirname "$0") -curdir=$( - cd "$curdir" - pwd -) +set -eo pipefail -export DORIS_HOME=$( - cd "$curdir/.." +curdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" + +DORIS_HOME="$( + cd "${curdir}/.." pwd -) -export PID_DIR=$( - cd "$curdir" +)" +export DORIS_HOME + +PID_DIR="$( + cd "${curdir}" pwd -) +)" +export PID_DIR signum=9 -if [ "x"$1 = "x--grace" ]; then +if [[ "$1" = "--grace" ]]; then signum=15 fi -while read line; do - envline=$(echo $line | sed 's/[[:blank:]]*=[[:blank:]]*/=/g' | sed 's/^[[:blank:]]*//g' | egrep "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*=") - envline=$(eval "echo $envline") - if [[ $envline == *"="* ]]; then - eval 'export "$envline"' - fi -done <$DORIS_HOME/conf/be.conf +pidfile="${PID_DIR}/be.pid" -pidfile=$PID_DIR/be.pid - -if [ -f $pidfile ]; then - pid=$(cat $pidfile) +if [[ -f "${pidfile}" ]]; then + pid="$(cat "${pidfile}")" # check if pid valid - if test -z "$pid"; then + if test -z "${pid}"; then echo "ERROR: invalid pid." exit 1 fi # check if pid process exist - if ! kill -0 $pid 2>&1; then - echo "ERROR: be process $pid does not exist." + if ! kill -0 "${pid}" 2>&1; then + echo "ERROR: be process ${pid} does not exist." exit 1 fi - pidcomm=$(ps -p $pid -o comm=) + pidcomm="$(ps -p "${pid}" -o comm=)" # check if pid process is backend process - if [ "doris_be"x != "$pidcomm"x ]; then + if [[ "doris_be" != "${pidcomm}" ]]; then echo "ERROR: pid process may not be be. " exit 1 fi # kill pid process and check it - if kill -${signum} $pid >/dev/null 2>&1; then + if kill "-${signum}" "${pid}" >/dev/null 2>&1; then while true; do - if kill -0 $pid >/dev/null 2>&1; then - echo "waiting be to stop, pid: $pid" + if kill -0 "${pid}" >/dev/null 2>&1; then + echo "waiting be to stop, pid: ${pid}" sleep 2 else - echo "stop $pidcomm, and remove pid file. " - if [ -f $pidfile ]; then rm $pidfile; fi + echo "stop ${pidcomm}, and remove pid file. " + if [[ -f "${pidfile}" ]]; then rm "${pidfile}"; fi exit 0 fi done else - echo "ERROR: failed to stop $pid" + echo "ERROR: failed to stop ${pid}" exit 1 fi else - echo "ERROR: $pidfile does not exist" + echo "ERROR: ${pidfile} does not exist" exit 1 fi diff --git a/bin/stop_fe.sh b/bin/stop_fe.sh index 8e1ed9c271..d107821f7f 100755 --- a/bin/stop_fe.sh +++ b/bin/stop_fe.sh @@ -16,75 +16,68 @@ # specific language governing permissions and limitations # under the License. -curdir=$(dirname "$0") -curdir=$( - cd "$curdir" - pwd -) +set -eo pipefail -export DORIS_HOME=$( - cd "$curdir/.." +curdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" + +DORIS_HOME="$( + cd "${curdir}/.." pwd -) -export PID_DIR=$( - cd "$curdir" +)" +export DORIS_HOME + +PID_DIR="$( + cd "${curdir}" pwd -) +)" +export PID_DIR signum=9 -if [ "x"$1 = "x--grace" ]; then +if [[ "$1" = "--grace" ]]; then signum=15 fi -while read line; do - envline=$(echo $line | sed 's/[[:blank:]]*=[[:blank:]]*/=/g' | sed 's/^[[:blank:]]*//g' | egrep "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*=") - envline=$(eval "echo $envline") - if [[ $envline == *"="* ]]; then - eval 'export "$envline"' - fi -done <$DORIS_HOME/conf/fe.conf +pidfile="${PID_DIR}/fe.pid" -pidfile=$PID_DIR/fe.pid - -if [ -f $pidfile ]; then - pid=$(cat $pidfile) +if [[ -f "${pidfile}" ]]; then + pid="$(cat "${pidfile}")" # check if pid valid - if test -z "$pid"; then + if test -z "${pid}"; then echo "ERROR: invalid pid." exit 1 fi # check if pid process exist - if ! kill -0 $pid 2>&1; then - echo "ERROR: fe process $pid does not exist." + if ! kill -0 "${pid}" 2>&1; then + echo "ERROR: fe process ${pid} does not exist." exit 1 fi - pidcomm=$(ps -p $pid -o comm=) + pidcomm="$(ps -p "${pid}" -o comm=)" # check if pid process is frontend process - if [ "java"x != "$pidcomm"x ]; then + if [[ "java" != "${pidcomm}" ]]; then echo "ERROR: pid process may not be fe. " exit 1 fi # kill pid process and check it - if kill -${signum} $pid >/dev/null 2>&1; then + if kill "-${signum}" "${pid}" >/dev/null 2>&1; then while true; do - if kill -0 $pid >/dev/null 2>&1; then - echo "waiting fe to stop, pid: $pid" + if kill -0 "${pid}" >/dev/null 2>&1; then + echo "waiting fe to stop, pid: ${pid}" sleep 2 else - echo "stop $pidcomm, and remove pid file. " - if [ -f $pidfile ]; then rm $pidfile; fi + echo "stop ${pidcomm}, and remove pid file. " + if [[ -f "${pidfile}" ]]; then rm "${pidfile}"; fi exit 0 fi done else - echo "ERROR: failed to stop $pid" + echo "ERROR: failed to stop ${pid}" exit 1 fi else - echo "ERROR: $pidfile does not exist" + echo "ERROR: ${pidfile} does not exist" exit 1 fi diff --git a/build-support/check-format.sh b/build-support/check-format.sh index f68a278985..4941673c07 100755 --- a/build-support/check-format.sh +++ b/build-support/check-format.sh @@ -23,11 +23,14 @@ set -eo pipefail -ROOT=`dirname "$0"` -ROOT=`cd "$ROOT"; pwd` +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" -export DORIS_HOME=`cd "${ROOT}/.."; pwd` +DORIS_HOME=$( + cd "${ROOT}/.." + pwd +) +export DORIS_HOME -CLANG_FORMAT=${CLANG_FORMAT_BINARY:=$(which clang-format)} +CLANG_FORMAT="${CLANG_FORMAT_BINARY:=$(which clang-format)}" -python ${DORIS_HOME}/build-support/run_clang_format.py "--clang-format-executable" "${CLANG_FORMAT}" "-r" "--style" "file" "--inplace" "false" "--extensions" "c,h,C,H,cpp,hpp,cc,hh,c++,h++,cxx,hxx" "--exclude" "none" "be/src be/test" +python "${DORIS_HOME}/build-support/run_clang_format.py" "--clang-format-executable" "${CLANG_FORMAT}" "-r" "--style" "file" "--inplace" "false" "--extensions" "c,h,C,H,cpp,hpp,cc,hh,c++,h++,cxx,hxx" "--exclude" "none" "be/src be/test" diff --git a/build-support/clang-format.sh b/build-support/clang-format.sh index 005458372c..9dfeb25167 100755 --- a/build-support/clang-format.sh +++ b/build-support/clang-format.sh @@ -23,11 +23,14 @@ set -eo pipefail -ROOT=`dirname "$0"` -ROOT=`cd "$ROOT"; pwd` +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" -export DORIS_HOME=`cd "${ROOT}/.."; pwd` +DORIS_HOME=$( + cd "${ROOT}/.." + pwd +) +export DORIS_HOME -CLANG_FORMAT=${CLANG_FORMAT_BINARY:=$(which clang-format)} +CLANG_FORMAT="${CLANG_FORMAT_BINARY:=$(which clang-format)}" -python ${DORIS_HOME}/build-support/run_clang_format.py "--clang-format-executable" "${CLANG_FORMAT}" "-r" "--style" "file" "--inplace" "true" "--extensions" "c,h,C,H,cpp,hpp,cc,hh,c++,h++,cxx,hxx" "--exclude" "none" "be/src be/test" +python "${DORIS_HOME}/build-support/run_clang_format.py" "--clang-format-executable" "${CLANG_FORMAT}" "-r" "--style" "file" "--inplace" "true" "--extensions" "c,h,C,H,cpp,hpp,cc,hh,c++,h++,cxx,hxx" "--exclude" "none" "be/src be/test" diff --git a/build-support/shell-check.sh b/build-support/shell-check.sh new file mode 100755 index 0000000000..6f5327f9f6 --- /dev/null +++ b/build-support/shell-check.sh @@ -0,0 +1,322 @@ +#!/bin/bash +# shellcheck disable=2311 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +DORIS_HOME="$( + cd "${SCRIPT_DIR}/.." &>/dev/null + pwd +)" +DORIS_THIRDPARTY="$( + source "${DORIS_HOME}/env.sh" &>/dev/null + echo "${DORIS_THIRDPARTY}" +)" +INSTALL_PATH="${DORIS_THIRDPARTY}/installed/bin" + +SHELLCHECK='' +SHFMT='' + +log() { + local level="${1}" + local message="${2}" + local date + date="$(date +'%Y-%m-%d %H:%M:%S')" + if [[ "${level}" == 'INFO' ]]; then + level="[\033[32;1m ${level} \033[0m]" + elif [[ "${level}" == 'WARNING' ]]; then + level="[\033[33;1m${level}\033[0m]" + elif [[ "${level}" == 'ERROR' ]]; then + level="[\033[31;1m ${level} \033[0m]" + fi + echo -e "${level} ${date} - ${message}" +} + +log_info() { + local message="${1}" + log 'INFO' "${message}" +} + +log_warning() { + local message="${1}" + log 'WARNING' "${message}" +} + +log_error() { + local message="${1}" + log 'ERROR' "${message}" + exit 1 +} + +get_version() { + local program="${1}" + local tool="${program##*/}" + + case "${tool}" in + 'shellcheck') "${program}" --version | sed -n 's/version: \(.*\)/\1/p' ;; + 'shfmt') "${program}" --version ;; + *) ;; + esac +} + +check_tool() { + local tool="${1}" + local version="${2}" + local program + + if program="$(command -v "${tool}" 2>/dev/null)" && [[ ! "$(get_version "${program}")" < "${version}" ]]; then + echo "${program}" + return 0 + fi + + program="${INSTALL_PATH}/${tool}" + if [[ -f "${program}" && ! "$(get_version "${program}")" < "${version}" ]]; then + echo "${program}" + return 0 + fi + + return 255 +} + +get_url() { + local tool="${1}" + local os + local arch + + os="$(uname -s | awk '{ print tolower($0) }')" + arch="$(uname -m | awk '{ print tolower($0) }')" + + if [[ "${tool}" == 'shellcheck' ]]; then + if [[ "${arch}" == 'arm64' ]]; then + arch='aarch64' + fi + echo "https://github.com/koalaman/shellcheck/releases/download/v0.8.0/shellcheck-v0.8.0.${os}.${arch}.tar.xz" + return 0 + elif [[ "${tool}" == 'shfmt' ]]; then + if [[ "${arch}" == 'x86_64' ]]; then + arch='amd64' + elif [[ "${arch}" == 'aarch64' ]]; then + arch='arm64' + fi + echo "https://github.com/mvdan/sh/releases/download/v3.5.1/shfmt_v3.5.1_${os}_${arch}" + return 0 + fi + return 255 +} + +get_md5() { + local tool="${1}" + local os + local arch + + os="$(uname -s | awk '{ print tolower($0) }')" + arch="$(uname -m | awk '{ print tolower($0) }')" + + case "${tool}" in + 'shellcheck') + case "${os}" in + 'linux') + case "${arch}" in + 'x86_64') echo '86ee889b1e771bc8292a7043df4b962a' ;; + 'arm64' | 'aarch64') echo 'a0338c733d1283a51777b27edf9ccc96' ;; + *) ;; + esac + ;; + 'darwin') + case "${arch}" in + 'x86_64') echo 'e744840256a77a1a277e81a3032f7bf4' ;; + *) ;; + esac + ;; + *) ;; + esac + ;; + + 'shfmt') + case "${os}" in + 'linux') + case "${arch}" in + 'x86_64') echo '1d234f204e249bf1c524015ce842b117' ;; + 'arm64' | 'aarch64') echo '5c8494fd257d5cd9db1eef48af967e47' ;; + *) ;; + esac + ;; + 'darwin') + case "${arch}" in + 'x86_64') echo '42f14325207a47f3177c96683c5085a4' ;; + 'arm64' | 'aarch64') echo 'a9c51e5d4aeebfa2c8cc70af3006bc4e' ;; + *) ;; + esac + ;; + *) ;; + esac + ;; + *) ;; + esac +} + +download_tool() { + local url="${1}" + local package="${2}" + local md5="${3}" + log_info "Download ${tool} from ${url}" + + if [[ -f "${package}" && "$(md5sum "${package}" | awk '{print $1}')" == "${md5}" ]]; then + log_info "${tool} has been downloaded succesfully!" + return 0 + fi + + for i in {0..2}; do + if [[ "${i}" -gt 0 ]]; then + log_warning "Retry #${i}..." + fi + if curl --connect-timeout 5 --speed-limit 1000 --speed-time 30 -L "${url}" -o "${package}" && + [[ "$(md5sum "${package}" | awk '{print $1}')" == "${md5}" ]]; then + log_info "${tool} has been downloaded succesfully!" + return 0 + fi + done + return 255 +} + +install_shellcheck() { + local tool="shellcheck" + local url + local package + local md5 + local dir + + SYSTEM_NAME="$(uname -s)" + if [[ "${SYSTEM_NAME}" == 'Darwin' ]] && command -v 'brew' &>/dev/null; then + brew install "${tool}" + else + url="$(get_url "${tool}")" + package="${url##*/}" + md5="$(get_md5 "${tool}")" + download_tool "${url}" "${package}" "${md5}" + + log_info "Set ${tool} up." + dir="$(mktemp -d)" + tar -xf "${package}" -C "${dir}" --strip-component=1 + mkdir -p "${INSTALL_PATH}" + mv "${dir}/${tool}" "${INSTALL_PATH}" + fi +} + +install_shfmt() { + local tool='shfmt' + local url + local package + local md5 + local dir + + SYSTEM_NAME="$(uname -s)" + if [[ "${SYSTEM_NAME}" == 'Darwin' ]] && command -v 'brew' &>/dev/null; then + brew install "${tool}" + else + url="$(get_url "${tool}")" + package="${url##*/}" + md5="$(get_md5 "${tool}")" + download_tool "${url}" "${package}" "${md5}" + + log_info "Set ${tool} up." + mkdir -p "${INSTALL_PATH}" + cp "${package}" "${INSTALL_PATH}/shfmt" + chmod a+x "${INSTALL_PATH}/shfmt" + fi +} + +install_tools_if_neccessary() { + while ! SHELLCHECK="$(check_tool 'shellcheck' '0.8.0')"; do + log_warning 'shellcheck was not found.' + install_shellcheck + done + log_info "shellcheck found: ${SHELLCHECK}" + + while ! SHFMT="$(check_tool 'shfmt' '3.5.1')"; do + log_warning 'shfmt was not found.' + install_shfmt + done + log_info "shfmt found: ${SHFMT}" +} + +find_shell_scripts() { + local path="${1}" + local exclude_patterns + local content + local files=() + content="$(grep 'sh_checker_exclude:' "${DORIS_HOME}/.github/workflows/shellcheck.yml")" + read -r -a exclude_patterns <<<"${content#*sh_checker_exclude: }" + while read -r file; do + local matched=false + for pattern in "${exclude_patterns[@]}"; do + if echo "${file:((${#DORIS_HOME} + 1))}" | grep -E "${pattern}" &>/dev/null; then + matched=true + break + fi + done + if ! "${matched}"; then + files+=("${file}") + fi + done < <(find "${path}" -type f -name '*.sh') + echo "${files[@]}" +} + +run_tool() { + set +e + local tool="${1}" + local opt="${2}" + local status + shift 2 + pushd "${DORIS_HOME}" >/dev/null + log_info "Run tool: ${tool}" + "${tool}" ${opt+${opt}} "${@}" + status="${?}" + popd >/dev/null + return "${status}" +} + +run_shellcheck() { + run_tool "${SHELLCHECK}" '' "${@}" +} + +run_shfmt() { + run_tool "${SHFMT}" '-d' "${@}" +} + +main() { + local files + local shellcheck_result=0 + local shfmt_result=0 + + install_tools_if_neccessary + read -r -a files < <(find_shell_scripts "${DORIS_HOME}") + run_shellcheck "${files[@]}" || shellcheck_result="${?}" + run_shfmt "${files[@]}" || shfmt_result="${?}" + + if [[ "${shellcheck_result}" -ne 0 || "${shfmt_result}" -ne 0 ]]; then + echo + log_error 'Some issues were detected!' + else + log_info 'Success!' + fi +} + +main "${@}" diff --git a/build.sh b/build.sh index ede2eeee50..53cdf4ecd0 100755 --- a/build.sh +++ b/build.sh @@ -27,16 +27,15 @@ set -eo pipefail -ROOT=`dirname "$0"` -ROOT=`cd "$ROOT"; pwd` +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" -export DORIS_HOME=${ROOT} +export DORIS_HOME="${ROOT}" -. ${DORIS_HOME}/env.sh +. "${DORIS_HOME}/env.sh" # Check args usage() { - echo " + echo " Usage: $0 Optional options: [no option] build all components @@ -68,78 +67,76 @@ Usage: $0 USE_AVX2=0 $0 --be build Backend and not using AVX2 instruction. USE_AVX2=0 STRIP_DEBUG_INFO=ON $0 build all and not using AVX2 instruction, and strip the debug info for Backend " - exit 1 + exit 1 } clean_gensrc() { - pushd ${DORIS_HOME}/gensrc + pushd "${DORIS_HOME}/gensrc" make clean - rm -rf ${DORIS_HOME}/fe/fe-common/target - rm -rf ${DORIS_HOME}/fe/fe-core/target + rm -rf "${DORIS_HOME}/fe/fe-common/target" + rm -rf "${DORIS_HOME}/fe/fe-core/target" popd } clean_be() { - pushd ${DORIS_HOME} + pushd "${DORIS_HOME}" # "build.sh --clean" just cleans and exits, however CMAKE_BUILD_DIR is set # while building be. - CMAKE_BUILD_TYPE=${BUILD_TYPE:-Release} - CMAKE_BUILD_DIR=${DORIS_HOME}/be/build_${CMAKE_BUILD_TYPE} + CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}" + CMAKE_BUILD_DIR="${DORIS_HOME}/be/build_${CMAKE_BUILD_TYPE}" - rm -rf $CMAKE_BUILD_DIR - rm -rf ${DORIS_HOME}/be/output/ + rm -rf "${CMAKE_BUILD_DIR}" + rm -rf "${DORIS_HOME}/be/output" popd } clean_fe() { - pushd ${DORIS_HOME}/fe - ${MVN_CMD} clean + pushd "${DORIS_HOME}/fe" + "${MVN_CMD}" clean popd } -OPTS=$(getopt \ - -n $0 \ - -o '' \ - -l 'fe' \ - -l 'be' \ - -l 'broker' \ - -l 'audit' \ - -l 'meta-tool' \ - -l 'spark-dpp' \ - -l 'java-udf' \ - -l 'hive-udf' \ - -l 'clean' \ - -l 'help' \ - -o 'hj:' \ - -- "$@") - -if [ $? != 0 ] ; then +if ! OPTS="$(getopt \ + -n "$0" \ + -o '' \ + -l 'fe' \ + -l 'be' \ + -l 'broker' \ + -l 'audit' \ + -l 'meta-tool' \ + -l 'spark-dpp' \ + -l 'java-udf' \ + -l 'hive-udf' \ + -l 'clean' \ + -l 'help' \ + -o 'hj:' \ + -- "$@")"; then usage fi -eval set -- "$OPTS" +eval set -- "${OPTS}" -PARALLEL=$[$(nproc)/4+1] +PARALLEL="$(($(nproc) / 4 + 1))" BUILD_FE=0 BUILD_BE=0 BUILD_BROKER=0 BUILD_AUDIT=0 -BUILD_META_TOOL=OFF +BUILD_META_TOOL='OFF' BUILD_SPARK_DPP=0 BUILD_JAVA_UDF=0 BUILD_HIVE_UDF=0 CLEAN=0 HELP=0 -PARAMETER_COUNT=$# +PARAMETER_COUNT="$#" PARAMETER_FLAG=0 -if [ $# == 1 ] ; then +if [[ "$#" == 1 ]]; then # default BUILD_FE=1 BUILD_BE=1 BUILD_BROKER=1 BUILD_AUDIT=1 - BUILD_META_TOOL=OFF + BUILD_META_TOOL='OFF' BUILD_SPARK_DPP=1 BUILD_JAVA_UDF=0 # TODO: open it when ready BUILD_HIVE_UDF=1 @@ -147,323 +144,378 @@ if [ $# == 1 ] ; then else while true; do case "$1" in - --fe) BUILD_FE=1 BUILD_SPARK_DPP=1 ; shift ;; - --be) BUILD_BE=1 ; shift ;; - --broker) BUILD_BROKER=1 ; shift ;; - --audit) BUILD_AUDIT=1 ; shift ;; - --meta-tool) BUILD_META_TOOL=ON ; shift ;; - --spark-dpp) BUILD_SPARK_DPP=1 ; shift ;; - --java-udf) BUILD_JAVA_UDF=1 BUILD_FE=1 BUILD_SPARK_DPP=1 ; shift ;; - --hive-udf) BUILD_HIVE_UDF=1 ; shift ;; - --clean) CLEAN=1 ; shift ;; - -h) HELP=1; shift ;; - --help) HELP=1; shift ;; - -j) PARALLEL=$2; PARAMETER_FLAG=1; shift 2 ;; - --) shift ; break ;; - *) echo "Internal error" ; exit 1 ;; + --fe) + BUILD_FE=1 + BUILD_SPARK_DPP=1 + shift + ;; + --be) + BUILD_BE=1 + shift + ;; + --broker) + BUILD_BROKER=1 + shift + ;; + --audit) + BUILD_AUDIT=1 + shift + ;; + --meta-tool) + BUILD_META_TOOL='ON' + shift + ;; + --spark-dpp) + BUILD_SPARK_DPP=1 + shift + ;; + --java-udf) + BUILD_JAVA_UDF=1 + BUILD_FE=1 + BUILD_SPARK_DPP=1 + shift + ;; + --hive-udf) + BUILD_HIVE_UDF=1 + shift + ;; + --clean) + CLEAN=1 + shift + ;; + -h) + HELP=1 + shift + ;; + --help) + HELP=1 + shift + ;; + -j) + PARALLEL="$2" + PARAMETER_FLAG=1 + shift 2 + ;; + --) + shift + break + ;; + *) + echo "Internal error" + exit 1 + ;; esac done #only ./build.sh -j xx then build all - if [[ ${PARAMETER_COUNT} -eq 3 ]] && [[ ${PARAMETER_FLAG} -eq 1 ]];then + if [[ "${PARAMETER_COUNT}" -eq 3 ]] && [[ "${PARAMETER_FLAG}" -eq 1 ]]; then BUILD_FE=1 BUILD_BE=1 BUILD_BROKER=1 BUILD_AUDIT=1 - BUILD_META_TOOL=ON + BUILD_META_TOOL='ON' BUILD_SPARK_DPP=1 BUILD_HIVE_UDF=1 CLEAN=0 fi fi -if [[ ${HELP} -eq 1 ]]; then +if [[ "${HELP}" -eq 1 ]]; then usage exit fi # build thirdparty libraries if necessary -if [[ ! -f ${DORIS_THIRDPARTY}/installed/lib/libbacktrace.a ]]; then +if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/libbacktrace.a" ]]; then echo "Thirdparty libraries need to be build ..." # need remove all installed pkgs because some lib like lz4 will throw error if its lib alreay exists - rm -rf ${DORIS_THIRDPARTY}/installed - ${DORIS_THIRDPARTY}/build-thirdparty.sh -j $PARALLEL + rm -rf "${DORIS_THIRDPARTY}/installed" + "${DORIS_THIRDPARTY}/build-thirdparty.sh" -j "${PARALLEL}" fi -if [ ${CLEAN} -eq 1 -a ${BUILD_BE} -eq 0 -a ${BUILD_FE} -eq 0 -a ${BUILD_SPARK_DPP} -eq 0 ]; then +if [[ "${CLEAN}" -eq 1 && "${BUILD_BE}" -eq 0 && "${BUILD_FE}" -eq 0 && "${BUILD_SPARK_DPP}" -eq 0 ]]; then clean_gensrc clean_be clean_fe exit 0 fi -if [[ -z ${WITH_MYSQL} ]]; then - WITH_MYSQL=OFF +if [[ -z "${WITH_MYSQL}" ]]; then + WITH_MYSQL='OFF' fi -if [[ -z ${GLIBC_COMPATIBILITY} ]]; then - GLIBC_COMPATIBILITY=ON +if [[ -z "${GLIBC_COMPATIBILITY}" ]]; then + GLIBC_COMPATIBILITY='ON' fi -if [[ -z ${USE_AVX2} ]]; then - USE_AVX2=ON +if [[ -z "${USE_AVX2}" ]]; then + USE_AVX2='ON' fi -if [[ -z ${WITH_LZO} ]]; then - WITH_LZO=OFF +if [[ -z "${WITH_LZO}" ]]; then + WITH_LZO='OFF' fi -if [[ -z ${USE_LIBCPP} ]]; then - USE_LIBCPP=OFF +if [[ -z "${USE_LIBCPP}" ]]; then + USE_LIBCPP='OFF' fi -if [[ -z ${STRIP_DEBUG_INFO} ]]; then - STRIP_DEBUG_INFO=OFF +if [[ -z "${STRIP_DEBUG_INFO}" ]]; then + STRIP_DEBUG_INFO='OFF' fi -if [[ -z ${USE_MEM_TRACKER} ]]; then - USE_MEM_TRACKER=ON +if [[ -z "${USE_MEM_TRACKER}" ]]; then + USE_MEM_TRACKER='ON' fi -if [[ -z ${USE_JEMALLOC} ]]; then - USE_JEMALLOC=OFF +if [[ -z "${USE_JEMALLOC}" ]]; then + USE_JEMALLOC='OFF' fi -if [[ -z ${STRICT_MEMORY_USE} ]]; then - STRICT_MEMORY_USE=OFF +if [[ -z "${STRICT_MEMORY_USE}" ]]; then + STRICT_MEMORY_USE='OFF' fi -if [[ -z ${USE_DWARF} ]]; then - USE_DWARF=OFF +if [[ -z "${USE_DWARF}" ]]; then + USE_DWARF='OFF' fi echo "Get params: - BUILD_FE -- $BUILD_FE - BUILD_BE -- $BUILD_BE - BUILD_BROKER -- $BUILD_BROKER - BUILD_AUDIT -- $BUILD_AUDIT - BUILD_META_TOOL -- $BUILD_META_TOOL - BUILD_SPARK_DPP -- $BUILD_SPARK_DPP - BUILD_JAVA_UDF -- $BUILD_JAVA_UDF - BUILD_HIVE_UDF -- $BUILD_HIVE_UDF - PARALLEL -- $PARALLEL - CLEAN -- $CLEAN - WITH_MYSQL -- $WITH_MYSQL - WITH_LZO -- $WITH_LZO - GLIBC_COMPATIBILITY -- $GLIBC_COMPATIBILITY - USE_AVX2 -- $USE_AVX2 - USE_LIBCPP -- $USE_LIBCPP - USE_DWARF -- $USE_DWARF - STRIP_DEBUG_INFO -- $STRIP_DEBUG_INFO - USE_MEM_TRACKER -- $USE_MEM_TRACKER - USE_JEMALLOC -- $USE_JEMALLOC - STRICT_MEMORY_USE -- $STRICT_MEMORY_USE + BUILD_FE -- ${BUILD_FE} + BUILD_BE -- ${BUILD_BE} + BUILD_BROKER -- ${BUILD_BROKER} + BUILD_AUDIT -- ${BUILD_AUDIT} + BUILD_META_TOOL -- ${BUILD_META_TOOL} + BUILD_SPARK_DPP -- ${BUILD_SPARK_DPP} + BUILD_JAVA_UDF -- ${BUILD_JAVA_UDF} + BUILD_HIVE_UDF -- ${BUILD_HIVE_UDF} + PARALLEL -- ${PARALLEL} + CLEAN -- ${CLEAN} + WITH_MYSQL -- ${WITH_MYSQL} + WITH_LZO -- ${WITH_LZO} + GLIBC_COMPATIBILITY -- ${GLIBC_COMPATIBILITY} + USE_AVX2 -- ${USE_AVX2} + USE_LIBCPP -- ${USE_LIBCPP} + USE_DWARF -- ${USE_DWARF} + STRIP_DEBUG_INFO -- ${STRIP_DEBUG_INFO} + USE_MEM_TRACKER -- ${USE_MEM_TRACKER} + USE_JEMALLOC -- ${USE_JEMALLOC} + STRICT_MEMORY_USE -- ${STRICT_MEMORY_USE} " # Clean and build generated code -if [ ${CLEAN} -eq 1 ]; then +if [[ "${CLEAN}" -eq 1 ]]; then clean_gensrc fi echo "Build generated code" -cd ${DORIS_HOME}/gensrc +cd "${DORIS_HOME}/gensrc" # DO NOT using parallel make(-j) for gensrc make # Assesmble FE modules -FE_MODULES= -BUILD_DOCS=OFF +FE_MODULES='' +BUILD_DOCS='OFF' modules=("") -if [ ${BUILD_FE} -eq 1 ]; then +if [[ "${BUILD_FE}" -eq 1 ]]; then modules+=("fe-common") modules+=("fe-core") - BUILD_DOCS=ON + BUILD_DOCS='ON' fi -if [ ${BUILD_SPARK_DPP} -eq 1 ]; then +if [[ "${BUILD_SPARK_DPP}" -eq 1 ]]; then modules+=("fe-common") modules+=("spark-dpp") fi -if [ ${BUILD_JAVA_UDF} -eq 1 ]; then +if [[ "${BUILD_JAVA_UDF}" -eq 1 ]]; then modules+=("java-udf") fi -if [ ${BUILD_HIVE_UDF} -eq 1 ]; then +if [[ "${BUILD_HIVE_UDF}" -eq 1 ]]; then modules+=("fe-common") modules+=("hive-udf") fi -FE_MODULES=$(IFS=, ; echo "${modules[*]}") +FE_MODULES="$( + IFS=',' + echo "${modules[*]}" +)" # Clean and build Backend -if [ ${BUILD_BE} -eq 1 ] ; then - if [ -e ${DORIS_HOME}/gensrc/build/gen_cpp/version.h ]; then rm -f ${DORIS_HOME}/gensrc/build/gen_cpp/version.h ; fi - CMAKE_BUILD_TYPE=${BUILD_TYPE:-Release} +if [[ "${BUILD_BE}" -eq 1 ]]; then + if [[ -e "${DORIS_HOME}/gensrc/build/gen_cpp/version.h" ]]; then + rm -f "${DORIS_HOME}/gensrc/build/gen_cpp/version.h" + fi + CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}" echo "Build Backend: ${CMAKE_BUILD_TYPE}" - CMAKE_BUILD_DIR=${DORIS_HOME}/be/build_${CMAKE_BUILD_TYPE} - if [ ${CLEAN} -eq 1 ]; then + CMAKE_BUILD_DIR="${DORIS_HOME}/be/build_${CMAKE_BUILD_TYPE}" + if [[ "${CLEAN}" -eq 1 ]]; then clean_be fi MAKE_PROGRAM="$(which "${BUILD_SYSTEM}")" echo "-- Make program: ${MAKE_PROGRAM}" echo "-- Use ccache: ${CMAKE_USE_CCACHE}" - echo "-- Extra cxx flags: ${EXTRA_CXX_FLAGS}" + echo "-- Extra cxx flags: ${EXTRA_CXX_FLAGS:-}" - mkdir -p ${CMAKE_BUILD_DIR} - cd ${CMAKE_BUILD_DIR} - ${CMAKE_CMD} -G "${GENERATOR}" \ - -DCMAKE_MAKE_PROGRAM="${MAKE_PROGRAM}" \ - -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ - -DMAKE_TEST=OFF \ - ${CMAKE_USE_CCACHE} \ - -DWITH_MYSQL=${WITH_MYSQL} \ - -DWITH_LZO=${WITH_LZO} \ - -DUSE_LIBCPP=${USE_LIBCPP} \ - -DBUILD_META_TOOL=${BUILD_META_TOOL} \ - -DBUILD_JAVA_UDF=${BUILD_JAVA_UDF} \ - -DSTRIP_DEBUG_INFO=${STRIP_DEBUG_INFO} \ - -DUSE_DWARF=${USE_DWARF} \ - -DUSE_MEM_TRACKER=${USE_MEM_TRACKER} \ - -DUSE_JEMALLOC=${USE_JEMALLOC} \ - -DSTRICT_MEMORY_USE=${STRICT_MEMORY_USE} \ - -DUSE_AVX2=${USE_AVX2} \ - -DGLIBC_COMPATIBILITY=${GLIBC_COMPATIBILITY} \ - -DEXTRA_CXX_FLAGS="${EXTRA_CXX_FLAGS}" \ - ${DORIS_HOME}/be/ - ${BUILD_SYSTEM} -j ${PARALLEL} - ${BUILD_SYSTEM} install - cd ${DORIS_HOME} + mkdir -p "${CMAKE_BUILD_DIR}" + cd "${CMAKE_BUILD_DIR}" + "${CMAKE_CMD}" -G "${GENERATOR}" \ + -DCMAKE_MAKE_PROGRAM="${MAKE_PROGRAM}" \ + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ + -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" \ + -DMAKE_TEST=OFF \ + ${CMAKE_USE_CCACHE:+${CMAKE_USE_CCACHE}} \ + -DWITH_MYSQL="${WITH_MYSQL}" \ + -DWITH_LZO="${WITH_LZO}" \ + -DUSE_LIBCPP="${USE_LIBCPP}" \ + -DBUILD_META_TOOL="${BUILD_META_TOOL}" \ + -DBUILD_JAVA_UDF="${BUILD_JAVA_UDF}" \ + -DSTRIP_DEBUG_INFO="${STRIP_DEBUG_INFO}" \ + -DUSE_DWARF="${USE_DWARF}" \ + -DUSE_MEM_TRACKER="${USE_MEM_TRACKER}" \ + -DUSE_JEMALLOC="${USE_JEMALLOC}" \ + -DSTRICT_MEMORY_USE="${STRICT_MEMORY_USE}" \ + -DUSE_AVX2="${USE_AVX2}" \ + -DGLIBC_COMPATIBILITY="${GLIBC_COMPATIBILITY}" \ + -DEXTRA_CXX_FLAGS="${EXTRA_CXX_FLAGS}" \ + "${DORIS_HOME}/be" + "${BUILD_SYSTEM}" -j "${PARALLEL}" + "${BUILD_SYSTEM}" install + cd "${DORIS_HOME}" fi -if [ "${BUILD_DOCS}" = "ON" ] ; then +if [[ "${BUILD_DOCS}" = "ON" ]]; then # Build docs, should be built before Frontend echo "Build docs" - cd ${DORIS_HOME}/docs + cd "${DORIS_HOME}/docs" ./build_help_zip.sh - cd ${DORIS_HOME} + cd "${DORIS_HOME}" fi function build_ui() { - NPM=npm + NPM='npm' if ! ${NPM} --version; then echo "Error: npm is not found" exit 1 fi - if [[ ! -z ${CUSTOM_NPM_REGISTRY} ]]; then - ${NPM} config set registry ${CUSTOM_NPM_REGISTRY} - npm_reg=`${NPM} get registry` - echo "NPM registry: $npm_reg" + if [[ -n "${CUSTOM_NPM_REGISTRY}" ]]; then + "${NPM}" config set registry "${CUSTOM_NPM_REGISTRY}" + npm_reg="$("${NPM}" get registry)" + echo "NPM registry: ${npm_reg}" fi echo "Build Frontend UI" - ui_dist=${DORIS_HOME}/ui/dist/ - if [[ ! -z ${CUSTOM_UI_DIST} ]]; then - ui_dist=${CUSTOM_UI_DIST} + ui_dist="${DORIS_HOME}/ui/dist" + if [[ -n "${CUSTOM_UI_DIST}" ]]; then + ui_dist="${CUSTOM_UI_DIST}" else - cd ${DORIS_HOME}/ui - ${NPM} install - ${NPM} run build + cd "${DORIS_HOME}/ui" + "${NPM}" install + "${NPM}" run build fi echo "ui dist: ${ui_dist}" - rm -rf ${DORIS_HOME}/fe/fe-core/src/main/resources/static/ - mkdir -p ${DORIS_HOME}/fe/fe-core/src/main/resources/static - cp -r ${ui_dist}/* ${DORIS_HOME}/fe/fe-core/src/main/resources/static + rm -rf "${DORIS_HOME}/fe/fe-core/src/main/resources/static" + mkdir -p "${DORIS_HOME}/fe/fe-core/src/main/resources/static" + cp -r "${ui_dist}"/* "${DORIS_HOME}/fe/fe-core/src/main/resources/static"/ } # FE UI must be built before building FE -if [ ${BUILD_FE} -eq 1 ] ; then +if [[ "${BUILD_FE}" -eq 1 ]]; then build_ui fi # Clean and build Frontend -if [ ${FE_MODULES}x != ""x ]; then - echo "Build Frontend Modules: $FE_MODULES" - cd ${DORIS_HOME}/fe - if [ ${CLEAN} -eq 1 ]; then +if [[ "${FE_MODULES}" != '' ]]; then + echo "Build Frontend Modules: ${FE_MODULES}" + cd "${DORIS_HOME}/fe" + if [[ "${CLEAN}" -eq 1 ]]; then clean_fe fi - ${MVN_CMD} package -pl ${FE_MODULES} -DskipTests - cd ${DORIS_HOME} + "${MVN_CMD}" package -pl ${FE_MODULES:+${FE_MODULES}} -DskipTests + cd "${DORIS_HOME}" fi # Clean and prepare output dir DORIS_OUTPUT=${DORIS_HOME}/output/ -mkdir -p ${DORIS_OUTPUT} +mkdir -p "${DORIS_OUTPUT}" # Copy Frontend and Backend -if [ ${BUILD_FE} -eq 1 ]; then - install -d ${DORIS_OUTPUT}/fe/bin ${DORIS_OUTPUT}/fe/conf \ - ${DORIS_OUTPUT}/fe/webroot/ ${DORIS_OUTPUT}/fe/lib/ +if [[ "${BUILD_FE}" -eq 1 ]]; then + install -d "${DORIS_OUTPUT}/fe/bin" "${DORIS_OUTPUT}/fe/conf" \ + "${DORIS_OUTPUT}/fe/webroot" "${DORIS_OUTPUT}/fe/lib" - cp -r -p ${DORIS_HOME}/bin/*_fe.sh ${DORIS_OUTPUT}/fe/bin/ - cp -r -p ${DORIS_HOME}/conf/fe.conf ${DORIS_OUTPUT}/fe/conf/ - cp -r -p ${DORIS_HOME}/conf/*.xml ${DORIS_OUTPUT}/fe/conf/ - rm -rf ${DORIS_OUTPUT}/fe/lib/* - cp -r -p ${DORIS_HOME}/fe/fe-core/target/lib/* ${DORIS_OUTPUT}/fe/lib/ - rm -f ${DORIS_OUTPUT}/fe/lib/palo-fe.jar - cp -r -p ${DORIS_HOME}/fe/fe-core/target/doris-fe.jar ${DORIS_OUTPUT}/fe/lib/ - cp -r -p ${DORIS_HOME}/docs/build/help-resource.zip ${DORIS_OUTPUT}/fe/lib/ - cp -r -p ${DORIS_HOME}/webroot/static ${DORIS_OUTPUT}/fe/webroot/ + cp -r -p "${DORIS_HOME}/bin"/*_fe.sh "${DORIS_OUTPUT}/fe/bin"/ + cp -r -p "${DORIS_HOME}/conf/fe.conf" "${DORIS_OUTPUT}/fe/conf"/ + cp -r -p "${DORIS_HOME}/conf"/*.xml "${DORIS_OUTPUT}/fe/conf"/ + rm -rf "${DORIS_OUTPUT}/fe/lib"/* + cp -r -p "${DORIS_HOME}/fe/fe-core/target/lib"/* "${DORIS_OUTPUT}/fe/lib"/ + rm -f "${DORIS_OUTPUT}/fe/lib/palo-fe.jar" + cp -r -p "${DORIS_HOME}/fe/fe-core/target/doris-fe.jar" "${DORIS_OUTPUT}/fe/lib"/ + cp -r -p "${DORIS_HOME}/docs/build/help-resource.zip" "${DORIS_OUTPUT}/fe/lib"/ + cp -r -p "${DORIS_HOME}/webroot/static" "${DORIS_OUTPUT}/fe/webroot"/ - cp -r -p ${DORIS_THIRDPARTY}/installed/webroot/* ${DORIS_OUTPUT}/fe/webroot/static/ - mkdir -p ${DORIS_OUTPUT}/fe/log - mkdir -p ${DORIS_OUTPUT}/fe/doris-meta + cp -r -p "${DORIS_THIRDPARTY}/installed/webroot"/* "${DORIS_OUTPUT}/fe/webroot/static"/ + mkdir -p "${DORIS_OUTPUT}/fe/log" + mkdir -p "${DORIS_OUTPUT}/fe/doris-meta" fi -if [ ${BUILD_SPARK_DPP} -eq 1 ]; then - install -d ${DORIS_OUTPUT}/fe/spark-dpp/ - rm -rf ${DORIS_OUTPUT}/fe/spark-dpp/* - cp -r -p ${DORIS_HOME}/fe/spark-dpp/target/spark-dpp-*-jar-with-dependencies.jar ${DORIS_OUTPUT}/fe/spark-dpp/ +if [[ "${BUILD_SPARK_DPP}" -eq 1 ]]; then + install -d "${DORIS_OUTPUT}/fe/spark-dpp" + rm -rf "${DORIS_OUTPUT}/fe/spark-dpp"/* + cp -r -p "${DORIS_HOME}/fe/spark-dpp/target"/spark-dpp-*-jar-with-dependencies.jar "${DORIS_OUTPUT}/fe/spark-dpp"/ fi -if [ ${BUILD_BE} -eq 1 ]; then - install -d ${DORIS_OUTPUT}/be/bin \ - ${DORIS_OUTPUT}/be/conf \ - ${DORIS_OUTPUT}/be/lib/ \ - ${DORIS_OUTPUT}/be/www \ - ${DORIS_OUTPUT}/udf/lib \ - ${DORIS_OUTPUT}/udf/include +if [[ "${BUILD_BE}" -eq 1 ]]; then + install -d "${DORIS_OUTPUT}/be/bin" \ + "${DORIS_OUTPUT}/be/conf" \ + "${DORIS_OUTPUT}/be/lib" \ + "${DORIS_OUTPUT}/be/www" \ + "${DORIS_OUTPUT}/udf/lib" \ + "${DORIS_OUTPUT}/udf/include" + + cp -r -p "${DORIS_HOME}/be/output/bin"/* "${DORIS_OUTPUT}/be/bin"/ + cp -r -p "${DORIS_HOME}/be/output/conf"/* "${DORIS_OUTPUT}/be/conf"/ + cp -r -p "${DORIS_HOME}/be/output/lib/doris_be" "${DORIS_OUTPUT}/be/lib"/ - cp -r -p ${DORIS_HOME}/be/output/bin/* ${DORIS_OUTPUT}/be/bin/ - cp -r -p ${DORIS_HOME}/be/output/conf/* ${DORIS_OUTPUT}/be/conf/ - cp -r -p ${DORIS_HOME}/be/output/lib/doris_be ${DORIS_OUTPUT}/be/lib/ # make a soft link palo_be point to doris_be, for forward compatibility - cd ${DORIS_OUTPUT}/be/lib && rm -f palo_be && ln -s doris_be palo_be && cd - + cd "${DORIS_OUTPUT}/be/lib" + rm -f palo_be + ln -s doris_be palo_be + cd - - if [ "${BUILD_META_TOOL}" = "ON" ]; then - cp -r -p ${DORIS_HOME}/be/output/lib/meta_tool ${DORIS_OUTPUT}/be/lib/ + if [[ "${BUILD_META_TOOL}" = "ON" ]]; then + cp -r -p "${DORIS_HOME}/be/output/lib/meta_tool" "${DORIS_OUTPUT}/be/lib"/ fi - cp -r -p ${DORIS_HOME}/be/output/udf/*.a ${DORIS_OUTPUT}/udf/lib/ - cp -r -p ${DORIS_HOME}/be/output/udf/include/* ${DORIS_OUTPUT}/udf/include/ - cp -r -p ${DORIS_HOME}/webroot/be/* ${DORIS_OUTPUT}/be/www/ - if [ "${STRIP_DEBUG_INFO}" = "ON" ]; then - cp -r -p ${DORIS_HOME}/be/output/lib/debug_info ${DORIS_OUTPUT}/be/lib/ + cp -r -p "${DORIS_HOME}/be/output/udf"/*.a "${DORIS_OUTPUT}/udf/lib"/ + cp -r -p "${DORIS_HOME}/be/output/udf/include"/* "${DORIS_OUTPUT}/udf/include"/ + cp -r -p "${DORIS_HOME}/webroot/be"/* "${DORIS_OUTPUT}/be/www"/ + if [[ "${STRIP_DEBUG_INFO}" = "ON" ]]; then + cp -r -p "${DORIS_HOME}/be/output/lib/debug_info" "${DORIS_OUTPUT}/be/lib"/ fi - java_udf_path=${DORIS_HOME}/fe/java-udf/target/java-udf-jar-with-dependencies.jar - if [ -f ${java_udf_path} ];then - cp ${java_udf_path} ${DORIS_OUTPUT}/be/lib/ + java_udf_path="${DORIS_HOME}/fe/java-udf/target/java-udf-jar-with-dependencies.jar" + if [[ -f "${java_udf_path}" ]]; then + cp "${java_udf_path}" "${DORIS_OUTPUT}/be/lib"/ fi - cp -r -p ${DORIS_THIRDPARTY}/installed/webroot/* ${DORIS_OUTPUT}/be/www/ - mkdir -p ${DORIS_OUTPUT}/be/log - mkdir -p ${DORIS_OUTPUT}/be/storage + cp -r -p "${DORIS_THIRDPARTY}/installed/webroot"/* "${DORIS_OUTPUT}/be/www"/ + mkdir -p "${DORIS_OUTPUT}/be/log" + mkdir -p "${DORIS_OUTPUT}/be/storage" fi -if [ ${BUILD_BROKER} -eq 1 ]; then - install -d ${DORIS_OUTPUT}/apache_hdfs_broker +if [[ "${BUILD_BROKER}" -eq 1 ]]; then + install -d "${DORIS_OUTPUT}/apache_hdfs_broker" - cd ${DORIS_HOME}/fs_brokers/apache_hdfs_broker/ + cd "${DORIS_HOME}/fs_brokers/apache_hdfs_broker" ./build.sh - rm -rf ${DORIS_OUTPUT}/apache_hdfs_broker/* - cp -r -p ${DORIS_HOME}/fs_brokers/apache_hdfs_broker/output/apache_hdfs_broker/* ${DORIS_OUTPUT}/apache_hdfs_broker/ - cd ${DORIS_HOME} + rm -rf "${DORIS_OUTPUT}/apache_hdfs_broker"/* + cp -r -p "${DORIS_HOME}/fs_brokers/apache_hdfs_broker/output/apache_hdfs_broker"/* "${DORIS_OUTPUT}/apache_hdfs_broker"/ + cd "${DORIS_HOME}" fi -if [ ${BUILD_AUDIT} -eq 1 ]; then - install -d ${DORIS_OUTPUT}/audit_loader +if [[ "${BUILD_AUDIT}" -eq 1 ]]; then + install -d "${DORIS_OUTPUT}/audit_loader" - cd ${DORIS_HOME}/fe_plugins/auditloader/ + cd "${DORIS_HOME}/fe_plugins/auditloader" ./build.sh - rm -rf ${DORIS_OUTPUT}/audit_loader/* - cp -r -p ${DORIS_HOME}/fe_plugins/auditloader/output/* ${DORIS_OUTPUT}/audit_loader/ - cd ${DORIS_HOME} + rm -rf "${DORIS_OUTPUT}/audit_loader"/* + cp -r -p "${DORIS_HOME}/fe_plugins/auditloader/output"/* "${DORIS_OUTPUT}/audit_loader"/ + cd "${DORIS_HOME}" fi echo "***************************************" echo "Successfully build Doris" echo "***************************************" -if [[ ! -z ${DORIS_POST_BUILD_HOOK} ]]; then - eval ${DORIS_POST_BUILD_HOOK} +if [[ -n "${DORIS_POST_BUILD_HOOK}" ]]; then + eval "${DORIS_POST_BUILD_HOOK}" fi exit 0 diff --git a/build_plugin.sh b/build_plugin.sh index d3f20ea23a..6b42862cd7 100755 --- a/build_plugin.sh +++ b/build_plugin.sh @@ -18,16 +18,15 @@ set -eo pipefail -ROOT=`dirname "$0"` -ROOT=`cd "$ROOT"; pwd` +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" -export DORIS_HOME=${ROOT} +export DORIS_HOME="${ROOT}" -. ${DORIS_HOME}/env.sh +. "${DORIS_HOME}/env.sh" # Check args usage() { - echo " + echo " Usage: $0 Optional options: --plugin build special plugin @@ -35,81 +34,97 @@ Usage: $0 $0 --plugin xxx build xxx plugin $0 build all plugins " - exit 1 + exit 1 } -OPTS=$(getopt \ - -n $0 \ - -o '' \ - -o 'h' \ - -l 'plugin' \ - -l 'clean' \ - -l 'help' \ - -- "$@") - -if [ $? != 0 ] ; then +if ! OPTS="$(getopt \ + -n "$0" \ + -o '' \ + -o 'h' \ + -l 'plugin' \ + -l 'clean' \ + -l 'help' \ + -- "$@")"; then usage fi -eval set -- "$OPTS" +eval set -- "${OPTS}" ALL_PLUGIN=1 CLEAN=0 -if [ $# == 1 ] ; then +if [[ "$#" == 1 ]]; then # defuat ALL_PLUGIN=1 CLEAN=0 else while true; do case "$1" in - --plugin) ALL_PLUGIN=0 ; shift ;; - --clean) CLEAN=1 ; shift ;; - -h) HELP=1; shift ;; - --help) HELP=1; shift ;; - --) shift ; break ;; - *) echo "Internal error" ; exit 1 ;; + --plugin) + ALL_PLUGIN=0 + shift + ;; + --clean) + CLEAN=1 + shift + ;; + -h) + HELP=1 + shift + ;; + --help) + HELP=1 + shift + ;; + --) + shift + break + ;; + *) + echo "Internal error" + exit 1 + ;; esac done fi -if [[ ${HELP} -eq 1 ]]; then +if [[ "${HELP}" -eq 1 ]]; then usage exit fi echo "Get params: - BUILD_ALL_PLUGIN -- $ALL_PLUGIN - CLEAN -- $CLEAN + BUILD_ALL_PLUGIN -- ${ALL_PLUGIN} + CLEAN -- ${CLEAN} " -cd ${DORIS_HOME} -PLUGIN_MODULE= -if [ ${ALL_PLUGIN} -eq 1 ] ; then - cd ${DORIS_HOME}/fe_plugins - if [ ${CLEAN} -eq 1 ]; then - ${MVN_CMD} clean +cd "${DORIS_HOME}" +PLUGIN_MODULE='' +if [[ "${ALL_PLUGIN}" -eq 1 ]]; then + cd "${DORIS_HOME}/fe_plugins" + if [[ "${CLEAN}" -eq 1 ]]; then + "${MVN_CMD}" clean fi echo "build all plugins" - ${MVN_CMD} package -DskipTests + "${MVN_CMD}" package -DskipTests else - PLUGIN_MODULE=$1 - cd ${DORIS_HOME}/fe_plugins/$PLUGIN_MODULE - if [ ${CLEAN} -eq 1 ]; then - ${MVN_CMD} clean + PLUGIN_MODULE="$1" + cd "${DORIS_HOME}/fe_plugins/${PLUGIN_MODULE}" + if [[ "${CLEAN}" -eq 1 ]]; then + "${MVN_CMD}" clean fi - echo "build plugin $PLUGIN_MODULE" - ${MVN_CMD} package -DskipTests + echo "build plugin ${PLUGIN_MODULE}" + "${MVN_CMD}" package -DskipTests fi -cd ${DORIS_HOME} +cd "${DORIS_HOME}" # Clean and prepare output dir -DORIS_OUTPUT=${DORIS_HOME}/fe_plugins/output/ -mkdir -p ${DORIS_OUTPUT} +DORIS_OUTPUT="${DORIS_HOME}/fe_plugins/output" +mkdir -p "${DORIS_OUTPUT}" -if [ ${ALL_PLUGIN} -eq 1 ] ; then - cp -p ${DORIS_HOME}/fe_plugins/*/target/*.zip ${DORIS_OUTPUT}/ +if [[ "${ALL_PLUGIN}" -eq 1 ]]; then + cp -p "${DORIS_HOME}/fe_plugins"/*/target/*.zip "${DORIS_OUTPUT}"/ else - cp -p ${DORIS_HOME}/fe_plugins/$PLUGIN_MODULE/target/*.zip ${DORIS_OUTPUT}/ + cp -p "${DORIS_HOME}/fe_plugins/${PLUGIN_MODULE}/target"/*.zip "${DORIS_OUTPUT}"/ fi echo "***************************************" @@ -117,4 +132,3 @@ echo "Successfully build Doris FE Plugin" echo "***************************************" exit 0 - diff --git a/contrib/udf/build_udf.sh b/contrib/udf/build_udf.sh index c73d45041c..bd793ee29e 100644 --- a/contrib/udf/build_udf.sh +++ b/contrib/udf/build_udf.sh @@ -17,7 +17,7 @@ # under the License. ############################################################## -# This script is used to compile UDF +# This script is used to compile UDF # Usage: # sh build-udf.sh build udf without clean. # sh build-udf.sh --clean clean previous output and build. @@ -26,20 +26,23 @@ set -eo pipefail -ROOT=`dirname "$0"` -ROOT=`cd "$ROOT"; pwd` +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" -export UDF_HOME=${ROOT} -export DORIS_HOME=$(cd ../..; printf %s "$PWD") -echo ${DORIS_HOME} +export UDF_HOME="${ROOT}" +DORIS_HOME="$( + cd ../.. + printf %s "${PWD}" +)" +export DORIS_HOME +echo "${DORIS_HOME}" -. ${DORIS_HOME}/env.sh +. "${DORIS_HOME}/env.sh" -PARALLEL=$[$(nproc)/4+1] +PARALLEL="$(($(nproc) / 4 + 1))" # Check args usage() { - echo " + echo " Usage: $0 Optional options: --clean clean and build target @@ -48,80 +51,93 @@ Usage: $0 $0 build UDF without clean $0 --clean clean and build UDF " - exit 1 + exit 1 } -OPTS=$(getopt \ - -n $0 \ - -o '' \ - -o 'h' \ - -l 'clean' \ - -l 'help' \ - -- "$@") - -if [ $? != 0 ] ; then +if ! OPTS="$(getopt \ + -n "$0" \ + -o '' \ + -o 'h' \ + -l 'clean' \ + -l 'help' \ + -- "$@")"; then usage fi -eval set -- "$OPTS" +eval set -- "${OPTS}" BUILD_UDF=1 CLEAN=0 HELP=0 -if [ $# == 1 ] ; then +if [[ "$#" == 1 ]]; then # default CLEAN=0 else CLEAN=0 while true; do case "$1" in - --clean) CLEAN=1 ; shift ;; - -h) HELP=1; shift ;; - --help) HELP=1; shift ;; - --) shift ; break ;; - *) echo "Internal error" ; exit 1 ;; + --clean) + CLEAN=1 + shift + ;; + -h) + HELP=1 + shift + ;; + --help) + HELP=1 + shift + ;; + --) + shift + break + ;; + *) + echo "Internal error" + exit 1 + ;; esac done fi -if [[ ${HELP} -eq 1 ]]; then +if [[ "${HELP}" -eq 1 ]]; then usage exit fi echo "Get params: - CLEAN -- $CLEAN + CLEAN -- ${CLEAN} " -cd ${UDF_HOME} +cd "${UDF_HOME}" # Clean and build UDF -if [ ${BUILD_UDF} -eq 1 ] ; then - CMAKE_BUILD_TYPE=${BUILD_TYPE:-Release} +if [[ "${BUILD_UDF}" -eq 1 ]]; then + CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}" echo "Build UDF: ${CMAKE_BUILD_TYPE}" - CMAKE_BUILD_DIR=${UDF_HOME}/build_${CMAKE_BUILD_TYPE} - if [ ${CLEAN} -eq 1 ]; then - rm -rf $CMAKE_BUILD_DIR - rm -rf ${UDF_HOME}/output/ + CMAKE_BUILD_DIR="${UDF_HOME}/build_${CMAKE_BUILD_TYPE}" + if [[ "${CLEAN}" -eq 1 ]]; then + rm -rf "${CMAKE_BUILD_DIR}" + rm -rf "${UDF_HOME}/output" fi - mkdir -p ${CMAKE_BUILD_DIR} - cd ${CMAKE_BUILD_DIR} - ${CMAKE_CMD} -G "${GENERATOR}" -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} ../ - ${BUILD_SYSTEM} -j${PARALLEL} - ${BUILD_SYSTEM} install - cd ${UDF_HOME} + mkdir -p "${CMAKE_BUILD_DIR}" + cd "${CMAKE_BUILD_DIR}" + "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" ../ + "${BUILD_SYSTEM}" -j "${PARALLEL}" + "${BUILD_SYSTEM}" install + cd "${UDF_HOME}" fi # Clean and prepare output dir -DORIS_OUTPUT=${DORIS_HOME}/output/ -mkdir -p ${DORIS_OUTPUT} +DORIS_OUTPUT="${DORIS_HOME}/output" +mkdir -p "${DORIS_OUTPUT}" #Copy UDF -if [ ${BUILD_UDF} -eq 1 ]; then - install -d ${DORIS_OUTPUT}/contrib/udf/lib - for dir in $(ls ${CMAKE_BUILD_DIR}/src) - do - mkdir -p ${DORIS_OUTPUT}/contrib/udf/lib/$dir - cp -r -p ${CMAKE_BUILD_DIR}/src/$dir/*.so ${DORIS_OUTPUT}/contrib/udf/lib/$dir/ +if [[ "${BUILD_UDF}" -eq 1 ]]; then + install -d "${DORIS_OUTPUT}/contrib/udf/lib" + for dir in "${CMAKE_BUILD_DIR}/src"/*; do + dir="$(basename "${dir}")" + mkdir -p "${DORIS_OUTPUT}/contrib/udf/lib/${dir}" + cp -r -p "${CMAKE_BUILD_DIR}/src/${dir}"/*.so "${DORIS_OUTPUT}/contrib/udf/lib/${dir}"/ done fi @@ -129,8 +145,8 @@ echo "***************************************" echo "Successfully build Doris UDF" echo "***************************************" -if [[ ! -z ${DORIS_POST_BUILD_HOOK} ]]; then - eval ${DORIS_POST_BUILD_HOOK} +if [[ -n "${DORIS_POST_BUILD_HOOK}" ]]; then + eval "${DORIS_POST_BUILD_HOOK}" fi exit 0 diff --git a/docs/build_help_zip.sh b/docs/build_help_zip.sh index 08fd107641..b77db9c073 100755 --- a/docs/build_help_zip.sh +++ b/docs/build_help_zip.sh @@ -24,21 +24,18 @@ set -eo pipefail -ROOT=`dirname "$0"` -ROOT=`cd "$ROOT"; pwd` +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" -BUILD_DIR=build -HELP_DIR=contents -HELP_ZIP_FILE=help-resource.zip -SQL_REF_DOC_DIR=zh-CN/docs/sql-manual/ +BUILD_DIR='build' +HELP_DIR='contents' +HELP_ZIP_FILE='help-resource.zip' +SQL_REF_DOC_DIR='zh-CN/docs/sql-manual' -cd $ROOT -rm -rf $BUILD_DIR $HELP_DIR $HELP_ZIP_FILE -mkdir -p $BUILD_DIR $HELP_DIR - -cp -r $SQL_REF_DOC_DIR/* $HELP_DIR/ - -zip -r $HELP_ZIP_FILE $HELP_DIR -mv $HELP_ZIP_FILE $BUILD_DIR/ +cd "${ROOT}" +rm -rf "${BUILD_DIR}" "${HELP_DIR}" "${HELP_ZIP_FILE}" +mkdir -p "${BUILD_DIR}" "${HELP_DIR}" +cp -r "${SQL_REF_DOC_DIR}"/* "${HELP_DIR}"/ +zip -r "${HELP_ZIP_FILE}" "${HELP_DIR}" +mv "${HELP_ZIP_FILE}" "${BUILD_DIR}"/ diff --git a/env.sh b/env.sh index 8ce9ee2984..30c9d977f2 100755 --- a/env.sh +++ b/env.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -18,31 +20,32 @@ # check DORIS_HOME export LC_ALL=C -if [[ -z ${DORIS_HOME} ]]; then +if [[ -z "${DORIS_HOME}" ]]; then echo "Error: DORIS_HOME is not set" exit 1 fi # check OS type -if [[ ! -z "$OSTYPE" ]]; then - if [[ ${OSTYPE} != "linux-gnu" ]] && [[ ${OSTYPE:0:6} != "darwin" ]]; then - echo "Error: Unsupported OS type: $OSTYPE" +if [[ -n "${OSTYPE}" ]]; then + if [[ "${OSTYPE}" != "linux-gnu" ]] && [[ "${OSTYPE:0:6}" != "darwin" ]]; then + echo "Error: Unsupported OS type: ${OSTYPE}" exit 1 fi fi # include custom environment variables -if [[ -f ${DORIS_HOME}/custom_env.sh ]]; then - . ${DORIS_HOME}/custom_env.sh +if [[ -f "${DORIS_HOME}/custom_env.sh" ]]; then + # shellcheck disable=1091 + . "${DORIS_HOME}/custom_env.sh" fi # set DORIS_THIRDPARTY -if [[ -z ${DORIS_THIRDPARTY} ]]; then - export DORIS_THIRDPARTY=${DORIS_HOME}/thirdparty +if [[ -z "${DORIS_THIRDPARTY}" ]]; then + export DORIS_THIRDPARTY="${DORIS_HOME}/thirdparty" fi # check python -export PYTHON=python +export PYTHON='python' if ! ${PYTHON} --version; then export PYTHON=python2.7 if ! ${PYTHON} --version; then @@ -51,114 +54,115 @@ if ! ${PYTHON} --version; then fi fi -if [[ -z ${DORIS_TOOLCHAIN} ]]; then +if [[ -z "${DORIS_TOOLCHAIN}" ]]; then DORIS_TOOLCHAIN=gcc fi if [[ "${DORIS_TOOLCHAIN}" == "gcc" ]]; then # set GCC HOME - if [[ -z ${DORIS_GCC_HOME} ]]; then - DORIS_GCC_HOME=$(dirname $(which gcc))/.. + if [[ -z "${DORIS_GCC_HOME}" ]]; then + DORIS_GCC_HOME="$(dirname "$(which gcc)")"/.. export DORIS_GCC_HOME fi - gcc_ver=$(${DORIS_GCC_HOME}/bin/gcc -dumpfullversion -dumpversion) + gcc_ver="$("${DORIS_GCC_HOME}/bin/gcc" -dumpfullversion -dumpversion)" required_ver="7.3.0" - if [[ ! "$(printf '%s\n' "$required_ver" "$gcc_ver" | sort -V | head -n1)" = "$required_ver" ]]; then + if [[ ! "$(printf '%s\n' "${required_ver}" "${gcc_ver}" | sort -V | head -n1)" = "${required_ver}" ]]; then echo "Error: GCC version (${gcc_ver}) must be greater than or equal to ${required_ver}" exit 1 fi - export CC=${DORIS_GCC_HOME}/bin/gcc - export CXX=${DORIS_GCC_HOME}/bin/g++ - if test -x ${DORIS_GCC_HOME}/bin/ld; then - export DORIS_BIN_UTILS=${DORIS_GCC_HOME}/bin/ + export CC="${DORIS_GCC_HOME}/bin/gcc" + export CXX="${DORIS_GCC_HOME}/bin/g++" + if test -x "${DORIS_GCC_HOME}/bin/ld"; then + export DORIS_BIN_UTILS="${DORIS_GCC_HOME}/bin/" fi elif [[ "${DORIS_TOOLCHAIN}" == "clang" ]]; then # set CLANG HOME - if [[ -z ${DORIS_CLANG_HOME} ]]; then - DORIS_CLANG_HOME=$(dirname $(which clang))/.. + if [[ -z "${DORIS_CLANG_HOME}" ]]; then + DORIS_CLANG_HOME="$(dirname "$(which clang)")"/.. export DORIS_CLANG_HOME fi - clang_ver=$(${DORIS_CLANG_HOME}/bin/clang -dumpfullversion -dumpversion) + clang_ver="$("${DORIS_CLANG_HOME}/bin/clang" -dumpfullversion -dumpversion)" required_ver="13.0.0" - if [[ ! "$(printf '%s\n' "$required_ver" "$clang_ver" | sort -V | head -n1)" = "$required_ver" ]]; then + if [[ ! "$(printf '%s\n' "${required_ver}" "${clang_ver}" | sort -V | head -n1)" = "${required_ver}" ]]; then echo "Error: CLANG version (${clang_ver}) must be greater than or equal to ${required_ver}" exit 1 fi - export CC=${DORIS_CLANG_HOME}/bin/clang - export CXX=${DORIS_CLANG_HOME}/bin/clang++ - if test -x ${DORIS_CLANG_HOME}/bin/ld.lld; then - export DORIS_BIN_UTILS=${DORIS_CLANG_HOME}/bin/ + export CC="${DORIS_CLANG_HOME}/bin/clang" + export CXX="${DORIS_CLANG_HOME}/bin/clang++" + if test -x "${DORIS_CLANG_HOME}/bin/ld.lld"; then + export DORIS_BIN_UTILS="${DORIS_CLANG_HOME}/bin/" fi - if [[ -f ${DORIS_CLANG_HOME}/bin/llvm-symbolizer ]]; then - export ASAN_SYMBOLIZER_PATH=${DORIS_CLANG_HOME}/bin/llvm-symbolizer + if [[ -f "${DORIS_CLANG_HOME}/bin/llvm-symbolizer" ]]; then + export ASAN_SYMBOLIZER_PATH="${DORIS_CLANG_HOME}/bin/llvm-symbolizer" fi else echo "Error: unknown DORIS_TOOLCHAIN=${DORIS_TOOLCHAIN}, currently only 'gcc' and 'clang' are supported" exit 1 fi -if [ -z "$DORIS_BIN_UTILS" ]; then - export DORIS_BIN_UTILS=/usr/bin/ +if [[ -z "${DORIS_BIN_UTILS}" ]]; then + export DORIS_BIN_UTILS='/usr/bin/' fi # export CLANG COMPATIBLE FLAGS -export CLANG_COMPATIBLE_FLAGS=$(echo | ${DORIS_GCC_HOME}/bin/gcc -Wp,-v -xc++ - -fsyntax-only 2>&1 | - grep -E '^\s+/' | awk '{print "-I" $1}' | tr '\n' ' ') - +CLANG_COMPATIBLE_FLAGS="$(echo | "${DORIS_GCC_HOME:-"$(dirname "$(which gcc)")/.."}/bin/gcc" -Wp,-v -xc++ - -fsyntax-only 2>&1 | + grep -E '^\s+/' | awk '{print "-I" $1}' | tr '\n' ' ')" +export CLANG_COMPATIBLE_FLAGS # if is called from build-thirdparty.sh, no need to check these tools -if test -z "${BUILD_THIRDPARTY_WIP}"; then +if test -z "${BUILD_THIRDPARTY_WIP:-}"; then # register keyword is forbidden to use in C++17 # the C++ code generated by flex that remove register keyword after version 2.6.0 # so we need check flex version here to avoid compilation failed - flex_ver=$(flex --version | awk '{print $2}') + flex_ver="$(flex --version | awk '{print $2}')" required_ver="2.6.0" - if [[ ! "$(printf '%s\n' "$required_ver" "$flex_ver" | sort -V | head -n1)" = "$required_ver" ]]; then + if [[ ! "$(printf '%s\n' "${required_ver}" "${flex_ver}" | sort -V | head -n1)" = "${required_ver}" ]]; then echo "Error: flex version (${flex_ver}) must be greater than or equal to ${required_ver}" exit 1 fi # check java home - if [ -z "$JAVA_HOME" ]; then - export JAVA=$(which java) - JAVAP=$(which javap) + if [[ -z "${JAVA_HOME}" ]]; then + JAVA="$(which java)" + JAVAP="$(which javap)" else - export JAVA="${JAVA_HOME}/bin/java" + JAVA="${JAVA_HOME}/bin/java" JAVAP="${JAVA_HOME}/bin/javap" fi + export JAVA - if [ ! -x "$JAVA" ]; then + if [[ ! -x "${JAVA}" ]]; then echo "The JAVA_HOME environment variable is not defined correctly" echo "This environment variable is needed to run this program" echo "NB: JAVA_HOME should point to a JDK not a JRE" exit 1 fi - JAVA_VER=$(${JAVAP} -verbose java.lang.String | grep "major version" | cut -d " " -f5) - if [[ $JAVA_VER -lt 52 ]]; then + JAVA_VER="$("${JAVAP}" -verbose java.lang.String | grep "major version" | cut -d " " -f5)" + if [[ "${JAVA_VER}" -lt 52 ]]; then echo "Error: require JAVA with JDK version at least 1.8" exit 1 fi # check maven - MVN_CMD=mvn - if [[ ! -z ${CUSTOM_MVN} ]]; then - MVN_CMD=${CUSTOM_MVN} + MVN_CMD='mvn' + if [[ -n "${CUSTOM_MVN}" ]]; then + MVN_CMD="${CUSTOM_MVN}" fi - if ! ${MVN_CMD} --version; then + if ! "${MVN_CMD}" --version; then echo "Error: mvn is not found" exit 1 fi export MVN_CMD fi -CMAKE_CMD=cmake -if [[ ! -z ${CUSTOM_CMAKE} ]]; then - CMAKE_CMD=${CUSTOM_CMAKE} +CMAKE_CMD='cmake' +if [[ -n "${CUSTOM_CMAKE}" ]]; then + CMAKE_CMD="${CUSTOM_CMAKE}" fi -if ! ${CMAKE_CMD} --version; then +if ! "${CMAKE_CMD}" --version; then echo "Error: cmake is not found" exit 1 fi @@ -166,16 +170,17 @@ export CMAKE_CMD GENERATOR="Unix Makefiles" BUILD_SYSTEM="make" -if ninja --version 2> /dev/null; then +if ninja --version 2>/dev/null; then GENERATOR="Ninja" BUILD_SYSTEM="ninja" fi -if ccache --version > /dev/null; then +if ccache --version >/dev/null; then + # shellcheck disable=2034 CMAKE_USE_CCACHE="-DCMAKE_CXX_COMPILER_LAUNCHER=ccache" fi export GENERATOR export BUILD_SYSTEM -export PKG_CONFIG_PATH=${DORIS_HOME}/thirdparty/installed/lib64/pkgconfig:$PKG_CONFIG_PATH +export PKG_CONFIG_PATH="${DORIS_HOME}/thirdparty/installed/lib64/pkgconfig:${PKG_CONFIG_PATH}" diff --git a/fe_plugins/auditloader/build.sh b/fe_plugins/auditloader/build.sh index ce3db1fc96..051fd0bc85 100755 --- a/fe_plugins/auditloader/build.sh +++ b/fe_plugins/auditloader/build.sh @@ -16,25 +16,23 @@ # specific language governing permissions and limitations # under the License. -set -e +set -eo pipefail -ROOT=`dirname "$0"` -ROOT=`cd "$ROOT"; pwd` +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" -export DORIS_HOME=${ROOT}/../.. +export DORIS_HOME="${ROOT}/../.." -. ${DORIS_HOME}/env.sh +. "${DORIS_HOME}/env.sh" -export AUDITLOADER_HOME=$ROOT +export AUDITLOADER_HOME="${ROOT}" - -$MVN_CMD clean package -DskipTests +"${MVN_CMD}" clean package -DskipTests echo "Install auditloader..." -AUDITLOADER_OUTPUT=${AUDITLOADER_HOME}/output/ -rm -rf ${AUDITLOADER_OUTPUT} -mkdir ${AUDITLOADER_OUTPUT} -cp ${AUDITLOADER_HOME}/target/auditloader.zip ${AUDITLOADER_HOME}/output/ +AUDITLOADER_OUTPUT="${AUDITLOADER_HOME}/output" +rm -rf "${AUDITLOADER_OUTPUT}" +mkdir "${AUDITLOADER_OUTPUT}" +cp "${AUDITLOADER_HOME}/target/auditloader.zip" "${AUDITLOADER_HOME}/output"/ echo "Build Auditloader Finished" diff --git a/fs_brokers/apache_hdfs_broker/bin/start_broker.sh b/fs_brokers/apache_hdfs_broker/bin/start_broker.sh index 6318a32fa0..c6e0e8fb63 100755 --- a/fs_brokers/apache_hdfs_broker/bin/start_broker.sh +++ b/fs_brokers/apache_hdfs_broker/bin/start_broker.sh @@ -16,19 +16,17 @@ # specific language governing permissions and limitations # under the License. -curdir=$(dirname "$0") -curdir=$( - cd "$curdir" - pwd -) +set -eo pipefail -OPTS=$(getopt \ - -n $0 \ +curdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" + +OPTS="$(getopt \ + -n "$0" \ -o '' \ -l 'daemon' \ - -- "$@") + -- "$@")" -eval set -- "$OPTS" +eval set -- "${OPTS}" RUN_DAEMON=0 while true; do @@ -48,25 +46,28 @@ while true; do esac done -export BROKER_HOME=$( - cd "$curdir/.." +BROKER_HOME="$( + cd "${curdir}/.." pwd -) -export PID_DIR=$( - cd "$curdir" +)" +export BROKER_HOME + +PID_DIR="$( + cd "${curdir}" pwd -) +)" +export PID_DIR export JAVA_OPTS="-Xmx1024m -Dfile.encoding=UTF-8" -export BROKER_LOG_DIR="$BROKER_HOME/log" +export BROKER_LOG_DIR="${BROKER_HOME}/log" # java -if [ -z "$JAVA_HOME" ]; then - JAVA=$(which java) +if [[ -z "${JAVA_HOME}" ]]; then + JAVA="$(which java)" else - JAVA="$JAVA_HOME/bin/java" + JAVA="${JAVA_HOME}/bin/java" fi -if [ ! -x "$JAVA" ]; then +if [[ ! -x "${JAVA}" ]]; then echo "The JAVA_HOME environment variable is not defined correctly" echo "This environment variable is needed to run this program" echo "NB: JAVA_HOME should point to a JDK not a JRE" @@ -74,38 +75,42 @@ if [ ! -x "$JAVA" ]; then fi # add libs to CLASSPATH -for f in $BROKER_HOME/lib/*.jar; do - CLASSPATH=$f:${CLASSPATH} +for f in "${BROKER_HOME}/lib"/*.jar; do + CLASSPATH="${f}:${CLASSPATH}" done -export CLASSPATH=${CLASSPATH}:${BROKER_HOME}/lib:$BROKER_HOME/conf +export CLASSPATH="${CLASSPATH}:${BROKER_HOME}/lib:${BROKER_HOME}/conf" -while read line; do - envline=$(echo $line | sed 's/[[:blank:]]*=[[:blank:]]*/=/g' | sed 's/^[[:blank:]]*//g' | egrep "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*=") - envline=$(eval "echo $envline") - if [[ $envline == *"="* ]]; then - eval 'export "$envline"' +while read -r line; do + envline="$(echo "${line}" | + sed 's/[[:blank:]]*=[[:blank:]]*/=/g' | + sed 's/^[[:blank:]]*//g' | + grep -E "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*=" || + true)" + envline="$(eval "echo ${envline}")" + if [[ "${envline}" == *"="* ]]; then + eval 'export "${envline}"' fi -done < $BROKER_HOME/conf/apache_hdfs_broker.conf +done <"${BROKER_HOME}/conf/apache_hdfs_broker.conf" -pidfile=$PID_DIR/apache_hdfs_broker.pid +pidfile="${PID_DIR}/apache_hdfs_broker.pid" -if [ -f $pidfile ]; then - if kill -0 $(cat $pidfile) > /dev/null 2>&1; then - echo "Broker running as process $(cat $pidfile). Stop it first." +if [[ -f "${pidfile}" ]]; then + if kill -0 "$(cat "${pidfile}")" >/dev/null 2>&1; then + echo "Broker running as process $(cat "${pidfile}"). Stop it first." exit 1 fi fi -if [ ! -d $BROKER_LOG_DIR ]; then - mkdir -p $BROKER_LOG_DIR +if [[ ! -d "${BROKER_LOG_DIR}" ]]; then + mkdir -p "${BROKER_LOG_DIR}" fi -echo $(date) >> $BROKER_LOG_DIR/apache_hdfs_broker.out +date >>"${BROKER_LOG_DIR}/apache_hdfs_broker.out" -if [ ${RUN_DAEMON} -eq 1 ]; then - nohup $LIMIT $JAVA $JAVA_OPTS org.apache.doris.broker.hdfs.BrokerBootstrap "$@" >> $BROKER_LOG_DIR/apache_hdfs_broker.out 2>&1 < /dev/null & +if [[ ${RUN_DAEMON} -eq 1 ]]; then + nohup ${LIMIT:+${LIMIT}} "${JAVA}" ${JAVA_OPTS:+${JAVA_OPTS}} org.apache.doris.broker.hdfs.BrokerBootstrap "$@" >>"${BROKER_LOG_DIR}/apache_hdfs_broker.out" 2>&1 > $BROKER_LOG_DIR/apache_hdfs_broker.out 2>&1 < /dev/null + ${LIMIT:+${LIMIT}} "${JAVA}" ${JAVA_OPTS:+${JAVA_OPTS}} org.apache.doris.broker.hdfs.BrokerBootstrap "$@" >>"${BROKER_LOG_DIR}/apache_hdfs_broker.out" 2>&1 $pidfile +echo $! >"${pidfile}" diff --git a/fs_brokers/apache_hdfs_broker/bin/stop_broker.sh b/fs_brokers/apache_hdfs_broker/bin/stop_broker.sh index aa65f21a1e..84c126f1ac 100755 --- a/fs_brokers/apache_hdfs_broker/bin/stop_broker.sh +++ b/fs_brokers/apache_hdfs_broker/bin/stop_broker.sh @@ -16,33 +16,34 @@ # specific language governing permissions and limitations # under the License. -curdir=`dirname "$0"` -curdir=`cd "$curdir"; pwd` +set -eo pipefail -export BROKER_HOME=`cd "$curdir/.."; pwd` -export PID_DIR=`cd "$curdir"; pwd` +curdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" -while read line; do - envline=`echo $line | sed 's/[[:blank:]]*=[[:blank:]]*/=/g' | sed 's/^[[:blank:]]*//g' | egrep "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*="` - envline=`eval "echo $envline"` - if [[ $envline == *"="* ]]; then - eval 'export "$envline"' +BROKER_HOME=$( + cd "${curdir}/.." + pwd +) +export BROKER_HOME + +PID_DIR=$( + cd "${curdir}" + pwd +) +export PID_DIR + +pidfile="${PID_DIR}/apache_hdfs_broker.pid" + +if [[ -f "${pidfile}" ]]; then + pid="$(cat "${pidfile}")" + pidcomm="$(ps -p "${pid}" -o comm=)" + + if [[ "java" != "${pidcomm}" ]]; then + echo "ERROR: pid process may not broker. " fi -done < $BROKER_HOME/conf/apache_hdfs_broker.conf -pidfile=$PID_DIR/apache_hdfs_broker.pid - -if [ -f $pidfile ]; then - pid=`cat $pidfile` - pidcomm=`ps -p $pid -o comm=` - - if [ "java" != "$pidcomm" ]; then - echo "ERROR: pid process may not broker. " - fi - - if kill -9 $pid > /dev/null 2>&1; then - echo "stop $pidcomm, and remove pid file. " - rm $pidfile - fi + if kill -9 "${pid}" >/dev/null 2>&1; then + echo "stop ${pidcomm}, and remove pid file. " + rm "${pidfile}" + fi fi - diff --git a/fs_brokers/apache_hdfs_broker/build.sh b/fs_brokers/apache_hdfs_broker/build.sh index b594bdda4b..9a21f7feaf 100755 --- a/fs_brokers/apache_hdfs_broker/build.sh +++ b/fs_brokers/apache_hdfs_broker/build.sh @@ -16,37 +16,36 @@ # specific language governing permissions and limitations # under the License. -set -e +set -eo pipefail -ROOT=`dirname "$0"` -ROOT=`cd "$ROOT"; pwd` +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" -export DORIS_HOME=${ROOT}/../.. +export DORIS_HOME="${ROOT}/../.." -. ${DORIS_HOME}/env.sh +. "${DORIS_HOME}/env.sh" -export BROKER_HOME=$ROOT +export BROKER_HOME="${ROOT}" # prepare thrift -mkdir -p ${BROKER_HOME}/src/main/resources/thrift -mkdir -p ${BROKER_HOME}/src/main/thrift +mkdir -p "${BROKER_HOME}/src/main/resources/thrift" +mkdir -p "${BROKER_HOME}/src/main/thrift" -cp ${BROKER_HOME}/../../gensrc/thrift/PaloBrokerService.thrift ${BROKER_HOME}/src/main/resources/thrift/ +cp "${BROKER_HOME}/../../gensrc/thrift/PaloBrokerService.thrift" "${BROKER_HOME}/src/main/resources/thrift"/ -$MVN_CMD package -DskipTests +"${MVN_CMD}" package -DskipTests echo "Install broker..." -BROKER_OUTPUT=${BROKER_HOME}/output/apache_hdfs_broker/ -rm -rf ${BROKER_OUTPUT} +BROKER_OUTPUT="${BROKER_HOME}/output/apache_hdfs_broker" +rm -rf "${BROKER_OUTPUT}" -install -d ${BROKER_OUTPUT}/bin ${BROKER_OUTPUT}/conf \ - ${BROKER_OUTPUT}lib/ +install -d "${BROKER_OUTPUT}/bin" "${BROKER_OUTPUT}/conf" \ + "${BROKER_OUTPUT}/lib" -cp -r -p ${BROKER_HOME}/bin/*.sh ${BROKER_OUTPUT}/bin/ -cp -r -p ${BROKER_HOME}/conf/*.conf ${BROKER_OUTPUT}/conf/ -cp -r -p ${BROKER_HOME}/conf/*.xml ${BROKER_OUTPUT}/conf/ -cp -r -p ${BROKER_HOME}/conf/log4j.properties ${BROKER_OUTPUT}/conf/ -cp -r -p ${BROKER_HOME}/target/lib/* ${BROKER_OUTPUT}/lib/ -cp -r -p ${BROKER_HOME}/target/apache_hdfs_broker.jar ${BROKER_OUTPUT}/lib/ +cp -r -p "${BROKER_HOME}/bin"/*.sh "${BROKER_OUTPUT}/bin"/ +cp -r -p "${BROKER_HOME}/conf"/*.conf "${BROKER_OUTPUT}/conf"/ +cp -r -p "${BROKER_HOME}/conf"/*.xml "${BROKER_OUTPUT}/conf"/ +cp -r -p "${BROKER_HOME}/conf/log4j.properties" "${BROKER_OUTPUT}/conf"/ +cp -r -p "${BROKER_HOME}/target/lib"/* "${BROKER_OUTPUT}/lib"/ +cp -r -p "${BROKER_HOME}/target/apache_hdfs_broker.jar" "${BROKER_OUTPUT}/lib"/ echo "Finished" diff --git a/gensrc/script/gen_build_version.sh b/gensrc/script/gen_build_version.sh index 2937cdb9aa..1a36c4210b 100755 --- a/gensrc/script/gen_build_version.sh +++ b/gensrc/script/gen_build_version.sh @@ -25,45 +25,43 @@ # contains the build version based on the git hash or svn revision. ############################################################## +set -eo pipefail + build_version="trunk" unset LANG unset LC_CTYPE -user=$(whoami) -date=$(date +"%a, %d %b %Y %H:%M:%S %Z") -hostname=$(hostname) +user="$(whoami)" +date="$(date +"%a, %d %b %Y %H:%M:%S %Z")" +hostname="$(hostname)" -cwd=$(pwd) +cwd="$(pwd)" -if [ -z ${DORIS_HOME+x} ]; then - ROOT=$(dirname "$0") - ROOT=$( - cd "$ROOT" - pwd - ) - DORIS_HOME=${ROOT}/../.. +if [[ -z "${DORIS_HOME}" ]]; then + ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" + DORIS_HOME="${ROOT}/../.." echo "DORIS_HOME: ${DORIS_HOME}" fi -if [[ -z ${DORIS_TEST_BINARY_DIR} ]]; then - if [ -e ${DORIS_HOME}/fe/fe-core/target/generated-sources/build/org/apache/doris/common/Version.java \ - -a -e ${DORIS_HOME}/gensrc/build/gen_cpp/version.h ]; then +if [[ -z "${DORIS_TEST_BINARY_DIR}" ]]; then + if [[ -e "${DORIS_HOME}/fe/fe-core/target/generated-sources/build/org/apache/doris/common/Version.java" && + -e "${DORIS_HOME}/gensrc/build/gen_cpp/version.h" ]]; then exit fi fi -cd ${DORIS_HOME} -if [ -d .svn ]; then +cd "${DORIS_HOME}" +if [[ -d '.svn' ]]; then revision=$(svn info | sed -n -e 's/Last Changed Rev: \(.*\)/\1/p') short_revision="${revision}" url=$(svn info | sed -n -e 's/^URL: \(.*\)/\1/p') - if echo ${url} | grep '\/tags\/' > /dev/null; then - build_version="$(echo ${url} | sed 's/.*_\([0-9-]\+\)_PD_BL.*/\1/g' | sed 's/-/\./g')" + if echo "${url}" | grep '\/tags\/' >/dev/null; then + build_version="$(echo "${url}" | sed 's/.*_\([0-9-]\+\)_PD_BL.*/\1/g' | sed 's/-/\./g')" fi -elif [ -d .git ]; then - revision=$(git log -1 --pretty=format:"%H") - short_revision=$(git log -1 --pretty=format:"%h") +elif [[ -d '.git' ]]; then + revision="$(git log -1 --pretty=format:"%H")" + short_revision="$(git log -1 --pretty=format:"%h")" url="git://${hostname}${DORIS_HOME}" else revision="Unknown" @@ -71,34 +69,33 @@ else url="file://${DORIS_HOME}" fi -cd ${cwd} +cd "${cwd}" build_hash="${url}@${revision}" build_short_hash="${short_revision}" build_time="${date}" build_info="${user}@${hostname}" -if [ -z "$JAVA_HOME" ]; then - java_cmd=$(which java) +if [[ -z "${JAVA_HOME}" ]]; then + java_cmd="$(which java)" else - java_cmd="$JAVA_HOME/bin/java" + java_cmd="${JAVA_HOME}/bin/java" fi -if [ ! -x "$java_cmd" ]; then +if [[ ! -x "${java_cmd}" ]]; then echo "The JAVA_HOME environment variable is not defined correctly" echo "This environment variable is needed to run this program" echo "NB: JAVA_HOME should point to a JDK not a JRE" exit 1 fi -java_version_str=$($java_cmd -fullversion 2>&1) -java_version_str=$(echo $java_version_str | sed -e 's/"/\\"/g') +java_version_str="$("${java_cmd}" -fullversion 2>&1 | sed -e 's/"/\\"/g')" -echo "get java cmd: $java_cmd" -echo "get java version: $java_version_str" +echo "get java cmd: ${java_cmd}" +echo "get java version: ${java_version_str}" VERSION_PACKAGE="${DORIS_HOME}/fe/fe-core/target/generated-sources/build/org/apache/doris/common" -mkdir -p ${VERSION_PACKAGE} -cat > "${VERSION_PACKAGE}/Version.java" << EOF +mkdir -p "${VERSION_PACKAGE}" +cat >"${VERSION_PACKAGE}/Version.java" < "${GEN_CPP_DIR}/version.h" << EOF +GEN_CPP_DIR="${DORIS_HOME}/gensrc/build/gen_cpp" +mkdir -p "${GEN_CPP_DIR}" +cat >"${GEN_CPP_DIR}/version.h" </dev/null && pwd)" + +export DORIS_HOME="${ROOT}" # Check args usage() { - echo " + echo " Usage: $0 Optional options: --benchmark build benchmark-tool @@ -61,74 +62,92 @@ Usage: $0 $0 --clean clean and build tests $0 --clean --run clean, build and run all tests " - exit 1 + exit 1 } -OPTS=$(getopt -n $0 -o vhj:f: -l benchmark,run,clean,filter: -- "$@") -if [ "$?" != "0" ]; then - usage +if ! OPTS="$(getopt -n "$0" -o vhj:f: -l benchmark,run,clean,filter: -- "$@")"; then + usage fi -set -eo pipefail +eval set -- "${OPTS}" -eval set -- "$OPTS" - -PARALLEL=$[$(nproc)/5+1] +PARALLEL="$(($(nproc) / 5 + 1))" CLEAN=0 RUN=0 -BUILD_BENCHMARK_TOOL=OFF +BUILD_BENCHMARK_TOOL='OFF' FILTER="" -if [ $# != 1 ] ; then - while true; do +if [[ "$#" != 1 ]]; then + while true; do case "$1" in - --clean) CLEAN=1 ; shift ;; - --run) RUN=1 ; shift ;; - --benchmark) BUILD_BENCHMARK_TOOL=ON ; shift ;; - -f | --filter) FILTER="--gtest_filter=$2"; shift 2;; - -j) PARALLEL=$2; shift 2 ;; - --) shift ; break ;; - *) usage ; exit 0 ;; + --clean) + CLEAN=1 + shift + ;; + --run) + RUN=1 + shift + ;; + --benchmark) + BUILD_BENCHMARK_TOOL='ON' + shift + ;; + -f | --filter) + FILTER="--gtest_filter=$2" + shift 2 + ;; + -j) + PARALLEL="$2" + shift 2 + ;; + --) + shift + break + ;; + *) + usage + exit 0 + ;; esac done fi -CMAKE_BUILD_TYPE=${BUILD_TYPE:-ASAN} +CMAKE_BUILD_TYPE="${BUILD_TYPE:-ASAN}" CMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE^^}" echo "Get params: - PARALLEL -- $PARALLEL - CLEAN -- $CLEAN + PARALLEL -- ${PARALLEL} + CLEAN -- ${CLEAN} " echo "Build Backend UT" -. ${DORIS_HOME}/env.sh +. "${DORIS_HOME}/env.sh" -CMAKE_BUILD_DIR=${DORIS_HOME}/be/ut_build_${CMAKE_BUILD_TYPE} -if [ ${CLEAN} -eq 1 ]; then - rm ${CMAKE_BUILD_DIR} -rf - rm ${DORIS_HOME}/be/output/ -rf +CMAKE_BUILD_DIR="${DORIS_HOME}/be/ut_build_${CMAKE_BUILD_TYPE}" +if [[ "${CLEAN}" -eq 1 ]]; then + rm -rf "${CMAKE_BUILD_DIR}" + rm -rf "${DORIS_HOME}/be/output" fi -if [ ! -d ${CMAKE_BUILD_DIR} ]; then - mkdir -p ${CMAKE_BUILD_DIR} +if [[ ! -d "${CMAKE_BUILD_DIR}" ]]; then + mkdir -p "${CMAKE_BUILD_DIR}" fi -if [[ -z ${GLIBC_COMPATIBILITY} ]]; then - GLIBC_COMPATIBILITY=ON +if [[ -z "${GLIBC_COMPATIBILITY}" ]]; then + GLIBC_COMPATIBILITY='ON' fi -if [[ -z ${USE_DWARF} ]]; then - USE_DWARF=OFF +if [[ -z "${USE_DWARF}" ]]; then + USE_DWARF='OFF' fi MAKE_PROGRAM="$(which "${BUILD_SYSTEM}")" echo "-- Make program: ${MAKE_PROGRAM}" echo "-- Use ccache: ${CMAKE_USE_CCACHE}" -echo "-- Extra cxx flags: ${EXTRA_CXX_FLAGS}" +echo "-- Extra cxx flags: ${EXTRA_CXX_FLAGS:-}" -cd ${CMAKE_BUILD_DIR} -${CMAKE_CMD} -G "${GENERATOR}" \ +cd "${CMAKE_BUILD_DIR}" +"${CMAKE_CMD}" -G "${GENERATOR}" \ -DCMAKE_MAKE_PROGRAM="${MAKE_PROGRAM}" \ -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" \ -DMAKE_TEST=ON \ @@ -136,16 +155,16 @@ ${CMAKE_CMD} -G "${GENERATOR}" \ -DBUILD_META_TOOL=OFF \ -DBUILD_BENCHMARK_TOOL="${BUILD_BENCHMARK_TOOL}" \ -DWITH_MYSQL=OFF \ - -DUSE_DWARF=${USE_DWARF} \ + -DUSE_DWARF="${USE_DWARF}" \ -DUSE_MEM_TRACKER=ON \ -DUSE_JEMALLOC=OFF \ -DSTRICT_MEMORY_USE=OFF \ -DEXTRA_CXX_FLAGS="${EXTRA_CXX_FLAGS}" \ - ${CMAKE_USE_CCACHE} \ - ${DORIS_HOME}/be/ -${BUILD_SYSTEM} -j ${PARALLEL} + ${CMAKE_USE_CCACHE:+${CMAKE_USE_CCACHE}} \ + "${DORIS_HOME}/be" +"${BUILD_SYSTEM}" -j "${PARALLEL}" -if [ ${RUN} -ne 1 ]; then +if [[ "${RUN}" -ne 1 ]]; then echo "Finished" exit 0 fi @@ -154,96 +173,94 @@ echo "******************************" echo " Running Backend Unit Test " echo "******************************" -cd ${DORIS_HOME} -export DORIS_TEST_BINARY_DIR=${CMAKE_BUILD_DIR} -export TERM=xterm -export UDF_RUNTIME_DIR=${DORIS_HOME}/lib/udf-runtime -export LOG_DIR=${DORIS_HOME}/log -for i in `sed 's/ //g' $DORIS_HOME/conf/be.conf | egrep "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*="`; do - eval "export $i"; -done +cd "${DORIS_HOME}" +export DORIS_TEST_BINARY_DIR="${CMAKE_BUILD_DIR}" +export TERM='xterm' +export UDF_RUNTIME_DIR="${DORIS_HOME}/lib/udf-runtime" +export LOG_DIR="${DORIS_HOME}/log" +while read -r variable; do + eval "export ${variable}" +done < <(sed 's/ //g' "${DORIS_HOME}/conf/be.conf" | grep -E "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*=") -mkdir -p $LOG_DIR -mkdir -p ${UDF_RUNTIME_DIR} -rm -f ${UDF_RUNTIME_DIR}/* +mkdir -p "${LOG_DIR}" +mkdir -p "${UDF_RUNTIME_DIR}" +rm -f "${UDF_RUNTIME_DIR}"/* # clean all gcda file +while read -r gcda_file; do + rm "${gcda_file}" +done < <(find "${DORIS_TEST_BINARY_DIR}" -name "*gcda") -gcda_files=`find ${DORIS_TEST_BINARY_DIR} -name "*gcda"` -for gcda_file in ${gcda_files[@]} -do - rm $gcda_file -done - -export DORIS_TEST_BINARY_DIR=${DORIS_TEST_BINARY_DIR}/test/ +export DORIS_TEST_BINARY_DIR="${DORIS_TEST_BINARY_DIR}/test" # prepare jvm if needed jdk_version() { local result - local java_cmd=$JAVA_HOME/bin/java + local java_cmd="${JAVA_HOME}/bin/java" local IFS=$'\n' - # remove \r for Cygwin - local lines=$("$java_cmd" -Xms32M -Xmx32M -version 2>&1 | tr '\r' '\n') - if [[ -z $java_cmd ]] - then + + if [[ -z "${java_cmd}" ]]; then result=no_java + return 1 else - for line in $lines; do - if [[ (-z $result) && ($line = *"version \""*) ]] - then - local ver=$(echo $line | sed -e 's/.*version "\(.*\)"\(.*\)/\1/; 1q') - # on macOS, sed doesn't support '?' - if [[ $ver = "1."* ]] - then - result=$(echo $ver | sed -e 's/1\.\([0-9]*\)\(.*\)/\1/; 1q') - else - result=$(echo $ver | sed -e 's/\([0-9]*\)\(.*\)/\1/; 1q') - fi - fi - done + local version + # remove \r for Cygwin + version="$("${java_cmd}" -Xms32M -Xmx32M -version 2>&1 | tr '\r' '\n' | grep version | awk '{print $3}')" + version="${version//\"/}" + if [[ "${version}" =~ ^1\. ]]; then + result="$(echo "${version}" | awk -F '.' '{print $2}')" + else + result="$(echo "${version}" | awk -F '.' '{print $1}')" + fi fi - echo "$result" + echo "${result}" + return 0 } jvm_arch="amd64" -MACHINE_TYPE=$(uname -m) +MACHINE_TYPE="$(uname -m)" if [[ "${MACHINE_TYPE}" == "aarch64" ]]; then jvm_arch="aarch64" fi -java_version=$(jdk_version) -if [[ $java_version -gt 8 ]]; then - export LD_LIBRARY_PATH=$JAVA_HOME/lib/server:$JAVA_HOME/lib:$LD_LIBRARY_PATH +java_version="$( + set -e + jdk_version +)" +if [[ "${java_version}" -gt 8 ]]; then + export LD_LIBRARY_PATH="${JAVA_HOME}/lib/server:${JAVA_HOME}/lib:${LD_LIBRARY_PATH}" # JAVA_HOME is jdk -elif [[ -d "$JAVA_HOME/jre" ]]; then - export LD_LIBRARY_PATH=$JAVA_HOME/jre/lib/$jvm_arch/server:$JAVA_HOME/jre/lib/$jvm_arch:$LD_LIBRARY_PATH +elif [[ -d "${JAVA_HOME}/jre" ]]; then + export LD_LIBRARY_PATH="${JAVA_HOME}/jre/lib/${jvm_arch}/server:${JAVA_HOME}/jre/lib/${jvm_arch}:${LD_LIBRARY_PATH}" # JAVA_HOME is jre else - export LD_LIBRARY_PATH=$JAVA_HOME/lib/$jvm_arch/server:$JAVA_HOME/lib/$jvm_arch:$LD_LIBRARY_PATH + export LD_LIBRARY_PATH="${JAVA_HOME}/lib/${jvm_arch}/server:${JAVA_HOME}/lib/${jvm_arch}:${LD_LIBRARY_PATH}" fi # prepare gtest output dir -GTEST_OUTPUT_DIR=${CMAKE_BUILD_DIR}/gtest_output -rm -rf ${GTEST_OUTPUT_DIR} && mkdir ${GTEST_OUTPUT_DIR} +GTEST_OUTPUT_DIR="${CMAKE_BUILD_DIR}/gtest_output" +rm -rf "${GTEST_OUTPUT_DIR}" +mkdir "${GTEST_OUTPUT_DIR}" # prepare util test_data -mkdir -p ${DORIS_TEST_BINARY_DIR}/util -if [ -d ${DORIS_TEST_BINARY_DIR}/util/test_data ]; then - rm -rf ${DORIS_TEST_BINARY_DIR}/util/test_data +mkdir -p "${DORIS_TEST_BINARY_DIR}/util" +if [[ -d "${DORIS_TEST_BINARY_DIR}/util/test_data" ]]; then + rm -rf "${DORIS_TEST_BINARY_DIR}/util/test_data" fi -cp -r ${DORIS_HOME}/be/test/util/test_data ${DORIS_TEST_BINARY_DIR}/util/ +cp -r "${DORIS_HOME}/be/test/util/test_data" "${DORIS_TEST_BINARY_DIR}/util"/ # prepare ut temp dir -UT_TMP_DIR=${DORIS_HOME}/ut_dir -rm -rf ${UT_TMP_DIR} && mkdir ${UT_TMP_DIR} -touch ${UT_TMP_DIR}/tmp_file +UT_TMP_DIR="${DORIS_HOME}/ut_dir" +rm -rf "${UT_TMP_DIR}" +mkdir "${UT_TMP_DIR}" +touch "${UT_TMP_DIR}/tmp_file" # find all executable test files -test=${DORIS_TEST_BINARY_DIR}doris_be_test -file_name=${test##*/} -if [ -f "$test" ]; then - $test --gtest_output=xml:${GTEST_OUTPUT_DIR}/${file_name}.xml --gtest_print_time=true "${FILTER}" +test="${DORIS_TEST_BINARY_DIR}/doris_be_test" +file_name="${test##*/}" +if [[ -f "${test}" ]]; then + "${test}" --gtest_output="xml:${GTEST_OUTPUT_DIR}/${file_name}.xml" --gtest_print_time=true "${FILTER}" echo "=== Finished. Gtest output: ${GTEST_OUTPUT_DIR}" -else - echo "unit test file: $test does not exist." +else + echo "unit test file: ${test} does not exist." fi diff --git a/run-fe-ut.sh b/run-fe-ut.sh index dd1062d6e1..bf32af5afb 100755 --- a/run-fe-ut.sh +++ b/run-fe-ut.sh @@ -18,16 +18,15 @@ set -eo pipefail -ROOT=`dirname "$0"` -ROOT=`cd "$ROOT"; pwd` +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" -export DORIS_HOME=${ROOT} +export DORIS_HOME="${ROOT}" -. ${DORIS_HOME}/env.sh +. "${DORIS_HOME}/env.sh" # Check args usage() { - echo " + echo " Usage: $0 Optional options: --clean clean and build ut @@ -38,25 +37,23 @@ Usage: $0 $0 --coverage build and run coverage statistic $0 --run xxx build and run the specified class " - exit 1 + exit 1 } -OPTS=$(getopt \ - -n $0 \ - -o '' \ - -l 'coverage' \ - -l 'run' \ - -- "$@") - -if [ $? != 0 ] ; then +if ! OPTS="$(getopt \ + -n "$0" \ + -o '' \ + -l 'coverage' \ + -l 'run' \ + -- "$@")"; then usage fi -eval set -- "$OPTS" +eval set -- "${OPTS}" -RUN= -COVERAGE= -if [ $# == 1 ] ; then +RUN=0 +COVERAGE=0 +if [[ "$#" == 1 ]]; then #default RUN=0 COVERAGE=0 @@ -65,10 +62,22 @@ else COVERAGE=0 while true; do case "$1" in - --coverage) COVERAGE=1 ; shift ;; - --run) RUN=1 ; shift ;; - --) shift ; break ;; - *) echo "Internal error" ; exit 1 ;; + --coverage) + COVERAGE=1 + shift + ;; + --run) + RUN=1 + shift + ;; + --) + shift + break + ;; + *) + echo "Internal error" + exit 1 + ;; esac done fi @@ -79,27 +88,27 @@ echo "******************************" echo " Runing DorisFe Unittest " echo "******************************" -cd ${DORIS_HOME}/fe/ +cd "${DORIS_HOME}/fe" mkdir -p build/compile -if [ -z "${FE_UT_PARALLEL}" ]; then +if [[ -z "${FE_UT_PARALLEL}" ]]; then # the default fe unit test parallel is 1 export FE_UT_PARALLEL=1 fi -echo "Unit test parallel is: $FE_UT_PARALLEL" +echo "Unit test parallel is: ${FE_UT_PARALLEL}" -if [ ${COVERAGE} -eq 1 ]; then +if [[ "${COVERAGE}" -eq 1 ]]; then echo "Run coverage statistic" ant cover-test else - if [ ${RUN} -eq 1 ]; then + if [[ "${RUN}" -eq 1 ]]; then echo "Run the specified class: $1" # eg: # sh run-fe-ut.sh --run org.apache.doris.utframe.Demo # sh run-fe-ut.sh --run org.apache.doris.utframe.Demo#testCreateDbAndTable+test2 - ${MVN_CMD} test -DfailIfNoTests=false -D test=$1 + "${MVN_CMD}" test -DfailIfNoTests=false -D test="$1" else echo "Run Frontend UT" - ${MVN_CMD} test -DfailIfNoTests=false + "${MVN_CMD}" test -DfailIfNoTests=false fi fi diff --git a/run-regression-test.sh b/run-regression-test.sh index 096b4cb933..6ca6867859 100755 --- a/run-regression-test.sh +++ b/run-regression-test.sh @@ -19,14 +19,13 @@ set -eo pipefail #set -x -ROOT=`dirname "$0"` -ROOT=`cd "$ROOT"; pwd` +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" -DORIS_HOME=${ROOT} +DORIS_HOME="${ROOT}" # Check args usage() { - echo " + echo " Usage: $0 Optional shell_options: --clean clean output of regression test framework @@ -63,14 +62,14 @@ Usage: $0 Log path: \${DORIS_HOME}/output/regression-test/log Default config file: \${DORIS_HOME}/regression-test/conf/regression-conf.groovy " - exit 1 + exit 1 } -CLEAN= -WRONG_CMD= -TEAMCITY= -RUN= -if [ $# == 0 ] ; then +CLEAN=0 +WRONG_CMD=0 +TEAMCITY=0 +RUN=0 +if [[ "$#" == 0 ]]; then #default CLEAN=0 WRONG_CMD=0 @@ -83,95 +82,103 @@ else WRONG_CMD=0 while true; do case "$1" in - --clean) CLEAN=1 ; shift ;; - --teamcity) TEAMCITY=1 ; shift ;; - --run) RUN=1 ; shift ;; - *) - if [ ${RUN} -eq 0 ] && [ ${CLEAN} -eq 0 ]; then - WRONG_CMD=1 - fi - break ;; + --clean) + CLEAN=1 + shift + ;; + --teamcity) + TEAMCITY=1 + shift + ;; + --run) + RUN=1 + shift + ;; + *) + if [[ "${RUN}" -eq 0 ]] && [[ "${CLEAN}" -eq 0 ]]; then + WRONG_CMD=1 + fi + break + ;; esac done fi -if [ ${WRONG_CMD} -eq 1 ]; then +if [[ "${WRONG_CMD}" -eq 1 ]]; then usage exit 1 fi # set maven -MVN_CMD=mvn -if [[ ! -z ${CUSTOM_MVN} ]]; then - MVN_CMD=${CUSTOM_MVN} +MVN_CMD='mvn' +if [[ -n "${CUSTOM_MVN}" ]]; then + MVN_CMD="${CUSTOM_MVN}" fi -if ! ${MVN_CMD} --version; then +if ! "${MVN_CMD}" --version; then echo "Error: mvn is not found" exit 1 fi export MVN_CMD -CONF_DIR=${DORIS_HOME}/regression-test/conf -CONFIG_FILE=${CONF_DIR}/regression-conf.groovy -LOG_CONFIG_FILE=${CONF_DIR}/logback.xml +CONF_DIR="${DORIS_HOME}/regression-test/conf" +CONFIG_FILE="${CONF_DIR}/regression-conf.groovy" +LOG_CONFIG_FILE="${CONF_DIR}/logback.xml" -FRAMEWORK_SOURCE_DIR=${DORIS_HOME}/regression-test/framework -REGRESSION_TEST_BUILD_DIR=${FRAMEWORK_SOURCE_DIR}/target +FRAMEWORK_SOURCE_DIR="${DORIS_HOME}/regression-test/framework" +REGRESSION_TEST_BUILD_DIR="${FRAMEWORK_SOURCE_DIR}/target" -OUTPUT_DIR=${DORIS_HOME}/output/regression-test -LOG_OUTPUT_FILE=${OUTPUT_DIR}/log -RUN_JAR=${OUTPUT_DIR}/lib/regression-test-*.jar +OUTPUT_DIR="${DORIS_HOME}/output/regression-test" +LOG_OUTPUT_FILE="${OUTPUT_DIR}/log" +RUN_JAR="${OUTPUT_DIR}/lib/regression-test-*.jar" -if [ ${CLEAN} -eq 1 ]; then - rm -rf ${REGRESSION_TEST_BUILD_DIR} - rm -rf ${OUTPUT_DIR} +if [[ "${CLEAN}" -eq 1 ]]; then + rm -rf "${REGRESSION_TEST_BUILD_DIR}" + rm -rf "${OUTPUT_DIR}" fi -if [ ${RUN} -ne 1 ]; then +if [[ "${RUN}" -ne 1 ]]; then echo "Finished" exit 0 fi -if [ ! -f ${RUN_JAR} ]; then +if [[ ! -f ${RUN_JAR} ]]; then echo "===== Build Regression Test Framework =====" - cd ${DORIS_HOME}/regression-test/framework - ${MVN_CMD} package - cd ${DORIS_HOME} + cd "${DORIS_HOME}/regression-test/framework" + "${MVN_CMD}" package + cd "${DORIS_HOME}" - mkdir -p ${OUTPUT_DIR}/{lib,log} - cp -r ${REGRESSION_TEST_BUILD_DIR}/regression-test-*.jar ${OUTPUT_DIR}/lib + mkdir -p "${OUTPUT_DIR}"/{lib,log} + cp -r "${REGRESSION_TEST_BUILD_DIR}"/regression-test-*.jar "${OUTPUT_DIR}/lib" fi # check java home -if [[ -z ${JAVA_HOME} ]]; then +if [[ -z "${JAVA_HOME}" ]]; then echo "Error: JAVA_HOME is not set" exit 1 fi # check java version -export JAVA=${JAVA_HOME}/bin/java +export JAVA="${JAVA_HOME}/bin/java" - -REGRESSION_OPTIONS_PREFIX= +REGRESSION_OPTIONS_PREFIX='' # contains framework options and not start with - # it should be suite name -if [ $# -ne 0 ] && [[ "$1" =~ ^[^-].* ]]; then +if [[ "$#" -ne 0 ]] && [[ "$1" =~ ^[^-].* ]]; then # specify suiteName REGRESSION_OPTIONS_PREFIX='-s' fi echo "===== Run Regression Test =====" -JAVA_OPTS=${JAVA_OPTS} -if [ ${TEAMCITY} -eq 1 ]; then - JAVA_OPTS="$JAVA_OPTS -DstdoutAppenderType=teamcity -Xmx2048m" +if [[ "${TEAMCITY}" -eq 1 ]]; then + JAVA_OPTS="${JAVA_OPTS} -DstdoutAppenderType=teamcity -Xmx2048m" fi -$JAVA -DDORIS_HOME=$DORIS_HOME \ - -DLOG_PATH=$LOG_OUTPUT_FILE \ - -Dlogback.configurationFile=${LOG_CONFIG_FILE} \ - ${JAVA_OPTS} \ - -jar ${RUN_JAR} \ - -cf ${CONFIG_FILE} \ - ${REGRESSION_OPTIONS_PREFIX} "$@" +"${JAVA}" -DDORIS_HOME="${DORIS_HOME}" \ + -DLOG_PATH="${LOG_OUTPUT_FILE}" \ + -Dlogback.configurationFile="${LOG_CONFIG_FILE}" \ + ${JAVA_OPTS:+${JAVA_OPTS}} \ + -jar ${RUN_JAR:+${RUN_JAR}} \ + -cf "${CONFIG_FILE}" \ + ${REGRESSION_OPTIONS_PREFIX:+${REGRESSION_OPTIONS_PREFIX}} "$@" diff --git a/samples/insert/shell/insert_utils.sh b/samples/insert/shell/insert_utils.sh index d33fda5c13..0ff3a5b442 100644 --- a/samples/insert/shell/insert_utils.sh +++ b/samples/insert/shell/insert_utils.sh @@ -15,35 +15,34 @@ # specific language governing permissions and limitations # under the License. +function check_insert_load_doris_func() { + local doris="$1" + local sql="$2" + insert_return_info="$(${doris} -e "${sql}" -vv)" -function check_insert_load_doris_func(){ - local doris=$1 - local sql=$2 - insert_return_info=`${doris} -e "${sql}" -vv` - - label=`echo ${insert_return_info}|perl -e '$_ = <>;chomp;/(\w{8}\-\w{4}\-\w{4}\-\w{4}\-\w{12})/;print $1'` + label="$(echo "${insert_return_info}" | perl -e '$_ = <>;chomp;/(\w{8}\-\w{4}\-\w{4}\-\w{4}\-\w{12})/;print $1')" echo "${label}" wait_seconds=3600 - while [[ "${wait_seconds}" > 0 && "${label}" == '' ]];do - label=`echo "${insert_return_info}"|perl -e '$_ = <>;chomp;/(\w{8}\-\w{4}\-\w{4}\-\w{4}\-\w{12})/;print $1'` + while [[ "${wait_seconds}" -gt 0 && "${label}" == '' ]]; do + label="$(echo "${insert_return_info}" | perl -e '$_ = <>;chomp;/(\w{8}\-\w{4}\-\w{4}\-\w{4}\-\w{12})/;print $1')" sleep 5s - ((wait_seconds=${wait_seconds}-5)) + wait_seconds="$((wait_seconds - 5))" done - if [[ "$label" == '' ]];then + if [[ "${label}" == '' ]]; then return 1 fi wait_seconds=3600 - while [[ "${wait_seconds}" > 0 ]];do + while [[ "${wait_seconds}" -gt 0 ]]; do echo "${wait_seconds}" echo "${doris} -e show load where label = '${label}' order by createtime desc limit 1" - result=`${doris} -e "show load where label = '${label}' order by createtime desc limit 1" -N` + result=$(${doris} -e "show load where label = '${label}' order by createtime desc limit 1" -N) - load_status=`echo "${result}"|perl -e '$_ = <>;chomp;/(\d+)\s(\w{8}\-\w{4}\-\w{4}\-\w{4}\-\w{12})\s(\w+)\s/;print $3'` + load_status=$(echo "${result}" | perl -e '$_ = <>;chomp;/(\d+)\s(\w{8}\-\w{4}\-\w{4}\-\w{4}\-\w{12})\s(\w+)\s/;print $3') if [[ "${load_status}" == 'FINISHED' || "${load_status}" == 'CANCELLED' ]]; then echo "insert status: ${load_status}" - if [[ "${load_status}" == 'FINISHED' ]];then + if [[ "${load_status}" == 'FINISHED' ]]; then return 0 else return 1 @@ -52,7 +51,7 @@ function check_insert_load_doris_func(){ else echo "insert status: ${load_status}" sleep 5s - ((wait_seconds=$wait_seconds-5)) + wait_seconds="$((wait_seconds - 5))" fi done @@ -66,6 +65,6 @@ doris_port=8080 doris_user="db_user" doris_password="db_password" doris_database="db_name" -doris="mysql -h$doris_host -P$doris_port -u$doris_user -p$doris_password -D$doris_database" +doris="mysql -h${doris_host} -P${doris_port} -u${doris_user} -p${doris_password} -D${doris_database}" sql="INSERT INTO TABLE_Y[(column1, column2,...,columnN)] SELECT column1, column2,..., columnN [FROM TABLE_X WHERE xxx]" -check_insert_load_doris_func ``"$doris"` `"$sql" +check_insert_load_doris_func "${doris}" "${sql}" diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index a70b843a61..5bd7fb21dd 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash +# shellcheck disable=2034 + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -26,16 +28,13 @@ # This script will run *download-thirdparty.sh* once again # to check if all thirdparties have been downloaded, unpacked and patched. ################################################################################# -set -e -curdir=$(dirname "$0") -curdir=$( - cd "$curdir" - pwd -) +set -eo pipefail -export DORIS_HOME=$curdir/.. -export TP_DIR=$curdir +curdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" + +export DORIS_HOME="${curdir}/.." +export TP_DIR="${curdir}" # Check args usage() { @@ -47,31 +46,31 @@ Usage: $0 exit 1 } -if ! OPTS=$(getopt \ +if ! OPTS="$(getopt \ -n "$0" \ -o '' \ -o 'h' \ -l 'help' \ -o 'j:' \ - -- "$@"); then + -- "$@")"; then usage fi -eval set -- "$OPTS" +eval set -- "${OPTS}" KERNEL="$(uname -s)" if [[ "${KERNEL}" == 'Darwin' ]]; then - PARALLEL=$(($(sysctl -n hw.logicalcpu) / 4 + 1)) + PARALLEL="$(($(sysctl -n hw.logicalcpu) / 4 + 1))" else - PARALLEL=$(($(nproc) / 4 + 1)) + PARALLEL="$(($(nproc) / 4 + 1))" fi -if [[ $# -ne 1 ]]; then +if [[ "$#" -ne 1 ]]; then while true; do case "$1" in -j) - PARALLEL=$2 + PARALLEL="$2" shift 2 ;; -h) @@ -94,28 +93,28 @@ if [[ $# -ne 1 ]]; then done fi -if [[ ${HELP} -eq 1 ]]; then +if [[ "${HELP}" -eq 1 ]]; then usage exit fi echo "Get params: - PARALLEL -- $PARALLEL + PARALLEL -- ${PARALLEL} " # include custom environment variables -if [[ -f ${DORIS_HOME}/env.sh ]]; then +if [[ -f "${DORIS_HOME}/env.sh" ]]; then export BUILD_THIRDPARTY_WIP=1 . "${DORIS_HOME}/env.sh" export BUILD_THIRDPARTY_WIP= fi -if [[ ! -f ${TP_DIR}/download-thirdparty.sh ]]; then +if [[ ! -f "${TP_DIR}/download-thirdparty.sh" ]]; then echo "Download thirdparty script is missing". exit 1 fi -if [[ ! -f ${TP_DIR}/vars.sh ]]; then +if [[ ! -f "${TP_DIR}/vars.sh" ]]; then echo "vars.sh is missing". exit 1 fi @@ -127,26 +126,26 @@ cd "${TP_DIR}" # Download thirdparties. "${TP_DIR}/download-thirdparty.sh" -export LD_LIBRARY_PATH=$TP_DIR/installed/lib:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH="${TP_DIR}/installed/lib:${LD_LIBRARY_PATH}" # toolchain specific warning options and settings -if [[ "$CC" == *gcc ]]; then - warning_uninitialized=-Wno-maybe-uninitialized - warning_stringop_truncation=-Wno-stringop-truncation - warning_class_memaccess=-Wno-class-memaccess - warning_array_parameter=-Wno-array-parameter - boost_toolset=gcc -elif [[ "$CC" == *clang ]]; then - warning_uninitialized=-Wno-uninitialized - warning_shadow=-Wno-shadow - warning_dangling_gsl=-Wno-dangling-gsl - warning_unused_but_set_variable=-Wno-unused-but-set-variable - warning_defaulted_function_deleted=-Wno-defaulted-function-deleted - warning_reserved_identifier=-Wno-reserved-identifier - warning_suggest_override="-Wno-suggest-override -Wno-suggest-destructor-override" - warning_option_ignored=-Wno-option-ignored - boost_toolset=clang - libhdfs_cxx17=-std=c++1z +if [[ "${CC}" == *gcc ]]; then + warning_uninitialized='-Wno-maybe-uninitialized' + warning_stringop_truncation='-Wno-stringop-truncation' + warning_class_memaccess='-Wno-class-memaccess' + warning_array_parameter='-Wno-array-parameter' + boost_toolset='gcc' +elif [[ "${CC}" == *clang ]]; then + warning_uninitialized='-Wno-uninitialized' + warning_shadow='-Wno-shadow' + warning_dangling_gsl='-Wno-dangling-gsl' + warning_unused_but_set_variable='-Wno-unused-but-set-variable' + warning_defaulted_function_deleted='-Wno-defaulted-function-deleted' + warning_reserved_identifier='-Wno-reserved-identifier' + warning_suggest_override='-Wno-suggest-override -Wno-suggest-destructor-override' + warning_option_ignored='-Wno-option-ignored' + boost_toolset='clang' + libhdfs_cxx17='-std=c++1z' fi # prepare installed prefix @@ -156,13 +155,13 @@ ln -sf lib64 lib popd check_prerequest() { - local CMD=$1 - local NAME=$2 - if ! $CMD; then - echo "$NAME is missing" + local CMD="$1" + local NAME="$2" + if ! ${CMD}; then + echo "${NAME} is missing" exit 1 else - echo "$NAME is found" + echo "${NAME} is found" fi } @@ -222,8 +221,8 @@ check_if_source_exist() { exit 1 fi - if [[ ! -d $TP_SOURCE_DIR/$1 ]]; then - echo "$TP_SOURCE_DIR/$1 does not exist." + if [[ ! -d "${TP_SOURCE_DIR}/$1" ]]; then + echo "${TP_SOURCE_DIR}/$1 does not exist." exit 1 fi echo "===== begin build $1" @@ -235,8 +234,8 @@ check_if_archieve_exist() { exit 1 fi - if [[ ! -f $TP_SOURCE_DIR/$1 ]]; then - echo "$TP_SOURCE_DIR/$1 does not exist." + if [[ ! -f "${TP_SOURCE_DIR}/$1" ]]; then + echo "${TP_SOURCE_DIR}/$1 does not exist." exit 1 fi } @@ -374,7 +373,7 @@ build_protobuf() { # This will casue protoc cannot run # If you really need to dynamically link protoc, please set the environment variable DYN_LINK_PROTOC=1 - if [[ "${DYN_LINK_PROTOC}" == "1" || "${KERNEL}" == 'Darwin' ]]; then + if [[ "${DYN_LINK_PROTOC:-0}" == "1" || "${KERNEL}" == 'Darwin' ]]; then echo "link protoc dynamiclly" else cd src @@ -940,22 +939,22 @@ build_bitshuffle() { # we still need to support non-AVX2-capable hardware. So, we build it twice, # once with the flag and once without, and use some linker tricks to # suffix the AVX2 symbols with '_avx2'. - arches="default avx2" + arches=('default' 'avx2') MACHINE_TYPE="$(uname -m)" # Becuase aarch64 don't support avx2, disable it. if [[ "${MACHINE_TYPE}" == "aarch64" || "${MACHINE_TYPE}" == 'arm64' ]]; then - arches="default" + arches=('default') fi to_link="" - for arch in $arches; do + for arch in "${arches[@]}"; do arch_flag="" if [[ "${arch}" == "avx2" ]]; then arch_flag="-mavx2" fi tmp_obj="bitshuffle_${arch}_tmp.o" dst_obj="bitshuffle_${arch}.o" - "${CC}" ${EXTRA_CFLAGS} ${arch_flag} -std=c99 "-I${PREFIX}/include/lz4" -O3 -DNDEBUG -c \ + "${CC}" ${EXTRA_CFLAGS:+${EXTRA_CFLAGS}} ${arch_flag:+${arch_flag}} -std=c99 "-I${PREFIX}/include/lz4" -O3 -DNDEBUG -c \ "src/bitshuffle_core.c" \ "src/bitshuffle.c" \ "src/iochain.c" @@ -964,7 +963,7 @@ build_bitshuffle() { # For the AVX2 symbols, suffix them. if [[ "${arch}" == "avx2" ]]; then # Create a mapping file with ' ' on each line. - "${DORIS_BIN_UTILS}/nm" --defined-only --extern-only "${tmp_obj}" | while read addr type sym; do + "${DORIS_BIN_UTILS}/nm" --defined-only --extern-only "${tmp_obj}" | while read -r addr type sym; do echo "${sym} ${sym}_${arch}" done >renames.txt "${DORIS_BIN_UTILS}/objcopy" --redefine-syms=renames.txt "${tmp_obj}" "${dst_obj}" @@ -973,17 +972,19 @@ build_bitshuffle() { fi to_link="${to_link} ${dst_obj}" done + local links + read -r -a links <<<"${to_link}" rm -f libbitshuffle.a - "${DORIS_BIN_UTILS}/ar" rs libbitshuffle.a ${to_link} + "${DORIS_BIN_UTILS}/ar" rs libbitshuffle.a "${links[@]}" mkdir -p "${PREFIX}/include/bitshuffle" cp libbitshuffle.a "${PREFIX}"/lib/ - cp "${TP_SOURCE_DIR}/$BITSHUFFLE_SOURCE/src/bitshuffle.h" "${PREFIX}/include/bitshuffle/bitshuffle.h" - cp "${TP_SOURCE_DIR}/$BITSHUFFLE_SOURCE/src/bitshuffle_core.h" "${PREFIX}/include/bitshuffle/bitshuffle_core.h" + cp "${TP_SOURCE_DIR}/${BITSHUFFLE_SOURCE}/src/bitshuffle.h" "${PREFIX}/include/bitshuffle/bitshuffle.h" + cp "${TP_SOURCE_DIR}/${BITSHUFFLE_SOURCE}/src/bitshuffle_core.h" "${PREFIX}/include/bitshuffle/bitshuffle_core.h" } # croaring bitmap build_croaringbitmap() { - avx_flag= + avx_flag='' if [[ -n "${USE_AVX2}" && "${USE_AVX2}" -eq 0 ]]; then echo "set USE_AVX2=${USE_AVX2} to FORCE disable AVX2 in croaringbitmap" avx_flag="-DROARING_DISABLE_AVX=ON" @@ -1005,7 +1006,7 @@ build_croaringbitmap() { CXXFLAGS="-O3" \ LDFLAGS="${ldflags}" \ - "${CMAKE_CMD}" -G "${GENERATOR}" ${avx_flag} -DROARING_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ + "${CMAKE_CMD}" -G "${GENERATOR}" ${avx_flag:+${avx_flag}} -DROARING_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ -DENABLE_ROARING_TESTS=OFF .. "${BUILD_SYSTEM}" -j "${PARALLEL}" @@ -1230,7 +1231,7 @@ build_krb5() { fi CFLAGS="-fcommon -I${TP_INSTALL_DIR}/include" LDFLAGS="-L${TP_INSTALL_DIR}/lib" \ - ../configure --prefix="${TP_INSTALL_DIR}" --disable-shared --enable-static ${with_crypto_impl} + ../configure --prefix="${TP_INSTALL_DIR}" --disable-shared --enable-static ${with_crypto_impl:+${with_crypto_impl}} make -j "${PARALLEL}" make install @@ -1262,7 +1263,7 @@ build_benchmark() { cmake -E make_directory "build" - if [[ "$KERNEL" != 'Darwin' ]]; then + if [[ "${KERNEL}" != 'Darwin' ]]; then cxxflags='-lresolv -pthread -lrt' else cxxflags='-lresolv -pthread' @@ -1288,8 +1289,8 @@ build_simdjson() { CXXFLAGS="-O3" \ CFLAGS="-O3" \ - $CMAKE_CMD .. - $CMAKE_CMD --build . + "${CMAKE_CMD}" .. + "${CMAKE_CMD}" --build . cp "${TP_SOURCE_DIR}/${SIMDJSON_SOURCE}/${BUILD_DIR}/libsimdjson.a" "${TP_INSTALL_DIR}/lib64" cp -r "${TP_SOURCE_DIR}/${SIMDJSON_SOURCE}/include"/* "${TP_INCLUDE_DIR}/" diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index 5e67ccd549..3856993c2f 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -16,138 +16,135 @@ # specific language governing permissions and limitations # under the License. -set -e ################################################################ # This script will download all thirdparties and java libraries # which are defined in *vars.sh*, unpack patch them if necessary. # You can run this script multi-times. # Things will only be downloaded, unpacked and patched once. ################################################################ -curdir=`dirname "$0"` -curdir=`cd "$curdir"; pwd` + +set -eo pipefail + +curdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" if [[ -z "${DORIS_HOME}" ]]; then - DORIS_HOME=$curdir/.. + DORIS_HOME="${curdir}/.." fi # include custom environment variables -if [[ -f ${DORIS_HOME}/custom_env.sh ]]; then - . ${DORIS_HOME}/custom_env.sh +if [[ -f "${DORIS_HOME}/custom_env.sh" ]]; then + # shellcheck disable=1091 + . "${DORIS_HOME}/custom_env.sh" fi if [[ -z "${TP_DIR}" ]]; then - TP_DIR=$curdir + TP_DIR="${curdir}" fi -if [ ! -f ${TP_DIR}/vars.sh ]; then - echo "vars.sh is missing". +if [[ ! -f "${TP_DIR}/vars.sh" ]]; then + echo 'vars.sh is missing'. exit 1 fi -. ${TP_DIR}/vars.sh -mkdir -p ${TP_DIR}/src +. "${TP_DIR}/vars.sh" -md5sum_bin=md5sum -if ! command -v ${md5sum_bin} >/dev/null 2>&1; then +mkdir -p "${TP_DIR}/src" + +md5sum_bin='md5sum' +if ! command -v "${md5sum_bin}" >/dev/null 2>&1; then echo "Warn: md5sum is not installed" md5sum_bin="" fi md5sum_func() { - local FILENAME=$1 - local DESC_DIR=$2 - local MD5SUM=$3 + local FILENAME="$1" + local DESC_DIR="$2" + local MD5SUM="$3" + local md5 - if [ "$md5sum_bin" == "" ]; then - return 0 + if [[ "${md5sum_bin}" == "" ]]; then + return 0 else - md5=`md5sum "$DESC_DIR/$FILENAME"` - if [ "$md5" != "$MD5SUM $DESC_DIR/$FILENAME" ]; then - echo "$DESC_DIR/$FILENAME md5sum check failed!" - echo -e "except-md5 $MD5SUM \nactual-md5 $md5" - return 1 - fi + md5="$(md5sum "${DESC_DIR}/${FILENAME}")" + if [[ "${md5}" != "${MD5SUM} ${DESC_DIR}/${FILENAME}" ]]; then + echo "${DESC_DIR}/${FILENAME} md5sum check failed!" + echo -e "except-md5 ${MD5SUM} \nactual-md5 ${md5}" + return 1 + fi fi - return 0 } # return 0 if download succeed. # return 1 if not. download_func() { - local FILENAME=$1 - local DOWNLOAD_URL=$2 - local DESC_DIR=$3 - local MD5SUM=$4 + local FILENAME="$1" + local DOWNLOAD_URL="$2" + local DESC_DIR="$3" + local MD5SUM="$4" - if [ -z "$FILENAME" ]; then + if [[ -z "${FILENAME}" ]]; then echo "Error: No file name specified to download" exit 1 fi - if [ -z "$DOWNLOAD_URL" ]; then - echo "Error: No download url specified for $FILENAME" + if [[ -z "${DOWNLOAD_URL}" ]]; then + echo "Error: No download url specified for ${FILENAME}" exit 1 fi - if [ -z "$DESC_DIR" ]; then - echo "Error: No dest dir specified for $FILENAME" + if [[ -z "${DESC_DIR}" ]]; then + echo "Error: No dest dir specified for ${FILENAME}" exit 1 fi - local STATUS=1 for attemp in 1 2; do - if [ -r "$DESC_DIR/$FILENAME" ]; then - if md5sum_func $FILENAME $DESC_DIR $MD5SUM; then - echo "Archive $FILENAME already exist." + if [[ -r "${DESC_DIR}/${FILENAME}" ]]; then + if md5sum_func "${FILENAME}" "${DESC_DIR}" "${MD5SUM}"; then + echo "Archive ${FILENAME} already exist." STATUS=0 - break; + break fi - echo "Archive $FILENAME will be removed and download again." - rm -f "$DESC_DIR/$FILENAME" + echo "Archive ${FILENAME} will be removed and download again." + rm -f "${DESC_DIR}/${FILENAME}" else - echo "Downloading $FILENAME from $DOWNLOAD_URL to $DESC_DIR" - wget --no-check-certificate -q $DOWNLOAD_URL -O $DESC_DIR/$FILENAME - if [ "$?"x == "0"x ]; then - if md5sum_func $FILENAME $DESC_DIR $MD5SUM; then + echo "Downloading ${FILENAME} from ${DOWNLOAD_URL} to ${DESC_DIR}" + if wget --no-check-certificate -q "${DOWNLOAD_URL}" -O "${DESC_DIR}/${FILENAME}"; then + if md5sum_func "${FILENAME}" "${DESC_DIR}" "${MD5SUM}"; then STATUS=0 - echo "Success to download $FILENAME" - break; + echo "Success to download ${FILENAME}" + break fi - echo "Archive $FILENAME will be removed and download again." - rm -f "$DESC_DIR/$FILENAME" + echo "Archive ${FILENAME} will be removed and download again." + rm -f "${DESC_DIR}/${FILENAME}" else - echo "Failed to download $FILENAME. attemp: $attemp" + echo "Failed to download ${FILENAME}. attemp: ${attemp}" fi fi done - if [ $STATUS -ne 0 ]; then - echo "Failed to download $FILENAME" + if [[ "${STATUS}" -ne 0 ]]; then + echo "Failed to download ${FILENAME}" fi - return $STATUS + return "${STATUS}" } # download thirdparty archives echo "===== Downloading thirdparty archives..." -for TP_ARCH in ${TP_ARCHIVES[*]} -do - NAME=$TP_ARCH"_NAME" - MD5SUM=$TP_ARCH"_MD5SUM" - if test "x$REPOSITORY_URL" = x; then - URL=$TP_ARCH"_DOWNLOAD" - download_func ${!NAME} ${!URL} $TP_SOURCE_DIR ${!MD5SUM} - if [ "$?"x != "0"x ]; then +for TP_ARCH in "${TP_ARCHIVES[@]}"; do + NAME="${TP_ARCH}_NAME" + MD5SUM="${TP_ARCH}_MD5SUM" + if [[ -z "${REPOSITORY_URL}" ]]; then + URL="${TP_ARCH}_DOWNLOAD" + if ! download_func "${!NAME}" "${!URL}" "${TP_SOURCE_DIR}" "${!MD5SUM}"; then echo "Failed to download ${!NAME}" exit 1 fi else URL="${REPOSITORY_URL}/${!NAME}" - download_func ${!NAME} ${URL} $TP_SOURCE_DIR ${!MD5SUM} - if [ "$?x" != "0x" ]; then + if ! download_func "${!NAME}" "${URL}" "${TP_SOURCE_DIR}" "${!MD5SUM}"; then #try to download from home - URL=$TP_ARCH"_DOWNLOAD" - download_func ${!NAME} ${!URL} $TP_SOURCE_DIR ${!MD5SUM} - if [ "$?"x != "0"x ]; then + URL="${TP_ARCH}_DOWNLOAD" + if ! download_func "${!NAME}" "${!URL}" "${TP_SOURCE_DIR}" "${!MD5SUM}"; then echo "Failed to download ${!NAME}" exit 1 # download failed again exit. fi @@ -158,10 +155,9 @@ echo "===== Downloading thirdparty archives...done" # check if all tp archives exists echo "===== Checking all thirdpart archives..." -for TP_ARCH in ${TP_ARCHIVES[*]} -do - NAME=$TP_ARCH"_NAME" - if [ ! -r $TP_SOURCE_DIR/${!NAME} ]; then +for TP_ARCH in "${TP_ARCHIVES[@]}"; do + NAME="${TP_ARCH}_NAME" + if [[ ! -r "${TP_SOURCE_DIR}/${!NAME}" ]]; then echo "Failed to fetch ${!NAME}" exit 1 fi @@ -176,37 +172,36 @@ SUFFIX_TGZ="\.(tar\.gz|tgz)$" SUFFIX_XZ="\.tar\.xz$" SUFFIX_ZIP="\.zip$" SUFFIX_BZ2="\.tar\.bz2$" -for TP_ARCH in ${TP_ARCHIVES[*]} -do - NAME=$TP_ARCH"_NAME" - SOURCE=$TP_ARCH"_SOURCE" +for TP_ARCH in "${TP_ARCHIVES[@]}"; do + NAME="${TP_ARCH}_NAME" + SOURCE="${TP_ARCH}_SOURCE" - if [ -z "${!SOURCE}" ]; then + if [[ -z "${!SOURCE}" ]]; then continue fi - if [ ! -d $TP_SOURCE_DIR/${!SOURCE} ]; then - if [[ "${!NAME}" =~ $SUFFIX_TGZ ]]; then - echo "$TP_SOURCE_DIR/${!NAME}" - echo "$TP_SOURCE_DIR/${!SOURCE}" - if ! $TAR_CMD xzf "$TP_SOURCE_DIR/${!NAME}" -C "$TP_SOURCE_DIR/"; then + if [[ ! -d "${TP_SOURCE_DIR}/${!SOURCE}" ]]; then + if [[ "${!NAME}" =~ ${SUFFIX_TGZ} ]]; then + echo "${TP_SOURCE_DIR}/${!NAME}" + echo "${TP_SOURCE_DIR}/${!SOURCE}" + if ! "${TAR_CMD}" xzf "${TP_SOURCE_DIR}/${!NAME}" -C "${TP_SOURCE_DIR}/"; then echo "Failed to untar ${!NAME}" exit 1 fi - elif [[ "${!NAME}" =~ $SUFFIX_XZ ]]; then - echo "$TP_SOURCE_DIR/${!NAME}" - echo "$TP_SOURCE_DIR/${!SOURCE}" - if ! $TAR_CMD xJf "$TP_SOURCE_DIR/${!NAME}" -C "$TP_SOURCE_DIR/"; then + elif [[ "${!NAME}" =~ ${SUFFIX_XZ} ]]; then + echo "${TP_SOURCE_DIR}/${!NAME}" + echo "${TP_SOURCE_DIR}/${!SOURCE}" + if ! "${TAR_CMD}" xJf "${TP_SOURCE_DIR}/${!NAME}" -C "${TP_SOURCE_DIR}/"; then echo "Failed to untar ${!NAME}" exit 1 fi - elif [[ "${!NAME}" =~ $SUFFIX_ZIP ]]; then - if ! $UNZIP_CMD -o -qq "$TP_SOURCE_DIR/${!NAME}" -d "$TP_SOURCE_DIR/"; then + elif [[ "${!NAME}" =~ ${SUFFIX_ZIP} ]]; then + if ! "${UNZIP_CMD}" -o -qq "${TP_SOURCE_DIR}/${!NAME}" -d "${TP_SOURCE_DIR}/"; then echo "Failed to unzip ${!NAME}" exit 1 fi - elif [[ "${!NAME}" =~ $SUFFIX_BZ2 ]]; then - if ! $TAR_CMD xf "$TP_SOURCE_DIR/${!NAME}" -C "$TP_SOURCE_DIR/"; then + elif [[ "${!NAME}" =~ ${SUFFIX_BZ2} ]]; then + if ! "${TAR_CMD}" xf "${TP_SOURCE_DIR}/${!NAME}" -C "${TP_SOURCE_DIR}/"; then echo "Failed to untar ${!NAME}" exit 1 fi @@ -227,134 +222,132 @@ echo "===== Patching thirdparty archives..." PATCHED_MARK="patched_mark" # glog patch -cd $TP_SOURCE_DIR/$GLOG_SOURCE -if [ ! -f $PATCHED_MARK ]; then - patch -p1 < $TP_PATCH_DIR/glog-0.4.0.patch - touch $PATCHED_MARK +cd "${TP_SOURCE_DIR}/${GLOG_SOURCE}" +if [[ ! -f "${PATCHED_MARK}" ]]; then + patch -p1 <"${TP_PATCH_DIR}/glog-0.4.0.patch" + touch "${PATCHED_MARK}" fi cd - -echo "Finished patching $GLOG_SOURCE" +echo "Finished patching ${GLOG_SOURCE}" # gtest patch -cd $TP_SOURCE_DIR/$GTEST_SOURCE -if [ ! -f $PATCHED_MARK ]; then - patch -p1 < $TP_PATCH_DIR/googletest-release-1.11.0.patch - touch $PATCHED_MARK +cd "${TP_SOURCE_DIR}/${GTEST_SOURCE}" +if [[ ! -f "${PATCHED_MARK}" ]]; then + patch -p1 <"${TP_PATCH_DIR}/googletest-release-1.11.0.patch" + touch "${PATCHED_MARK}" fi cd - -echo "Finished patching $GTEST_SOURCE" +echo "Finished patching ${GTEST_SOURCE}" # mysql patch -cd $TP_SOURCE_DIR/$MYSQL_SOURCE -if [ ! -f $PATCHED_MARK ]; then - patch -p1 < $TP_PATCH_DIR/mysql-server-mysql-5.7.18.patch - touch $PATCHED_MARK +cd "${TP_SOURCE_DIR}/${MYSQL_SOURCE}" +if [[ ! -f "${PATCHED_MARK}" ]]; then + patch -p1 <"${TP_PATCH_DIR}/mysql-server-mysql-5.7.18.patch" + touch "${PATCHED_MARK}" fi cd - -echo "Finished patching $MYSQL_SOURCE" +echo "Finished patching ${MYSQL_SOURCE}" # libevent patch -cd $TP_SOURCE_DIR/$LIBEVENT_SOURCE -if [ ! -f $PATCHED_MARK ]; then - patch -p1 < $TP_PATCH_DIR/libevent.patch - touch $PATCHED_MARK +cd "${TP_SOURCE_DIR}/${LIBEVENT_SOURCE}" +if [[ ! -f "${PATCHED_MARK}" ]]; then + patch -p1 <"${TP_PATCH_DIR}/libevent.patch" + touch "${PATCHED_MARK}" fi cd - -echo "Finished patching $LIBEVENT_SOURCE" +echo "Finished patching ${LIBEVENT_SOURCE}" # s2 patch to disable shared library -cd $TP_SOURCE_DIR/$S2_SOURCE -if [ ! -f $PATCHED_MARK ]; then - patch -p1 < $TP_PATCH_DIR/s2geometry-0.9.0.patch - touch $PATCHED_MARK +cd "${TP_SOURCE_DIR}/${S2_SOURCE}" +if [[ ! -f "${PATCHED_MARK}" ]]; then + patch -p1 <"${TP_PATCH_DIR}/s2geometry-0.9.0.patch" + touch "${PATCHED_MARK}" fi cd - -echo "Finished patching $S2_SOURCE" +echo "Finished patching ${S2_SOURCE}" # gsasl2 patch to fix link error such as mutilple func defination # when link target with kerberos -cd $TP_SOURCE_DIR/$GSASL_SOURCE -if [ ! -f $PATCHED_MARK ]; then - patch -p1 < $TP_PATCH_DIR/libgsasl-1.8.0.patch - touch $PATCHED_MARK +cd "${TP_SOURCE_DIR}/${GSASL_SOURCE}" +if [[ ! -f ${PATCHED_MARK} ]]; then + patch -p1 <"${TP_PATCH_DIR}/libgsasl-1.8.0.patch" + touch "${PATCHED_MARK}" fi cd - -echo "Finished patching $GSASL_SOURCE" +echo "Finished patching ${GSASL_SOURCE}" # rocksdb patch to fix compile error -if [ $ROCKSDB_SOURCE == "rocksdb-5.14.2" ]; then - cd $TP_SOURCE_DIR/$ROCKSDB_SOURCE - if [ ! -f $PATCHED_MARK ]; then - patch -p1 < $TP_PATCH_DIR/rocksdb-5.14.2.patch - touch $PATCHED_MARK +if [[ "${ROCKSDB_SOURCE}" == "rocksdb-5.14.2" ]]; then + cd "${TP_SOURCE_DIR}/${ROCKSDB_SOURCE}" + if [[ ! -f "${PATCHED_MARK}" ]]; then + patch -p1 <"${TP_PATCH_DIR}/rocksdb-5.14.2.patch" + touch "${PATCHED_MARK}" fi cd - fi -echo "Finished patching $ROCKSDB_SOURCE" +echo "Finished patching ${ROCKSDB_SOURCE}" # opentelemetry patch is used to solve the problem that threadlocal depends on GLIBC_2.18 # see: https://github.com/apache/doris/pull/7911 -if [ $OPENTELEMETRY_SOURCE == "opentelemetry-cpp-1.4.0" ]; then - rm -rf $TP_SOURCE_DIR/$OPENTELEMETRY_SOURCE/third_party/opentelemetry-proto/* - cp -r $TP_SOURCE_DIR/$OPENTELEMETRY_PROTO_SOURCE/* $TP_SOURCE_DIR/$OPENTELEMETRY_SOURCE/third_party/opentelemetry-proto - mkdir -p $TP_SOURCE_DIR/$OPENTELEMETRY_SOURCE/third_party/opentelemetry-proto/.git +if [[ "${OPENTELEMETRY_SOURCE}" == "opentelemetry-cpp-1.4.0" ]]; then + rm -rf "${TP_SOURCE_DIR}/${OPENTELEMETRY_SOURCE}/third_party/opentelemetry-proto"/* + cp -r "${TP_SOURCE_DIR}/${OPENTELEMETRY_PROTO_SOURCE}"/* "${TP_SOURCE_DIR}/${OPENTELEMETRY_SOURCE}/third_party/opentelemetry-proto" + mkdir -p "${TP_SOURCE_DIR}/${OPENTELEMETRY_SOURCE}/third_party/opentelemetry-proto/.git" - cd $TP_SOURCE_DIR/$OPENTELEMETRY_SOURCE - if [ ! -f $PATCHED_MARK ]; then - patch -p1 < $TP_PATCH_DIR/opentelemetry-cpp-1.4.0.patch - touch $PATCHED_MARK + cd "${TP_SOURCE_DIR}/${OPENTELEMETRY_SOURCE}" + if [[ ! -f "${PATCHED_MARK}" ]]; then + patch -p1 <"${TP_PATCH_DIR}/opentelemetry-cpp-1.4.0.patch" + touch "${PATCHED_MARK}" fi cd - fi -echo "Finished patching $OPENTELEMETRY_SOURCE" +echo "Finished patching ${OPENTELEMETRY_SOURCE}" # arrow patch is used to get the raw orc reader for filter prune. -if [ $ARROW_SOURCE == "apache-arrow-7.0.0" ]; then - cd $TP_SOURCE_DIR/$ARROW_SOURCE - if [ ! -f $PATCHED_MARK ]; then - patch -p1 < $TP_PATCH_DIR/apache-arrow-7.0.0.patch - touch $PATCHED_MARK +if [[ "${ARROW_SOURCE}" == "apache-arrow-7.0.0" ]]; then + cd "${TP_SOURCE_DIR}/${ARROW_SOURCE}" + if [[ ! -f "${PATCHED_MARK}" ]]; then + patch -p1 <"${TP_PATCH_DIR}/apache-arrow-7.0.0.patch" + touch "${PATCHED_MARK}" fi cd - fi -echo "Finished patching $ARROW_SOURCE" +echo "Finished patching ${ARROW_SOURCE}" # patch librdkafka to avoid crash -if [ $LIBRDKAFKA_SOURCE == "librdkafka-1.8.2" ]; then - cd $TP_SOURCE_DIR/$LIBRDKAFKA_SOURCE - if [ ! -f $PATCHED_MARK ]; then - patch -p0 < $TP_PATCH_DIR/librdkafka-1.8.2.patch - touch $PATCHED_MARK +if [[ "${LIBRDKAFKA_SOURCE}" == "librdkafka-1.8.2" ]]; then + cd "${TP_SOURCE_DIR}/${LIBRDKAFKA_SOURCE}" + if [[ ! -f "${PATCHED_MARK}" ]]; then + patch -p0 <"${TP_PATCH_DIR}/librdkafka-1.8.2.patch" + touch "${PATCHED_MARK}" fi cd - fi -echo "Finished patching $LIBRDKAFKA_SOURCE" +echo "Finished patching ${LIBRDKAFKA_SOURCE}" # patch hyperscan # https://github.com/intel/hyperscan/issues/292 -if [ $HYPERSCAN_SOURCE == "hyperscan-5.4.0" ]; then - cd $TP_SOURCE_DIR/$HYPERSCAN_SOURCE - if [ ! -f $PATCHED_MARK ]; then - patch -p0 < $TP_PATCH_DIR/hyperscan-5.4.0.patch - touch $PATCHED_MARK +if [[ "${HYPERSCAN_SOURCE}" == "hyperscan-5.4.0" ]]; then + cd "${TP_SOURCE_DIR}/${HYPERSCAN_SOURCE}" + if [[ ! -f "${PATCHED_MARK}" ]]; then + patch -p0 <"${TP_PATCH_DIR}/hyperscan-5.4.0.patch" + touch "${PATCHED_MARK}" fi cd - -elif [ $HYPERSCAN_SOURCE == "vectorscan-vectorscan-5.4.7" ]; then - cd $TP_SOURCE_DIR/$HYPERSCAN_SOURCE - if [ ! -f $PATCHED_MARK ]; then - patch -p0 < $TP_PATCH_DIR/vectorscan-5.4.7.patch - touch $PATCHED_MARK +elif [[ "${HYPERSCAN_SOURCE}" == "vectorscan-vectorscan-5.4.7" ]]; then + cd "${TP_SOURCE_DIR}/${HYPERSCAN_SOURCE}" + if [[ ! -f "${PATCHED_MARK}" ]]; then + patch -p0 <"${TP_PATCH_DIR}/vectorscan-5.4.7.patch" + touch "${PATCHED_MARK}" fi cd - fi -echo "Finished patching $HYPERSCAN_SOURCE" +echo "Finished patching ${HYPERSCAN_SOURCE}" -cd $TP_SOURCE_DIR/$AWS_SDK_SOURCE -if [ ! -f $PATCHED_MARK ]; then - if [ $AWS_SDK_SOURCE == "aws-sdk-cpp-1.9.211" ]; then - wget --no-check-certificate -q https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/aws-crt-cpp-1.9.211.tar.gz -O aws-crt-cpp-1.9.211.tar.gz - ret="$?" - if [ $ret -eq 0 ] ; then +cd "${TP_SOURCE_DIR}/${AWS_SDK_SOURCE}" +if [[ ! -f "${PATCHED_MARK}" ]]; then + if [[ "${AWS_SDK_SOURCE}" == "aws-sdk-cpp-1.9.211" ]]; then + if wget --no-check-certificate -q https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/aws-crt-cpp-1.9.211.tar.gz -O aws-crt-cpp-1.9.211.tar.gz; then tar xzf aws-crt-cpp-1.9.211.tar.gz else bash ./prefetch_crt_dependency.sh @@ -362,16 +355,15 @@ if [ ! -f $PATCHED_MARK ]; then else bash ./prefetch_crt_dependency.sh fi - touch $PATCHED_MARK + touch "${PATCHED_MARK}" fi cd - -echo "Finished patching $AWS_SDK_SOURCE" +echo "Finished patching ${AWS_SDK_SOURCE}" cd "${TP_SOURCE_DIR}/${BRPC_SOURCE}" -if [[ ! -f $PATCHED_MARK ]]; then +if [[ ! -f "${PATCHED_MARK}" ]]; then patch -p1 <"${TP_PATCH_DIR}/brpc-1.2.0.patch" - touch ${PATCHED_MARK} + touch "${PATCHED_MARK}" fi cd - echo "Finished patching ${BRPC_SOURCE}" - diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index 0ed5f039e4..96148f6628 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -1,4 +1,6 @@ #!/bin/bash +# shellcheck disable=2034 + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -27,22 +29,22 @@ ################################################### # thirdparties will be downloaded and unpacked here -export TP_SOURCE_DIR=$TP_DIR/src +export TP_SOURCE_DIR="${TP_DIR:-.}/src" # thirdparties will be installed to here -export TP_INSTALL_DIR=$TP_DIR/installed +export TP_INSTALL_DIR="${TP_DIR:-.}/installed" # patches for all thirdparties -export TP_PATCH_DIR=$TP_DIR/patches +export TP_PATCH_DIR="${TP_DIR:-.}/patches" # header files of all thirdparties will be intalled to here -export TP_INCLUDE_DIR=$TP_INSTALL_DIR/include +export TP_INCLUDE_DIR="${TP_INSTALL_DIR}/include" # libraries of all thirdparties will be intalled to here -export TP_LIB_DIR=$TP_INSTALL_DIR/lib +export TP_LIB_DIR="${TP_INSTALL_DIR}/lib" # all java libraries will be unpacked to here -export TP_JAR_DIR=$TP_INSTALL_DIR/lib/jar +export TP_JAR_DIR="${TP_INSTALL_DIR}/lib/jar" # source of all dependencies, default unuse it # export REPOSITORY_URL= @@ -422,62 +424,64 @@ SSE2NEON_SOURCE=sse2neon-1.5.1 SSE2NEON_MD5SUM="9de5dc2970aa7efac7faee59e2826c51" # all thirdparties which need to be downloaded is set in array TP_ARCHIVES -export TP_ARCHIVES="LIBEVENT -OPENSSL -THRIFT -PROTOBUF -GFLAGS -GLOG -GTEST -RAPIDJSON -SNAPPY -GPERFTOOLS -ZLIB -LZ4 -BZIP -LZO2 -CURL -RE2 -HYPERSCAN -RAGEL -BOOST -MYSQL -ODBC -LEVELDB -BRPC -ROCKSDB -CYRUS_SASL -LIBRDKAFKA -FLATBUFFERS -ARROW -BROTLI -ZSTD -S2 -BITSHUFFLE -CROARINGBITMAP -FMT -PARALLEL_HASHMAP -ORC -JEMALLOC -CCTZ -DATATABLES -BOOTSTRAP_TABLE_JS -BOOTSTRAP_TABLE_CSS -TSAN_HEADER -AWS_SDK -LZMA -XML2 -IDN -GSASL -KRB5 -HDFS3 -LIBDIVIDE -PDQSORT -BENCHMARK -XSIMD -SIMDJSON -NLOHMANN_JSON -OPENTELEMETRY_PROTO -OPENTELEMETRY -LIBBACKTRACE -SSE2NEON" +export TP_ARCHIVES=( + 'LIBEVENT' + 'OPENSSL' + 'THRIFT' + 'PROTOBUF' + 'GFLAGS' + 'GLOG' + 'GTEST' + 'RAPIDJSON' + 'SNAPPY' + 'GPERFTOOLS' + 'ZLIB' + 'LZ4' + 'BZIP' + 'LZO2' + 'CURL' + 'RE2' + 'HYPERSCAN' + 'RAGEL' + 'BOOST' + 'MYSQL' + 'ODBC' + 'LEVELDB' + 'BRPC' + 'ROCKSDB' + 'CYRUS_SASL' + 'LIBRDKAFKA' + 'FLATBUFFERS' + 'ARROW' + 'BROTLI' + 'ZSTD' + 'S2' + 'BITSHUFFLE' + 'CROARINGBITMAP' + 'FMT' + 'PARALLEL_HASHMAP' + 'ORC' + 'JEMALLOC' + 'CCTZ' + 'DATATABLES' + 'BOOTSTRAP_TABLE_JS' + 'BOOTSTRAP_TABLE_CSS' + 'TSAN_HEADER' + 'AWS_SDK' + 'LZMA' + 'XML2' + 'IDN' + 'GSASL' + 'KRB5' + 'HDFS3' + 'LIBDIVIDE' + 'PDQSORT' + 'BENCHMARK' + 'XSIMD' + 'SIMDJSON' + 'NLOHMANN_JSON' + 'OPENTELEMETRY_PROTO' + 'OPENTELEMETRY' + 'LIBBACKTRACE' + 'SSE2NEON' +) diff --git a/tools/find_libjvm.sh b/tools/find_libjvm.sh index 4017616874..039342070d 100644 --- a/tools/find_libjvm.sh +++ b/tools/find_libjvm.sh @@ -18,43 +18,40 @@ jdk_version() { local result - local java_cmd=$JAVA_HOME/bin/java + local java_cmd="${JAVA_HOME:-.}/bin/java" local IFS=$'\n' - # remove \r for Cygwin - local lines=$("$java_cmd" -Xms32M -Xmx32M -version 2>&1 | tr '\r' '\n') - if [[ -z $java_cmd ]] - then + + if [[ -z "${java_cmd}" ]]; then result=no_java + return 1 else - for line in $lines; do - if [[ (-z $result) && ($line = *"version \""*) ]] - then - local ver=$(echo $line | sed -e 's/.*version "\(.*\)"\(.*\)/\1/; 1q') - # on macOS, sed doesn't support '?' - if [[ $ver = "1."* ]] - then - result=$(echo $ver | sed -e 's/1\.\([0-9]*\)\(.*\)/\1/; 1q') - else - result=$(echo $ver | sed -e 's/\([0-9]*\)\(.*\)/\1/; 1q') - fi - fi - done + local version + # remove \r for Cygwin + version="$("${java_cmd}" -Xms32M -Xmx32M -version 2>&1 | tr '\r' '\n' | grep version | awk '{print $3}')" + version="${version//\"/}" + if [[ "${version}" =~ ^1\. ]]; then + result="$(echo "${version}" | awk -F '.' '{print $2}')" + else + result="$(echo "${version}" | awk -F '.' '{print $1}')" + fi fi - echo "$result" + echo "${result}" + return 0 } + java_version=$(jdk_version) MACHINE_TYPE=$(uname -m) jvm_arch="amd64" if [[ "${MACHINE_TYPE}" == "aarch64" ]]; then jvm_arch="aarch64" fi -if [[ $java_version -gt 8 ]]; then - export LIBJVM_PATH=$JAVA_HOME/lib +if [[ "${java_version}" -gt 8 ]]; then + export LIBJVM_PATH="${JAVA_HOME}/lib" # JAVA_HOME is jdk -elif [[ -d "$JAVA_HOME/jre" ]]; then - export LIBJVM_PATH=$JAVA_HOME/jre/lib/$jvm_arch +elif [[ -d "${JAVA_HOME}/jre" ]]; then + export LIBJVM_PATH="${JAVA_HOME}/jre/lib/${jvm_arch}" # JAVA_HOME is jre else - export LIBJVM_PATH=$JAVA_HOME/lib/$jvm_arch + export LIBJVM_PATH="${JAVA_HOME}/lib/${jvm_arch}" fi -echo ${LIBJVM_PATH}/*/libjvm.so +echo "${LIBJVM_PATH}"/*/libjvm.so diff --git a/tools/restore_tablet_tool.sh b/tools/restore_tablet_tool.sh index 17c1e2ca4f..f98d90d97a 100644 --- a/tools/restore_tablet_tool.sh +++ b/tools/restore_tablet_tool.sh @@ -39,63 +39,81 @@ Usage: $0 Examples: batch mode: - sh restore_tablet_tool.sh -b "http://127.0.0.1:8040" -f tablets.txt - sh restore_tablet_tool.sh --backend "http://127.0.0.1:8040" --file tablets.txt + bash restore_tablet_tool.sh -b http://127.0.0.1:8040 -f tablets.txt + bash restore_tablet_tool.sh --backend http://127.0.0.1:8040 --file tablets.txt single mode: - sh restore_tablet_tool.sh -b "http://127.0.0.1:8040" -t 12345 -s 11111 - sh restore_tablet_tool.sh --backend "http://127.0.0.1:8040" --tablet_id 12345 --schema_hash 11111 + bash restore_tablet_tool.sh -b http://127.0.0.1:8040 -t 12345 -s 11111 + bash restore_tablet_tool.sh --backend http://127.0.0.1:8040 --tablet_id 12345 --schema_hash 11111 " exit 1 } -OPTS=$(getopt \ - -n $0 \ - -o 'b:t:s:f:' \ - -l 'server:,tablet_id:,schema_hash:,file:,help' \ - -- "$@") - -if [ $? != 0 ] ; then +if ! OPTS=$(getopt \ + -n "$0" \ + -o 'b:t:s:f:' \ + -l 'server:,tablet_id:,schema_hash:,file:,help' \ + -- "$@"); then usage fi -eval set -- "$OPTS" +eval set -- "${OPTS}" -SERVER="http://127.0.0.1/8040" -TABLET_ID= -SCHEMA_HASH= -FILENAME= +SERVER='http://127.0.0.1/8040' +TABLET_ID='' +SCHEMA_HASH='' +FILENAME='' BATCH_MODE=false while true; do case "$1" in - -b|--backend) SERVER=$2 ; shift 2 ;; - -f|--file) FILENAME=$2 ; BATCH_MODE=true ; shift 2 ;; - -t|--tablet_id) TABLET_ID=$2 ; shift 2 ;; - -s|--schema_hash) SCHEMA_HASH=$2 ; shift 2 ;; - -h|--help) usage ; shift ;; - --) shift ; break ;; - *) echo "Internal error!" ; exit 1 ;; + -b | --backend) + SERVER="$2" + shift 2 + ;; + -f | --file) + FILENAME="$2" + BATCH_MODE=true + shift 2 + ;; + -t | --tablet_id) + TABLET_ID="$2" + shift 2 + ;; + -s | --schema_hash) + SCHEMA_HASH="$2" + shift 2 + ;; + -h | --help) + usage + shift + ;; + --) + shift + break + ;; + *) + echo "Internal error!" + exit 1 + ;; esac done restore_tablet() { - echo "start to restore tablet id:"$2", schema hash:"$3 + echo "start to restore tablet id:$2, schema hash:$3" curl -X POST "$1/api/restore_tablet?tablet_id=$2&schema_hash=$3" echo -e "\n" } -if [ $BATCH_MODE = true ] ; then - lines=`cat $FILENAME` - for line in $lines - do +if ${BATCH_MODE}; then + while read -r line; do # split the comma-split line # format: tablet_id,schema_hash - fields=(${line/,/ }) - TABLET_ID=${fields[0]} - SCHEMA_HASH=${fields[1]} - restore_tablet $SERVER $TABLET_ID $SCHEMA_HASH - done + read -r -a fields <<<"${line/,/ }" + TABLET_ID="${fields[0]}" + SCHEMA_HASH="${fields[1]}" + restore_tablet "${SERVER}" "${TABLET_ID}" "${SCHEMA_HASH}" + done <"${FILENAME}" else - restore_tablet $SERVER $TABLET_ID $SCHEMA_HASH + restore_tablet "${SERVER}" "${TABLET_ID}" "${SCHEMA_HASH}" fi