[chore](workflow) Add shellcheck to check shell scripts (#11744)
This commit is contained in:
Adonis Ling
1
.github/actions/action-sh-checker
vendored
Submodule
1
.github/actions/action-sh-checker
vendored
Submodule
Submodule .github/actions/action-sh-checker added at edd0e45ecf
38
.github/workflows/shellcheck.yml
vendored
Normal file
38
.github/workflows/shellcheck.yml
vendored
Normal file
@ -0,0 +1,38 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
name: ShellCheck
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
shellcheck:
|
||||
name: ShellCheck
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout ${{ github.ref }} ( ${{ github.sha }} )
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Run ShellCheck
|
||||
uses: ./.github/actions/action-sh-checker
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
sh_checker_comment: true
|
||||
sh_checker_exclude: .git .github ^thirdparty/src ^thirdparty/installed ^ui ^docs/node_modules ^tools/ssb-tools ^tools/tpch-tools ^tools/clickbench-tools ^extension ^output ^fs_brokers/apache_hdfs_broker/output
|
||||
3
.gitmodules
vendored
3
.gitmodules
vendored
@ -19,3 +19,6 @@
|
||||
[submodule ".github/actions/ccache-action"]
|
||||
path = .github/actions/ccache-action
|
||||
url = https://github.com/hendrikmuhs/ccache-action
|
||||
[submodule ".github/actions/action-sh-checker"]
|
||||
path = .github/actions/action-sh-checker
|
||||
url = https://github.com/luizm/action-sh-checker
|
||||
|
||||
27
.shellcheckrc
Normal file
27
.shellcheckrc
Normal file
@ -0,0 +1,27 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
source-path=SCRIPTDIR
|
||||
source-path=SCRIPTDIR/..
|
||||
source-path=SCRIPTDIR/../..
|
||||
source-path=thirdparty
|
||||
|
||||
external-sources=true
|
||||
|
||||
enable=all
|
||||
disable=SC2310
|
||||
disable=SC2312
|
||||
166
bin/start_be.sh
166
bin/start_be.sh
@ -16,19 +16,17 @@
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
curdir=$(dirname "$0")
|
||||
curdir=$(
|
||||
cd "$curdir"
|
||||
pwd
|
||||
)
|
||||
set -eo pipefail
|
||||
|
||||
OPTS=$(getopt \
|
||||
-n $0 \
|
||||
curdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
|
||||
|
||||
OPTS="$(getopt \
|
||||
-n "$0" \
|
||||
-o '' \
|
||||
-l 'daemon' \
|
||||
-- "$@")
|
||||
-- "$@")"
|
||||
|
||||
eval set -- "$OPTS"
|
||||
eval set -- "${OPTS}"
|
||||
|
||||
RUN_DAEMON=0
|
||||
RUN_IN_AWS=0
|
||||
@ -53,69 +51,69 @@ while true; do
|
||||
esac
|
||||
done
|
||||
|
||||
export DORIS_HOME=$(
|
||||
cd "$curdir/.."
|
||||
DORIS_HOME="$(
|
||||
cd "${curdir}/.."
|
||||
pwd
|
||||
)
|
||||
)"
|
||||
export DORIS_HOME
|
||||
|
||||
MAX_MAP_COUNT=`sysctl -n vm.max_map_count`
|
||||
if [ $MAX_MAP_COUNT -lt 2000000 ]; then
|
||||
MAX_MAP_COUNT="$(sysctl -n vm.max_map_count)"
|
||||
if [[ "${MAX_MAP_COUNT}" -lt 2000000 ]]; then
|
||||
echo "Please set vm.max_map_count to be 2000000. sysctl -w vm.max_map_count=2000000"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# add libs to CLASSPATH
|
||||
for f in $DORIS_HOME/lib/*.jar; do
|
||||
if [ ! -n "${DORIS_JNI_CLASSPATH_PARAMETER}" ]; then
|
||||
export DORIS_JNI_CLASSPATH_PARAMETER=$f
|
||||
else
|
||||
export DORIS_JNI_CLASSPATH_PARAMETER=$f:${DORIS_JNI_CLASSPATH_PARAMETER}
|
||||
fi
|
||||
for f in "${DORIS_HOME}/lib"/*.jar; do
|
||||
if [[ -z "${DORIS_JNI_CLASSPATH_PARAMETER}" ]]; then
|
||||
export DORIS_JNI_CLASSPATH_PARAMETER="${f}"
|
||||
else
|
||||
export DORIS_JNI_CLASSPATH_PARAMETER="${f}:${DORIS_JNI_CLASSPATH_PARAMETER}"
|
||||
fi
|
||||
done
|
||||
# DORIS_JNI_CLASSPATH_PARAMETER is used to configure additional jar path to jvm. e.g. -Djava.class.path=$DORIS_HOME/lib/java-udf.jar
|
||||
export DORIS_JNI_CLASSPATH_PARAMETER="-Djava.class.path=${DORIS_JNI_CLASSPATH_PARAMETER}"
|
||||
|
||||
jdk_version() {
|
||||
local result
|
||||
local java_cmd=$JAVA_HOME/bin/java
|
||||
local java_cmd="${JAVA_HOME:-.}/bin/java"
|
||||
local IFS=$'\n'
|
||||
# remove \r for Cygwin
|
||||
local lines=$("$java_cmd" -Xms32M -Xmx32M -version 2>&1 | tr '\r' '\n')
|
||||
if [[ -z $java_cmd ]]
|
||||
then
|
||||
|
||||
if [[ -z "${java_cmd}" ]]; then
|
||||
result=no_java
|
||||
return 1
|
||||
else
|
||||
for line in $lines; do
|
||||
if [[ (-z $result) && ($line = *"version \""*) ]]
|
||||
then
|
||||
local ver=$(echo $line | sed -e 's/.*version "\(.*\)"\(.*\)/\1/; 1q')
|
||||
# on macOS, sed doesn't support '?'
|
||||
if [[ $ver = "1."* ]]
|
||||
then
|
||||
result=$(echo $ver | sed -e 's/1\.\([0-9]*\)\(.*\)/\1/; 1q')
|
||||
else
|
||||
result=$(echo $ver | sed -e 's/\([0-9]*\)\(.*\)/\1/; 1q')
|
||||
fi
|
||||
fi
|
||||
done
|
||||
local version
|
||||
# remove \r for Cygwin
|
||||
version="$("${java_cmd}" -Xms32M -Xmx32M -version 2>&1 | tr '\r' '\n' | grep version | awk '{print $3}')"
|
||||
version="${version//\"/}"
|
||||
if [[ "${version}" =~ ^1\. ]]; then
|
||||
result="$(echo "${version}" | awk -F '.' '{print $2}')"
|
||||
else
|
||||
result="$(echo "${version}" | awk -F '.' '{print $1}')"
|
||||
fi
|
||||
fi
|
||||
echo "$result"
|
||||
echo "${result}"
|
||||
return 0
|
||||
}
|
||||
|
||||
jvm_arch="amd64"
|
||||
MACHINE_TYPE=$(uname -m)
|
||||
MACHINE_TYPE="$(uname -m)"
|
||||
if [[ "${MACHINE_TYPE}" == "aarch64" ]]; then
|
||||
jvm_arch="aarch64"
|
||||
fi
|
||||
java_version=$(jdk_version)
|
||||
if [[ $java_version -gt 8 ]]; then
|
||||
export LD_LIBRARY_PATH=$JAVA_HOME/lib/server:$JAVA_HOME/lib:$LD_LIBRARY_PATH
|
||||
java_version="$(
|
||||
set -e
|
||||
jdk_version
|
||||
)"
|
||||
if [[ "${java_version}" -gt 8 ]]; then
|
||||
export LD_LIBRARY_PATH="${JAVA_HOME}/lib/server:${JAVA_HOME}/lib:${LD_LIBRARY_PATH}"
|
||||
# JAVA_HOME is jdk
|
||||
elif [[ -d "$JAVA_HOME/jre" ]]; then
|
||||
export LD_LIBRARY_PATH=$JAVA_HOME/jre/lib/$jvm_arch/server:$JAVA_HOME/jre/lib/$jvm_arch:$LD_LIBRARY_PATH
|
||||
elif [[ -d "${JAVA_HOME}/jre" ]]; then
|
||||
export LD_LIBRARY_PATH="${JAVA_HOME}/jre/lib/${jvm_arch}/server:${JAVA_HOME}/jre/lib/${jvm_arch}:${LD_LIBRARY_PATH}"
|
||||
# JAVA_HOME is jre
|
||||
else
|
||||
export LD_LIBRARY_PATH=$JAVA_HOME/lib/$jvm_arch/server:$JAVA_HOME/lib/$jvm_arch:$LD_LIBRARY_PATH
|
||||
export LD_LIBRARY_PATH="${JAVA_HOME}/lib/${jvm_arch}/server:${JAVA_HOME}/lib/${jvm_arch}:${LD_LIBRARY_PATH}"
|
||||
fi
|
||||
|
||||
# export env variables from be.conf
|
||||
@ -123,75 +121,81 @@ fi
|
||||
# UDF_RUNTIME_DIR
|
||||
# LOG_DIR
|
||||
# PID_DIR
|
||||
export UDF_RUNTIME_DIR=${DORIS_HOME}/lib/udf-runtime
|
||||
export LOG_DIR=${DORIS_HOME}/log
|
||||
export PID_DIR=$(
|
||||
cd "$curdir"
|
||||
export UDF_RUNTIME_DIR="${DORIS_HOME}/lib/udf-runtime"
|
||||
export LOG_DIR="${DORIS_HOME}/log"
|
||||
PID_DIR="$(
|
||||
cd "${curdir}"
|
||||
pwd
|
||||
)
|
||||
)"
|
||||
export PID_DIR
|
||||
|
||||
# set odbc conf path
|
||||
export ODBCSYSINI=$DORIS_HOME/conf
|
||||
export ODBCSYSINI="${DORIS_HOME}/conf"
|
||||
|
||||
# support utf8 for oracle database
|
||||
export NLS_LANG=AMERICAN_AMERICA.AL32UTF8
|
||||
export NLS_LANG='AMERICAN_AMERICA.AL32UTF8'
|
||||
|
||||
#filter known leak for lsan.
|
||||
export LSAN_OPTIONS=suppressions=${DORIS_HOME}/conf/asan_suppr.conf
|
||||
export LSAN_OPTIONS="suppressions=${DORIS_HOME}/conf/asan_suppr.conf"
|
||||
|
||||
while read line; do
|
||||
envline=$(echo $line | sed 's/[[:blank:]]*=[[:blank:]]*/=/g' | sed 's/^[[:blank:]]*//g' | egrep "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*=")
|
||||
envline=$(eval "echo $envline")
|
||||
if [[ $envline == *"="* ]]; then
|
||||
eval 'export "$envline"'
|
||||
while read -r line; do
|
||||
envline="$(echo "${line}" |
|
||||
sed 's/[[:blank:]]*=[[:blank:]]*/=/g' |
|
||||
sed 's/^[[:blank:]]*//g' |
|
||||
grep -E "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*=" ||
|
||||
true)"
|
||||
envline="$(eval "echo ${envline}")"
|
||||
if [[ "${envline}" == *"="* ]]; then
|
||||
eval 'export "${envline}"'
|
||||
fi
|
||||
done < $DORIS_HOME/conf/be.conf
|
||||
done <"${DORIS_HOME}/conf/be.conf"
|
||||
|
||||
if [ -e $DORIS_HOME/bin/palo_env.sh ]; then
|
||||
source $DORIS_HOME/bin/palo_env.sh
|
||||
if [[ -e "${DORIS_HOME}/bin/palo_env.sh" ]]; then
|
||||
# shellcheck disable=1091
|
||||
source "${DORIS_HOME}/bin/palo_env.sh"
|
||||
fi
|
||||
|
||||
if [ ! -d $LOG_DIR ]; then
|
||||
mkdir -p $LOG_DIR
|
||||
if [[ ! -d "${LOG_DIR}" ]]; then
|
||||
mkdir -p "${LOG_DIR}"
|
||||
fi
|
||||
|
||||
if [ ! -d $UDF_RUNTIME_DIR ]; then
|
||||
mkdir -p ${UDF_RUNTIME_DIR}
|
||||
if [[ ! -d "${UDF_RUNTIME_DIR}" ]]; then
|
||||
mkdir -p "${UDF_RUNTIME_DIR}"
|
||||
fi
|
||||
|
||||
rm -f ${UDF_RUNTIME_DIR}/*
|
||||
rm -f "${UDF_RUNTIME_DIR}"/*
|
||||
|
||||
pidfile=$PID_DIR/be.pid
|
||||
pidfile="${PID_DIR}/be.pid"
|
||||
|
||||
if [ -f $pidfile ]; then
|
||||
if kill -0 $(cat $pidfile) > /dev/null 2>&1; then
|
||||
echo "Backend running as process $(cat $pidfile). Stop it first."
|
||||
if [[ -f "${pidfile}" ]]; then
|
||||
if kill -0 "$(cat "${pidfile}")" >/dev/null 2>&1; then
|
||||
echo "Backend running as process $(cat "${pidfile}"). Stop it first."
|
||||
exit 1
|
||||
else
|
||||
rm $pidfile
|
||||
rm "${pidfile}"
|
||||
fi
|
||||
fi
|
||||
|
||||
chmod 755 ${DORIS_HOME}/lib/doris_be
|
||||
echo "start time: "$(date) >> $LOG_DIR/be.out
|
||||
chmod 755 "${DORIS_HOME}/lib/doris_be"
|
||||
echo "start time: $(date)" >>"${LOG_DIR}/be.out"
|
||||
|
||||
if [ ! -f /bin/limit3 ]; then
|
||||
LIMIT=
|
||||
if [[ ! -f '/bin/limit3' ]]; then
|
||||
LIMIT=''
|
||||
else
|
||||
LIMIT="/bin/limit3 -c 0 -n 65536"
|
||||
fi
|
||||
|
||||
## If you are not running in aws cloud, disable this env since https://github.com/aws/aws-sdk-cpp/issues/1410.
|
||||
if [ ${RUN_IN_AWS} -eq 0 ]; then
|
||||
if [[ "${RUN_IN_AWS}" -eq 0 ]]; then
|
||||
export AWS_EC2_METADATA_DISABLED=true
|
||||
fi
|
||||
|
||||
## set hdfs conf
|
||||
export LIBHDFS3_CONF=${DORIS_HOME}/conf/hdfs-site.xml
|
||||
export LIBHDFS3_CONF="${DORIS_HOME}/conf/hdfs-site.xml"
|
||||
|
||||
if [ ${RUN_DAEMON} -eq 1 ]; then
|
||||
nohup $LIMIT ${DORIS_HOME}/lib/doris_be "$@" >> $LOG_DIR/be.out 2>&1 < /dev/null &
|
||||
if [[ "${RUN_DAEMON}" -eq 1 ]]; then
|
||||
nohup ${LIMIT:+${LIMIT}} "${DORIS_HOME}/lib/doris_be" "$@" >>"${LOG_DIR}/be.out" 2>&1 </dev/null &
|
||||
else
|
||||
export DORIS_LOG_TO_STDERR=1
|
||||
$LIMIT ${DORIS_HOME}/lib/doris_be "$@" 2>&1 < /dev/null
|
||||
${LIMIT:+${LIMIT}} "${DORIS_HOME}/lib/doris_be" "$@" 2>&1 </dev/null
|
||||
fi
|
||||
|
||||
164
bin/start_fe.sh
164
bin/start_fe.sh
@ -16,28 +16,26 @@
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
curdir=$(dirname "$0")
|
||||
curdir=$(
|
||||
cd "$curdir"
|
||||
pwd
|
||||
)
|
||||
set -eo pipefail
|
||||
|
||||
OPTS=$(getopt \
|
||||
-n $0 \
|
||||
curdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
|
||||
|
||||
OPTS="$(getopt \
|
||||
-n "$0" \
|
||||
-o '' \
|
||||
-l 'daemon' \
|
||||
-l 'helper:' \
|
||||
-l 'image:' \
|
||||
-l 'version' \
|
||||
-- "$@")
|
||||
-- "$@")"
|
||||
|
||||
eval set -- "$OPTS"
|
||||
eval set -- "${OPTS}"
|
||||
|
||||
RUN_DAEMON=0
|
||||
HELPER=
|
||||
IMAGE_PATH=
|
||||
IMAGE_TOOL=
|
||||
OPT_VERSION=
|
||||
HELPER=''
|
||||
IMAGE_PATH=''
|
||||
IMAGE_TOOL=''
|
||||
OPT_VERSION=''
|
||||
while true; do
|
||||
case "$1" in
|
||||
--daemon)
|
||||
@ -49,12 +47,12 @@ while true; do
|
||||
shift
|
||||
;;
|
||||
--helper)
|
||||
HELPER=$2
|
||||
HELPER="$2"
|
||||
shift 2
|
||||
;;
|
||||
--image)
|
||||
IMAGE_TOOL=1
|
||||
IMAGE_PATH=$2
|
||||
IMAGE_PATH="$2"
|
||||
shift 2
|
||||
;;
|
||||
--)
|
||||
@ -68,10 +66,11 @@ while true; do
|
||||
esac
|
||||
done
|
||||
|
||||
export DORIS_HOME=$(
|
||||
cd "$curdir/.."
|
||||
DORIS_HOME="$(
|
||||
cd "${curdir}/.."
|
||||
pwd
|
||||
)
|
||||
)"
|
||||
export DORIS_HOME
|
||||
|
||||
# export env variables from fe.conf
|
||||
#
|
||||
@ -79,31 +78,37 @@ export DORIS_HOME=$(
|
||||
# LOG_DIR
|
||||
# PID_DIR
|
||||
export JAVA_OPTS="-Xmx1024m"
|
||||
export LOG_DIR="$DORIS_HOME/log"
|
||||
export PID_DIR=$(
|
||||
cd "$curdir"
|
||||
export LOG_DIR="${DORIS_HOME}/log"
|
||||
PID_DIR="$(
|
||||
cd "${curdir}"
|
||||
pwd
|
||||
)
|
||||
)"
|
||||
export PID_DIR
|
||||
|
||||
while read line; do
|
||||
envline=$(echo $line | sed 's/[[:blank:]]*=[[:blank:]]*/=/g' | sed 's/^[[:blank:]]*//g' | egrep "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*=")
|
||||
envline=$(eval "echo $envline")
|
||||
if [[ $envline == *"="* ]]; then
|
||||
eval 'export "$envline"'
|
||||
while read -r line; do
|
||||
envline="$(echo "${line}" |
|
||||
sed 's/[[:blank:]]*=[[:blank:]]*/=/g' |
|
||||
sed 's/^[[:blank:]]*//g' |
|
||||
grep -E "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*=" ||
|
||||
true)"
|
||||
envline="$(eval "echo ${envline}")"
|
||||
if [[ "${envline}" == *"="* ]]; then
|
||||
eval 'export "${envline}"'
|
||||
fi
|
||||
done <$DORIS_HOME/conf/fe.conf
|
||||
done <"${DORIS_HOME}/conf/fe.conf"
|
||||
|
||||
if [ -e $DORIS_HOME/bin/palo_env.sh ]; then
|
||||
source $DORIS_HOME/bin/palo_env.sh
|
||||
if [[ -e "${DORIS_HOME}/bin/palo_env.sh" ]]; then
|
||||
# shellcheck disable=1091
|
||||
source "${DORIS_HOME}/bin/palo_env.sh"
|
||||
fi
|
||||
|
||||
if [ -z "$JAVA_HOME" ]; then
|
||||
JAVA=$(which java)
|
||||
if [[ -z "${JAVA_HOME}" ]]; then
|
||||
JAVA="$(which java)"
|
||||
else
|
||||
JAVA="$JAVA_HOME/bin/java"
|
||||
JAVA="${JAVA_HOME}/bin/java"
|
||||
fi
|
||||
|
||||
if [ ! -x "$JAVA" ]; then
|
||||
if [[ ! -x "${JAVA}" ]]; then
|
||||
echo "The JAVA_HOME environment variable is not defined correctly"
|
||||
echo "This environment variable is needed to run this program"
|
||||
echo "NB: JAVA_HOME should point to a JDK not a JRE"
|
||||
@ -114,80 +119,87 @@ fi
|
||||
# 1.8 => 8, 13.0 => 13
|
||||
jdk_version() {
|
||||
local result
|
||||
local java_cmd="${JAVA_HOME:-.}/bin/java"
|
||||
local IFS=$'\n'
|
||||
# remove \r for Cygwin
|
||||
local lines=$("$JAVA" -Xms32M -Xmx32M -version 2>&1 | tr '\r' '\n')
|
||||
for line in $lines; do
|
||||
if [[ (-z $result) && ($line = *"version \""*) ]]; then
|
||||
local ver=$(echo $line | sed -e 's/.*version "\(.*\)"\(.*\)/\1/; 1q')
|
||||
# on macOS, sed doesn't support '?'
|
||||
if [[ $ver = "1."* ]]; then
|
||||
result=$(echo $ver | sed -e 's/1\.\([0-9]*\)\(.*\)/\1/; 1q')
|
||||
else
|
||||
result=$(echo $ver | sed -e 's/\([0-9]*\)\(.*\)/\1/; 1q')
|
||||
fi
|
||||
|
||||
if [[ -z "${java_cmd}" ]]; then
|
||||
result=no_java
|
||||
return 1
|
||||
else
|
||||
local version
|
||||
# remove \r for Cygwin
|
||||
version="$("${java_cmd}" -Xms32M -Xmx32M -version 2>&1 | tr '\r' '\n' | grep version | awk '{print $3}')"
|
||||
version="${version//\"/}"
|
||||
if [[ "${version}" =~ ^1\. ]]; then
|
||||
result="$(echo "${version}" | awk -F '.' '{print $2}')"
|
||||
else
|
||||
result="$(echo "${version}" | awk -F '.' '{print $1}')"
|
||||
fi
|
||||
done
|
||||
echo "$result"
|
||||
fi
|
||||
echo "${result}"
|
||||
return 0
|
||||
}
|
||||
|
||||
# need check and create if the log directory existed before outing message to the log file.
|
||||
if [ ! -d $LOG_DIR ]; then
|
||||
mkdir -p $LOG_DIR
|
||||
if [[ ! -d "${LOG_DIR}" ]]; then
|
||||
mkdir -p "${LOG_DIR}"
|
||||
fi
|
||||
|
||||
# check java version and choose correct JAVA_OPTS
|
||||
java_version=$(jdk_version)
|
||||
final_java_opt=$JAVA_OPTS
|
||||
if [ $java_version -gt 8 ]; then
|
||||
if [ -z "$JAVA_OPTS_FOR_JDK_9" ]; then
|
||||
echo "JAVA_OPTS_FOR_JDK_9 is not set in fe.conf" >>$LOG_DIR/fe.out
|
||||
java_version="$(
|
||||
set -e
|
||||
jdk_version
|
||||
)"
|
||||
final_java_opt="${JAVA_OPTS}"
|
||||
if [[ "${java_version}" -gt 8 ]]; then
|
||||
if [[ -z "${JAVA_OPTS_FOR_JDK_9}" ]]; then
|
||||
echo "JAVA_OPTS_FOR_JDK_9 is not set in fe.conf" >>"${LOG_DIR}/fe.out"
|
||||
exit 1
|
||||
fi
|
||||
final_java_opt=$JAVA_OPTS_FOR_JDK_9
|
||||
final_java_opt="${JAVA_OPTS_FOR_JDK_9}"
|
||||
fi
|
||||
echo "using java version $java_version" >>$LOG_DIR/fe.out
|
||||
echo $final_java_opt >>$LOG_DIR/fe.out
|
||||
echo "using java version ${java_version}" >>"${LOG_DIR}/fe.out"
|
||||
echo "${final_java_opt}" >>"${LOG_DIR}/fe.out"
|
||||
|
||||
# add libs to CLASSPATH
|
||||
for f in $DORIS_HOME/lib/*.jar; do
|
||||
CLASSPATH=$f:${CLASSPATH}
|
||||
for f in "${DORIS_HOME}/lib"/*.jar; do
|
||||
CLASSPATH="${f}:${CLASSPATH}"
|
||||
done
|
||||
export CLASSPATH=${CLASSPATH}:${DORIS_HOME}/lib:${DORIS_HOME}/conf
|
||||
export CLASSPATH="${CLASSPATH}:${DORIS_HOME}/lib:${DORIS_HOME}/conf"
|
||||
|
||||
pidfile=$PID_DIR/fe.pid
|
||||
pidfile="${PID_DIR}/fe.pid"
|
||||
|
||||
if [ -f $pidfile ]; then
|
||||
if kill -0 $(cat $pidfile) >/dev/null 2>&1; then
|
||||
echo Frontend running as process $(cat $pidfile). Stop it first.
|
||||
if [[ -f "${pidfile}" ]]; then
|
||||
if kill -0 "$(cat "${pidfile}")" >/dev/null 2>&1; then
|
||||
echo "Frontend running as process $(cat "${pidfile}"). Stop it first."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ! -f /bin/limit ]; then
|
||||
LIMIT=
|
||||
if [[ ! -f "/bin/limit" ]]; then
|
||||
LIMIT=''
|
||||
else
|
||||
LIMIT=/bin/limit
|
||||
fi
|
||||
|
||||
echo $(date) >>$LOG_DIR/fe.out
|
||||
date >>"${LOG_DIR}/fe.out"
|
||||
|
||||
if [ x"$HELPER" != x"" ]; then
|
||||
if [[ "${HELPER}" != "" ]]; then
|
||||
# change it to '-helper' to be compatible with code in Frontend
|
||||
HELPER="-helper $HELPER"
|
||||
HELPER="-helper ${HELPER}"
|
||||
fi
|
||||
|
||||
if [[ ${IMAGE_TOOL} -eq 1 ]]; then
|
||||
if [ ! -z ${IMAGE_PATH} ]; then
|
||||
$LIMIT $JAVA $final_java_opt org.apache.doris.PaloFe -i ${IMAGE_PATH}
|
||||
if [[ "${IMAGE_TOOL}" -eq 1 ]]; then
|
||||
if [[ -n "${IMAGE_PATH}" ]]; then
|
||||
${LIMIT:+${LIMIT}} "${JAVA}" ${final_java_opt:+${final_java_opt}} org.apache.doris.PaloFe -i "${IMAGE_PATH}"
|
||||
else
|
||||
echo "Internal Error. USE IMAGE_TOOL like : ./start_fe.sh --image image_path"
|
||||
fi
|
||||
elif [[ ${RUN_DAEMON} -eq 1 ]]; then
|
||||
nohup $LIMIT $JAVA $final_java_opt -XX:OnOutOfMemoryError="kill -9 %p" org.apache.doris.PaloFe ${HELPER} "$@" >>$LOG_DIR/fe.out 2>&1 </dev/null &
|
||||
elif [[ "${RUN_DAEMON}" -eq 1 ]]; then
|
||||
nohup ${LIMIT:+${LIMIT}} "${JAVA}" ${final_java_opt:+${final_java_opt}} -XX:OnOutOfMemoryError="kill -9 %p" org.apache.doris.PaloFe ${HELPER:+${HELPER}} "$@" >>"${LOG_DIR}/fe.out" 2>&1 </dev/null &
|
||||
else
|
||||
export DORIS_LOG_TO_STDERR=1
|
||||
$LIMIT $JAVA $final_java_opt -XX:OnOutOfMemoryError="kill -9 %p" org.apache.doris.PaloFe ${HELPER} ${OPT_VERSION} "$@" </dev/null
|
||||
${LIMIT:+${LIMIT}} "${JAVA}" ${final_java_opt:+${final_java_opt}} -XX:OnOutOfMemoryError="kill -9 %p" org.apache.doris.PaloFe ${HELPER:+${HELPER}} ${OPT_VERSION:+${OPT_VERSION}} "$@" </dev/null
|
||||
fi
|
||||
|
||||
echo $! >$pidfile
|
||||
echo $! >"${pidfile}"
|
||||
|
||||
@ -16,75 +16,68 @@
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
curdir=$(dirname "$0")
|
||||
curdir=$(
|
||||
cd "$curdir"
|
||||
pwd
|
||||
)
|
||||
set -eo pipefail
|
||||
|
||||
export DORIS_HOME=$(
|
||||
cd "$curdir/.."
|
||||
curdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
|
||||
|
||||
DORIS_HOME="$(
|
||||
cd "${curdir}/.."
|
||||
pwd
|
||||
)
|
||||
export PID_DIR=$(
|
||||
cd "$curdir"
|
||||
)"
|
||||
export DORIS_HOME
|
||||
|
||||
PID_DIR="$(
|
||||
cd "${curdir}"
|
||||
pwd
|
||||
)
|
||||
)"
|
||||
export PID_DIR
|
||||
|
||||
signum=9
|
||||
if [ "x"$1 = "x--grace" ]; then
|
||||
if [[ "$1" = "--grace" ]]; then
|
||||
signum=15
|
||||
fi
|
||||
|
||||
while read line; do
|
||||
envline=$(echo $line | sed 's/[[:blank:]]*=[[:blank:]]*/=/g' | sed 's/^[[:blank:]]*//g' | egrep "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*=")
|
||||
envline=$(eval "echo $envline")
|
||||
if [[ $envline == *"="* ]]; then
|
||||
eval 'export "$envline"'
|
||||
fi
|
||||
done <$DORIS_HOME/conf/be.conf
|
||||
pidfile="${PID_DIR}/be.pid"
|
||||
|
||||
pidfile=$PID_DIR/be.pid
|
||||
|
||||
if [ -f $pidfile ]; then
|
||||
pid=$(cat $pidfile)
|
||||
if [[ -f "${pidfile}" ]]; then
|
||||
pid="$(cat "${pidfile}")"
|
||||
|
||||
# check if pid valid
|
||||
if test -z "$pid"; then
|
||||
if test -z "${pid}"; then
|
||||
echo "ERROR: invalid pid."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# check if pid process exist
|
||||
if ! kill -0 $pid 2>&1; then
|
||||
echo "ERROR: be process $pid does not exist."
|
||||
if ! kill -0 "${pid}" 2>&1; then
|
||||
echo "ERROR: be process ${pid} does not exist."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
pidcomm=$(ps -p $pid -o comm=)
|
||||
pidcomm="$(ps -p "${pid}" -o comm=)"
|
||||
# check if pid process is backend process
|
||||
if [ "doris_be"x != "$pidcomm"x ]; then
|
||||
if [[ "doris_be" != "${pidcomm}" ]]; then
|
||||
echo "ERROR: pid process may not be be. "
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# kill pid process and check it
|
||||
if kill -${signum} $pid >/dev/null 2>&1; then
|
||||
if kill "-${signum}" "${pid}" >/dev/null 2>&1; then
|
||||
while true; do
|
||||
if kill -0 $pid >/dev/null 2>&1; then
|
||||
echo "waiting be to stop, pid: $pid"
|
||||
if kill -0 "${pid}" >/dev/null 2>&1; then
|
||||
echo "waiting be to stop, pid: ${pid}"
|
||||
sleep 2
|
||||
else
|
||||
echo "stop $pidcomm, and remove pid file. "
|
||||
if [ -f $pidfile ]; then rm $pidfile; fi
|
||||
echo "stop ${pidcomm}, and remove pid file. "
|
||||
if [[ -f "${pidfile}" ]]; then rm "${pidfile}"; fi
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
else
|
||||
echo "ERROR: failed to stop $pid"
|
||||
echo "ERROR: failed to stop ${pid}"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "ERROR: $pidfile does not exist"
|
||||
echo "ERROR: ${pidfile} does not exist"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@ -16,75 +16,68 @@
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
curdir=$(dirname "$0")
|
||||
curdir=$(
|
||||
cd "$curdir"
|
||||
pwd
|
||||
)
|
||||
set -eo pipefail
|
||||
|
||||
export DORIS_HOME=$(
|
||||
cd "$curdir/.."
|
||||
curdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
|
||||
|
||||
DORIS_HOME="$(
|
||||
cd "${curdir}/.."
|
||||
pwd
|
||||
)
|
||||
export PID_DIR=$(
|
||||
cd "$curdir"
|
||||
)"
|
||||
export DORIS_HOME
|
||||
|
||||
PID_DIR="$(
|
||||
cd "${curdir}"
|
||||
pwd
|
||||
)
|
||||
)"
|
||||
export PID_DIR
|
||||
|
||||
signum=9
|
||||
if [ "x"$1 = "x--grace" ]; then
|
||||
if [[ "$1" = "--grace" ]]; then
|
||||
signum=15
|
||||
fi
|
||||
|
||||
while read line; do
|
||||
envline=$(echo $line | sed 's/[[:blank:]]*=[[:blank:]]*/=/g' | sed 's/^[[:blank:]]*//g' | egrep "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*=")
|
||||
envline=$(eval "echo $envline")
|
||||
if [[ $envline == *"="* ]]; then
|
||||
eval 'export "$envline"'
|
||||
fi
|
||||
done <$DORIS_HOME/conf/fe.conf
|
||||
pidfile="${PID_DIR}/fe.pid"
|
||||
|
||||
pidfile=$PID_DIR/fe.pid
|
||||
|
||||
if [ -f $pidfile ]; then
|
||||
pid=$(cat $pidfile)
|
||||
if [[ -f "${pidfile}" ]]; then
|
||||
pid="$(cat "${pidfile}")"
|
||||
|
||||
# check if pid valid
|
||||
if test -z "$pid"; then
|
||||
if test -z "${pid}"; then
|
||||
echo "ERROR: invalid pid."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# check if pid process exist
|
||||
if ! kill -0 $pid 2>&1; then
|
||||
echo "ERROR: fe process $pid does not exist."
|
||||
if ! kill -0 "${pid}" 2>&1; then
|
||||
echo "ERROR: fe process ${pid} does not exist."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
pidcomm=$(ps -p $pid -o comm=)
|
||||
pidcomm="$(ps -p "${pid}" -o comm=)"
|
||||
# check if pid process is frontend process
|
||||
if [ "java"x != "$pidcomm"x ]; then
|
||||
if [[ "java" != "${pidcomm}" ]]; then
|
||||
echo "ERROR: pid process may not be fe. "
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# kill pid process and check it
|
||||
if kill -${signum} $pid >/dev/null 2>&1; then
|
||||
if kill "-${signum}" "${pid}" >/dev/null 2>&1; then
|
||||
while true; do
|
||||
if kill -0 $pid >/dev/null 2>&1; then
|
||||
echo "waiting fe to stop, pid: $pid"
|
||||
if kill -0 "${pid}" >/dev/null 2>&1; then
|
||||
echo "waiting fe to stop, pid: ${pid}"
|
||||
sleep 2
|
||||
else
|
||||
echo "stop $pidcomm, and remove pid file. "
|
||||
if [ -f $pidfile ]; then rm $pidfile; fi
|
||||
echo "stop ${pidcomm}, and remove pid file. "
|
||||
if [[ -f "${pidfile}" ]]; then rm "${pidfile}"; fi
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
else
|
||||
echo "ERROR: failed to stop $pid"
|
||||
echo "ERROR: failed to stop ${pid}"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "ERROR: $pidfile does not exist"
|
||||
echo "ERROR: ${pidfile} does not exist"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@ -23,11 +23,14 @@
|
||||
|
||||
set -eo pipefail
|
||||
|
||||
ROOT=`dirname "$0"`
|
||||
ROOT=`cd "$ROOT"; pwd`
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
|
||||
|
||||
export DORIS_HOME=`cd "${ROOT}/.."; pwd`
|
||||
DORIS_HOME=$(
|
||||
cd "${ROOT}/.."
|
||||
pwd
|
||||
)
|
||||
export DORIS_HOME
|
||||
|
||||
CLANG_FORMAT=${CLANG_FORMAT_BINARY:=$(which clang-format)}
|
||||
CLANG_FORMAT="${CLANG_FORMAT_BINARY:=$(which clang-format)}"
|
||||
|
||||
python ${DORIS_HOME}/build-support/run_clang_format.py "--clang-format-executable" "${CLANG_FORMAT}" "-r" "--style" "file" "--inplace" "false" "--extensions" "c,h,C,H,cpp,hpp,cc,hh,c++,h++,cxx,hxx" "--exclude" "none" "be/src be/test"
|
||||
python "${DORIS_HOME}/build-support/run_clang_format.py" "--clang-format-executable" "${CLANG_FORMAT}" "-r" "--style" "file" "--inplace" "false" "--extensions" "c,h,C,H,cpp,hpp,cc,hh,c++,h++,cxx,hxx" "--exclude" "none" "be/src be/test"
|
||||
|
||||
@ -23,11 +23,14 @@
|
||||
|
||||
set -eo pipefail
|
||||
|
||||
ROOT=`dirname "$0"`
|
||||
ROOT=`cd "$ROOT"; pwd`
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
|
||||
|
||||
export DORIS_HOME=`cd "${ROOT}/.."; pwd`
|
||||
DORIS_HOME=$(
|
||||
cd "${ROOT}/.."
|
||||
pwd
|
||||
)
|
||||
export DORIS_HOME
|
||||
|
||||
CLANG_FORMAT=${CLANG_FORMAT_BINARY:=$(which clang-format)}
|
||||
CLANG_FORMAT="${CLANG_FORMAT_BINARY:=$(which clang-format)}"
|
||||
|
||||
python ${DORIS_HOME}/build-support/run_clang_format.py "--clang-format-executable" "${CLANG_FORMAT}" "-r" "--style" "file" "--inplace" "true" "--extensions" "c,h,C,H,cpp,hpp,cc,hh,c++,h++,cxx,hxx" "--exclude" "none" "be/src be/test"
|
||||
python "${DORIS_HOME}/build-support/run_clang_format.py" "--clang-format-executable" "${CLANG_FORMAT}" "-r" "--style" "file" "--inplace" "true" "--extensions" "c,h,C,H,cpp,hpp,cc,hh,c++,h++,cxx,hxx" "--exclude" "none" "be/src be/test"
|
||||
|
||||
322
build-support/shell-check.sh
Executable file
322
build-support/shell-check.sh
Executable file
@ -0,0 +1,322 @@
|
||||
#!/bin/bash
|
||||
# shellcheck disable=2311
|
||||
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
set -eo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
|
||||
DORIS_HOME="$(
|
||||
cd "${SCRIPT_DIR}/.." &>/dev/null
|
||||
pwd
|
||||
)"
|
||||
DORIS_THIRDPARTY="$(
|
||||
source "${DORIS_HOME}/env.sh" &>/dev/null
|
||||
echo "${DORIS_THIRDPARTY}"
|
||||
)"
|
||||
INSTALL_PATH="${DORIS_THIRDPARTY}/installed/bin"
|
||||
|
||||
SHELLCHECK=''
|
||||
SHFMT=''
|
||||
|
||||
log() {
|
||||
local level="${1}"
|
||||
local message="${2}"
|
||||
local date
|
||||
date="$(date +'%Y-%m-%d %H:%M:%S')"
|
||||
if [[ "${level}" == 'INFO' ]]; then
|
||||
level="[\033[32;1m ${level} \033[0m]"
|
||||
elif [[ "${level}" == 'WARNING' ]]; then
|
||||
level="[\033[33;1m${level}\033[0m]"
|
||||
elif [[ "${level}" == 'ERROR' ]]; then
|
||||
level="[\033[31;1m ${level} \033[0m]"
|
||||
fi
|
||||
echo -e "${level} ${date} - ${message}"
|
||||
}
|
||||
|
||||
log_info() {
|
||||
local message="${1}"
|
||||
log 'INFO' "${message}"
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
local message="${1}"
|
||||
log 'WARNING' "${message}"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
local message="${1}"
|
||||
log 'ERROR' "${message}"
|
||||
exit 1
|
||||
}
|
||||
|
||||
get_version() {
|
||||
local program="${1}"
|
||||
local tool="${program##*/}"
|
||||
|
||||
case "${tool}" in
|
||||
'shellcheck') "${program}" --version | sed -n 's/version: \(.*\)/\1/p' ;;
|
||||
'shfmt') "${program}" --version ;;
|
||||
*) ;;
|
||||
esac
|
||||
}
|
||||
|
||||
check_tool() {
|
||||
local tool="${1}"
|
||||
local version="${2}"
|
||||
local program
|
||||
|
||||
if program="$(command -v "${tool}" 2>/dev/null)" && [[ ! "$(get_version "${program}")" < "${version}" ]]; then
|
||||
echo "${program}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
program="${INSTALL_PATH}/${tool}"
|
||||
if [[ -f "${program}" && ! "$(get_version "${program}")" < "${version}" ]]; then
|
||||
echo "${program}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
return 255
|
||||
}
|
||||
|
||||
get_url() {
|
||||
local tool="${1}"
|
||||
local os
|
||||
local arch
|
||||
|
||||
os="$(uname -s | awk '{ print tolower($0) }')"
|
||||
arch="$(uname -m | awk '{ print tolower($0) }')"
|
||||
|
||||
if [[ "${tool}" == 'shellcheck' ]]; then
|
||||
if [[ "${arch}" == 'arm64' ]]; then
|
||||
arch='aarch64'
|
||||
fi
|
||||
echo "https://github.com/koalaman/shellcheck/releases/download/v0.8.0/shellcheck-v0.8.0.${os}.${arch}.tar.xz"
|
||||
return 0
|
||||
elif [[ "${tool}" == 'shfmt' ]]; then
|
||||
if [[ "${arch}" == 'x86_64' ]]; then
|
||||
arch='amd64'
|
||||
elif [[ "${arch}" == 'aarch64' ]]; then
|
||||
arch='arm64'
|
||||
fi
|
||||
echo "https://github.com/mvdan/sh/releases/download/v3.5.1/shfmt_v3.5.1_${os}_${arch}"
|
||||
return 0
|
||||
fi
|
||||
return 255
|
||||
}
|
||||
|
||||
get_md5() {
|
||||
local tool="${1}"
|
||||
local os
|
||||
local arch
|
||||
|
||||
os="$(uname -s | awk '{ print tolower($0) }')"
|
||||
arch="$(uname -m | awk '{ print tolower($0) }')"
|
||||
|
||||
case "${tool}" in
|
||||
'shellcheck')
|
||||
case "${os}" in
|
||||
'linux')
|
||||
case "${arch}" in
|
||||
'x86_64') echo '86ee889b1e771bc8292a7043df4b962a' ;;
|
||||
'arm64' | 'aarch64') echo 'a0338c733d1283a51777b27edf9ccc96' ;;
|
||||
*) ;;
|
||||
esac
|
||||
;;
|
||||
'darwin')
|
||||
case "${arch}" in
|
||||
'x86_64') echo 'e744840256a77a1a277e81a3032f7bf4' ;;
|
||||
*) ;;
|
||||
esac
|
||||
;;
|
||||
*) ;;
|
||||
esac
|
||||
;;
|
||||
|
||||
'shfmt')
|
||||
case "${os}" in
|
||||
'linux')
|
||||
case "${arch}" in
|
||||
'x86_64') echo '1d234f204e249bf1c524015ce842b117' ;;
|
||||
'arm64' | 'aarch64') echo '5c8494fd257d5cd9db1eef48af967e47' ;;
|
||||
*) ;;
|
||||
esac
|
||||
;;
|
||||
'darwin')
|
||||
case "${arch}" in
|
||||
'x86_64') echo '42f14325207a47f3177c96683c5085a4' ;;
|
||||
'arm64' | 'aarch64') echo 'a9c51e5d4aeebfa2c8cc70af3006bc4e' ;;
|
||||
*) ;;
|
||||
esac
|
||||
;;
|
||||
*) ;;
|
||||
esac
|
||||
;;
|
||||
*) ;;
|
||||
esac
|
||||
}
|
||||
|
||||
download_tool() {
|
||||
local url="${1}"
|
||||
local package="${2}"
|
||||
local md5="${3}"
|
||||
log_info "Download ${tool} from ${url}"
|
||||
|
||||
if [[ -f "${package}" && "$(md5sum "${package}" | awk '{print $1}')" == "${md5}" ]]; then
|
||||
log_info "${tool} has been downloaded succesfully!"
|
||||
return 0
|
||||
fi
|
||||
|
||||
for i in {0..2}; do
|
||||
if [[ "${i}" -gt 0 ]]; then
|
||||
log_warning "Retry #${i}..."
|
||||
fi
|
||||
if curl --connect-timeout 5 --speed-limit 1000 --speed-time 30 -L "${url}" -o "${package}" &&
|
||||
[[ "$(md5sum "${package}" | awk '{print $1}')" == "${md5}" ]]; then
|
||||
log_info "${tool} has been downloaded succesfully!"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 255
|
||||
}
|
||||
|
||||
install_shellcheck() {
|
||||
local tool="shellcheck"
|
||||
local url
|
||||
local package
|
||||
local md5
|
||||
local dir
|
||||
|
||||
SYSTEM_NAME="$(uname -s)"
|
||||
if [[ "${SYSTEM_NAME}" == 'Darwin' ]] && command -v 'brew' &>/dev/null; then
|
||||
brew install "${tool}"
|
||||
else
|
||||
url="$(get_url "${tool}")"
|
||||
package="${url##*/}"
|
||||
md5="$(get_md5 "${tool}")"
|
||||
download_tool "${url}" "${package}" "${md5}"
|
||||
|
||||
log_info "Set ${tool} up."
|
||||
dir="$(mktemp -d)"
|
||||
tar -xf "${package}" -C "${dir}" --strip-component=1
|
||||
mkdir -p "${INSTALL_PATH}"
|
||||
mv "${dir}/${tool}" "${INSTALL_PATH}"
|
||||
fi
|
||||
}
|
||||
|
||||
install_shfmt() {
|
||||
local tool='shfmt'
|
||||
local url
|
||||
local package
|
||||
local md5
|
||||
local dir
|
||||
|
||||
SYSTEM_NAME="$(uname -s)"
|
||||
if [[ "${SYSTEM_NAME}" == 'Darwin' ]] && command -v 'brew' &>/dev/null; then
|
||||
brew install "${tool}"
|
||||
else
|
||||
url="$(get_url "${tool}")"
|
||||
package="${url##*/}"
|
||||
md5="$(get_md5 "${tool}")"
|
||||
download_tool "${url}" "${package}" "${md5}"
|
||||
|
||||
log_info "Set ${tool} up."
|
||||
mkdir -p "${INSTALL_PATH}"
|
||||
cp "${package}" "${INSTALL_PATH}/shfmt"
|
||||
chmod a+x "${INSTALL_PATH}/shfmt"
|
||||
fi
|
||||
}
|
||||
|
||||
install_tools_if_neccessary() {
|
||||
while ! SHELLCHECK="$(check_tool 'shellcheck' '0.8.0')"; do
|
||||
log_warning 'shellcheck was not found.'
|
||||
install_shellcheck
|
||||
done
|
||||
log_info "shellcheck found: ${SHELLCHECK}"
|
||||
|
||||
while ! SHFMT="$(check_tool 'shfmt' '3.5.1')"; do
|
||||
log_warning 'shfmt was not found.'
|
||||
install_shfmt
|
||||
done
|
||||
log_info "shfmt found: ${SHFMT}"
|
||||
}
|
||||
|
||||
find_shell_scripts() {
|
||||
local path="${1}"
|
||||
local exclude_patterns
|
||||
local content
|
||||
local files=()
|
||||
content="$(grep 'sh_checker_exclude:' "${DORIS_HOME}/.github/workflows/shellcheck.yml")"
|
||||
read -r -a exclude_patterns <<<"${content#*sh_checker_exclude: }"
|
||||
while read -r file; do
|
||||
local matched=false
|
||||
for pattern in "${exclude_patterns[@]}"; do
|
||||
if echo "${file:((${#DORIS_HOME} + 1))}" | grep -E "${pattern}" &>/dev/null; then
|
||||
matched=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
if ! "${matched}"; then
|
||||
files+=("${file}")
|
||||
fi
|
||||
done < <(find "${path}" -type f -name '*.sh')
|
||||
echo "${files[@]}"
|
||||
}
|
||||
|
||||
run_tool() {
|
||||
set +e
|
||||
local tool="${1}"
|
||||
local opt="${2}"
|
||||
local status
|
||||
shift 2
|
||||
pushd "${DORIS_HOME}" >/dev/null
|
||||
log_info "Run tool: ${tool}"
|
||||
"${tool}" ${opt+${opt}} "${@}"
|
||||
status="${?}"
|
||||
popd >/dev/null
|
||||
return "${status}"
|
||||
}
|
||||
|
||||
run_shellcheck() {
|
||||
run_tool "${SHELLCHECK}" '' "${@}"
|
||||
}
|
||||
|
||||
run_shfmt() {
|
||||
run_tool "${SHFMT}" '-d' "${@}"
|
||||
}
|
||||
|
||||
main() {
|
||||
local files
|
||||
local shellcheck_result=0
|
||||
local shfmt_result=0
|
||||
|
||||
install_tools_if_neccessary
|
||||
read -r -a files < <(find_shell_scripts "${DORIS_HOME}")
|
||||
run_shellcheck "${files[@]}" || shellcheck_result="${?}"
|
||||
run_shfmt "${files[@]}" || shfmt_result="${?}"
|
||||
|
||||
if [[ "${shellcheck_result}" -ne 0 || "${shfmt_result}" -ne 0 ]]; then
|
||||
echo
|
||||
log_error 'Some issues were detected!'
|
||||
else
|
||||
log_info 'Success!'
|
||||
fi
|
||||
}
|
||||
|
||||
main "${@}"
|
||||
498
build.sh
498
build.sh
@ -27,16 +27,15 @@
|
||||
|
||||
set -eo pipefail
|
||||
|
||||
ROOT=`dirname "$0"`
|
||||
ROOT=`cd "$ROOT"; pwd`
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
|
||||
|
||||
export DORIS_HOME=${ROOT}
|
||||
export DORIS_HOME="${ROOT}"
|
||||
|
||||
. ${DORIS_HOME}/env.sh
|
||||
. "${DORIS_HOME}/env.sh"
|
||||
|
||||
# Check args
|
||||
usage() {
|
||||
echo "
|
||||
echo "
|
||||
Usage: $0 <options>
|
||||
Optional options:
|
||||
[no option] build all components
|
||||
@ -68,78 +67,76 @@ Usage: $0 <options>
|
||||
USE_AVX2=0 $0 --be build Backend and not using AVX2 instruction.
|
||||
USE_AVX2=0 STRIP_DEBUG_INFO=ON $0 build all and not using AVX2 instruction, and strip the debug info for Backend
|
||||
"
|
||||
exit 1
|
||||
exit 1
|
||||
}
|
||||
|
||||
clean_gensrc() {
|
||||
pushd ${DORIS_HOME}/gensrc
|
||||
pushd "${DORIS_HOME}/gensrc"
|
||||
make clean
|
||||
rm -rf ${DORIS_HOME}/fe/fe-common/target
|
||||
rm -rf ${DORIS_HOME}/fe/fe-core/target
|
||||
rm -rf "${DORIS_HOME}/fe/fe-common/target"
|
||||
rm -rf "${DORIS_HOME}/fe/fe-core/target"
|
||||
popd
|
||||
}
|
||||
|
||||
clean_be() {
|
||||
pushd ${DORIS_HOME}
|
||||
pushd "${DORIS_HOME}"
|
||||
|
||||
# "build.sh --clean" just cleans and exits, however CMAKE_BUILD_DIR is set
|
||||
# while building be.
|
||||
CMAKE_BUILD_TYPE=${BUILD_TYPE:-Release}
|
||||
CMAKE_BUILD_DIR=${DORIS_HOME}/be/build_${CMAKE_BUILD_TYPE}
|
||||
CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}"
|
||||
CMAKE_BUILD_DIR="${DORIS_HOME}/be/build_${CMAKE_BUILD_TYPE}"
|
||||
|
||||
rm -rf $CMAKE_BUILD_DIR
|
||||
rm -rf ${DORIS_HOME}/be/output/
|
||||
rm -rf "${CMAKE_BUILD_DIR}"
|
||||
rm -rf "${DORIS_HOME}/be/output"
|
||||
popd
|
||||
}
|
||||
|
||||
clean_fe() {
|
||||
pushd ${DORIS_HOME}/fe
|
||||
${MVN_CMD} clean
|
||||
pushd "${DORIS_HOME}/fe"
|
||||
"${MVN_CMD}" clean
|
||||
popd
|
||||
}
|
||||
|
||||
OPTS=$(getopt \
|
||||
-n $0 \
|
||||
-o '' \
|
||||
-l 'fe' \
|
||||
-l 'be' \
|
||||
-l 'broker' \
|
||||
-l 'audit' \
|
||||
-l 'meta-tool' \
|
||||
-l 'spark-dpp' \
|
||||
-l 'java-udf' \
|
||||
-l 'hive-udf' \
|
||||
-l 'clean' \
|
||||
-l 'help' \
|
||||
-o 'hj:' \
|
||||
-- "$@")
|
||||
|
||||
if [ $? != 0 ] ; then
|
||||
if ! OPTS="$(getopt \
|
||||
-n "$0" \
|
||||
-o '' \
|
||||
-l 'fe' \
|
||||
-l 'be' \
|
||||
-l 'broker' \
|
||||
-l 'audit' \
|
||||
-l 'meta-tool' \
|
||||
-l 'spark-dpp' \
|
||||
-l 'java-udf' \
|
||||
-l 'hive-udf' \
|
||||
-l 'clean' \
|
||||
-l 'help' \
|
||||
-o 'hj:' \
|
||||
-- "$@")"; then
|
||||
usage
|
||||
fi
|
||||
|
||||
eval set -- "$OPTS"
|
||||
eval set -- "${OPTS}"
|
||||
|
||||
PARALLEL=$[$(nproc)/4+1]
|
||||
PARALLEL="$(($(nproc) / 4 + 1))"
|
||||
BUILD_FE=0
|
||||
BUILD_BE=0
|
||||
BUILD_BROKER=0
|
||||
BUILD_AUDIT=0
|
||||
BUILD_META_TOOL=OFF
|
||||
BUILD_META_TOOL='OFF'
|
||||
BUILD_SPARK_DPP=0
|
||||
BUILD_JAVA_UDF=0
|
||||
BUILD_HIVE_UDF=0
|
||||
CLEAN=0
|
||||
HELP=0
|
||||
PARAMETER_COUNT=$#
|
||||
PARAMETER_COUNT="$#"
|
||||
PARAMETER_FLAG=0
|
||||
if [ $# == 1 ] ; then
|
||||
if [[ "$#" == 1 ]]; then
|
||||
# default
|
||||
BUILD_FE=1
|
||||
BUILD_BE=1
|
||||
BUILD_BROKER=1
|
||||
BUILD_AUDIT=1
|
||||
BUILD_META_TOOL=OFF
|
||||
BUILD_META_TOOL='OFF'
|
||||
BUILD_SPARK_DPP=1
|
||||
BUILD_JAVA_UDF=0 # TODO: open it when ready
|
||||
BUILD_HIVE_UDF=1
|
||||
@ -147,323 +144,378 @@ if [ $# == 1 ] ; then
|
||||
else
|
||||
while true; do
|
||||
case "$1" in
|
||||
--fe) BUILD_FE=1 BUILD_SPARK_DPP=1 ; shift ;;
|
||||
--be) BUILD_BE=1 ; shift ;;
|
||||
--broker) BUILD_BROKER=1 ; shift ;;
|
||||
--audit) BUILD_AUDIT=1 ; shift ;;
|
||||
--meta-tool) BUILD_META_TOOL=ON ; shift ;;
|
||||
--spark-dpp) BUILD_SPARK_DPP=1 ; shift ;;
|
||||
--java-udf) BUILD_JAVA_UDF=1 BUILD_FE=1 BUILD_SPARK_DPP=1 ; shift ;;
|
||||
--hive-udf) BUILD_HIVE_UDF=1 ; shift ;;
|
||||
--clean) CLEAN=1 ; shift ;;
|
||||
-h) HELP=1; shift ;;
|
||||
--help) HELP=1; shift ;;
|
||||
-j) PARALLEL=$2; PARAMETER_FLAG=1; shift 2 ;;
|
||||
--) shift ; break ;;
|
||||
*) echo "Internal error" ; exit 1 ;;
|
||||
--fe)
|
||||
BUILD_FE=1
|
||||
BUILD_SPARK_DPP=1
|
||||
shift
|
||||
;;
|
||||
--be)
|
||||
BUILD_BE=1
|
||||
shift
|
||||
;;
|
||||
--broker)
|
||||
BUILD_BROKER=1
|
||||
shift
|
||||
;;
|
||||
--audit)
|
||||
BUILD_AUDIT=1
|
||||
shift
|
||||
;;
|
||||
--meta-tool)
|
||||
BUILD_META_TOOL='ON'
|
||||
shift
|
||||
;;
|
||||
--spark-dpp)
|
||||
BUILD_SPARK_DPP=1
|
||||
shift
|
||||
;;
|
||||
--java-udf)
|
||||
BUILD_JAVA_UDF=1
|
||||
BUILD_FE=1
|
||||
BUILD_SPARK_DPP=1
|
||||
shift
|
||||
;;
|
||||
--hive-udf)
|
||||
BUILD_HIVE_UDF=1
|
||||
shift
|
||||
;;
|
||||
--clean)
|
||||
CLEAN=1
|
||||
shift
|
||||
;;
|
||||
-h)
|
||||
HELP=1
|
||||
shift
|
||||
;;
|
||||
--help)
|
||||
HELP=1
|
||||
shift
|
||||
;;
|
||||
-j)
|
||||
PARALLEL="$2"
|
||||
PARAMETER_FLAG=1
|
||||
shift 2
|
||||
;;
|
||||
--)
|
||||
shift
|
||||
break
|
||||
;;
|
||||
*)
|
||||
echo "Internal error"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
#only ./build.sh -j xx then build all
|
||||
if [[ ${PARAMETER_COUNT} -eq 3 ]] && [[ ${PARAMETER_FLAG} -eq 1 ]];then
|
||||
if [[ "${PARAMETER_COUNT}" -eq 3 ]] && [[ "${PARAMETER_FLAG}" -eq 1 ]]; then
|
||||
BUILD_FE=1
|
||||
BUILD_BE=1
|
||||
BUILD_BROKER=1
|
||||
BUILD_AUDIT=1
|
||||
BUILD_META_TOOL=ON
|
||||
BUILD_META_TOOL='ON'
|
||||
BUILD_SPARK_DPP=1
|
||||
BUILD_HIVE_UDF=1
|
||||
CLEAN=0
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ ${HELP} -eq 1 ]]; then
|
||||
if [[ "${HELP}" -eq 1 ]]; then
|
||||
usage
|
||||
exit
|
||||
fi
|
||||
# build thirdparty libraries if necessary
|
||||
if [[ ! -f ${DORIS_THIRDPARTY}/installed/lib/libbacktrace.a ]]; then
|
||||
if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/libbacktrace.a" ]]; then
|
||||
echo "Thirdparty libraries need to be build ..."
|
||||
# need remove all installed pkgs because some lib like lz4 will throw error if its lib alreay exists
|
||||
rm -rf ${DORIS_THIRDPARTY}/installed
|
||||
${DORIS_THIRDPARTY}/build-thirdparty.sh -j $PARALLEL
|
||||
rm -rf "${DORIS_THIRDPARTY}/installed"
|
||||
"${DORIS_THIRDPARTY}/build-thirdparty.sh" -j "${PARALLEL}"
|
||||
fi
|
||||
|
||||
if [ ${CLEAN} -eq 1 -a ${BUILD_BE} -eq 0 -a ${BUILD_FE} -eq 0 -a ${BUILD_SPARK_DPP} -eq 0 ]; then
|
||||
if [[ "${CLEAN}" -eq 1 && "${BUILD_BE}" -eq 0 && "${BUILD_FE}" -eq 0 && "${BUILD_SPARK_DPP}" -eq 0 ]]; then
|
||||
clean_gensrc
|
||||
clean_be
|
||||
clean_fe
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ -z ${WITH_MYSQL} ]]; then
|
||||
WITH_MYSQL=OFF
|
||||
if [[ -z "${WITH_MYSQL}" ]]; then
|
||||
WITH_MYSQL='OFF'
|
||||
fi
|
||||
if [[ -z ${GLIBC_COMPATIBILITY} ]]; then
|
||||
GLIBC_COMPATIBILITY=ON
|
||||
if [[ -z "${GLIBC_COMPATIBILITY}" ]]; then
|
||||
GLIBC_COMPATIBILITY='ON'
|
||||
fi
|
||||
if [[ -z ${USE_AVX2} ]]; then
|
||||
USE_AVX2=ON
|
||||
if [[ -z "${USE_AVX2}" ]]; then
|
||||
USE_AVX2='ON'
|
||||
fi
|
||||
if [[ -z ${WITH_LZO} ]]; then
|
||||
WITH_LZO=OFF
|
||||
if [[ -z "${WITH_LZO}" ]]; then
|
||||
WITH_LZO='OFF'
|
||||
fi
|
||||
if [[ -z ${USE_LIBCPP} ]]; then
|
||||
USE_LIBCPP=OFF
|
||||
if [[ -z "${USE_LIBCPP}" ]]; then
|
||||
USE_LIBCPP='OFF'
|
||||
fi
|
||||
if [[ -z ${STRIP_DEBUG_INFO} ]]; then
|
||||
STRIP_DEBUG_INFO=OFF
|
||||
if [[ -z "${STRIP_DEBUG_INFO}" ]]; then
|
||||
STRIP_DEBUG_INFO='OFF'
|
||||
fi
|
||||
if [[ -z ${USE_MEM_TRACKER} ]]; then
|
||||
USE_MEM_TRACKER=ON
|
||||
if [[ -z "${USE_MEM_TRACKER}" ]]; then
|
||||
USE_MEM_TRACKER='ON'
|
||||
fi
|
||||
if [[ -z ${USE_JEMALLOC} ]]; then
|
||||
USE_JEMALLOC=OFF
|
||||
if [[ -z "${USE_JEMALLOC}" ]]; then
|
||||
USE_JEMALLOC='OFF'
|
||||
fi
|
||||
if [[ -z ${STRICT_MEMORY_USE} ]]; then
|
||||
STRICT_MEMORY_USE=OFF
|
||||
if [[ -z "${STRICT_MEMORY_USE}" ]]; then
|
||||
STRICT_MEMORY_USE='OFF'
|
||||
fi
|
||||
|
||||
if [[ -z ${USE_DWARF} ]]; then
|
||||
USE_DWARF=OFF
|
||||
if [[ -z "${USE_DWARF}" ]]; then
|
||||
USE_DWARF='OFF'
|
||||
fi
|
||||
|
||||
echo "Get params:
|
||||
BUILD_FE -- $BUILD_FE
|
||||
BUILD_BE -- $BUILD_BE
|
||||
BUILD_BROKER -- $BUILD_BROKER
|
||||
BUILD_AUDIT -- $BUILD_AUDIT
|
||||
BUILD_META_TOOL -- $BUILD_META_TOOL
|
||||
BUILD_SPARK_DPP -- $BUILD_SPARK_DPP
|
||||
BUILD_JAVA_UDF -- $BUILD_JAVA_UDF
|
||||
BUILD_HIVE_UDF -- $BUILD_HIVE_UDF
|
||||
PARALLEL -- $PARALLEL
|
||||
CLEAN -- $CLEAN
|
||||
WITH_MYSQL -- $WITH_MYSQL
|
||||
WITH_LZO -- $WITH_LZO
|
||||
GLIBC_COMPATIBILITY -- $GLIBC_COMPATIBILITY
|
||||
USE_AVX2 -- $USE_AVX2
|
||||
USE_LIBCPP -- $USE_LIBCPP
|
||||
USE_DWARF -- $USE_DWARF
|
||||
STRIP_DEBUG_INFO -- $STRIP_DEBUG_INFO
|
||||
USE_MEM_TRACKER -- $USE_MEM_TRACKER
|
||||
USE_JEMALLOC -- $USE_JEMALLOC
|
||||
STRICT_MEMORY_USE -- $STRICT_MEMORY_USE
|
||||
BUILD_FE -- ${BUILD_FE}
|
||||
BUILD_BE -- ${BUILD_BE}
|
||||
BUILD_BROKER -- ${BUILD_BROKER}
|
||||
BUILD_AUDIT -- ${BUILD_AUDIT}
|
||||
BUILD_META_TOOL -- ${BUILD_META_TOOL}
|
||||
BUILD_SPARK_DPP -- ${BUILD_SPARK_DPP}
|
||||
BUILD_JAVA_UDF -- ${BUILD_JAVA_UDF}
|
||||
BUILD_HIVE_UDF -- ${BUILD_HIVE_UDF}
|
||||
PARALLEL -- ${PARALLEL}
|
||||
CLEAN -- ${CLEAN}
|
||||
WITH_MYSQL -- ${WITH_MYSQL}
|
||||
WITH_LZO -- ${WITH_LZO}
|
||||
GLIBC_COMPATIBILITY -- ${GLIBC_COMPATIBILITY}
|
||||
USE_AVX2 -- ${USE_AVX2}
|
||||
USE_LIBCPP -- ${USE_LIBCPP}
|
||||
USE_DWARF -- ${USE_DWARF}
|
||||
STRIP_DEBUG_INFO -- ${STRIP_DEBUG_INFO}
|
||||
USE_MEM_TRACKER -- ${USE_MEM_TRACKER}
|
||||
USE_JEMALLOC -- ${USE_JEMALLOC}
|
||||
STRICT_MEMORY_USE -- ${STRICT_MEMORY_USE}
|
||||
"
|
||||
|
||||
# Clean and build generated code
|
||||
if [ ${CLEAN} -eq 1 ]; then
|
||||
if [[ "${CLEAN}" -eq 1 ]]; then
|
||||
clean_gensrc
|
||||
fi
|
||||
echo "Build generated code"
|
||||
cd ${DORIS_HOME}/gensrc
|
||||
cd "${DORIS_HOME}/gensrc"
|
||||
# DO NOT using parallel make(-j) for gensrc
|
||||
make
|
||||
|
||||
# Assesmble FE modules
|
||||
FE_MODULES=
|
||||
BUILD_DOCS=OFF
|
||||
FE_MODULES=''
|
||||
BUILD_DOCS='OFF'
|
||||
modules=("")
|
||||
if [ ${BUILD_FE} -eq 1 ]; then
|
||||
if [[ "${BUILD_FE}" -eq 1 ]]; then
|
||||
modules+=("fe-common")
|
||||
modules+=("fe-core")
|
||||
BUILD_DOCS=ON
|
||||
BUILD_DOCS='ON'
|
||||
fi
|
||||
if [ ${BUILD_SPARK_DPP} -eq 1 ]; then
|
||||
if [[ "${BUILD_SPARK_DPP}" -eq 1 ]]; then
|
||||
modules+=("fe-common")
|
||||
modules+=("spark-dpp")
|
||||
fi
|
||||
if [ ${BUILD_JAVA_UDF} -eq 1 ]; then
|
||||
if [[ "${BUILD_JAVA_UDF}" -eq 1 ]]; then
|
||||
modules+=("java-udf")
|
||||
fi
|
||||
if [ ${BUILD_HIVE_UDF} -eq 1 ]; then
|
||||
if [[ "${BUILD_HIVE_UDF}" -eq 1 ]]; then
|
||||
modules+=("fe-common")
|
||||
modules+=("hive-udf")
|
||||
fi
|
||||
FE_MODULES=$(IFS=, ; echo "${modules[*]}")
|
||||
FE_MODULES="$(
|
||||
IFS=','
|
||||
echo "${modules[*]}"
|
||||
)"
|
||||
|
||||
# Clean and build Backend
|
||||
if [ ${BUILD_BE} -eq 1 ] ; then
|
||||
if [ -e ${DORIS_HOME}/gensrc/build/gen_cpp/version.h ]; then rm -f ${DORIS_HOME}/gensrc/build/gen_cpp/version.h ; fi
|
||||
CMAKE_BUILD_TYPE=${BUILD_TYPE:-Release}
|
||||
if [[ "${BUILD_BE}" -eq 1 ]]; then
|
||||
if [[ -e "${DORIS_HOME}/gensrc/build/gen_cpp/version.h" ]]; then
|
||||
rm -f "${DORIS_HOME}/gensrc/build/gen_cpp/version.h"
|
||||
fi
|
||||
CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}"
|
||||
echo "Build Backend: ${CMAKE_BUILD_TYPE}"
|
||||
CMAKE_BUILD_DIR=${DORIS_HOME}/be/build_${CMAKE_BUILD_TYPE}
|
||||
if [ ${CLEAN} -eq 1 ]; then
|
||||
CMAKE_BUILD_DIR="${DORIS_HOME}/be/build_${CMAKE_BUILD_TYPE}"
|
||||
if [[ "${CLEAN}" -eq 1 ]]; then
|
||||
clean_be
|
||||
fi
|
||||
MAKE_PROGRAM="$(which "${BUILD_SYSTEM}")"
|
||||
echo "-- Make program: ${MAKE_PROGRAM}"
|
||||
echo "-- Use ccache: ${CMAKE_USE_CCACHE}"
|
||||
echo "-- Extra cxx flags: ${EXTRA_CXX_FLAGS}"
|
||||
echo "-- Extra cxx flags: ${EXTRA_CXX_FLAGS:-}"
|
||||
|
||||
mkdir -p ${CMAKE_BUILD_DIR}
|
||||
cd ${CMAKE_BUILD_DIR}
|
||||
${CMAKE_CMD} -G "${GENERATOR}" \
|
||||
-DCMAKE_MAKE_PROGRAM="${MAKE_PROGRAM}" \
|
||||
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
|
||||
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
|
||||
-DMAKE_TEST=OFF \
|
||||
${CMAKE_USE_CCACHE} \
|
||||
-DWITH_MYSQL=${WITH_MYSQL} \
|
||||
-DWITH_LZO=${WITH_LZO} \
|
||||
-DUSE_LIBCPP=${USE_LIBCPP} \
|
||||
-DBUILD_META_TOOL=${BUILD_META_TOOL} \
|
||||
-DBUILD_JAVA_UDF=${BUILD_JAVA_UDF} \
|
||||
-DSTRIP_DEBUG_INFO=${STRIP_DEBUG_INFO} \
|
||||
-DUSE_DWARF=${USE_DWARF} \
|
||||
-DUSE_MEM_TRACKER=${USE_MEM_TRACKER} \
|
||||
-DUSE_JEMALLOC=${USE_JEMALLOC} \
|
||||
-DSTRICT_MEMORY_USE=${STRICT_MEMORY_USE} \
|
||||
-DUSE_AVX2=${USE_AVX2} \
|
||||
-DGLIBC_COMPATIBILITY=${GLIBC_COMPATIBILITY} \
|
||||
-DEXTRA_CXX_FLAGS="${EXTRA_CXX_FLAGS}" \
|
||||
${DORIS_HOME}/be/
|
||||
${BUILD_SYSTEM} -j ${PARALLEL}
|
||||
${BUILD_SYSTEM} install
|
||||
cd ${DORIS_HOME}
|
||||
mkdir -p "${CMAKE_BUILD_DIR}"
|
||||
cd "${CMAKE_BUILD_DIR}"
|
||||
"${CMAKE_CMD}" -G "${GENERATOR}" \
|
||||
-DCMAKE_MAKE_PROGRAM="${MAKE_PROGRAM}" \
|
||||
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
|
||||
-DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" \
|
||||
-DMAKE_TEST=OFF \
|
||||
${CMAKE_USE_CCACHE:+${CMAKE_USE_CCACHE}} \
|
||||
-DWITH_MYSQL="${WITH_MYSQL}" \
|
||||
-DWITH_LZO="${WITH_LZO}" \
|
||||
-DUSE_LIBCPP="${USE_LIBCPP}" \
|
||||
-DBUILD_META_TOOL="${BUILD_META_TOOL}" \
|
||||
-DBUILD_JAVA_UDF="${BUILD_JAVA_UDF}" \
|
||||
-DSTRIP_DEBUG_INFO="${STRIP_DEBUG_INFO}" \
|
||||
-DUSE_DWARF="${USE_DWARF}" \
|
||||
-DUSE_MEM_TRACKER="${USE_MEM_TRACKER}" \
|
||||
-DUSE_JEMALLOC="${USE_JEMALLOC}" \
|
||||
-DSTRICT_MEMORY_USE="${STRICT_MEMORY_USE}" \
|
||||
-DUSE_AVX2="${USE_AVX2}" \
|
||||
-DGLIBC_COMPATIBILITY="${GLIBC_COMPATIBILITY}" \
|
||||
-DEXTRA_CXX_FLAGS="${EXTRA_CXX_FLAGS}" \
|
||||
"${DORIS_HOME}/be"
|
||||
"${BUILD_SYSTEM}" -j "${PARALLEL}"
|
||||
"${BUILD_SYSTEM}" install
|
||||
cd "${DORIS_HOME}"
|
||||
fi
|
||||
|
||||
if [ "${BUILD_DOCS}" = "ON" ] ; then
|
||||
if [[ "${BUILD_DOCS}" = "ON" ]]; then
|
||||
# Build docs, should be built before Frontend
|
||||
echo "Build docs"
|
||||
cd ${DORIS_HOME}/docs
|
||||
cd "${DORIS_HOME}/docs"
|
||||
./build_help_zip.sh
|
||||
cd ${DORIS_HOME}
|
||||
cd "${DORIS_HOME}"
|
||||
fi
|
||||
|
||||
function build_ui() {
|
||||
NPM=npm
|
||||
NPM='npm'
|
||||
if ! ${NPM} --version; then
|
||||
echo "Error: npm is not found"
|
||||
exit 1
|
||||
fi
|
||||
if [[ ! -z ${CUSTOM_NPM_REGISTRY} ]]; then
|
||||
${NPM} config set registry ${CUSTOM_NPM_REGISTRY}
|
||||
npm_reg=`${NPM} get registry`
|
||||
echo "NPM registry: $npm_reg"
|
||||
if [[ -n "${CUSTOM_NPM_REGISTRY}" ]]; then
|
||||
"${NPM}" config set registry "${CUSTOM_NPM_REGISTRY}"
|
||||
npm_reg="$("${NPM}" get registry)"
|
||||
echo "NPM registry: ${npm_reg}"
|
||||
fi
|
||||
|
||||
echo "Build Frontend UI"
|
||||
ui_dist=${DORIS_HOME}/ui/dist/
|
||||
if [[ ! -z ${CUSTOM_UI_DIST} ]]; then
|
||||
ui_dist=${CUSTOM_UI_DIST}
|
||||
ui_dist="${DORIS_HOME}/ui/dist"
|
||||
if [[ -n "${CUSTOM_UI_DIST}" ]]; then
|
||||
ui_dist="${CUSTOM_UI_DIST}"
|
||||
else
|
||||
cd ${DORIS_HOME}/ui
|
||||
${NPM} install
|
||||
${NPM} run build
|
||||
cd "${DORIS_HOME}/ui"
|
||||
"${NPM}" install
|
||||
"${NPM}" run build
|
||||
fi
|
||||
echo "ui dist: ${ui_dist}"
|
||||
rm -rf ${DORIS_HOME}/fe/fe-core/src/main/resources/static/
|
||||
mkdir -p ${DORIS_HOME}/fe/fe-core/src/main/resources/static
|
||||
cp -r ${ui_dist}/* ${DORIS_HOME}/fe/fe-core/src/main/resources/static
|
||||
rm -rf "${DORIS_HOME}/fe/fe-core/src/main/resources/static"
|
||||
mkdir -p "${DORIS_HOME}/fe/fe-core/src/main/resources/static"
|
||||
cp -r "${ui_dist}"/* "${DORIS_HOME}/fe/fe-core/src/main/resources/static"/
|
||||
}
|
||||
|
||||
# FE UI must be built before building FE
|
||||
if [ ${BUILD_FE} -eq 1 ] ; then
|
||||
if [[ "${BUILD_FE}" -eq 1 ]]; then
|
||||
build_ui
|
||||
fi
|
||||
|
||||
# Clean and build Frontend
|
||||
if [ ${FE_MODULES}x != ""x ]; then
|
||||
echo "Build Frontend Modules: $FE_MODULES"
|
||||
cd ${DORIS_HOME}/fe
|
||||
if [ ${CLEAN} -eq 1 ]; then
|
||||
if [[ "${FE_MODULES}" != '' ]]; then
|
||||
echo "Build Frontend Modules: ${FE_MODULES}"
|
||||
cd "${DORIS_HOME}/fe"
|
||||
if [[ "${CLEAN}" -eq 1 ]]; then
|
||||
clean_fe
|
||||
fi
|
||||
${MVN_CMD} package -pl ${FE_MODULES} -DskipTests
|
||||
cd ${DORIS_HOME}
|
||||
"${MVN_CMD}" package -pl ${FE_MODULES:+${FE_MODULES}} -DskipTests
|
||||
cd "${DORIS_HOME}"
|
||||
fi
|
||||
|
||||
# Clean and prepare output dir
|
||||
DORIS_OUTPUT=${DORIS_HOME}/output/
|
||||
mkdir -p ${DORIS_OUTPUT}
|
||||
mkdir -p "${DORIS_OUTPUT}"
|
||||
|
||||
# Copy Frontend and Backend
|
||||
if [ ${BUILD_FE} -eq 1 ]; then
|
||||
install -d ${DORIS_OUTPUT}/fe/bin ${DORIS_OUTPUT}/fe/conf \
|
||||
${DORIS_OUTPUT}/fe/webroot/ ${DORIS_OUTPUT}/fe/lib/
|
||||
if [[ "${BUILD_FE}" -eq 1 ]]; then
|
||||
install -d "${DORIS_OUTPUT}/fe/bin" "${DORIS_OUTPUT}/fe/conf" \
|
||||
"${DORIS_OUTPUT}/fe/webroot" "${DORIS_OUTPUT}/fe/lib"
|
||||
|
||||
cp -r -p ${DORIS_HOME}/bin/*_fe.sh ${DORIS_OUTPUT}/fe/bin/
|
||||
cp -r -p ${DORIS_HOME}/conf/fe.conf ${DORIS_OUTPUT}/fe/conf/
|
||||
cp -r -p ${DORIS_HOME}/conf/*.xml ${DORIS_OUTPUT}/fe/conf/
|
||||
rm -rf ${DORIS_OUTPUT}/fe/lib/*
|
||||
cp -r -p ${DORIS_HOME}/fe/fe-core/target/lib/* ${DORIS_OUTPUT}/fe/lib/
|
||||
rm -f ${DORIS_OUTPUT}/fe/lib/palo-fe.jar
|
||||
cp -r -p ${DORIS_HOME}/fe/fe-core/target/doris-fe.jar ${DORIS_OUTPUT}/fe/lib/
|
||||
cp -r -p ${DORIS_HOME}/docs/build/help-resource.zip ${DORIS_OUTPUT}/fe/lib/
|
||||
cp -r -p ${DORIS_HOME}/webroot/static ${DORIS_OUTPUT}/fe/webroot/
|
||||
cp -r -p "${DORIS_HOME}/bin"/*_fe.sh "${DORIS_OUTPUT}/fe/bin"/
|
||||
cp -r -p "${DORIS_HOME}/conf/fe.conf" "${DORIS_OUTPUT}/fe/conf"/
|
||||
cp -r -p "${DORIS_HOME}/conf"/*.xml "${DORIS_OUTPUT}/fe/conf"/
|
||||
rm -rf "${DORIS_OUTPUT}/fe/lib"/*
|
||||
cp -r -p "${DORIS_HOME}/fe/fe-core/target/lib"/* "${DORIS_OUTPUT}/fe/lib"/
|
||||
rm -f "${DORIS_OUTPUT}/fe/lib/palo-fe.jar"
|
||||
cp -r -p "${DORIS_HOME}/fe/fe-core/target/doris-fe.jar" "${DORIS_OUTPUT}/fe/lib"/
|
||||
cp -r -p "${DORIS_HOME}/docs/build/help-resource.zip" "${DORIS_OUTPUT}/fe/lib"/
|
||||
cp -r -p "${DORIS_HOME}/webroot/static" "${DORIS_OUTPUT}/fe/webroot"/
|
||||
|
||||
cp -r -p ${DORIS_THIRDPARTY}/installed/webroot/* ${DORIS_OUTPUT}/fe/webroot/static/
|
||||
mkdir -p ${DORIS_OUTPUT}/fe/log
|
||||
mkdir -p ${DORIS_OUTPUT}/fe/doris-meta
|
||||
cp -r -p "${DORIS_THIRDPARTY}/installed/webroot"/* "${DORIS_OUTPUT}/fe/webroot/static"/
|
||||
mkdir -p "${DORIS_OUTPUT}/fe/log"
|
||||
mkdir -p "${DORIS_OUTPUT}/fe/doris-meta"
|
||||
fi
|
||||
|
||||
if [ ${BUILD_SPARK_DPP} -eq 1 ]; then
|
||||
install -d ${DORIS_OUTPUT}/fe/spark-dpp/
|
||||
rm -rf ${DORIS_OUTPUT}/fe/spark-dpp/*
|
||||
cp -r -p ${DORIS_HOME}/fe/spark-dpp/target/spark-dpp-*-jar-with-dependencies.jar ${DORIS_OUTPUT}/fe/spark-dpp/
|
||||
if [[ "${BUILD_SPARK_DPP}" -eq 1 ]]; then
|
||||
install -d "${DORIS_OUTPUT}/fe/spark-dpp"
|
||||
rm -rf "${DORIS_OUTPUT}/fe/spark-dpp"/*
|
||||
cp -r -p "${DORIS_HOME}/fe/spark-dpp/target"/spark-dpp-*-jar-with-dependencies.jar "${DORIS_OUTPUT}/fe/spark-dpp"/
|
||||
fi
|
||||
|
||||
if [ ${BUILD_BE} -eq 1 ]; then
|
||||
install -d ${DORIS_OUTPUT}/be/bin \
|
||||
${DORIS_OUTPUT}/be/conf \
|
||||
${DORIS_OUTPUT}/be/lib/ \
|
||||
${DORIS_OUTPUT}/be/www \
|
||||
${DORIS_OUTPUT}/udf/lib \
|
||||
${DORIS_OUTPUT}/udf/include
|
||||
if [[ "${BUILD_BE}" -eq 1 ]]; then
|
||||
install -d "${DORIS_OUTPUT}/be/bin" \
|
||||
"${DORIS_OUTPUT}/be/conf" \
|
||||
"${DORIS_OUTPUT}/be/lib" \
|
||||
"${DORIS_OUTPUT}/be/www" \
|
||||
"${DORIS_OUTPUT}/udf/lib" \
|
||||
"${DORIS_OUTPUT}/udf/include"
|
||||
|
||||
cp -r -p "${DORIS_HOME}/be/output/bin"/* "${DORIS_OUTPUT}/be/bin"/
|
||||
cp -r -p "${DORIS_HOME}/be/output/conf"/* "${DORIS_OUTPUT}/be/conf"/
|
||||
cp -r -p "${DORIS_HOME}/be/output/lib/doris_be" "${DORIS_OUTPUT}/be/lib"/
|
||||
|
||||
cp -r -p ${DORIS_HOME}/be/output/bin/* ${DORIS_OUTPUT}/be/bin/
|
||||
cp -r -p ${DORIS_HOME}/be/output/conf/* ${DORIS_OUTPUT}/be/conf/
|
||||
cp -r -p ${DORIS_HOME}/be/output/lib/doris_be ${DORIS_OUTPUT}/be/lib/
|
||||
# make a soft link palo_be point to doris_be, for forward compatibility
|
||||
cd ${DORIS_OUTPUT}/be/lib && rm -f palo_be && ln -s doris_be palo_be && cd -
|
||||
cd "${DORIS_OUTPUT}/be/lib"
|
||||
rm -f palo_be
|
||||
ln -s doris_be palo_be
|
||||
cd -
|
||||
|
||||
if [ "${BUILD_META_TOOL}" = "ON" ]; then
|
||||
cp -r -p ${DORIS_HOME}/be/output/lib/meta_tool ${DORIS_OUTPUT}/be/lib/
|
||||
if [[ "${BUILD_META_TOOL}" = "ON" ]]; then
|
||||
cp -r -p "${DORIS_HOME}/be/output/lib/meta_tool" "${DORIS_OUTPUT}/be/lib"/
|
||||
fi
|
||||
|
||||
cp -r -p ${DORIS_HOME}/be/output/udf/*.a ${DORIS_OUTPUT}/udf/lib/
|
||||
cp -r -p ${DORIS_HOME}/be/output/udf/include/* ${DORIS_OUTPUT}/udf/include/
|
||||
cp -r -p ${DORIS_HOME}/webroot/be/* ${DORIS_OUTPUT}/be/www/
|
||||
if [ "${STRIP_DEBUG_INFO}" = "ON" ]; then
|
||||
cp -r -p ${DORIS_HOME}/be/output/lib/debug_info ${DORIS_OUTPUT}/be/lib/
|
||||
cp -r -p "${DORIS_HOME}/be/output/udf"/*.a "${DORIS_OUTPUT}/udf/lib"/
|
||||
cp -r -p "${DORIS_HOME}/be/output/udf/include"/* "${DORIS_OUTPUT}/udf/include"/
|
||||
cp -r -p "${DORIS_HOME}/webroot/be"/* "${DORIS_OUTPUT}/be/www"/
|
||||
if [[ "${STRIP_DEBUG_INFO}" = "ON" ]]; then
|
||||
cp -r -p "${DORIS_HOME}/be/output/lib/debug_info" "${DORIS_OUTPUT}/be/lib"/
|
||||
fi
|
||||
|
||||
java_udf_path=${DORIS_HOME}/fe/java-udf/target/java-udf-jar-with-dependencies.jar
|
||||
if [ -f ${java_udf_path} ];then
|
||||
cp ${java_udf_path} ${DORIS_OUTPUT}/be/lib/
|
||||
java_udf_path="${DORIS_HOME}/fe/java-udf/target/java-udf-jar-with-dependencies.jar"
|
||||
if [[ -f "${java_udf_path}" ]]; then
|
||||
cp "${java_udf_path}" "${DORIS_OUTPUT}/be/lib"/
|
||||
fi
|
||||
|
||||
cp -r -p ${DORIS_THIRDPARTY}/installed/webroot/* ${DORIS_OUTPUT}/be/www/
|
||||
mkdir -p ${DORIS_OUTPUT}/be/log
|
||||
mkdir -p ${DORIS_OUTPUT}/be/storage
|
||||
cp -r -p "${DORIS_THIRDPARTY}/installed/webroot"/* "${DORIS_OUTPUT}/be/www"/
|
||||
mkdir -p "${DORIS_OUTPUT}/be/log"
|
||||
mkdir -p "${DORIS_OUTPUT}/be/storage"
|
||||
fi
|
||||
|
||||
if [ ${BUILD_BROKER} -eq 1 ]; then
|
||||
install -d ${DORIS_OUTPUT}/apache_hdfs_broker
|
||||
if [[ "${BUILD_BROKER}" -eq 1 ]]; then
|
||||
install -d "${DORIS_OUTPUT}/apache_hdfs_broker"
|
||||
|
||||
cd ${DORIS_HOME}/fs_brokers/apache_hdfs_broker/
|
||||
cd "${DORIS_HOME}/fs_brokers/apache_hdfs_broker"
|
||||
./build.sh
|
||||
rm -rf ${DORIS_OUTPUT}/apache_hdfs_broker/*
|
||||
cp -r -p ${DORIS_HOME}/fs_brokers/apache_hdfs_broker/output/apache_hdfs_broker/* ${DORIS_OUTPUT}/apache_hdfs_broker/
|
||||
cd ${DORIS_HOME}
|
||||
rm -rf "${DORIS_OUTPUT}/apache_hdfs_broker"/*
|
||||
cp -r -p "${DORIS_HOME}/fs_brokers/apache_hdfs_broker/output/apache_hdfs_broker"/* "${DORIS_OUTPUT}/apache_hdfs_broker"/
|
||||
cd "${DORIS_HOME}"
|
||||
fi
|
||||
|
||||
if [ ${BUILD_AUDIT} -eq 1 ]; then
|
||||
install -d ${DORIS_OUTPUT}/audit_loader
|
||||
if [[ "${BUILD_AUDIT}" -eq 1 ]]; then
|
||||
install -d "${DORIS_OUTPUT}/audit_loader"
|
||||
|
||||
cd ${DORIS_HOME}/fe_plugins/auditloader/
|
||||
cd "${DORIS_HOME}/fe_plugins/auditloader"
|
||||
./build.sh
|
||||
rm -rf ${DORIS_OUTPUT}/audit_loader/*
|
||||
cp -r -p ${DORIS_HOME}/fe_plugins/auditloader/output/* ${DORIS_OUTPUT}/audit_loader/
|
||||
cd ${DORIS_HOME}
|
||||
rm -rf "${DORIS_OUTPUT}/audit_loader"/*
|
||||
cp -r -p "${DORIS_HOME}/fe_plugins/auditloader/output"/* "${DORIS_OUTPUT}/audit_loader"/
|
||||
cd "${DORIS_HOME}"
|
||||
fi
|
||||
|
||||
echo "***************************************"
|
||||
echo "Successfully build Doris"
|
||||
echo "***************************************"
|
||||
|
||||
if [[ ! -z ${DORIS_POST_BUILD_HOOK} ]]; then
|
||||
eval ${DORIS_POST_BUILD_HOOK}
|
||||
if [[ -n "${DORIS_POST_BUILD_HOOK}" ]]; then
|
||||
eval "${DORIS_POST_BUILD_HOOK}"
|
||||
fi
|
||||
|
||||
exit 0
|
||||
|
||||
108
build_plugin.sh
108
build_plugin.sh
@ -18,16 +18,15 @@
|
||||
|
||||
set -eo pipefail
|
||||
|
||||
ROOT=`dirname "$0"`
|
||||
ROOT=`cd "$ROOT"; pwd`
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
|
||||
|
||||
export DORIS_HOME=${ROOT}
|
||||
export DORIS_HOME="${ROOT}"
|
||||
|
||||
. ${DORIS_HOME}/env.sh
|
||||
. "${DORIS_HOME}/env.sh"
|
||||
|
||||
# Check args
|
||||
usage() {
|
||||
echo "
|
||||
echo "
|
||||
Usage: $0 <options>
|
||||
Optional options:
|
||||
--plugin build special plugin
|
||||
@ -35,81 +34,97 @@ Usage: $0 <options>
|
||||
$0 --plugin xxx build xxx plugin
|
||||
$0 build all plugins
|
||||
"
|
||||
exit 1
|
||||
exit 1
|
||||
}
|
||||
|
||||
OPTS=$(getopt \
|
||||
-n $0 \
|
||||
-o '' \
|
||||
-o 'h' \
|
||||
-l 'plugin' \
|
||||
-l 'clean' \
|
||||
-l 'help' \
|
||||
-- "$@")
|
||||
|
||||
if [ $? != 0 ] ; then
|
||||
if ! OPTS="$(getopt \
|
||||
-n "$0" \
|
||||
-o '' \
|
||||
-o 'h' \
|
||||
-l 'plugin' \
|
||||
-l 'clean' \
|
||||
-l 'help' \
|
||||
-- "$@")"; then
|
||||
usage
|
||||
fi
|
||||
|
||||
eval set -- "$OPTS"
|
||||
eval set -- "${OPTS}"
|
||||
|
||||
ALL_PLUGIN=1
|
||||
CLEAN=0
|
||||
if [ $# == 1 ] ; then
|
||||
if [[ "$#" == 1 ]]; then
|
||||
# defuat
|
||||
ALL_PLUGIN=1
|
||||
CLEAN=0
|
||||
else
|
||||
while true; do
|
||||
case "$1" in
|
||||
--plugin) ALL_PLUGIN=0 ; shift ;;
|
||||
--clean) CLEAN=1 ; shift ;;
|
||||
-h) HELP=1; shift ;;
|
||||
--help) HELP=1; shift ;;
|
||||
--) shift ; break ;;
|
||||
*) echo "Internal error" ; exit 1 ;;
|
||||
--plugin)
|
||||
ALL_PLUGIN=0
|
||||
shift
|
||||
;;
|
||||
--clean)
|
||||
CLEAN=1
|
||||
shift
|
||||
;;
|
||||
-h)
|
||||
HELP=1
|
||||
shift
|
||||
;;
|
||||
--help)
|
||||
HELP=1
|
||||
shift
|
||||
;;
|
||||
--)
|
||||
shift
|
||||
break
|
||||
;;
|
||||
*)
|
||||
echo "Internal error"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
fi
|
||||
|
||||
if [[ ${HELP} -eq 1 ]]; then
|
||||
if [[ "${HELP}" -eq 1 ]]; then
|
||||
usage
|
||||
exit
|
||||
fi
|
||||
|
||||
echo "Get params:
|
||||
BUILD_ALL_PLUGIN -- $ALL_PLUGIN
|
||||
CLEAN -- $CLEAN
|
||||
BUILD_ALL_PLUGIN -- ${ALL_PLUGIN}
|
||||
CLEAN -- ${CLEAN}
|
||||
"
|
||||
|
||||
cd ${DORIS_HOME}
|
||||
PLUGIN_MODULE=
|
||||
if [ ${ALL_PLUGIN} -eq 1 ] ; then
|
||||
cd ${DORIS_HOME}/fe_plugins
|
||||
if [ ${CLEAN} -eq 1 ]; then
|
||||
${MVN_CMD} clean
|
||||
cd "${DORIS_HOME}"
|
||||
PLUGIN_MODULE=''
|
||||
if [[ "${ALL_PLUGIN}" -eq 1 ]]; then
|
||||
cd "${DORIS_HOME}/fe_plugins"
|
||||
if [[ "${CLEAN}" -eq 1 ]]; then
|
||||
"${MVN_CMD}" clean
|
||||
fi
|
||||
echo "build all plugins"
|
||||
${MVN_CMD} package -DskipTests
|
||||
"${MVN_CMD}" package -DskipTests
|
||||
else
|
||||
PLUGIN_MODULE=$1
|
||||
cd ${DORIS_HOME}/fe_plugins/$PLUGIN_MODULE
|
||||
if [ ${CLEAN} -eq 1 ]; then
|
||||
${MVN_CMD} clean
|
||||
PLUGIN_MODULE="$1"
|
||||
cd "${DORIS_HOME}/fe_plugins/${PLUGIN_MODULE}"
|
||||
if [[ "${CLEAN}" -eq 1 ]]; then
|
||||
"${MVN_CMD}" clean
|
||||
fi
|
||||
echo "build plugin $PLUGIN_MODULE"
|
||||
${MVN_CMD} package -DskipTests
|
||||
echo "build plugin ${PLUGIN_MODULE}"
|
||||
"${MVN_CMD}" package -DskipTests
|
||||
fi
|
||||
|
||||
cd ${DORIS_HOME}
|
||||
cd "${DORIS_HOME}"
|
||||
# Clean and prepare output dir
|
||||
DORIS_OUTPUT=${DORIS_HOME}/fe_plugins/output/
|
||||
mkdir -p ${DORIS_OUTPUT}
|
||||
DORIS_OUTPUT="${DORIS_HOME}/fe_plugins/output"
|
||||
mkdir -p "${DORIS_OUTPUT}"
|
||||
|
||||
if [ ${ALL_PLUGIN} -eq 1 ] ; then
|
||||
cp -p ${DORIS_HOME}/fe_plugins/*/target/*.zip ${DORIS_OUTPUT}/
|
||||
if [[ "${ALL_PLUGIN}" -eq 1 ]]; then
|
||||
cp -p "${DORIS_HOME}/fe_plugins"/*/target/*.zip "${DORIS_OUTPUT}"/
|
||||
else
|
||||
cp -p ${DORIS_HOME}/fe_plugins/$PLUGIN_MODULE/target/*.zip ${DORIS_OUTPUT}/
|
||||
cp -p "${DORIS_HOME}/fe_plugins/${PLUGIN_MODULE}/target"/*.zip "${DORIS_OUTPUT}"/
|
||||
fi
|
||||
|
||||
echo "***************************************"
|
||||
@ -117,4 +132,3 @@ echo "Successfully build Doris FE Plugin"
|
||||
echo "***************************************"
|
||||
|
||||
exit 0
|
||||
|
||||
|
||||
@ -17,7 +17,7 @@
|
||||
# under the License.
|
||||
|
||||
##############################################################
|
||||
# This script is used to compile UDF
|
||||
# This script is used to compile UDF
|
||||
# Usage:
|
||||
# sh build-udf.sh build udf without clean.
|
||||
# sh build-udf.sh --clean clean previous output and build.
|
||||
@ -26,20 +26,23 @@
|
||||
|
||||
set -eo pipefail
|
||||
|
||||
ROOT=`dirname "$0"`
|
||||
ROOT=`cd "$ROOT"; pwd`
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
|
||||
|
||||
export UDF_HOME=${ROOT}
|
||||
export DORIS_HOME=$(cd ../..; printf %s "$PWD")
|
||||
echo ${DORIS_HOME}
|
||||
export UDF_HOME="${ROOT}"
|
||||
DORIS_HOME="$(
|
||||
cd ../..
|
||||
printf %s "${PWD}"
|
||||
)"
|
||||
export DORIS_HOME
|
||||
echo "${DORIS_HOME}"
|
||||
|
||||
. ${DORIS_HOME}/env.sh
|
||||
. "${DORIS_HOME}/env.sh"
|
||||
|
||||
PARALLEL=$[$(nproc)/4+1]
|
||||
PARALLEL="$(($(nproc) / 4 + 1))"
|
||||
|
||||
# Check args
|
||||
usage() {
|
||||
echo "
|
||||
echo "
|
||||
Usage: $0 <options>
|
||||
Optional options:
|
||||
--clean clean and build target
|
||||
@ -48,80 +51,93 @@ Usage: $0 <options>
|
||||
$0 build UDF without clean
|
||||
$0 --clean clean and build UDF
|
||||
"
|
||||
exit 1
|
||||
exit 1
|
||||
}
|
||||
|
||||
OPTS=$(getopt \
|
||||
-n $0 \
|
||||
-o '' \
|
||||
-o 'h' \
|
||||
-l 'clean' \
|
||||
-l 'help' \
|
||||
-- "$@")
|
||||
|
||||
if [ $? != 0 ] ; then
|
||||
if ! OPTS="$(getopt \
|
||||
-n "$0" \
|
||||
-o '' \
|
||||
-o 'h' \
|
||||
-l 'clean' \
|
||||
-l 'help' \
|
||||
-- "$@")"; then
|
||||
usage
|
||||
fi
|
||||
|
||||
eval set -- "$OPTS"
|
||||
eval set -- "${OPTS}"
|
||||
|
||||
BUILD_UDF=1
|
||||
CLEAN=0
|
||||
HELP=0
|
||||
if [ $# == 1 ] ; then
|
||||
if [[ "$#" == 1 ]]; then
|
||||
# default
|
||||
CLEAN=0
|
||||
else
|
||||
CLEAN=0
|
||||
while true; do
|
||||
case "$1" in
|
||||
--clean) CLEAN=1 ; shift ;;
|
||||
-h) HELP=1; shift ;;
|
||||
--help) HELP=1; shift ;;
|
||||
--) shift ; break ;;
|
||||
*) echo "Internal error" ; exit 1 ;;
|
||||
--clean)
|
||||
CLEAN=1
|
||||
shift
|
||||
;;
|
||||
-h)
|
||||
HELP=1
|
||||
shift
|
||||
;;
|
||||
--help)
|
||||
HELP=1
|
||||
shift
|
||||
;;
|
||||
--)
|
||||
shift
|
||||
break
|
||||
;;
|
||||
*)
|
||||
echo "Internal error"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
fi
|
||||
|
||||
if [[ ${HELP} -eq 1 ]]; then
|
||||
if [[ "${HELP}" -eq 1 ]]; then
|
||||
usage
|
||||
exit
|
||||
fi
|
||||
|
||||
echo "Get params:
|
||||
CLEAN -- $CLEAN
|
||||
CLEAN -- ${CLEAN}
|
||||
"
|
||||
|
||||
cd ${UDF_HOME}
|
||||
cd "${UDF_HOME}"
|
||||
# Clean and build UDF
|
||||
if [ ${BUILD_UDF} -eq 1 ] ; then
|
||||
CMAKE_BUILD_TYPE=${BUILD_TYPE:-Release}
|
||||
if [[ "${BUILD_UDF}" -eq 1 ]]; then
|
||||
CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}"
|
||||
echo "Build UDF: ${CMAKE_BUILD_TYPE}"
|
||||
CMAKE_BUILD_DIR=${UDF_HOME}/build_${CMAKE_BUILD_TYPE}
|
||||
if [ ${CLEAN} -eq 1 ]; then
|
||||
rm -rf $CMAKE_BUILD_DIR
|
||||
rm -rf ${UDF_HOME}/output/
|
||||
CMAKE_BUILD_DIR="${UDF_HOME}/build_${CMAKE_BUILD_TYPE}"
|
||||
if [[ "${CLEAN}" -eq 1 ]]; then
|
||||
rm -rf "${CMAKE_BUILD_DIR}"
|
||||
rm -rf "${UDF_HOME}/output"
|
||||
fi
|
||||
mkdir -p ${CMAKE_BUILD_DIR}
|
||||
cd ${CMAKE_BUILD_DIR}
|
||||
${CMAKE_CMD} -G "${GENERATOR}" -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} ../
|
||||
${BUILD_SYSTEM} -j${PARALLEL}
|
||||
${BUILD_SYSTEM} install
|
||||
cd ${UDF_HOME}
|
||||
mkdir -p "${CMAKE_BUILD_DIR}"
|
||||
cd "${CMAKE_BUILD_DIR}"
|
||||
"${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" ../
|
||||
"${BUILD_SYSTEM}" -j "${PARALLEL}"
|
||||
"${BUILD_SYSTEM}" install
|
||||
cd "${UDF_HOME}"
|
||||
fi
|
||||
|
||||
# Clean and prepare output dir
|
||||
DORIS_OUTPUT=${DORIS_HOME}/output/
|
||||
mkdir -p ${DORIS_OUTPUT}
|
||||
DORIS_OUTPUT="${DORIS_HOME}/output"
|
||||
mkdir -p "${DORIS_OUTPUT}"
|
||||
|
||||
#Copy UDF
|
||||
if [ ${BUILD_UDF} -eq 1 ]; then
|
||||
install -d ${DORIS_OUTPUT}/contrib/udf/lib
|
||||
for dir in $(ls ${CMAKE_BUILD_DIR}/src)
|
||||
do
|
||||
mkdir -p ${DORIS_OUTPUT}/contrib/udf/lib/$dir
|
||||
cp -r -p ${CMAKE_BUILD_DIR}/src/$dir/*.so ${DORIS_OUTPUT}/contrib/udf/lib/$dir/
|
||||
if [[ "${BUILD_UDF}" -eq 1 ]]; then
|
||||
install -d "${DORIS_OUTPUT}/contrib/udf/lib"
|
||||
for dir in "${CMAKE_BUILD_DIR}/src"/*; do
|
||||
dir="$(basename "${dir}")"
|
||||
mkdir -p "${DORIS_OUTPUT}/contrib/udf/lib/${dir}"
|
||||
cp -r -p "${CMAKE_BUILD_DIR}/src/${dir}"/*.so "${DORIS_OUTPUT}/contrib/udf/lib/${dir}"/
|
||||
done
|
||||
fi
|
||||
|
||||
@ -129,8 +145,8 @@ echo "***************************************"
|
||||
echo "Successfully build Doris UDF"
|
||||
echo "***************************************"
|
||||
|
||||
if [[ ! -z ${DORIS_POST_BUILD_HOOK} ]]; then
|
||||
eval ${DORIS_POST_BUILD_HOOK}
|
||||
if [[ -n "${DORIS_POST_BUILD_HOOK}" ]]; then
|
||||
eval "${DORIS_POST_BUILD_HOOK}"
|
||||
fi
|
||||
|
||||
exit 0
|
||||
|
||||
@ -24,21 +24,18 @@
|
||||
|
||||
set -eo pipefail
|
||||
|
||||
ROOT=`dirname "$0"`
|
||||
ROOT=`cd "$ROOT"; pwd`
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
|
||||
|
||||
BUILD_DIR=build
|
||||
HELP_DIR=contents
|
||||
HELP_ZIP_FILE=help-resource.zip
|
||||
SQL_REF_DOC_DIR=zh-CN/docs/sql-manual/
|
||||
BUILD_DIR='build'
|
||||
HELP_DIR='contents'
|
||||
HELP_ZIP_FILE='help-resource.zip'
|
||||
SQL_REF_DOC_DIR='zh-CN/docs/sql-manual'
|
||||
|
||||
cd $ROOT
|
||||
rm -rf $BUILD_DIR $HELP_DIR $HELP_ZIP_FILE
|
||||
mkdir -p $BUILD_DIR $HELP_DIR
|
||||
|
||||
cp -r $SQL_REF_DOC_DIR/* $HELP_DIR/
|
||||
|
||||
zip -r $HELP_ZIP_FILE $HELP_DIR
|
||||
mv $HELP_ZIP_FILE $BUILD_DIR/
|
||||
cd "${ROOT}"
|
||||
rm -rf "${BUILD_DIR}" "${HELP_DIR}" "${HELP_ZIP_FILE}"
|
||||
mkdir -p "${BUILD_DIR}" "${HELP_DIR}"
|
||||
|
||||
cp -r "${SQL_REF_DOC_DIR}"/* "${HELP_DIR}"/
|
||||
|
||||
zip -r "${HELP_ZIP_FILE}" "${HELP_DIR}"
|
||||
mv "${HELP_ZIP_FILE}" "${BUILD_DIR}"/
|
||||
|
||||
113
env.sh
113
env.sh
@ -1,3 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
@ -18,31 +20,32 @@
|
||||
# check DORIS_HOME
|
||||
export LC_ALL=C
|
||||
|
||||
if [[ -z ${DORIS_HOME} ]]; then
|
||||
if [[ -z "${DORIS_HOME}" ]]; then
|
||||
echo "Error: DORIS_HOME is not set"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# check OS type
|
||||
if [[ ! -z "$OSTYPE" ]]; then
|
||||
if [[ ${OSTYPE} != "linux-gnu" ]] && [[ ${OSTYPE:0:6} != "darwin" ]]; then
|
||||
echo "Error: Unsupported OS type: $OSTYPE"
|
||||
if [[ -n "${OSTYPE}" ]]; then
|
||||
if [[ "${OSTYPE}" != "linux-gnu" ]] && [[ "${OSTYPE:0:6}" != "darwin" ]]; then
|
||||
echo "Error: Unsupported OS type: ${OSTYPE}"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# include custom environment variables
|
||||
if [[ -f ${DORIS_HOME}/custom_env.sh ]]; then
|
||||
. ${DORIS_HOME}/custom_env.sh
|
||||
if [[ -f "${DORIS_HOME}/custom_env.sh" ]]; then
|
||||
# shellcheck disable=1091
|
||||
. "${DORIS_HOME}/custom_env.sh"
|
||||
fi
|
||||
|
||||
# set DORIS_THIRDPARTY
|
||||
if [[ -z ${DORIS_THIRDPARTY} ]]; then
|
||||
export DORIS_THIRDPARTY=${DORIS_HOME}/thirdparty
|
||||
if [[ -z "${DORIS_THIRDPARTY}" ]]; then
|
||||
export DORIS_THIRDPARTY="${DORIS_HOME}/thirdparty"
|
||||
fi
|
||||
|
||||
# check python
|
||||
export PYTHON=python
|
||||
export PYTHON='python'
|
||||
if ! ${PYTHON} --version; then
|
||||
export PYTHON=python2.7
|
||||
if ! ${PYTHON} --version; then
|
||||
@ -51,114 +54,115 @@ if ! ${PYTHON} --version; then
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -z ${DORIS_TOOLCHAIN} ]]; then
|
||||
if [[ -z "${DORIS_TOOLCHAIN}" ]]; then
|
||||
DORIS_TOOLCHAIN=gcc
|
||||
fi
|
||||
|
||||
if [[ "${DORIS_TOOLCHAIN}" == "gcc" ]]; then
|
||||
# set GCC HOME
|
||||
if [[ -z ${DORIS_GCC_HOME} ]]; then
|
||||
DORIS_GCC_HOME=$(dirname $(which gcc))/..
|
||||
if [[ -z "${DORIS_GCC_HOME}" ]]; then
|
||||
DORIS_GCC_HOME="$(dirname "$(which gcc)")"/..
|
||||
export DORIS_GCC_HOME
|
||||
fi
|
||||
|
||||
gcc_ver=$(${DORIS_GCC_HOME}/bin/gcc -dumpfullversion -dumpversion)
|
||||
gcc_ver="$("${DORIS_GCC_HOME}/bin/gcc" -dumpfullversion -dumpversion)"
|
||||
required_ver="7.3.0"
|
||||
if [[ ! "$(printf '%s\n' "$required_ver" "$gcc_ver" | sort -V | head -n1)" = "$required_ver" ]]; then
|
||||
if [[ ! "$(printf '%s\n' "${required_ver}" "${gcc_ver}" | sort -V | head -n1)" = "${required_ver}" ]]; then
|
||||
echo "Error: GCC version (${gcc_ver}) must be greater than or equal to ${required_ver}"
|
||||
exit 1
|
||||
fi
|
||||
export CC=${DORIS_GCC_HOME}/bin/gcc
|
||||
export CXX=${DORIS_GCC_HOME}/bin/g++
|
||||
if test -x ${DORIS_GCC_HOME}/bin/ld; then
|
||||
export DORIS_BIN_UTILS=${DORIS_GCC_HOME}/bin/
|
||||
export CC="${DORIS_GCC_HOME}/bin/gcc"
|
||||
export CXX="${DORIS_GCC_HOME}/bin/g++"
|
||||
if test -x "${DORIS_GCC_HOME}/bin/ld"; then
|
||||
export DORIS_BIN_UTILS="${DORIS_GCC_HOME}/bin/"
|
||||
fi
|
||||
elif [[ "${DORIS_TOOLCHAIN}" == "clang" ]]; then
|
||||
# set CLANG HOME
|
||||
if [[ -z ${DORIS_CLANG_HOME} ]]; then
|
||||
DORIS_CLANG_HOME=$(dirname $(which clang))/..
|
||||
if [[ -z "${DORIS_CLANG_HOME}" ]]; then
|
||||
DORIS_CLANG_HOME="$(dirname "$(which clang)")"/..
|
||||
export DORIS_CLANG_HOME
|
||||
fi
|
||||
|
||||
clang_ver=$(${DORIS_CLANG_HOME}/bin/clang -dumpfullversion -dumpversion)
|
||||
clang_ver="$("${DORIS_CLANG_HOME}/bin/clang" -dumpfullversion -dumpversion)"
|
||||
required_ver="13.0.0"
|
||||
if [[ ! "$(printf '%s\n' "$required_ver" "$clang_ver" | sort -V | head -n1)" = "$required_ver" ]]; then
|
||||
if [[ ! "$(printf '%s\n' "${required_ver}" "${clang_ver}" | sort -V | head -n1)" = "${required_ver}" ]]; then
|
||||
echo "Error: CLANG version (${clang_ver}) must be greater than or equal to ${required_ver}"
|
||||
exit 1
|
||||
fi
|
||||
export CC=${DORIS_CLANG_HOME}/bin/clang
|
||||
export CXX=${DORIS_CLANG_HOME}/bin/clang++
|
||||
if test -x ${DORIS_CLANG_HOME}/bin/ld.lld; then
|
||||
export DORIS_BIN_UTILS=${DORIS_CLANG_HOME}/bin/
|
||||
export CC="${DORIS_CLANG_HOME}/bin/clang"
|
||||
export CXX="${DORIS_CLANG_HOME}/bin/clang++"
|
||||
if test -x "${DORIS_CLANG_HOME}/bin/ld.lld"; then
|
||||
export DORIS_BIN_UTILS="${DORIS_CLANG_HOME}/bin/"
|
||||
fi
|
||||
if [[ -f ${DORIS_CLANG_HOME}/bin/llvm-symbolizer ]]; then
|
||||
export ASAN_SYMBOLIZER_PATH=${DORIS_CLANG_HOME}/bin/llvm-symbolizer
|
||||
if [[ -f "${DORIS_CLANG_HOME}/bin/llvm-symbolizer" ]]; then
|
||||
export ASAN_SYMBOLIZER_PATH="${DORIS_CLANG_HOME}/bin/llvm-symbolizer"
|
||||
fi
|
||||
else
|
||||
echo "Error: unknown DORIS_TOOLCHAIN=${DORIS_TOOLCHAIN}, currently only 'gcc' and 'clang' are supported"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$DORIS_BIN_UTILS" ]; then
|
||||
export DORIS_BIN_UTILS=/usr/bin/
|
||||
if [[ -z "${DORIS_BIN_UTILS}" ]]; then
|
||||
export DORIS_BIN_UTILS='/usr/bin/'
|
||||
fi
|
||||
|
||||
# export CLANG COMPATIBLE FLAGS
|
||||
export CLANG_COMPATIBLE_FLAGS=$(echo | ${DORIS_GCC_HOME}/bin/gcc -Wp,-v -xc++ - -fsyntax-only 2>&1 |
|
||||
grep -E '^\s+/' | awk '{print "-I" $1}' | tr '\n' ' ')
|
||||
|
||||
CLANG_COMPATIBLE_FLAGS="$(echo | "${DORIS_GCC_HOME:-"$(dirname "$(which gcc)")/.."}/bin/gcc" -Wp,-v -xc++ - -fsyntax-only 2>&1 |
|
||||
grep -E '^\s+/' | awk '{print "-I" $1}' | tr '\n' ' ')"
|
||||
export CLANG_COMPATIBLE_FLAGS
|
||||
|
||||
# if is called from build-thirdparty.sh, no need to check these tools
|
||||
if test -z "${BUILD_THIRDPARTY_WIP}"; then
|
||||
if test -z "${BUILD_THIRDPARTY_WIP:-}"; then
|
||||
# register keyword is forbidden to use in C++17
|
||||
# the C++ code generated by flex that remove register keyword after version 2.6.0
|
||||
# so we need check flex version here to avoid compilation failed
|
||||
flex_ver=$(flex --version | awk '{print $2}')
|
||||
flex_ver="$(flex --version | awk '{print $2}')"
|
||||
required_ver="2.6.0"
|
||||
if [[ ! "$(printf '%s\n' "$required_ver" "$flex_ver" | sort -V | head -n1)" = "$required_ver" ]]; then
|
||||
if [[ ! "$(printf '%s\n' "${required_ver}" "${flex_ver}" | sort -V | head -n1)" = "${required_ver}" ]]; then
|
||||
echo "Error: flex version (${flex_ver}) must be greater than or equal to ${required_ver}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# check java home
|
||||
if [ -z "$JAVA_HOME" ]; then
|
||||
export JAVA=$(which java)
|
||||
JAVAP=$(which javap)
|
||||
if [[ -z "${JAVA_HOME}" ]]; then
|
||||
JAVA="$(which java)"
|
||||
JAVAP="$(which javap)"
|
||||
else
|
||||
export JAVA="${JAVA_HOME}/bin/java"
|
||||
JAVA="${JAVA_HOME}/bin/java"
|
||||
JAVAP="${JAVA_HOME}/bin/javap"
|
||||
fi
|
||||
export JAVA
|
||||
|
||||
if [ ! -x "$JAVA" ]; then
|
||||
if [[ ! -x "${JAVA}" ]]; then
|
||||
echo "The JAVA_HOME environment variable is not defined correctly"
|
||||
echo "This environment variable is needed to run this program"
|
||||
echo "NB: JAVA_HOME should point to a JDK not a JRE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
JAVA_VER=$(${JAVAP} -verbose java.lang.String | grep "major version" | cut -d " " -f5)
|
||||
if [[ $JAVA_VER -lt 52 ]]; then
|
||||
JAVA_VER="$("${JAVAP}" -verbose java.lang.String | grep "major version" | cut -d " " -f5)"
|
||||
if [[ "${JAVA_VER}" -lt 52 ]]; then
|
||||
echo "Error: require JAVA with JDK version at least 1.8"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# check maven
|
||||
MVN_CMD=mvn
|
||||
if [[ ! -z ${CUSTOM_MVN} ]]; then
|
||||
MVN_CMD=${CUSTOM_MVN}
|
||||
MVN_CMD='mvn'
|
||||
if [[ -n "${CUSTOM_MVN}" ]]; then
|
||||
MVN_CMD="${CUSTOM_MVN}"
|
||||
fi
|
||||
if ! ${MVN_CMD} --version; then
|
||||
if ! "${MVN_CMD}" --version; then
|
||||
echo "Error: mvn is not found"
|
||||
exit 1
|
||||
fi
|
||||
export MVN_CMD
|
||||
fi
|
||||
|
||||
CMAKE_CMD=cmake
|
||||
if [[ ! -z ${CUSTOM_CMAKE} ]]; then
|
||||
CMAKE_CMD=${CUSTOM_CMAKE}
|
||||
CMAKE_CMD='cmake'
|
||||
if [[ -n "${CUSTOM_CMAKE}" ]]; then
|
||||
CMAKE_CMD="${CUSTOM_CMAKE}"
|
||||
fi
|
||||
if ! ${CMAKE_CMD} --version; then
|
||||
if ! "${CMAKE_CMD}" --version; then
|
||||
echo "Error: cmake is not found"
|
||||
exit 1
|
||||
fi
|
||||
@ -166,16 +170,17 @@ export CMAKE_CMD
|
||||
|
||||
GENERATOR="Unix Makefiles"
|
||||
BUILD_SYSTEM="make"
|
||||
if ninja --version 2> /dev/null; then
|
||||
if ninja --version 2>/dev/null; then
|
||||
GENERATOR="Ninja"
|
||||
BUILD_SYSTEM="ninja"
|
||||
fi
|
||||
|
||||
if ccache --version > /dev/null; then
|
||||
if ccache --version >/dev/null; then
|
||||
# shellcheck disable=2034
|
||||
CMAKE_USE_CCACHE="-DCMAKE_CXX_COMPILER_LAUNCHER=ccache"
|
||||
fi
|
||||
|
||||
export GENERATOR
|
||||
export BUILD_SYSTEM
|
||||
|
||||
export PKG_CONFIG_PATH=${DORIS_HOME}/thirdparty/installed/lib64/pkgconfig:$PKG_CONFIG_PATH
|
||||
export PKG_CONFIG_PATH="${DORIS_HOME}/thirdparty/installed/lib64/pkgconfig:${PKG_CONFIG_PATH}"
|
||||
|
||||
@ -16,25 +16,23 @@
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
set -e
|
||||
set -eo pipefail
|
||||
|
||||
ROOT=`dirname "$0"`
|
||||
ROOT=`cd "$ROOT"; pwd`
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
|
||||
|
||||
export DORIS_HOME=${ROOT}/../..
|
||||
export DORIS_HOME="${ROOT}/../.."
|
||||
|
||||
. ${DORIS_HOME}/env.sh
|
||||
. "${DORIS_HOME}/env.sh"
|
||||
|
||||
export AUDITLOADER_HOME=$ROOT
|
||||
export AUDITLOADER_HOME="${ROOT}"
|
||||
|
||||
|
||||
$MVN_CMD clean package -DskipTests
|
||||
"${MVN_CMD}" clean package -DskipTests
|
||||
|
||||
echo "Install auditloader..."
|
||||
|
||||
AUDITLOADER_OUTPUT=${AUDITLOADER_HOME}/output/
|
||||
rm -rf ${AUDITLOADER_OUTPUT}
|
||||
mkdir ${AUDITLOADER_OUTPUT}
|
||||
cp ${AUDITLOADER_HOME}/target/auditloader.zip ${AUDITLOADER_HOME}/output/
|
||||
AUDITLOADER_OUTPUT="${AUDITLOADER_HOME}/output"
|
||||
rm -rf "${AUDITLOADER_OUTPUT}"
|
||||
mkdir "${AUDITLOADER_OUTPUT}"
|
||||
cp "${AUDITLOADER_HOME}/target/auditloader.zip" "${AUDITLOADER_HOME}/output"/
|
||||
|
||||
echo "Build Auditloader Finished"
|
||||
|
||||
@ -16,19 +16,17 @@
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
curdir=$(dirname "$0")
|
||||
curdir=$(
|
||||
cd "$curdir"
|
||||
pwd
|
||||
)
|
||||
set -eo pipefail
|
||||
|
||||
OPTS=$(getopt \
|
||||
-n $0 \
|
||||
curdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
|
||||
|
||||
OPTS="$(getopt \
|
||||
-n "$0" \
|
||||
-o '' \
|
||||
-l 'daemon' \
|
||||
-- "$@")
|
||||
-- "$@")"
|
||||
|
||||
eval set -- "$OPTS"
|
||||
eval set -- "${OPTS}"
|
||||
|
||||
RUN_DAEMON=0
|
||||
while true; do
|
||||
@ -48,25 +46,28 @@ while true; do
|
||||
esac
|
||||
done
|
||||
|
||||
export BROKER_HOME=$(
|
||||
cd "$curdir/.."
|
||||
BROKER_HOME="$(
|
||||
cd "${curdir}/.."
|
||||
pwd
|
||||
)
|
||||
export PID_DIR=$(
|
||||
cd "$curdir"
|
||||
)"
|
||||
export BROKER_HOME
|
||||
|
||||
PID_DIR="$(
|
||||
cd "${curdir}"
|
||||
pwd
|
||||
)
|
||||
)"
|
||||
export PID_DIR
|
||||
|
||||
export JAVA_OPTS="-Xmx1024m -Dfile.encoding=UTF-8"
|
||||
export BROKER_LOG_DIR="$BROKER_HOME/log"
|
||||
export BROKER_LOG_DIR="${BROKER_HOME}/log"
|
||||
# java
|
||||
if [ -z "$JAVA_HOME" ]; then
|
||||
JAVA=$(which java)
|
||||
if [[ -z "${JAVA_HOME}" ]]; then
|
||||
JAVA="$(which java)"
|
||||
else
|
||||
JAVA="$JAVA_HOME/bin/java"
|
||||
JAVA="${JAVA_HOME}/bin/java"
|
||||
fi
|
||||
|
||||
if [ ! -x "$JAVA" ]; then
|
||||
if [[ ! -x "${JAVA}" ]]; then
|
||||
echo "The JAVA_HOME environment variable is not defined correctly"
|
||||
echo "This environment variable is needed to run this program"
|
||||
echo "NB: JAVA_HOME should point to a JDK not a JRE"
|
||||
@ -74,38 +75,42 @@ if [ ! -x "$JAVA" ]; then
|
||||
fi
|
||||
|
||||
# add libs to CLASSPATH
|
||||
for f in $BROKER_HOME/lib/*.jar; do
|
||||
CLASSPATH=$f:${CLASSPATH}
|
||||
for f in "${BROKER_HOME}/lib"/*.jar; do
|
||||
CLASSPATH="${f}:${CLASSPATH}"
|
||||
done
|
||||
export CLASSPATH=${CLASSPATH}:${BROKER_HOME}/lib:$BROKER_HOME/conf
|
||||
export CLASSPATH="${CLASSPATH}:${BROKER_HOME}/lib:${BROKER_HOME}/conf"
|
||||
|
||||
while read line; do
|
||||
envline=$(echo $line | sed 's/[[:blank:]]*=[[:blank:]]*/=/g' | sed 's/^[[:blank:]]*//g' | egrep "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*=")
|
||||
envline=$(eval "echo $envline")
|
||||
if [[ $envline == *"="* ]]; then
|
||||
eval 'export "$envline"'
|
||||
while read -r line; do
|
||||
envline="$(echo "${line}" |
|
||||
sed 's/[[:blank:]]*=[[:blank:]]*/=/g' |
|
||||
sed 's/^[[:blank:]]*//g' |
|
||||
grep -E "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*=" ||
|
||||
true)"
|
||||
envline="$(eval "echo ${envline}")"
|
||||
if [[ "${envline}" == *"="* ]]; then
|
||||
eval 'export "${envline}"'
|
||||
fi
|
||||
done < $BROKER_HOME/conf/apache_hdfs_broker.conf
|
||||
done <"${BROKER_HOME}/conf/apache_hdfs_broker.conf"
|
||||
|
||||
pidfile=$PID_DIR/apache_hdfs_broker.pid
|
||||
pidfile="${PID_DIR}/apache_hdfs_broker.pid"
|
||||
|
||||
if [ -f $pidfile ]; then
|
||||
if kill -0 $(cat $pidfile) > /dev/null 2>&1; then
|
||||
echo "Broker running as process $(cat $pidfile). Stop it first."
|
||||
if [[ -f "${pidfile}" ]]; then
|
||||
if kill -0 "$(cat "${pidfile}")" >/dev/null 2>&1; then
|
||||
echo "Broker running as process $(cat "${pidfile}"). Stop it first."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ! -d $BROKER_LOG_DIR ]; then
|
||||
mkdir -p $BROKER_LOG_DIR
|
||||
if [[ ! -d "${BROKER_LOG_DIR}" ]]; then
|
||||
mkdir -p "${BROKER_LOG_DIR}"
|
||||
fi
|
||||
|
||||
echo $(date) >> $BROKER_LOG_DIR/apache_hdfs_broker.out
|
||||
date >>"${BROKER_LOG_DIR}/apache_hdfs_broker.out"
|
||||
|
||||
if [ ${RUN_DAEMON} -eq 1 ]; then
|
||||
nohup $LIMIT $JAVA $JAVA_OPTS org.apache.doris.broker.hdfs.BrokerBootstrap "$@" >> $BROKER_LOG_DIR/apache_hdfs_broker.out 2>&1 < /dev/null &
|
||||
if [[ ${RUN_DAEMON} -eq 1 ]]; then
|
||||
nohup ${LIMIT:+${LIMIT}} "${JAVA}" ${JAVA_OPTS:+${JAVA_OPTS}} org.apache.doris.broker.hdfs.BrokerBootstrap "$@" >>"${BROKER_LOG_DIR}/apache_hdfs_broker.out" 2>&1 </dev/null &
|
||||
else
|
||||
$LIMIT $JAVA $JAVA_OPTS org.apache.doris.broker.hdfs.BrokerBootstrap "$@" >> $BROKER_LOG_DIR/apache_hdfs_broker.out 2>&1 < /dev/null
|
||||
${LIMIT:+${LIMIT}} "${JAVA}" ${JAVA_OPTS:+${JAVA_OPTS}} org.apache.doris.broker.hdfs.BrokerBootstrap "$@" >>"${BROKER_LOG_DIR}/apache_hdfs_broker.out" 2>&1 </dev/null
|
||||
fi
|
||||
|
||||
echo $! > $pidfile
|
||||
echo $! >"${pidfile}"
|
||||
|
||||
@ -16,33 +16,34 @@
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
curdir=`dirname "$0"`
|
||||
curdir=`cd "$curdir"; pwd`
|
||||
set -eo pipefail
|
||||
|
||||
export BROKER_HOME=`cd "$curdir/.."; pwd`
|
||||
export PID_DIR=`cd "$curdir"; pwd`
|
||||
curdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
|
||||
|
||||
while read line; do
|
||||
envline=`echo $line | sed 's/[[:blank:]]*=[[:blank:]]*/=/g' | sed 's/^[[:blank:]]*//g' | egrep "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*="`
|
||||
envline=`eval "echo $envline"`
|
||||
if [[ $envline == *"="* ]]; then
|
||||
eval 'export "$envline"'
|
||||
BROKER_HOME=$(
|
||||
cd "${curdir}/.."
|
||||
pwd
|
||||
)
|
||||
export BROKER_HOME
|
||||
|
||||
PID_DIR=$(
|
||||
cd "${curdir}"
|
||||
pwd
|
||||
)
|
||||
export PID_DIR
|
||||
|
||||
pidfile="${PID_DIR}/apache_hdfs_broker.pid"
|
||||
|
||||
if [[ -f "${pidfile}" ]]; then
|
||||
pid="$(cat "${pidfile}")"
|
||||
pidcomm="$(ps -p "${pid}" -o comm=)"
|
||||
|
||||
if [[ "java" != "${pidcomm}" ]]; then
|
||||
echo "ERROR: pid process may not broker. "
|
||||
fi
|
||||
done < $BROKER_HOME/conf/apache_hdfs_broker.conf
|
||||
|
||||
pidfile=$PID_DIR/apache_hdfs_broker.pid
|
||||
|
||||
if [ -f $pidfile ]; then
|
||||
pid=`cat $pidfile`
|
||||
pidcomm=`ps -p $pid -o comm=`
|
||||
|
||||
if [ "java" != "$pidcomm" ]; then
|
||||
echo "ERROR: pid process may not broker. "
|
||||
fi
|
||||
|
||||
if kill -9 $pid > /dev/null 2>&1; then
|
||||
echo "stop $pidcomm, and remove pid file. "
|
||||
rm $pidfile
|
||||
fi
|
||||
if kill -9 "${pid}" >/dev/null 2>&1; then
|
||||
echo "stop ${pidcomm}, and remove pid file. "
|
||||
rm "${pidfile}"
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
@ -16,37 +16,36 @@
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
set -e
|
||||
set -eo pipefail
|
||||
|
||||
ROOT=`dirname "$0"`
|
||||
ROOT=`cd "$ROOT"; pwd`
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
|
||||
|
||||
export DORIS_HOME=${ROOT}/../..
|
||||
export DORIS_HOME="${ROOT}/../.."
|
||||
|
||||
. ${DORIS_HOME}/env.sh
|
||||
. "${DORIS_HOME}/env.sh"
|
||||
|
||||
export BROKER_HOME=$ROOT
|
||||
export BROKER_HOME="${ROOT}"
|
||||
|
||||
# prepare thrift
|
||||
mkdir -p ${BROKER_HOME}/src/main/resources/thrift
|
||||
mkdir -p ${BROKER_HOME}/src/main/thrift
|
||||
mkdir -p "${BROKER_HOME}/src/main/resources/thrift"
|
||||
mkdir -p "${BROKER_HOME}/src/main/thrift"
|
||||
|
||||
cp ${BROKER_HOME}/../../gensrc/thrift/PaloBrokerService.thrift ${BROKER_HOME}/src/main/resources/thrift/
|
||||
cp "${BROKER_HOME}/../../gensrc/thrift/PaloBrokerService.thrift" "${BROKER_HOME}/src/main/resources/thrift"/
|
||||
|
||||
$MVN_CMD package -DskipTests
|
||||
"${MVN_CMD}" package -DskipTests
|
||||
|
||||
echo "Install broker..."
|
||||
BROKER_OUTPUT=${BROKER_HOME}/output/apache_hdfs_broker/
|
||||
rm -rf ${BROKER_OUTPUT}
|
||||
BROKER_OUTPUT="${BROKER_HOME}/output/apache_hdfs_broker"
|
||||
rm -rf "${BROKER_OUTPUT}"
|
||||
|
||||
install -d ${BROKER_OUTPUT}/bin ${BROKER_OUTPUT}/conf \
|
||||
${BROKER_OUTPUT}lib/
|
||||
install -d "${BROKER_OUTPUT}/bin" "${BROKER_OUTPUT}/conf" \
|
||||
"${BROKER_OUTPUT}/lib"
|
||||
|
||||
cp -r -p ${BROKER_HOME}/bin/*.sh ${BROKER_OUTPUT}/bin/
|
||||
cp -r -p ${BROKER_HOME}/conf/*.conf ${BROKER_OUTPUT}/conf/
|
||||
cp -r -p ${BROKER_HOME}/conf/*.xml ${BROKER_OUTPUT}/conf/
|
||||
cp -r -p ${BROKER_HOME}/conf/log4j.properties ${BROKER_OUTPUT}/conf/
|
||||
cp -r -p ${BROKER_HOME}/target/lib/* ${BROKER_OUTPUT}/lib/
|
||||
cp -r -p ${BROKER_HOME}/target/apache_hdfs_broker.jar ${BROKER_OUTPUT}/lib/
|
||||
cp -r -p "${BROKER_HOME}/bin"/*.sh "${BROKER_OUTPUT}/bin"/
|
||||
cp -r -p "${BROKER_HOME}/conf"/*.conf "${BROKER_OUTPUT}/conf"/
|
||||
cp -r -p "${BROKER_HOME}/conf"/*.xml "${BROKER_OUTPUT}/conf"/
|
||||
cp -r -p "${BROKER_HOME}/conf/log4j.properties" "${BROKER_OUTPUT}/conf"/
|
||||
cp -r -p "${BROKER_HOME}/target/lib"/* "${BROKER_OUTPUT}/lib"/
|
||||
cp -r -p "${BROKER_HOME}/target/apache_hdfs_broker.jar" "${BROKER_OUTPUT}/lib"/
|
||||
|
||||
echo "Finished"
|
||||
|
||||
@ -25,45 +25,43 @@
|
||||
# contains the build version based on the git hash or svn revision.
|
||||
##############################################################
|
||||
|
||||
set -eo pipefail
|
||||
|
||||
build_version="trunk"
|
||||
|
||||
unset LANG
|
||||
unset LC_CTYPE
|
||||
|
||||
user=$(whoami)
|
||||
date=$(date +"%a, %d %b %Y %H:%M:%S %Z")
|
||||
hostname=$(hostname)
|
||||
user="$(whoami)"
|
||||
date="$(date +"%a, %d %b %Y %H:%M:%S %Z")"
|
||||
hostname="$(hostname)"
|
||||
|
||||
cwd=$(pwd)
|
||||
cwd="$(pwd)"
|
||||
|
||||
if [ -z ${DORIS_HOME+x} ]; then
|
||||
ROOT=$(dirname "$0")
|
||||
ROOT=$(
|
||||
cd "$ROOT"
|
||||
pwd
|
||||
)
|
||||
DORIS_HOME=${ROOT}/../..
|
||||
if [[ -z "${DORIS_HOME}" ]]; then
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
|
||||
DORIS_HOME="${ROOT}/../.."
|
||||
echo "DORIS_HOME: ${DORIS_HOME}"
|
||||
fi
|
||||
|
||||
if [[ -z ${DORIS_TEST_BINARY_DIR} ]]; then
|
||||
if [ -e ${DORIS_HOME}/fe/fe-core/target/generated-sources/build/org/apache/doris/common/Version.java \
|
||||
-a -e ${DORIS_HOME}/gensrc/build/gen_cpp/version.h ]; then
|
||||
if [[ -z "${DORIS_TEST_BINARY_DIR}" ]]; then
|
||||
if [[ -e "${DORIS_HOME}/fe/fe-core/target/generated-sources/build/org/apache/doris/common/Version.java" &&
|
||||
-e "${DORIS_HOME}/gensrc/build/gen_cpp/version.h" ]]; then
|
||||
exit
|
||||
fi
|
||||
fi
|
||||
|
||||
cd ${DORIS_HOME}
|
||||
if [ -d .svn ]; then
|
||||
cd "${DORIS_HOME}"
|
||||
if [[ -d '.svn' ]]; then
|
||||
revision=$(svn info | sed -n -e 's/Last Changed Rev: \(.*\)/\1/p')
|
||||
short_revision="${revision}"
|
||||
url=$(svn info | sed -n -e 's/^URL: \(.*\)/\1/p')
|
||||
if echo ${url} | grep '\/tags\/' > /dev/null; then
|
||||
build_version="$(echo ${url} | sed 's/.*_\([0-9-]\+\)_PD_BL.*/\1/g' | sed 's/-/\./g')"
|
||||
if echo "${url}" | grep '\/tags\/' >/dev/null; then
|
||||
build_version="$(echo "${url}" | sed 's/.*_\([0-9-]\+\)_PD_BL.*/\1/g' | sed 's/-/\./g')"
|
||||
fi
|
||||
elif [ -d .git ]; then
|
||||
revision=$(git log -1 --pretty=format:"%H")
|
||||
short_revision=$(git log -1 --pretty=format:"%h")
|
||||
elif [[ -d '.git' ]]; then
|
||||
revision="$(git log -1 --pretty=format:"%H")"
|
||||
short_revision="$(git log -1 --pretty=format:"%h")"
|
||||
url="git://${hostname}${DORIS_HOME}"
|
||||
else
|
||||
revision="Unknown"
|
||||
@ -71,34 +69,33 @@ else
|
||||
url="file://${DORIS_HOME}"
|
||||
fi
|
||||
|
||||
cd ${cwd}
|
||||
cd "${cwd}"
|
||||
|
||||
build_hash="${url}@${revision}"
|
||||
build_short_hash="${short_revision}"
|
||||
build_time="${date}"
|
||||
build_info="${user}@${hostname}"
|
||||
|
||||
if [ -z "$JAVA_HOME" ]; then
|
||||
java_cmd=$(which java)
|
||||
if [[ -z "${JAVA_HOME}" ]]; then
|
||||
java_cmd="$(which java)"
|
||||
else
|
||||
java_cmd="$JAVA_HOME/bin/java"
|
||||
java_cmd="${JAVA_HOME}/bin/java"
|
||||
fi
|
||||
|
||||
if [ ! -x "$java_cmd" ]; then
|
||||
if [[ ! -x "${java_cmd}" ]]; then
|
||||
echo "The JAVA_HOME environment variable is not defined correctly"
|
||||
echo "This environment variable is needed to run this program"
|
||||
echo "NB: JAVA_HOME should point to a JDK not a JRE"
|
||||
exit 1
|
||||
fi
|
||||
java_version_str=$($java_cmd -fullversion 2>&1)
|
||||
java_version_str=$(echo $java_version_str | sed -e 's/"/\\"/g')
|
||||
java_version_str="$("${java_cmd}" -fullversion 2>&1 | sed -e 's/"/\\"/g')"
|
||||
|
||||
echo "get java cmd: $java_cmd"
|
||||
echo "get java version: $java_version_str"
|
||||
echo "get java cmd: ${java_cmd}"
|
||||
echo "get java version: ${java_version_str}"
|
||||
|
||||
VERSION_PACKAGE="${DORIS_HOME}/fe/fe-core/target/generated-sources/build/org/apache/doris/common"
|
||||
mkdir -p ${VERSION_PACKAGE}
|
||||
cat > "${VERSION_PACKAGE}/Version.java" << EOF
|
||||
mkdir -p "${VERSION_PACKAGE}"
|
||||
cat >"${VERSION_PACKAGE}/Version.java" <<EOF
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
@ -143,9 +140,9 @@ public class Version {
|
||||
}
|
||||
EOF
|
||||
|
||||
GEN_CPP_DIR=${DORIS_HOME}/gensrc/build/gen_cpp/
|
||||
mkdir -p ${GEN_CPP_DIR}
|
||||
cat > "${GEN_CPP_DIR}/version.h" << EOF
|
||||
GEN_CPP_DIR="${DORIS_HOME}/gensrc/build/gen_cpp"
|
||||
mkdir -p "${GEN_CPP_DIR}"
|
||||
cat >"${GEN_CPP_DIR}/version.h" <<EOF
|
||||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
|
||||
223
run-be-ut.sh
223
run-be-ut.sh
@ -32,14 +32,15 @@
|
||||
# GTest result xml files will be in "be/ut_build_ASAN/gtest_output/"
|
||||
#####################################################################
|
||||
|
||||
ROOT=`dirname "$0"`
|
||||
ROOT=`cd "$ROOT"; pwd`
|
||||
set -eo pipefail
|
||||
|
||||
export DORIS_HOME=${ROOT}
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
|
||||
|
||||
export DORIS_HOME="${ROOT}"
|
||||
|
||||
# Check args
|
||||
usage() {
|
||||
echo "
|
||||
echo "
|
||||
Usage: $0 <options>
|
||||
Optional options:
|
||||
--benchmark build benchmark-tool
|
||||
@ -61,74 +62,92 @@ Usage: $0 <options>
|
||||
$0 --clean clean and build tests
|
||||
$0 --clean --run clean, build and run all tests
|
||||
"
|
||||
exit 1
|
||||
exit 1
|
||||
}
|
||||
|
||||
OPTS=$(getopt -n $0 -o vhj:f: -l benchmark,run,clean,filter: -- "$@")
|
||||
if [ "$?" != "0" ]; then
|
||||
usage
|
||||
if ! OPTS="$(getopt -n "$0" -o vhj:f: -l benchmark,run,clean,filter: -- "$@")"; then
|
||||
usage
|
||||
fi
|
||||
|
||||
set -eo pipefail
|
||||
eval set -- "${OPTS}"
|
||||
|
||||
eval set -- "$OPTS"
|
||||
|
||||
PARALLEL=$[$(nproc)/5+1]
|
||||
PARALLEL="$(($(nproc) / 5 + 1))"
|
||||
|
||||
CLEAN=0
|
||||
RUN=0
|
||||
BUILD_BENCHMARK_TOOL=OFF
|
||||
BUILD_BENCHMARK_TOOL='OFF'
|
||||
FILTER=""
|
||||
if [ $# != 1 ] ; then
|
||||
while true; do
|
||||
if [[ "$#" != 1 ]]; then
|
||||
while true; do
|
||||
case "$1" in
|
||||
--clean) CLEAN=1 ; shift ;;
|
||||
--run) RUN=1 ; shift ;;
|
||||
--benchmark) BUILD_BENCHMARK_TOOL=ON ; shift ;;
|
||||
-f | --filter) FILTER="--gtest_filter=$2"; shift 2;;
|
||||
-j) PARALLEL=$2; shift 2 ;;
|
||||
--) shift ; break ;;
|
||||
*) usage ; exit 0 ;;
|
||||
--clean)
|
||||
CLEAN=1
|
||||
shift
|
||||
;;
|
||||
--run)
|
||||
RUN=1
|
||||
shift
|
||||
;;
|
||||
--benchmark)
|
||||
BUILD_BENCHMARK_TOOL='ON'
|
||||
shift
|
||||
;;
|
||||
-f | --filter)
|
||||
FILTER="--gtest_filter=$2"
|
||||
shift 2
|
||||
;;
|
||||
-j)
|
||||
PARALLEL="$2"
|
||||
shift 2
|
||||
;;
|
||||
--)
|
||||
shift
|
||||
break
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
done
|
||||
fi
|
||||
|
||||
CMAKE_BUILD_TYPE=${BUILD_TYPE:-ASAN}
|
||||
CMAKE_BUILD_TYPE="${BUILD_TYPE:-ASAN}"
|
||||
CMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE^^}"
|
||||
|
||||
echo "Get params:
|
||||
PARALLEL -- $PARALLEL
|
||||
CLEAN -- $CLEAN
|
||||
PARALLEL -- ${PARALLEL}
|
||||
CLEAN -- ${CLEAN}
|
||||
"
|
||||
echo "Build Backend UT"
|
||||
|
||||
. ${DORIS_HOME}/env.sh
|
||||
. "${DORIS_HOME}/env.sh"
|
||||
|
||||
CMAKE_BUILD_DIR=${DORIS_HOME}/be/ut_build_${CMAKE_BUILD_TYPE}
|
||||
if [ ${CLEAN} -eq 1 ]; then
|
||||
rm ${CMAKE_BUILD_DIR} -rf
|
||||
rm ${DORIS_HOME}/be/output/ -rf
|
||||
CMAKE_BUILD_DIR="${DORIS_HOME}/be/ut_build_${CMAKE_BUILD_TYPE}"
|
||||
if [[ "${CLEAN}" -eq 1 ]]; then
|
||||
rm -rf "${CMAKE_BUILD_DIR}"
|
||||
rm -rf "${DORIS_HOME}/be/output"
|
||||
fi
|
||||
|
||||
if [ ! -d ${CMAKE_BUILD_DIR} ]; then
|
||||
mkdir -p ${CMAKE_BUILD_DIR}
|
||||
if [[ ! -d "${CMAKE_BUILD_DIR}" ]]; then
|
||||
mkdir -p "${CMAKE_BUILD_DIR}"
|
||||
fi
|
||||
|
||||
if [[ -z ${GLIBC_COMPATIBILITY} ]]; then
|
||||
GLIBC_COMPATIBILITY=ON
|
||||
if [[ -z "${GLIBC_COMPATIBILITY}" ]]; then
|
||||
GLIBC_COMPATIBILITY='ON'
|
||||
fi
|
||||
|
||||
if [[ -z ${USE_DWARF} ]]; then
|
||||
USE_DWARF=OFF
|
||||
if [[ -z "${USE_DWARF}" ]]; then
|
||||
USE_DWARF='OFF'
|
||||
fi
|
||||
|
||||
MAKE_PROGRAM="$(which "${BUILD_SYSTEM}")"
|
||||
echo "-- Make program: ${MAKE_PROGRAM}"
|
||||
echo "-- Use ccache: ${CMAKE_USE_CCACHE}"
|
||||
echo "-- Extra cxx flags: ${EXTRA_CXX_FLAGS}"
|
||||
echo "-- Extra cxx flags: ${EXTRA_CXX_FLAGS:-}"
|
||||
|
||||
cd ${CMAKE_BUILD_DIR}
|
||||
${CMAKE_CMD} -G "${GENERATOR}" \
|
||||
cd "${CMAKE_BUILD_DIR}"
|
||||
"${CMAKE_CMD}" -G "${GENERATOR}" \
|
||||
-DCMAKE_MAKE_PROGRAM="${MAKE_PROGRAM}" \
|
||||
-DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" \
|
||||
-DMAKE_TEST=ON \
|
||||
@ -136,16 +155,16 @@ ${CMAKE_CMD} -G "${GENERATOR}" \
|
||||
-DBUILD_META_TOOL=OFF \
|
||||
-DBUILD_BENCHMARK_TOOL="${BUILD_BENCHMARK_TOOL}" \
|
||||
-DWITH_MYSQL=OFF \
|
||||
-DUSE_DWARF=${USE_DWARF} \
|
||||
-DUSE_DWARF="${USE_DWARF}" \
|
||||
-DUSE_MEM_TRACKER=ON \
|
||||
-DUSE_JEMALLOC=OFF \
|
||||
-DSTRICT_MEMORY_USE=OFF \
|
||||
-DEXTRA_CXX_FLAGS="${EXTRA_CXX_FLAGS}" \
|
||||
${CMAKE_USE_CCACHE} \
|
||||
${DORIS_HOME}/be/
|
||||
${BUILD_SYSTEM} -j ${PARALLEL}
|
||||
${CMAKE_USE_CCACHE:+${CMAKE_USE_CCACHE}} \
|
||||
"${DORIS_HOME}/be"
|
||||
"${BUILD_SYSTEM}" -j "${PARALLEL}"
|
||||
|
||||
if [ ${RUN} -ne 1 ]; then
|
||||
if [[ "${RUN}" -ne 1 ]]; then
|
||||
echo "Finished"
|
||||
exit 0
|
||||
fi
|
||||
@ -154,96 +173,94 @@ echo "******************************"
|
||||
echo " Running Backend Unit Test "
|
||||
echo "******************************"
|
||||
|
||||
cd ${DORIS_HOME}
|
||||
export DORIS_TEST_BINARY_DIR=${CMAKE_BUILD_DIR}
|
||||
export TERM=xterm
|
||||
export UDF_RUNTIME_DIR=${DORIS_HOME}/lib/udf-runtime
|
||||
export LOG_DIR=${DORIS_HOME}/log
|
||||
for i in `sed 's/ //g' $DORIS_HOME/conf/be.conf | egrep "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*="`; do
|
||||
eval "export $i";
|
||||
done
|
||||
cd "${DORIS_HOME}"
|
||||
export DORIS_TEST_BINARY_DIR="${CMAKE_BUILD_DIR}"
|
||||
export TERM='xterm'
|
||||
export UDF_RUNTIME_DIR="${DORIS_HOME}/lib/udf-runtime"
|
||||
export LOG_DIR="${DORIS_HOME}/log"
|
||||
while read -r variable; do
|
||||
eval "export ${variable}"
|
||||
done < <(sed 's/ //g' "${DORIS_HOME}/conf/be.conf" | grep -E "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*=")
|
||||
|
||||
mkdir -p $LOG_DIR
|
||||
mkdir -p ${UDF_RUNTIME_DIR}
|
||||
rm -f ${UDF_RUNTIME_DIR}/*
|
||||
mkdir -p "${LOG_DIR}"
|
||||
mkdir -p "${UDF_RUNTIME_DIR}"
|
||||
rm -f "${UDF_RUNTIME_DIR}"/*
|
||||
|
||||
# clean all gcda file
|
||||
while read -r gcda_file; do
|
||||
rm "${gcda_file}"
|
||||
done < <(find "${DORIS_TEST_BINARY_DIR}" -name "*gcda")
|
||||
|
||||
gcda_files=`find ${DORIS_TEST_BINARY_DIR} -name "*gcda"`
|
||||
for gcda_file in ${gcda_files[@]}
|
||||
do
|
||||
rm $gcda_file
|
||||
done
|
||||
|
||||
export DORIS_TEST_BINARY_DIR=${DORIS_TEST_BINARY_DIR}/test/
|
||||
export DORIS_TEST_BINARY_DIR="${DORIS_TEST_BINARY_DIR}/test"
|
||||
|
||||
# prepare jvm if needed
|
||||
jdk_version() {
|
||||
local result
|
||||
local java_cmd=$JAVA_HOME/bin/java
|
||||
local java_cmd="${JAVA_HOME}/bin/java"
|
||||
local IFS=$'\n'
|
||||
# remove \r for Cygwin
|
||||
local lines=$("$java_cmd" -Xms32M -Xmx32M -version 2>&1 | tr '\r' '\n')
|
||||
if [[ -z $java_cmd ]]
|
||||
then
|
||||
|
||||
if [[ -z "${java_cmd}" ]]; then
|
||||
result=no_java
|
||||
return 1
|
||||
else
|
||||
for line in $lines; do
|
||||
if [[ (-z $result) && ($line = *"version \""*) ]]
|
||||
then
|
||||
local ver=$(echo $line | sed -e 's/.*version "\(.*\)"\(.*\)/\1/; 1q')
|
||||
# on macOS, sed doesn't support '?'
|
||||
if [[ $ver = "1."* ]]
|
||||
then
|
||||
result=$(echo $ver | sed -e 's/1\.\([0-9]*\)\(.*\)/\1/; 1q')
|
||||
else
|
||||
result=$(echo $ver | sed -e 's/\([0-9]*\)\(.*\)/\1/; 1q')
|
||||
fi
|
||||
fi
|
||||
done
|
||||
local version
|
||||
# remove \r for Cygwin
|
||||
version="$("${java_cmd}" -Xms32M -Xmx32M -version 2>&1 | tr '\r' '\n' | grep version | awk '{print $3}')"
|
||||
version="${version//\"/}"
|
||||
if [[ "${version}" =~ ^1\. ]]; then
|
||||
result="$(echo "${version}" | awk -F '.' '{print $2}')"
|
||||
else
|
||||
result="$(echo "${version}" | awk -F '.' '{print $1}')"
|
||||
fi
|
||||
fi
|
||||
echo "$result"
|
||||
echo "${result}"
|
||||
return 0
|
||||
}
|
||||
|
||||
jvm_arch="amd64"
|
||||
MACHINE_TYPE=$(uname -m)
|
||||
MACHINE_TYPE="$(uname -m)"
|
||||
if [[ "${MACHINE_TYPE}" == "aarch64" ]]; then
|
||||
jvm_arch="aarch64"
|
||||
fi
|
||||
java_version=$(jdk_version)
|
||||
if [[ $java_version -gt 8 ]]; then
|
||||
export LD_LIBRARY_PATH=$JAVA_HOME/lib/server:$JAVA_HOME/lib:$LD_LIBRARY_PATH
|
||||
java_version="$(
|
||||
set -e
|
||||
jdk_version
|
||||
)"
|
||||
if [[ "${java_version}" -gt 8 ]]; then
|
||||
export LD_LIBRARY_PATH="${JAVA_HOME}/lib/server:${JAVA_HOME}/lib:${LD_LIBRARY_PATH}"
|
||||
# JAVA_HOME is jdk
|
||||
elif [[ -d "$JAVA_HOME/jre" ]]; then
|
||||
export LD_LIBRARY_PATH=$JAVA_HOME/jre/lib/$jvm_arch/server:$JAVA_HOME/jre/lib/$jvm_arch:$LD_LIBRARY_PATH
|
||||
elif [[ -d "${JAVA_HOME}/jre" ]]; then
|
||||
export LD_LIBRARY_PATH="${JAVA_HOME}/jre/lib/${jvm_arch}/server:${JAVA_HOME}/jre/lib/${jvm_arch}:${LD_LIBRARY_PATH}"
|
||||
# JAVA_HOME is jre
|
||||
else
|
||||
export LD_LIBRARY_PATH=$JAVA_HOME/lib/$jvm_arch/server:$JAVA_HOME/lib/$jvm_arch:$LD_LIBRARY_PATH
|
||||
export LD_LIBRARY_PATH="${JAVA_HOME}/lib/${jvm_arch}/server:${JAVA_HOME}/lib/${jvm_arch}:${LD_LIBRARY_PATH}"
|
||||
fi
|
||||
|
||||
# prepare gtest output dir
|
||||
GTEST_OUTPUT_DIR=${CMAKE_BUILD_DIR}/gtest_output
|
||||
rm -rf ${GTEST_OUTPUT_DIR} && mkdir ${GTEST_OUTPUT_DIR}
|
||||
GTEST_OUTPUT_DIR="${CMAKE_BUILD_DIR}/gtest_output"
|
||||
rm -rf "${GTEST_OUTPUT_DIR}"
|
||||
mkdir "${GTEST_OUTPUT_DIR}"
|
||||
|
||||
# prepare util test_data
|
||||
mkdir -p ${DORIS_TEST_BINARY_DIR}/util
|
||||
if [ -d ${DORIS_TEST_BINARY_DIR}/util/test_data ]; then
|
||||
rm -rf ${DORIS_TEST_BINARY_DIR}/util/test_data
|
||||
mkdir -p "${DORIS_TEST_BINARY_DIR}/util"
|
||||
if [[ -d "${DORIS_TEST_BINARY_DIR}/util/test_data" ]]; then
|
||||
rm -rf "${DORIS_TEST_BINARY_DIR}/util/test_data"
|
||||
fi
|
||||
cp -r ${DORIS_HOME}/be/test/util/test_data ${DORIS_TEST_BINARY_DIR}/util/
|
||||
cp -r "${DORIS_HOME}/be/test/util/test_data" "${DORIS_TEST_BINARY_DIR}/util"/
|
||||
|
||||
# prepare ut temp dir
|
||||
UT_TMP_DIR=${DORIS_HOME}/ut_dir
|
||||
rm -rf ${UT_TMP_DIR} && mkdir ${UT_TMP_DIR}
|
||||
touch ${UT_TMP_DIR}/tmp_file
|
||||
UT_TMP_DIR="${DORIS_HOME}/ut_dir"
|
||||
rm -rf "${UT_TMP_DIR}"
|
||||
mkdir "${UT_TMP_DIR}"
|
||||
touch "${UT_TMP_DIR}/tmp_file"
|
||||
|
||||
# find all executable test files
|
||||
|
||||
test=${DORIS_TEST_BINARY_DIR}doris_be_test
|
||||
file_name=${test##*/}
|
||||
if [ -f "$test" ]; then
|
||||
$test --gtest_output=xml:${GTEST_OUTPUT_DIR}/${file_name}.xml --gtest_print_time=true "${FILTER}"
|
||||
test="${DORIS_TEST_BINARY_DIR}/doris_be_test"
|
||||
file_name="${test##*/}"
|
||||
if [[ -f "${test}" ]]; then
|
||||
"${test}" --gtest_output="xml:${GTEST_OUTPUT_DIR}/${file_name}.xml" --gtest_print_time=true "${FILTER}"
|
||||
echo "=== Finished. Gtest output: ${GTEST_OUTPUT_DIR}"
|
||||
else
|
||||
echo "unit test file: $test does not exist."
|
||||
else
|
||||
echo "unit test file: ${test} does not exist."
|
||||
fi
|
||||
|
||||
67
run-fe-ut.sh
67
run-fe-ut.sh
@ -18,16 +18,15 @@
|
||||
|
||||
set -eo pipefail
|
||||
|
||||
ROOT=`dirname "$0"`
|
||||
ROOT=`cd "$ROOT"; pwd`
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
|
||||
|
||||
export DORIS_HOME=${ROOT}
|
||||
export DORIS_HOME="${ROOT}"
|
||||
|
||||
. ${DORIS_HOME}/env.sh
|
||||
. "${DORIS_HOME}/env.sh"
|
||||
|
||||
# Check args
|
||||
usage() {
|
||||
echo "
|
||||
echo "
|
||||
Usage: $0 <options>
|
||||
Optional options:
|
||||
--clean clean and build ut
|
||||
@ -38,25 +37,23 @@ Usage: $0 <options>
|
||||
$0 --coverage build and run coverage statistic
|
||||
$0 --run xxx build and run the specified class
|
||||
"
|
||||
exit 1
|
||||
exit 1
|
||||
}
|
||||
|
||||
OPTS=$(getopt \
|
||||
-n $0 \
|
||||
-o '' \
|
||||
-l 'coverage' \
|
||||
-l 'run' \
|
||||
-- "$@")
|
||||
|
||||
if [ $? != 0 ] ; then
|
||||
if ! OPTS="$(getopt \
|
||||
-n "$0" \
|
||||
-o '' \
|
||||
-l 'coverage' \
|
||||
-l 'run' \
|
||||
-- "$@")"; then
|
||||
usage
|
||||
fi
|
||||
|
||||
eval set -- "$OPTS"
|
||||
eval set -- "${OPTS}"
|
||||
|
||||
RUN=
|
||||
COVERAGE=
|
||||
if [ $# == 1 ] ; then
|
||||
RUN=0
|
||||
COVERAGE=0
|
||||
if [[ "$#" == 1 ]]; then
|
||||
#default
|
||||
RUN=0
|
||||
COVERAGE=0
|
||||
@ -65,10 +62,22 @@ else
|
||||
COVERAGE=0
|
||||
while true; do
|
||||
case "$1" in
|
||||
--coverage) COVERAGE=1 ; shift ;;
|
||||
--run) RUN=1 ; shift ;;
|
||||
--) shift ; break ;;
|
||||
*) echo "Internal error" ; exit 1 ;;
|
||||
--coverage)
|
||||
COVERAGE=1
|
||||
shift
|
||||
;;
|
||||
--run)
|
||||
RUN=1
|
||||
shift
|
||||
;;
|
||||
--)
|
||||
shift
|
||||
break
|
||||
;;
|
||||
*)
|
||||
echo "Internal error"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
fi
|
||||
@ -79,27 +88,27 @@ echo "******************************"
|
||||
echo " Runing DorisFe Unittest "
|
||||
echo "******************************"
|
||||
|
||||
cd ${DORIS_HOME}/fe/
|
||||
cd "${DORIS_HOME}/fe"
|
||||
mkdir -p build/compile
|
||||
|
||||
if [ -z "${FE_UT_PARALLEL}" ]; then
|
||||
if [[ -z "${FE_UT_PARALLEL}" ]]; then
|
||||
# the default fe unit test parallel is 1
|
||||
export FE_UT_PARALLEL=1
|
||||
fi
|
||||
echo "Unit test parallel is: $FE_UT_PARALLEL"
|
||||
echo "Unit test parallel is: ${FE_UT_PARALLEL}"
|
||||
|
||||
if [ ${COVERAGE} -eq 1 ]; then
|
||||
if [[ "${COVERAGE}" -eq 1 ]]; then
|
||||
echo "Run coverage statistic"
|
||||
ant cover-test
|
||||
else
|
||||
if [ ${RUN} -eq 1 ]; then
|
||||
if [[ "${RUN}" -eq 1 ]]; then
|
||||
echo "Run the specified class: $1"
|
||||
# eg:
|
||||
# sh run-fe-ut.sh --run org.apache.doris.utframe.Demo
|
||||
# sh run-fe-ut.sh --run org.apache.doris.utframe.Demo#testCreateDbAndTable+test2
|
||||
${MVN_CMD} test -DfailIfNoTests=false -D test=$1
|
||||
"${MVN_CMD}" test -DfailIfNoTests=false -D test="$1"
|
||||
else
|
||||
echo "Run Frontend UT"
|
||||
${MVN_CMD} test -DfailIfNoTests=false
|
||||
"${MVN_CMD}" test -DfailIfNoTests=false
|
||||
fi
|
||||
fi
|
||||
|
||||
@ -19,14 +19,13 @@
|
||||
set -eo pipefail
|
||||
#set -x
|
||||
|
||||
ROOT=`dirname "$0"`
|
||||
ROOT=`cd "$ROOT"; pwd`
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
|
||||
|
||||
DORIS_HOME=${ROOT}
|
||||
DORIS_HOME="${ROOT}"
|
||||
|
||||
# Check args
|
||||
usage() {
|
||||
echo "
|
||||
echo "
|
||||
Usage: $0 <shell_options> <framework_options>
|
||||
Optional shell_options:
|
||||
--clean clean output of regression test framework
|
||||
@ -63,14 +62,14 @@ Usage: $0 <shell_options> <framework_options>
|
||||
Log path: \${DORIS_HOME}/output/regression-test/log
|
||||
Default config file: \${DORIS_HOME}/regression-test/conf/regression-conf.groovy
|
||||
"
|
||||
exit 1
|
||||
exit 1
|
||||
}
|
||||
|
||||
CLEAN=
|
||||
WRONG_CMD=
|
||||
TEAMCITY=
|
||||
RUN=
|
||||
if [ $# == 0 ] ; then
|
||||
CLEAN=0
|
||||
WRONG_CMD=0
|
||||
TEAMCITY=0
|
||||
RUN=0
|
||||
if [[ "$#" == 0 ]]; then
|
||||
#default
|
||||
CLEAN=0
|
||||
WRONG_CMD=0
|
||||
@ -83,95 +82,103 @@ else
|
||||
WRONG_CMD=0
|
||||
while true; do
|
||||
case "$1" in
|
||||
--clean) CLEAN=1 ; shift ;;
|
||||
--teamcity) TEAMCITY=1 ; shift ;;
|
||||
--run) RUN=1 ; shift ;;
|
||||
*)
|
||||
if [ ${RUN} -eq 0 ] && [ ${CLEAN} -eq 0 ]; then
|
||||
WRONG_CMD=1
|
||||
fi
|
||||
break ;;
|
||||
--clean)
|
||||
CLEAN=1
|
||||
shift
|
||||
;;
|
||||
--teamcity)
|
||||
TEAMCITY=1
|
||||
shift
|
||||
;;
|
||||
--run)
|
||||
RUN=1
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
if [[ "${RUN}" -eq 0 ]] && [[ "${CLEAN}" -eq 0 ]]; then
|
||||
WRONG_CMD=1
|
||||
fi
|
||||
break
|
||||
;;
|
||||
esac
|
||||
done
|
||||
fi
|
||||
|
||||
if [ ${WRONG_CMD} -eq 1 ]; then
|
||||
if [[ "${WRONG_CMD}" -eq 1 ]]; then
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# set maven
|
||||
MVN_CMD=mvn
|
||||
if [[ ! -z ${CUSTOM_MVN} ]]; then
|
||||
MVN_CMD=${CUSTOM_MVN}
|
||||
MVN_CMD='mvn'
|
||||
if [[ -n "${CUSTOM_MVN}" ]]; then
|
||||
MVN_CMD="${CUSTOM_MVN}"
|
||||
fi
|
||||
if ! ${MVN_CMD} --version; then
|
||||
if ! "${MVN_CMD}" --version; then
|
||||
echo "Error: mvn is not found"
|
||||
exit 1
|
||||
fi
|
||||
export MVN_CMD
|
||||
|
||||
CONF_DIR=${DORIS_HOME}/regression-test/conf
|
||||
CONFIG_FILE=${CONF_DIR}/regression-conf.groovy
|
||||
LOG_CONFIG_FILE=${CONF_DIR}/logback.xml
|
||||
CONF_DIR="${DORIS_HOME}/regression-test/conf"
|
||||
CONFIG_FILE="${CONF_DIR}/regression-conf.groovy"
|
||||
LOG_CONFIG_FILE="${CONF_DIR}/logback.xml"
|
||||
|
||||
FRAMEWORK_SOURCE_DIR=${DORIS_HOME}/regression-test/framework
|
||||
REGRESSION_TEST_BUILD_DIR=${FRAMEWORK_SOURCE_DIR}/target
|
||||
FRAMEWORK_SOURCE_DIR="${DORIS_HOME}/regression-test/framework"
|
||||
REGRESSION_TEST_BUILD_DIR="${FRAMEWORK_SOURCE_DIR}/target"
|
||||
|
||||
OUTPUT_DIR=${DORIS_HOME}/output/regression-test
|
||||
LOG_OUTPUT_FILE=${OUTPUT_DIR}/log
|
||||
RUN_JAR=${OUTPUT_DIR}/lib/regression-test-*.jar
|
||||
OUTPUT_DIR="${DORIS_HOME}/output/regression-test"
|
||||
LOG_OUTPUT_FILE="${OUTPUT_DIR}/log"
|
||||
RUN_JAR="${OUTPUT_DIR}/lib/regression-test-*.jar"
|
||||
|
||||
if [ ${CLEAN} -eq 1 ]; then
|
||||
rm -rf ${REGRESSION_TEST_BUILD_DIR}
|
||||
rm -rf ${OUTPUT_DIR}
|
||||
if [[ "${CLEAN}" -eq 1 ]]; then
|
||||
rm -rf "${REGRESSION_TEST_BUILD_DIR}"
|
||||
rm -rf "${OUTPUT_DIR}"
|
||||
fi
|
||||
|
||||
if [ ${RUN} -ne 1 ]; then
|
||||
if [[ "${RUN}" -ne 1 ]]; then
|
||||
echo "Finished"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ ! -f ${RUN_JAR} ]; then
|
||||
if [[ ! -f ${RUN_JAR} ]]; then
|
||||
echo "===== Build Regression Test Framework ====="
|
||||
cd ${DORIS_HOME}/regression-test/framework
|
||||
${MVN_CMD} package
|
||||
cd ${DORIS_HOME}
|
||||
cd "${DORIS_HOME}/regression-test/framework"
|
||||
"${MVN_CMD}" package
|
||||
cd "${DORIS_HOME}"
|
||||
|
||||
mkdir -p ${OUTPUT_DIR}/{lib,log}
|
||||
cp -r ${REGRESSION_TEST_BUILD_DIR}/regression-test-*.jar ${OUTPUT_DIR}/lib
|
||||
mkdir -p "${OUTPUT_DIR}"/{lib,log}
|
||||
cp -r "${REGRESSION_TEST_BUILD_DIR}"/regression-test-*.jar "${OUTPUT_DIR}/lib"
|
||||
fi
|
||||
|
||||
# check java home
|
||||
if [[ -z ${JAVA_HOME} ]]; then
|
||||
if [[ -z "${JAVA_HOME}" ]]; then
|
||||
echo "Error: JAVA_HOME is not set"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# check java version
|
||||
export JAVA=${JAVA_HOME}/bin/java
|
||||
export JAVA="${JAVA_HOME}/bin/java"
|
||||
|
||||
|
||||
REGRESSION_OPTIONS_PREFIX=
|
||||
REGRESSION_OPTIONS_PREFIX=''
|
||||
|
||||
# contains framework options and not start with -
|
||||
# it should be suite name
|
||||
if [ $# -ne 0 ] && [[ "$1" =~ ^[^-].* ]]; then
|
||||
if [[ "$#" -ne 0 ]] && [[ "$1" =~ ^[^-].* ]]; then
|
||||
# specify suiteName
|
||||
REGRESSION_OPTIONS_PREFIX='-s'
|
||||
fi
|
||||
|
||||
echo "===== Run Regression Test ====="
|
||||
|
||||
JAVA_OPTS=${JAVA_OPTS}
|
||||
if [ ${TEAMCITY} -eq 1 ]; then
|
||||
JAVA_OPTS="$JAVA_OPTS -DstdoutAppenderType=teamcity -Xmx2048m"
|
||||
if [[ "${TEAMCITY}" -eq 1 ]]; then
|
||||
JAVA_OPTS="${JAVA_OPTS} -DstdoutAppenderType=teamcity -Xmx2048m"
|
||||
fi
|
||||
|
||||
$JAVA -DDORIS_HOME=$DORIS_HOME \
|
||||
-DLOG_PATH=$LOG_OUTPUT_FILE \
|
||||
-Dlogback.configurationFile=${LOG_CONFIG_FILE} \
|
||||
${JAVA_OPTS} \
|
||||
-jar ${RUN_JAR} \
|
||||
-cf ${CONFIG_FILE} \
|
||||
${REGRESSION_OPTIONS_PREFIX} "$@"
|
||||
"${JAVA}" -DDORIS_HOME="${DORIS_HOME}" \
|
||||
-DLOG_PATH="${LOG_OUTPUT_FILE}" \
|
||||
-Dlogback.configurationFile="${LOG_CONFIG_FILE}" \
|
||||
${JAVA_OPTS:+${JAVA_OPTS}} \
|
||||
-jar ${RUN_JAR:+${RUN_JAR}} \
|
||||
-cf "${CONFIG_FILE}" \
|
||||
${REGRESSION_OPTIONS_PREFIX:+${REGRESSION_OPTIONS_PREFIX}} "$@"
|
||||
|
||||
@ -15,35 +15,34 @@
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
function check_insert_load_doris_func() {
|
||||
local doris="$1"
|
||||
local sql="$2"
|
||||
insert_return_info="$(${doris} -e "${sql}" -vv)"
|
||||
|
||||
function check_insert_load_doris_func(){
|
||||
local doris=$1
|
||||
local sql=$2
|
||||
insert_return_info=`${doris} -e "${sql}" -vv`
|
||||
|
||||
label=`echo ${insert_return_info}|perl -e '$_ = <>;chomp;/(\w{8}\-\w{4}\-\w{4}\-\w{4}\-\w{12})/;print $1'`
|
||||
label="$(echo "${insert_return_info}" | perl -e '$_ = <>;chomp;/(\w{8}\-\w{4}\-\w{4}\-\w{4}\-\w{12})/;print $1')"
|
||||
echo "${label}"
|
||||
wait_seconds=3600
|
||||
while [[ "${wait_seconds}" > 0 && "${label}" == '' ]];do
|
||||
label=`echo "${insert_return_info}"|perl -e '$_ = <>;chomp;/(\w{8}\-\w{4}\-\w{4}\-\w{4}\-\w{12})/;print $1'`
|
||||
while [[ "${wait_seconds}" -gt 0 && "${label}" == '' ]]; do
|
||||
label="$(echo "${insert_return_info}" | perl -e '$_ = <>;chomp;/(\w{8}\-\w{4}\-\w{4}\-\w{4}\-\w{12})/;print $1')"
|
||||
sleep 5s
|
||||
((wait_seconds=${wait_seconds}-5))
|
||||
wait_seconds="$((wait_seconds - 5))"
|
||||
done
|
||||
|
||||
if [[ "$label" == '' ]];then
|
||||
if [[ "${label}" == '' ]]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
wait_seconds=3600
|
||||
while [[ "${wait_seconds}" > 0 ]];do
|
||||
while [[ "${wait_seconds}" -gt 0 ]]; do
|
||||
echo "${wait_seconds}"
|
||||
echo "${doris} -e show load where label = '${label}' order by createtime desc limit 1"
|
||||
result=`${doris} -e "show load where label = '${label}' order by createtime desc limit 1" -N`
|
||||
result=$(${doris} -e "show load where label = '${label}' order by createtime desc limit 1" -N)
|
||||
|
||||
load_status=`echo "${result}"|perl -e '$_ = <>;chomp;/(\d+)\s(\w{8}\-\w{4}\-\w{4}\-\w{4}\-\w{12})\s(\w+)\s/;print $3'`
|
||||
load_status=$(echo "${result}" | perl -e '$_ = <>;chomp;/(\d+)\s(\w{8}\-\w{4}\-\w{4}\-\w{4}\-\w{12})\s(\w+)\s/;print $3')
|
||||
if [[ "${load_status}" == 'FINISHED' || "${load_status}" == 'CANCELLED' ]]; then
|
||||
echo "insert status: ${load_status}"
|
||||
if [[ "${load_status}" == 'FINISHED' ]];then
|
||||
if [[ "${load_status}" == 'FINISHED' ]]; then
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
@ -52,7 +51,7 @@ function check_insert_load_doris_func(){
|
||||
else
|
||||
echo "insert status: ${load_status}"
|
||||
sleep 5s
|
||||
((wait_seconds=$wait_seconds-5))
|
||||
wait_seconds="$((wait_seconds - 5))"
|
||||
fi
|
||||
done
|
||||
|
||||
@ -66,6 +65,6 @@ doris_port=8080
|
||||
doris_user="db_user"
|
||||
doris_password="db_password"
|
||||
doris_database="db_name"
|
||||
doris="mysql -h$doris_host -P$doris_port -u$doris_user -p$doris_password -D$doris_database"
|
||||
doris="mysql -h${doris_host} -P${doris_port} -u${doris_user} -p${doris_password} -D${doris_database}"
|
||||
sql="INSERT INTO TABLE_Y[(column1, column2,...,columnN)] SELECT column1, column2,..., columnN [FROM TABLE_X WHERE xxx]"
|
||||
check_insert_load_doris_func ``"$doris"` `"$sql"
|
||||
check_insert_load_doris_func "${doris}" "${sql}"
|
||||
|
||||
125
thirdparty/build-thirdparty.sh
vendored
125
thirdparty/build-thirdparty.sh
vendored
@ -1,4 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
# shellcheck disable=2034
|
||||
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
@ -26,16 +28,13 @@
|
||||
# This script will run *download-thirdparty.sh* once again
|
||||
# to check if all thirdparties have been downloaded, unpacked and patched.
|
||||
#################################################################################
|
||||
set -e
|
||||
|
||||
curdir=$(dirname "$0")
|
||||
curdir=$(
|
||||
cd "$curdir"
|
||||
pwd
|
||||
)
|
||||
set -eo pipefail
|
||||
|
||||
export DORIS_HOME=$curdir/..
|
||||
export TP_DIR=$curdir
|
||||
curdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
|
||||
|
||||
export DORIS_HOME="${curdir}/.."
|
||||
export TP_DIR="${curdir}"
|
||||
|
||||
# Check args
|
||||
usage() {
|
||||
@ -47,31 +46,31 @@ Usage: $0 <options>
|
||||
exit 1
|
||||
}
|
||||
|
||||
if ! OPTS=$(getopt \
|
||||
if ! OPTS="$(getopt \
|
||||
-n "$0" \
|
||||
-o '' \
|
||||
-o 'h' \
|
||||
-l 'help' \
|
||||
-o 'j:' \
|
||||
-- "$@"); then
|
||||
-- "$@")"; then
|
||||
usage
|
||||
fi
|
||||
|
||||
eval set -- "$OPTS"
|
||||
eval set -- "${OPTS}"
|
||||
|
||||
KERNEL="$(uname -s)"
|
||||
|
||||
if [[ "${KERNEL}" == 'Darwin' ]]; then
|
||||
PARALLEL=$(($(sysctl -n hw.logicalcpu) / 4 + 1))
|
||||
PARALLEL="$(($(sysctl -n hw.logicalcpu) / 4 + 1))"
|
||||
else
|
||||
PARALLEL=$(($(nproc) / 4 + 1))
|
||||
PARALLEL="$(($(nproc) / 4 + 1))"
|
||||
fi
|
||||
|
||||
if [[ $# -ne 1 ]]; then
|
||||
if [[ "$#" -ne 1 ]]; then
|
||||
while true; do
|
||||
case "$1" in
|
||||
-j)
|
||||
PARALLEL=$2
|
||||
PARALLEL="$2"
|
||||
shift 2
|
||||
;;
|
||||
-h)
|
||||
@ -94,28 +93,28 @@ if [[ $# -ne 1 ]]; then
|
||||
done
|
||||
fi
|
||||
|
||||
if [[ ${HELP} -eq 1 ]]; then
|
||||
if [[ "${HELP}" -eq 1 ]]; then
|
||||
usage
|
||||
exit
|
||||
fi
|
||||
|
||||
echo "Get params:
|
||||
PARALLEL -- $PARALLEL
|
||||
PARALLEL -- ${PARALLEL}
|
||||
"
|
||||
|
||||
# include custom environment variables
|
||||
if [[ -f ${DORIS_HOME}/env.sh ]]; then
|
||||
if [[ -f "${DORIS_HOME}/env.sh" ]]; then
|
||||
export BUILD_THIRDPARTY_WIP=1
|
||||
. "${DORIS_HOME}/env.sh"
|
||||
export BUILD_THIRDPARTY_WIP=
|
||||
fi
|
||||
|
||||
if [[ ! -f ${TP_DIR}/download-thirdparty.sh ]]; then
|
||||
if [[ ! -f "${TP_DIR}/download-thirdparty.sh" ]]; then
|
||||
echo "Download thirdparty script is missing".
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f ${TP_DIR}/vars.sh ]]; then
|
||||
if [[ ! -f "${TP_DIR}/vars.sh" ]]; then
|
||||
echo "vars.sh is missing".
|
||||
exit 1
|
||||
fi
|
||||
@ -127,26 +126,26 @@ cd "${TP_DIR}"
|
||||
# Download thirdparties.
|
||||
"${TP_DIR}/download-thirdparty.sh"
|
||||
|
||||
export LD_LIBRARY_PATH=$TP_DIR/installed/lib:$LD_LIBRARY_PATH
|
||||
export LD_LIBRARY_PATH="${TP_DIR}/installed/lib:${LD_LIBRARY_PATH}"
|
||||
|
||||
# toolchain specific warning options and settings
|
||||
if [[ "$CC" == *gcc ]]; then
|
||||
warning_uninitialized=-Wno-maybe-uninitialized
|
||||
warning_stringop_truncation=-Wno-stringop-truncation
|
||||
warning_class_memaccess=-Wno-class-memaccess
|
||||
warning_array_parameter=-Wno-array-parameter
|
||||
boost_toolset=gcc
|
||||
elif [[ "$CC" == *clang ]]; then
|
||||
warning_uninitialized=-Wno-uninitialized
|
||||
warning_shadow=-Wno-shadow
|
||||
warning_dangling_gsl=-Wno-dangling-gsl
|
||||
warning_unused_but_set_variable=-Wno-unused-but-set-variable
|
||||
warning_defaulted_function_deleted=-Wno-defaulted-function-deleted
|
||||
warning_reserved_identifier=-Wno-reserved-identifier
|
||||
warning_suggest_override="-Wno-suggest-override -Wno-suggest-destructor-override"
|
||||
warning_option_ignored=-Wno-option-ignored
|
||||
boost_toolset=clang
|
||||
libhdfs_cxx17=-std=c++1z
|
||||
if [[ "${CC}" == *gcc ]]; then
|
||||
warning_uninitialized='-Wno-maybe-uninitialized'
|
||||
warning_stringop_truncation='-Wno-stringop-truncation'
|
||||
warning_class_memaccess='-Wno-class-memaccess'
|
||||
warning_array_parameter='-Wno-array-parameter'
|
||||
boost_toolset='gcc'
|
||||
elif [[ "${CC}" == *clang ]]; then
|
||||
warning_uninitialized='-Wno-uninitialized'
|
||||
warning_shadow='-Wno-shadow'
|
||||
warning_dangling_gsl='-Wno-dangling-gsl'
|
||||
warning_unused_but_set_variable='-Wno-unused-but-set-variable'
|
||||
warning_defaulted_function_deleted='-Wno-defaulted-function-deleted'
|
||||
warning_reserved_identifier='-Wno-reserved-identifier'
|
||||
warning_suggest_override='-Wno-suggest-override -Wno-suggest-destructor-override'
|
||||
warning_option_ignored='-Wno-option-ignored'
|
||||
boost_toolset='clang'
|
||||
libhdfs_cxx17='-std=c++1z'
|
||||
fi
|
||||
|
||||
# prepare installed prefix
|
||||
@ -156,13 +155,13 @@ ln -sf lib64 lib
|
||||
popd
|
||||
|
||||
check_prerequest() {
|
||||
local CMD=$1
|
||||
local NAME=$2
|
||||
if ! $CMD; then
|
||||
echo "$NAME is missing"
|
||||
local CMD="$1"
|
||||
local NAME="$2"
|
||||
if ! ${CMD}; then
|
||||
echo "${NAME} is missing"
|
||||
exit 1
|
||||
else
|
||||
echo "$NAME is found"
|
||||
echo "${NAME} is found"
|
||||
fi
|
||||
}
|
||||
|
||||
@ -222,8 +221,8 @@ check_if_source_exist() {
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -d $TP_SOURCE_DIR/$1 ]]; then
|
||||
echo "$TP_SOURCE_DIR/$1 does not exist."
|
||||
if [[ ! -d "${TP_SOURCE_DIR}/$1" ]]; then
|
||||
echo "${TP_SOURCE_DIR}/$1 does not exist."
|
||||
exit 1
|
||||
fi
|
||||
echo "===== begin build $1"
|
||||
@ -235,8 +234,8 @@ check_if_archieve_exist() {
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f $TP_SOURCE_DIR/$1 ]]; then
|
||||
echo "$TP_SOURCE_DIR/$1 does not exist."
|
||||
if [[ ! -f "${TP_SOURCE_DIR}/$1" ]]; then
|
||||
echo "${TP_SOURCE_DIR}/$1 does not exist."
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
@ -374,7 +373,7 @@ build_protobuf() {
|
||||
# This will casue protoc cannot run
|
||||
# If you really need to dynamically link protoc, please set the environment variable DYN_LINK_PROTOC=1
|
||||
|
||||
if [[ "${DYN_LINK_PROTOC}" == "1" || "${KERNEL}" == 'Darwin' ]]; then
|
||||
if [[ "${DYN_LINK_PROTOC:-0}" == "1" || "${KERNEL}" == 'Darwin' ]]; then
|
||||
echo "link protoc dynamiclly"
|
||||
else
|
||||
cd src
|
||||
@ -940,22 +939,22 @@ build_bitshuffle() {
|
||||
# we still need to support non-AVX2-capable hardware. So, we build it twice,
|
||||
# once with the flag and once without, and use some linker tricks to
|
||||
# suffix the AVX2 symbols with '_avx2'.
|
||||
arches="default avx2"
|
||||
arches=('default' 'avx2')
|
||||
MACHINE_TYPE="$(uname -m)"
|
||||
# Becuase aarch64 don't support avx2, disable it.
|
||||
if [[ "${MACHINE_TYPE}" == "aarch64" || "${MACHINE_TYPE}" == 'arm64' ]]; then
|
||||
arches="default"
|
||||
arches=('default')
|
||||
fi
|
||||
|
||||
to_link=""
|
||||
for arch in $arches; do
|
||||
for arch in "${arches[@]}"; do
|
||||
arch_flag=""
|
||||
if [[ "${arch}" == "avx2" ]]; then
|
||||
arch_flag="-mavx2"
|
||||
fi
|
||||
tmp_obj="bitshuffle_${arch}_tmp.o"
|
||||
dst_obj="bitshuffle_${arch}.o"
|
||||
"${CC}" ${EXTRA_CFLAGS} ${arch_flag} -std=c99 "-I${PREFIX}/include/lz4" -O3 -DNDEBUG -c \
|
||||
"${CC}" ${EXTRA_CFLAGS:+${EXTRA_CFLAGS}} ${arch_flag:+${arch_flag}} -std=c99 "-I${PREFIX}/include/lz4" -O3 -DNDEBUG -c \
|
||||
"src/bitshuffle_core.c" \
|
||||
"src/bitshuffle.c" \
|
||||
"src/iochain.c"
|
||||
@ -964,7 +963,7 @@ build_bitshuffle() {
|
||||
# For the AVX2 symbols, suffix them.
|
||||
if [[ "${arch}" == "avx2" ]]; then
|
||||
# Create a mapping file with '<old_sym> <suffixed_sym>' on each line.
|
||||
"${DORIS_BIN_UTILS}/nm" --defined-only --extern-only "${tmp_obj}" | while read addr type sym; do
|
||||
"${DORIS_BIN_UTILS}/nm" --defined-only --extern-only "${tmp_obj}" | while read -r addr type sym; do
|
||||
echo "${sym} ${sym}_${arch}"
|
||||
done >renames.txt
|
||||
"${DORIS_BIN_UTILS}/objcopy" --redefine-syms=renames.txt "${tmp_obj}" "${dst_obj}"
|
||||
@ -973,17 +972,19 @@ build_bitshuffle() {
|
||||
fi
|
||||
to_link="${to_link} ${dst_obj}"
|
||||
done
|
||||
local links
|
||||
read -r -a links <<<"${to_link}"
|
||||
rm -f libbitshuffle.a
|
||||
"${DORIS_BIN_UTILS}/ar" rs libbitshuffle.a ${to_link}
|
||||
"${DORIS_BIN_UTILS}/ar" rs libbitshuffle.a "${links[@]}"
|
||||
mkdir -p "${PREFIX}/include/bitshuffle"
|
||||
cp libbitshuffle.a "${PREFIX}"/lib/
|
||||
cp "${TP_SOURCE_DIR}/$BITSHUFFLE_SOURCE/src/bitshuffle.h" "${PREFIX}/include/bitshuffle/bitshuffle.h"
|
||||
cp "${TP_SOURCE_DIR}/$BITSHUFFLE_SOURCE/src/bitshuffle_core.h" "${PREFIX}/include/bitshuffle/bitshuffle_core.h"
|
||||
cp "${TP_SOURCE_DIR}/${BITSHUFFLE_SOURCE}/src/bitshuffle.h" "${PREFIX}/include/bitshuffle/bitshuffle.h"
|
||||
cp "${TP_SOURCE_DIR}/${BITSHUFFLE_SOURCE}/src/bitshuffle_core.h" "${PREFIX}/include/bitshuffle/bitshuffle_core.h"
|
||||
}
|
||||
|
||||
# croaring bitmap
|
||||
build_croaringbitmap() {
|
||||
avx_flag=
|
||||
avx_flag=''
|
||||
if [[ -n "${USE_AVX2}" && "${USE_AVX2}" -eq 0 ]]; then
|
||||
echo "set USE_AVX2=${USE_AVX2} to FORCE disable AVX2 in croaringbitmap"
|
||||
avx_flag="-DROARING_DISABLE_AVX=ON"
|
||||
@ -1005,7 +1006,7 @@ build_croaringbitmap() {
|
||||
|
||||
CXXFLAGS="-O3" \
|
||||
LDFLAGS="${ldflags}" \
|
||||
"${CMAKE_CMD}" -G "${GENERATOR}" ${avx_flag} -DROARING_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \
|
||||
"${CMAKE_CMD}" -G "${GENERATOR}" ${avx_flag:+${avx_flag}} -DROARING_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \
|
||||
-DENABLE_ROARING_TESTS=OFF ..
|
||||
|
||||
"${BUILD_SYSTEM}" -j "${PARALLEL}"
|
||||
@ -1230,7 +1231,7 @@ build_krb5() {
|
||||
fi
|
||||
|
||||
CFLAGS="-fcommon -I${TP_INSTALL_DIR}/include" LDFLAGS="-L${TP_INSTALL_DIR}/lib" \
|
||||
../configure --prefix="${TP_INSTALL_DIR}" --disable-shared --enable-static ${with_crypto_impl}
|
||||
../configure --prefix="${TP_INSTALL_DIR}" --disable-shared --enable-static ${with_crypto_impl:+${with_crypto_impl}}
|
||||
|
||||
make -j "${PARALLEL}"
|
||||
make install
|
||||
@ -1262,7 +1263,7 @@ build_benchmark() {
|
||||
|
||||
cmake -E make_directory "build"
|
||||
|
||||
if [[ "$KERNEL" != 'Darwin' ]]; then
|
||||
if [[ "${KERNEL}" != 'Darwin' ]]; then
|
||||
cxxflags='-lresolv -pthread -lrt'
|
||||
else
|
||||
cxxflags='-lresolv -pthread'
|
||||
@ -1288,8 +1289,8 @@ build_simdjson() {
|
||||
|
||||
CXXFLAGS="-O3" \
|
||||
CFLAGS="-O3" \
|
||||
$CMAKE_CMD ..
|
||||
$CMAKE_CMD --build .
|
||||
"${CMAKE_CMD}" ..
|
||||
"${CMAKE_CMD}" --build .
|
||||
|
||||
cp "${TP_SOURCE_DIR}/${SIMDJSON_SOURCE}/${BUILD_DIR}/libsimdjson.a" "${TP_INSTALL_DIR}/lib64"
|
||||
cp -r "${TP_SOURCE_DIR}/${SIMDJSON_SOURCE}/include"/* "${TP_INCLUDE_DIR}/"
|
||||
|
||||
326
thirdparty/download-thirdparty.sh
vendored
326
thirdparty/download-thirdparty.sh
vendored
@ -16,138 +16,135 @@
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
set -e
|
||||
################################################################
|
||||
# This script will download all thirdparties and java libraries
|
||||
# which are defined in *vars.sh*, unpack patch them if necessary.
|
||||
# You can run this script multi-times.
|
||||
# Things will only be downloaded, unpacked and patched once.
|
||||
################################################################
|
||||
curdir=`dirname "$0"`
|
||||
curdir=`cd "$curdir"; pwd`
|
||||
|
||||
set -eo pipefail
|
||||
|
||||
curdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
|
||||
|
||||
if [[ -z "${DORIS_HOME}" ]]; then
|
||||
DORIS_HOME=$curdir/..
|
||||
DORIS_HOME="${curdir}/.."
|
||||
fi
|
||||
|
||||
# include custom environment variables
|
||||
if [[ -f ${DORIS_HOME}/custom_env.sh ]]; then
|
||||
. ${DORIS_HOME}/custom_env.sh
|
||||
if [[ -f "${DORIS_HOME}/custom_env.sh" ]]; then
|
||||
# shellcheck disable=1091
|
||||
. "${DORIS_HOME}/custom_env.sh"
|
||||
fi
|
||||
|
||||
if [[ -z "${TP_DIR}" ]]; then
|
||||
TP_DIR=$curdir
|
||||
TP_DIR="${curdir}"
|
||||
fi
|
||||
|
||||
if [ ! -f ${TP_DIR}/vars.sh ]; then
|
||||
echo "vars.sh is missing".
|
||||
if [[ ! -f "${TP_DIR}/vars.sh" ]]; then
|
||||
echo 'vars.sh is missing'.
|
||||
exit 1
|
||||
fi
|
||||
. ${TP_DIR}/vars.sh
|
||||
|
||||
mkdir -p ${TP_DIR}/src
|
||||
. "${TP_DIR}/vars.sh"
|
||||
|
||||
md5sum_bin=md5sum
|
||||
if ! command -v ${md5sum_bin} >/dev/null 2>&1; then
|
||||
mkdir -p "${TP_DIR}/src"
|
||||
|
||||
md5sum_bin='md5sum'
|
||||
if ! command -v "${md5sum_bin}" >/dev/null 2>&1; then
|
||||
echo "Warn: md5sum is not installed"
|
||||
md5sum_bin=""
|
||||
fi
|
||||
|
||||
md5sum_func() {
|
||||
local FILENAME=$1
|
||||
local DESC_DIR=$2
|
||||
local MD5SUM=$3
|
||||
local FILENAME="$1"
|
||||
local DESC_DIR="$2"
|
||||
local MD5SUM="$3"
|
||||
local md5
|
||||
|
||||
if [ "$md5sum_bin" == "" ]; then
|
||||
return 0
|
||||
if [[ "${md5sum_bin}" == "" ]]; then
|
||||
return 0
|
||||
else
|
||||
md5=`md5sum "$DESC_DIR/$FILENAME"`
|
||||
if [ "$md5" != "$MD5SUM $DESC_DIR/$FILENAME" ]; then
|
||||
echo "$DESC_DIR/$FILENAME md5sum check failed!"
|
||||
echo -e "except-md5 $MD5SUM \nactual-md5 $md5"
|
||||
return 1
|
||||
fi
|
||||
md5="$(md5sum "${DESC_DIR}/${FILENAME}")"
|
||||
if [[ "${md5}" != "${MD5SUM} ${DESC_DIR}/${FILENAME}" ]]; then
|
||||
echo "${DESC_DIR}/${FILENAME} md5sum check failed!"
|
||||
echo -e "except-md5 ${MD5SUM} \nactual-md5 ${md5}"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# return 0 if download succeed.
|
||||
# return 1 if not.
|
||||
download_func() {
|
||||
local FILENAME=$1
|
||||
local DOWNLOAD_URL=$2
|
||||
local DESC_DIR=$3
|
||||
local MD5SUM=$4
|
||||
local FILENAME="$1"
|
||||
local DOWNLOAD_URL="$2"
|
||||
local DESC_DIR="$3"
|
||||
local MD5SUM="$4"
|
||||
|
||||
if [ -z "$FILENAME" ]; then
|
||||
if [[ -z "${FILENAME}" ]]; then
|
||||
echo "Error: No file name specified to download"
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$DOWNLOAD_URL" ]; then
|
||||
echo "Error: No download url specified for $FILENAME"
|
||||
if [[ -z "${DOWNLOAD_URL}" ]]; then
|
||||
echo "Error: No download url specified for ${FILENAME}"
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$DESC_DIR" ]; then
|
||||
echo "Error: No dest dir specified for $FILENAME"
|
||||
if [[ -z "${DESC_DIR}" ]]; then
|
||||
echo "Error: No dest dir specified for ${FILENAME}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
local STATUS=1
|
||||
for attemp in 1 2; do
|
||||
if [ -r "$DESC_DIR/$FILENAME" ]; then
|
||||
if md5sum_func $FILENAME $DESC_DIR $MD5SUM; then
|
||||
echo "Archive $FILENAME already exist."
|
||||
if [[ -r "${DESC_DIR}/${FILENAME}" ]]; then
|
||||
if md5sum_func "${FILENAME}" "${DESC_DIR}" "${MD5SUM}"; then
|
||||
echo "Archive ${FILENAME} already exist."
|
||||
STATUS=0
|
||||
break;
|
||||
break
|
||||
fi
|
||||
echo "Archive $FILENAME will be removed and download again."
|
||||
rm -f "$DESC_DIR/$FILENAME"
|
||||
echo "Archive ${FILENAME} will be removed and download again."
|
||||
rm -f "${DESC_DIR}/${FILENAME}"
|
||||
else
|
||||
echo "Downloading $FILENAME from $DOWNLOAD_URL to $DESC_DIR"
|
||||
wget --no-check-certificate -q $DOWNLOAD_URL -O $DESC_DIR/$FILENAME
|
||||
if [ "$?"x == "0"x ]; then
|
||||
if md5sum_func $FILENAME $DESC_DIR $MD5SUM; then
|
||||
echo "Downloading ${FILENAME} from ${DOWNLOAD_URL} to ${DESC_DIR}"
|
||||
if wget --no-check-certificate -q "${DOWNLOAD_URL}" -O "${DESC_DIR}/${FILENAME}"; then
|
||||
if md5sum_func "${FILENAME}" "${DESC_DIR}" "${MD5SUM}"; then
|
||||
STATUS=0
|
||||
echo "Success to download $FILENAME"
|
||||
break;
|
||||
echo "Success to download ${FILENAME}"
|
||||
break
|
||||
fi
|
||||
echo "Archive $FILENAME will be removed and download again."
|
||||
rm -f "$DESC_DIR/$FILENAME"
|
||||
echo "Archive ${FILENAME} will be removed and download again."
|
||||
rm -f "${DESC_DIR}/${FILENAME}"
|
||||
else
|
||||
echo "Failed to download $FILENAME. attemp: $attemp"
|
||||
echo "Failed to download ${FILENAME}. attemp: ${attemp}"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
if [ $STATUS -ne 0 ]; then
|
||||
echo "Failed to download $FILENAME"
|
||||
if [[ "${STATUS}" -ne 0 ]]; then
|
||||
echo "Failed to download ${FILENAME}"
|
||||
fi
|
||||
return $STATUS
|
||||
return "${STATUS}"
|
||||
}
|
||||
|
||||
# download thirdparty archives
|
||||
echo "===== Downloading thirdparty archives..."
|
||||
for TP_ARCH in ${TP_ARCHIVES[*]}
|
||||
do
|
||||
NAME=$TP_ARCH"_NAME"
|
||||
MD5SUM=$TP_ARCH"_MD5SUM"
|
||||
if test "x$REPOSITORY_URL" = x; then
|
||||
URL=$TP_ARCH"_DOWNLOAD"
|
||||
download_func ${!NAME} ${!URL} $TP_SOURCE_DIR ${!MD5SUM}
|
||||
if [ "$?"x != "0"x ]; then
|
||||
for TP_ARCH in "${TP_ARCHIVES[@]}"; do
|
||||
NAME="${TP_ARCH}_NAME"
|
||||
MD5SUM="${TP_ARCH}_MD5SUM"
|
||||
if [[ -z "${REPOSITORY_URL}" ]]; then
|
||||
URL="${TP_ARCH}_DOWNLOAD"
|
||||
if ! download_func "${!NAME}" "${!URL}" "${TP_SOURCE_DIR}" "${!MD5SUM}"; then
|
||||
echo "Failed to download ${!NAME}"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
URL="${REPOSITORY_URL}/${!NAME}"
|
||||
download_func ${!NAME} ${URL} $TP_SOURCE_DIR ${!MD5SUM}
|
||||
if [ "$?x" != "0x" ]; then
|
||||
if ! download_func "${!NAME}" "${URL}" "${TP_SOURCE_DIR}" "${!MD5SUM}"; then
|
||||
#try to download from home
|
||||
URL=$TP_ARCH"_DOWNLOAD"
|
||||
download_func ${!NAME} ${!URL} $TP_SOURCE_DIR ${!MD5SUM}
|
||||
if [ "$?"x != "0"x ]; then
|
||||
URL="${TP_ARCH}_DOWNLOAD"
|
||||
if ! download_func "${!NAME}" "${!URL}" "${TP_SOURCE_DIR}" "${!MD5SUM}"; then
|
||||
echo "Failed to download ${!NAME}"
|
||||
exit 1 # download failed again exit.
|
||||
fi
|
||||
@ -158,10 +155,9 @@ echo "===== Downloading thirdparty archives...done"
|
||||
|
||||
# check if all tp archives exists
|
||||
echo "===== Checking all thirdpart archives..."
|
||||
for TP_ARCH in ${TP_ARCHIVES[*]}
|
||||
do
|
||||
NAME=$TP_ARCH"_NAME"
|
||||
if [ ! -r $TP_SOURCE_DIR/${!NAME} ]; then
|
||||
for TP_ARCH in "${TP_ARCHIVES[@]}"; do
|
||||
NAME="${TP_ARCH}_NAME"
|
||||
if [[ ! -r "${TP_SOURCE_DIR}/${!NAME}" ]]; then
|
||||
echo "Failed to fetch ${!NAME}"
|
||||
exit 1
|
||||
fi
|
||||
@ -176,37 +172,36 @@ SUFFIX_TGZ="\.(tar\.gz|tgz)$"
|
||||
SUFFIX_XZ="\.tar\.xz$"
|
||||
SUFFIX_ZIP="\.zip$"
|
||||
SUFFIX_BZ2="\.tar\.bz2$"
|
||||
for TP_ARCH in ${TP_ARCHIVES[*]}
|
||||
do
|
||||
NAME=$TP_ARCH"_NAME"
|
||||
SOURCE=$TP_ARCH"_SOURCE"
|
||||
for TP_ARCH in "${TP_ARCHIVES[@]}"; do
|
||||
NAME="${TP_ARCH}_NAME"
|
||||
SOURCE="${TP_ARCH}_SOURCE"
|
||||
|
||||
if [ -z "${!SOURCE}" ]; then
|
||||
if [[ -z "${!SOURCE}" ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
if [ ! -d $TP_SOURCE_DIR/${!SOURCE} ]; then
|
||||
if [[ "${!NAME}" =~ $SUFFIX_TGZ ]]; then
|
||||
echo "$TP_SOURCE_DIR/${!NAME}"
|
||||
echo "$TP_SOURCE_DIR/${!SOURCE}"
|
||||
if ! $TAR_CMD xzf "$TP_SOURCE_DIR/${!NAME}" -C "$TP_SOURCE_DIR/"; then
|
||||
if [[ ! -d "${TP_SOURCE_DIR}/${!SOURCE}" ]]; then
|
||||
if [[ "${!NAME}" =~ ${SUFFIX_TGZ} ]]; then
|
||||
echo "${TP_SOURCE_DIR}/${!NAME}"
|
||||
echo "${TP_SOURCE_DIR}/${!SOURCE}"
|
||||
if ! "${TAR_CMD}" xzf "${TP_SOURCE_DIR}/${!NAME}" -C "${TP_SOURCE_DIR}/"; then
|
||||
echo "Failed to untar ${!NAME}"
|
||||
exit 1
|
||||
fi
|
||||
elif [[ "${!NAME}" =~ $SUFFIX_XZ ]]; then
|
||||
echo "$TP_SOURCE_DIR/${!NAME}"
|
||||
echo "$TP_SOURCE_DIR/${!SOURCE}"
|
||||
if ! $TAR_CMD xJf "$TP_SOURCE_DIR/${!NAME}" -C "$TP_SOURCE_DIR/"; then
|
||||
elif [[ "${!NAME}" =~ ${SUFFIX_XZ} ]]; then
|
||||
echo "${TP_SOURCE_DIR}/${!NAME}"
|
||||
echo "${TP_SOURCE_DIR}/${!SOURCE}"
|
||||
if ! "${TAR_CMD}" xJf "${TP_SOURCE_DIR}/${!NAME}" -C "${TP_SOURCE_DIR}/"; then
|
||||
echo "Failed to untar ${!NAME}"
|
||||
exit 1
|
||||
fi
|
||||
elif [[ "${!NAME}" =~ $SUFFIX_ZIP ]]; then
|
||||
if ! $UNZIP_CMD -o -qq "$TP_SOURCE_DIR/${!NAME}" -d "$TP_SOURCE_DIR/"; then
|
||||
elif [[ "${!NAME}" =~ ${SUFFIX_ZIP} ]]; then
|
||||
if ! "${UNZIP_CMD}" -o -qq "${TP_SOURCE_DIR}/${!NAME}" -d "${TP_SOURCE_DIR}/"; then
|
||||
echo "Failed to unzip ${!NAME}"
|
||||
exit 1
|
||||
fi
|
||||
elif [[ "${!NAME}" =~ $SUFFIX_BZ2 ]]; then
|
||||
if ! $TAR_CMD xf "$TP_SOURCE_DIR/${!NAME}" -C "$TP_SOURCE_DIR/"; then
|
||||
elif [[ "${!NAME}" =~ ${SUFFIX_BZ2} ]]; then
|
||||
if ! "${TAR_CMD}" xf "${TP_SOURCE_DIR}/${!NAME}" -C "${TP_SOURCE_DIR}/"; then
|
||||
echo "Failed to untar ${!NAME}"
|
||||
exit 1
|
||||
fi
|
||||
@ -227,134 +222,132 @@ echo "===== Patching thirdparty archives..."
|
||||
PATCHED_MARK="patched_mark"
|
||||
|
||||
# glog patch
|
||||
cd $TP_SOURCE_DIR/$GLOG_SOURCE
|
||||
if [ ! -f $PATCHED_MARK ]; then
|
||||
patch -p1 < $TP_PATCH_DIR/glog-0.4.0.patch
|
||||
touch $PATCHED_MARK
|
||||
cd "${TP_SOURCE_DIR}/${GLOG_SOURCE}"
|
||||
if [[ ! -f "${PATCHED_MARK}" ]]; then
|
||||
patch -p1 <"${TP_PATCH_DIR}/glog-0.4.0.patch"
|
||||
touch "${PATCHED_MARK}"
|
||||
fi
|
||||
cd -
|
||||
echo "Finished patching $GLOG_SOURCE"
|
||||
echo "Finished patching ${GLOG_SOURCE}"
|
||||
|
||||
# gtest patch
|
||||
cd $TP_SOURCE_DIR/$GTEST_SOURCE
|
||||
if [ ! -f $PATCHED_MARK ]; then
|
||||
patch -p1 < $TP_PATCH_DIR/googletest-release-1.11.0.patch
|
||||
touch $PATCHED_MARK
|
||||
cd "${TP_SOURCE_DIR}/${GTEST_SOURCE}"
|
||||
if [[ ! -f "${PATCHED_MARK}" ]]; then
|
||||
patch -p1 <"${TP_PATCH_DIR}/googletest-release-1.11.0.patch"
|
||||
touch "${PATCHED_MARK}"
|
||||
fi
|
||||
cd -
|
||||
echo "Finished patching $GTEST_SOURCE"
|
||||
echo "Finished patching ${GTEST_SOURCE}"
|
||||
|
||||
# mysql patch
|
||||
cd $TP_SOURCE_DIR/$MYSQL_SOURCE
|
||||
if [ ! -f $PATCHED_MARK ]; then
|
||||
patch -p1 < $TP_PATCH_DIR/mysql-server-mysql-5.7.18.patch
|
||||
touch $PATCHED_MARK
|
||||
cd "${TP_SOURCE_DIR}/${MYSQL_SOURCE}"
|
||||
if [[ ! -f "${PATCHED_MARK}" ]]; then
|
||||
patch -p1 <"${TP_PATCH_DIR}/mysql-server-mysql-5.7.18.patch"
|
||||
touch "${PATCHED_MARK}"
|
||||
fi
|
||||
cd -
|
||||
echo "Finished patching $MYSQL_SOURCE"
|
||||
echo "Finished patching ${MYSQL_SOURCE}"
|
||||
|
||||
# libevent patch
|
||||
cd $TP_SOURCE_DIR/$LIBEVENT_SOURCE
|
||||
if [ ! -f $PATCHED_MARK ]; then
|
||||
patch -p1 < $TP_PATCH_DIR/libevent.patch
|
||||
touch $PATCHED_MARK
|
||||
cd "${TP_SOURCE_DIR}/${LIBEVENT_SOURCE}"
|
||||
if [[ ! -f "${PATCHED_MARK}" ]]; then
|
||||
patch -p1 <"${TP_PATCH_DIR}/libevent.patch"
|
||||
touch "${PATCHED_MARK}"
|
||||
fi
|
||||
cd -
|
||||
echo "Finished patching $LIBEVENT_SOURCE"
|
||||
echo "Finished patching ${LIBEVENT_SOURCE}"
|
||||
|
||||
# s2 patch to disable shared library
|
||||
cd $TP_SOURCE_DIR/$S2_SOURCE
|
||||
if [ ! -f $PATCHED_MARK ]; then
|
||||
patch -p1 < $TP_PATCH_DIR/s2geometry-0.9.0.patch
|
||||
touch $PATCHED_MARK
|
||||
cd "${TP_SOURCE_DIR}/${S2_SOURCE}"
|
||||
if [[ ! -f "${PATCHED_MARK}" ]]; then
|
||||
patch -p1 <"${TP_PATCH_DIR}/s2geometry-0.9.0.patch"
|
||||
touch "${PATCHED_MARK}"
|
||||
fi
|
||||
cd -
|
||||
echo "Finished patching $S2_SOURCE"
|
||||
echo "Finished patching ${S2_SOURCE}"
|
||||
|
||||
# gsasl2 patch to fix link error such as mutilple func defination
|
||||
# when link target with kerberos
|
||||
cd $TP_SOURCE_DIR/$GSASL_SOURCE
|
||||
if [ ! -f $PATCHED_MARK ]; then
|
||||
patch -p1 < $TP_PATCH_DIR/libgsasl-1.8.0.patch
|
||||
touch $PATCHED_MARK
|
||||
cd "${TP_SOURCE_DIR}/${GSASL_SOURCE}"
|
||||
if [[ ! -f ${PATCHED_MARK} ]]; then
|
||||
patch -p1 <"${TP_PATCH_DIR}/libgsasl-1.8.0.patch"
|
||||
touch "${PATCHED_MARK}"
|
||||
fi
|
||||
cd -
|
||||
echo "Finished patching $GSASL_SOURCE"
|
||||
echo "Finished patching ${GSASL_SOURCE}"
|
||||
|
||||
# rocksdb patch to fix compile error
|
||||
if [ $ROCKSDB_SOURCE == "rocksdb-5.14.2" ]; then
|
||||
cd $TP_SOURCE_DIR/$ROCKSDB_SOURCE
|
||||
if [ ! -f $PATCHED_MARK ]; then
|
||||
patch -p1 < $TP_PATCH_DIR/rocksdb-5.14.2.patch
|
||||
touch $PATCHED_MARK
|
||||
if [[ "${ROCKSDB_SOURCE}" == "rocksdb-5.14.2" ]]; then
|
||||
cd "${TP_SOURCE_DIR}/${ROCKSDB_SOURCE}"
|
||||
if [[ ! -f "${PATCHED_MARK}" ]]; then
|
||||
patch -p1 <"${TP_PATCH_DIR}/rocksdb-5.14.2.patch"
|
||||
touch "${PATCHED_MARK}"
|
||||
fi
|
||||
cd -
|
||||
fi
|
||||
echo "Finished patching $ROCKSDB_SOURCE"
|
||||
echo "Finished patching ${ROCKSDB_SOURCE}"
|
||||
|
||||
# opentelemetry patch is used to solve the problem that threadlocal depends on GLIBC_2.18
|
||||
# see: https://github.com/apache/doris/pull/7911
|
||||
if [ $OPENTELEMETRY_SOURCE == "opentelemetry-cpp-1.4.0" ]; then
|
||||
rm -rf $TP_SOURCE_DIR/$OPENTELEMETRY_SOURCE/third_party/opentelemetry-proto/*
|
||||
cp -r $TP_SOURCE_DIR/$OPENTELEMETRY_PROTO_SOURCE/* $TP_SOURCE_DIR/$OPENTELEMETRY_SOURCE/third_party/opentelemetry-proto
|
||||
mkdir -p $TP_SOURCE_DIR/$OPENTELEMETRY_SOURCE/third_party/opentelemetry-proto/.git
|
||||
if [[ "${OPENTELEMETRY_SOURCE}" == "opentelemetry-cpp-1.4.0" ]]; then
|
||||
rm -rf "${TP_SOURCE_DIR}/${OPENTELEMETRY_SOURCE}/third_party/opentelemetry-proto"/*
|
||||
cp -r "${TP_SOURCE_DIR}/${OPENTELEMETRY_PROTO_SOURCE}"/* "${TP_SOURCE_DIR}/${OPENTELEMETRY_SOURCE}/third_party/opentelemetry-proto"
|
||||
mkdir -p "${TP_SOURCE_DIR}/${OPENTELEMETRY_SOURCE}/third_party/opentelemetry-proto/.git"
|
||||
|
||||
cd $TP_SOURCE_DIR/$OPENTELEMETRY_SOURCE
|
||||
if [ ! -f $PATCHED_MARK ]; then
|
||||
patch -p1 < $TP_PATCH_DIR/opentelemetry-cpp-1.4.0.patch
|
||||
touch $PATCHED_MARK
|
||||
cd "${TP_SOURCE_DIR}/${OPENTELEMETRY_SOURCE}"
|
||||
if [[ ! -f "${PATCHED_MARK}" ]]; then
|
||||
patch -p1 <"${TP_PATCH_DIR}/opentelemetry-cpp-1.4.0.patch"
|
||||
touch "${PATCHED_MARK}"
|
||||
fi
|
||||
cd -
|
||||
fi
|
||||
echo "Finished patching $OPENTELEMETRY_SOURCE"
|
||||
echo "Finished patching ${OPENTELEMETRY_SOURCE}"
|
||||
|
||||
# arrow patch is used to get the raw orc reader for filter prune.
|
||||
if [ $ARROW_SOURCE == "apache-arrow-7.0.0" ]; then
|
||||
cd $TP_SOURCE_DIR/$ARROW_SOURCE
|
||||
if [ ! -f $PATCHED_MARK ]; then
|
||||
patch -p1 < $TP_PATCH_DIR/apache-arrow-7.0.0.patch
|
||||
touch $PATCHED_MARK
|
||||
if [[ "${ARROW_SOURCE}" == "apache-arrow-7.0.0" ]]; then
|
||||
cd "${TP_SOURCE_DIR}/${ARROW_SOURCE}"
|
||||
if [[ ! -f "${PATCHED_MARK}" ]]; then
|
||||
patch -p1 <"${TP_PATCH_DIR}/apache-arrow-7.0.0.patch"
|
||||
touch "${PATCHED_MARK}"
|
||||
fi
|
||||
cd -
|
||||
fi
|
||||
echo "Finished patching $ARROW_SOURCE"
|
||||
echo "Finished patching ${ARROW_SOURCE}"
|
||||
|
||||
# patch librdkafka to avoid crash
|
||||
if [ $LIBRDKAFKA_SOURCE == "librdkafka-1.8.2" ]; then
|
||||
cd $TP_SOURCE_DIR/$LIBRDKAFKA_SOURCE
|
||||
if [ ! -f $PATCHED_MARK ]; then
|
||||
patch -p0 < $TP_PATCH_DIR/librdkafka-1.8.2.patch
|
||||
touch $PATCHED_MARK
|
||||
if [[ "${LIBRDKAFKA_SOURCE}" == "librdkafka-1.8.2" ]]; then
|
||||
cd "${TP_SOURCE_DIR}/${LIBRDKAFKA_SOURCE}"
|
||||
if [[ ! -f "${PATCHED_MARK}" ]]; then
|
||||
patch -p0 <"${TP_PATCH_DIR}/librdkafka-1.8.2.patch"
|
||||
touch "${PATCHED_MARK}"
|
||||
fi
|
||||
cd -
|
||||
fi
|
||||
echo "Finished patching $LIBRDKAFKA_SOURCE"
|
||||
echo "Finished patching ${LIBRDKAFKA_SOURCE}"
|
||||
|
||||
# patch hyperscan
|
||||
# https://github.com/intel/hyperscan/issues/292
|
||||
if [ $HYPERSCAN_SOURCE == "hyperscan-5.4.0" ]; then
|
||||
cd $TP_SOURCE_DIR/$HYPERSCAN_SOURCE
|
||||
if [ ! -f $PATCHED_MARK ]; then
|
||||
patch -p0 < $TP_PATCH_DIR/hyperscan-5.4.0.patch
|
||||
touch $PATCHED_MARK
|
||||
if [[ "${HYPERSCAN_SOURCE}" == "hyperscan-5.4.0" ]]; then
|
||||
cd "${TP_SOURCE_DIR}/${HYPERSCAN_SOURCE}"
|
||||
if [[ ! -f "${PATCHED_MARK}" ]]; then
|
||||
patch -p0 <"${TP_PATCH_DIR}/hyperscan-5.4.0.patch"
|
||||
touch "${PATCHED_MARK}"
|
||||
fi
|
||||
cd -
|
||||
elif [ $HYPERSCAN_SOURCE == "vectorscan-vectorscan-5.4.7" ]; then
|
||||
cd $TP_SOURCE_DIR/$HYPERSCAN_SOURCE
|
||||
if [ ! -f $PATCHED_MARK ]; then
|
||||
patch -p0 < $TP_PATCH_DIR/vectorscan-5.4.7.patch
|
||||
touch $PATCHED_MARK
|
||||
elif [[ "${HYPERSCAN_SOURCE}" == "vectorscan-vectorscan-5.4.7" ]]; then
|
||||
cd "${TP_SOURCE_DIR}/${HYPERSCAN_SOURCE}"
|
||||
if [[ ! -f "${PATCHED_MARK}" ]]; then
|
||||
patch -p0 <"${TP_PATCH_DIR}/vectorscan-5.4.7.patch"
|
||||
touch "${PATCHED_MARK}"
|
||||
fi
|
||||
cd -
|
||||
fi
|
||||
echo "Finished patching $HYPERSCAN_SOURCE"
|
||||
echo "Finished patching ${HYPERSCAN_SOURCE}"
|
||||
|
||||
cd $TP_SOURCE_DIR/$AWS_SDK_SOURCE
|
||||
if [ ! -f $PATCHED_MARK ]; then
|
||||
if [ $AWS_SDK_SOURCE == "aws-sdk-cpp-1.9.211" ]; then
|
||||
wget --no-check-certificate -q https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/aws-crt-cpp-1.9.211.tar.gz -O aws-crt-cpp-1.9.211.tar.gz
|
||||
ret="$?"
|
||||
if [ $ret -eq 0 ] ; then
|
||||
cd "${TP_SOURCE_DIR}/${AWS_SDK_SOURCE}"
|
||||
if [[ ! -f "${PATCHED_MARK}" ]]; then
|
||||
if [[ "${AWS_SDK_SOURCE}" == "aws-sdk-cpp-1.9.211" ]]; then
|
||||
if wget --no-check-certificate -q https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/aws-crt-cpp-1.9.211.tar.gz -O aws-crt-cpp-1.9.211.tar.gz; then
|
||||
tar xzf aws-crt-cpp-1.9.211.tar.gz
|
||||
else
|
||||
bash ./prefetch_crt_dependency.sh
|
||||
@ -362,16 +355,15 @@ if [ ! -f $PATCHED_MARK ]; then
|
||||
else
|
||||
bash ./prefetch_crt_dependency.sh
|
||||
fi
|
||||
touch $PATCHED_MARK
|
||||
touch "${PATCHED_MARK}"
|
||||
fi
|
||||
cd -
|
||||
echo "Finished patching $AWS_SDK_SOURCE"
|
||||
echo "Finished patching ${AWS_SDK_SOURCE}"
|
||||
|
||||
cd "${TP_SOURCE_DIR}/${BRPC_SOURCE}"
|
||||
if [[ ! -f $PATCHED_MARK ]]; then
|
||||
if [[ ! -f "${PATCHED_MARK}" ]]; then
|
||||
patch -p1 <"${TP_PATCH_DIR}/brpc-1.2.0.patch"
|
||||
touch ${PATCHED_MARK}
|
||||
touch "${PATCHED_MARK}"
|
||||
fi
|
||||
cd -
|
||||
echo "Finished patching ${BRPC_SOURCE}"
|
||||
|
||||
|
||||
134
thirdparty/vars.sh
vendored
134
thirdparty/vars.sh
vendored
@ -1,4 +1,6 @@
|
||||
#!/bin/bash
|
||||
# shellcheck disable=2034
|
||||
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
@ -27,22 +29,22 @@
|
||||
###################################################
|
||||
|
||||
# thirdparties will be downloaded and unpacked here
|
||||
export TP_SOURCE_DIR=$TP_DIR/src
|
||||
export TP_SOURCE_DIR="${TP_DIR:-.}/src"
|
||||
|
||||
# thirdparties will be installed to here
|
||||
export TP_INSTALL_DIR=$TP_DIR/installed
|
||||
export TP_INSTALL_DIR="${TP_DIR:-.}/installed"
|
||||
|
||||
# patches for all thirdparties
|
||||
export TP_PATCH_DIR=$TP_DIR/patches
|
||||
export TP_PATCH_DIR="${TP_DIR:-.}/patches"
|
||||
|
||||
# header files of all thirdparties will be intalled to here
|
||||
export TP_INCLUDE_DIR=$TP_INSTALL_DIR/include
|
||||
export TP_INCLUDE_DIR="${TP_INSTALL_DIR}/include"
|
||||
|
||||
# libraries of all thirdparties will be intalled to here
|
||||
export TP_LIB_DIR=$TP_INSTALL_DIR/lib
|
||||
export TP_LIB_DIR="${TP_INSTALL_DIR}/lib"
|
||||
|
||||
# all java libraries will be unpacked to here
|
||||
export TP_JAR_DIR=$TP_INSTALL_DIR/lib/jar
|
||||
export TP_JAR_DIR="${TP_INSTALL_DIR}/lib/jar"
|
||||
|
||||
# source of all dependencies, default unuse it
|
||||
# export REPOSITORY_URL=
|
||||
@ -422,62 +424,64 @@ SSE2NEON_SOURCE=sse2neon-1.5.1
|
||||
SSE2NEON_MD5SUM="9de5dc2970aa7efac7faee59e2826c51"
|
||||
|
||||
# all thirdparties which need to be downloaded is set in array TP_ARCHIVES
|
||||
export TP_ARCHIVES="LIBEVENT
|
||||
OPENSSL
|
||||
THRIFT
|
||||
PROTOBUF
|
||||
GFLAGS
|
||||
GLOG
|
||||
GTEST
|
||||
RAPIDJSON
|
||||
SNAPPY
|
||||
GPERFTOOLS
|
||||
ZLIB
|
||||
LZ4
|
||||
BZIP
|
||||
LZO2
|
||||
CURL
|
||||
RE2
|
||||
HYPERSCAN
|
||||
RAGEL
|
||||
BOOST
|
||||
MYSQL
|
||||
ODBC
|
||||
LEVELDB
|
||||
BRPC
|
||||
ROCKSDB
|
||||
CYRUS_SASL
|
||||
LIBRDKAFKA
|
||||
FLATBUFFERS
|
||||
ARROW
|
||||
BROTLI
|
||||
ZSTD
|
||||
S2
|
||||
BITSHUFFLE
|
||||
CROARINGBITMAP
|
||||
FMT
|
||||
PARALLEL_HASHMAP
|
||||
ORC
|
||||
JEMALLOC
|
||||
CCTZ
|
||||
DATATABLES
|
||||
BOOTSTRAP_TABLE_JS
|
||||
BOOTSTRAP_TABLE_CSS
|
||||
TSAN_HEADER
|
||||
AWS_SDK
|
||||
LZMA
|
||||
XML2
|
||||
IDN
|
||||
GSASL
|
||||
KRB5
|
||||
HDFS3
|
||||
LIBDIVIDE
|
||||
PDQSORT
|
||||
BENCHMARK
|
||||
XSIMD
|
||||
SIMDJSON
|
||||
NLOHMANN_JSON
|
||||
OPENTELEMETRY_PROTO
|
||||
OPENTELEMETRY
|
||||
LIBBACKTRACE
|
||||
SSE2NEON"
|
||||
export TP_ARCHIVES=(
|
||||
'LIBEVENT'
|
||||
'OPENSSL'
|
||||
'THRIFT'
|
||||
'PROTOBUF'
|
||||
'GFLAGS'
|
||||
'GLOG'
|
||||
'GTEST'
|
||||
'RAPIDJSON'
|
||||
'SNAPPY'
|
||||
'GPERFTOOLS'
|
||||
'ZLIB'
|
||||
'LZ4'
|
||||
'BZIP'
|
||||
'LZO2'
|
||||
'CURL'
|
||||
'RE2'
|
||||
'HYPERSCAN'
|
||||
'RAGEL'
|
||||
'BOOST'
|
||||
'MYSQL'
|
||||
'ODBC'
|
||||
'LEVELDB'
|
||||
'BRPC'
|
||||
'ROCKSDB'
|
||||
'CYRUS_SASL'
|
||||
'LIBRDKAFKA'
|
||||
'FLATBUFFERS'
|
||||
'ARROW'
|
||||
'BROTLI'
|
||||
'ZSTD'
|
||||
'S2'
|
||||
'BITSHUFFLE'
|
||||
'CROARINGBITMAP'
|
||||
'FMT'
|
||||
'PARALLEL_HASHMAP'
|
||||
'ORC'
|
||||
'JEMALLOC'
|
||||
'CCTZ'
|
||||
'DATATABLES'
|
||||
'BOOTSTRAP_TABLE_JS'
|
||||
'BOOTSTRAP_TABLE_CSS'
|
||||
'TSAN_HEADER'
|
||||
'AWS_SDK'
|
||||
'LZMA'
|
||||
'XML2'
|
||||
'IDN'
|
||||
'GSASL'
|
||||
'KRB5'
|
||||
'HDFS3'
|
||||
'LIBDIVIDE'
|
||||
'PDQSORT'
|
||||
'BENCHMARK'
|
||||
'XSIMD'
|
||||
'SIMDJSON'
|
||||
'NLOHMANN_JSON'
|
||||
'OPENTELEMETRY_PROTO'
|
||||
'OPENTELEMETRY'
|
||||
'LIBBACKTRACE'
|
||||
'SSE2NEON'
|
||||
)
|
||||
|
||||
@ -18,43 +18,40 @@
|
||||
|
||||
jdk_version() {
|
||||
local result
|
||||
local java_cmd=$JAVA_HOME/bin/java
|
||||
local java_cmd="${JAVA_HOME:-.}/bin/java"
|
||||
local IFS=$'\n'
|
||||
# remove \r for Cygwin
|
||||
local lines=$("$java_cmd" -Xms32M -Xmx32M -version 2>&1 | tr '\r' '\n')
|
||||
if [[ -z $java_cmd ]]
|
||||
then
|
||||
|
||||
if [[ -z "${java_cmd}" ]]; then
|
||||
result=no_java
|
||||
return 1
|
||||
else
|
||||
for line in $lines; do
|
||||
if [[ (-z $result) && ($line = *"version \""*) ]]
|
||||
then
|
||||
local ver=$(echo $line | sed -e 's/.*version "\(.*\)"\(.*\)/\1/; 1q')
|
||||
# on macOS, sed doesn't support '?'
|
||||
if [[ $ver = "1."* ]]
|
||||
then
|
||||
result=$(echo $ver | sed -e 's/1\.\([0-9]*\)\(.*\)/\1/; 1q')
|
||||
else
|
||||
result=$(echo $ver | sed -e 's/\([0-9]*\)\(.*\)/\1/; 1q')
|
||||
fi
|
||||
fi
|
||||
done
|
||||
local version
|
||||
# remove \r for Cygwin
|
||||
version="$("${java_cmd}" -Xms32M -Xmx32M -version 2>&1 | tr '\r' '\n' | grep version | awk '{print $3}')"
|
||||
version="${version//\"/}"
|
||||
if [[ "${version}" =~ ^1\. ]]; then
|
||||
result="$(echo "${version}" | awk -F '.' '{print $2}')"
|
||||
else
|
||||
result="$(echo "${version}" | awk -F '.' '{print $1}')"
|
||||
fi
|
||||
fi
|
||||
echo "$result"
|
||||
echo "${result}"
|
||||
return 0
|
||||
}
|
||||
|
||||
java_version=$(jdk_version)
|
||||
MACHINE_TYPE=$(uname -m)
|
||||
jvm_arch="amd64"
|
||||
if [[ "${MACHINE_TYPE}" == "aarch64" ]]; then
|
||||
jvm_arch="aarch64"
|
||||
fi
|
||||
if [[ $java_version -gt 8 ]]; then
|
||||
export LIBJVM_PATH=$JAVA_HOME/lib
|
||||
if [[ "${java_version}" -gt 8 ]]; then
|
||||
export LIBJVM_PATH="${JAVA_HOME}/lib"
|
||||
# JAVA_HOME is jdk
|
||||
elif [[ -d "$JAVA_HOME/jre" ]]; then
|
||||
export LIBJVM_PATH=$JAVA_HOME/jre/lib/$jvm_arch
|
||||
elif [[ -d "${JAVA_HOME}/jre" ]]; then
|
||||
export LIBJVM_PATH="${JAVA_HOME}/jre/lib/${jvm_arch}"
|
||||
# JAVA_HOME is jre
|
||||
else
|
||||
export LIBJVM_PATH=$JAVA_HOME/lib/$jvm_arch
|
||||
export LIBJVM_PATH="${JAVA_HOME}/lib/${jvm_arch}"
|
||||
fi
|
||||
echo ${LIBJVM_PATH}/*/libjvm.so
|
||||
echo "${LIBJVM_PATH}"/*/libjvm.so
|
||||
|
||||
@ -39,63 +39,81 @@ Usage: $0 <options>
|
||||
|
||||
Examples:
|
||||
batch mode:
|
||||
sh restore_tablet_tool.sh -b "http://127.0.0.1:8040" -f tablets.txt
|
||||
sh restore_tablet_tool.sh --backend "http://127.0.0.1:8040" --file tablets.txt
|
||||
bash restore_tablet_tool.sh -b http://127.0.0.1:8040 -f tablets.txt
|
||||
bash restore_tablet_tool.sh --backend http://127.0.0.1:8040 --file tablets.txt
|
||||
|
||||
single mode:
|
||||
sh restore_tablet_tool.sh -b "http://127.0.0.1:8040" -t 12345 -s 11111
|
||||
sh restore_tablet_tool.sh --backend "http://127.0.0.1:8040" --tablet_id 12345 --schema_hash 11111
|
||||
bash restore_tablet_tool.sh -b http://127.0.0.1:8040 -t 12345 -s 11111
|
||||
bash restore_tablet_tool.sh --backend http://127.0.0.1:8040 --tablet_id 12345 --schema_hash 11111
|
||||
"
|
||||
exit 1
|
||||
}
|
||||
|
||||
OPTS=$(getopt \
|
||||
-n $0 \
|
||||
-o 'b:t:s:f:' \
|
||||
-l 'server:,tablet_id:,schema_hash:,file:,help' \
|
||||
-- "$@")
|
||||
|
||||
if [ $? != 0 ] ; then
|
||||
if ! OPTS=$(getopt \
|
||||
-n "$0" \
|
||||
-o 'b:t:s:f:' \
|
||||
-l 'server:,tablet_id:,schema_hash:,file:,help' \
|
||||
-- "$@"); then
|
||||
usage
|
||||
fi
|
||||
|
||||
eval set -- "$OPTS"
|
||||
eval set -- "${OPTS}"
|
||||
|
||||
SERVER="http://127.0.0.1/8040"
|
||||
TABLET_ID=
|
||||
SCHEMA_HASH=
|
||||
FILENAME=
|
||||
SERVER='http://127.0.0.1/8040'
|
||||
TABLET_ID=''
|
||||
SCHEMA_HASH=''
|
||||
FILENAME=''
|
||||
BATCH_MODE=false
|
||||
|
||||
while true; do
|
||||
case "$1" in
|
||||
-b|--backend) SERVER=$2 ; shift 2 ;;
|
||||
-f|--file) FILENAME=$2 ; BATCH_MODE=true ; shift 2 ;;
|
||||
-t|--tablet_id) TABLET_ID=$2 ; shift 2 ;;
|
||||
-s|--schema_hash) SCHEMA_HASH=$2 ; shift 2 ;;
|
||||
-h|--help) usage ; shift ;;
|
||||
--) shift ; break ;;
|
||||
*) echo "Internal error!" ; exit 1 ;;
|
||||
-b | --backend)
|
||||
SERVER="$2"
|
||||
shift 2
|
||||
;;
|
||||
-f | --file)
|
||||
FILENAME="$2"
|
||||
BATCH_MODE=true
|
||||
shift 2
|
||||
;;
|
||||
-t | --tablet_id)
|
||||
TABLET_ID="$2"
|
||||
shift 2
|
||||
;;
|
||||
-s | --schema_hash)
|
||||
SCHEMA_HASH="$2"
|
||||
shift 2
|
||||
;;
|
||||
-h | --help)
|
||||
usage
|
||||
shift
|
||||
;;
|
||||
--)
|
||||
shift
|
||||
break
|
||||
;;
|
||||
*)
|
||||
echo "Internal error!"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
restore_tablet() {
|
||||
echo "start to restore tablet id:"$2", schema hash:"$3
|
||||
echo "start to restore tablet id:$2, schema hash:$3"
|
||||
curl -X POST "$1/api/restore_tablet?tablet_id=$2&schema_hash=$3"
|
||||
echo -e "\n"
|
||||
}
|
||||
|
||||
if [ $BATCH_MODE = true ] ; then
|
||||
lines=`cat $FILENAME`
|
||||
for line in $lines
|
||||
do
|
||||
if ${BATCH_MODE}; then
|
||||
while read -r line; do
|
||||
# split the comma-split line
|
||||
# format: tablet_id,schema_hash
|
||||
fields=(${line/,/ })
|
||||
TABLET_ID=${fields[0]}
|
||||
SCHEMA_HASH=${fields[1]}
|
||||
restore_tablet $SERVER $TABLET_ID $SCHEMA_HASH
|
||||
done
|
||||
read -r -a fields <<<"${line/,/ }"
|
||||
TABLET_ID="${fields[0]}"
|
||||
SCHEMA_HASH="${fields[1]}"
|
||||
restore_tablet "${SERVER}" "${TABLET_ID}" "${SCHEMA_HASH}"
|
||||
done <"${FILENAME}"
|
||||
else
|
||||
restore_tablet $SERVER $TABLET_ID $SCHEMA_HASH
|
||||
restore_tablet "${SERVER}" "${TABLET_ID}" "${SCHEMA_HASH}"
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user