database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[improvement] optimize java cmd find (#7428)

Browse Source 
* optimize java cmd find, if java_home not set use java in PATH
This commit is contained in:
Zhengguo Yang
2021-12-30 10:16:56 +08:00
committed by GitHub
parent 2872dbfeb8
commit bc4ceeca44
5 changed files with 202 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
bin/start_be.sh
View File

@ -16,27 +16,42 @@
# specific language governing permissions and limitations
# under the License.
curdir=`dirname "$0"`
curdir=`cd "$curdir"; pwd`
curdir=$(dirname "$0")
curdir=$(
cd "$curdir"
pwd
)
OPTS=$(getopt \
-n $0 \
-o '' \
-l 'daemon' \
-- "$@")
-n $0 \
-o '' \
-l 'daemon' \
-- "$@")
eval set -- "$OPTS"
RUN_DAEMON=0
while true; do
case "$1" in
--daemon) RUN_DAEMON=1 ; shift ;;
--) shift ; break ;;
*) echo "Internal error" ; exit 1 ;;
--daemon)
RUN_DAEMON=1
shift
;;
--)
shift
break
;;
*)
echo "Internal error"
exit 1
;;
esac
done
export DORIS_HOME=`cd "$curdir/.."; pwd`
export DORIS_HOME=$(
cd "$curdir/.."
pwd
)
# export env variables from be.conf
#
@ -45,7 +60,10 @@ export DORIS_HOME=`cd "$curdir/.."; pwd`
# PID_DIR
export UDF_RUNTIME_DIR=${DORIS_HOME}/lib/udf-runtime
export LOG_DIR=${DORIS_HOME}/log
export PID_DIR=`cd "$curdir"; pwd`
export PID_DIR=$(
cd "$curdir"
pwd
)
# set odbc conf path
export ODBCSYSINI=$DORIS_HOME/conf
@ -54,8 +72,8 @@ export ODBCSYSINI=$DORIS_HOME/conf
export NLS_LANG=AMERICAN_AMERICA.AL32UTF8
while read line; do
envline=`echo $line | sed 's/[[:blank:]]*=[[:blank:]]*/=/g' | sed 's/^[[:blank:]]*//g' | egrep "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*="`
envline=`eval "echo $envline"`
envline=$(echo $line | sed 's/[[:blank:]]*=[[:blank:]]*/=/g' | sed 's/^[[:blank:]]*//g' | egrep "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*=")
envline=$(eval "echo $envline")
if [[ $envline == *"="* ]]; then
eval 'export "$envline"'
fi
@ -79,7 +97,7 @@ pidfile=$PID_DIR/be.pid
if [ -f $pidfile ]; then
if kill -0 $(cat $pidfile) > /dev/null 2>&1; then
echo "Backend running as process `cat $pidfile`. Stop it first."
echo "Backend running as process $(cat $pidfile). Stop it first."
exit 1
else
rm $pidfile
@ -96,7 +114,7 @@ else
fi
if [ ${RUN_DAEMON} -eq 1 ]; then
nohup $LIMIT ${DORIS_HOME}/lib/palo_be "$@" >> $LOG_DIR/be.out 2>&1 </dev/null &
nohup $LIMIT ${DORIS_HOME}/lib/palo_be "$@" >> $LOG_DIR/be.out 2>&1 < /dev/null &
else
$LIMIT ${DORIS_HOME}/lib/palo_be "$@" >> $LOG_DIR/be.out 2>&1 </dev/null
$LIMIT ${DORIS_HOME}/lib/palo_be "$@" >> $LOG_DIR/be.out 2>&1 < /dev/null
fi

120
bin/start_fe.sh
View File

@ -16,15 +16,18 @@
# specific language governing permissions and limitations
# under the License.
curdir=`dirname "$0"`
curdir=`cd "$curdir"; pwd`
curdir=$(dirname "$0")
curdir=$(
cd "$curdir"
pwd
)
OPTS=$(getopt \
-n $0 \
-o '' \
-l 'daemon' \
-l 'helper:' \
-- "$@")
-n $0 \
-o '' \
-l 'daemon' \
-l 'helper:' \
-- "$@")
eval set -- "$OPTS"
@ -32,14 +35,29 @@ RUN_DAEMON=0
HELPER=
while true; do
case "$1" in
--daemon) RUN_DAEMON=1 ; shift ;;
--helper) HELPER=$2 ; shift 2 ;;
--) shift ; break ;;
*) ehco "Internal error" ; exit 1 ;;
--daemon)
RUN_DAEMON=1
shift
;;
--helper)
HELPER=$2
shift 2
;;
--)
shift
break
;;
*)
ehco "Internal error"
exit 1
;;
esac
done
export DORIS_HOME=`cd "$curdir/.."; pwd`
export DORIS_HOME=$(
cd "$curdir/.."
pwd
)
# export env variables from fe.conf
#
@ -48,11 +66,14 @@ export DORIS_HOME=`cd "$curdir/.."; pwd`
# PID_DIR
export JAVA_OPTS="-Xmx1024m"
export LOG_DIR="$DORIS_HOME/log"
export PID_DIR=`cd "$curdir"; pwd`
export PID_DIR=$(
cd "$curdir"
pwd
)
while read line; do
envline=`echo $line | sed 's/[[:blank:]]*=[[:blank:]]*/=/g' | sed 's/^[[:blank:]]*//g' | egrep "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*="`
envline=`eval "echo $envline"`
envline=$(echo $line | sed 's/[[:blank:]]*=[[:blank:]]*/=/g' | sed 's/^[[:blank:]]*//g' | egrep "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*=")
envline=$(eval "echo $envline")
if [[ $envline == *"="* ]]; then
eval 'export "$envline"'
fi
@ -62,39 +83,37 @@ if [ -e $DORIS_HOME/bin/palo_env.sh ]; then
source $DORIS_HOME/bin/palo_env.sh
fi
# java
if [ "$JAVA_HOME" = "" ]; then
echo "Error: JAVA_HOME is not set."
exit 1
if [ -z "$JAVA_HOME" ]; then
JAVA=$(which java)
else
JAVA="$JAVA_HOME/bin/java"
fi
if [ ! -x "$JAVA" ]; then
echo "The JAVA_HOME environment variable is not defined correctly"
echo "This environment variable is needed to run this program"
echo "NB: JAVA_HOME should point to a JDK not a JRE"
exit 1
fi
JAVA=$JAVA_HOME/bin/java
# get jdk version, return version as an Integer.
# 1.8 => 8, 13.0 => 13
jdk_version() {
local result
local java_cmd=$JAVA_HOME/bin/java
local IFS=$'\n'
# remove \r for Cygwin
local lines=$("$java_cmd" -Xms32M -Xmx32M -version 2>&1 | tr '\r' '\n')
if [[ -z $java_cmd ]]
then
result=no_java
else
for line in $lines; do
if [[ (-z $result) && ($line = *"version \""*) ]]
then
local ver=$(echo $line | sed -e 's/.*version "\(.*\)"\(.*\)/\1/; 1q')
# on macOS, sed doesn't support '?'
if [[ $ver = "1."* ]]
then
result=$(echo $ver | sed -e 's/1\.\([0-9]*\)\(.*\)/\1/; 1q')
else
result=$(echo $ver | sed -e 's/\([0-9]*\)\(.*\)/\1/; 1q')
fi
local lines=$("$JAVA" -Xms32M -Xmx32M -version 2>&1 | tr '\r' '\n')
for line in $lines; do
if [[ (-z $result) && ($line = *"version \""*) ]]; then
local ver=$(echo $line | sed -e 's/.*version "\(.*\)"\(.*\)/\1/; 1q')
# on macOS, sed doesn't support '?'
if [[ $ver = "1."* ]]; then
result=$(echo $ver | sed -e 's/1\.\([0-9]*\)\(.*\)/\1/; 1q')
else
result=$(echo $ver | sed -e 's/\([0-9]*\)\(.*\)/\1/; 1q')
fi
done
fi
fi
done
echo "$result"
}
@ -102,15 +121,15 @@ jdk_version() {
if [ ! -d $LOG_DIR ]; then
mkdir -p $LOG_DIR
fi
# check java version and choose correct JAVA_OPTS
java_version=$(jdk_version)
final_java_opt=$JAVA_OPTS
if [ $java_version -gt 8 ]; then
if [ -z "$JAVA_OPTS_FOR_JDK_9" ]; then
echo "JAVA_OPTS_FOR_JDK_9 is not set in fe.conf" >> $LOG_DIR/fe.out
exit -1
fi
exit 1
fi
final_java_opt=$JAVA_OPTS_FOR_JDK_9
fi
echo "using java version $java_version" >> $LOG_DIR/fe.out
@ -118,18 +137,17 @@ echo $final_java_opt >> $LOG_DIR/fe.out
# add libs to CLASSPATH
for f in $DORIS_HOME/lib/*.jar; do
CLASSPATH=$f:${CLASSPATH};
CLASSPATH=$f:${CLASSPATH}
done
export CLASSPATH=${CLASSPATH}:${DORIS_HOME}/lib
pidfile=$PID_DIR/fe.pid
if [ -f $pidfile ]; then
if kill -0 `cat $pidfile` > /dev/null 2>&1; then
echo Frontend running as process `cat $pidfile`. Stop it first.
exit 1
fi
if kill -0 $(cat $pidfile) > /dev/null 2>&1; then
echo Frontend running as process $(cat $pidfile). Stop it first.
exit 1
fi
fi
if [ ! -f /bin/limit ]; then
@ -138,7 +156,7 @@ else
LIMIT=/bin/limit
fi
echo `date` >> $LOG_DIR/fe.out
echo $(date) >> $LOG_DIR/fe.out
if [ x"$HELPER" != x"" ]; then
# change it to '-helper' to be compatible with code in Frontend
@ -146,9 +164,9 @@ if [ x"$HELPER" != x"" ]; then
fi
if [ ${RUN_DAEMON} -eq 1 ]; then
nohup $LIMIT $JAVA $final_java_opt org.apache.doris.PaloFe ${HELPER} "$@" >> $LOG_DIR/fe.out 2>&1 </dev/null &
nohup $LIMIT $JAVA $final_java_opt org.apache.doris.PaloFe ${HELPER} "$@" >> $LOG_DIR/fe.out 2>&1 < /dev/null &
else
$LIMIT $JAVA $final_java_opt org.apache.doris.PaloFe ${HELPER} "$@" >> $LOG_DIR/fe.out 2>&1 </dev/null
$LIMIT $JAVA $final_java_opt org.apache.doris.PaloFe ${HELPER} "$@" >> $LOG_DIR/fe.out 2>&1 < /dev/null
fi
echo $! > $pidfile

35
env.sh
View File

@ -53,12 +53,12 @@ fi
# set GCC HOME
if [[ -z ${DORIS_GCC_HOME} ]]; then
export DORIS_GCC_HOME=$(dirname `which gcc`)/..
export DORIS_GCC_HOME=$(dirname $(which gcc))/..
fi
gcc_ver=`${DORIS_GCC_HOME}/bin/gcc -dumpfullversion -dumpversion`
gcc_ver=$(${DORIS_GCC_HOME}/bin/gcc -dumpfullversion -dumpversion)
required_ver="7.3.0"
if [[ ! "$(printf '%s\n' "$required_ver" "$gcc_ver" | sort -V | head -n1)" = "$required_ver" ]]; then
if [[ ! "$(printf '%s\n' "$required_ver" "$gcc_ver" | sort -V | head -n1)" = "$required_ver" ]]; then
echo "Error: GCC version (${gcc_ver}) must be greater than or equal to ${required_ver}"
exit 1
fi
@ -66,33 +66,40 @@ fi
# register keyword is forbidden to use in C++17
# the C++ code generated by flex that remove register keyword after version 2.6.0
# so we need check flex version here to avoid compilation failed
flex_ver=`flex --version | awk '{print $2}'`
flex_ver=$(flex --version | awk '{print $2}')
required_ver="2.6.0"
if [[ ! "$(printf '%s\n' "$required_ver" "$flex_ver" | sort -V | head -n1)" = "$required_ver" ]]; then
if [[ ! "$(printf '%s\n' "$required_ver" "$flex_ver" | sort -V | head -n1)" = "$required_ver" ]]; then
echo "Error: flex version (${flex_ver}) must be greater than or equal to ${required_ver}"
exit 1
fi
# find binutils
if test -x ${DORIS_GCC_HOME}/bin/ld ;then
if test -x ${DORIS_GCC_HOME}/bin/ld; then
export DORIS_BIN_UTILS=${DORIS_GCC_HOME}/bin/
else
export DORIS_BIN_UTILS=/usr/bin/
fi
# export CLANG COMPATIBLE FLAGS
export CLANG_COMPATIBLE_FLAGS=`echo | ${DORIS_GCC_HOME}/bin/gcc -Wp,-v -xc++ - -fsyntax-only 2>&1 \
| grep -E '^\s+/' | awk '{print "-I" $1}' | tr '\n' ' '`
export CLANG_COMPATIBLE_FLAGS=$(echo | ${DORIS_GCC_HOME}/bin/gcc -Wp,-v -xc++ - -fsyntax-only 2>&1 |
grep -E '^\s+/' | awk '{print "-I" $1}' | tr '\n' ' ')
# check java home
if [[ -z ${JAVA_HOME} ]]; then
echo "Error: JAVA_HOME is not set"
if [ -z "$JAVA_HOME" ]; then
export JAVACMD=$(which java)
JAVAP=$(which javap)
else
export JAVA="${JAVA_HOME}/bin/java"
JAVAP="${JAVA_HOME}/bin/javap"
fi
if [ ! -x "$JAVA" ]; then
echo "The JAVA_HOME environment variable is not defined correctly"
echo "This environment variable is needed to run this program"
echo "NB: JAVA_HOME should point to a JDK not a JRE"
exit 1
fi
# check java version
export JAVA=${JAVA_HOME}/bin/java
JAVAP=${JAVA_HOME}/bin/javap
JAVA_VER=$(${JAVAP} -verbose java.lang.String | grep "major version" | cut -d " " -f5)
if [[ $JAVA_VER -lt 52 ]]; then
echo "Error: require JAVA with JDK version at least 1.8"
@ -122,7 +129,7 @@ export CMAKE_CMD
GENERATOR="Unix Makefiles"
BUILD_SYSTEM="make"
if ninja --version 2>/dev/null; then
if ninja --version 2> /dev/null; then
GENERATOR="Ninja"
BUILD_SYSTEM="ninja"
fi

71
fs_brokers/apache_hdfs_broker/bin/start_broker.sh
View File

@ -16,49 +16,72 @@
# specific language governing permissions and limitations
# under the License.
curdir=`dirname "$0"`
curdir=`cd "$curdir"; pwd`
curdir=$(dirname "$0")
curdir=$(
cd "$curdir"
pwd
)
OPTS=$(getopt \
-n $0 \
-o '' \
-l 'daemon' \
-- "$@")
-n $0 \
-o '' \
-l 'daemon' \
-- "$@")
eval set -- "$OPTS"
RUN_DAEMON=0
while true; do
case "$1" in
--daemon) RUN_DAEMON=1 ; shift ;;
--) shift ; break ;;
*) ehco "Internal error" ; exit 1 ;;
--daemon)
RUN_DAEMON=1
shift
;;
--)
shift
break
;;
*)
ehco "Internal error"
exit 1
;;
esac
done
export BROKER_HOME=`cd "$curdir/.."; pwd`
export PID_DIR=`cd "$curdir"; pwd`
export BROKER_HOME=$(
cd "$curdir/.."
pwd
)
export PID_DIR=$(
cd "$curdir"
pwd
)
export JAVA_OPTS="-Xmx1024m -Dfile.encoding=UTF-8"
export BROKER_LOG_DIR="$BROKER_HOME/log"
# export JAVA_HOME="/usr/java/jdk1.8.0_131"
# java
if [ "$JAVA_HOME" = "" ]; then
echo "Error: JAVA_HOME is not set."
exit 1
if [ -z "$JAVA_HOME" ]; then
JAVA=$(which java)
else
JAVA="$JAVA_HOME/bin/java"
fi
JAVA=$JAVA_HOME/bin/java
if [ ! -x "$JAVA" ]; then
echo "The JAVA_HOME environment variable is not defined correctly"
echo "This environment variable is needed to run this program"
echo "NB: JAVA_HOME should point to a JDK not a JRE"
exit 1
fi
# add libs to CLASSPATH
for f in $BROKER_HOME/lib/*.jar; do
CLASSPATH=$f:${CLASSPATH};
CLASSPATH=$f:${CLASSPATH}
done
export CLASSPATH=${CLASSPATH}:${BROKER_HOME}/lib:$BROKER_HOME/conf
while read line; do
envline=`echo $line | sed 's/[[:blank:]]*=[[:blank:]]*/=/g' | sed 's/^[[:blank:]]*//g' | egrep "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*="`
envline=`eval "echo $envline"`
envline=$(echo $line | sed 's/[[:blank:]]*=[[:blank:]]*/=/g' | sed 's/^[[:blank:]]*//g' | egrep "^[[:upper:]]([[:upper:]]|_|[[:digit:]])*=")
envline=$(eval "echo $envline")
if [[ $envline == *"="* ]]; then
eval 'export "$envline"'
fi
@ -67,8 +90,8 @@ done < $BROKER_HOME/conf/apache_hdfs_broker.conf
pidfile=$PID_DIR/apache_hdfs_broker.pid
if [ -f $pidfile ]; then
if kill -0 `cat $pidfile` > /dev/null 2>&1; then
echo "Broker running as process `cat $pidfile`. Stop it first."
if kill -0 $(cat $pidfile) > /dev/null 2>&1; then
echo "Broker running as process $(cat $pidfile). Stop it first."
exit 1
fi
fi
@ -77,12 +100,12 @@ if [ ! -d $BROKER_LOG_DIR ]; then
mkdir -p $BROKER_LOG_DIR
fi
echo `date` >> $BROKER_LOG_DIR/apache_hdfs_broker.out
echo $(date) >> $BROKER_LOG_DIR/apache_hdfs_broker.out
if [ ${RUN_DAEMON} -eq 1 ]; then
nohup $LIMIT $JAVA $JAVA_OPTS org.apache.doris.broker.hdfs.BrokerBootstrap "$@" >> $BROKER_LOG_DIR/apache_hdfs_broker.out 2>&1 </dev/null &
nohup $LIMIT $JAVA $JAVA_OPTS org.apache.doris.broker.hdfs.BrokerBootstrap "$@" >> $BROKER_LOG_DIR/apache_hdfs_broker.out 2>&1 < /dev/null &
else
$LIMIT $JAVA $JAVA_OPTS org.apache.doris.broker.hdfs.BrokerBootstrap "$@" >> $BROKER_LOG_DIR/apache_hdfs_broker.out 2>&1 </dev/null
$LIMIT $JAVA $JAVA_OPTS org.apache.doris.broker.hdfs.BrokerBootstrap "$@" >> $BROKER_LOG_DIR/apache_hdfs_broker.out 2>&1 < /dev/null
fi
echo $! > $pidfile

57
gensrc/script/gen_build_version.sh
View File

@ -20,8 +20,8 @@
# This script is used to build Doris used in Baidu
##############################################################
# the script generates src/be/src/common/version.h and
# fe/src/main/java/org/apache/doris/common/Version.java which
# the script generates src/be/src/common/version.h and
# fe/src/main/java/org/apache/doris/common/Version.java which
# contains the build version based on the git hash or svn revision.
##############################################################
@ -30,39 +30,40 @@ build_version="trunk"
unset LANG
unset LC_CTYPE
user=`whoami`
date=`date +"%a, %d %b %Y %H:%M:%S %Z"`
hostname=`hostname`
user=$(whoami)
date=$(date +"%a, %d %b %Y %H:%M:%S %Z")
hostname=$(hostname)
cwd=`pwd`
cwd=$(pwd)
if [ -z ${DORIS_HOME+x} ]
then
ROOT=`dirname "$0"`
ROOT=`cd "$ROOT"; pwd`
if [ -z ${DORIS_HOME+x} ]; then
ROOT=$(dirname "$0")
ROOT=$(
cd "$ROOT"
pwd
)
DORIS_HOME=${ROOT}/../..
echo "DORIS_HOME: ${DORIS_HOME}"
fi
if [[ -z ${DORIS_TEST_BINARY_DIR} ]]; then
if [ -e ${DORIS_HOME}/fe/fe-core/target/generated-sources/build/org/apache/doris/common/Version.java \
-a -e ${DORIS_HOME}/gensrc/build/gen_cpp/version.h ]; then
-a -e ${DORIS_HOME}/gensrc/build/gen_cpp/version.h ]; then
exit
fi
fi
cd ${DORIS_HOME}
if [ -d .svn ]; then
revision=`svn info | sed -n -e 's/Last Changed Rev: \(.*\)/\1/p'`
revision=$(svn info | sed -n -e 's/Last Changed Rev: \(.*\)/\1/p')
short_revision="${revision}"
url=`svn info | sed -n -e 's/^URL: \(.*\)/\1/p'`
if echo ${url} | grep '\/tags\/' > /dev/null
then
build_version="`echo ${url} | sed 's/.*_\([0-9-]\+\)_PD_BL.*/\1/g' | sed 's/-/\./g'`"
url=$(svn info | sed -n -e 's/^URL: \(.*\)/\1/p')
if echo ${url} | grep '\/tags\/' > /dev/null; then
build_version="$(echo ${url} | sed 's/.*_\([0-9-]\+\)_PD_BL.*/\1/g' | sed 's/-/\./g')"
fi
elif [ -d .git ]; then
revision=`git log -1 --pretty=format:"%H"`
short_revision=`git log -1 --pretty=format:"%h"`
revision=$(git log -1 --pretty=format:"%H")
short_revision=$(git log -1 --pretty=format:"%h")
url="git://${hostname}${DORIS_HOME}"
else
revision="Unknown"
@ -77,15 +78,19 @@ build_short_hash="${short_revision}"
build_time="${date}"
build_info="${user}@${hostname}"
java_cmd=
if [[ (-n "$JAVA_HOME") && (-x "$JAVA_HOME/bin/java") ]]; then
java_cmd="$JAVA_HOME/bin/java"
if [ -z "$JAVA_HOME" ]; then
java_cmd=$(which java)
else
echo "JAVA_HOME is not set, or java bin is not found"
exit -1
java_cmd="$JAVA_HOME/bin/java"
fi
java_version_str=`$java_cmd -fullversion 2>&1`
if [ ! -x "$java_cmd" ]; then
echo "The JAVA_HOME environment variable is not defined correctly"
echo "This environment variable is needed to run this program"
echo "NB: JAVA_HOME should point to a JDK not a JRE"
exit 1
fi
java_version_str=$($java_cmd -fullversion 2>&1)
java_version_str=$(echo $java_version_str | sed -e 's/"/\\"/g')
echo "get java cmd: $java_cmd"
@ -93,7 +98,7 @@ echo "get java version: $java_version_str"
VERSION_PACKAGE="${DORIS_HOME}/fe/fe-core/target/generated-sources/build/org/apache/doris/common"
mkdir -p ${VERSION_PACKAGE}
cat >"${VERSION_PACKAGE}/Version.java" <<EOF
cat > "${VERSION_PACKAGE}/Version.java" << EOF
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
@ -140,7 +145,7 @@ EOF
GEN_CPP_DIR=${DORIS_HOME}/gensrc/build/gen_cpp/
mkdir -p ${GEN_CPP_DIR}
cat >"${GEN_CPP_DIR}/version.h" <<EOF
cat > "${GEN_CPP_DIR}/version.h" << EOF
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information